Merge branch 'master' of git.fnexe.com:github.com/pypiserver

* chore(rc-changes): update CHANGES.rst

* chore: update changes.rst

---------

Co-authored-by: github-actions <github-actions@github.com>
Co-authored-by: Dmitrii Orlov <dmtree.dev@yahoo.com>

.github/workflows/ci.yml

@@ -129,8 +129,6 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - "tests"
-    # only if a tag is pushed
-    if: startsWith(github.event.ref, 'refs/tags/v')
     steps:
       - uses: actions/checkout@master
       - name: Set up Python
@@ -138,12 +136,17 @@ jobs:
         with:
           python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
 
+      - name: Install dev dependencies
+        run: pip install -r "requirements/dev.pip"
+
       - name: Build distribution _wheel_.
         run: |
           ./bin/package.sh
 
       - name: Publish distribution 📦 to PyPI.
         uses: pypa/gh-action-pypi-publish@release/v1
+        # Push to PyPi only if a tag is pushed
+        if: startsWith(github.event.ref, 'refs/tags/v')
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
           print-hash: true

.github/workflows/rt.yml

@@ -22,14 +22,16 @@ jobs:
     if: ${{ github.ref_name == 'master' }}
     runs-on: ubuntu-latest
     env:
-      CHANGES_FILE: CHANGES.rst
+      CHANGE_FILE: CHANGES.rst
       EXPECTED_DIFF_COUNT: 1
     steps:
       - uses: actions/checkout@v3
 
       - id: get-version
         run: |
+          CHANGE_FILE=${{ env.CHANGE_FILE }}
           LAST_VERSION=$(grep -m1 -E ' \([0-9]+-[0-9]+-[0-9]+\)$' ${CHANGE_FILE} | awk '{ print $1 }')
+          echo "👀 Version detected: ${LAST_VERSION}"
           echo "LAST_VERSION=${LAST_VERSION}" >> "$GITHUB_OUTPUT"
 
       - uses: actions/setup-python@v4
@@ -45,7 +47,7 @@ jobs:
         run: |
           echo ${{ inputs.dryrun && '💡 Running in dry-run mode' || 'Preparing release...' }}
 
-          CHANGE_FILE=${{ env.CHANGES_FILE }}
+          CHANGE_FILE=${{ env.CHANGE_FILE }}
           LAST_VERSION=${{ steps.get-version.outputs.LAST_VERSION }}
           git config user.name github-actions
           git config user.email github-actions@github.com

CHANGES.rst

@@ -4,6 +4,22 @@ Changelog
 3.0.0 (tbd)
 -----------
 
+2.1.1 (2024-04-24)
+--------------------------
+
+- 31c9cf1 FIX: deprecated `setuptools.py` when building in `package.sh` (#568)
+- 2619c17 FIX: use the right env variables in `release-tag` workflow (#569)
+
+2.1.0 (2024-04-24)
+--------------------------
+
+- d588913 ENH: Bump github action versions and add multiarch support (#553)
+- a558dbc ENH:  Handle tar.xz archives (#536)
+- 2f0a56c FIX: support Python 3.12 (#539)
+- 84bf12c MAINT: make the last supported python version explicit in `ci.yaml` (#558)
+- 946fbfe MAINT: Update setuptools requirement from <62.0.0,>=40.0 to >=40.0,<70.0.0 in /requirements (#557)
+- 50c7a78 MAINT: add tar xz test case (#538)
+
 2.0.1 (2023-10-01)
 --------------------------
 

README.md

@@ -9,8 +9,8 @@
 
 | name        | description                                                                                                                                                                                                                                     |
 | :---------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Version     | 2.0.1                                                                                                                                                                                                                                           |
+| Version     | 2.1.1                                                                                                                                                                                                                                           |
-| Date:       | 2023-10-01                                                                                                                                                                                                                                      |
+| Date:       | 2024-04-25                                                                                                                                                                                                                                      |
 | Source      | <https://github.com/pypiserver/pypiserver>                                                                                                                                                                                                      |
 | PyPI        | <https://pypi.org/project/pypiserver/>                                                                                                                                                                                                          |
 | Tests       | <https://github.com/pypiserver/pypiserver/actions>                                                                                                                                                                                              |

bin/package.sh

@@ -6,4 +6,4 @@ my_dir=`dirname "$0"`
 cd $my_dir/..
 
 rm -r build/* dist/* || echo "no build/* or dist/* folder is found"
-python3 setup.py bdist_wheel sdist
+python3 -m build

docker/docker-requirements.txt

@@ -8,4 +8,4 @@ bcrypt==3.2.0
 # whatever bottle chooses as a default. Since the wsgiref server is not
 # production-ready, install waitress as a fallback for these cases.
 waitress==2.1.2
-watchdog==4.0.0
+watchdog==1.0.2

pypiserver/__init__.py

@@ -7,9 +7,9 @@ import typing as t
 from pypiserver.bottle import Bottle
 from pypiserver.config import Config, RunConfig, strtobool
 
-version = __version__ = "2.0.1"
+version = __version__ = "2.1.1"
 __version_info__ = tuple(_re.split("[.-]", __version__))
-__updated__ = "2023-10-01 16:14:10"
+__updated__ = "2024-04-25 01:23:25"
 
 __title__ = "pypiserver"
 __summary__ = "A minimal PyPI server for use with pip/easy_install."

pypiserver/_app.py

@@ -14,6 +14,7 @@ from urllib.parse import urljoin, urlparse
 from pypiserver.config import RunConfig
 from . import __version__
 from . import core
+from . import mirror_cache
 from .bottle import (
     static_file,
     redirect,
@@ -286,7 +287,9 @@ def simple(project):
         key=lambda x: (x.parsed_version, x.relfn),
     )
     if not packages:
-        if not config.disable_fallback:
+        if config.mirror:
+            return mirror_cache.MirrorCache.add(project=project, config=config)
+        elif not config.disable_fallback:
             return redirect(f"{config.fallback_url.rstrip('/')}/{project}/")
         return HTTPError(404, f"Not Found ({normalized} does not exist)\n\n")
 
@@ -364,7 +367,8 @@ def server_static(filename):
                     "Cache-Control", f"public, max-age={config.cache_control}"
                 )
             return response
-
+    if config.mirror and mirror_cache.MirrorCache.has_project(filename):
+        return mirror_cache.MirrorCache.get_static_file(filename=filename, config=config)
     return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
 
 

pypiserver/config.py

@@ -517,6 +517,14 @@ def get_parser() -> argparse.ArgumentParser:
             "to '%%s' to see them all."
         ),
     )
+    run_parser.add_argument(
+        "--mirror",
+        default=0,
+        action="count",
+        help=(
+            "Mirror packages to local disk"
+        ),
+    )
 
     update_parser = subparsers.add_parser(
         "update",
@@ -720,6 +728,7 @@ class RunConfig(_ConfigCommon):
         overwrite: bool,
         welcome_msg: str,
         cache_control: t.Optional[int],
+        mirror: bool,
         log_req_frmt: str,
         log_res_frmt: str,
         log_err_frmt: str,
@@ -745,6 +754,7 @@ class RunConfig(_ConfigCommon):
         # Derived properties
         self._derived_properties = self._derived_properties + ("auther",)
         self.auther = self.get_auther(auther)
+        self.mirror = mirror
 
     @classmethod
     def kwargs_from_namespace(
@@ -764,6 +774,7 @@ class RunConfig(_ConfigCommon):
             "overwrite": namespace.overwrite,
             "welcome_msg": namespace.welcome,
             "cache_control": namespace.cache_control,
+            "mirror": namespace.mirror,
             "log_req_frmt": namespace.log_req_frmt,
             "log_res_frmt": namespace.log_res_frmt,
             "log_err_frmt": namespace.log_err_frmt,

pypiserver/mirror_cache.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import logging
+from collections import OrderedDict
+from pypiserver.bottle import HTTPError, redirect
+from pypiserver.config import RunConfig
+log = logging.getLogger(__name__)
+try:
+    import requests
+    from bs4 import BeautifulSoup
+    import_ok = True
+except ImportError:
+    import_ok = False
+    logging.error("mirror_cache import dependencies error")
+
+
+class CacheElement:
+    def __init__(self, project: str):
+        self.project = project
+        self.html = ""
+        self.cache = dict()
+
+    def add(self, href: str):
+        targz = href.split("/")[-1]
+        pkg_name = targz.split("#")[0]
+        self.cache[f"{self.project}/{pkg_name}"] = href
+        return f"/packages/{self.project}/{targz}"
+
+
+class MirrorCache:
+    cache: OrderedDict[str, CacheElement] = dict()
+    cache_limit = 10
+
+    @classmethod
+    def add(cls, project: str, config: RunConfig) -> str:
+        if not import_ok:
+            return redirect(f"{config.fallback_url.rstrip('/')}/{project}/")
+
+        if project in cls.cache:
+            log.info(f"mirror_cache serve html from cache {project}")
+            return cls.cache[project].html
+
+        element = CacheElement(project=project)
+
+        resp = requests.get(f"{config.fallback_url.rstrip('/')}/{project}/")
+        soup = BeautifulSoup(resp.content, "html.parser")
+        links = soup.find_all("a")
+        for link in links:
+            # new href with mapping to old href for later
+            new_href = element.add(href=link["href"])
+            # create new link
+            new_link = soup.new_tag("a")
+            new_link.string = link.text.strip()
+            new_link["href"] = new_href
+            link.replace_with(new_link)
+        element.html = str(soup)
+        cls.cache[project] = element
+        log.info(f"mirror_cache add project '{project}' to cache")
+        # purge
+        if len(cls.cache) > cls.cache_limit:
+            item = cls.cache.popitem(last=False)
+            log.info(f"mirror_cache limit '{cls.cache_limit}' exceeded, purged last item - {item}")
+        return element.html
+
+    @classmethod
+    def has_project(cls, filename):
+        project = filename.split("/")[0]
+        return project in cls.cache
+
+    @classmethod
+    def get_static_file(cls, filename, config: RunConfig):
+        if not import_ok:
+            return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
+        project = filename.split("/")[0]
+        element = cls.cache[project]
+        if filename in element.cache:
+            href = element.cache[filename]
+            resp = requests.get(href)
+            cls.add_to_cache(filename=filename, resp=resp, config=config)
+            return resp
+        log.info(f"mirror_cache not found in cache {filename} ")
+        return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
+
+    @classmethod
+    def add_to_cache(cls, filename: str, resp: requests.Response, config: RunConfig):
+        project = filename.split("/")[0]
+        os.makedirs(os.path.join(config.package_root, project), exist_ok=True)
+        log.info(f"mirror_cache add file '{filename}' to cache")
+        with open(f"{config.package_root}/{filename}", "wb+") as f:
+            f.write(resp.content)

requirements/mirror-cache-requirements.txt

@@ -0,0 +1,2 @@
+beautifulsoup4==4.12.3
+requests==2.31.0