Merge branch 'master' of git.fnexe.com:github.com/pypiserver

chore(ver): bump 2.1.0-->2.1.1
chore(release-candidate): v2.1.1 (#570 )
2024-05-01 01:13:33 +02:00 · 2024-04-25 01:23:26 +02:00 · 2024-04-25 01:17:42 +02:00 · 2024-04-25 01:07:24 +02:00 · 2024-04-25 00:55:36 +02:00 · 2024-04-24 18:30:24 +02:00
19 changed files with 305 additions and 112 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -12,13 +12,26 @@ on:
  # Allowing to run on fork and other pull requests
  pull_request:

+env:
+  LAST_SUPPORTED_PYTHON: "3.12"
+
 jobs:
  test-python:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "pypy3.9", "3.11"] # "3.12-dev"
+        # make sure to align the `python-version`s in the Matrix with env.LAST_SUPPORTED_PYTHON
+        python-version: [
+            "3.7",
+            "3.8",
+            "3.9",
+            "3.10",
+            "pypy3.9",
+            "3.11",
+            "3.12",
+            "3.x", # make sure to test the current stable Python version
+          ]

    steps:
      - uses: actions/checkout@v3
@ -43,7 +56,7 @@ jobs:
        uses: actions/setup-python@v4
        with:
          # Use the current version of Python
-          python-version: "3.x"
+          python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
      - name: Install dependencies
        run: |
          pip install -r "requirements/dev.pip"
@ -90,7 +103,7 @@ jobs:
        uses: actions/setup-python@v4
        with:
          # Use the current version of Python
-          python-version: "3.x"
+          python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
      - name: Install test dependencies
        run: pip install -r "requirements/test.pip"
      - name: Install package
@ -116,14 +129,15 @@ jobs:
    runs-on: ubuntu-latest
    needs:
      - "tests"
-    # only if a tag is pushed
-    if: startsWith(github.event.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@master
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
-          python-version: 3.x
+          python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
+
+      - name: Install dev dependencies
+        run: pip install -r "requirements/dev.pip"

      - name: Build distribution _wheel_.
        run: |
@ -131,6 +145,8 @@ jobs:

      - name: Publish distribution 📦 to PyPI.
        uses: pypa/gh-action-pypi-publish@release/v1
+        # Push to PyPi only if a tag is pushed
+        if: startsWith(github.event.ref, 'refs/tags/v')
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
          print-hash: true
@ -154,7 +170,7 @@ jobs:

      - uses: "actions/setup-python@v4"
        with:
-          python-version: "3.x"
+          python-version: ${{ env.LAST_SUPPORTED_PYTHON }}

        # This script prints a JSON array of needed docker tags, depending on the
        # ref. That array is then used to construct the matrix of the
@ -195,27 +211,31 @@ jobs:
            ${{ runner.os }}-buildx-

      - name: "Login to Docker Hub"
-        uses: "docker/login-action@v1"
+        uses: "docker/login-action@v3"
        with:
          username: "${{ secrets.DOCKER_HUB_USER }}"
          password: "${{ secrets.DOCKER_HUB_TOKEN }}"

      - name: "Login to GitHub Container Registry"
-        uses: "docker/login-action@v2"
+        uses: "docker/login-action@v3"
        with:
          registry: "ghcr.io"
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

+      - name: "Set up QEMU"
+        uses: "docker/setup-qemu-action@v3"
+
      - name: "Set up Docker Buildx"
        id: "buildx"
-        uses: "docker/setup-buildx-action@v1"
+        uses: "docker/setup-buildx-action@v3"

      - name: "Build and push"
        id: "docker_build"
-        uses: "docker/build-push-action@v2"
+        uses: "docker/build-push-action@v5"
        with:
          context: "./"
+          platforms: linux/amd64,linux/arm64
          file: "./Dockerfile"
          builder: "${{ steps.buildx.outputs.name }}"
          push: true
--- a/.github/workflows/rt.yml
+++ b/.github/workflows/rt.yml
@ -22,14 +22,16 @@ jobs:
    if: ${{ github.ref_name == 'master' }}
    runs-on: ubuntu-latest
    env:
-      CHANGES_FILE: CHANGES.rst
+      CHANGE_FILE: CHANGES.rst
      EXPECTED_DIFF_COUNT: 1
    steps:
      - uses: actions/checkout@v3

      - id: get-version
        run: |
+          CHANGE_FILE=${{ env.CHANGE_FILE }}
          LAST_VERSION=$(grep -m1 -E ' \([0-9]+-[0-9]+-[0-9]+\)$' ${CHANGE_FILE} | awk '{ print $1 }')
+          echo "👀 Version detected: ${LAST_VERSION}"
          echo "LAST_VERSION=${LAST_VERSION}" >> "$GITHUB_OUTPUT"

      - uses: actions/setup-python@v4
@ -45,7 +47,7 @@ jobs:
        run: |
          echo ${{ inputs.dryrun && '💡 Running in dry-run mode' || 'Preparing release...' }}

-          CHANGE_FILE=${{ env.CHANGES_FILE }}
+          CHANGE_FILE=${{ env.CHANGE_FILE }}
          LAST_VERSION=${{ steps.get-version.outputs.LAST_VERSION }}
          git config user.name github-actions
          git config user.email github-actions@github.com
--- a/CHANGES.rst
+++ b/CHANGES.rst
@ -4,6 +4,22 @@ Changelog
 3.0.0 (tbd)
 -----------

+2.1.1 (2024-04-24)
+--------------------------
+
+- 31c9cf1 FIX: deprecated `setuptools.py` when building in `package.sh` (#568)
+- 2619c17 FIX: use the right env variables in `release-tag` workflow (#569)
+
+2.1.0 (2024-04-24)
+--------------------------
+
+- d588913 ENH: Bump github action versions and add multiarch support (#553)
+- a558dbc ENH:  Handle tar.xz archives (#536)
+- 2f0a56c FIX: support Python 3.12 (#539)
+- 84bf12c MAINT: make the last supported python version explicit in `ci.yaml` (#558)
+- 946fbfe MAINT: Update setuptools requirement from <62.0.0,>=40.0 to >=40.0,<70.0.0 in /requirements (#557)
+- 50c7a78 MAINT: add tar xz test case (#538)
+
 2.0.1 (2023-10-01)
 --------------------------

--- a/README.md
+++ b/README.md
@ -9,14 +9,14 @@

 | name        | description                                                                                                                                                                                                                                     |
 | :---------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Version     | 2.0.1                                                                                                                                                                                                                                           |
-| Date:       | 2023-10-01                                                                                                                                                                                                                                      |
-| Source      | https://github.com/pypiserver/pypiserver                                                                                                                                                                                                        |
-| PyPI        | https://pypi.org/project/pypiserver/                                                                                                                                                                                                            |
-| Tests       | https://github.com/pypiserver/pypiserver/actions                                                                                                                                                                                                |
+| Version     | 2.1.1                                                                                                                                                                                                                                           |
+| Date:       | 2024-04-25                                                                                                                                                                                                                                      |
+| Source      | <https://github.com/pypiserver/pypiserver>                                                                                                                                                                                                      |
+| PyPI        | <https://pypi.org/project/pypiserver/>                                                                                                                                                                                                          |
+| Tests       | <https://github.com/pypiserver/pypiserver/actions>                                                                                                                                                                                              |
 | Maintainers | Kostis Anagnostopoulos <ankostis@gmail.com>, Matthew Planchard <mplanchard@gmail.com>,  Dmitrii Orlov <dmtree.dev@yahoo.com>,  **Someone new?** We are looking for new maintainers! [#397](https://github.com/pypiserver/pypiserver/issues/397) |
 | License     | zlib/libpng + MIT                                                                                                                                                                                                                               |
-| Community   | https://pypiserver.zulipchat.com                                                                                                                                                                                                                |
+| Community   | <https://pypiserver.zulipchat.com>                                                                                                                                                                                                              |

 Chat with us on [Zulip](https://pypiserver.zulipchat.com)!

@ -44,47 +44,47 @@ making it much easier to get a running index server.

 Table of Contents

- [pypiserver - minimal PyPI server for use with pip/easy_install](#pypiserver---minimal-pypi-server-for-use-with-pipeasy_install)
-  - [Quickstart Installation and Usage](#Quickstart-Installation-and-Usage)
-    - [More details about pypi-server run](#More-details-about-pypi-server-run)
-    - [More details about pypi-server update](#More-details-about-pypi-server-update)
-  - [Client-Side Configurations](#Client-Side-Configurations)
-    - [Configuring pip](#Configuring-pip)
-    - [Configuring easy_install](#Configuring-easy_install)
-  - [Uploading Packages Remotely](#Uploading-Packages-Remotely)
-    - [Apache like Authentication ( htpasswd )](#Apache-like-Authentication)
-    - [Upload with setuptools](#Upload-with-setuptools)
-    - [Upload with twine](#Upload-with-twine)
-  - [Using the Docker Image](#Using-the-Docker-Image)
-  - [Alternative Installation methods](#Alternative-Installation-methods)
-    - [Installing the Very Latest Version](#Installing-the-Very-Latest-Version)
-  - [Recipes](#Recipes)
-    - [Managing the Package Directory](#Managing-the-Package-Directory)
-    - [Serving Thousands of Packages](#Serving-Thousands-of-Packages)
-    - [Managing Automated Startup](#Managing-Automated-Startup)
-      - [Running as a systemd service](#Running-as-a-systemd-service)
-      - [Launching through supervisor](#Launching-through-supervisor)
-      - [Running as a service with NSSM (Windows)](#Running-as-a-service-with-NSSM)
-    - [Using a Different WSGI Server](#Using-a-Different-WSGI-Server)
-      - [Apache](#Apache)
-      - [Gunicorn](#Gunicorn)
-      - [Paste](#Paste)
-    - [Behind a Reverse Proxy](#Behind-a-Reverse-Proxy)
-      - [Nginx](#Nginx)
-      - [Supporting HTTPS](#Supporting-HTTPS)
-      - [Traefik](#Traefik)
-    - [Utilizing the API](#Utilizing-the-API)
-      - [Using Ad-Hoc Authentication Providers](#Using-Ad-Hoc-Authentication-Providers)
-    - [Use with MicroPython](#Use-with-MicroPython)
-    - [Custom Health Check Endpoint](#Custom-Health-Check-Endpoint)
-      - [Configure a custom health check by CLI arguments](#Configure-a-custom-health-check-by-CLI-arguments)
-      - [Configure a custom health endpoint by script](#Configure-a-custom-health-endpoint-by-script)
-    - [Sources](#Sources)
-    - [Known Limitations](#known-limitations)
-    - [Similar Projects](#similar-projects)
-      - [Unmaintained or archived](#unmaintained-or-archived)
-    - [Related Projects](#related-projects)
-    - [License](#license)
+- [pypiserver](#pypiserver)
+  - [Quickstart Installation and Usage](#quickstart-installation-and-usage)
+    - [More details about pypi server run](#more-details-about-pypi-server-run)
+    - [More details about pypi-server update](#more-details-about-pypi-server-update)
+  - [Client-Side Configurations](#client-side-configurations)
+    - [Configuring pip](#configuring-pip)
+    - [Configuring easy_install](#configuring-easy_install)
+    - [Uploading Packages Remotely](#uploading-packages-remotely)
+      - [Apache Like Authentication (htpasswd)](#apache-like-authentication-htpasswd)
+      - [Upload with setuptools](#upload-with-setuptools)
+      - [Upload with twine](#upload-with-twine)
+  - [Using the Docker Image](#using-the-docker-image)
+  - [Alternative Installation Methods](#alternative-installation-methods)
+    - [Installing the Very Latest Version](#installing-the-very-latest-version)
+  - [Recipes](#recipes)
+    - [Managing the Package Directory](#managing-the-package-directory)
+    - [Serving Thousands of Packages](#serving-thousands-of-packages)
+    - [Managing Automated Startup](#managing-automated-startup)
+      - [Running As a systemd Service](#running-as-a-systemd-service)
+      - [Launching through supervisor](#launching-through-supervisor)
+      - [Running As a service with NSSM](#running-as-a-service-with-nssm)
+    - [Using a Different WSGI Server](#using-a-different-wsgi-server)
+      - [Apache](#apache)
+      - [gunicorn](#gunicorn)
+      - [paste](#paste)
+    - [Behind a Reverse Proxy](#behind-a-reverse-proxy)
+      - [Nginx](#nginx)
+      - [Supporting HTTPS](#supporting-https)
+      - [Traefik](#traefik)
+    - [Utilizing the API](#utilizing-the-api)
+      - [Using Ad-Hoc Authentication Providers](#using-ad-hoc-authentication-providers)
+    - [Use with MicroPython](#use-with-micropython)
+    - [Custom Health Check Endpoint](#custom-health-check-endpoint)
+      - [Configure a custom health endpoint by CLI arguments](#configure-a-custom-health-endpoint-by-cli-arguments)
+      - [Configure a custom health endpoint by script](#configure-a-custom-health-endpoint-by-script)
+  - [Sources](#sources)
+  - [Known Limitations](#known-limitations)
+  - [Similar Projects](#similar-projects)
+    - [Unmaintained or archived](#unmaintained-or-archived)
+  - [Related Software](#related-software)
+- [Licensing](#licensing)

 ## Quickstart Installation and Usage

@ -134,7 +134,7 @@ See also [Alternative Installation methods](<>)
   # Note that pip search does not currently work with the /simple/ endpoint.
 ```

-See also [Client-side configurations](#Client-Side-Configurations) for avoiding tedious typing.
+See also [Client-side configurations](#client-side-configurations) for avoiding tedious typing.

 4. Enter **pypi-server -h** in the cmd-line to print a detailed usage message

@ -461,7 +461,7 @@ Please see `Using Ad-hoc authentication providers`\_ for more information.
     password: <some_passwd>
 ```

-2. Then from within the directory of the python-project you wish to upload,
+1. Then from within the directory of the python-project you wish to upload,
   issue this command:

 ```shell
@ -693,7 +693,7 @@ Adjusting the paths and adding this file as **pypiserver.service** into your
 **systemctl**, e.g. **systemctl start pypiserver**.

 More useful information about *systemd* can be found at
-https://www.digitalocean.com/community/tutorials/how-to-use-systemctl-to-manage-systemd-services-and-units
+<https://www.digitalocean.com/community/tutorials/how-to-use-systemctl-to-manage-systemd-services-and-units>

 #### Launching through supervisor

@ -716,7 +716,7 @@ From there, the process can be managed via **supervisord** using **supervisorctl

 #### Running As a service with NSSM

-For Windows download NSSM from https://nssm.cc unzip to a desired location such as Program Files. Decide whether you are going
+For Windows download NSSM from <https://nssm.cc> unzip to a desired location such as Program Files. Decide whether you are going
 to use win32 or win64, and add that exe to environment PATH.

 Create a start_pypiserver.bat
@ -762,7 +762,7 @@ Other useful commands

 ```

-For detailed information please visit https://nssm.cc
+For detailed information please visit <https://nssm.cc>

 ### Using a Different WSGI Server

@ -1057,7 +1057,7 @@ these steps:
 3. Invoke the python-script to start-up **pypiserver**

 ```shell
-   $ python pypiserver-start.py
+   python pypiserver-start.py
 ```

 Note
@ -1092,7 +1092,7 @@ Installing packages from the REPL of an embedded device works in this way:
    upip.install("micropython-foobar")
 ```

-Further information on micropython-packaging can be found here: https://docs.micropython.org/en/latest/reference/packages.html
+Further information on micropython-packaging can be found here: <https://docs.micropython.org/en/latest/reference/packages.html>

 ### Custom Health Check Endpoint

@ -1127,7 +1127,7 @@ Run pypiserver with **--health-endpoint** argument:
    bottle.run(app=app, host="0.0.0.0", port=8080, server="auto")
 ````

-Try **curl http://localhost:8080/action/health**
+Try **curl <http://localhost:8080/action/health>**

 ## Sources

--- a/bin/package.sh
+++ b/bin/package.sh
@ -6,4 +6,4 @@ my_dir=`dirname "$0"`
 cd $my_dir/..

 rm -r build/* dist/* || echo "no build/* or dist/* folder is found"
-python3 setup.py bdist_wheel sdist
+python3 -m build
--- a/docker/test_docker.py
+++ b/docker/test_docker.py
@ -396,6 +396,7 @@ class TestBasics:
            "-m",
            "pip",
            "install",
+            "--force-reinstall",
            "--index-url",
            f"http://localhost:{container.port}/simple",
            TEST_DEMO_PIP_PACKAGE,
@ -562,6 +563,7 @@ class TestAuthed:
            "-m",
            "pip",
            "install",
+            "--force-reinstall",
            "--index-url",
            f"http://a:a@localhost:{self.HOST_PORT}/simple",
            TEST_DEMO_PIP_PACKAGE,
@ -581,9 +583,10 @@ class TestAuthed:
            "-m",
            "pip",
            "install",
+            "--force-reinstall",
            "--no-cache",
            "--index-url",
-            f"http://localhost:{self.HOST_PORT}/simple",
+            f"http://foo:bar@localhost:{self.HOST_PORT}/simple",
            TEST_DEMO_PIP_PACKAGE,
            check_code=lambda c: c != 0,
        )
--- a/pypiserver/init.py
+++ b/pypiserver/init.py
@ -7,9 +7,9 @@ import typing as t
 from pypiserver.bottle import Bottle
 from pypiserver.config import Config, RunConfig, strtobool

-version = __version__ = "2.0.1"
+version = __version__ = "2.1.1"
 __version_info__ = tuple(_re.split("[.-]", __version__))
-__updated__ = "2023-10-01 16:14:10"
+__updated__ = "2024-04-25 01:23:25"

 __title__ = "pypiserver"
 __summary__ = "A minimal PyPI server for use with pip/easy_install."
--- a/pypiserver/_app.py
+++ b/pypiserver/_app.py
@ -14,6 +14,7 @@ from urllib.parse import urljoin, urlparse
 from pypiserver.config import RunConfig
 from . import __version__
 from . import core
+from . import mirror_cache
 from .bottle import (
    static_file,
    redirect,
@ -286,7 +287,9 @@ def simple(project):
        key=lambda x: (x.parsed_version, x.relfn),
    )
    if not packages:
-        if not config.disable_fallback:
+        if config.mirror:
+            return mirror_cache.MirrorCache.add(project=project, config=config)
+        elif not config.disable_fallback:
            return redirect(f"{config.fallback_url.rstrip('/')}/{project}/")
        return HTTPError(404, f"Not Found ({normalized} does not exist)\n\n")

@ -364,7 +367,8 @@ def server_static(filename):
                    "Cache-Control", f"public, max-age={config.cache_control}"
                )
            return response
-
+    if config.mirror and mirror_cache.MirrorCache.has_project(filename):
+        return mirror_cache.MirrorCache.get_static_file(filename=filename, config=config)
    return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")


--- a/pypiserver/config.py
+++ b/pypiserver/config.py
@ -43,9 +43,26 @@ import re
 import sys
 import textwrap
 import typing as t
-from distutils.util import strtobool as strtoint

-import pkg_resources
+try:
+    # `importlib_resources` is required for Python versions below 3.12
+    # See more in the package docs: https://pypi.org/project/importlib-resources/
+    try:
+        from importlib_resources import files as import_files
+    except ImportError:
+        from importlib.resources import files as import_files
+
+    def get_resource_bytes(package: str, resource: str) -> bytes:
+        ref = import_files(package).joinpath(resource)
+        return ref.read_bytes()
+
+except ImportError:
+    # The `pkg_resources` is deprecated in Python 3.12
+    import pkg_resources
+
+    def get_resource_bytes(package: str, resource: str) -> bytes:
+        return pkg_resources.resource_string(package, resource)
+

 from pypiserver.backend import (
    SimpleFileBackend,
@ -63,10 +80,29 @@ except ImportError:
    HtpasswdFile = None


-# The "strtobool" function in distutils does a nice job at parsing strings,
-# but returns an integer. This just wraps it in a boolean call so that we
-# get a bool.
-strtobool: t.Callable[[str], bool] = lambda val: bool(strtoint(val))
+def legacy_strtoint(val: str) -> int:
+    """Convert a string representation of truth to true (1) or false (0).
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
+    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
+    'val' is anything else.
+
+    The "strtobool" function in distutils does a nice job at parsing strings,
+    but returns an integer. This just wraps it in a boolean call so that we
+    get a bool.
+
+    Borrowed from deprecated distutils.
+    """
+    val = val.lower()
+    if val in ("y", "yes", "t", "true", "on", "1"):
+        return 1
+    elif val in ("n", "no", "f", "false", "off", "0"):
+        return 0
+    else:
+        raise ValueError("invalid truth value {!r}".format(val))
+
+
+strtobool: t.Callable[[str], bool] = lambda val: bool(legacy_strtoint(val))


 # Specify defaults here so that we can use them in tests &c. and not need
@ -151,9 +187,7 @@ def health_endpoint_arg(arg: str) -> str:
 def html_file_arg(arg: t.Optional[str]) -> str:
    """Parse the provided HTML file and return its contents."""
    if arg is None or arg == "pypiserver/welcome.html":
-        return pkg_resources.resource_string(__name__, "welcome.html").decode(
-            "utf-8"
-        )
+        return get_resource_bytes(__name__, "welcome.html").decode("utf-8")
    with open(arg, "r", encoding="utf-8") as f:
        msg = f.read()
    return msg
@ -483,6 +517,14 @@ def get_parser() -> argparse.ArgumentParser:
            "to '%%s' to see them all."
        ),
    )
+    run_parser.add_argument(
+        "--mirror",
+        default=0,
+        action="count",
+        help=(
+            "Mirror packages to local disk"
+        ),
+    )

    update_parser = subparsers.add_parser(
        "update",
@ -686,6 +728,7 @@ class RunConfig(_ConfigCommon):
        overwrite: bool,
        welcome_msg: str,
        cache_control: t.Optional[int],
+        mirror: bool,
        log_req_frmt: str,
        log_res_frmt: str,
        log_err_frmt: str,
@ -711,6 +754,7 @@ class RunConfig(_ConfigCommon):
        # Derived properties
        self._derived_properties = self._derived_properties + ("auther",)
        self.auther = self.get_auther(auther)
+        self.mirror = mirror

    @classmethod
    def kwargs_from_namespace(
@ -730,6 +774,7 @@ class RunConfig(_ConfigCommon):
            "overwrite": namespace.overwrite,
            "welcome_msg": namespace.welcome,
            "cache_control": namespace.cache_control,
+            "mirror": namespace.mirror,
            "log_req_frmt": namespace.log_req_frmt,
            "log_res_frmt": namespace.log_res_frmt,
            "log_err_frmt": namespace.log_err_frmt,
--- a/pypiserver/manage.py
+++ b/pypiserver/manage.py
@ -5,7 +5,8 @@ from __future__ import absolute_import, print_function, unicode_literals
 import itertools
 import os
 import sys
-from distutils.version import LooseVersion
+
+from packaging.version import parse as packaging_parse
 from pathlib import Path
 from subprocess import call
 from xmlrpc.client import Server
@ -112,12 +113,14 @@ class PipCmd:

    @staticmethod
    def update_root(pip_version):
-        """Yield an appropriate root command depending on pip version."""
-        # legacy_pip = StrictVersion(pip_version) < StrictVersion('10.0')
-        legacy_pip = LooseVersion(pip_version) < LooseVersion("10.0")
-        for part in ("pip", "-q"):
+        """Yield an appropriate root command depending on pip version.
+
+        Use `pip install` for `pip` 9 or lower, and `pip download` otherwise.
+        """
+        legacy_pip = packaging_parse(pip_version).major < 10
+        pip_command = "install" if legacy_pip else "download"
+        for part in ("pip", "-q", pip_command):
            yield part
-        yield "install" if legacy_pip else "download"

    @staticmethod
    def update(
--- a/pypiserver/mirror_cache.py
+++ b/pypiserver/mirror_cache.py
@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import logging
+from collections import OrderedDict
+from pypiserver.bottle import HTTPError, redirect
+from pypiserver.config import RunConfig
+log = logging.getLogger(__name__)
+try:
+    import requests
+    from bs4 import BeautifulSoup
+    import_ok = True
+except ImportError:
+    import_ok = False
+    logging.error("mirror_cache import dependencies error")
+
+
+class CacheElement:
+    def __init__(self, project: str):
+        self.project = project
+        self.html = ""
+        self.cache = dict()
+
+    def add(self, href: str):
+        targz = href.split("/")[-1]
+        pkg_name = targz.split("#")[0]
+        self.cache[f"{self.project}/{pkg_name}"] = href
+        return f"/packages/{self.project}/{targz}"
+
+
+class MirrorCache:
+    cache: OrderedDict[str, CacheElement] = dict()
+    cache_limit = 10
+
+    @classmethod
+    def add(cls, project: str, config: RunConfig) -> str:
+        if not import_ok:
+            return redirect(f"{config.fallback_url.rstrip('/')}/{project}/")
+
+        if project in cls.cache:
+            log.info(f"mirror_cache serve html from cache {project}")
+            return cls.cache[project].html
+
+        element = CacheElement(project=project)
+
+        resp = requests.get(f"{config.fallback_url.rstrip('/')}/{project}/")
+        soup = BeautifulSoup(resp.content, "html.parser")
+        links = soup.find_all("a")
+        for link in links:
+            # new href with mapping to old href for later
+            new_href = element.add(href=link["href"])
+            # create new link
+            new_link = soup.new_tag("a")
+            new_link.string = link.text.strip()
+            new_link["href"] = new_href
+            link.replace_with(new_link)
+        element.html = str(soup)
+        cls.cache[project] = element
+        log.info(f"mirror_cache add project '{project}' to cache")
+        # purge
+        if len(cls.cache) > cls.cache_limit:
+            item = cls.cache.popitem(last=False)
+            log.info(f"mirror_cache limit '{cls.cache_limit}' exceeded, purged last item - {item}")
+        return element.html
+
+    @classmethod
+    def has_project(cls, filename):
+        project = filename.split("/")[0]
+        return project in cls.cache
+
+    @classmethod
+    def get_static_file(cls, filename, config: RunConfig):
+        if not import_ok:
+            return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
+        project = filename.split("/")[0]
+        element = cls.cache[project]
+        if filename in element.cache:
+            href = element.cache[filename]
+            resp = requests.get(href)
+            cls.add_to_cache(filename=filename, resp=resp, config=config)
+            return resp
+        log.info(f"mirror_cache not found in cache {filename} ")
+        return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
+
+    @classmethod
+    def add_to_cache(cls, filename: str, resp: requests.Response, config: RunConfig):
+        project = filename.split("/")[0]
+        os.makedirs(os.path.join(config.package_root, project), exist_ok=True)
+        log.info(f"mirror_cache add file '{filename}' to cache")
+        with open(f"{config.package_root}/{filename}", "wb+") as f:
+            f.write(resp.content)
--- a/pypiserver/pkg_helpers.py
+++ b/pypiserver/pkg_helpers.py
@ -58,7 +58,7 @@ def is_listed_path(path_part: t.Union[PurePath, str]) -> bool:


 _archive_suffix_rx = re.compile(
-    r"(\.zip|\.tar\.gz|\.tgz|\.tar\.bz2|-py[23]\.\d-.*|"
+    r"(\.zip|\.tar\.gz|\.tgz|\.tar\.bz2|\.tar\.xz|-py[23]\.\d-.*|"
    r"\.win-amd64-py[23]\.\d\..*|\.win32-py[23]\.\d\..*|\.egg)$",
    re.I,
 )
--- a/pypiserver/plugin.py
+++ b/pypiserver/plugin.py
@ -18,6 +18,7 @@ with the following keyword arguments
 In the future, the plugin callable may be called with additional keyword
 arguments, so a plugin should accept a **kwargs variadic keyword argument.
 """
+
 from pypiserver.backend import SimpleFileBackend, CachingFileBackend
 from pypiserver import get_file_backend

--- a/requirements/mirror-cache-requirements.txt
+++ b/requirements/mirror-cache-requirements.txt
@ -0,0 +1,2 @@
+beautifulsoup4==4.12.3
+requests==2.31.0
--- a/requirements/test-requirements.txt
+++ b/requirements/test-requirements.txt
@ -4,11 +4,12 @@ pip
 passlib>=1.6
 pytest>=6.2.2
 pytest-cov
-setuptools>=40.0,<62.0.0
+setuptools>=40.0,<70.0.0
 tox
 twine
 webtest
 wheel>=0.25.0
+build>=1.2.0; python_version >= '3.8'
 mdformat-gfm
 mdformat-frontmatter
 mdformat-footnote
--- a/setup.py
+++ b/setup.py
@ -11,10 +11,19 @@ tests_require = [
    "twine",
    "passlib>=1.6",
    "webtest",
+    "build>=1.2.0;python_version>='3.8'",
 ]

-setup_requires = ["setuptools", "setuptools-git >= 0.3", "wheel >= 0.25.0"]
-install_requires = ["pip>=7"]
+setup_requires = [
+    "setuptools",
+    "setuptools-git>=0.3",
+    "wheel>=0.25.0",
+]
+install_requires = [
+    "pip>=7",
+    "packaging>=23.2",
+    "importlib_resources;python_version>'3.8' and python_version<'3.12'",
+]


 def read_file(rel_path: str):
--- a/tests/test_init.py
+++ b/tests/test_init.py
@ -1,6 +1,7 @@
 """
 Test module for app initialization
 """
+
 # Standard library imports
 import logging
 import pathlib
--- a/tests/test_pkg_helpers.py
+++ b/tests/test_pkg_helpers.py
@ -10,6 +10,7 @@ files = [
    ("pytz-2012b.tgz", "pytz", "2012b"),
    ("pytz-2012b.ZIP", "pytz", "2012b"),
    ("pytz-2012a.zip", "pytz", "2012a"),
+    ("pytz-2012b.tar.xz", "pytz", "2012b"),
    ("gevent-1.0b1.win32-py2.6.exe", "gevent", "1.0b1"),
    ("gevent-1.0b1.win32-py2.7.msi", "gevent", "1.0b1"),
    ("greenlet-0.3.4-py3.1-win-amd64.egg", "greenlet", "0.3.4"),
--- a/tests/test_server.py
+++ b/tests/test_server.py
@ -16,6 +16,7 @@ import itertools
 import os
 import shutil
 import socket
+import re
 import sys
 import time
 import typing as t
@ -99,10 +100,14 @@ def build_url(port: t.Union[int, str], user: str = "", pswd: str = "") -> str:
    return f"http://{auth}localhost:{port}"


-def run_setup_py(path: Path, arguments: str):
+def run_setup_py(path: Path, arguments: str) -> int:
    return os.system(f"{sys.executable} {path / 'setup.py'} {arguments}")


+def run_py_build(srcdir: Path, flags: str) -> int:
+    return os.system(f"{sys.executable} -m build {flags} {srcdir}")
+
+
 # A test-distribution to check if
 #    bottle supports uploading 100's of packages,
 #    see: https://github.com/pypiserver/pypiserver/issues/82
@ -140,8 +145,13 @@ def server_root(tmp_path_factory):
@pytest.fixture(scope="module")
 def wheel_file(project, tmp_path_factory):
    distdir = tmp_path_factory.mktemp("dist")
-    assert run_setup_py(project, f"bdist_wheel -d {distdir}") == 0
-    return list(distdir.glob("centodeps*.whl"))[0]
+    if re.match("^3\.7", sys.version):
+        assert run_setup_py(project, f"bdist_wheel -d {distdir}") == 0
+    else:
+        assert run_py_build(project, f"--wheel --outdir {distdir}") == 0
+    wheels = list(distdir.glob("centodeps*.whl"))
+    assert len(wheels) > 0
+    return wheels[0]


@pytest.fixture()
@ -317,22 +327,6 @@ def test_pip_install_authed_succeeds(authed_server, hosted_wheel_file, pipdir):
    assert pipdir.joinpath(hosted_wheel_file.name).is_file()


-@pytest.mark.parametrize("pkg_frmt", ["bdist", "bdist_wheel"])
-@pytest.mark.parametrize(["server_fixture", "pypirc_fixture"], all_servers)
-def test_setuptools_upload(
-    server_fixture, pypirc_fixture, project, pkg_frmt, server_root, request
-):
-    request.getfixturevalue(server_fixture)
-    request.getfixturevalue(pypirc_fixture)
-
-    assert len(list(server_root.iterdir())) == 0
-
-    for i in range(5):
-        print(f"++Attempt #{i}")
-        assert run_setup_py(project, f"-vvv {pkg_frmt} upload -r test") == 0
-    assert len(list(server_root.iterdir())) == 1
-
-
 def test_partial_authed_open_download(partial_authed_server):
    """Validate that partial auth still allows downloads."""
    url = build_url(partial_authed_server.port) + "/simple"
Author	SHA1	Message	Date
Michal Szczepanski	f65bc5bf6e	Merge branch 'master' of git.fnexe.com:github.com/pypiserver	2024-05-01 01:13:33 +02:00
Dmitrii Orlov	acff1bbab8	chore(ver): bump 2.1.0-->2.1.1	2024-04-25 01:23:26 +02:00
github-actions[bot]	5ca6004ebe	chore(release-candidate): v2.1.1 (#570 ) * chore(rc-changes): update CHANGES.rst * chore: update changes.rst --------- Co-authored-by: github-actions <github-actions@github.com> Co-authored-by: Dmitrii Orlov <dmtree.dev@yahoo.com>	2024-04-25 01:17:42 +02:00
Mitja O	31c9cf14d1	fix: deprecated `setuptools.py` when building in `package.sh` (#568 ) * fix: deprecated-setuptools-in-package.sh * chore: include package build in non-tag ci * chore: install requirements before building	2024-04-25 01:07:24 +02:00
Mitja O	2619c17602	fix: use the right env variables in `release-tag` workflow (#569 ) fix: env variable	2024-04-25 00:55:36 +02:00
Dmitrii Orlov	d5886ae3d5	chore(ver): bump 2.0.1-->2.1.0	2024-04-24 18:30:24 +02:00
github-actions[bot]	6bfeddc1fc	chore(release-candidate): v2.1.0 (#567 ) * chore(rc-changes): update CHANGES.rst * chore: update CHANGES.rst * chore: bump to 2.1.0 --------- Co-authored-by: github-actions <github-actions@github.com> Co-authored-by: Dmitrii Orlov <dmtree.dev@yahoo.com>	2024-04-24 18:21:44 +02:00
Michal Szczepanski	4ddfc3a077	feat: add local pypi package mirror (#333 )	2024-04-15 14:58:39 +02:00
Mitja O	2f0a56c380	fix: support Python 3.12 (#539 ) * chore: pin explicit Python 3.12 * chore: add a `test-python` for stable Python * chore: empty commit * chore: add some FIXMEs * chore: add `packaging` * chore(wip): replace `LegacyVersion` with `packaging`'s `parse` * chore(wip): bypass `strtobool` usage * chore(wip): `pkg_resources` are deprecated * chore(wip): naive way to support Python <3.12 * chore(wip): swap import order * chore(wip): try fixing version check * chore: add a fixme * chore(wip): reverse legacy pip check * chore(wip): legacy pip check for 9 or lower * fix: fix the legacy pip check * chore: small cleanup * chore(wip): try the `importlib_resources` * chore: add small comment * chore(wip): avoid `setup.py` in fixtures * chore(wip): version-compatible wheel build * chore: install `build` for `3.8` too * fix: mypy issues * chore: fix comments * fix: more formatting fixes * fix: mdformat * fix: pass wrong auth to `failed_auth` test * chore: cleanup packages before and after test runs * chore(wip): try to bypass test error * chore: add a tech debt comment * chore: undo too many changes * chore(wip): small debug experiment * chore(wip): skip some tests * chore(wip): use nonsense code * fix(chore): small fix to the nonsense code * chore(wip): try `--force-reinstall` * chore: finalize the docker tests	2024-04-01 21:30:02 +02:00
Mitja O	84bf12cdd4	chore: make the last supported python version explicit in `ci.yaml` (#558 ) * chore: make the last supported python version explicit * fix: formatting	2024-04-01 12:23:42 +02:00
dependabot[bot]	946fbfe64e	chore: Update setuptools requirement from <62.0.0,>=40.0 to >=40.0,<70.0.0 in /requirements (#557 ) chore: Update setuptools requirement in /requirements Updates the requirements on [setuptools](https://github.com/pypa/setuptools) to permit the latest version. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v40.0.0...v69.2.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-04-01 12:14:44 +02:00
Chuck CBW	d588913e75	feat: Bump github action versions and add multiarch support (#553 ) feat: bump github action versions and add multiarch support	2024-04-01 11:33:17 +02:00
Mitja O	50c7a78f4f	chore: add tar xz test case (#538 )	2023-11-13 16:19:52 +01:00
Daniel M. Weeks	a558dbcfb2	Handle tar.xz archives (#536 )	2023-11-13 16:02:36 +01:00