From 2f0a56c380d7564c2de827cf6a4f8c2d096f72b2 Mon Sep 17 00:00:00 2001 From: Mitja O Date: Mon, 1 Apr 2024 21:30:02 +0200 Subject: [PATCH] fix: support Python 3.12 (#539) * chore: pin explicit Python 3.12 * chore: add a `test-python` for stable Python * chore: empty commit * chore: add some FIXMEs * chore: add `packaging` * chore(wip): replace `LegacyVersion` with `packaging`'s `parse` * chore(wip): bypass `strtobool` usage * chore(wip): `pkg_resources` are deprecated * chore(wip): naive way to support Python <3.12 * chore(wip): swap import order * chore(wip): try fixing version check * chore: add a fixme * chore(wip): reverse legacy pip check * chore(wip): legacy pip check for 9 or lower * fix: fix the legacy pip check * chore: small cleanup * chore(wip): try the `importlib_resources` * chore: add small comment * chore(wip): avoid `setup.py` in fixtures * chore(wip): version-compatible wheel build * chore: install `build` for `3.8` too * fix: mypy issues * chore: fix comments * fix: more formatting fixes * fix: mdformat * fix: pass wrong auth to `failed_auth` test * chore: cleanup packages before and after test runs * chore(wip): try to bypass test error * chore: add a tech debt comment * chore: undo too many changes * chore(wip): small debug experiment * chore(wip): skip some tests * chore(wip): use nonsense code * fix(chore): small fix to the nonsense code * chore(wip): try `--force-reinstall` * chore: finalize the docker tests --- .github/workflows/ci.yml | 14 +++- README.md | 106 ++++++++++++++--------------- docker/test_docker.py | 5 +- pypiserver/config.py | 52 +++++++++++--- pypiserver/manage.py | 15 ++-- requirements/test-requirements.txt | 1 + setup.py | 13 +++- tests/test_server.py | 32 ++++----- 8 files changed, 146 insertions(+), 92 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a9e273..cfc26e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: pull_request: env: - LAST_SUPPORTED_PYTHON: "3.11" + LAST_SUPPORTED_PYTHON: "3.12" jobs: test-python: @@ -21,7 +21,17 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "pypy3.9", "3.11"] # "3.12-dev" + # make sure to align the `python-version`s in the Matrix with env.LAST_SUPPORTED_PYTHON + python-version: [ + "3.7", + "3.8", + "3.9", + "3.10", + "pypy3.9", + "3.11", + "3.12", + "3.x", # make sure to test the current stable Python version + ] steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index 761d1a6..d8abc75 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,12 @@ | :---------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Version | 2.0.1 | | Date: | 2023-10-01 | -| Source | https://github.com/pypiserver/pypiserver | -| PyPI | https://pypi.org/project/pypiserver/ | -| Tests | https://github.com/pypiserver/pypiserver/actions | +| Source | | +| PyPI | | +| Tests | | | Maintainers | Kostis Anagnostopoulos , Matthew Planchard , Dmitrii Orlov , **Someone new?** We are looking for new maintainers! [#397](https://github.com/pypiserver/pypiserver/issues/397) | | License | zlib/libpng + MIT | -| Community | https://pypiserver.zulipchat.com | +| Community | | Chat with us on [Zulip](https://pypiserver.zulipchat.com)! @@ -44,47 +44,47 @@ making it much easier to get a running index server. Table of Contents -- [pypiserver - minimal PyPI server for use with pip/easy_install](#pypiserver---minimal-pypi-server-for-use-with-pipeasy_install) - - [Quickstart Installation and Usage](#Quickstart-Installation-and-Usage) - - [More details about pypi-server run](#More-details-about-pypi-server-run) - - [More details about pypi-server update](#More-details-about-pypi-server-update) - - [Client-Side Configurations](#Client-Side-Configurations) - - [Configuring pip](#Configuring-pip) - - [Configuring easy_install](#Configuring-easy_install) - - [Uploading Packages Remotely](#Uploading-Packages-Remotely) - - [Apache like Authentication ( htpasswd )](#Apache-like-Authentication) - - [Upload with setuptools](#Upload-with-setuptools) - - [Upload with twine](#Upload-with-twine) - - [Using the Docker Image](#Using-the-Docker-Image) - - [Alternative Installation methods](#Alternative-Installation-methods) - - [Installing the Very Latest Version](#Installing-the-Very-Latest-Version) - - [Recipes](#Recipes) - - [Managing the Package Directory](#Managing-the-Package-Directory) - - [Serving Thousands of Packages](#Serving-Thousands-of-Packages) - - [Managing Automated Startup](#Managing-Automated-Startup) - - [Running as a systemd service](#Running-as-a-systemd-service) - - [Launching through supervisor](#Launching-through-supervisor) - - [Running as a service with NSSM (Windows)](#Running-as-a-service-with-NSSM) - - [Using a Different WSGI Server](#Using-a-Different-WSGI-Server) - - [Apache](#Apache) - - [Gunicorn](#Gunicorn) - - [Paste](#Paste) - - [Behind a Reverse Proxy](#Behind-a-Reverse-Proxy) - - [Nginx](#Nginx) - - [Supporting HTTPS](#Supporting-HTTPS) - - [Traefik](#Traefik) - - [Utilizing the API](#Utilizing-the-API) - - [Using Ad-Hoc Authentication Providers](#Using-Ad-Hoc-Authentication-Providers) - - [Use with MicroPython](#Use-with-MicroPython) - - [Custom Health Check Endpoint](#Custom-Health-Check-Endpoint) - - [Configure a custom health check by CLI arguments](#Configure-a-custom-health-check-by-CLI-arguments) - - [Configure a custom health endpoint by script](#Configure-a-custom-health-endpoint-by-script) - - [Sources](#Sources) - - [Known Limitations](#known-limitations) - - [Similar Projects](#similar-projects) - - [Unmaintained or archived](#unmaintained-or-archived) - - [Related Projects](#related-projects) - - [License](#license) +- [pypiserver](#pypiserver) + - [Quickstart Installation and Usage](#quickstart-installation-and-usage) + - [More details about pypi server run](#more-details-about-pypi-server-run) + - [More details about pypi-server update](#more-details-about-pypi-server-update) + - [Client-Side Configurations](#client-side-configurations) + - [Configuring pip](#configuring-pip) + - [Configuring easy_install](#configuring-easy_install) + - [Uploading Packages Remotely](#uploading-packages-remotely) + - [Apache Like Authentication (htpasswd)](#apache-like-authentication-htpasswd) + - [Upload with setuptools](#upload-with-setuptools) + - [Upload with twine](#upload-with-twine) + - [Using the Docker Image](#using-the-docker-image) + - [Alternative Installation Methods](#alternative-installation-methods) + - [Installing the Very Latest Version](#installing-the-very-latest-version) + - [Recipes](#recipes) + - [Managing the Package Directory](#managing-the-package-directory) + - [Serving Thousands of Packages](#serving-thousands-of-packages) + - [Managing Automated Startup](#managing-automated-startup) + - [Running As a systemd Service](#running-as-a-systemd-service) + - [Launching through supervisor](#launching-through-supervisor) + - [Running As a service with NSSM](#running-as-a-service-with-nssm) + - [Using a Different WSGI Server](#using-a-different-wsgi-server) + - [Apache](#apache) + - [gunicorn](#gunicorn) + - [paste](#paste) + - [Behind a Reverse Proxy](#behind-a-reverse-proxy) + - [Nginx](#nginx) + - [Supporting HTTPS](#supporting-https) + - [Traefik](#traefik) + - [Utilizing the API](#utilizing-the-api) + - [Using Ad-Hoc Authentication Providers](#using-ad-hoc-authentication-providers) + - [Use with MicroPython](#use-with-micropython) + - [Custom Health Check Endpoint](#custom-health-check-endpoint) + - [Configure a custom health endpoint by CLI arguments](#configure-a-custom-health-endpoint-by-cli-arguments) + - [Configure a custom health endpoint by script](#configure-a-custom-health-endpoint-by-script) + - [Sources](#sources) + - [Known Limitations](#known-limitations) + - [Similar Projects](#similar-projects) + - [Unmaintained or archived](#unmaintained-or-archived) + - [Related Software](#related-software) +- [Licensing](#licensing) ## Quickstart Installation and Usage @@ -134,7 +134,7 @@ See also [Alternative Installation methods](<>) # Note that pip search does not currently work with the /simple/ endpoint. ``` -See also [Client-side configurations](#Client-Side-Configurations) for avoiding tedious typing. +See also [Client-side configurations](#client-side-configurations) for avoiding tedious typing. 4. Enter **pypi-server -h** in the cmd-line to print a detailed usage message @@ -461,7 +461,7 @@ Please see `Using Ad-hoc authentication providers`\_ for more information. password: ``` -2. Then from within the directory of the python-project you wish to upload, +1. Then from within the directory of the python-project you wish to upload, issue this command: ```shell @@ -693,7 +693,7 @@ Adjusting the paths and adding this file as **pypiserver.service** into your **systemctl**, e.g. **systemctl start pypiserver**. More useful information about *systemd* can be found at -https://www.digitalocean.com/community/tutorials/how-to-use-systemctl-to-manage-systemd-services-and-units + #### Launching through supervisor @@ -716,7 +716,7 @@ From there, the process can be managed via **supervisord** using **supervisorctl #### Running As a service with NSSM -For Windows download NSSM from https://nssm.cc unzip to a desired location such as Program Files. Decide whether you are going +For Windows download NSSM from unzip to a desired location such as Program Files. Decide whether you are going to use win32 or win64, and add that exe to environment PATH. Create a start_pypiserver.bat @@ -762,7 +762,7 @@ Other useful commands ``` -For detailed information please visit https://nssm.cc +For detailed information please visit ### Using a Different WSGI Server @@ -1057,7 +1057,7 @@ these steps: 3. Invoke the python-script to start-up **pypiserver** ```shell - $ python pypiserver-start.py + python pypiserver-start.py ``` Note @@ -1092,7 +1092,7 @@ Installing packages from the REPL of an embedded device works in this way: upip.install("micropython-foobar") ``` -Further information on micropython-packaging can be found here: https://docs.micropython.org/en/latest/reference/packages.html +Further information on micropython-packaging can be found here: ### Custom Health Check Endpoint @@ -1127,7 +1127,7 @@ Run pypiserver with **--health-endpoint** argument: bottle.run(app=app, host="0.0.0.0", port=8080, server="auto") ```` -Try **curl http://localhost:8080/action/health** +Try **curl ** ## Sources diff --git a/docker/test_docker.py b/docker/test_docker.py index 60e46bb..0bf3cc5 100644 --- a/docker/test_docker.py +++ b/docker/test_docker.py @@ -396,6 +396,7 @@ class TestBasics: "-m", "pip", "install", + "--force-reinstall", "--index-url", f"http://localhost:{container.port}/simple", TEST_DEMO_PIP_PACKAGE, @@ -562,6 +563,7 @@ class TestAuthed: "-m", "pip", "install", + "--force-reinstall", "--index-url", f"http://a:a@localhost:{self.HOST_PORT}/simple", TEST_DEMO_PIP_PACKAGE, @@ -581,9 +583,10 @@ class TestAuthed: "-m", "pip", "install", + "--force-reinstall", "--no-cache", "--index-url", - f"http://localhost:{self.HOST_PORT}/simple", + f"http://foo:bar@localhost:{self.HOST_PORT}/simple", TEST_DEMO_PIP_PACKAGE, check_code=lambda c: c != 0, ) diff --git a/pypiserver/config.py b/pypiserver/config.py index f2af277..5850c90 100644 --- a/pypiserver/config.py +++ b/pypiserver/config.py @@ -43,9 +43,26 @@ import re import sys import textwrap import typing as t -from distutils.util import strtobool as strtoint -import pkg_resources +try: + # `importlib_resources` is required for Python versions below 3.12 + # See more in the package docs: https://pypi.org/project/importlib-resources/ + try: + from importlib_resources import files as import_files + except ImportError: + from importlib.resources import files as import_files + + def get_resource_bytes(package: str, resource: str) -> bytes: + ref = import_files(package).joinpath(resource) + return ref.read_bytes() + +except ImportError: + # The `pkg_resources` is deprecated in Python 3.12 + import pkg_resources + + def get_resource_bytes(package: str, resource: str) -> bytes: + return pkg_resources.resource_string(package, resource) + from pypiserver.backend import ( SimpleFileBackend, @@ -63,10 +80,29 @@ except ImportError: HtpasswdFile = None -# The "strtobool" function in distutils does a nice job at parsing strings, -# but returns an integer. This just wraps it in a boolean call so that we -# get a bool. -strtobool: t.Callable[[str], bool] = lambda val: bool(strtoint(val)) +def legacy_strtoint(val: str) -> int: + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + + The "strtobool" function in distutils does a nice job at parsing strings, + but returns an integer. This just wraps it in a boolean call so that we + get a bool. + + Borrowed from deprecated distutils. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return 1 + elif val in ("n", "no", "f", "false", "off", "0"): + return 0 + else: + raise ValueError("invalid truth value {!r}".format(val)) + + +strtobool: t.Callable[[str], bool] = lambda val: bool(legacy_strtoint(val)) # Specify defaults here so that we can use them in tests &c. and not need @@ -151,9 +187,7 @@ def health_endpoint_arg(arg: str) -> str: def html_file_arg(arg: t.Optional[str]) -> str: """Parse the provided HTML file and return its contents.""" if arg is None or arg == "pypiserver/welcome.html": - return pkg_resources.resource_string(__name__, "welcome.html").decode( - "utf-8" - ) + return get_resource_bytes(__name__, "welcome.html").decode("utf-8") with open(arg, "r", encoding="utf-8") as f: msg = f.read() return msg diff --git a/pypiserver/manage.py b/pypiserver/manage.py index 8f3676a..fe1442f 100644 --- a/pypiserver/manage.py +++ b/pypiserver/manage.py @@ -5,7 +5,8 @@ from __future__ import absolute_import, print_function, unicode_literals import itertools import os import sys -from distutils.version import LooseVersion + +from packaging.version import parse as packaging_parse from pathlib import Path from subprocess import call from xmlrpc.client import Server @@ -112,12 +113,14 @@ class PipCmd: @staticmethod def update_root(pip_version): - """Yield an appropriate root command depending on pip version.""" - # legacy_pip = StrictVersion(pip_version) < StrictVersion('10.0') - legacy_pip = LooseVersion(pip_version) < LooseVersion("10.0") - for part in ("pip", "-q"): + """Yield an appropriate root command depending on pip version. + + Use `pip install` for `pip` 9 or lower, and `pip download` otherwise. + """ + legacy_pip = packaging_parse(pip_version).major < 10 + pip_command = "install" if legacy_pip else "download" + for part in ("pip", "-q", pip_command): yield part - yield "install" if legacy_pip else "download" @staticmethod def update( diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 71e8a54..25f70fb 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -9,6 +9,7 @@ tox twine webtest wheel>=0.25.0 +build>=1.2.0; python_version >= '3.8' mdformat-gfm mdformat-frontmatter mdformat-footnote diff --git a/setup.py b/setup.py index 5e9d7d7..9af1e83 100644 --- a/setup.py +++ b/setup.py @@ -11,10 +11,19 @@ tests_require = [ "twine", "passlib>=1.6", "webtest", + "build>=1.2.0;python_version>='3.8'", ] -setup_requires = ["setuptools", "setuptools-git >= 0.3", "wheel >= 0.25.0"] -install_requires = ["pip>=7"] +setup_requires = [ + "setuptools", + "setuptools-git>=0.3", + "wheel>=0.25.0", +] +install_requires = [ + "pip>=7", + "packaging>=23.2", + "importlib_resources;python_version>'3.8' and python_version<'3.12'", +] def read_file(rel_path: str): diff --git a/tests/test_server.py b/tests/test_server.py index 051a1ff..92ecdda 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -16,6 +16,7 @@ import itertools import os import shutil import socket +import re import sys import time import typing as t @@ -99,10 +100,14 @@ def build_url(port: t.Union[int, str], user: str = "", pswd: str = "") -> str: return f"http://{auth}localhost:{port}" -def run_setup_py(path: Path, arguments: str): +def run_setup_py(path: Path, arguments: str) -> int: return os.system(f"{sys.executable} {path / 'setup.py'} {arguments}") +def run_py_build(srcdir: Path, flags: str) -> int: + return os.system(f"{sys.executable} -m build {flags} {srcdir}") + + # A test-distribution to check if # bottle supports uploading 100's of packages, # see: https://github.com/pypiserver/pypiserver/issues/82 @@ -140,8 +145,13 @@ def server_root(tmp_path_factory): @pytest.fixture(scope="module") def wheel_file(project, tmp_path_factory): distdir = tmp_path_factory.mktemp("dist") - assert run_setup_py(project, f"bdist_wheel -d {distdir}") == 0 - return list(distdir.glob("centodeps*.whl"))[0] + if re.match("^3\.7", sys.version): + assert run_setup_py(project, f"bdist_wheel -d {distdir}") == 0 + else: + assert run_py_build(project, f"--wheel --outdir {distdir}") == 0 + wheels = list(distdir.glob("centodeps*.whl")) + assert len(wheels) > 0 + return wheels[0] @pytest.fixture() @@ -317,22 +327,6 @@ def test_pip_install_authed_succeeds(authed_server, hosted_wheel_file, pipdir): assert pipdir.joinpath(hosted_wheel_file.name).is_file() -@pytest.mark.parametrize("pkg_frmt", ["bdist", "bdist_wheel"]) -@pytest.mark.parametrize(["server_fixture", "pypirc_fixture"], all_servers) -def test_setuptools_upload( - server_fixture, pypirc_fixture, project, pkg_frmt, server_root, request -): - request.getfixturevalue(server_fixture) - request.getfixturevalue(pypirc_fixture) - - assert len(list(server_root.iterdir())) == 0 - - for i in range(5): - print(f"++Attempt #{i}") - assert run_setup_py(project, f"-vvv {pkg_frmt} upload -r test") == 0 - assert len(list(server_root.iterdir())) == 1 - - def test_partial_authed_open_download(partial_authed_server): """Validate that partial auth still allows downloads.""" url = build_url(partial_authed_server.port) + "/simple"