Compare commits

..

1 Commits

Author SHA1 Message Date
Dmitry Orlov
5fc2b8d318
Add --diff flag to black formatter check
This makes the output more explicit. Check this for details: https://github.com/psf/black#command-line-options
2020-11-03 22:31:04 +01:00
69 changed files with 2195 additions and 4867 deletions

@ -1,2 +0,0 @@
[run]
omit = pypiserver/bottle.py

@ -1,9 +1,8 @@
*
!pypiserver
!requirements
!docker/docker-requirements.txt
!docker/gunicorn.conf.py
!docker/entrypoint.sh
!README.md
!docker-requirements.txt
!entrypoint.sh
!README.rst
!setup.cfg
!setup.py

@ -1,13 +0,0 @@
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "pip"
directory: "/docker"
schedule:
interval: "monthly"
- package-ecosystem: "pip"
directory: "/requirements"
schedule:
interval: "monthly"

@ -1,272 +0,0 @@
# Run tests
name: CI
on:
# This will run when any branch or tag is pushed
push:
branches:
- "master"
tags:
- "v**"
# Allowing to run on fork and other pull requests
pull_request:
env:
LAST_SUPPORTED_PYTHON: "3.12"
jobs:
test-python:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
# make sure to align the `python-version`s in the Matrix with env.LAST_SUPPORTED_PYTHON
python-version: [
"3.7",
"3.8",
"3.9",
"3.10",
"pypy3.9",
"3.11",
"3.12",
"3.x", # make sure to test the current stable Python version
]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
pip install --upgrade setuptools
pip install tox==3.27.*
- name: Run tests
run: tox -e py
check:
# These checks only need to be done once, not for every python version we
# support
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
# Use the current version of Python
python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
- name: Install dependencies
run: |
pip install -r "requirements/dev.pip"
pip install types-pkg_resources # one of mypy required stubs
- name: Check types
# individual mypy files for now, until we get the rest
# of the project typechecking
run: >-
mypy
docker/test_docker.py
pypiserver/config.py
tests/test_init.py
- name: Check formatting
run: black --diff --check .
- name: Validate README
id: validate_readme
run: mdformat --check README.md
continue-on-error: true
- name: check mdformat result
run: |
if [ "${{ steps.validate_readme.outcome }}" == "failure" ]; then
echo "copy readme to /tmp/pypiserver"
mkdir -p /tmp/pypiserver
cp README.md /tmp/pypiserver
echo "README.md is not formatted correctly. Please run 'mdformat README.md' and commit the result."
mdformat /tmp/pypiserver/README.md
diff -u README.md /tmp/pypiserver/README.md
exit 1
else
echo "README.md is formatted correctly."
fi
# Full-flow docker tests, again not python version dependent
# We _could_ test this on MacOS, but it takes forever to get docker
# installed. I'm going to say for now probably 99% of people using
# the docker image will be doing so from a linux system, e.g. for
# a k8s deploy, and I've verified manually that things work on
# MacOS, so /shrug.
test-docker:
runs-on: "ubuntu-latest"
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
# Use the current version of Python
python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
- name: Install test dependencies
run: pip install -r "requirements/test.pip"
- name: Install package
run: pip install -r "requirements/exe.pip"
- name: Run tests
run: "pytest docker/test_docker.py"
tests:
runs-on: "ubuntu-latest"
needs:
- "check"
- "test-docker"
- "test-python"
steps:
- name: "Everything is good!"
run: "echo true"
# RELEASES
## PYPI
build-wheel-and-push-to-pypi:
runs-on: ubuntu-latest
needs:
- "tests"
steps:
- uses: actions/checkout@master
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
- name: Install dev dependencies
run: pip install -r "requirements/dev.pip"
- name: Build distribution _wheel_.
run: |
./bin/package.sh
- name: Publish distribution 📦 to PyPI.
uses: pypa/gh-action-pypi-publish@release/v1
# Push to PyPi only if a tag is pushed
if: startsWith(github.event.ref, 'refs/tags/v')
with:
password: ${{ secrets.PYPI_API_TOKEN }}
print-hash: true
## DOCKER (DOCKER HUB & CONTAINER REGISTRY)
# figure out which docker tags we need to push
docker-determine-tags:
runs-on: "ubuntu-latest"
needs:
- "tests"
env:
STABLE_IMAGES: '["pypiserver/pypiserver", "ghcr.io/pypiserver/pypiserver"]'
FLEXIBLE_IMAGES: '["pypiserver/pypiserver"]'
outputs:
tags: "${{ steps.tags.outputs.tags }}"
has_tags: "${{ steps.has_tags.outputs.has_tags }}"
images: ${{ contains(steps.tags.outputs.tags, 'unstable') && env.FLEXIBLE_IMAGES || env.STABLE_IMAGES }}
steps:
- uses: "actions/checkout@v3"
- uses: "actions/setup-python@v4"
with:
python-version: ${{ env.LAST_SUPPORTED_PYTHON }}
# This script prints a JSON array of needed docker tags, depending on the
# ref. That array is then used to construct the matrix of the
# deploy-docker job
- name: "Get expected docker tags"
id: "tags"
run: >-
echo "::set-output name=tags::$(bin/ci_helper.py ${{ github.ref }} docker_tags)"
# This is needed because GH actions will fail on an empty matrix, so
# we need to be sure the `if` condition is false on the next job if
# the matrix will be empty. The script prints 'true' if the array is
# not empty, or 'false' otherwise.
- name: "Determine whether any tags are needed"
id: "has_tags"
run: >-
echo "::set-output name=has_tags::$(bin/ci_helper.py ${{ github.ref }} has_tags)"
# Deploy any needed docker tags
deploy-docker:
runs-on: "ubuntu-latest"
needs:
- "docker-determine-tags"
if: "${{ fromJson(needs.docker-determine-tags.outputs.has_tags) }}"
strategy:
matrix:
tag: "${{ fromJson(needs.docker-determine-tags.outputs.tags) }}"
image: "${{ fromJson(needs.docker-determine-tags.outputs.images) }}"
steps:
- uses: "actions/checkout@v3"
- name: "Cache Docker layers"
uses: "actions/cache@v3"
with:
path: "/tmp/.buildx-cache"
key: "${{ runner.os }}-buildx-${{ github.sha }}"
restore-keys: |
${{ runner.os }}-buildx-
- name: "Login to Docker Hub"
uses: "docker/login-action@v3"
with:
username: "${{ secrets.DOCKER_HUB_USER }}"
password: "${{ secrets.DOCKER_HUB_TOKEN }}"
- name: "Login to GitHub Container Registry"
uses: "docker/login-action@v3"
with:
registry: "ghcr.io"
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: "Set up QEMU"
uses: "docker/setup-qemu-action@v3"
- name: "Set up Docker Buildx"
id: "buildx"
uses: "docker/setup-buildx-action@v3"
- name: "Build and push"
id: "docker_build"
uses: "docker/build-push-action@v5"
with:
context: "./"
platforms: linux/amd64,linux/arm64
file: "./Dockerfile"
builder: "${{ steps.buildx.outputs.name }}"
push: true
tags: "${{ matrix.image }}:${{ matrix.tag }}"
cache-from: "type=local,src=/tmp/.buildx-cache"
cache-to: "type=local,dest=/tmp/.buildx-cache"
- name: "Image digest"
run: "echo ${{ steps.docker_build.outputs.digest }}"
- name: "Docker Hub Description"
uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
repository: pypiserver/pypiserver
## GITHUB RELEASE DRAFT
create_release:
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
runs-on: "ubuntu-latest"
needs:
- "tests"
steps:
- uses: actions/checkout@v3
- uses: softprops/action-gh-release@v1
with:
body: 👋 This is a draft release. Please update it manually.
prerelease: false
draft: true
files: |
CHANGES.rst

@ -1,77 +0,0 @@
# Release Candidate GitHub Action
name: release_candidate
# Performed actions:
# - [x] create a new AUTO-RC-<DATE> branch
# - [x] prepare RC metadata and description
# - [x] update CHANGES.rst (+ rc-title, + date)
# - [x] create changes commit
# - [x] push to GH
# - [x] open a PR to `master`
on:
schedule:
- cron: "0 0 1 * *" # each 1st day of the month
workflow_dispatch: # on manual trigger
jobs:
new-rc:
runs-on: ubuntu-latest
env:
CHANGES_FILE: CHANGES.rst
PR_BODY_FILE: /tmp/pr-body.md
RF_DOCS_FILE: ./docs/contents/repo-maintenance/release-work.md
steps:
- uses: actions/checkout@v3
with:
# Flag to fetch all history.
# @see https://github.com/marketplace/actions/checkout#Fetch-all-history-for-all-tags-and-branches
fetch-depth: 0
- id: get-rc-date
run: echo "RC_DATE=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
- id: make-pr-body-file
run: |
PR_BODY_FILE=${{ env.PR_BODY_FILE }}
RC_DATE=${{ steps.get-rc-date.outputs.RC_DATE }}
touch ${PR_BODY_FILE}
echo "📦 Automated release candidate for ${RC_DATE}." >> ${PR_BODY_FILE}
echo "" >> ${PR_BODY_FILE}
echo "_TODO:_" >> ${PR_BODY_FILE}
echo "- [ ] Manually adjust generated CHANGES lines" >> ${PR_BODY_FILE}
echo "- [ ] Manually adjust generated CHANGES title" >> ${PR_BODY_FILE}
echo "- [ ] Manually adjust generated CHANGES date" >> ${PR_BODY_FILE}
echo "- [ ] Approve and merge this PR" >> ${PR_BODY_FILE}
echo "- [ ] See \`${{ env.RF_DOCS_FILE }}\` to continue" >> ${PR_BODY_FILE}
echo "${PR_BODY_FILE}:"
cat ${PR_BODY_FILE}
- id: propose-rc
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
CHANGES_FILE=${{ env.CHANGES_FILE }}
PR_BODY_FILE=${{ env.PR_BODY_FILE }}
RC_DATE=${{ steps.get-rc-date.outputs.RC_DATE }}
git config user.name github-actions
git config user.email github-actions@github.com
git checkout -b auto-release-candidate-${RC_DATE}
git push -u origin auto-release-candidate-${RC_DATE}
git status
git fetch
./bin/update_changelog.sh
git add ${CHANGES_FILE}
git commit -m "chore(rc-changes): update ${CHANGES_FILE}"
git push
gh pr create --title "chore(auto-release-candidate-${RC_DATE})" \
--body-file ${PR_BODY_FILE} \
--base master \
--draft

@ -1,56 +0,0 @@
# Release Tag workflow
name: release_tag
# Performed actions:
# - [x] infer the last RC version
# - [x] run bumpver.py with the new version
# - [x] push the commit and new tag
# - [x] support dry-run mode
on:
workflow_dispatch: # on manual trigger
inputs:
dryrun:
description: "Whether to run the release in a dry-run mode"
default: true
required: true
type: boolean
jobs:
new-tag:
if: ${{ github.ref_name == 'master' }}
runs-on: ubuntu-latest
env:
CHANGE_FILE: CHANGES.rst
EXPECTED_DIFF_COUNT: 1
steps:
- uses: actions/checkout@v3
- id: get-version
run: |
CHANGE_FILE=${{ env.CHANGE_FILE }}
LAST_VERSION=$(grep -m1 -E ' \([0-9]+-[0-9]+-[0-9]+\)$' ${CHANGE_FILE} | awk '{ print $1 }')
echo "👀 Version detected: ${LAST_VERSION}"
echo "LAST_VERSION=${LAST_VERSION}" >> "$GITHUB_OUTPUT"
- uses: actions/setup-python@v4
with:
python-version: "3.x"
- id: install-requirements
run: pip install -r "requirements/dev.pip"
- name: run `bumpver`
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo ${{ inputs.dryrun && '💡 Running in dry-run mode' || 'Preparing release...' }}
CHANGE_FILE=${{ env.CHANGE_FILE }}
LAST_VERSION=${{ steps.get-version.outputs.LAST_VERSION }}
git config user.name github-actions
git config user.email github-actions@github.com
python3 bin/bumpver.py ${{ inputs.dryrun && '-n' || '' }} -t "Automated release ${LAST_VERSION}" ${LAST_VERSION}
git push --follow-tags

16
.gitignore vendored

@ -16,8 +16,8 @@
.ropeproject
ID
__pycache__/
**/build/
**/dist/
/build/
/dist/
/*.egg*
/MANIFEST
/README.html
@ -26,10 +26,9 @@ __pycache__/
/.pydevproject
/.pytest_cache
/.tox/
**/*.egg-info/
/*.egg-info/
/.standalone
/.coverage*
!/.coveragerc
/htmlcov/
/.installed.cfg
/develop-eggs/
@ -47,12 +46,3 @@ __pycache__/
.venv/
venv/
# Temporary / runtime generated files
traefik/
auth/
# Release Candidate Artifacts
rc/
# Various local temporary artifacts
.tmp/

28
.travis.yml Normal file

@ -0,0 +1,28 @@
sudo: required
language: python
services: docker
python:
- 3.6
- 3.7
- 3.8
- 3.9
- pypy3
install:
- pip install -U setuptools twine pip sphinx tox tox-travis
script:
- ./bin/test-docker.sh
- tox
- ./bin/check_readme.sh
branches:
except:
- standalone
jobs:
include:
- python: 3.8
install: pip install -U black
script: black --diff --check .

@ -1,111 +1,9 @@
Changelog
=========
3.0.0 (tbd)
2.0.0 (tbd)
-----------
2.1.1 (2024-04-24)
--------------------------
- 31c9cf1 FIX: deprecated `setuptools.py` when building in `package.sh` (#568)
- 2619c17 FIX: use the right env variables in `release-tag` workflow (#569)
2.1.0 (2024-04-24)
--------------------------
- d588913 ENH: Bump github action versions and add multiarch support (#553)
- a558dbc ENH: Handle tar.xz archives (#536)
- 2f0a56c FIX: support Python 3.12 (#539)
- 84bf12c MAINT: make the last supported python version explicit in `ci.yaml` (#558)
- 946fbfe MAINT: Update setuptools requirement from <62.0.0,>=40.0 to >=40.0,<70.0.0 in /requirements (#557)
- 50c7a78 MAINT: add tar xz test case (#538)
2.0.1 (2023-10-01)
--------------------------
- e788785 FIX: dockerhub description readme (#533)
- 904f50f FIX: specify long_description as MD type (#532)
- ad9fb4a MAINT: simpler release_tag action (#530)
2.0.0 (2023-10-01)
--------------------------
- df99872 FIX: Json Info for the same version (#511)
- c235ef4 ENH: Switch default hash-algo to sha256 (#459)
- 6417769 MAINT: add GitHub container registry for stable images (#521)
- 4645f7b MAINT: cleanup release process (#516)
- e542702 MAINT: TOC internal links (#520)
- f14b92c MAINT: readme output diff on format error (#512)
- 9edae77 ENH: Feature/markdown conversion (#503)
1.5.2 (2023-07-30)
--------------------------
- 3f520cd FIX: Add missing pip dependency (#500)
- 85e065e MAINT: Feat/dependabot (#493)
- 73dbe15 FIX: Health endpoint usage is missing. (#481)
- e0c9723 MAINT: Bump waitress from 1.4.4 to 2.1.2 in /docker (#454)
- a95f456 MAINT: update docs folder (#479)
- 8cc8e80 MAINT: Update README.rst and config.py (#470)
- 754b0f4 MAINT: add help output for `run` and `update` to README (#478)
- 5fd6400 MAINT: Update README to reflect run/update commands (#451)
- abc4bfb MAINT: Upgrade to psf/black stable style 2023 (#474)
- 383c936 MAINT: disable tests on Python3.6 (#471)
- d716d0f FIX: explicit optional types in `config.py` (#472)
- ae3dcf2 ENH: :stethoscope: allow customized health check endpoint (#442)
- 2f3b997 FIX: correct 1.5.1 tag date in CHANGES (#457)
- 4a0c6fb MAINT: from importlib import reload for Python 3 (#448)
- 0ba44b5 FIX: force setuptools update + no duplicate runs in GH Actions (#445)
- 530852b MAINT: Support current versions of CPython (#453)
- 6ea316c MAINT: Upgrade GitHub Actions (#447)
1.5.1 (2022-10-18)
--------------------------
- 61e4487 ENH: add extremely basic /health endpoint (#396)
- bbd2a47 FIX: docker tests in cicd (#444)
- 784a9a1 MAINT: Replace usage of deprecated inspect.getargspec (#436)
- 640a748 MAINT: Add traefik/ and auth/ dirs to gitignore (#398)
- a67829e MAINT: Fix typos in README (#431)
1.5.0 (2022-05-01)
------------------
- CHORE: pull CHANGES.rst from v1.4.x
- DOC: add call for maintainers to README
- DOC: Fixes in README examples (#380)
- DOC: start of empty contributor documentation directory (#383)
- ENH: added JSON Topic for use with micropython-upip (#395, thanks @awachtler)
- ENH: Backwards-compatible argparse config (not yet in use) (#339)
- ENH: Refactor storage operations into separate Backend classes (#348)
- ENH: Restore ability to drop hashing in new config (#347)
- ENH: Specify doctype for generated pages.
Fixed violation of PEP 503 warning with pip>=22.
(#413 and #414, thanks @domdfcoding and @luismsgomes)
- ENH: Use argparse config throughout app (#349)
- FIX: Fix silly typo in the readme :) (#362)
- FIX: fix small typing and lint errors in config.py (#421)
- FIX: fix version incompatibility that breaks twine in docker test (#356)
- FIX: fix(setuptools-in-tests) add `packages=[]` in test setup (#425)
- FIX: Gunicorn/gevent docker, log fixes, cache busting (#371)
- MAINT: Add aggregate "tests" job (#370)
- MAINT: Add release to PyPi job (#428)
- MAINT: add github action for release management (#417)
- MAINT: Add python 3.9 testing (#351, disabled in #407)
- MAINT: Cleanup code to python 3.6 (#342)
- MAINT: contributor update in README (#415)
- MAINT: disable unstable test for Python 3.9 (#407, planned temporary)
- MAINT: Docker improvements (#365)
- MAINT: drop standalone, drop py 2.7 and 3.5 (#338)
- MAINT: Merge branch 'v1.4.x'
- MAINT: Push to Docker Hub from CI (#375)
- MAINT: Refactor test_server to increase speed (#354)
- MAINT: Run black on codebase (#336)
- MAINT: run gh actions on PRs from forks #401
- MAINT: small code style fixes
- MAINT: Switch to GH actions (#361)
1.4.2 (2020-10-10)
------------------

@ -31,37 +31,26 @@ RUN apk add --no-cache --virtual .build-deps \
FROM base AS builder_dependencies
WORKDIR /code
COPY pypiserver /code/pypiserver
COPY requirements /code/requirements
COPY docker-requirements.txt /code
COPY setup.cfg /code
COPY setup.py /code
COPY README.rst /code
COPY docker/docker-requirements.txt .
# Install requirements
RUN apk add --no-cache --virtual .build-deps \
build-base \
libffi-dev \
&& mkdir /install \
&& python -m pip install \
--no-warn-script-location \
--prefix=/install \
--requirement docker-requirements.txt
# Install pypiserver
# - do this separately from deps so that when developing, every change does not
# require reinstalling deps
COPY pypiserver pypiserver
COPY setup.cfg .
COPY setup.py .
COPY README.md .
RUN python -m pip install --no-warn-script-location --prefix=/install .
&& python -m pip install --no-warn-script-location \
--prefix=/install \
/code --requirement /code/docker-requirements.txt
FROM base
WORKDIR /data
# Copy the libraries installed via pip
COPY --from=builder_dependencies /install /usr/local
COPY --from=builder_gosu /usr/local/bin/gosu /usr/local/bin/gosu
COPY docker/entrypoint.sh /entrypoint.sh
COPY docker/gunicorn.conf.py /data
COPY entrypoint.sh /entrypoint.sh
# Use a consistent user and group ID so that linux users
# can create a corresponding system user and set permissions
@ -72,14 +61,11 @@ RUN apk add bash \
&& addgroup -S -g 9898 pypiserver \
&& adduser -S -u 9898 -G pypiserver pypiserver --home /data\
&& mkdir -p /data/packages \
&& chmod +x /entrypoint.sh
&& chmod +x /entrypoint.sh
VOLUME /data/packages
ENV PYPISERVER_PORT=8080
# PORT is deprecated. Please use PYPISERVER_PORT instead
ENV PORT=$PYPISERVER_PORT
# Flush logs immediately to stdout
ENV PYTHONUNBUFFERED=t
EXPOSE $PYPISERVER_PORT
WORKDIR /data
ENV PORT=8080
EXPOSE $PORT
ENTRYPOINT ["/entrypoint.sh"]

@ -1,23 +0,0 @@
##
# pypiserver
#
# this makefile is used to help with building resources needed for testing
#
# @file
# @version 0.1
SHELL = /bin/sh
MYPKG_SRC = fixtures/mypkg/setup.py $(shell find fixtures/mypkg/mypkg -type f -name '*.py')
# Build the test fixture package.
mypkg: fixtures/mypkg/dist/pypiserver_mypkg-1.0.0.tar.gz
mypkg: fixtures/mypkg/dist/pypiserver_mypkg-1.0.0-py2.py3-none-any.whl
fixtures/mypkg/dist/pypiserver_mypkg-1.0.0.tar.gz: $(MYPKG_SRC)
cd fixtures/mypkg; python setup.py sdist
fixtures/mypkg/dist/pypiserver_mypkg-1.0.0-py2.py3-none-any.whl: $(MYPKG_SRC)
cd fixtures/mypkg; python setup.py bdist_wheel
# end

1212
README.md

File diff suppressed because it is too large Load Diff

1005
README.rst Normal file

File diff suppressed because it is too large Load Diff

@ -1,31 +0,0 @@
# Build scripts folder
## Highlight files
- `bumpver.py` : Bump, commit and tag new project versions
- `package.sh` : Build deployable artifact (wheel) in `/dist/` folder.
## Fully manual release check-list
1. Update `/CHANGES.rst` (+ Title + Date).
2. Push to GitHub to run all TCs once more.
3. Bump version: commit & tag it with `/bin/bumpver.py`. Use `--help`.
> 💡 Read [PEP-440](https://www.python.org/dev/peps/pep-0440/) to decide the version.
4. Push it in GitHub with `--follow-tags`.
### Manually publishing a new package
1. Generate package *wheel* with `/bin/package.sh`.
2. Upload to PyPi with `twine upload -s -i <gpg-user> dist/*`
3. Ensure that the new tag is built on
[`hub.docker.com`](https://hub.docker.com/r/pypiserver/pypiserver)
as `latest` and as a direct tag reference.
4. Copy release notes from `/CHANGES.rst` in GitHub as new *"release"*
page on the new tag.
> 💡 Check syntactic differences between `.md` and `.rst` files.

36
bin/README.rst Normal file

@ -0,0 +1,36 @@
====================
Build scripts folder
====================
Files:
======
- ``bumpver.py`` : Bump, commit and tag new project versions
- ``check_readme.sh`` : Check that README has no RsT-syntactic errors.
- ``package.sh`` : Build deployable artifact (wheel) in ``/dist/`` folder.
- ``README.rst`` : This file.
Release check-list:
===================
1. Update ``/CHANGES.rst`` (+ Title + Date) & ``/README.rst`` (Date,
not version).
2. Push to GitHub to run all TCs once more.
3. Bump version: commit & tag it with ``/bin/bumpver.py``.
Use ``--help``.
Read `PEP-440 <https://www.python.org/dev/peps/pep-0440/>`_ to decide the version.
4. Push it in GitHub with ``--follow-tags``.
5. Generate package *wheel* with ``/bin/package.sh``.
6. Upload to PyPi with ``twine upload -s -i <gpg-user> dist/*``
7. Ensure that the new tag is built on hub.docker.com as ``latest`` and as a
direct tag reference.
8. Copy release notes from ``/CHANGES.rst`` in GitHub as new *"release"* page
on the new tag. Check syntactic differences between ``.md`` and ``.rst`` files.

@ -35,22 +35,21 @@ EXAMPLE:
"""
import functools as fnt
import os.path as osp
import re
import sys
from datetime import datetime
import re
import functools as fnt
import docopt
my_dir = osp.dirname(__file__)
VFILE = osp.join(my_dir, "..", "pypiserver", "__init__.py")
VFILE_regex_version = re.compile(r'version *= *__version__ *= *"([^"]+)"')
VFILE_regex_datetime = re.compile(r'__updated__ *= *"([^"]+)"')
VFILE_regex_date = re.compile(r'__updated__ *= *"([^"\s]+)\s')
VFILE_regex_v = re.compile(r'version *= *__version__ *= *"([^"]+)"')
VFILE_regex_d = re.compile(r'__updated__ *= *"([^"]+)"')
RFILE = osp.join(my_dir, "..", "README.md")
RFILE = osp.join(my_dir, "..", "README.rst")
PYTEST_ARGS = [osp.join("tests", "test_docs.py")]
@ -59,13 +58,6 @@ class CmdException(Exception):
pass
def get_current_date_info() -> (str, str):
now = datetime.now()
new_datetime = now.strftime("%Y-%m-%d %H:%M:%S%z")
new_date = now.strftime("%Y-%m-%d")
return (new_datetime, new_date)
@fnt.lru_cache()
def read_txtfile(fpath):
with open(fpath, "rt", encoding="utf-8") as fp:
@ -146,6 +138,7 @@ def exec_cmd(cmd):
def do_commit(new_ver, old_ver, dry_run, amend, ver_files):
import pathlib
# new_ver = strip_ver2_commonprefix(old_ver, new_ver)
cmt_msg = "chore(ver): bump %s-->%s" % (old_ver, new_ver)
ver_files = [pathlib.Path(f).as_posix() for f in ver_files]
@ -190,12 +183,11 @@ def bumpver(
cmd.append(RFILE)
exec_cmd(cmd)
regexes = [VFILE_regex_version, VFILE_regex_datetime, VFILE_regex_date]
old_ver, old_datetime, old_date = extract_file_regexes(VFILE, regexes)
regexes = [VFILE_regex_v, VFILE_regex_d]
old_ver, old_date = extract_file_regexes(VFILE, regexes)
if not new_ver:
yield old_ver
yield old_datetime
yield old_date
else:
if new_ver == old_ver:
@ -207,13 +199,12 @@ def bumpver(
msg += "!\n Use of --force recommended."
raise CmdException(msg % new_ver)
new_datetime, new_date = get_current_date_info()
from datetime import datetime
new_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S%z")
ver_files = [osp.normpath(f) for f in [VFILE, RFILE]]
subst_pairs = [
(old_ver, new_ver),
(old_datetime, new_datetime),
(old_date, new_date),
]
subst_pairs = [(old_ver, new_ver), (old_date, new_date)]
for repl in replace_substrings(ver_files, subst_pairs):
new_txt, fpath, replacements = repl
@ -267,7 +258,6 @@ def main(*args):
except CmdException as ex:
sys.exit(str(ex))
except Exception as ex:
print("Unexpected error happened.")
raise ex

37
bin/check_readme.sh Executable file

@ -0,0 +1,37 @@
#!/bin/bash
#-*- coding: utf-8 -*-
#
## Checks that README has no RsT-syntactic errors.
# Since it is used by `setup.py`'s `description` if it has any errors,
# PyPi would fail parsing them, ending up with an ugly landing page,
# when uploaded.
>&2 echo "+++ Checking README for PyPy...."
set +x ## Enable for debug
my_dir=`dirname "$0"`
cd $my_dir/..
py=""
rst="rst2html"
if [ ! -x "`which $rst 2>/dev/null`" ]; then
## In WinPython, only a python-script exist in PATH,
# so execute it with python-interpreter.
#
exe="`which rst2html.py 2> /dev/null`"
if [ $? -eq 0 ]; then
py=python
rst="$exe"
else
echo -e "Cannot find 'rst2html'! \n Sphinx installed? `pip show sphinx`" &&
exit 1
fi
if [ -x "`which cygpath`" ]; then
rst="`cygpath -w $rst`"
fi
fi
export PYTHONPATH='$my_dir/..'
#python setup.py --long-description > t.rst ## Uncomment to inspect it.
python setup.py --long-description | $py "$rst" --halt=warning > /dev/null && echo OK

@ -1,88 +0,0 @@
#!/usr/bin/env python3
"""Output expected docker tags to build in CI."""
import json
import typing as t
import re
from argparse import ArgumentParser, Namespace
RELEASE_RE = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?")
PRE_RELEASE_RE = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+(a|b|c|\.?dev)[0-9]+")
def parse_args() -> Namespace:
"""Parse cmdline args."""
parser = ArgumentParser()
parser.add_argument(
"ref",
help=(
"The github ref for which CI is running. This may be a full ref "
"like refs/tags/v1.2.3 or refs/heads/master, or just a tag/branch "
"name like v1.2.3 or master."
),
)
parser.add_argument(
"action",
help=("The action to perform"),
choices=("docker_tags", "pypi_release", "has_tags"),
)
return parser.parse_args()
def strip_ref_to_name(ref: str) -> str:
"""Strip a full ref to a name."""
strips = ("refs/heads/", "refs/tags/")
for strip in strips:
if ref.startswith(strip):
return ref[len(strip) :]
return ref
def name_to_array(name: str) -> t.Tuple[str, ...]:
"""Convert a ref name to an array of tags to build."""
tags: t.Dict[str, t.Callable[[str], bool]] = {
# unstable for any master build
"unstable": lambda i: i == "master",
# latest goes for full releases
"latest": lambda i: RELEASE_RE.fullmatch(i) is not None,
# the tag itself for any release or pre-release tag
name: lambda i: (
RELEASE_RE.fullmatch(i) is not None
or PRE_RELEASE_RE.fullmatch(i) is not None
),
}
return tuple(tag for tag, test in tags.items() if test(name))
def ref_to_json(ref: str) -> str:
"""Convert a ref to a JSON array and return it as a string."""
array = name_to_array(strip_ref_to_name(ref))
return json.dumps(array)
def should_deploy_to_pypi(ref: str) -> str:
"""Return a JSON bool indicating whether we should deploy to PyPI."""
name = strip_ref_to_name(ref)
return json.dumps(
RELEASE_RE.fullmatch(name) is not None
or PRE_RELEASE_RE.fullmatch(name) is not None
)
def main() -> None:
"""Parse args and print the JSON array."""
args = parse_args()
action_switch: t.Dict[str, t.Callable[[], None]] = {
"docker_tags": lambda: print(ref_to_json(args.ref)),
"has_tags": lambda: print(
json.dumps(len(name_to_array(strip_ref_to_name(args.ref))) > 0)
),
"pypi_release": lambda: print(should_deploy_to_pypi(args.ref)),
}
action_switch[args.action]()
if __name__ == "__main__":
main()

@ -5,5 +5,5 @@
my_dir=`dirname "$0"`
cd $my_dir/..
rm -r build/* dist/* || echo "no build/* or dist/* folder is found"
python3 -m build
rm -r build/* dist/*
python setup.py bdist_wheel sdist

Binary file not shown.

48
bin/test-docker.sh Executable file

@ -0,0 +1,48 @@
#!/usr/bin/env sh
# Perform some simple validation to make sure the Docker image works
# Should be run from the repo root.
set -xe # exit on any error, show debug output
DIR="$( cd "$( dirname "$0" )" >/dev/null 2>&1 && pwd )"
docker build . -t pypiserver:test
docker run pypiserver:test --help > /dev/null
# Mount our htpasswd file, which contains a test user with a bcrypt-encrypted
# "test" password
CONTAINER_ID=$(docker run \
-d \
-v "${DIR}/test.htpasswd:/data/.htpasswd" \
-p 8080:8080 \
pypiserver:test -a "list,update,download" -P /data/.htpasswd packages)
trap "docker container stop $CONTAINER_ID" EXIT
sleep 15 # give the container some time to get going
# Ensure we can authenticate locally
RET=$(curl localhost:8080)
echo $RET
echo $RET | grep -q "pypiserver"
RET=$(curl localhost:8080/packages/)
echo $RET
echo $RET | grep -q "401"
RET=$(curl test:test@localhost:8080/packages/)
echo $RET
echo $RET | grep -q "Index of packages"
twine upload \
-u test \
-p test \
--repository-url http://localhost:8080 \
"${DIR}/pypiserver-1.2.6-py2.py3-none-any.whl"
RET=$(curl test:test@localhost:8080/packages/)
echo $RET
echo $RET | grep -q "pypiserver-1.2.6"

2
bin/test.htpasswd Normal file

@ -0,0 +1,2 @@
test:$2y$05$0wU8vmgucWeyLyqxB.mm1OOPf660/exARXPN5uC.gHaWziv7C4t/m

@ -1,116 +0,0 @@
#!/usr/bin/env bash
# Script to create a new RC entry
#
# Actions:
# 1. find latest published version
# 2. find the TBD planned big version
# 3. create new RC version entry
# 4. add change log entries
set -e # exit on errors
# TODO: provide that as parameters?
CHANGE_FILE='CHANGES.rst'
RC_DATE=$(date +'%m-%d-%Y')
WORKSPACE_DIR="${GITHUB_WORKSPACE:-.}/rc"
TMP_CHANGE_LOG="${WORKSPACE_DIR}/rc-${RC_DATE}.txt"
############
# CLEANUPS #
############
rm -rf $TMP_CHANGE_LOG
mkdir -p $WORKSPACE_DIR
##################
# INITIALIZATION #
##################
echo "Updating $CHANGE_FILE:"
# TODO(tech-debt): get `LAST_VERSION` with a separate bash script
LAST_VERSION=$(grep -m1 -E ' \([0-9]+-[0-9]+-[0-9]+\)$' $CHANGE_FILE | awk '{ print $1 }')
echo "Detected last release version: $LAST_VERSION"
###################
# VERSION BUMPING #
###################
echo "Bumping patch version..."
MAJOR_COLUMN=1
MINOR_COLUMN=2
PATCH_COLUMN=3
# `awk` is used to bump the PATCH version since the last public release.
# -F - gives a separator for splitting the original release into columns.
# -v - provides a value for variable to be used in the `awk` command.
# -v K=$PATCH_COLUMN - provides value for `K` - the version column to bump.
# This attempts to preserve the a standard syntax for GNU Awk.
# More can be found here: https://www.gnu.org/software/gawk/manual/gawk.html
BUMPED_VERSION=$(echo $LAST_VERSION | awk -F. -v K=$PATCH_COLUMN '{$K+=1; print $0}' OFS='.')
echo "Bumped to new candidate version: $BUMPED_VERSION"
RC_VERSION=${BUMPED_VERSION}rc${RC_DATE}
echo "Final RC version: $RC_VERSION"
###################
# CHANGELOG ENTRY #
###################
CHANGE_DIFF_TARGETS="v${LAST_VERSION}..HEAD"
VERSION_TITLE="${RC_VERSION} (__rc__)"
# Using GNU Awk syntax: -v LL specifies the title pattern variable.
TITLE_LINE=$(awk -v LL=${#VERSION_TITLE} 'BEGIN{for(c=0;c<LL;c++) printf "-"}')
VERSION_HEADER="$VERSION_TITLE\n${TITLE_LINE}"
# DEBUG INFO
echo -e "Comparing versions between: $CHANGE_DIFF_TARGETS\n"
# VERSION HEADER:
echo -e "$VERSION_HEADER\n" >> $TMP_CHANGE_LOG
# COLLECT ALL COMMITS:
git log --pretty=oneline --abbrev-commit $CHANGE_DIFF_TARGETS | sed 's/^/- /' >> $TMP_CHANGE_LOG
# DEBUG:
git log --pretty=oneline --abbrev-commit $CHANGE_DIFF_TARGETS | sed 's/^/- /'
# CHECK FINAL CONTENT
echo -e "\nCollected info:"
ls $WORKSPACE_DIR
cat $TMP_CHANGE_LOG
# APPEND INFO TO CHANGE FILE:
# 1. Finds the first (tbd) release
# 2. Populates space between (tbd) release and the latest one with RC changes
# NB: supporting macos and linux interoperability
# see https://stackoverflow.com/questions/43171648/sed-gives-sed-cant-read-no-such-file-or-directory
if [[ "$OSTYPE" == "darwin"* ]]; then
# begin: mac os support
sed -i '' "/^[0-9]\.0\.0.*\(tbd\)/{N;G;r\
\
$TMP_CHANGE_LOG
\
}" $CHANGE_FILE
# end;
else
# begin: linux support
sed -i "/^[0-9]\.0\.0.*\(tbd\)/{N;G;r\
\
$TMP_CHANGE_LOG
\
}" $CHANGE_FILE
# end;
fi
# CHANGE_LOG_CONTENTS=$(cat $TMP_CHANGE_LOG)

@ -88,8 +88,6 @@ except ImportError:
ez["use_setuptools"](**setup_args)
if to_reload:
from importlib import reload
reload(pkg_resources)
import pkg_resources

3
docker-requirements.txt Normal file

@ -0,0 +1,3 @@
passlib==1.7.2
bcrypt==3.1.7
watchdog==0.10.3

@ -1,14 +0,0 @@
<!-- -*-GFM-*- -->
# Docker Resources and Tests
This directory contains resources and tests for the docker image.
Note that for these tests to run, the pytest process must be able to run
`docker`. If you are on a system where that requires `sudo`, you will need to
run the tests with `sudo`.
Tests are here rather than in `/tests` because there's no reason to run these
tests as part of the usual `tox` process, which is run in CI against every
supported Python version. We only need to run the Docker tests once.

@ -1,11 +0,0 @@
# We use gunicorn as the default server in the docker container, with gevent
# workers
gevent==21.12.0
gunicorn==20.0.4
passlib==1.7.4
bcrypt==3.2.0
# If a user overrides args but does not override the server arg, we fall back to
# whatever bottle chooses as a default. Since the wsgiref server is not
# production-ready, install waitress as a fallback for these cases.
waitress==2.1.2
watchdog==1.0.2

@ -1,132 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
function run() {
# we're not root. Run as who we are.
if [[ "$EUID" -ne 0 ]]; then
eval "$@"
else
gosu pypiserver "$@"
fi
}
if [[ "$EUID" -ne 0 && "$EUID" -ne $(id -u pypiserver) ]]; then
USER_ID="$EUID"
WARN=(
"The pypiserver container was run as a non-root, non-pypiserver user."
"Pypiserver will be run as this user if possible, but this is not"
"officially supported."
)
echo "" 1>&2
echo "${WARN[@]}" 1>&2
echo "" 1>&2
else
USER_ID=$(id -u pypiserver)
fi
function print_permissions_help() {
MSG1=(
"If you are mounting a volume at /data or /data/packages and are running the"
"container on a linux system, you may need to add add a pypiserver"
"group to the host and give it permission to access the directories."
"Please see https://github.com/pypiserver/pypiserver/issues/256 for more"
"details."
)
MSG2=(
"Please see https://github.com/pypiserver/pypiserver/issues/256 for more"
"details."
)
echo "" 1>&2
echo "${MSG1[@]}" 1>&2
echo "" 1>&2
echo "${MSG2[@]}" 1>&2
}
# the user must have read and execute access to the /data directory
# (execute to be able to cd into directory and list content metadata)
if ! run test -r /data -a -x /data; then
chown -R "$USER_ID:pypiserver" /data || true
if ! run test -r /data -a -x /data; then
FAIL_MSG=(
"Cannot start pypiserver:"
"pypiserver user (UID $USER_ID)"
"or pypiserver group (GID $(id -g pypiserver))"
"must have read/execute access to /data"
)
echo "${FAIL_MSG[@]}" 1>&2
echo "" 1>&2
print_permissions_help
exit 1
fi
fi
# The /data/packages directory must exist
# It not existing is very unlikely, possibly impossible, because the VOLUME
# specification in the Dockerfile leads to its being created even if someone is
# mounting a volume at /data that does not contain a /packages subdirectory
if [[ ! -d "/data/packages" ]]; then
if ! run test -w /data; then
FAIL_MSG=(
"Cannot start pypiserver:"
"/data/packages does not exist and"
"pypiserver user (UID $USER_ID)"
"or pypiserver group (GID $(id -g pypiserver))"
"does not have write access to /data to create it"
)
echo "" 1>&2
echo "${FAIL_MSG[@]}" 1>&2
print_permissions_help
exit 1
fi
run mkdir /data/packages
fi
# The pypiserver user needs read/write/execute access to the packages directory
if ! run \
test -w /data/packages \
-a -r /data/packages \
-a -x /data/packages; then
# We'll try to chown as a last resort.
# Don't complain if it fails, since we'll bomb on the next check anyway.
chown -R "$USER_ID:pypiserver" /data/packages || true
if ! run \
test -w /data/packages \
-a -r /data/packages \
-a -x /data/packages; then
FAIL_MSG=(
"Cannot start pypiserver:"
"pypiserver user (UID $USER_ID)"
"or pypiserver group (GID $(id -g pypiserver))"
"must have read/write/execute access to /data/packages"
)
echo "" 1>&2
echo "${FAIL_MSG[@]}" 1>&2
print_permissions_help
exit 1
fi
fi
if [[ "$*" == "" ]]; then
# Use the gunicorn server by default, since it's more performant than
# bottle's default server
CMD=("run" "-p" "${PYPISERVER_PORT:-$PORT}" "--server" "gunicorn")
else
# this reassigns the array to the CMD variable
CMD=( "${@}" )
fi
if [[ "$EUID" -ne 0 ]]; then
exec pypi-server "${CMD[@]}"
else
exec gosu pypiserver pypi-server "${CMD[@]}"
fi

@ -1,14 +0,0 @@
"""Default gunicorn config for the docker environment.
To override, mount a new gunicorn config at /data/gunicorn.conf.py in your
Docker container.
"""
# pylint: disable=invalid-name
# Enable to log every request
# accesslog = "-"
errorlog = "-"
preload_app = True
workers = 1
worker_class = "gevent"

@ -1,598 +0,0 @@
"""Tests for the Pypiserver Docker image."""
import contextlib
import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import typing as t
from pathlib import Path
import httpx
import pypiserver
import pytest
PYPISERVER_PROCESS_NAME = "pypi-server"
TEST_DEMO_PIP_PACKAGE = "pypiserver-mypkg"
THIS_DIR = Path(__file__).parent
ROOT_DIR = THIS_DIR.parent
DOCKERFILE = ROOT_DIR / "Dockerfile"
FIXTURES = ROOT_DIR / "fixtures"
MYPKG_ROOT = FIXTURES / "mypkg"
HTPASS_FILE = FIXTURES / "htpasswd.a.a"
# This rule is largely useless when using pytest because of the need to use the
# name of the fixture as an argument to the test function or fixture using it
# pylint: disable=redefined-outer-name
#
# Also useless rule for our test context, where we may want to group test
# functions in a class to share common fixtures, but where we don't care about
# the `self` instance.
# pylint: disable=no-self-use
@pytest.fixture(scope="session")
def image() -> str:
"""Build the docker image for pypiserver.
Return the tag.
"""
tag = "pypiserver:test"
run(
"docker",
"build",
"--file",
str(DOCKERFILE),
"--tag",
tag,
str(ROOT_DIR),
cwd=ROOT_DIR,
)
return tag
@pytest.fixture(scope="session")
def mypkg_build() -> None:
"""Ensure the mypkg test fixture package is build."""
# Use make for this so that it will skip the build step if it's not needed
run("make", "mypkg", cwd=ROOT_DIR)
@pytest.fixture(scope="session")
def mypkg_paths(
mypkg_build: None, # pylint: disable=unused-argument
) -> t.Dict[str, Path]:
"""The path to the mypkg sdist file."""
dist_dir = Path(MYPKG_ROOT) / "dist"
assert dist_dir.exists()
sdist = dist_dir / "pypiserver_mypkg-1.0.0.tar.gz"
assert sdist.exists()
wheel = dist_dir / "pypiserver_mypkg-1.0.0-py2.py3-none-any.whl"
assert wheel.exists()
return {
"dist_dir": dist_dir,
"sdist": sdist,
"wheel": wheel,
}
def wait_for_container(port: int) -> None:
"""Wait for the container to be available."""
for _ in range(60):
try:
httpx.get(f"http://localhost:{port}").raise_for_status()
except (httpx.RequestError, httpx.HTTPStatusError):
time.sleep(1)
else:
return
# If we reach here, we've tried 60 times without success, meaning either
# the container is broken or it took more than about a minute to become
# functional, either of which cases is something we will want to look into.
raise RuntimeError("Could not connect to pypiserver container")
def get_socket() -> int:
"""Find a random, open socket and return it."""
# Close the socket automatically upon exiting the block
with contextlib.closing(
socket.socket(socket.AF_INET, socket.SOCK_STREAM)
) as sock:
# Bind to a random open socket >=1024
sock.bind(("", 0))
# Return the socket number
return sock.getsockname()[1]
class RunReturn(t.NamedTuple):
"""Simple wrapper around a simple subprocess call's results."""
returncode: int
out: str
err: str
def run(
*cmd: str,
capture: bool = False,
raise_on_err: bool = True,
check_code: t.Callable[[int], bool] = lambda c: c == 0,
**popen_kwargs: t.Any,
) -> RunReturn:
"""Run a command to completion."""
stdout = subprocess.PIPE if capture else None
stderr = subprocess.PIPE if capture else None
proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, **popen_kwargs)
out, err = proc.communicate()
result = RunReturn(
proc.returncode,
"" if out is None else out.decode(),
"" if err is None else err.decode(),
)
if raise_on_err and not check_code(result.returncode):
raise RuntimeError(result)
return result
def uninstall_pkgs() -> None:
"""Uninstall any packages we've installed."""
res = run("pip", "freeze", capture=True)
if any(
ln.strip().startswith(TEST_DEMO_PIP_PACKAGE)
for ln in res.out.splitlines()
):
run("pip", "uninstall", "-y", TEST_DEMO_PIP_PACKAGE)
@pytest.fixture(scope="session", autouse=True)
def session_cleanup() -> t.Iterator[None]:
"""Deal with any pollution of the local env."""
yield
uninstall_pkgs()
@pytest.fixture()
def cleanup() -> t.Iterator[None]:
"""Clean up after tests that may have affected the env."""
yield
uninstall_pkgs()
class TestCommands:
"""Test commands other than `run`."""
def test_help(self, image: str) -> None:
"""We can get help from the docker container."""
res = run("docker", "run", image, "--help", capture=True)
assert PYPISERVER_PROCESS_NAME in res.out
def test_version(self, image: str) -> None:
"""We can get the version from the docker container."""
res = run("docker", "run", image, "--version", capture=True)
assert res.out.strip() == pypiserver.__version__
class TestPermissions:
"""Test permission validation, especially with mounted volumes."""
@pytest.mark.parametrize("perms", (0o706, 0o701, 0o704))
def test_needs_rx_on_data(self, image: str, perms: int) -> None:
"""Read and execute permissions are required on /data."""
# Note we can't run this one as root because then we have to make a file
# that even we can't delete.
with tempfile.TemporaryDirectory() as tmpdir:
# Make sure the directory is not readable for anyone other than
# the owner
data_dir = Path(tmpdir) / "data"
data_dir.mkdir(mode=perms)
res = run(
"docker",
"run",
"--rm",
"--user",
# Run as a not us user ID, so access to /data will be
# determined by the "all other users" setting
str(os.getuid() + 1),
"-v",
# Mount the temporary directory as the /data directory
f"{data_dir}:/data",
image,
capture=True,
# This should error out, so we check that the code is non-zero
check_code=lambda c: c != 0,
)
assert "must have read/execute access" in res.err
@pytest.mark.parametrize(
"extra_args",
(("--user", str(os.getuid())), ("--user", str(os.getuid() + 1))),
)
def test_needs_rwx_on_packages(self, image: str, extra_args: tuple) -> None:
"""RWX permission is required on /data/packages."""
with tempfile.TemporaryDirectory() as tmpdir:
td_path = Path(tmpdir)
# Make the /data directory read/writable by anyone
td_path.chmod(0o777)
# Add the /data/packages directory, and make it readable by anyone,
# but writable only by the owner
(td_path / "packages").mkdir(mode=0o444)
res = run(
"docker",
"run",
"--rm",
*extra_args,
"-v",
# Mount the temporary directory as the /data directory
f"{tmpdir}:/data",
image,
capture=True,
# We should error out in this case
check_code=lambda c: c != 0,
)
assert "must have read/write/execute access" in res.err
def test_runs_as_pypiserver_user(self, image: str) -> None:
"""Test that the default run uses the pypiserver user."""
host_port = get_socket()
res = run(
"docker",
"run",
"--rm",
"--detach",
"--publish",
f"{host_port}:8080",
image,
capture=True,
)
container_id = res.out.strip()
try:
wait_for_container(host_port)
res = run(
"docker",
"container",
"exec",
container_id,
"ps",
"a",
capture=True,
)
proc_line = next(
filter(
# grab the process line for the pypi-server process
lambda ln: PYPISERVER_PROCESS_NAME in ln,
res.out.splitlines(),
)
)
user = proc_line.split()[1]
# the ps command on these alpine containers doesn't always show the
# full user name, so we only check for the first bit
assert user.startswith("pypi")
finally:
run("docker", "container", "rm", "-f", container_id)
class ContainerInfo(t.NamedTuple):
"""Info about a running container"""
container_id: str
port: int
args: tuple
class TestBasics:
"""Test basic pypiserver functionality in a simple unauthed container."""
# We want to automatically parametrize this class' tests with a variety of
# pypiserver args, since it should work the same in all of these cases
@pytest.fixture(
scope="class",
params=[
# default (gunicorn) server with cached backend
(),
# default (gunicorn) server with non-cached backend
("--backend", "simple-dir"),
# explicit gunicorn server with a non-cached backend
("--server", "gunicorn", "--backend", "simple-dir"),
# explicit gunicorn server
("--server", "gunicorn"),
# explicit waitress server
("--server", "wsgiref"),
# explicit wsgiref server
("--server", "wsgiref"),
],
)
def container(
self, request: pytest.FixtureRequest, image: str
) -> t.Iterator[ContainerInfo]:
"""Run the pypiserver container.
Returns the container ID.
"""
port = get_socket()
args = (
"docker",
"run",
"--rm",
"--publish",
f"{port}:8080",
"--detach",
image,
"run",
"--passwords",
".",
"--authenticate",
".",
*request.param,
)
res = run(*args, capture=True)
wait_for_container(port)
container_id = res.out.strip()
yield ContainerInfo(container_id, port, args)
run("docker", "container", "rm", "-f", container_id)
@pytest.fixture(scope="class")
def upload_mypkg(
self,
container: ContainerInfo,
mypkg_paths: t.Dict[str, Path],
) -> None:
"""Upload mypkg to the container."""
run(
sys.executable,
"-m",
"twine",
"upload",
"--repository-url",
f"http://localhost:{container.port}",
"--username",
"",
"--password",
"",
f"{mypkg_paths['dist_dir']}/*",
)
@pytest.mark.usefixtures("upload_mypkg")
def test_download(self, container: ContainerInfo) -> None:
"""Download mypkg from the container."""
with tempfile.TemporaryDirectory() as tmpdir:
run(
sys.executable,
"-m",
"pip",
"download",
"--index-url",
f"http://localhost:{container.port}/simple",
"--dest",
tmpdir,
"pypiserver_mypkg",
)
assert any(
"pypiserver_mypkg" in path.name
for path in Path(tmpdir).iterdir()
)
@pytest.mark.usefixtures("upload_mypkg", "cleanup")
def test_install(self, container: ContainerInfo) -> None:
"""Install mypkg from the container.
Note this also ensures that name normalization is working,
since we are requesting the package name with a dash, rather
than an underscore.
"""
run(
sys.executable,
"-m",
"pip",
"install",
"--force-reinstall",
"--index-url",
f"http://localhost:{container.port}/simple",
TEST_DEMO_PIP_PACKAGE,
)
run("python", "-c", "'import pypiserver_mypkg; mypkg.pkg_name()'")
def test_expected_server(self, container: ContainerInfo) -> None:
"""Ensure we run the server we think we're running."""
resp = httpx.get(f"http://localhost:{container.port}")
server = resp.headers["server"].lower()
arg_pairs = tuple(zip(container.args, container.args[1:]))
if (
container.args[-1] == "pypiserver:test"
or ("--server", "gunicorn") in arg_pairs
):
# We specified no overriding args, so we should run gunicorn, or
# we specified gunicorn in overriding args.
assert "gunicorn" in server
elif ("--server", "wsgiref") in arg_pairs:
# We explicitly specified the wsgiref server
assert "wsgiserver" in server
elif ("--server", "waitress") in arg_pairs:
# We explicitly specified the wsgiref server
assert "waitress" in server
else:
# We overrode args, so instead of using the gunicorn default,
# we use the `auto` option. Bottle won't choose gunicorn as an
# auto server, so we have waitress installed in the docker container
# as a fallback for these scenarios, since wsgiref is not production
# ready
assert "waitress" in server
def test_welcome(self, container: ContainerInfo) -> None:
"""View the welcome page."""
resp = httpx.get(f"http://localhost:{container.port}")
assert resp.status_code == 200
assert "pypiserver" in resp.text
class TestAuthed:
"""Test basic pypiserver functionality in a simple unauthed container."""
HOST_PORT = get_socket()
@pytest.fixture(scope="class")
def container(self, image: str) -> t.Iterator[str]:
"""Run the pypiserver container.
Returns the container ID.
"""
with tempfile.TemporaryDirectory() as tmpdir:
dirpath = Path(tmpdir)
shutil.copy2(HTPASS_FILE, dirpath / "htpasswd")
pkg_path = dirpath / "packages"
pkg_path.mkdir(mode=0o777)
res = run(
"docker",
"run",
"--rm",
"--publish",
f"{self.HOST_PORT}:8080",
"-v",
f"{dirpath / 'htpasswd'}:/data/htpasswd",
"--detach",
image,
"run",
"--passwords",
"/data/htpasswd",
"--authenticate",
"download, update",
capture=True,
)
wait_for_container(self.HOST_PORT)
container_id = res.out.strip()
yield container_id
run("docker", "container", "rm", "-f", container_id)
@pytest.fixture(scope="class")
def upload_mypkg(
self,
container: str, # pylint: disable=unused-argument
mypkg_paths: t.Dict[str, Path],
) -> None:
"""Upload mypkg to the container."""
run(
sys.executable,
"-m",
"twine",
"upload",
"--repository-url",
f"http://localhost:{self.HOST_PORT}",
"--username",
"a",
"--password",
"a",
f"{mypkg_paths['dist_dir']}/*",
)
def test_upload_failed_auth(
self,
container: str, # pylint: disable=unused-argument
mypkg_paths: t.Dict[str, Path],
) -> None:
"""Upload mypkg to the container."""
run(
sys.executable,
"-m",
"twine",
"upload",
"--repository-url",
f"http://localhost:{self.HOST_PORT}",
f"{mypkg_paths['dist_dir']}/*",
check_code=lambda c: c != 0,
)
@pytest.mark.usefixtures("upload_mypkg")
def test_download(self) -> None:
"""Download mypkg from the container."""
with tempfile.TemporaryDirectory() as tmpdir:
run(
sys.executable,
"-m",
"pip",
"download",
"--index-url",
f"http://a:a@localhost:{self.HOST_PORT}/simple",
"--dest",
tmpdir,
"pypiserver_mypkg",
)
assert any(
"pypiserver_mypkg" in path.name
for path in Path(tmpdir).iterdir()
)
@pytest.mark.usefixtures("upload_mypkg")
def test_download_failed_auth(self) -> None:
"""Download mypkg from the container."""
with tempfile.TemporaryDirectory() as tmpdir:
run(
sys.executable,
"-m",
"pip",
"download",
"--index-url",
f"http://foo:bar@localhost:{self.HOST_PORT}/simple",
"--dest",
tmpdir,
"pypiserver_mypkg",
check_code=lambda c: c != 0,
)
@pytest.mark.usefixtures("upload_mypkg", "cleanup")
def test_install(self) -> None:
"""Install mypkg from the container.
Note this also ensures that name normalization is working,
since we are requesting the package name with a dash, rather
than an underscore.
"""
run(
sys.executable,
"-m",
"pip",
"install",
"--force-reinstall",
"--index-url",
f"http://a:a@localhost:{self.HOST_PORT}/simple",
TEST_DEMO_PIP_PACKAGE,
)
run("python", "-c", "'import pypiserver_mypkg; mypkg.pkg_name()'")
@pytest.mark.usefixtures("upload_mypkg", "cleanup")
def test_install_failed_auth(self) -> None:
"""Install mypkg from the container.
Note this also ensures that name normalization is working,
since we are requesting the package name with a dash, rather
than an underscore.
"""
run(
sys.executable,
"-m",
"pip",
"install",
"--force-reinstall",
"--no-cache",
"--index-url",
f"http://foo:bar@localhost:{self.HOST_PORT}/simple",
TEST_DEMO_PIP_PACKAGE,
check_code=lambda c: c != 0,
)
def test_welcome(self) -> None:
"""View the welcome page."""
resp = httpx.get(f"http://localhost:{self.HOST_PORT}")
assert resp.status_code == 200
assert "pypiserver" in resp.text

@ -1,33 +0,0 @@
# pypiserver docs
Welcome to the documentation section of `pypiserver`.
> :bulb: This documentation is still a work in progress
> and would benefit from your help.
## More information
If something is missing in the documentation here, maybe it is covered in
[README.md](../README.md).
## Documentation setup
> Some areas to describe are highlighted in
[this issue](https://github.com/pypiserver/pypiserver/issues/368)
([#368](https://github.com/pypiserver/pypiserver/issues/368)),
but feel free to add if you see something missing there.
The [`./contents`](./contents/) directory is designed to
all the documentation files.
### How to contribute
Everyone is very welcome to extend `pypiserver` documentation.
1. :thought_balloon: If you think of a topic you would like to describe, this is great.
2. :deciduous_tree: Fork this project and clone it locally to start your contribution.
3. :page_facing_up: Create a new Markdown (`<doc title>.md`) file/subdirectory+file inside the [`./contents`](./contents/):
- :file_folder: Use subdirectories to organize related documentation into topics.
4. :octocat: Push your changes to your fork and open a PR to this repository.
5. :bell: Tag someone from the maintainers if you'd like, but in any case we will try to review the PR as soon as possible.
6. :dizzy: Once reviewed and accepted, your documentation will be merged into this repository.

@ -1,123 +0,0 @@
# `Pypi-server` Release Workflow Reference
The official `pypi-server` releases are handled using
[GitHub Actions workflows](../../../.github/workflows/).
## General release process
```mermaid
flowchart LR
rc["release-candidate ⭐️"]
rn["release-notes 📝"]
rm["confirmed-tag ✅"]
ci["code-checks 🧪"]
pk["build-and-pack 📦"]
py["pypi-index 🗃️"]
do["docker-hub 🐳"]
gh["gh-container-registry 🚀"]
gr["github-release 📣"]
subgraph "Preparation 🌱"
rc-->rn-->rm
end
subgraph "Integration 🪴"
rm-->ci-->pk
end
subgraph "Deploy 🌳"
pk--> py & do & gh & gr
end
```
## Process walkthrough
> 🗺️ ***This description approximates the real GitHub workflows and steps.***
> 👀 *For a more detailed view, do check out the linked resources as you read.*
### Preparation 🌱
> 🛠️ *These step are applicable only for maintainers.*
#### Release candidate ⭐️
A new release candidate can be initiated ***manually** or **on a monthly schedule***.
This is done via the [`rc.yml`](../../../.github/workflows/rc.yml) GH
Workflow's `workflow_dispatch` or `schedule` trigger.
The workflow automatically prepares a list of changes for the `CHANGES.rst` and
creates a new Pull Request *(rc PR)* named
`chore(auto-release-candidate-YYY-MM-DD)` including these draft change notes.
#### Release notes 📝
In the created rc PR, open the `CHANGES.rst` and:
1. ***adjust the suggested changelog items***
2. ***choose & set the next released version***
3. ***set the right release date***
Commit the changes and push them to the head branch of the rc PR.
#### Confirmed tag ✅
1. Once everything is looking good, ***approve and merge*** the rc PR.
It will create the new *commit* with the updated `CHANGES.rst`
on the default branch.
2. Next, to create a release tag, ***manually run*** the
[`rt.yml`](../../../.github/workflows/rt.yml) GH Workflow.
First, it executes all the [`bumpver`](../../../bin/README.md) procedures.
Next, it commits and pushes the new **version tag** to the default branch.
### Integration 🪴
#### Code checks 🧪
Once any *commmit* or *tag* is pushed to the default branch,
[`ci.yml`](../../../.github/workflows/ci.yml) GH Workflow automatically
executes diverse code checks: e.g. *linting*, *formatting*, *tests*.
#### Build and pack 📦
If all the checks are successful, [`ci.yml`](../../../.github/workflows/ci.yml)
builds all the code artifacts: e.g. *wheels*, *docker images*.
### Deploy 🌳
#### Publish to PyPi 🗃️
> 🏷️ This happens only on new *version tags*.
Once everythig is built, [`ci.yml`](../../../.github/workflows/ci.yml) uploads
the wheels to the [`pypiserver` PyPi project](https://pypi.org/project/pypiserver/).
#### Publish to Docker Hub 🐳
> 🏷️ Docker image *tags* are determined on the fly.
If all is successful so far, [`ci.yml`](../../../.github/workflows/ci.yml) tags
the built docker images and pushes them to the
[`pypiserver` Docker Hub repository](https://hub.docker.com/r/pypiserver/pypiserver).
#### Publish to GitHub Container Registry 🚀
> 🏷️ Docker image *tags* are determined on the fly.
For all `stable` (i.e. `latest`, tag, release ...) tags derived by
[`ci.yml`](../../../.github/workflows/ci.yml) tags,
the built docker images are *also* pushed to
[`pypiserver` GitHub Container Registry](https://github.com/orgs/pypiserver/packages?repo_name=pypiserver).
#### Publish a GitHub Release draft 📣
> 🛠️ *This step is applicable only for maintainers.*
> 🏷️ This happens only on new *version tags*.
To make the release noticeable, [`ci.yml`](../../../.github/workflows/ci.yml)
also creates a *draft*
[GitHub Release entry in the `pypiserver` repository](https://github.com/pypiserver/pypiserver/releases).
> 📝 Since it is a *draft*, the entry should be *manually* adjusted further.

11
entrypoint.sh Executable file

@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
# chown the VOLUME mount set in the dockerfile
# If you're using an alternative directory for packages,
# you'll need to ensure that pypiserver has read and
# write access to that directory
chown -R pypiserver:pypiserver /data/packages
exec gosu pypiserver pypi-server -p "$PORT" $@

@ -1,6 +0,0 @@
"""A very simple python file to package for testing."""
def pkg_name() -> None:
"""Print the package name."""
print("mypkg")

@ -1,7 +0,0 @@
[wheel]
universal=1
[mypy]
follow_imports = silent
ignore_missing_imports = True

@ -1,10 +0,0 @@
"""A simple setup file for this test package."""
from setuptools import setup
setup(
name="pypiserver_mypkg",
description="Test pkg",
version="1.0.0",
packages=["mypkg"],
)

@ -7,9 +7,9 @@ import typing as t
from pypiserver.bottle import Bottle
from pypiserver.config import Config, RunConfig, strtobool
version = __version__ = "2.1.1"
version = __version__ = "2.0.0dev1"
__version_info__ = tuple(_re.split("[.-]", __version__))
__updated__ = "2024-04-25 01:23:25"
__updated__ = "2020-10-11 11:23:15"
__title__ = "pypiserver"
__summary__ = "A minimal PyPI server for use with pip/easy_install."
@ -121,7 +121,7 @@ def app(**kwargs: t.Any) -> Bottle:
(or its base), defined in `pypiserver.config`, may be overridden.
"""
config = Config.default_with_overrides(**backwards_compat_kwargs(kwargs))
return setup_routes_from_config(app_from_config(config), config)
return app_from_config(config)
def app_from_config(config: RunConfig) -> Bottle:
@ -141,20 +141,6 @@ def app_from_config(config: RunConfig) -> Bottle:
return _app.app
def setup_routes_from_config(app: Bottle, config: RunConfig) -> Bottle:
"""Set up additional routes supplied from the config."""
def _setup_health_endpoint(app, config):
if config.health_endpoint in [route.rule for route in app.routes]:
raise RuntimeError(
"Provided health endpoint overlaps with existing routes"
)
app.route(config.health_endpoint, "GET", lambda: "Ok")
_setup_health_endpoint(app, config)
return app
T = t.TypeVar("T")

@ -1,28 +1,26 @@
#! /usr/bin/env python3
#! /usr/bin/env python
"""Entrypoint for pypiserver."""
import enum
import importlib
from __future__ import print_function
import logging
import sys
import typing as t
from pathlib import Path
from wsgiref.simple_server import WSGIRequestHandler
import functools as ft
from pypiserver.config import Config, UpdateConfig
log = logging.getLogger("pypiserver.main")
def init_logging(
level: int = logging.NOTSET,
frmt: str = None,
filename: t.Union[str, Path] = None,
level=logging.NOTSET,
frmt=None,
filename=None,
stream: t.Optional[t.IO] = sys.stderr,
logger: logging.Logger = None,
) -> None:
"""Configure the specified logger, or the root logger otherwise."""
logger=None,
):
logger = logger or logging.getLogger()
logger.setLevel(level)
@ -38,90 +36,14 @@ def init_logging(
logger.addHandler(handler)
class WsgiHandler(WSGIRequestHandler):
"""A simple request handler to configure logging."""
# The default `FixedHandler` that bottle's `WSGIRefServer` uses does not
# log in a particularly predictable or configurable way. We'll pass this
# in to use instead.
def address_string(self) -> str: # Prevent reverse DNS lookups please.
# This method copied directly from bottle's `FixedHandler` and
# maintained on the Chesterton's fence principle (i.e. I don't know
# why it's important, so I'm not going to get rid of it)
return self.client_address[0]
def log_message(
self, format: str, *args: t.Any # pylint: disable=redefined-builtin
) -> None:
"""Log a message."""
# The log_message method on the `HttpRequestHandler` base class just
# writes directly to stderr. We'll use its same formatting, but pass
# it through the logger instead.
log.info(
"%s - - [%s] %s\n",
self.address_string(),
self.log_date_time_string(),
format % args,
)
class AutoServer(enum.Enum):
"""Expected servers that can be automaticlaly selected by bottle."""
Waitress = enum.auto()
Paste = enum.auto()
Twisted = enum.auto()
CherryPy = enum.auto()
WsgiRef = enum.auto()
# Possible automatically selected servers. This MUST match the available
# auto servers in bottle.py
AUTO_SERVER_IMPORTS = (
(AutoServer.Waitress, "waitress"),
(AutoServer.Paste, "paste"),
(AutoServer.Twisted, "twisted.web"),
(AutoServer.CherryPy, "cheroot.wsgi"),
(AutoServer.CherryPy, "cherrypy.wsgiserver"),
# this should always be available because it's part of the stdlib
(AutoServer.WsgiRef, "wsgiref"),
)
def _can_import(name: str) -> bool:
"""Attempt to import a module. Return a bool indicating success."""
try:
importlib.import_module(name)
return True
except ImportError:
return False
def guess_auto_server() -> AutoServer:
"""Guess which server bottle will use for the auto setting."""
# Return the first server that can be imported.
server = next(
(s for s, i in AUTO_SERVER_IMPORTS if _can_import(i)),
None,
)
if server is None:
raise RuntimeError(
"Unexpected error determining bottle auto server. There may be an "
"issue with this python environment. Please report this bug at "
"https://github.com/pypiserver/pypiserver/issues"
)
return server
def main(argv: t.Sequence[str] = None) -> None:
def main(argv=None):
"""Application entrypoint for pypiserver.
This function drives the application (as opposed to the library)
implementation of pypiserver. Usage from the commandline will result in
this function being called.
"""
# pylint: disable=import-outside-toplevel
import pypiserver # pylint: disable=redefined-outer-name
import pypiserver
if argv is None:
# The first item in sys.argv is the name of the python file being
@ -162,56 +84,25 @@ def main(argv: t.Sequence[str] = None) -> None:
from pypiserver import bottle
bottle.debug(config.verbosity > 1)
bottle._stderr = ft.partial( # pylint: disable=protected-access
bottle._stderr = ft.partial(
_logwrite, logging.getLogger(bottle.__name__), logging.INFO
)
# Here `app` is a Bottle instance, which we pass to bottle.run() to run
# the server
app = pypiserver.app_from_config(config)
app = pypiserver.setup_routes_from_config(app, config)
if config.server_method == "gunicorn":
# When bottle runs gunicorn, gunicorn tries to pull its arguments from
# sys.argv. Because pypiserver's arguments don't match gunicorn's,
# this leads to errors.
# Gunicorn can be configured by using a `gunicorn.conf.py` config file
# or by specifying the `GUNICORN_CMD_ARGS` env var. See gunicorn
# docs for more info.
sys.argv = ["gunicorn"]
wsgi_kwargs = {"handler_class": WsgiHandler}
if config.server_method == "auto":
expected_server = guess_auto_server()
extra_kwargs = (
wsgi_kwargs if expected_server is AutoServer.WsgiRef else {}
)
log.debug(
"Server 'auto' selected. Expecting bottle to run '%s'. "
"Passing extra keyword args: %s",
expected_server.name,
extra_kwargs,
)
else:
extra_kwargs = wsgi_kwargs if config.server_method == "wsgiref" else {}
log.debug(
"Running bottle with selected server '%s'", config.server_method
)
bottle.run(
app=app,
host=config.host,
port=config.port,
server=config.server_method,
**extra_kwargs,
)
def _logwrite(logger, level, msg):
if msg:
line_endings = ["\r\n", "\n\r", "\n"]
for le in line_endings: # pylint: disable=invalid-name
for le in line_endings:
if msg.endswith(le):
msg = msg[: -len(le)]
if msg:

@ -1,20 +1,14 @@
from collections import namedtuple
import logging
import mimetypes
import os
import re
import xml.dom.minidom
import xmlrpc.client as xmlrpclib
import zipfile
from collections import defaultdict
from collections import namedtuple
from io import BytesIO
from json import dumps
from urllib.parse import urljoin, urlparse
import xml.dom.minidom
from pypiserver.config import RunConfig
from . import __version__
from . import core
from . import mirror_cache
from .bottle import (
static_file,
redirect,
@ -24,10 +18,26 @@ from .bottle import (
Bottle,
template,
)
from .pkg_helpers import guess_pkgname_and_version, normalize_pkgname_for_url
try:
import xmlrpc.client as xmlrpclib # py3
except ImportError:
import xmlrpclib # py2
try:
from io import BytesIO
except ImportError:
from StringIO import StringIO as BytesIO
try: # PY3
from urllib.parse import urljoin, urlparse
except ImportError: # PY2
from urlparse import urljoin, urlparse
log = logging.getLogger(__name__)
config: RunConfig
app = Bottle()
@ -93,13 +103,19 @@ def favicon():
def root():
fp = request.custom_fullpath
try:
numpkgs = len(list(config.iter_packages()))
except Exception as exc:
log.error(f"Could not list packages: {exc}")
numpkgs = 0
# Ensure template() does not consider `msg` as filename!
msg = config.welcome_msg + "\n"
return template(
msg,
URL=request.url.rstrip("/") + "/",
VERSION=__version__,
NUMPKGS=config.backend.package_count(),
NUMPKGS=numpkgs,
PACKAGES=fp.rstrip("/") + "/packages/",
SIMPLE=fp.rstrip("/") + "/simple/",
)
@ -132,12 +148,16 @@ def remove_pkg():
if not name or not version:
msg = f"Missing 'name'/'version' fields: name={name}, version={version}"
raise HTTPError(400, msg)
pkgs = list(config.backend.find_version(name, version))
if not pkgs:
pkgs = list(
filter(
lambda pkg: pkg.pkgname == name and pkg.version == version,
core.find_packages(config.iter_packages()),
)
)
if len(pkgs) == 0:
raise HTTPError(404, f"{name} ({version}) not found")
for pkg in pkgs:
config.backend.remove_package(pkg)
os.unlink(pkg.fn)
Upload = namedtuple("Upload", "pkg sig")
@ -163,11 +183,13 @@ def file_upload():
continue
if (
not is_valid_pkg_filename(uf.raw_filename)
or guess_pkgname_and_version(uf.raw_filename) is None
or core.guess_pkgname_and_version(uf.raw_filename) is None
):
raise HTTPError(400, f"Bad filename: {uf.raw_filename}")
if not config.overwrite and config.backend.exists(uf.raw_filename):
if not config.overwrite and core.exists(
config.package_root, uf.raw_filename
):
log.warning(
f"Cannot upload {uf.raw_filename!r} since it already exists! \n"
" You may start server with `--overwrite` option. "
@ -178,7 +200,7 @@ def file_upload():
" You may start server with `--overwrite` option.",
)
config.backend.add_package(uf.raw_filename, uf.file)
core.store(config.package_root, uf.raw_filename, uf.save)
if request.auth:
user = request.auth[0]
else:
@ -209,10 +231,10 @@ def update():
@app.route("/simple")
@app.route("/simple/:project")
@app.route("/simple/:prefix")
@app.route("/packages")
@auth("list")
def pep_503_redirects(project=None):
def pep_503_redirects(prefix=None):
return redirect(request.custom_fullpath + "/", 301)
@ -226,7 +248,7 @@ def handle_rpc():
.childNodes[0]
.wholeText.strip()
)
log.debug(f"Processing RPC2 request for '{methodname}'")
log.info(f"Processing RPC2 request for '{methodname}'")
if methodname == "search":
value = (
parser.getElementsByTagName("string")[0]
@ -235,7 +257,7 @@ def handle_rpc():
)
response = []
ordering = 0
for p in config.backend.get_all_packages():
for p in config.iter_packages():
if p.pkgname.count(value) > 0:
# We do not presently have any description/summary, returning
# version instead
@ -256,9 +278,8 @@ def handle_rpc():
@app.route("/simple/")
@auth("list")
def simpleindex():
links = sorted(config.backend.get_projects())
links = sorted(core.get_prefixes(config.iter_packages()))
tmpl = """\
<!DOCTYPE html>
<html>
<head>
<title>Simple Index</title>
@ -274,67 +295,60 @@ def simpleindex():
return template(tmpl, links=links)
@app.route("/simple/:project/")
@app.route("/simple/:prefix/")
@auth("list")
def simple(project):
# PEP 503: require normalized project
normalized = normalize_pkgname_for_url(project)
if project != normalized:
return redirect(f"/simple/{normalized}/", 301)
def simple(prefix=""):
# PEP 503: require normalized prefix
normalized = core.normalize_pkgname_for_url(prefix)
if prefix != normalized:
return redirect("/simple/{0}/".format(normalized), 301)
packages = sorted(
config.backend.find_project_packages(project),
files = sorted(
core.find_packages(config.iter_packages(), prefix=prefix),
key=lambda x: (x.parsed_version, x.relfn),
)
if not packages:
if config.mirror:
return mirror_cache.MirrorCache.add(project=project, config=config)
elif not config.disable_fallback:
return redirect(f"{config.fallback_url.rstrip('/')}/{project}/")
if not files:
if not config.disable_fallback:
return redirect(f"{config.fallback_url.rstrip('/')}/{prefix}/")
return HTTPError(404, f"Not Found ({normalized} does not exist)\n\n")
current_uri = request.custom_fullpath
links = (
fp = request.custom_fullpath
links = [
(
os.path.basename(pkg.relfn),
urljoin(current_uri, f"../../packages/{pkg.fname_and_hash}"),
os.path.basename(f.relfn),
urljoin(fp, f"../../packages/{f.fname_and_hash(config.hash_algo)}"),
)
for pkg in packages
)
for f in files
]
tmpl = """\
<!DOCTYPE html>
<html>
<head>
<title>Links for {{project}}</title>
<title>Links for {{prefix}}</title>
</head>
<body>
<h1>Links for {{project}}</h1>
<h1>Links for {{prefix}}</h1>
% for file, href in links:
<a href="{{href}}">{{file}}</a><br>
% end
</body>
</html>
"""
return template(tmpl, project=project, links=links)
return template(tmpl, prefix=prefix, links=links)
@app.route("/packages/")
@auth("list")
def list_packages():
fp = request.custom_fullpath
packages = sorted(
config.backend.get_all_packages(),
files = sorted(
core.find_packages(config.iter_packages()),
key=lambda x: (os.path.dirname(x.relfn), x.pkgname, x.parsed_version),
)
links = (
(pkg.relfn_unix, urljoin(fp, pkg.fname_and_hash)) for pkg in packages
)
links = [
(f.relfn_unix, urljoin(fp, f.fname_and_hash(config.hash_algo)))
for f in files
]
tmpl = """\
<!DOCTYPE html>
<html>
<head>
<title>Index of packages</title>
@ -353,7 +367,7 @@ def list_packages():
@app.route("/packages/:filename#.*#")
@auth("download")
def server_static(filename):
entries = config.backend.get_all_packages()
entries = core.find_packages(config.iter_packages())
for x in entries:
f = x.relfn_unix
if f == filename:
@ -367,43 +381,12 @@ def server_static(filename):
"Cache-Control", f"public, max-age={config.cache_control}"
)
return response
if config.mirror and mirror_cache.MirrorCache.has_project(filename):
return mirror_cache.MirrorCache.get_static_file(filename=filename, config=config)
return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
@app.route("/:project/json")
@auth("list")
def json_info(project):
# PEP 503: require normalized project
normalized = normalize_pkgname_for_url(project)
if project != normalized:
return redirect(f"/{normalized}/json", 301)
packages = sorted(
config.backend.find_project_packages(project),
key=lambda x: x.parsed_version,
reverse=True,
)
if not packages:
raise HTTPError(404, f"package {project} not found")
latest_version = packages[0].version
releases = defaultdict(list)
req_url = request.url
for x in packages:
releases[x.version].append(
{"url": urljoin(req_url, "../../packages/" + x.relfn)}
)
rv = {"info": {"version": latest_version}, "releases": releases}
response.content_type = "application/json"
return dumps(rv)
@app.route("/:project")
@app.route("/:project/")
def bad_url(project):
@app.route("/:prefix")
@app.route("/:prefix/")
def bad_url(prefix):
"""Redirect unknown root URLs to /simple/."""
return redirect(core.get_bad_url_redirect_path(request, project))
return redirect(core.get_bad_url_redirect_path(request, prefix))

@ -1,325 +0,0 @@
import abc
import functools
import hashlib
import itertools
import logging
import os
import typing as t
from pathlib import Path
from .cache import CacheManager, ENABLE_CACHING
from .core import PkgFile
from .pkg_helpers import (
normalize_pkgname,
is_listed_path,
guess_pkgname_and_version,
)
if t.TYPE_CHECKING:
from .config import _ConfigCommon as Configuration
log = logging.getLogger(__name__)
PathLike = t.Union[str, os.PathLike]
class IBackend(abc.ABC):
@abc.abstractmethod
def get_all_packages(self) -> t.Iterable[PkgFile]:
pass
@abc.abstractmethod
def find_project_packages(self, project: str) -> t.Iterable[PkgFile]:
pass
@abc.abstractmethod
def find_version(self, name: str, version: str) -> t.Iterable[PkgFile]:
pass
@abc.abstractmethod
def get_projects(self) -> t.Iterable[str]:
pass
@abc.abstractmethod
def exists(self, filename: str) -> bool:
pass
@abc.abstractmethod
def digest(self, pkg: PkgFile) -> t.Optional[str]:
pass
@abc.abstractmethod
def package_count(self) -> int:
pass
@abc.abstractmethod
def add_package(self, filename: str, stream: t.BinaryIO) -> None:
pass
@abc.abstractmethod
def remove_package(self, pkg: PkgFile) -> None:
pass
class Backend(IBackend, abc.ABC):
def __init__(self, config: "Configuration"):
self.hash_algo = config.hash_algo
@abc.abstractmethod
def get_all_packages(self) -> t.Iterable[PkgFile]:
"""Implement this method to return an Iterable of all packages (as
PkgFile objects) that are available in the Backend.
"""
pass
@abc.abstractmethod
def add_package(self, filename: str, stream: t.BinaryIO) -> None:
"""Add a package to the Backend. `filename` is the package's filename
(without any directory parts). It is just a name, there is no file by
that name (yet). `stream` is an open file-like object that can be used
to read the file's content. To convert the package into an actual file
on disk, run `write_file(filename, stream)`.
"""
pass
@abc.abstractmethod
def remove_package(self, pkg: PkgFile) -> None:
"""Remove a package from the Backend"""
pass
@abc.abstractmethod
def exists(self, filename: str) -> bool:
"""Does a package by the given name exist?"""
pass
def digest(self, pkg: PkgFile) -> t.Optional[str]:
if self.hash_algo is None or pkg.fn is None:
return None
return digest_file(pkg.fn, self.hash_algo)
def package_count(self) -> int:
"""Return a count of all available packages. When implementing a Backend
class, either use this method as is, or override it with a more
performant version.
"""
return sum(1 for _ in self.get_all_packages())
def get_projects(self) -> t.Iterable[str]:
"""Return an iterable of all (unique) projects available in the store
in their PEP503 normalized form. When implementing a Backend class,
either use this method as is, or override it with a more performant
version.
"""
return set(package.pkgname_norm for package in self.get_all_packages())
def find_project_packages(self, project: str) -> t.Iterable[PkgFile]:
"""Find all packages from a given project. The project may be given
as either the normalized or canonical name. When implementing a
Backend class, either use this method as is, or override it with a
more performant version.
"""
return (
x
for x in self.get_all_packages()
if normalize_pkgname(project) == x.pkgname_norm
)
def find_version(self, name: str, version: str) -> t.Iterable[PkgFile]:
"""Return all packages that match PkgFile.pkgname == name and
PkgFile.version == version` When implementing a Backend class,
either use this method as is, or override it with a more performant
version.
"""
return filter(
lambda pkg: pkg.pkgname == name and pkg.version == version,
self.get_all_packages(),
)
class SimpleFileBackend(Backend):
def __init__(self, config: "Configuration"):
super().__init__(config)
self.roots = [Path(root).resolve() for root in config.roots]
def get_all_packages(self) -> t.Iterable[PkgFile]:
return itertools.chain.from_iterable(listdir(r) for r in self.roots)
def add_package(self, filename: str, stream: t.BinaryIO) -> None:
write_file(stream, self.roots[0].joinpath(filename))
def remove_package(self, pkg: PkgFile) -> None:
if pkg.fn is not None:
try:
os.remove(pkg.fn)
except FileNotFoundError:
log.warning(
"Tried to remove %s, but it is already gone", pkg.fn
)
except OSError:
log.exception("Unexpected error removing package: %s", pkg.fn)
raise
def exists(self, filename: str) -> bool:
return any(
filename == existing_file.name
for root in self.roots
for existing_file in all_listed_files(root)
)
class CachingFileBackend(SimpleFileBackend):
def __init__(
self,
config: "Configuration",
cache_manager: t.Optional[CacheManager] = None,
):
super().__init__(config)
self.cache_manager = cache_manager or CacheManager() # type: ignore
def add_package(self, filename: str, stream: t.BinaryIO) -> None:
super().add_package(filename, stream)
self.cache_manager.invalidate_root_cache(self.roots[0])
def remove_package(self, pkg: PkgFile) -> None:
super().remove_package(pkg)
self.cache_manager.invalidate_root_cache(pkg.root)
def get_all_packages(self) -> t.Iterable[PkgFile]:
return itertools.chain.from_iterable(
self.cache_manager.listdir(r, listdir) for r in self.roots
)
def digest(self, pkg: PkgFile) -> t.Optional[str]:
if self.hash_algo is None or pkg.fn is None:
return None
return self.cache_manager.digest_file(
pkg.fn, self.hash_algo, digest_file
)
def write_file(fh: t.BinaryIO, destination: PathLike) -> None:
"""write a byte stream into a destination file. Writes are chunked to reduce
the memory footprint
"""
chunk_size = 2**20 # 1 MB
offset = fh.tell()
try:
with open(destination, "wb") as dest:
for chunk in iter(lambda: fh.read(chunk_size), b""):
dest.write(chunk)
finally:
fh.seek(offset)
def listdir(root: Path) -> t.Iterator[PkgFile]:
root = root.resolve()
files = all_listed_files(root)
yield from valid_packages(root, files)
def all_listed_files(root: Path) -> t.Iterator[Path]:
for dirpath, dirnames, filenames in os.walk(root):
dirnames[:] = (
dirname for dirname in dirnames if is_listed_path(Path(dirname))
)
for filename in filenames:
if not is_listed_path(Path(filename)):
continue
filepath = root / dirpath / filename
if Path(filepath).is_file():
yield filepath
def valid_packages(root: Path, files: t.Iterable[Path]) -> t.Iterator[PkgFile]:
for file in files:
res = guess_pkgname_and_version(str(file.name))
if res is not None:
pkgname, version = res
fn = str(file)
root_name = str(root)
yield PkgFile(
pkgname=pkgname,
version=version,
fn=fn,
root=root_name,
relfn=fn[len(root_name) + 1 :],
)
def digest_file(file_path: PathLike, hash_algo: str) -> str:
"""
Reads and digests a file according to specified hashing-algorith.
:param file_path: path to a file on disk
:param hash_algo: any algo contained in :mod:`hashlib`
:return: <hash_algo>=<hex_digest>
From http://stackoverflow.com/a/21565932/548792
"""
blocksize = 2**16
digester = hashlib.new(hash_algo)
with open(file_path, "rb") as f:
for block in iter(lambda: f.read(blocksize), b""):
digester.update(block)
return f"{hash_algo}={digester.hexdigest()}"
def get_file_backend(config: "Configuration") -> Backend:
if ENABLE_CACHING:
return CachingFileBackend(config)
return SimpleFileBackend(config)
PkgFunc = t.TypeVar("PkgFunc", bound=t.Callable[..., t.Iterable[PkgFile]])
def with_digester(func: PkgFunc) -> PkgFunc:
@functools.wraps(func)
def add_digester_method(
self: "BackendProxy", *args: t.Any, **kwargs: t.Any
) -> t.Iterable[PkgFile]:
packages = func(self, *args, **kwargs)
for package in packages:
package.digester = self.backend.digest
yield package
return t.cast(PkgFunc, add_digester_method)
class BackendProxy(IBackend):
def __init__(self, wraps: Backend):
self.backend = wraps
@with_digester
def get_all_packages(self) -> t.Iterable[PkgFile]:
return self.backend.get_all_packages()
@with_digester
def find_project_packages(self, project: str) -> t.Iterable[PkgFile]:
return self.backend.find_project_packages(project)
def find_version(self, name: str, version: str) -> t.Iterable[PkgFile]:
return self.backend.find_version(name, version)
def get_projects(self) -> t.Iterable[str]:
return self.backend.get_projects()
def exists(self, filename: str) -> bool:
assert "/" not in filename
return self.backend.exists(filename)
def package_count(self) -> int:
return self.backend.package_count()
def add_package(self, filename: str, fh: t.BinaryIO) -> None:
assert "/" not in filename
return self.backend.add_package(filename, fh)
def remove_package(self, pkg: PkgFile) -> None:
return self.backend.remove_package(pkg)
def digest(self, pkg: PkgFile) -> t.Optional[str]:
return self.backend.digest(pkg)

@ -41,6 +41,7 @@ import base64, cgi, email.utils, functools, hmac, itertools, mimetypes,\
from datetime import date as datedate, datetime, timedelta
from tempfile import TemporaryFile
from traceback import format_exc, print_exc
from inspect import getargspec
from unicodedata import normalize
@ -78,7 +79,6 @@ except IOError:
# Lots of stdlib and builtin differences.
if py3k:
import http.client as httplib
from inspect import getfullargspec as getargspec
import _thread as thread
from urllib.parse import urljoin, SplitResult as UrlSplitResult
from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote
@ -101,7 +101,6 @@ if py3k:
def _raise(*a): raise a[0](a[1]).with_traceback(a[2])
else: # 2.x
import httplib
from inspect import getargspec
import thread
from urlparse import urljoin, SplitResult as UrlSplitResult
from urllib import urlencode, quote as urlquote, unquote as urlunquote
@ -1753,7 +1752,7 @@ class JSONPlugin(object):
if isinstance(rv, dict):
#Attempt to serialize, raises exception on failure
json_response = dumps(rv)
#Set content type only if serialization successful
#Set content type only if serialization succesful
response.content_type = 'application/json'
return json_response
elif isinstance(rv, HTTPResponse) and isinstance(rv.body, dict):
@ -2328,7 +2327,7 @@ class ResourceManager(object):
''' Search for a resource and return an absolute file path, or `None`.
The :attr:`path` list is searched in order. The first match is
returned. Symlinks are followed. The result is cached to speed up
returend. Symlinks are followed. The result is cached to speed up
future lookups. '''
if name not in self.cache or DEBUG:
for path in self.path:

@ -4,23 +4,10 @@
#
from os.path import dirname
from pathlib import Path
import typing as t
from watchdog.observers import Observer
import threading
try:
from watchdog.observers import Observer
ENABLE_CACHING = True
except ImportError:
Observer = None
ENABLE_CACHING = False
if t.TYPE_CHECKING:
from pypiserver.core import PkgFile
class CacheManager:
"""
@ -39,11 +26,6 @@ class CacheManager:
"""
def __init__(self):
if not ENABLE_CACHING:
raise RuntimeError(
"Please install the extra cache requirements by running 'pip "
"install pypiserver[cache]' to use the CachingFileBackend"
)
# Cache for listdir output
self.listdir_cache = {}
@ -64,12 +46,7 @@ class CacheManager:
self.digest_lock = threading.Lock()
self.listdir_lock = threading.Lock()
def listdir(
self,
root: t.Union[Path, str],
impl_fn: t.Callable[[Path], t.Iterable["PkgFile"]],
) -> t.Iterable["PkgFile"]:
root = str(root)
def listdir(self, root, impl_fn):
with self.listdir_lock:
try:
return self.listdir_cache[root]
@ -79,13 +56,11 @@ class CacheManager:
if root not in self.watched:
self._watch(root)
v = list(impl_fn(Path(root)))
v = list(impl_fn(root))
self.listdir_cache[root] = v
return v
def digest_file(
self, fpath: str, hash_algo: str, impl_fn: t.Callable[[str, str], str]
) -> str:
def digest_file(self, fpath, hash_algo, impl_fn):
with self.digest_lock:
try:
cache = self.digest_cache[hash_algo]
@ -107,17 +82,13 @@ class CacheManager:
cache[fpath] = v
return v
def _watch(self, root: str):
def _watch(self, root):
self.watched.add(root)
self.observer.schedule(_EventHandler(self, root), root, recursive=True)
def invalidate_root_cache(self, root: t.Union[Path, str]):
with self.listdir_lock:
self.listdir_cache.pop(str(root), None)
class _EventHandler:
def __init__(self, cache: CacheManager, root: str):
def __init__(self, cache, root):
self.cache = cache
self.root = root
@ -130,7 +101,8 @@ class _EventHandler:
return
# Lazy: just invalidate the whole cache
cache.invalidate_root_cache(self.root)
with cache.listdir_lock:
cache.listdir_cache.pop(self.root, None)
# Digests are more expensive: invalidate specific paths
paths = []
@ -145,3 +117,6 @@ class _EventHandler:
for _, subcache in cache.digest_cache.items():
for path in paths:
subcache.pop(path, None)
cache_manager = CacheManager()

@ -37,72 +37,30 @@ import argparse
import contextlib
import hashlib
import io
import itertools
import logging
import pathlib
import pkg_resources
import re
import sys
import textwrap
import typing as t
try:
# `importlib_resources` is required for Python versions below 3.12
# See more in the package docs: https://pypi.org/project/importlib-resources/
try:
from importlib_resources import files as import_files
except ImportError:
from importlib.resources import files as import_files
def get_resource_bytes(package: str, resource: str) -> bytes:
ref = import_files(package).joinpath(resource)
return ref.read_bytes()
except ImportError:
# The `pkg_resources` is deprecated in Python 3.12
import pkg_resources
def get_resource_bytes(package: str, resource: str) -> bytes:
return pkg_resources.resource_string(package, resource)
from pypiserver.backend import (
SimpleFileBackend,
CachingFileBackend,
Backend,
IBackend,
get_file_backend,
BackendProxy,
)
from distutils.util import strtobool as strtoint
# The `passlib` requirement is optional, so we need to verify its import here.
try:
from passlib.apache import HtpasswdFile
except ImportError:
HtpasswdFile = None
def legacy_strtoint(val: str) -> int:
"""Convert a string representation of truth to true (1) or false (0).
True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
'val' is anything else.
The "strtobool" function in distutils does a nice job at parsing strings,
but returns an integer. This just wraps it in a boolean call so that we
get a bool.
Borrowed from deprecated distutils.
"""
val = val.lower()
if val in ("y", "yes", "t", "true", "on", "1"):
return 1
elif val in ("n", "no", "f", "false", "off", "0"):
return 0
else:
raise ValueError("invalid truth value {!r}".format(val))
from pypiserver import core
strtobool: t.Callable[[str], bool] = lambda val: bool(legacy_strtoint(val))
# The "strtobool" function in distutils does a nice job at parsing strings,
# but returns an integer. This just wraps it in a boolean call so that we
# get a bool.
strtobool: t.Callable[[str], bool] = lambda val: bool(strtoint(val))
# Specify defaults here so that we can use them in tests &c. and not need
@ -112,8 +70,7 @@ class DEFAULTS:
AUTHENTICATE = ["update"]
FALLBACK_URL = "https://pypi.org/simple/"
HEALTH_ENDPOINT = "/health"
HASH_ALGO = "sha256"
HASH_ALGO = "md5"
INTERFACE = "0.0.0.0"
LOG_FRMT = "%(asctime)s|%(name)s|%(levelname)s|%(thread)d|%(message)s"
LOG_ERR_FRMT = "%(body)s: %(exception)s \n%(traceback)s"
@ -123,7 +80,6 @@ class DEFAULTS:
PACKAGE_DIRECTORIES = [pathlib.Path("~/packages").expanduser().resolve()]
PORT = 8080
SERVER_METHOD = "auto"
BACKEND = "auto"
def auth_arg(arg: str) -> t.List[str]:
@ -171,23 +127,12 @@ def hash_algo_arg(arg: str) -> t.Optional[str]:
)
def health_endpoint_arg(arg: str) -> str:
"""Verify the health_endpoint and raises ValueError if invalid."""
rule_regex = r"^/[a-z0-9/_-]+$"
if re.fullmatch(rule_regex, arg, re.I) is not None:
return arg
raise argparse.ArgumentTypeError(
"Invalid path for the health endpoint. Make sure that it contains only "
"alphanumeric characters, hyphens, forward slashes and underscores. "
f"In other words, make sure to match the following regex: {rule_regex}"
)
def html_file_arg(arg: t.Optional[str]) -> str:
"""Parse the provided HTML file and return its contents."""
if arg is None or arg == "pypiserver/welcome.html":
return get_resource_bytes(__name__, "welcome.html").decode("utf-8")
return pkg_resources.resource_string(__name__, "welcome.html").decode(
"utf-8"
)
with open(arg, "r", encoding="utf-8") as f:
msg = f.read()
return msg
@ -291,28 +236,6 @@ def add_common_args(parser: argparse.ArgumentParser) -> None:
"standard python library)"
),
)
parser.add_argument(
"--hash-algo",
default=DEFAULTS.HASH_ALGO,
type=hash_algo_arg,
help=(
"Any `hashlib` available algorithm to use for generating fragments "
"on package links. Can be disabled with one of (0, no, off, false)."
),
)
parser.add_argument(
"--backend",
default=DEFAULTS.BACKEND,
choices=("auto", "simple-dir", "cached-dir"),
dest="backend_arg",
help=(
"A backend implementation. Keep the default 'auto' to automatically"
" determine whether to activate caching or not"
),
)
parser.add_argument(
"--version",
action="version",
@ -331,6 +254,7 @@ def get_parser() -> argparse.ArgumentParser:
"directories starting with a dot. Multiple package directories "
"may be specified."
),
# formatter_class=argparse.RawTextHelpFormatter,
formatter_class=PreserveWhitespaceRawTextHelpFormatter,
epilog=(
"Visit https://github.com/pypiserver/pypiserver "
@ -430,15 +354,6 @@ def get_parser() -> argparse.ArgumentParser:
"index."
),
)
run_parser.add_argument(
"--health-endpoint",
default=DEFAULTS.HEALTH_ENDPOINT,
type=health_endpoint_arg,
help=(
"Configure a custom liveness endpoint. It always returns 200 Ok if "
"the service is up. Otherwise, it means that the service is not responsive."
),
)
run_parser.add_argument(
"--server",
metavar="METHOD",
@ -466,6 +381,15 @@ def get_parser() -> argparse.ArgumentParser:
action="store_true",
help="Allow overwriting existing package files during upload.",
)
run_parser.add_argument(
"--hash-algo",
default=DEFAULTS.HASH_ALGO,
type=hash_algo_arg,
help=(
"Any `hashlib` available algorithm to use for generating fragments "
"on package links. Can be disabled with one of (0, no, off, false)."
),
)
run_parser.add_argument(
"--welcome",
metavar="HTML_FILE",
@ -487,7 +411,6 @@ def get_parser() -> argparse.ArgumentParser:
help=(
'Add "Cache-Control: max-age=AGE" header to package downloads. '
"Pip 6+ requires this for caching."
"AGE is specified in seconds."
),
)
run_parser.add_argument(
@ -517,14 +440,6 @@ def get_parser() -> argparse.ArgumentParser:
"to '%%s' to see them all."
),
)
run_parser.add_argument(
"--mirror",
default=0,
action="count",
help=(
"Mirror packages to local disk"
),
)
update_parser = subparsers.add_parser(
"update",
@ -589,12 +504,9 @@ def get_parser() -> argparse.ArgumentParser:
TConf = t.TypeVar("TConf", bound="_ConfigCommon")
BackendFactory = t.Callable[["_ConfigCommon"], Backend]
class _ConfigCommon:
hash_algo: t.Optional[str] = None
def __init__(
self,
roots: t.List[pathlib.Path],
@ -602,8 +514,6 @@ class _ConfigCommon:
log_frmt: str,
log_file: t.Optional[str],
log_stream: t.Optional[t.IO],
hash_algo: t.Optional[str],
backend_arg: str,
) -> None:
"""Construct a RuntimeConfig."""
# Global arguments
@ -611,24 +521,18 @@ class _ConfigCommon:
self.log_file = log_file
self.log_stream = log_stream
self.log_frmt = log_frmt
self.roots = roots
self.hash_algo = hash_algo
self.backend_arg = backend_arg
# Derived properties are directly based on other properties and are not
# included in equality checks.
self._derived_properties: t.Tuple[str, ...] = (
"iter_packages",
"package_root",
"backend",
)
# The first package directory is considered the root. This is used
# for uploads.
self.package_root = self.roots[0]
self.backend = self.get_backend(backend_arg)
@classmethod
def from_namespace(
cls: t.Type[TConf], namespace: argparse.Namespace
@ -647,8 +551,6 @@ class _ConfigCommon:
log_stream=namespace.log_stream,
log_frmt=namespace.log_frmt,
roots=namespace.package_directory,
hash_algo=namespace.hash_algo,
backend_arg=namespace.backend_arg,
)
@property
@ -663,16 +565,13 @@ class _ConfigCommon:
# If we've specified 3 or more levels of verbosity, just return not set.
return levels.get(self.verbosity, logging.NOTSET)
def get_backend(self, arg: str) -> IBackend:
available_backends: t.Dict[str, BackendFactory] = {
"auto": get_file_backend,
"simple-dir": SimpleFileBackend,
"cached-dir": CachingFileBackend,
}
backend = available_backends[arg]
return BackendProxy(backend(self))
def iter_packages(self) -> t.Iterator[core.PkgFile]:
"""Iterate over packages in root directories."""
yield from (
itertools.chain.from_iterable(
core.listdir(str(r)) for r in self.roots
)
)
def with_updates(self: TConf, **kwargs: t.Any) -> TConf:
"""Create a new config with the specified updates.
@ -680,7 +579,7 @@ class _ConfigCommon:
The current config is used as a base. Any properties not specified in
keyword arguments will remain unchanged.
"""
return self.__class__(**{**dict(self), **kwargs})
return self.__class__(**{**dict(self), **kwargs}) # type: ignore
def __repr__(self) -> str:
"""A string representation indicating the class and its properties."""
@ -700,7 +599,7 @@ class _ConfigCommon:
return all(
getattr(other, k) == v
for k, v in self
if k not in self._derived_properties
if not k in self._derived_properties
)
def __iter__(self) -> t.Iterator[t.Tuple[str, t.Any]]:
@ -723,16 +622,15 @@ class RunConfig(_ConfigCommon):
password_file: t.Optional[str],
disable_fallback: bool,
fallback_url: str,
health_endpoint: str,
server_method: str,
overwrite: bool,
hash_algo: t.Optional[str],
welcome_msg: str,
cache_control: t.Optional[int],
mirror: bool,
log_req_frmt: str,
log_res_frmt: str,
log_err_frmt: str,
auther: t.Optional[t.Callable[[str, str], bool]] = None,
auther: t.Callable[[str, str], bool] = None,
**kwargs: t.Any,
) -> None:
"""Construct a RuntimeConfig."""
@ -743,18 +641,18 @@ class RunConfig(_ConfigCommon):
self.password_file = password_file
self.disable_fallback = disable_fallback
self.fallback_url = fallback_url
self.health_endpoint = health_endpoint
self.server_method = server_method
self.overwrite = overwrite
self.hash_algo = hash_algo
self.welcome_msg = welcome_msg
self.cache_control = cache_control
self.log_req_frmt = log_req_frmt
self.log_res_frmt = log_res_frmt
self.log_err_frmt = log_err_frmt
# Derived properties
self._derived_properties = self._derived_properties + ("auther",)
self.auther = self.get_auther(auther)
self.mirror = mirror
@classmethod
def kwargs_from_namespace(
@ -769,12 +667,11 @@ class RunConfig(_ConfigCommon):
"password_file": namespace.passwords,
"disable_fallback": namespace.disable_fallback,
"fallback_url": namespace.fallback_url,
"health_endpoint": namespace.health_endpoint,
"server_method": namespace.server,
"overwrite": namespace.overwrite,
"hash_algo": namespace.hash_algo,
"welcome_msg": namespace.welcome,
"cache_control": namespace.cache_control,
"mirror": namespace.mirror,
"log_req_frmt": namespace.log_req_frmt,
"log_res_frmt": namespace.log_res_frmt,
"log_err_frmt": namespace.log_err_frmt,
@ -855,9 +752,6 @@ class UpdateConfig(_ConfigCommon):
}
Configuration = t.Union[RunConfig, UpdateConfig]
class Config:
"""Config constructor for building a config from args."""
@ -874,8 +768,8 @@ class Config:
@classmethod
def from_args(
cls, args: t.Optional[t.Sequence[str]] = None
) -> Configuration:
cls, args: t.Sequence[str] = None
) -> t.Union[RunConfig, UpdateConfig]:
"""Construct a Config from the passed args or sys.argv."""
# If pulling args from sys.argv (commandline arguments), argv[0] will
# be the program name, (i.e. pypi-server), so we don't need to
@ -952,7 +846,7 @@ class Config:
def _adjust_old_args(args: t.Sequence[str]) -> t.List[str]:
"""Adjust args for backwards compatibility.
Should only be called once args have been verified to be unparsable.
Should only be called once args have been verified to be unparseable.
"""
# Backwards compatibility hack: for most of pypiserver's life, "run"
# and "update" were not separate subcommands. The `-U` flag being

@ -1,69 +1,154 @@
#! /usr/bin/env python3
#! /usr/bin/env python
"""minimal PyPI like server for use with pip/easy_install"""
import hashlib
import logging
import mimetypes
import os
import re
import typing as t
from urllib.parse import quote
from pypiserver.pkg_helpers import normalize_pkgname, parse_version
log = logging.getLogger(__name__)
mimetypes.add_type("application/octet-stream", ".egg")
mimetypes.add_type("application/octet-stream", ".whl")
mimetypes.add_type("text/plain", ".asc")
def get_bad_url_redirect_path(request, project):
"""Get the path for a bad root url."""
uri = request.custom_fullpath
if uri.endswith("/"):
uri = uri[:-1]
uri = uri.rsplit("/", 1)[0]
project = quote(project)
uri += f"/simple/{project}/"
return uri
# ### Next 2 functions adapted from :mod:`distribute.pkg_resources`.
#
component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.I | re.VERBOSE)
replace = {"pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@"}.get
def _parse_version_parts(s):
for part in component_re.split(s):
part = replace(part, part)
if part in ["", "."]:
continue
if part[:1] in "0123456789":
yield part.zfill(8) # pad for numeric comparison
else:
yield "*" + part
yield "*final" # ensure that alpha/beta/candidate are before final
def parse_version(s):
parts = []
for part in _parse_version_parts(s.lower()):
if part.startswith("*"):
# remove trailing zeros from each series of numeric parts
while parts and parts[-1] == "00000000":
parts.pop()
parts.append(part)
return tuple(parts)
#
#### -- End of distribute's code.
_archive_suffix_rx = re.compile(
r"(\.zip|\.tar\.gz|\.tgz|\.tar\.bz2|-py[23]\.\d-.*|"
r"\.win-amd64-py[23]\.\d\..*|\.win32-py[23]\.\d\..*|\.egg)$",
re.I,
)
wheel_file_re = re.compile(
r"""^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
\.whl|\.dist-info)$""",
re.VERBOSE,
)
_pkgname_re = re.compile(r"-\d+[a-z_.!+]", re.I)
_pkgname_parts_re = re.compile(
r"[\.\-](?=cp\d|py\d|macosx|linux|sunos|solaris|irix|aix|cygwin|win)", re.I
)
def _guess_pkgname_and_version_wheel(basename):
m = wheel_file_re.match(basename)
if not m:
return None, None
name = m.group("name")
ver = m.group("ver")
build = m.group("build")
if build:
return name, ver + "-" + build
else:
return name, ver
def guess_pkgname_and_version(path):
path = os.path.basename(path)
if path.endswith(".asc"):
path = path.rstrip(".asc")
if path.endswith(".whl"):
return _guess_pkgname_and_version_wheel(path)
if not _archive_suffix_rx.search(path):
return
path = _archive_suffix_rx.sub("", path)
if "-" not in path:
pkgname, version = path, ""
elif path.count("-") == 1:
pkgname, version = path.split("-", 1)
elif "." not in path:
pkgname, version = path.rsplit("-", 1)
else:
pkgname = _pkgname_re.split(path)[0]
ver_spec = path[len(pkgname) + 1 :]
parts = _pkgname_parts_re.split(ver_spec)
version = parts[0]
return pkgname, version
def normalize_pkgname(name):
"""Perform PEP 503 normalization"""
return re.sub(r"[-_.]+", "-", name).lower()
def normalize_pkgname_for_url(name):
"""Perform PEP 503 normalization and ensure the value is safe for URLs."""
return quote(re.sub(r"[-_.]+", "-", name).lower())
def is_allowed_path(path_part):
p = path_part.replace("\\", "/")
return not (p.startswith(".") or "/." in p)
class PkgFile:
__slots__ = [
"pkgname", # The projects/package name with possible capitalization
"version", # The package version as a string
"fn", # The full file path
"root", # An optional root directory of the file
"relfn", # The file path relative to the root
"replaces", # The previous version of the package (used by manage.py)
"pkgname_norm", # The PEP503 normalized project name
"digest", # The file digest in the form of <algo>=<hash>
"relfn_unix", # The relative file path in unix notation
"parsed_version", # The package version as a tuple of parts
"digester", # a function that calculates the digest for the package
"fn",
"root",
"_fname_and_hash",
"relfn",
"relfn_unix",
"pkgname_norm",
"pkgname",
"version",
"parsed_version",
"replaces",
]
digest: t.Optional[str]
digester: t.Optional[t.Callable[["PkgFile"], t.Optional[str]]]
parsed_version: tuple
relfn_unix: t.Optional[str]
def __init__(
self,
pkgname: str,
version: str,
fn: t.Optional[str] = None,
root: t.Optional[str] = None,
relfn: t.Optional[str] = None,
replaces: t.Optional["PkgFile"] = None,
self, pkgname, version, fn=None, root=None, relfn=None, replaces=None
):
self.pkgname = pkgname
self.pkgname_norm = normalize_pkgname(pkgname)
self.version = version
self.parsed_version: tuple = parse_version(version)
self.parsed_version = parse_version(version)
self.fn = fn
self.root = root
self.relfn = relfn
self.relfn_unix = None if relfn is None else relfn.replace("\\", "/")
self.replaces = replaces
self.digest = None
self.digester = None
def __repr__(self) -> str:
def __repr__(self):
return "{}({})".format(
self.__class__.__name__,
", ".join(
@ -74,9 +159,109 @@ class PkgFile:
),
)
@property
def fname_and_hash(self) -> str:
if self.digest is None and self.digester is not None:
self.digest = self.digester(self)
hashpart = f"#{self.digest}" if self.digest else ""
return self.relfn_unix + hashpart # type: ignore
def fname_and_hash(self, hash_algo):
if not hasattr(self, "_fname_and_hash"):
if hash_algo:
self._fname_and_hash = (
f"{self.relfn_unix}#{hash_algo}="
f"{digest_file(self.fn, hash_algo)}"
)
else:
self._fname_and_hash = self.relfn_unix
return self._fname_and_hash
def _listdir(root: str) -> t.Iterable[PkgFile]:
root = os.path.abspath(root)
for dirpath, dirnames, filenames in os.walk(root):
dirnames[:] = [x for x in dirnames if is_allowed_path(x)]
for x in filenames:
fn = os.path.join(root, dirpath, x)
if not is_allowed_path(x) or not os.path.isfile(fn):
continue
res = guess_pkgname_and_version(x)
if not res:
# #Seems the current file isn't a proper package
continue
pkgname, version = res
if pkgname:
yield PkgFile(
pkgname=pkgname,
version=version,
fn=fn,
root=root,
relfn=fn[len(root) + 1 :],
)
def find_packages(pkgs, prefix=""):
prefix = normalize_pkgname(prefix)
for x in pkgs:
if prefix and x.pkgname_norm != prefix:
continue
yield x
def get_prefixes(pkgs):
normalized_pkgnames = set()
for x in pkgs:
if x.pkgname:
normalized_pkgnames.add(x.pkgname_norm)
return normalized_pkgnames
def exists(root, filename):
assert "/" not in filename
dest_fn = os.path.join(root, filename)
return os.path.exists(dest_fn)
def store(root, filename, save_method):
assert "/" not in filename
dest_fn = os.path.join(root, filename)
save_method(dest_fn, overwrite=True) # Overwite check earlier.
def get_bad_url_redirect_path(request, prefix):
"""Get the path for a bad root url."""
p = request.custom_fullpath
if p.endswith("/"):
p = p[:-1]
p = p.rsplit("/", 1)[0]
prefix = quote(prefix)
p += "/simple/{}/".format(prefix)
return p
def _digest_file(fpath, hash_algo):
"""
Reads and digests a file according to specified hashing-algorith.
:param str sha256: any algo contained in :mod:`hashlib`
:return: <hash_algo>=<hex_digest>
From http://stackoverflow.com/a/21565932/548792
"""
blocksize = 2 ** 16
digester = hashlib.new(hash_algo)
with open(fpath, "rb") as f:
for block in iter(lambda: f.read(blocksize), b""):
digester.update(block)
return digester.hexdigest()
try:
from .cache import cache_manager
def listdir(root: str) -> t.Iterable[PkgFile]:
# root must be absolute path
return cache_manager.listdir(root, _listdir)
def digest_file(fpath, hash_algo):
# fpath must be absolute path
return cache_manager.digest_file(fpath, hash_algo, _digest_file)
except ImportError:
listdir = _listdir
digest_file = _digest_file

@ -5,17 +5,14 @@ from __future__ import absolute_import, print_function, unicode_literals
import itertools
import os
import sys
from packaging.version import parse as packaging_parse
from pathlib import Path
from distutils.version import LooseVersion
from subprocess import call
from xmlrpc.client import Server
import pip
from .backend import listdir
from .core import PkgFile
from .pkg_helpers import normalize_pkgname, parse_version
from . import core
from xmlrpc.client import Server
def make_pypi_client(url):
@ -44,7 +41,7 @@ def filter_latest_pkgs(pkgs):
pkgname2latest = {}
for x in pkgs:
pkgname = normalize_pkgname(x.pkgname)
pkgname = core.normalize_pkgname(x.pkgname)
if pkgname not in pkgname2latest:
pkgname2latest[pkgname] = x
@ -56,9 +53,9 @@ def filter_latest_pkgs(pkgs):
def build_releases(pkg, versions):
for x in versions:
parsed_version = parse_version(x)
parsed_version = core.parse_version(x)
if parsed_version > pkg.parsed_version:
yield PkgFile(pkgname=pkg.pkgname, version=x, replaces=pkg)
yield core.PkgFile(pkgname=pkg.pkgname, version=x, replaces=pkg)
def find_updates(pkgset, stable_only=True):
@ -101,8 +98,7 @@ def find_updates(pkgset, stable_only=True):
if no_releases:
sys.stdout.write(
f"no releases found on pypi for"
f" {', '.join(sorted(no_releases))}\n\n"
f"no releases found on pypi for {', '.join(sorted(no_releases))}\n\n"
)
return need_update
@ -113,14 +109,12 @@ class PipCmd:
@staticmethod
def update_root(pip_version):
"""Yield an appropriate root command depending on pip version.
Use `pip install` for `pip` 9 or lower, and `pip download` otherwise.
"""
legacy_pip = packaging_parse(pip_version).major < 10
pip_command = "install" if legacy_pip else "download"
for part in ("pip", "-q", pip_command):
"""Yield an appropriate root command depending on pip version."""
# legacy_pip = StrictVersion(pip_version) < StrictVersion('10.0')
legacy_pip = LooseVersion(pip_version) < LooseVersion("10.0")
for part in ("pip", "-q"):
yield part
yield "install" if legacy_pip else "download"
@staticmethod
def update(
@ -141,7 +135,8 @@ class PipCmd:
def update_package(pkg, destdir, dry_run=False):
"""Print and optionally execute a package update."""
print(
f"# update {pkg.pkgname} from {pkg.replaces.version} to {pkg.version}"
"# update {0.pkgname} from {0.replaces.version} to "
"{0.version}".format(pkg)
)
cmd = tuple(
@ -153,7 +148,7 @@ def update_package(pkg, destdir, dry_run=False):
)
)
print(" ".join(cmd), end="\n\n")
print("{}\n".format(" ".join(cmd)))
if not dry_run:
call(cmd)
@ -176,9 +171,7 @@ def update(pkgset, destdir=None, dry_run=False, stable_only=True):
def update_all_packages(
roots, destdir=None, dry_run=False, stable_only=True, ignorelist=None
):
all_packages = itertools.chain.from_iterable(
listdir(Path(r)) for r in roots
)
all_packages = itertools.chain(*[core.listdir(r) for r in roots])
skip_packages = set(ignorelist or ())

@ -1,91 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import logging
from collections import OrderedDict
from pypiserver.bottle import HTTPError, redirect
from pypiserver.config import RunConfig
log = logging.getLogger(__name__)
try:
import requests
from bs4 import BeautifulSoup
import_ok = True
except ImportError:
import_ok = False
logging.error("mirror_cache import dependencies error")
class CacheElement:
def __init__(self, project: str):
self.project = project
self.html = ""
self.cache = dict()
def add(self, href: str):
targz = href.split("/")[-1]
pkg_name = targz.split("#")[0]
self.cache[f"{self.project}/{pkg_name}"] = href
return f"/packages/{self.project}/{targz}"
class MirrorCache:
cache: OrderedDict[str, CacheElement] = dict()
cache_limit = 10
@classmethod
def add(cls, project: str, config: RunConfig) -> str:
if not import_ok:
return redirect(f"{config.fallback_url.rstrip('/')}/{project}/")
if project in cls.cache:
log.info(f"mirror_cache serve html from cache {project}")
return cls.cache[project].html
element = CacheElement(project=project)
resp = requests.get(f"{config.fallback_url.rstrip('/')}/{project}/")
soup = BeautifulSoup(resp.content, "html.parser")
links = soup.find_all("a")
for link in links:
# new href with mapping to old href for later
new_href = element.add(href=link["href"])
# create new link
new_link = soup.new_tag("a")
new_link.string = link.text.strip()
new_link["href"] = new_href
link.replace_with(new_link)
element.html = str(soup)
cls.cache[project] = element
log.info(f"mirror_cache add project '{project}' to cache")
# purge
if len(cls.cache) > cls.cache_limit:
item = cls.cache.popitem(last=False)
log.info(f"mirror_cache limit '{cls.cache_limit}' exceeded, purged last item - {item}")
return element.html
@classmethod
def has_project(cls, filename):
project = filename.split("/")[0]
return project in cls.cache
@classmethod
def get_static_file(cls, filename, config: RunConfig):
if not import_ok:
return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
project = filename.split("/")[0]
element = cls.cache[project]
if filename in element.cache:
href = element.cache[filename]
resp = requests.get(href)
cls.add_to_cache(filename=filename, resp=resp, config=config)
return resp
log.info(f"mirror_cache not found in cache {filename} ")
return HTTPError(404, f"Not Found ({filename} does not exist)\n\n")
@classmethod
def add_to_cache(cls, filename: str, resp: requests.Response, config: RunConfig):
project = filename.split("/")[0]
os.makedirs(os.path.join(config.package_root, project), exist_ok=True)
log.info(f"mirror_cache add file '{filename}' to cache")
with open(f"{config.package_root}/{filename}", "wb+") as f:
f.write(resp.content)

@ -1,112 +0,0 @@
import os
import re
import typing as t
from pathlib import PurePath, Path
from urllib.parse import quote
def normalize_pkgname(name: str) -> str:
"""Perform PEP 503 normalization"""
return re.sub(r"[-_.]+", "-", name).lower()
def normalize_pkgname_for_url(name: str) -> str:
"""Perform PEP 503 normalization and ensure the value is safe for URLs."""
return quote(normalize_pkgname(name))
# ### Next 2 functions adapted from :mod:`distribute.pkg_resources`.
#
component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.I | re.VERBOSE)
replace = {"pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@"}.get
def _parse_version_parts(s):
for part in component_re.split(s):
part = replace(part, part)
if part in ["", "."]:
continue
if part[:1] in "0123456789":
yield part.zfill(8) # pad for numeric comparison
else:
yield "*" + part
yield "*final" # ensure that alpha/beta/candidate are before final
def parse_version(s: str) -> tuple:
parts = []
for part in _parse_version_parts(s.lower()):
if part.startswith("*"):
# remove trailing zeros from each series of numeric parts
while parts and parts[-1] == "00000000":
parts.pop()
parts.append(part)
return tuple(parts)
#
# ### -- End of distribute's code.
def is_listed_path(path_part: t.Union[PurePath, str]) -> bool:
if isinstance(path_part, str):
path_part = PurePath(path_part)
return not any(part.startswith(".") for part in path_part.parts)
_archive_suffix_rx = re.compile(
r"(\.zip|\.tar\.gz|\.tgz|\.tar\.bz2|\.tar\.xz|-py[23]\.\d-.*|"
r"\.win-amd64-py[23]\.\d\..*|\.win32-py[23]\.\d\..*|\.egg)$",
re.I,
)
wheel_file_re = re.compile(
r"""^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
\.whl|\.dist-info)$""",
re.VERBOSE,
)
_pkgname_re = re.compile(r"-\d+[a-z_.!+]", re.I)
_pkgname_parts_re = re.compile(
r"[\.\-](?=cp\d|py\d|macosx|linux|sunos|solaris|irix|aix|cygwin|win)", re.I
)
def _guess_pkgname_and_version_wheel(
basename: str,
) -> t.Optional[t.Tuple[str, str]]:
m = wheel_file_re.match(basename)
if not m:
return None
name = m.group("name")
ver = m.group("ver")
build = m.group("build")
if build:
return name, ver + "-" + build
else:
return name, ver
def guess_pkgname_and_version(path: str) -> t.Optional[t.Tuple[str, str]]:
path = os.path.basename(path)
if path.endswith(".asc"):
path = path.rstrip(".asc")
if path.endswith(".whl"):
return _guess_pkgname_and_version_wheel(path)
if not _archive_suffix_rx.search(path):
return None
path = _archive_suffix_rx.sub("", path)
if "-" not in path:
pkgname, version = path, ""
elif path.count("-") == 1:
pkgname, version = path.split("-", 1)
elif "." not in path:
pkgname, version = path.rsplit("-", 1)
else:
pkgname = _pkgname_re.split(path)[0]
ver_spec = path[len(pkgname) + 1 :]
parts = _pkgname_parts_re.split(ver_spec)
version = parts[0]
return pkgname, version

@ -1,42 +0,0 @@
""" NOT YET IMPLEMENTED
Plugins are callable setuptools entrypoints that are invoked at startup that
a developer may use to extend the behaviour of pypiserver. A plugin for example
may add an additional Backend to the system. A plugin will be called
with the following keyword arguments
* app: the Bottle App object
* add_argument: A callable for registering command line arguments for your
plugin using the argparse cli library
* backends: A Dict[str, callable] object that you may register a backend to.
The key is the identifier for the backend in the `--backend` command line
argument.
The callable must take a single argument `config` as a Configuration object
and return a Backend instance. It may be the class constructor or a factory
function to construct a Backend object
In the future, the plugin callable may be called with additional keyword
arguments, so a plugin should accept a **kwargs variadic keyword argument.
"""
from pypiserver.backend import SimpleFileBackend, CachingFileBackend
from pypiserver import get_file_backend
DEFAULT_PACKAGE_DIRECTORIES = ["~/packages"]
# register this as a setuptools entrypoint under the 'pypiserver.plugin' key
def my_plugin(add_argument, backends, **_):
add_argument(
"package_directory",
default=DEFAULT_PACKAGE_DIRECTORIES,
nargs="*",
help="The directory from which to serve packages.",
)
backends.update(
{
"auto": get_file_backend,
"simple-dir": SimpleFileBackend,
"cached-dir": CachingFileBackend,
}
)

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

@ -4,8 +4,17 @@
# pip install -r requirements/dev.pip
-r exe.pip
-r test.pip
black
docopt # For `/bin/bumpver.py`.
gevent>=1.1b4; python_version >= '3'
mypy; implementation_name == 'cpython'
pip>=7
passlib>=1.6
pytest>=2.3
pytest-cov
setuptools
setuptools-git>=0.3
tox
twine
webtest
wheel>=0.25.0

@ -1,2 +0,0 @@
beautifulsoup4==4.12.3
requests==2.31.0

@ -1,15 +0,0 @@
gevent>=1.1b4; python_version >= '3'
httpx
pip
passlib>=1.6
pytest>=6.2.2
pytest-cov
setuptools>=40.0,<70.0.0
tox
twine
webtest
wheel>=0.25.0
build>=1.2.0; python_version >= '3.8'
mdformat-gfm
mdformat-frontmatter
mdformat-footnote

@ -1,3 +0,0 @@
# Just the absolutely necessary extra requirements for
# running tests
-r test-requirements.txt

@ -27,6 +27,3 @@ warn_unused_ignores = True
[mypy-tests.*]
disallow_untyped_decorators = False
[mypy-test_docker.*]
disallow_untyped_decorators = False

@ -9,21 +9,12 @@ tests_require = [
"pytest>=2.3",
"tox",
"twine",
"pip>=7",
"passlib>=1.6",
"webtest",
"build>=1.2.0;python_version>='3.8'",
]
setup_requires = [
"setuptools",
"setuptools-git>=0.3",
"wheel>=0.25.0",
]
install_requires = [
"pip>=7",
"packaging>=23.2",
"importlib_resources;python_version>'3.8' and python_version<'3.12'",
]
setup_requires = ["setuptools", "setuptools-git >= 0.3", "wheel >= 0.25.0"]
def read_file(rel_path: str):
@ -49,13 +40,11 @@ def get_version():
setup(
name="pypiserver",
description="A minimal PyPI server for use with pip/easy_install.",
long_description=read_file("README.md"),
long_description_content_type="text/markdown",
long_description=read_file("README.rst"),
version=get_version(),
packages=["pypiserver"],
package_data={"pypiserver": ["welcome.html"]},
python_requires=">=3.6",
install_requires=install_requires,
setup_requires=setup_requires,
extras_require={"passlib": ["passlib>=1.6"], "cache": ["watchdog"]},
tests_require=tests_require,
@ -80,8 +69,6 @@ setup(
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
"Topic :: Software Development :: Build Tools",

16
tests/centodeps-setup.py Normal file

@ -0,0 +1,16 @@
## A test-distribution to check if
# bottle supports uploading 100's of packages,
# see: https://github.com/pypiserver/pypiserver/issues/82
#
# Has been run once `pip wheel .`, just to generate:
# ./wheelhouse/centodeps-0.0.0-cp34-none-win_amd64.whl
#
from setuptools import setup
setup(
name="centodeps",
install_requires=["a==1.0"] * 200,
options={
"bdist_wheel": {"universal": True},
},
)

@ -4,21 +4,34 @@
import logging
import os
import pathlib
import xmlrpc.client as xmlrpclib
from html import unescape
try: # python 3
from html.parser import HTMLParser
from html import unescape
except ImportError:
from HTMLParser import HTMLParser
unescape = HTMLParser().unescape
try:
import xmlrpc.client as xmlrpclib
except ImportError:
import xmlrpclib # legacy Python
# Third party imports
import pytest
import webtest
# Local Imports
from tests.test_pkg_helpers import files, invalid_files
from pypiserver import __main__, bottle, core, Bottle
from pypiserver.backend import CachingFileBackend, SimpleFileBackend
from pypiserver import __main__, bottle
import tests.test_core as test_core
# Enable logging to detect any problems with it
##
__main__.init_logging()
@ -30,14 +43,12 @@ def app(tmpdir):
roots=[pathlib.Path(tmpdir.strpath)],
authenticate=[],
password_file=".",
backend_arg="simple-dir",
)
@pytest.fixture
def testapp(app):
"""Return a webtest TestApp initiated with pypiserver app"""
bottle.debug(True)
return webtest.TestApp(app)
@ -100,18 +111,7 @@ def welcome_file_all_vars(request, root):
return wfile
@pytest.fixture
def add_file_to_root(app):
def file_adder(root, filename, content=""):
root.join(filename).write(content)
backend = app.config.backend
if isinstance(backend, CachingFileBackend):
backend.cache_manager.invalidate_root_cache(root)
return file_adder
def test_root_count(root, testapp, add_file_to_root):
def test_root_count(root, testapp):
"""Test that the welcome page count updates with added packages
:param root: root temporary directory fixture
@ -119,7 +119,7 @@ def test_root_count(root, testapp, add_file_to_root):
"""
resp = testapp.get("/")
resp.mustcontain("PyPI compatible package index serving 0 packages")
add_file_to_root(root, "Twisted-11.0.0.tar.bz2")
root.join("Twisted-11.0.0.tar.bz2").write("")
resp = testapp.get("/")
resp.mustcontain("PyPI compatible package index serving 1 packages")
@ -188,29 +188,6 @@ def test_packages_empty(testapp):
assert len(resp.html("a")) == 0
def test_health_default_endpoint(testapp):
resp = testapp.get("/health")
assert resp.status_int == 200
assert "Ok" in resp
def test_health_customized_endpoint(root):
from pypiserver import app
_app = app(root=root.strpath, health_endpoint="/healthz")
testapp = webtest.TestApp(_app)
resp = testapp.get("/healthz")
assert resp.status_int == 200
assert "Ok" in resp
def test_health_invalid_customized_endpoint(root):
from pypiserver import app
with pytest.raises(RuntimeError, match="overlaps with existing routes"):
app(root=root.strpath, health_endpoint="/simple")
def test_favicon(testapp):
testapp.get("/favicon.ico", status=404)
@ -353,19 +330,16 @@ def test_nonroot_root_with_x_forwarded_host_without_trailing_slash(testapp):
resp.mustcontain("""<a href="/priv/packages/">here</a>""")
def test_nonroot_simple_index(root, testpriv, add_file_to_root):
add_file_to_root(root, "foobar-1.0.zip", "123")
def test_nonroot_simple_index(root, testpriv):
root.join("foobar-1.0.zip").write("")
resp = testpriv.get("/priv/simple/foobar/")
links = resp.html("a")
assert len(links) == 1
assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#")
def test_nonroot_simple_index_with_x_forwarded_host(
root, testapp, add_file_to_root
):
add_file_to_root(root, "foobar-1.0.zip", "123")
def test_nonroot_simple_index_with_x_forwarded_host(root, testapp):
root.join("foobar-1.0.zip").write("")
resp = testapp.get(
"/simple/foobar/", headers={"X-Forwarded-Host": "forwarded.ed/priv/"}
)
@ -374,25 +348,22 @@ def test_nonroot_simple_index_with_x_forwarded_host(
assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#")
def test_nonroot_simple_packages(root, testpriv, add_file_to_root):
add_file_to_root(root, "foobar-1.0.zip", "123")
def test_nonroot_simple_packages(root, testpriv):
root.join("foobar-1.0.zip").write("123")
resp = testpriv.get("/priv/packages/")
links = resp.html("a")
assert len(links) == 1
assert "/priv/packages/foobar-1.0.zip#" in links[0]["href"]
assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#")
def test_nonroot_simple_packages_with_x_forwarded_host(
root, testapp, add_file_to_root
):
add_file_to_root(root, "foobar-1.0.zip", "123")
def test_nonroot_simple_packages_with_x_forwarded_host(root, testapp):
root.join("foobar-1.0.zip").write("123")
resp = testapp.get(
"/packages/", headers={"X-Forwarded-Host": "forwarded/priv/"}
)
links = resp.html("a")
assert len(links) == 1
assert "/priv/packages/foobar-1.0.zip#" in links[0]["href"]
assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#")
def test_root_no_relative_paths(testpriv):
@ -442,42 +413,6 @@ def test_simple_index_list_name_with_underscore_no_egg(root, testapp):
assert hrefs == {"foo-bar/"}
def test_json_info(root, testapp):
root.join("foobar-1.0.zip").write("")
root.join("foobar-1.1.zip").write("")
root.join("foobar-1.1-linux.zip").write("")
root.join("foobarX-1.1.zip").write("")
resp = testapp.get("/foobar/json")
assert "info" in resp.json
assert "releases" in resp.json
assert len(resp.json["info"]) == 1
assert len(resp.json["releases"]) == 2
assert len(resp.json["releases"]["1.0"]) == 1
assert len(resp.json["releases"]["1.1"]) == 2
def test_json_info_package_not_existing(root, testapp):
resp = testapp.get("/foobar/json", status=404)
@pytest.mark.parametrize(
"package,normalized",
[
("FooBar", "foobar"),
("Foo.Bar", "foo-bar"),
("foo_bar", "foo-bar"),
("Foo-Bar", "foo-bar"),
("foo--_.bar", "foo-bar"),
],
)
def test_json_info_normalized_name_redirect(testapp, package, normalized):
resp = testapp.get("/{0}/json".format(package))
assert resp.status_code >= 300
assert resp.status_code < 400
assert resp.location.endswith("/{0}/json".format(normalized))
def test_no_cache_control_set(root, testapp):
assert not testapp.app._pypiserver_config.cache_control
root.join("foo_bar-1.0.tar.gz").write("")
@ -509,7 +444,7 @@ def test_upload_badAction(testapp):
@pytest.mark.parametrize(
"package", [f[0] for f in files if f[1] and "/" not in f[0]]
"package", [f[0] for f in test_core.files if f[1] and "/" not in f[0]]
)
def test_upload(package, root, testapp):
resp = testapp.post(
@ -523,23 +458,8 @@ def test_upload(package, root, testapp):
assert uploaded_pkgs[0].lower() == package.lower()
def test_upload_conflict_on_existing(root, testapp):
package = "foo_bar-1.0.tar.gz"
root.join("foo_bar-1.0.tar.gz").write("")
resp = testapp.post(
"/",
params={":action": "file_upload"},
upload_files=[("content", package, b"")],
status=409,
)
assert resp.status_int == 409
assert "Package 'foo_bar-1.0.tar.gz' already exists!" in unescape(resp.text)
@pytest.mark.parametrize(
"package", [f[0] for f in files if f[1] and "/" not in f[0]]
"package", [f[0] for f in test_core.files if f[1] and "/" not in f[0]]
)
def test_upload_with_signature(package, root, testapp):
resp = testapp.post(
@ -557,7 +477,9 @@ def test_upload_with_signature(package, root, testapp):
assert f"{package.lower()}.asc" in uploaded_pkgs
@pytest.mark.parametrize("package", invalid_files)
@pytest.mark.parametrize(
"package", [f[0] for f in test_core.files if f[1] is None]
)
def test_upload_badFilename(package, root, testapp):
resp = testapp.post(
"/",
@ -604,7 +526,7 @@ def test_search(root, testapp, search_xml, pkgs, matches):
expected name and version matches for a search for the "test"
package as specified by the search_xml fixture.
:param root: root temporary directory fixture; used as packages dir
:param root: root temporry directory fixture; used as packages dir
for testapp
:param testapp: webtest TestApp
:param str search_xml: XML string roughly equivalent to a pip search

@ -1,42 +0,0 @@
from pathlib import Path
import pytest
from pypiserver.backend import listdir
def create_path(root: Path, path: Path):
if path.is_absolute():
raise ValueError(
"Only test using relative paths"
" to prevent leaking outside test environment"
)
fullpath = root / path
if not fullpath.parent.exists():
fullpath.parent.mkdir(parents=True)
fullpath.touch()
valid_paths = ["direct-in-root.zip", "some/nested/pkg.zip"]
@pytest.mark.parametrize("path_name", valid_paths)
def test_listdir_generates_pkgfile_for_valid_package(tmp_path, path_name):
path = Path(path_name)
create_path(tmp_path, path)
assert len(list(listdir(tmp_path))) == 1
invalid_paths = [
".hidden-pkg.zip",
".hidden/dir/pkg.zip",
"in/between/.hidden/pkg.zip",
"invalid-wheel.whl",
]
@pytest.mark.parametrize("path_name", invalid_paths)
def test_listdir_doesnt_generate_pkgfile_for_invalid_file(tmp_path, path_name):
path = Path(path_name)
create_path(tmp_path, path)
assert not list(listdir(tmp_path))

@ -8,13 +8,12 @@ import sys
import pytest
from pypiserver.backend import SimpleFileBackend, BackendProxy
from pypiserver.config import DEFAULTS, Config, RunConfig, UpdateConfig
FILE_DIR = pathlib.Path(__file__).parent.resolve()
# Username and password stored in the htpasswd.a.a test file.
HTPASS_TEST_FILE = str(FILE_DIR / "../fixtures/htpasswd.a.a")
HTPASS_TEST_FILE = str(FILE_DIR / "htpasswd.a.a")
HTPASS_TEST_USER = "a"
HTPASS_TEST_PASS = "a"
@ -369,21 +368,6 @@ _CONFIG_TEST_PARAMS: t.Tuple[ConfigTestCase, ...] = (
exp_config_type=RunConfig,
exp_config_values={"fallback_url": "foobar.com"},
),
# health-endpoint
ConfigTestCase(
case="Run: health-endpoint unspecified",
args=["run"],
legacy_args=[],
exp_config_type=RunConfig,
exp_config_values={"health_endpoint": DEFAULTS.HEALTH_ENDPOINT},
),
ConfigTestCase(
case="Run: health-endpoint specified",
args=["run", "--health-endpoint", "/healthz"],
legacy_args=["--health-endpoint", "/healthz"],
exp_config_type=RunConfig,
exp_config_values={"health_endpoint": "/healthz"},
),
# server method
ConfigTestCase(
case="Run: server method unspecified",
@ -546,39 +530,10 @@ _CONFIG_TEST_PARAMS: t.Tuple[ConfigTestCase, ...] = (
exp_config_type=RunConfig,
exp_config_values={"log_err_frmt": "foo"},
),
# backend
ConfigTestCase(
"Run: backend unspecified",
args=["run"],
legacy_args=[],
exp_config_type=RunConfig,
exp_config_values={
"backend_arg": "auto",
"_test": (
lambda conf: (
isinstance(conf.backend, BackendProxy)
and isinstance(conf.backend.backend, SimpleFileBackend)
)
),
},
),
ConfigTestCase(
"Run: simple backend specified",
args=["run", "--backend", "simple-dir"],
legacy_args=["--backend", "simple-dir"],
exp_config_type=RunConfig,
exp_config_values={
"_test": (
lambda conf: (
isinstance(conf.backend.backend, SimpleFileBackend)
)
),
},
),
# ******************************************************************
# Update subcommand args
# ******************************************************************
# execute
# exeucte
ConfigTestCase(
case="Update: execute not specified",
args=["update"],
@ -698,14 +653,6 @@ _CONFIG_ERROR_CASES = (
)
for val in ("true", "foo", "1", "md6")
),
*(
ConfigErrorCase(
case=f"Invalid health endpoint: {val}",
args=["run", "--health-endpoint", val],
exp_txt="Invalid path for the health endpoint",
)
for val in ("/", "health", "/health!", "/:health", "/health?check=True")
),
)
# pylint: disable=unsubscriptable-object
CONFIG_ERROR_PARAMS = (i[1:] for i in _CONFIG_ERROR_CASES)
@ -748,7 +695,7 @@ def test_config(
@pytest.mark.parametrize(
"args, exp_txt",
CONFIG_ERROR_PARAMS,
ids=CONFIG_ERROR_IDS,
ids=CONFIG_TEST_IDS,
)
def test_config_error(
args: t.List[str],
@ -758,7 +705,7 @@ def test_config_error(
"""Validate error cases."""
with pytest.raises(SystemExit):
Config.from_args(args)
# Unfortunately the error text is printed before the SystemExit is
# Unfortunatley the error text is printed before the SystemExit is
# raised, rather than being raised _with_ the systemexit, so we
# need to capture stderr and check it for our expected text, if
# any was specified in the test case.

@ -6,10 +6,7 @@ import os
import pytest
from pypiserver import __main__, core, backend
from pypiserver.pkg_helpers import (
normalize_pkgname_for_url,
)
from pypiserver import __main__, core
from tests.doubles import Namespace
@ -18,9 +15,98 @@ from tests.doubles import Namespace
__main__.init_logging()
def test_listdir_bad_name(tmp_path):
tmp_path.joinpath("foo.whl").touch()
res = list(backend.listdir(tmp_path))
files = [
("pytz-2012b.tar.bz2", "pytz", "2012b"),
("pytz-2012b.tgz", "pytz", "2012b"),
("pytz-2012b.ZIP", "pytz", "2012b"),
("pytz-2012a.zip", "pytz", "2012a"),
("gevent-1.0b1.win32-py2.6.exe", "gevent", "1.0b1"),
("gevent-1.0b1.win32-py2.7.msi", "gevent", "1.0b1"),
("greenlet-0.3.4-py3.1-win-amd64.egg", "greenlet", "0.3.4"),
("greenlet-0.3.4.win-amd64-py3.2.exe", "greenlet", "0.3.4"),
("greenlet-0.3.4-py3.2-win32.egg", "greenlet", "0.3.4"),
("greenlet-0.3.4-py2.7-linux-x86_64.egg", "greenlet", "0.3.4"),
("pep8-0.6.0.zip", "pep8", "0.6.0"),
("ABC12-34_V1X-1.2.3.zip", "ABC12", "34_V1X-1.2.3"),
("A100-200-XYZ-1.2.3.zip", "A100-200-XYZ", "1.2.3"),
("flup-1.0.3.dev-20110405.tar.gz", "flup", "1.0.3.dev-20110405"),
("package-1.0.0-alpha.1.zip", "package", "1.0.0-alpha.1"),
("package-1.3.7+build.11.e0f985a.zip", "package", "1.3.7+build.11.e0f985a"),
("package-v1-8.1.301.ga0df26f.zip", "package-v1", "8.1.301.ga0df26f"),
("package-v1.1-8.1.301.ga0df26f.zip", "package-v1.1", "8.1.301.ga0df26f"),
("package-2013.02.17.dev123.zip", "package", "2013.02.17.dev123"),
("package-20000101.zip", "package", "20000101"),
("flup-123-1.0.3.dev-20110405.tar.gz", "flup-123", "1.0.3.dev-20110405"),
("package-123-1.0.0-alpha.1.zip", "package-123", "1.0.0-alpha.1"),
(
"package-123-1.3.7+build.11.e0f985a.zip",
"package-123",
"1.3.7+build.11.e0f985a",
),
("package-123-v1.1_3-8.1.zip", "package-123-v1.1_3", "8.1"),
("package-123-2013.02.17.dev123.zip", "package-123", "2013.02.17.dev123"),
("package-123-20000101.zip", "package-123", "20000101"),
(
"pyelasticsearch-0.5-brainbot-1-20130712.zip",
"pyelasticsearch",
"0.5-brainbot-1-20130712",
),
("pywin32-217-cp27-none-win32.whl", "pywin32", "217"),
("pywin32-217-55-cp27-none-win32.whl", "pywin32", "217-55"),
("pywin32-217.1-cp27-none-win32.whl", "pywin32", "217.1"),
("package.zip", "package", ""),
(
"package-name-0.0.1.dev0.linux-x86_64.tar.gz",
"package-name",
"0.0.1.dev0",
),
(
"package-name-0.0.1.dev0.macosx-10.10-intel.tar.gz",
"package-name",
"0.0.1.dev0",
),
(
"package-name-0.0.1.alpha.1.win-amd64-py3.2.exe",
"package-name",
"0.0.1.alpha.1",
),
("pkg-3!1.0-0.1.tgz", "pkg", "3!1.0-0.1"), # TO BE FIXED
("pkg-3!1+.0-0.1.tgz", "pkg", "3!1+.0-0.1"), # TO BE FIXED
("pkg.zip", "pkg", ""),
("foo/pkg.zip", "pkg", ""),
("foo/pkg-1b.zip", "pkg", "1b"),
(
"package-name-0.0.1.alpha.1.win-amd64-py3.2.exe",
"package-name",
"0.0.1.alpha.1",
),
]
def _capitalize_ext(fpath):
f, e = os.path.splitext(fpath)
if e != ".whl":
e = e.upper()
return f + e
@pytest.mark.parametrize(("filename", "pkgname", "version"), files)
def test_guess_pkgname_and_version(filename, pkgname, version):
exp = (pkgname, version)
assert core.guess_pkgname_and_version(filename) == exp
assert core.guess_pkgname_and_version(_capitalize_ext(filename)) == exp
@pytest.mark.parametrize(("filename", "pkgname", "version"), files)
def test_guess_pkgname_and_version_asc(filename, pkgname, version):
exp = (pkgname, version)
filename = f"{filename}.asc"
assert core.guess_pkgname_and_version(filename) == exp
def test_listdir_bad_name(tmpdir):
tmpdir.join("foo.whl").ensure()
res = list(core.listdir(tmpdir.strpath))
assert res == []
@ -36,37 +122,33 @@ hashes = (
@pytest.mark.parametrize(("algo", "digest"), hashes)
def test_hashfile(tmp_path, algo, digest):
f = tmp_path.joinpath("empty")
f.touch()
assert backend.digest_file(str(f), algo) == f"{algo}={digest}"
def test_hashfile(tmpdir, algo, digest):
f = tmpdir.join("empty")
f.ensure()
assert core.digest_file(f.strpath, algo) == digest
@pytest.mark.parametrize("hash_algo", ("md5", "sha256", "sha512"))
def test_fname_and_hash(tmp_path, hash_algo):
def test_fname_and_hash(tmpdir, hash_algo):
"""Ensure we are returning the expected hashes for files."""
def digester(pkg):
digest = backend.digest_file(pkg.fn, hash_algo)
pkg.digest = digest
return digest
f = tmp_path.joinpath("tmpfile")
f.touch()
pkgfile = core.PkgFile("tmp", "1.0.0", str(f), f.parent, f.name)
pkgfile.digester = digester
assert pkgfile.fname_and_hash == f"{f.name}#{digester(pkgfile)}"
f = tmpdir.join("tmpfile")
f.ensure()
pkgfile = core.PkgFile("tmp", "1.0.0", f.strpath, f.dirname, f.basename)
assert pkgfile.fname_and_hash(hash_algo) == "{}#{}={}".format(
f.basename, hash_algo, str(f.computehash(hashtype=hash_algo))
)
def test_redirect_project_encodes_newlines():
def test_redirect_prefix_encodes_newlines():
"""Ensure raw newlines are url encoded in the generated redirect."""
request = Namespace(custom_fullpath="/\nSet-Cookie:malicious=1;")
project = "\nSet-Cookie:malicious=1;"
newpath = core.get_bad_url_redirect_path(request, project)
prefix = "\nSet-Cookie:malicious=1;"
newpath = core.get_bad_url_redirect_path(request, prefix)
assert "\n" not in newpath
def test_normalize_pkgname_for_url_encodes_newlines():
"""Ensure newlines are url encoded in package names for urls."""
assert "\n" not in normalize_pkgname_for_url("/\nSet-Cookie:malicious=1;")
assert "\n" not in core.normalize_pkgname_for_url(
"/\nSet-Cookie:malicious=1;"
)

@ -8,12 +8,10 @@ from pypiserver import version as my_ver
@pytest.fixture()
def readme():
return Path(__file__).parents[1].joinpath("README.md").read_text()
return Path(__file__).parents[1].joinpath("README.rst").read_text()
def test_READMEversion(readme):
m = re.compile(
r"^\|\s*Version\s*\|\s*(\d+\.\d+\.\d+)\s*\|$", re.MULTILINE
).search(readme)
m = re.compile(r"^\s*:Version:\s*(.+)\s*$", re.MULTILINE).search(readme)
assert m, "Could not find version on README!"
assert m.group(1) == my_ver, f"Updated version({m.group(1)}) on README!"

@ -1,7 +1,6 @@
"""
Test module for app initialization
"""
# Standard library imports
import logging
import pathlib
@ -13,12 +12,11 @@ import pytest
# Local imports
import pypiserver
from pypiserver.config import Config
logger = logging.getLogger(__name__)
TEST_DIR = pathlib.Path(__file__).parent
HTPASS_FILE = TEST_DIR / "../fixtures/htpasswd.a.a"
HTPASS_FILE = TEST_DIR / "htpasswd.a.a"
WELCOME_FILE = TEST_DIR / "sample_msg.html"
@ -105,19 +103,3 @@ def test_backwards_compat_kwargs_duplicate_check(
with pytest.raises(ValueError) as err:
pypiserver.backwards_compat_kwargs(kwargs)
assert "('redirect_to_fallback', 'disable_fallback')" in str(err.value)
def test_setup_routes_from_config_customized_endpoint() -> None:
_app = pypiserver.setup_routes_from_config(
pypiserver.app(),
Config.default_with_overrides(health_endpoint="/healthz"),
)
assert "/healthz" in (route.rule for route in _app.routes)
def test_setup_routes_from_config_invalid_customized_endpoint() -> None:
with pytest.raises(RuntimeError, match="overlaps with existing routes"):
pypiserver.setup_routes_from_config(
pypiserver.app(),
Config.default_with_overrides(health_endpoint="/simple"),
)

@ -7,13 +7,12 @@ from unittest import mock
import pytest
import pypiserver.bottle
from pypiserver import __main__
from pypiserver.bottle import Bottle
THIS_DIR = pathlib.Path(__file__).parent
HTPASS_FILE = THIS_DIR / "../fixtures/htpasswd.a.a"
HTPASS_FILE = THIS_DIR / "htpasswd.a.a"
IGNORELIST_FILE = THIS_DIR / "test-ignorelist"
@ -39,6 +38,7 @@ class main_wrapper:
@pytest.fixture()
def main(monkeypatch):
main = main_wrapper()
def run(**kwargs):
@ -66,18 +66,14 @@ def test_noargs(main):
# Assert we're calling with the default host, port, and server, and
# assume that we've popped `app` off of the bottle args in our `main`
# fixture.
exp_kwargs = {"host": "0.0.0.0", "port": 8080, "server": "auto"}
actual_kwargs = main([])
# Only assert our expected are are present. We may pass extra kwargs
# for particular servers, depending on what is available in the python
# path.
assert all(map(lambda k: exp_kwargs[k] == actual_kwargs[k], exp_kwargs))
assert main([]) == {"host": "0.0.0.0", "port": 8080, "server": "auto"}
def test_port(main):
assert main(["--port=8081"])["port"] == 8081
assert main(["--port", "8081"])["port"] == 8081
assert main(["-p", "8081"])["port"] == 8081
expected = dict(host="0.0.0.0", port=8081, server="auto")
assert main(["--port=8081"]) == expected
assert main(["--port", "8081"]) == expected
assert main(["-p", "8081"]) == expected
def test_server(main):
@ -85,26 +81,6 @@ def test_server(main):
assert main(["--server", "cherrypy"])["server"] == "cherrypy"
def test_wsgiserver_extra_args_present(monkeypatch, main):
"""The wsgi server gets extra keyword arguments."""
monkeypatch.setattr(
__main__,
"guess_auto_server",
lambda: __main__.AutoServer.WsgiRef,
)
assert main([])["handler_class"] is __main__.WsgiHandler
def test_wsgiserver_extra_kwargs_absent(monkeypatch, main):
"""Other servers don't get wsgiserver args."""
monkeypatch.setattr(
__main__,
"guess_auto_server",
lambda: __main__.AutoServer.Waitress,
)
assert "handler_class" not in main([])
def test_root_multiple(main):
# Remember we're already setting THIS_DIR as a root in the `main` fixture
main([str(THIS_DIR.parent)])
@ -130,12 +106,12 @@ def test_fallback_url_default(main):
def test_hash_algo_default(main):
main([])
assert main.app._pypiserver_config.hash_algo == "sha256"
assert main.app._pypiserver_config.hash_algo == "md5"
def test_hash_algo(main):
main(["--hash-algo=md5"])
assert main.app._pypiserver_config.hash_algo == "md5"
main(["--hash-algo=sha256"])
assert main.app._pypiserver_config.hash_algo == "sha256"
def test_hash_algo_off(main):
@ -256,57 +232,3 @@ def test_blacklist_file(main):
"""
main(["-U", "--blacklist-file", str(IGNORELIST_FILE)])
assert main.update_kwargs["ignorelist"] == ["mypiserver", "something"]
def test_auto_servers() -> None:
"""Test auto servers."""
# A list of bottle ServerAdapters
bottle_adapters = tuple(
a.__name__.lower() for a in pypiserver.bottle.AutoServer.adapters
)
# We are going to expect that our AutoServer enum names must match those
# at least closely enough to be recognizable.
our_mappings = tuple(map(str.lower, __main__.AutoServer.__members__))
# Assert that all of our mappings are represented in bottle adapters
assert all(
any(mapping in a for a in bottle_adapters) for mapping in our_mappings
)
# Assert that our import checking order matches the order in which the
# adapters are defined in the AutoServer
our_check_order = tuple(i[0] for i in __main__.AUTO_SERVER_IMPORTS)
# Some of the servers have more than one check, so we need to remove
# duplicates before we check for identity with the AutoServer definition.
seen: t.Dict[__main__.AutoServer, __main__.AutoServer] = {}
our_check_order = tuple(
seen.setdefault(i, i) for i in our_check_order if i not in seen
)
# We should have the same number of deduped checkers as there are bottle
# adapters
assert len(our_check_order) == len(bottle_adapters)
# And the order should be the same
assert all(
us.name.lower() in them
for us, them in zip(our_check_order, bottle_adapters)
)
def test_health_endpoint_default(main):
main([])
assert main.app._pypiserver_config.health_endpoint == "/health"
assert "/health" in (route.rule for route in main.app.routes)
def test_health_endpoint_customized(main):
main(["--health-endpoint", "/healthz"])
assert main.app._pypiserver_config.health_endpoint == "/healthz"
assert "/healthz" in (route.rule for route in main.app.routes)
def test_health_endpoint_invalid_customized(main):
with pytest.raises(SystemExit):
main(["--health-endpoint", "/health!"])

@ -3,15 +3,20 @@
from __future__ import absolute_import, print_function, unicode_literals
from pathlib import Path
from unittest.mock import Mock
try:
from unittest.mock import Mock
except ImportError:
from mock import Mock
import py
import pytest
from pypiserver import manage
from pypiserver.core import PkgFile
from pypiserver.pkg_helpers import guess_pkgname_and_version, parse_version
from pypiserver.core import (
PkgFile,
guess_pkgname_and_version,
parse_version,
)
from pypiserver.manage import (
PipCmd,
build_releases,
@ -205,17 +210,17 @@ def test_update_all_packages(monkeypatch):
private_pkg_2 = PkgFile("my_other_private_pkg", "1.0")
roots_mock = {
Path("/opt/pypi"): [
"/opt/pypi": [
public_pkg_1,
private_pkg_1,
],
Path("/data/pypi"): [public_pkg_2, private_pkg_2],
"/data/pypi": [public_pkg_2, private_pkg_2],
}
def core_listdir_mock(path: Path):
return roots_mock.get(path, [])
def core_listdir_mock(directory):
return roots_mock.get(directory, [])
monkeypatch.setattr(manage, "listdir", core_listdir_mock)
monkeypatch.setattr(manage.core, "listdir", core_listdir_mock)
monkeypatch.setattr(manage, "update", Mock(return_value=None))
destdir = None
@ -238,7 +243,7 @@ def test_update_all_packages(monkeypatch):
)
def test_update_all_packages_with_blacklist(monkeypatch):
def test_update_all_packages_with_ignorelist(monkeypatch):
"""Test calling update_all_packages()"""
public_pkg_1 = PkgFile("Flask", "1.0")
public_pkg_2 = PkgFile("requests", "1.0")
@ -246,17 +251,17 @@ def test_update_all_packages_with_blacklist(monkeypatch):
private_pkg_2 = PkgFile("my_other_private_pkg", "1.0")
roots_mock = {
Path("/opt/pypi"): [
"/opt/pypi": [
public_pkg_1,
private_pkg_1,
],
Path("/data/pypi"): [public_pkg_2, private_pkg_2],
"/data/pypi": [public_pkg_2, private_pkg_2],
}
def core_listdir_mock(path: Path):
return roots_mock.get(path, [])
def core_listdir_mock(directory):
return roots_mock.get(directory, [])
monkeypatch.setattr(manage, "listdir", core_listdir_mock)
monkeypatch.setattr(manage.core, "listdir", core_listdir_mock)
monkeypatch.setattr(manage, "update", Mock(return_value=None))
destdir = None

@ -1,117 +0,0 @@
import os
from pathlib import WindowsPath, PureWindowsPath
import pytest
from pypiserver.pkg_helpers import guess_pkgname_and_version, is_listed_path
files = [
("pytz-2012b.tar.bz2", "pytz", "2012b"),
("pytz-2012b.tgz", "pytz", "2012b"),
("pytz-2012b.ZIP", "pytz", "2012b"),
("pytz-2012a.zip", "pytz", "2012a"),
("pytz-2012b.tar.xz", "pytz", "2012b"),
("gevent-1.0b1.win32-py2.6.exe", "gevent", "1.0b1"),
("gevent-1.0b1.win32-py2.7.msi", "gevent", "1.0b1"),
("greenlet-0.3.4-py3.1-win-amd64.egg", "greenlet", "0.3.4"),
("greenlet-0.3.4.win-amd64-py3.2.exe", "greenlet", "0.3.4"),
("greenlet-0.3.4-py3.2-win32.egg", "greenlet", "0.3.4"),
("greenlet-0.3.4-py2.7-linux-x86_64.egg", "greenlet", "0.3.4"),
("pep8-0.6.0.zip", "pep8", "0.6.0"),
("ABC12-34_V1X-1.2.3.zip", "ABC12", "34_V1X-1.2.3"),
("A100-200-XYZ-1.2.3.zip", "A100-200-XYZ", "1.2.3"),
("flup-1.0.3.dev-20110405.tar.gz", "flup", "1.0.3.dev-20110405"),
("package-1.0.0-alpha.1.zip", "package", "1.0.0-alpha.1"),
("package-1.3.7+build.11.e0f985a.zip", "package", "1.3.7+build.11.e0f985a"),
("package-v1-8.1.301.ga0df26f.zip", "package-v1", "8.1.301.ga0df26f"),
("package-v1.1-8.1.301.ga0df26f.zip", "package-v1.1", "8.1.301.ga0df26f"),
("package-2013.02.17.dev123.zip", "package", "2013.02.17.dev123"),
("package-20000101.zip", "package", "20000101"),
("flup-123-1.0.3.dev-20110405.tar.gz", "flup-123", "1.0.3.dev-20110405"),
("package-123-1.0.0-alpha.1.zip", "package-123", "1.0.0-alpha.1"),
(
"package-123-1.3.7+build.11.e0f985a.zip",
"package-123",
"1.3.7+build.11.e0f985a",
),
("package-123-v1.1_3-8.1.zip", "package-123-v1.1_3", "8.1"),
("package-123-2013.02.17.dev123.zip", "package-123", "2013.02.17.dev123"),
("package-123-20000101.zip", "package-123", "20000101"),
(
"pyelasticsearch-0.5-brainbot-1-20130712.zip",
"pyelasticsearch",
"0.5-brainbot-1-20130712",
),
("pywin32-217-cp27-none-win32.whl", "pywin32", "217"),
("pywin32-217-55-cp27-none-win32.whl", "pywin32", "217-55"),
("pywin32-217.1-cp27-none-win32.whl", "pywin32", "217.1"),
("package.zip", "package", ""),
(
"package-name-0.0.1.dev0.linux-x86_64.tar.gz",
"package-name",
"0.0.1.dev0",
),
(
"package-name-0.0.1.dev0.macosx-10.10-intel.tar.gz",
"package-name",
"0.0.1.dev0",
),
(
"package-name-0.0.1.alpha.1.win-amd64-py3.2.exe",
"package-name",
"0.0.1.alpha.1",
),
("pkg-3!1.0-0.1.tgz", "pkg", "3!1.0-0.1"), # TO BE FIXED
("pkg-3!1+.0-0.1.tgz", "pkg", "3!1+.0-0.1"), # TO BE FIXED
("pkg.zip", "pkg", ""),
("foo/pkg.zip", "pkg", ""),
("foo/pkg-1b.zip", "pkg", "1b"),
("foo/pywin32-217.1-cp27-none-win32.whl", "pywin32", "217.1"),
(
"package-name-0.0.1.alpha.1.win-amd64-py3.2.exe",
"package-name",
"0.0.1.alpha.1",
),
]
def _capitalize_ext(fpath):
f, e = os.path.splitext(fpath)
if e != ".whl":
e = e.upper()
return f + e
@pytest.mark.parametrize(("filename", "pkgname", "version"), files)
def test_guess_pkgname_and_version(filename, pkgname, version):
exp = (pkgname, version)
assert guess_pkgname_and_version(filename) == exp
assert guess_pkgname_and_version(_capitalize_ext(filename)) == exp
@pytest.mark.parametrize(("filename", "pkgname", "version"), files)
def test_guess_pkgname_and_version_asc(filename, pkgname, version):
exp = (pkgname, version)
filename = f"{filename}.asc"
assert guess_pkgname_and_version(filename) == exp
invalid_files = ["some_file", "some_file.ext", "some_wheel.whl"]
@pytest.mark.parametrize("filename", invalid_files)
def test_guess_pkgname_and_version_invalid_files(filename):
assert guess_pkgname_and_version(filename) is None
paths = [
("/some/path", True),
(PureWindowsPath(r"c:\some\windows\path"), True),
("/.hidden", False),
(PureWindowsPath(r"c:\.hidden\windows\path"), False),
]
@pytest.mark.parametrize(("pathname", "allowed"), paths)
def test_allowed_path_check(pathname, allowed):
assert is_listed_path(pathname) == allowed

@ -11,217 +11,172 @@ The tests below are using 3 ways to startup pypi-servers:
- "new_server": starting a new server with any configurations on each test.
"""
import contextlib
import itertools
import os
import shutil
import socket
import re
import sys
import time
import typing as t
from __future__ import print_function
from collections import namedtuple
import contextlib
import functools
import os
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from shlex import split
from subprocess import Popen
from urllib.error import URLError
from urllib.request import urlopen
from textwrap import dedent
try:
from urllib.request import urlopen
except ImportError:
from urllib import urlopen
from py import path # @UnresolvedImport
import pytest
# ######################################################################
# Fixtures & Helper Functions
# ######################################################################
CURRENT_PATH = Path(__file__).parent
ports = itertools.count(10000)
Srv = namedtuple("Srv", ("port", "root"))
_BUFF_SIZE = 2 ** 16
_port = 8090
SLEEP_AFTER_SRV = 3 # sec
@pytest.fixture
def port():
global _port
_port += 1
return _port
Srv = namedtuple("Srv", ("proc", "port", "package"))
def _run_server(packdir, port, authed, other_cli=""):
"""Run a server, optionally with partial auth enabled."""
pswd_opt_choices = {
True: "-Ptests/htpasswd.a.a -a update,download",
False: "-P. -a.",
"partial": "-Ptests/htpasswd.a.a -a update",
}
pswd_opts = pswd_opt_choices[authed]
cmd = (
f"{sys.executable} -m pypiserver.__main__ -vvv --overwrite -i 127.0.0.1 "
f"-p {port} {pswd_opts} {other_cli} {packdir}"
)
proc = subprocess.Popen(cmd.split(), bufsize=_BUFF_SIZE)
time.sleep(SLEEP_AFTER_SRV)
assert proc.poll() is None
return Srv(proc, int(port), packdir)
def _kill_server(srv):
print(f"Killing {srv}")
try:
srv.proc.terminate()
time.sleep(1)
finally:
srv.proc.kill()
@contextlib.contextmanager
def run_server(root, authed=False, other_cli=""):
"""Run a server, optionally with partial auth enabled."""
htpasswd = (
CURRENT_PATH.joinpath("../fixtures/htpasswd.a.a").expanduser().resolve()
)
pswd_opt_choices = {
True: f"-P {htpasswd} -a update,download",
False: "-P. -a.",
"partial": f"-P {htpasswd} -a update",
}
pswd_opts = pswd_opt_choices[authed]
port = next(ports)
cmd = (
f"{sys.executable} -m pypiserver.__main__ "
f"run -vvv --overwrite -i 127.0.0.1 "
f"-p {port} {pswd_opts} {other_cli} {root}"
)
proc = Popen(cmd.split(), bufsize=2**16)
srv = Srv(port, root)
def new_server(packdir, port, authed=False, other_cli=""):
srv = _run_server(packdir, port, authed=authed, other_cli=other_cli)
try:
wait_until_ready(srv)
assert proc.poll() is None
yield srv
finally:
print(f"Killing {srv}")
_kill_proc(proc)
_kill_server(srv)
def wait_until_ready(srv: Srv, n_tries=10):
for _ in range(n_tries):
if is_ready(srv):
return True
time.sleep(0.5)
raise TimeoutError
def is_ready(srv: Srv):
@contextlib.contextmanager
def chdir(d):
old_d = os.getcwd()
try:
return urlopen(build_url(srv.port), timeout=0.5).getcode() in (
200,
401,
)
except (URLError, socket.timeout):
return False
def _kill_proc(proc):
proc.terminate()
try:
proc.wait(timeout=1)
os.chdir(d)
yield
finally:
proc.kill()
os.chdir(old_d)
def build_url(port: t.Union[int, str], user: str = "", pswd: str = "") -> str:
def _run_python(cmd):
return os.system(f"{sys.executable} {cmd}")
@pytest.fixture(scope="module")
def project(request):
def fin():
tmpdir.remove(True)
tmpdir = path.local(tempfile.mkdtemp())
request.addfinalizer(fin)
src_setup_py = path.local().join("tests", "centodeps-setup.py")
assert src_setup_py.check()
projdir = tmpdir.join("centodeps")
projdir.mkdir()
dst_setup_py = projdir.join("setup.py")
src_setup_py.copy(dst_setup_py)
assert dst_setup_py.check()
return projdir
@pytest.fixture(scope="module")
def package(project, request):
with chdir(project.strpath):
cmd = "setup.py bdist_wheel"
assert _run_python(cmd) == 0
pkgs = list(project.join("dist").visit("centodeps*.whl"))
assert len(pkgs) == 1
pkg = path.local(pkgs[0])
assert pkg.check()
return pkg
@pytest.fixture(scope="module")
def packdir(package):
return package.dirpath()
open_port = 8081
@pytest.fixture(scope="module")
def open_server(packdir, request):
srv = _run_server(packdir, open_port, authed=False)
fin = functools.partial(_kill_server, srv)
request.addfinalizer(fin)
return srv
protected_port = 8082
@pytest.fixture(scope="module")
def protected_server(packdir, request):
srv = _run_server(packdir, protected_port, authed=True)
fin = functools.partial(_kill_server, srv)
request.addfinalizer(fin)
return srv
@pytest.fixture
def empty_packdir(tmpdir):
return tmpdir.mkdir("dists")
def _build_url(port, user="", pswd=""):
auth = f"{user}:{pswd}@" if user or pswd else ""
return f"http://{auth}localhost:{port}"
def run_setup_py(path: Path, arguments: str) -> int:
return os.system(f"{sys.executable} {path / 'setup.py'} {arguments}")
def run_py_build(srcdir: Path, flags: str) -> int:
return os.system(f"{sys.executable} -m build {flags} {srcdir}")
# A test-distribution to check if
# bottle supports uploading 100's of packages,
# see: https://github.com/pypiserver/pypiserver/issues/82
#
# Has been run once `pip wheel .`, just to generate:
# ./wheelhouse/centodeps-0.0.0-cp34-none-win_amd64.whl
#
SETUP_PY = """\
from setuptools import setup
setup(
name="centodeps",
install_requires=["a==1.0"] * 200,
options={
"bdist_wheel": {"universal": True},
},
packages=[],
)
"""
@pytest.fixture(scope="module")
def project(tmp_path_factory):
projdir = tmp_path_factory.mktemp("project") / "centodeps"
projdir.mkdir(parents=True, exist_ok=True)
projdir.joinpath("setup.py").write_text(SETUP_PY)
return projdir
@pytest.fixture(scope="session")
def server_root(tmp_path_factory):
return tmp_path_factory.mktemp("root")
@pytest.fixture(scope="module")
def wheel_file(project, tmp_path_factory):
distdir = tmp_path_factory.mktemp("dist")
if re.match("^3\.7", sys.version):
assert run_setup_py(project, f"bdist_wheel -d {distdir}") == 0
else:
assert run_py_build(project, f"--wheel --outdir {distdir}") == 0
wheels = list(distdir.glob("centodeps*.whl"))
assert len(wheels) > 0
return wheels[0]
@pytest.fixture()
def hosted_wheel_file(wheel_file, server_root):
dst = server_root / wheel_file.name
shutil.copy(wheel_file, dst)
yield dst
if dst.is_file():
dst.unlink()
def clear_directory(root: Path):
for path in root.iterdir():
if path.is_file():
path.unlink()
@pytest.fixture(scope="module")
def _open_server(server_root):
with run_server(server_root, authed=False) as srv:
yield srv
@pytest.fixture
def open_server(_open_server: Srv):
yield _open_server
clear_directory(_open_server.root)
@pytest.fixture(scope="module")
def _authed_server(server_root):
with run_server(server_root, authed=True) as srv:
yield srv
@pytest.fixture
def authed_server(_authed_server):
yield _authed_server
clear_directory(_authed_server.root)
@pytest.fixture(scope="module")
def _partial_auth_server(server_root):
with run_server(server_root, authed="partial") as srv:
yield srv
@pytest.fixture
def partial_authed_server(_partial_auth_server):
yield _partial_auth_server
clear_directory(_partial_auth_server.root)
@pytest.fixture
def empty_packdir(tmp_path_factory):
return tmp_path_factory.mktemp("dists")
def pip_download(
cmd: str,
port: t.Union[int, str],
install_dir: str,
user: str = None,
pswd: str = None,
) -> int:
url = build_url(port, user, pswd)
return _run_pip(f"-vv download -d {install_dir} -i {url} {cmd}")
def _run_pip(cmd: str) -> int:
def _run_pip(cmd):
ncmd = (
"pip --no-cache-dir --disable-pip-version-check "
f"--retries 0 --timeout 5 --no-input {cmd}"
@ -232,139 +187,281 @@ def _run_pip(cmd: str) -> int:
return proc.returncode
def _run_pip_install(cmd, port, install_dir, user=None, pswd=None):
url = _build_url(port, user, pswd)
return _run_pip(f"-vv download -d {install_dir} -i {url} {cmd}")
@pytest.fixture
def pipdir(tmp_path_factory):
return tmp_path_factory.mktemp("pip")
def pipdir(tmpdir):
return tmpdir.mkdir("pip")
@contextlib.contextmanager
def pypirc_file(repo, username="''", password="''"):
pypirc_path = Path.home() / ".pypirc"
old_pypirc = pypirc_path.read_text() if pypirc_path.is_file() else None
pypirc_path.write_text(
def pypirc_tmpfile(port, user, password):
"""Create a temporary pypirc file."""
fd, filepath = tempfile.mkstemp()
os.close(fd)
Path(filepath).write_text(
"\n".join(
(
"[distutils]",
"index-servers: test",
"",
"[test]",
f"repository: {repo}",
f"username: {username}",
"" "[test]",
f"repository: {_build_url(port)}",
f"username: {user}",
f"password: {password}",
)
)
)
print(Path(filepath).read_text())
yield filepath
os.remove(filepath)
@contextlib.contextmanager
def pypirc_file(txt):
pypirc_path = path.local("~/.pypirc", expanduser=1)
old_pypirc = pypirc_path.read() if pypirc_path.check() else None
pypirc_path.write(txt)
try:
yield pypirc_path
yield
finally:
if old_pypirc:
pypirc_path.write_text(old_pypirc)
pypirc_path.write(old_pypirc)
else:
pypirc_path.unlink()
pypirc_path.remove()
@pytest.fixture
def open_pypirc(open_server):
with pypirc_file(repo=build_url(open_server.port)) as path:
yield path
@pytest.fixture
def authed_pypirc(authed_server):
username, password = "a", "a"
with pypirc_file(
repo=build_url(authed_server.port),
username=username,
password=password,
) as path:
yield path
def run_twine(command: str, package: str, conf: str) -> None:
def twine_upload(
packages, repository="test", conf="pypirc", expect_failure=False
):
"""Call 'twine upload' with appropriate arguments"""
proc = Popen(
split(
f"twine {command} --repository test --config-file {conf} {package}"
(
"twine",
"upload",
"--repository",
repository,
"--config-file",
conf,
" ".join(packages),
)
)
proc.communicate()
assert not proc.returncode, f"Twine {command} failed. See stdout/err"
if not expect_failure and proc.returncode:
assert False, "Twine upload failed. See stdout/err"
def twine_register(
packages, repository="test", conf="pypirc", expect_failure=False
):
"""Call 'twine register' with appropriate args"""
proc = Popen(
(
"twine",
"register",
"--repository",
repository,
"--config-file",
conf,
" ".join(packages),
)
)
proc.communicate()
if not expect_failure and proc.returncode:
assert False, "Twine register failed. See stdout/err"
# ######################################################################
# Tests
# ######################################################################
all_servers = [
("open_server", "open_pypirc"),
("authed_server", "authed_pypirc"),
("partial_authed_server", "authed_pypirc"),
]
def test_pipInstall_packageNotFound(empty_packdir, port, pipdir, package):
with new_server(empty_packdir, port):
cmd = "centodeps"
assert _run_pip_install(cmd, port, pipdir) != 0
assert not pipdir.listdir()
def test_pip_install_package_not_found(open_server, pipdir):
assert pip_download("centodeps", open_server.port, pipdir) != 0
assert not list(pipdir.iterdir())
def test_pipInstall_openOk(open_server, package, pipdir):
cmd = "centodeps"
assert _run_pip_install(cmd, open_server.port, pipdir) == 0
assert pipdir.join(package.basename).check()
def test_pip_install_open_succeeds(open_server, hosted_wheel_file, pipdir):
assert pip_download("centodeps", open_server.port, pipdir) == 0
assert pipdir.joinpath(hosted_wheel_file.name).is_file()
def test_pipInstall_authedFails(protected_server, pipdir):
cmd = "centodeps"
assert _run_pip_install(cmd, protected_server.port, pipdir) != 0
assert not pipdir.listdir()
@pytest.mark.usefixtures("wheel_file")
def test_pip_install_authed_fails(authed_server, pipdir):
assert pip_download("centodeps", authed_server.port, pipdir) != 0
assert not list(pipdir.iterdir())
def test_pip_install_authed_succeeds(authed_server, hosted_wheel_file, pipdir):
def test_pipInstall_authedOk(protected_server, package, pipdir):
cmd = "centodeps"
assert (
pip_download(
"centodeps", authed_server.port, pipdir, user="a", pswd="a"
)
_run_pip_install(cmd, protected_server.port, pipdir, user="a", pswd="a")
== 0
)
assert pipdir.joinpath(hosted_wheel_file.name).is_file()
assert pipdir.join(package.basename).check()
def test_partial_authed_open_download(partial_authed_server):
@pytest.mark.parametrize("pkg_frmt", ["bdist", "bdist_wheel"])
def test_setuptoolsUpload_open(empty_packdir, port, project, package, pkg_frmt):
url = _build_url(port, None, None)
with pypirc_file(
dedent(
f"""\
[distutils]
index-servers: test
[test]
repository: {url}
username: ''
password: ''
"""
)
):
with new_server(empty_packdir, port):
with chdir(project.strpath):
cmd = f"setup.py -vvv {pkg_frmt} upload -r {url}"
for i in range(5):
print(f"++Attempt #{i}")
assert _run_python(cmd) == 0
time.sleep(SLEEP_AFTER_SRV)
assert len(empty_packdir.listdir()) == 1
@pytest.mark.parametrize("pkg_frmt", ["bdist", "bdist_wheel"])
def test_setuptoolsUpload_authed(
empty_packdir, port, project, package, pkg_frmt, monkeypatch
):
url = _build_url(port)
with pypirc_file(
dedent(
f"""\
[distutils]
index-servers: test
[test]
repository: {url}
username: a
password: a
"""
)
):
with new_server(empty_packdir, port, authed=True):
with chdir(project.strpath):
cmd = (
f"setup.py -vvv {pkg_frmt} register -r test upload -r test"
)
for i in range(5):
print(f"++Attempt #{i}")
assert _run_python(cmd) == 0
time.sleep(SLEEP_AFTER_SRV)
assert len(empty_packdir.listdir()) == 1
@pytest.mark.parametrize("pkg_frmt", ["bdist", "bdist_wheel"])
def test_setuptools_upload_partial_authed(
empty_packdir, port, project, pkg_frmt
):
"""Test uploading a package with setuptools with partial auth."""
url = _build_url(port)
with pypirc_file(
dedent(
f"""\
[distutils]
index-servers: test
[test]
repository: {url}
username: a
password: a
"""
)
):
with new_server(empty_packdir, port, authed="partial"):
with chdir(project.strpath):
cmd = (
f"setup.py -vvv {pkg_frmt} register -r test upload -r test"
)
for i in range(5):
print(f"++Attempt #{i}")
assert _run_python(cmd) == 0
time.sleep(SLEEP_AFTER_SRV)
assert len(empty_packdir.listdir()) == 1
def test_partial_authed_open_download(empty_packdir, port):
"""Validate that partial auth still allows downloads."""
url = build_url(partial_authed_server.port) + "/simple"
resp = urlopen(url)
assert resp.getcode() == 200
url = _build_url(port) + "/simple"
with new_server(empty_packdir, port, authed="partial"):
resp = urlopen(url)
assert resp.getcode() == 200
def test_twine_upload_open(empty_packdir, port, package):
"""Test twine upload with no authentication"""
user, pswd = "foo", "bar"
with new_server(empty_packdir, port):
with pypirc_tmpfile(port, user, pswd) as rcfile:
twine_upload([package.strpath], repository="test", conf=rcfile)
time.sleep(SLEEP_AFTER_SRV)
assert len(empty_packdir.listdir()) == 1
@pytest.mark.parametrize("hash_algo", ("md5", "sha256", "sha512"))
@pytest.mark.usefixtures("hosted_wheel_file")
def test_hash_algos(server_root, pipdir, hash_algo):
def test_hash_algos(empty_packdir, port, package, pipdir, hash_algo):
"""Test twine upload with no authentication"""
with run_server(
server_root, other_cli="--hash-algo {}".format(hash_algo)
) as srv:
assert pip_download("centodeps", srv.port, pipdir) == 0
user, pswd = "foo", "bar"
with new_server(
empty_packdir, port, other_cli="--hash-algo {}".format(hash_algo)
):
with pypirc_tmpfile(port, user, pswd) as rcfile:
twine_upload([package.strpath], repository="test", conf=rcfile)
time.sleep(SLEEP_AFTER_SRV)
assert _run_pip_install("centodeps", port, pipdir) == 0
@pytest.mark.parametrize(["server_fixture", "pypirc_fixture"], all_servers)
def test_twine_upload(
server_fixture, pypirc_fixture, server_root, wheel_file, request
):
"""Test twine upload with no authentication"""
assert len(list(server_root.iterdir())) == 0
request.getfixturevalue(server_fixture)
pypirc = request.getfixturevalue(pypirc_fixture)
def test_twine_upload_authed(empty_packdir, port, package):
"""Test authenticated twine upload"""
user, pswd = "a", "a"
with new_server(empty_packdir, port, authed=False):
with pypirc_tmpfile(port, user, pswd) as rcfile:
twine_upload([package.strpath], repository="test", conf=rcfile)
time.sleep(SLEEP_AFTER_SRV)
assert len(empty_packdir.listdir()) == 1
run_twine("upload", wheel_file, conf=pypirc)
assert len(list(server_root.iterdir())) == 1
assert server_root.joinpath(wheel_file.name).is_file(), (
wheel_file.name,
list(server_root.iterdir()),
assert empty_packdir.join(package.basename).check(), (
package.basename,
empty_packdir.listdir(),
)
@pytest.mark.parametrize(["server_fixture", "pypirc_fixture"], all_servers)
def test_twine_register(server_fixture, pypirc_fixture, wheel_file, request):
def test_twine_upload_partial_authed(empty_packdir, port, package):
"""Test partially authenticated twine upload"""
user, pswd = "a", "a"
with new_server(empty_packdir, port, authed="partial"):
with pypirc_tmpfile(port, user, pswd) as rcfile:
twine_upload([package.strpath], repository="test", conf=rcfile)
time.sleep(SLEEP_AFTER_SRV)
assert len(empty_packdir.listdir()) == 1
def test_twine_register_open(open_server, package):
"""Test unauthenticated twine registration"""
request.getfixturevalue(server_fixture)
pypirc = request.getfixturevalue(pypirc_fixture)
run_twine("register", wheel_file, conf=pypirc)
srv = open_server
with pypirc_tmpfile(srv.port, "foo", "bar") as rcfile:
twine_register([package.strpath], repository="test", conf=rcfile)
def test_twine_register_authed_ok(protected_server, package):
"""Test authenticated twine registration"""
srv = protected_server
user, pswd = "a", "a"
with pypirc_tmpfile(srv.port, user, pswd) as rcfile:
twine_register([package.strpath], repository="test", conf=rcfile)

20
tox.ini

@ -1,14 +1,28 @@
[tox]
envlist = py36, py37, py38, py39, py310, py311, pypy3
envlist = py36, py37, py38, py39, pypy3
[testenv]
deps=-r{toxinidir}/requirements/test.pip
deps=-r{toxinidir}/requirements/dev.pip
allowlist_externals=
/bin/sh
mypy
sitepackages=False
[testenv:py{36,37,38,39}]
commands=
/bin/sh -c "{env:PYPISERVER_SETUP_CMD:true}"
pytest --cov=pypiserver {posargs:tests}
# individual mypy files for now, until we get the rest
# of the project typechecking
mypy \
pypiserver/config.py \
tests/test_init.py
pytest --cov=pypiserver {posargs}
[testenv:pypy3]
commands=
/bin/sh -c "{env:PYPISERVER_SETUP_CMD:true}"
# no mypy in pypy
pytest --cov=pypiserver {posargs}
[pytest]