From 9edae77659aff731c6797a6f81eca1e11f5fe0f9 Mon Sep 17 00:00:00 2001 From: Glenn Date: Tue, 15 Aug 2023 05:16:30 -0400 Subject: [PATCH] Feature/markdown conversion (#503) * feat: markdown conversion logo and badges * feat: markdown conversion fix logo path * feat: markdown table changes * feat: markdown table alignment * feat: markdown check toc * feat: markdown toc additions * feat: markdown quickstart section * feat: dependabot more details section in quick start * feat: dependabot correct bold * feat: markdown client side config, pip * feat: markdown typo in title * feat: markdown typo in configuring pip * feat: markdown apache like authentication section an initial cut to view the markdown in github * feat: markdown typo in markdown link * feat: markdown remove trailing colon * feat: markdown typo in shell markdown * feat: markdown standardize on 4 space indent in shell code block * feat: markdown complete markdown for section up to alternate installation methods * feat: markdown add more of the contents to test with * feat: markdown contents * feat: markdown contents * feat: markdown contents * feat: markdown dquote> dquote> recipes * feat: markdown dquote> dquote> recipes * feat: markdown dquote> dquote> up to licensing * feat: markdown dquote> dquote> contents * Update README.md Missing exclamation mark * Update README.md missing link * Update README.md remove duplicated text * Update README.md bold differences it Table of contents * Update README.md additional bold changes in table of contents * Update README.md broken link * Update README.md typo in link fix * Update README.md change code block to text as shell highlighting was showing some items in red * Update README.md code block shell to text * Update README.md correct pypi-server update section * feat: markdown dquote> dquote> link back to TOC title * Update README.md change link to TOC title * Update README.md link test * Update README.md link update * Update README.md link update * Update README.md link update * feat: markdown links * Update README.md change the level of indent for uploading packages remotely * Update README.md add link to python-pam * feat: markdown apache link to TOC not working. * Update README.md grammar * Update README.md typo bold * feat: markdown undo bolded text in TOC and titles as linking does not work * feat: markdown remove bold from TOC * feat: feature more link issues * feat: markdown fixing broken links * feat: markdown change text slightly as markdown only links to plain text * feat: markdown typo * feat: markdown more link typos * Update README.md typo in link * Update README.md link will not work with braces in the titles * feat: markdown run mdformat and apply changes, :) lint! * feat: markdown - check via mdformat - remove old check script - update test-requirements.txt * feat: markdown correct the errors in the mdformat run command * feat: markdown for testing remove all the actual jobs * feat: markdown re-run mdformat * feat: markdown put the jobs back in after testing the mdformat cmd for passing and failing via workflow dispatch * feat: markdown remove references to README.md * feat: markdown change action to workflow dispatch for testing * feat: markdown - update docker igore - alter unit test to look for version number after md changes * feat: markdown black linting * feat: markdown update comments * feat: markdown update bumpver to look at md rather than rst file * feat: markdown replace workflow dispatch with pull request to get ready for the final PR * feat: markdown-delete-original delete the original rst file * feat: markdown-delete-original change ci to workflow dispatch for testing * feat: markdown-delete-original revert workflow dispatch * feat: markdown-badge-links set the links back to the original URLs. * feat: markdown-badge-links fix brackets * feat: markdown update the version and date * feat: markdown conversion markdown changes to conform to mdformat tooling. --- .dockerignore | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/rc.yml | 2 +- Dockerfile | 2 +- README.md | 1212 ++++++++++++++++++++++++++++ README.rst | 1078 ------------------------- bin/README.rst | 2 +- bin/bumpver.py | 2 +- bin/check_readme.sh | 37 - docs/README.md | 2 +- requirements/test-requirements.txt | 3 + setup.py | 2 +- tests/test_docs.py | 6 +- 13 files changed, 1227 insertions(+), 1125 deletions(-) create mode 100644 README.md delete mode 100644 README.rst delete mode 100755 bin/check_readme.sh diff --git a/.dockerignore b/.dockerignore index 4c84ef5..8fb4196 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,6 @@ !docker/docker-requirements.txt !docker/gunicorn.conf.py !docker/entrypoint.sh -!README.rst +!README.md !setup.cfg !setup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70044a7..686f27c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: - name: Check formatting run: black --diff --check . - name: Validate README - run: ./bin/check_readme.sh + run: mdformat --check README.md # Full-flow docker tests, again not python version dependent # We _could_ test this on MacOS, but it takes forever to get docker diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index dbd6e7d..f5c48ae 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -7,7 +7,7 @@ name: release_candidate # - [x] update CHANGES.rst # - [x] create changes commit # - [x] push to GH -# - [ ] update README.rst +# - [ ] update README.md # - [ ] create readme commit # - [ ] push to GH # - [ ] open a PR to `master` diff --git a/Dockerfile b/Dockerfile index 852f874..75c59c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ RUN apk add --no-cache --virtual .build-deps \ COPY pypiserver pypiserver COPY setup.cfg . COPY setup.py . -COPY README.rst . +COPY README.md . RUN python -m pip install --no-warn-script-location --prefix=/install . FROM base diff --git a/README.md b/README.md new file mode 100644 index 0000000..ae6aa38 --- /dev/null +++ b/README.md @@ -0,0 +1,1212 @@ +![pypi server logo](docs/__resources__/pypiserver_logo.png) + +[**pypiserver - minimal PyPI server for use with pip/easy_install**](<>) + +[![pypi badge](https://img.shields.io/pypi/v/pypiserver.svg)](https://shields.io/) +[![ci workflow](https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml/badge.svg)](https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml) +[![Generic badge](https://img.shields.io/badge/python-3.6%7C3.7%7C3.8+-blue.svg)](https://pypi.org/project/pypiserver/) +[![Generic badge](https://img.shields.io/badge/license-MIT%7Czlib/libpng-blue.svg)](https://raw.githubusercontent.com/pypiserver/pypiserver/master/LICENSE.txt) + +| name | description | +| :---------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Version | 1.5.2 | +| Date: | 2023-07-30 | +| Source | https://github.com/pypiserver/pypiserver | +| PyPI | https://pypi.org/project/pypiserver/ | +| Tests | https://github.com/pypiserver/pypiserver/actions | +| Maintainers | Kostis Anagnostopoulos , Matthew Planchard , Dmitrii Orlov , **Someone new?** We are looking for new maintainers! [#397](https://github.com/pypiserver/pypiserver/issues/397) | +| License | zlib/libpng + MIT | +| Community | https://pypiserver.zulipchat.com | + +Chat with us on [Zulip](https://pypiserver.zulipchat.com)! + +**pypiserver** is a minimal [PyPI](https://pypi.org/) compatible server for **pip** or **easy_install**. +It is based on [bottle](http://bottlepy.org/) and serves packages from regular directories. +Wheels, bdists, eggs and accompanying PGP-signatures can be uploaded +either with **pip**, **setuptools**, **twine**, **pypi-uploader**, or simply copied +with **scp**. + +Note +The official software powering [PyPI](https://pypi.org/) is +[Warehouse](https://github.com/pypa/warehouse/). However, +[Warehouse](https://github.com/pypa/warehouse/) +is fairly specialized to be **pypi.org**'s own software, and should not +be used in other contexts. In particular, it does not officially support +being used as a custom package index by users wishing to serve their own +packages. + +**pypiserver** implements the same interfaces as [PyPI](https://pypi.org/), allowing +standard Python packaging tooling such as **pip** and **twine** to +interact with it as a package index just as they would with [PyPI](https://pypi.org/), while +making it much easier to get a running index server. + +# pypiserver + +Table of Contents + +- [pypiserver - minimal PyPI server for use with pip/easy_install](#pypiserver---minimal-pypi-server-for-use-with-pipeasy_install) + - [Quickstart Installation and Usage](#Quickstart-Installation-and-Usage) + - [More details about pypi-server run](#More-details-about-pypi-server-run) + - [More details about pypi-server update](#More-details-about-pypi-server-update) + - [Client-Side Configurations](#Client-Side-Configurations) + - [Configuring pip](#Configuring-pip) + - [Configuring easy_install](#Configuring-easy_install) + - [Uploading Packages Remotely](#Uploading-Packages-Remotely) + - [Apache like Authentication ( htpasswd )](#Apache-like-Authentication) + - [Upload with setuptools](#Upload-with-setuptools) + - [Upload with twine](#Upload-with-twine) + - [Using the Docker Image](#Using-the-Docker-Image) + - [Alternative Installation methods](#Alternative-Installation-methods) + - [Installing the Very Latest Version](#Installing-the-Very-Latest-Version) + - [Recipes](#Recipes) + - [Managing the Package Directory](#Managing-the-Package-Directory) + - [Serving Thousands of Packages](#Serving-Thousands-of-Packages) + - [Managing Automated Startup](#Managing-Automated-Startup) + - [Running as a systemd service](#Running-as-a-systemd-service) + - [Launching through supervisor](#Launching-through-supervisor) + - [Running as a service with NSSM (Windows)](#Running-as-a-service-with-NSSM) + - [Using a Different WSGI Server](#Using-a-Different-WSGI-Server) + - [Apache](#Apache) + - [Gunicorn](#Gunicorn) + - [Paste](#Paste) + - [Behind a Reverse Proxy](#Behind-a-Reverse-Proxy) + - [Nginx](#Nginx) + - [Supporting HTTPS](#Supporting-HTTPS) + - [Traefik](#Traefik) + - [Utilizing the API](#Utilizing-the-API) + - [Using Ad-Hoc Authentication Providers](#Using-Ad-Hoc-Authentication-Providers) + - [Use with MicroPython](#Use-with-MicroPython) + - [Custom Health Check Endpoint](#Custom-Health-Check-Endpoint) + - [Configure a custom health check by CLI arguments](#Configure-a-custom-health-check-by-CLI-arguments) + - [Configure a custom health endpoint by script](#Configure-a-custom-health-endpoint-by-script) + - [Sources](#Sources) + - [Known Limitations](#known-limitations) + - [Similar Projects](#similar-projects) + - [Unmaintained or archived](#unmaintained-or-archived) + - [Related Projects](#related-projects) + - [License](#license) + +## [Quickstart Installation and Usage](#pypiserver) + +**pypiserver** works with Python 3.6+ and PyPy3. + +Older Python versions may still work, but they are not tested. + +For legacy Python versions, use **pypiserver-1.x** series. Note that these are +not officially supported, and will not receive bugfixes or new features. + +Tip + +The commands below work on a unix-like operating system with a posix shell. +The **'~'** character expands to user's home directory. + +If you're using Windows, you'll have to use their "Windows counterparts". +The same is true for the rest of this documentation. + +1. Install **pypiserver** with this command + +```shell + pip install pypiserver # Or: pypiserver[passlib,cache] + mkdir ~/packages # Copy packages into this directory. +``` + +See also [Alternative Installation methods](<>) + +2. Copy some packages into your **~/packages** folder and then + get your **pypiserver** up and running + +```shell + pypi-server run -p 8080 ~/packages & # Will listen to all IPs. +``` + +3. From the client computer, type this + +```shell + # Download and install hosted packages. + pip install --extra-index-url http://localhost:8080/simple/ ... + + # or + pip install --extra-index-url http://localhost:8080 ... + + # Search hosted packages. + pip search --index http://localhost:8080 ... + + # Note that pip search does not currently work with the /simple/ endpoint. +``` + +See also [Client-side configurations](#Client-Side-Configurations) for avoiding tedious typing. + +4. Enter **pypi-server -h** in the cmd-line to print a detailed usage message + +```text +usage: pypi-server [-h] [-v] [--log-file FILE] [--log-stream STREAM] + [--log-frmt FORMAT] [--hash-algo HASH_ALGO] + [--backend {auto,simple-dir,cached-dir}] [--version] + {run,update} ... + +start PyPI compatible package server serving packages from PACKAGES_DIRECTORY. If PACKAGES_DIRECTORY is not given on the command line, it uses the default ~/packages. pypiserver scans this directory recursively for packages. It skips packages and directories starting with a dot. Multiple package directories may be specified. + +positional arguments: + {run,update} + run Run pypiserver, serving packages from + PACKAGES_DIRECTORY + update Handle updates of packages managed by pypiserver. By + default, a pip command to update the packages is + printed to stdout for introspection or pipelining. See + the `-x` option for updating packages directly. + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Enable verbose logging; repeat for more verbosity. + --log-file FILE Write logging info into this FILE, as well as to + stdout or stderr, if configured. + --log-stream STREAM Log messages to the specified STREAM. Valid values are + stdout, stderr, and none + --log-frmt FORMAT The logging format-string. (see `logging.LogRecord` + class from standard python library) + --hash-algo HASH_ALGO + Any `hashlib` available algorithm to use for + generating fragments on package links. Can be disabled + with one of (0, no, off, false). + --backend {auto,simple-dir,cached-dir} + A backend implementation. Keep the default 'auto' to + automatically determine whether to activate caching or + not + --version show program's version number and exit + +Visit https://github.com/pypiserver/pypiserver for more information + + +``` + +### [More details about pypi server run](#pypiserver) + +Enter **pypi-server run -h** in the cmd-line to print a detailed usage + +```text +usage: pypi-server run [-h] [-v] [--log-file FILE] [--log-stream STREAM] + [--log-frmt FORMAT] [--hash-algo HASH_ALGO] + [--backend {auto,simple-dir,cached-dir}] [--version] + [-p PORT] [-i HOST] [-a AUTHENTICATE] + [-P PASSWORD_FILE] [--disable-fallback] + [--fallback-url FALLBACK_URL] + [--health-endpoint HEALTH_ENDPOINT] [--server METHOD] + [-o] [--welcome HTML_FILE] [--cache-control AGE] + [--log-req-frmt FORMAT] [--log-res-frmt FORMAT] + [--log-err-frmt FORMAT] + [package_directory [package_directory ...]] + +positional arguments: + package_directory The directory from which to serve packages. + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Enable verbose logging; repeat for more verbosity. + --log-file FILE Write logging info into this FILE, as well as to + stdout or stderr, if configured. + --log-stream STREAM Log messages to the specified STREAM. Valid values are + stdout, stderr, and none + --log-frmt FORMAT The logging format-string. (see `logging.LogRecord` + class from standard python library) + --hash-algo HASH_ALGO + Any `hashlib` available algorithm to use for + generating fragments on package links. Can be disabled + with one of (0, no, off, false). + --backend {auto,simple-dir,cached-dir} + A backend implementation. Keep the default 'auto' to + automatically determine whether to activate caching or + not + --version show program's version number and exit + -p PORT, --port PORT Listen on port PORT (default: 8080) + -i HOST, -H HOST, --interface HOST, --host HOST + Listen on interface INTERFACE (default: 0.0.0.0) + -a AUTHENTICATE, --authenticate AUTHENTICATE + Comma-separated list of (case-insensitive) actions to + authenticate (options: download, list, update; + default: update). + + Any actions not specified are not authenticated, so + to authenticate downloads and updates, but allow + unauthenticated viewing of the package list, you would + use: + + pypi-server -a 'download, update' -P + ./my_passwords.htaccess + + To disable authentication, use: + + pypi-server -a . -P . + + See the `-P` option for configuring users and + passwords. + + Note that when uploads are not protected, the + `register` command is not necessary, but `~/.pypirc` + still needs username and password fields, even if + bogus. + -P PASSWORD_FILE, --passwords PASSWORD_FILE + Use an apache htpasswd file PASSWORD_FILE to set + usernames and passwords for authentication. + + To allow unauthorized access, use: + + pypi-server -a . -P . + + --disable-fallback Disable the default redirect to PyPI for packages not + found in the local index. + --fallback-url FALLBACK_URL + Redirect to FALLBACK_URL for packages not found in the + local index. + --health-endpoint HEALTH_ENDPOINT + Configure a custom liveness endpoint. It always + returns 200 Ok if the service is up. Otherwise, it + means that the service is not responsive. + --server METHOD Use METHOD to run the server. Valid values include + paste, cherrypy, twisted, gunicorn, gevent, wsgiref, + and auto. The default is to use "auto", which chooses + one of paste, cherrypy, twisted, or wsgiref. + -o, --overwrite Allow overwriting existing package files during + upload. + --welcome HTML_FILE Use the contents of HTML_FILE as a custom welcome + message on the home page. + --cache-control AGE Add "Cache-Control: max-age=AGE" header to package + downloads. Pip 6+ requires this for caching.AGE is + specified in seconds. + --log-req-frmt FORMAT + A format-string selecting Http-Request properties to + log; set to '%s' to see them all. + --log-res-frmt FORMAT + A format-string selecting Http-Response properties to + log; set to '%s' to see them all. + --log-err-frmt FORMAT + A format-string selecting Http-Error properties to + log; set to '%s' to see them all. + +``` + +### [More details about pypi-server update](#pypiserver) + +More details about **pypi-server update** + +```text +usage: pypi-server update [-h] [-v] [--log-file FILE] [--log-stream STREAM] + [--log-frmt FORMAT] [--hash-algo HASH_ALGO] + [--backend {auto,simple-dir,cached-dir}] [--version] + [-x] [-d DOWNLOAD_DIRECTORY] [-u] + [--blacklist-file IGNORELIST_FILE] + [package_directory [package_directory ...]] + +positional arguments: + package_directory The directory from which to serve packages. + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Enable verbose logging; repeat for more verbosity. + --log-file FILE Write logging info into this FILE, as well as to + stdout or stderr, if configured. + --log-stream STREAM Log messages to the specified STREAM. Valid values are + stdout, stderr, and none + --log-frmt FORMAT The logging format-string. (see `logging.LogRecord` + class from standard python library) + --hash-algo HASH_ALGO + Any `hashlib` available algorithm to use for + generating fragments on package links. Can be disabled + with one of (0, no, off, false). + --backend {auto,simple-dir,cached-dir} + A backend implementation. Keep the default 'auto' to + automatically determine whether to activate caching or + not + --version show program's version number and exit + -x, --execute Execute the pip commands rather than printing to + stdout + -d DOWNLOAD_DIRECTORY, --download-directory DOWNLOAD_DIRECTORY + Specify a directory where packages updates will be + downloaded. The default behavior is to use the + directory which contains the package being updated. + -u, --allow-unstable Allow updating to unstable versions (alpha, beta, rc, + dev, etc.) + --blacklist-file IGNORELIST_FILE, --ignorelist-file IGNORELIST_FILE + Don't update packages listed in this file (one package + name per line, without versions, '#' comments + honored). This can be useful if you upload private + packages into pypiserver, but also keep a mirror of + public packages that you regularly update. Attempting + to pull an update of a private package from `pypi.org` + might pose a security risk - e.g. a malicious user + might publish a higher version of the private package, + containing arbitrary code. + + +``` + +## [Client-Side Configurations](#pypiserver) + +Always specifying the pypi url on the command line is a bit +cumbersome. Since **pypiserver** redirects **pip/easy_install** to the +**pypi.org** index if it doesn't have a requested package, it is a +good idea to configure them to always use your local pypi index. + +### [Configuring pip](#pypiserver) + +For **pip** command this can be done by setting the environment variable +**[PIP_EXTRA_INDEX_URL](https://packaging.python.org/en/latest/guides/hosting-your-own-index/)** in your **.bashr/.profile/.zshrc** + +```shell +export PIP_EXTRA_INDEX_URL=http://localhost:8080/simple/ +``` + +or by adding the following lines to **~/.pip/pip.conf** + +```shell +[global] +extra-index-url = http://localhost:8080/simple/ +``` + +Note + +If you have installed **pypiserver** on a remote url without *https* +you will receive an "untrusted" warning from *pip*, urging you to append +the **--trusted-host** option. You can also include this option permanently +in your configuration-files or environment variables. + +### [Configuring easy_install](#pypiserver) + +For **easy_install** command you may set the following configuration in +**~/.pydistutils.cfg** + +```shell +[easy_install] +index_url = http://localhost:8080/simple/ +``` + +### [Uploading Packages Remotely](#pypiserver) + +Instead of copying packages directly to the server's folder (i.e. with **scp**), +you may use python tools for the task, e.g. **python setup.py upload**. +In that case, **pypiserver** is responsible for authenticating the upload-requests. + +Note + +We strongly advise to password-protected your uploads! + +It is possible to disable authentication for uploads (e.g. in intranets). +To avoid lazy security decisions, read help for **-P** and **-a** options. + +#### [Apache Like Authentication (htpasswd)](#pypiserver) + +1. First make sure you have the **passlib** module installed (note that + **passlib>=1.6** is required), which is needed for parsing the Apache + *htpasswd* file specified by the **-P**, **--passwords** option + (see next steps) + +```shell + pip install passlib +``` + +2. Create the Apache **htpasswd** file with at least one user/password pair + with this command (you'll be prompted for a password) + +```shell + htpasswd -sc htpasswd.txt + +``` + +Tip + +Read this [SO](http://serverfault.com/questions/152950/how-to-create-and-edit-htaccess-and-htpasswd-locally-on-my-computer-and-then-u) question for running `htpasswd` cmd under *Windows* +or if you have bogus passwords that you don't care because they are for +an internal service (which is still "bad", from a security perspective...) +you may use this [public service](http://www.htaccesstools.com/htpasswd-generator/) + +Tip + +When accessing pypiserver via the api, alternate authentication +methods are available via the **auther** config flag. Any callable +returning a boolean can be passed through to the pypiserver config in +order to provide custom authentication. For example, to configure +pypiserver to authenticate using the [python-pam](https://pypi.org/project/python-pam/) + +```shell + import pam + pypiserver.default_config(auther=pam.authenticate) + +``` + +Please see `Using Ad-hoc authentication providers`\_ for more information. + +3. You need to restart the server with the **-P** option only once + (but user/password pairs can later be added or updated on the fly) + +```shell + ./pypi-server run -p 8080 -P htpasswd.txt ~/packages & +``` + +#### [Upload with setuptools](#pypiserver) + +1. On client-side, edit or create a **~/.pypirc** file with a similar content: + +```shell + [distutils] + index-servers = + pypi + local + + [pypi] + username: + password: + + [local] + repository: http://localhost:8080 + username: + password: +``` + +2. Then from within the directory of the python-project you wish to upload, + issue this command: + +```shell + python setup.py sdist upload -r local +``` + +#### [Upload with twine](#pypiserver) + +To avoid storing you passwords on disk, in clear text, you may either: + +- use the **register** *setuptools*'s command with the **-r** option, + like that + +```shell + python setup.py sdist register -r local upload -r local +``` + +- use *twine* library, which + breaks the procedure in two steps. In addition, it supports signing + your files with PGP-Signatures and uploading the generated *.asc* files + to **pypiserver**:: + +```shell + twine upload -r local --sign -identity user_name ./foo-1.zip +``` + +## [Using the Docker Image](#pypiserver) + +Starting with version 1.2.5, official Docker images will be built for each +push to master, each dev, alpha, or beta release, and each final release. +The most recent full release will always be available under the tag **latest**, +and the current master branch will always be available under the tag +**unstable**. + +You can always check to see what tags are currently available at our +[*Docker Repo*](https://hub.docker.com/r/pypiserver/pypiserver/tags/). + +To run the most recent release of **pypiserver** with Docker, simply + +```shell + docker run pypiserver/pypiserver:latest run +``` + +This starts **pypiserver** serving packages from the **/data/packages** +directory inside the container, listening on the container port 8080. + +The container takes all the same arguments as the normal **pypi-server** +executable, with the exception of the internal container port (**-p**), +which will always be 8080. + +Of course, just running a container isn't that interesting. To map +port 80 on the host to port 8080 on the container:: + +```shell + docker run -p 80:8080 pypiserver/pypiserver:latest run +``` + +You can now access your **pypiserver** at **localhost:80** in a web browser. + +To serve packages from a directory on the host, e.g. **~/packages** + +```shell + docker run -p 80:8080 -v ~/packages:/data/packages pypiserver/pypiserver:latest run +``` + +To authenticate against a local **.htpasswd** file:: + +```shell + docker run -p 80:8080 -v ~/.htpasswd:/data/.htpasswd pypiserver/pypiserver:latest run -P .htpasswd packages +``` + +You can also specify **pypiserver** to run as a Docker service using a +composefile. An example composefile is [provided](https://github.com/pypiserver/pypiserver/blob/master/docker-compose.yml) + +## [Alternative Installation Methods](#pypiserver) + +When trying the methods below, first use the following command to check whether +previous versions of **pypiserver** already exist, and (optionally) uninstall them:: + +```shell +# VERSION-CHECK: Fails if not installed. +pypi-server --version + +# UNINSTALL: Invoke again until it fails. +pip uninstall pypiserver +``` + +### [Installing the Very Latest Version](#pypiserver) + +In case the latest version in *pypi* is a pre-release, you have to use +*pip*'s *--pre* option. And to update an existing installation combine it +with `--ignore-installed` + +```shell +pip install pypiserver --pre -I +``` + +You can even install the latest **pypiserver** directly from *github* with the +following command, assuming you have *git* installed on your **PATH** + +```shell +pip install git+git://github.com/pypiserver/pypiserver.git +``` + +## [Recipes](#pypiserver) + +### [Managing the Package Directory](#pypiserver) + +The **pypi-server** command has the **update** command that searches for updates of +available packages. It scans the package directory for available +packages and searches on pypi.org for updates. Without further +options **pypi-server update** will just print a list of commands which must +be run in order to get the latest version of each package. Output +looks like + +```shell + $ ./pypi-server update + checking 106 packages for newer version + + .........u.e...........e..u............. + .....e..............................e... + .......................... + + no releases found on pypi for PyXML, Pymacs, mercurial, setuptools + + # update raven from 1.4.3 to 1.4.4 + pip -q install --no-deps --extra-index-url https://pypi.org/simple/ -d /home/ralf/packages/mirror raven==1.4.4 + + # update greenlet from 0.3.3 to 0.3.4 + pip -q install --no-deps --extra-index-url https://pypi.org/simple/ -d /home/ralf/packages/mirror greenlet==0.3.4 +``` + +It first prints for each package a single character after checking the +available versions on pypi. A dot(.) means the package is up-to-date, **'u'** +means the package can be updated and **'e'** means the list of releases on +pypi is empty. After that it shows a *pip* command line which can be used +to update a one package. Either copy and paste that or run +**pypi-server update -x** in order to really execute those commands. You need +to have *pip* installed for that to work however. + +Specifying an additional **-u** option will also allow alpha, beta and +release candidates to be downloaded. Without this option these +releases won't be considered. + +### [Serving Thousands of Packages](#pypiserver) + +By default, **pypiserver** scans the entire packages directory each time an +incoming HTTP request occurs. This isn't a problem for a small number of +packages, but causes noticeable slow-downs when serving thousands of packages. + +If you run into this problem, significant speedups can be gained by enabling +pypiserver's directory caching functionality. The only requirement is to +install the **watchdog** package, or it can be installed during **pypiserver** +installation, by specifying the **cache** extras option:: + +```shell + pip install pypiserver[cache] +``` + +Additional speedups can be obtained by using your webserver's builtin +caching functionality. For example, if you are using `nginx` as a +reverse-proxy as described below in `Behind a reverse proxy`, you can +easily enable caching. For example, to allow nginx to cache up to +10 gigabytes of data for up to 1 hour:: + +```shell + proxy_cache_path /data/nginx/cache + levels=1:2 + keys_zone=pypiserver_cache:10m + max_size=10g + inactive=60m + use_temp_path=off; + + server { + # ... + location / { + proxy_cache pypiserver_cache; + proxy_pass http://localhost:8080; + } + } + +``` + +Using webserver caching is especially helpful if you have high request +volume. Using nginx caching, a real-world pypiserver installation was +able to easily support over 1000 package downloads/min at peak load. + +### [Managing Automated Startup](#pypiserver) + +There are a variety of options for handling the automated starting of +pypiserver upon system startup. Two of the most common are *systemd* and +*supervisor* for linux systems. For windows creating services with scripts isn't +an easy task without a third party tool such as *NSSM*. + +#### [Running As a systemd Service](#pypiserver) + +**systemd** is installed by default on most modern Linux systems and as such, +it is an excellent option for managing the pypiserver process. An example +config file for **systemd** can be seen below + +```shell + [Unit] + Description=A minimal PyPI server for use with pip/easy_install. + After=network.target + + [Service] + Type=simple + # systemd requires absolute path here too. + PIDFile=/var/run/pypiserver.pid + User=www-data + Group=www-data + + ExecStart=/usr/local/bin/pypi-server run -p 8080 -a update,download --log-file /var/log/pypiserver.log -P /etc/nginx/.htpasswd /var/www/pypi + ExecStop=/bin/kill -TERM $MAINPID + ExecReload=/bin/kill -HUP $MAINPID + Restart=always + + WorkingDirectory=/var/www/pypi + + TimeoutStartSec=3 + RestartSec=5 + + [Install] + WantedBy=multi-user.target +``` + +Adjusting the paths and adding this file as **pypiserver.service** into your +**systemd/system** directory will allow management of the pypiserver process with +**systemctl**, e.g. **systemctl start pypiserver**. + +More useful information about *systemd* can be found at +https://www.digitalocean.com/community/tutorials/how-to-use-systemctl-to-manage-systemd-services-and-units + +#### [Launching through supervisor](#pypiserver) + +[supervisor](http://supervisord.org/) has the benefit of being a pure python +package and as such, it provides excellent cross-platform support for process +management. An example configuration file for **supervisor** is given below + +```shell + [program:pypi] + command=/home/pypi/pypi-venv/bin/pypi-server run -p 7001 -P /home/pypi/.htpasswd /home/pypi/packages + directory=/home/pypi + user=pypi + autostart=true + autorestart=true + stderr_logfile=/var/log/pypiserver.err.log + stdout_logfile=/var/log/pypiserver.out.log +``` + +From there, the process can be managed via **supervisord** using **supervisorctl**. + +#### [Running As a service with NSSM](#pypiserver) + +For Windows download NSSM from https://nssm.cc unzip to a desired location such as Program Files. Decide whether you are going +to use win32 or win64, and add that exe to environment PATH. + +Create a start_pypiserver.bat + +```shell + pypi-server run -p 8080 C:\Path\To\Packages & +``` + +Test the batch file by running it first before creating the service. Make sure you can access +the server remotely, and install packages. If you can, proceed, if not troubleshoot until you can. +This will ensure you know the server works, before adding NSSM into the mix. + +From the command prompt + +```shell + nssm install pypiserver +``` + +This command will launch a NSSM gui application + +```shell + Path: C:\Path\To\start_pypiserver.bat + Startup directory: Auto generates when selecting path + Service name: pypiserver +``` + +There are more tabs, but that is the basic setup. If the service needs to be running with a certain +login credentials, make sure you enter those credentials in the logon tab. + +Start the service + +```shell + nssm start pypiserver +``` + +Other useful commands + +```shell + nssm --help + nssm stop + nssm restart + nssm status + +``` + +For detailed information please visit https://nssm.cc + +### [Using a Different WSGI Server](#pypiserver) + +- The **bottle** web-server which supports many WSGI-servers, among others, + **paste**, **cherrypy**, **twisted** and **wsgiref** (part of Python); you select + them using the **--server** flag. + +- You may view all supported WSGI servers using the following interactive code + +```python + >>> from pypiserver import bottle + >>> list(bottle.server_names.keys()) + ['cgi', 'gunicorn', 'cherrypy', 'eventlet', 'tornado', 'geventSocketIO', + 'rocket', 'diesel', 'twisted', 'wsgiref', 'fapws3', 'bjoern', 'gevent', + 'meinheld', 'auto', 'aiohttp', 'flup', 'gae', 'paste', 'waitress'] + +``` + +- If none of the above servers matches your needs, invoke just the + **pypiserver:app()** method which returns the internal WSGI-app WITHOUT + starting-up a server - you may then send it to any WSGI server you like. + Read also the [Utilizing the API](#utilizing-the-api) section. + +- Some examples are given below - you may find more details in [bottle + site](http://bottlepy.org/docs/dev/deployment.html#switching-the-server-backend%3E). + +#### [Apache](#pypiserver) + +To use your *Apache2* with **pypiserver**, prefer to utilize **mod_wsgi** as +explained in [bottle's documentation](http://bottlepy.org/docs/dev/deployment.html#apache-mod-wsgi%3E). + +Note +If you choose instead to go with **mod_proxy**, mind that you may bump into problems +with the prefix-path (see [#155](https://github.com/pypiserver/pypiserver/issues/155%3E)). + +1. Adapt and place the following *Apache* configuration either into top-level scope, + or inside some **** (contributed by Thomas Waldmann): + +```shell + WSGIScriptAlias / /yoursite/wsgi/pypiserver-wsgi.py + WSGIDaemonProcess pypisrv user=pypisrv group=pypisrv umask=0007 \ + processes=1 threads=5 maximum-requests=500 \ + display-name=wsgi-pypisrv inactivity-timeout=300 + WSGIProcessGroup pypisrv + WSGIPassAuthorization On # Required for authentication (https://github.com/pypiserver/pypiserver/issues/288) + + + Require all granted + +``` + +or if using older **Apache \< 2.4**, substitute the last part with this:: + +```shell + + Order deny,allow + Allow from all + +``` + +2. Then create the **/yoursite/cfg/pypiserver.wsgi** file and make sure that + the **user** and **group** of the **WSGIDaemonProcess** directive + (**pypisrv:pypisrv** in the example) have the read permission on it + +```python + + import pypiserver + + conf = pypiserver.default_config( + root = "/yoursite/packages", + password_file = "/yoursite/htpasswd", ) + application = pypiserver.app(**conf) + +``` + +Tip +If you have installed **pypiserver** in a virtualenv, follow **mod_wsgi**'s +[instructions](http://modwsgi.readthedocs.io/en/develop/user-guides/virtual-environments.html) +and prepend the python code above with the following + +```python + import site + + site.addsitedir('/yoursite/venv/lib/pythonX.X/site-packages') +``` + +Note +For security reasons, notice that the **Directory** directive grants access +to a directory holding the **wsgi** start-up script, alone; nothing else. + +Note +To enable HTTPS support on Apache, configure the directive that contains the +WSGI configuration to use SSL. + +#### [gunicorn](#pypiserver) + +The following command uses **gunicorn** to start **pypiserver** + +```shell + gunicorn -w4 'pypiserver:app(root="/home/ralf/packages")' +``` + +or when using multiple roots + +```shell + gunicorn -w4 'pypiserver:app(root=["/home/ralf/packages", "/home/ralf/experimental"])' +``` + +#### [paste](#pypiserver) + +[paste](http://pythonpaste.org) allows to run multiple WSGI applications +under different URL paths. Therefore, it is possible to serve different set +of packages on different paths. + +The following example **paste.ini** could be used to serve stable and +unstable packages on different paths + +```shell + [composite:main] + use = egg:Paste#urlmap + /unstable/ = unstable + / = stable + + [app:stable] + use = egg:pypiserver#main + root = ~/stable-packages + + [app:unstable] + use = egg:pypiserver#main + root = ~/stable-packages + ~/unstable-packages + + [server:main] + use = egg:gunicorn#main + host = 0.0.0.0 + port = 9000 + workers = 5 + accesslog = - + +``` + +Note +You need to install some more dependencies for this to work, like:: + +```shell + pip install paste pastedeploy gunicorn pypiserver +``` + +The server can then start with + +```shell + gunicorn_paster paste.ini +``` + +### [Behind a Reverse Proxy](#pypiserver) + +You can run **pypiserver** behind a reverse proxy as well. + +#### [Nginx](#pypiserver) + +Extend your nginx configuration + +```shell + upstream pypi { + server pypiserver.example.com:12345 fail_timeout=0; + } + + server { + server_name myproxy.example.com; + + location / { + proxy_set_header Host $host:$server_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://pypi; + } + } + +``` + +As of pypiserver 1.3, you may also use the `X-Forwarded-Host` header in your +reverse proxy config to enable changing the base URL. For example if you +want to host pypiserver under a particular path on your server + +```shell + upstream pypi { + server localhost:8000; + } + + server { + location /pypi/ { + proxy_set_header X-Forwarded-Host $host:$server_port/pypi; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://pypi; + } + } +``` + +#### [Supporting HTTPS](#pypiserver) + +Using a reverse proxy is the preferred way of getting pypiserver behind +HTTPS. For example, to put pypiserver behind HTTPS on port 443, with +automatic HTTP redirection, using `nginx` + +```shell + upstream pypi { + server localhost:8000; + } + + server { + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; + } + + server { + listen 443 ssl; + server_name pypiserver.example.com; + + ssl_certificate /etc/star.example.com.crt; + ssl_certificate_key /etc/star.example.com.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + location / { + proxy_set_header Host $host:$server_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass http://pypi; + } + } + +``` + +Please see [nginx's HTTPS docs for more details](http://nginx.org/en/docs/http/configuring_https_servers.html). + +Getting and keeping your certificates up-to-date can be simplified using, +for example, using [certbot and letsencrypt](https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-18-04%3E). + +#### [Traefik](#pypiserver) + +It is also possible to use [Traefik](https://docs.traefik.io/) to put pypiserver behind HTTPS on port 443, with +automatic HTTP redirection using Docker Compose. Please see the provided [docker-compose.yml](https://github.com/pypiserver/pypiserver/blob/master/docker-compose.yml) example for more information. + +### [Utilizing the API](#pypiserver) + +In order to enable ad-hoc authentication-providers or to use WSGI-servers +not supported by *bottle* out-of-the-box, you needed to launch **pypiserver** +via its API. + +- The main entry-point for configuring **pypiserver** is the [pypiserver:app()](https://github.com/pypiserver/pypiserver/blob/master/pypiserver/__init__.py#L116) + function. This function returns the internal WSGI-app that you my then + send to any WSGI-server you like. + +- To get all **pypiserver:app()** keywords and their explanations, read the + function [pypiserver:default_config()](https://github.com/pypiserver/pypiserver/blob/master/pypiserver/__init__.py#L35) + +- Finally, to fire-up a WSGI-server with the configured app, invoke + the **bottle:run(app, host, port, server)** function. + Note that **pypiserver** ships with its own copy of *bottle*; to use it, + import it like that: **from pypiserver import bottle** + +#### [Using Ad-Hoc Authentication Providers](#pypiserver) + +The **auther** keyword of **pypiserver:app()** function maybe set only using +the API. This can be any callable that returns a boolean when passed +the *username* and the *password* for a given request. + +For example, to authenticate users based on the **/etc/passwd** file under Unix, +you may delegate such decisions to the [python-pam](https://pypi.org/project/python-pam/) library by following +these steps: + +1. Ensure **python-pam** module is installed + + pip install python-pam + +1. Create a python-script along these lines + +```shell + $ cat > pypiserver-start.py + import pypiserver + from pypiserver import bottle + import pam + app = pypiserver.app(root='./packages', auther=pam.authenticate) + bottle.run(app=app, host='0.0.0.0', port=80, server='auto') + + [Ctrl+ D] +``` + +3. Invoke the python-script to start-up **pypiserver** + +```shell + $ python pypiserver-start.py +``` + +Note +The [python-pam](https://pypi.org/project/python-pam/) module, requires *read* access to **/etc/shadow** file; +you may add the user under which **pypiserver** runs into the *shadow* +group, with a command like this: **sudo usermod -a -G shadow pypy-user**. + +### [Use with MicroPython](#pypiserver) + +The MicroPython interpreter for embedded devices can install packages with the +module **upip.py**. The module uses a specialized json-endpoint to retrieve +package information. This endpoint is supported by **pypiserver**. + +It can be tested with the UNIX port of **micropython** + +```shell + cd micropython + ports/unix/micropython -m tools.upip install -i http://my-server:8080 -p /tmp/mymodules micropython-foobar + +``` + +Installing packages from the REPL of an embedded device works in this way: + +```python + import network + import upip + + sta_if = network.WLAN(network.STA_IF) + sta_if.active(True) + sta_if.connect('', '') + upip.index_urls = ["http://my-server:8080"] + upip.install("micropython-foobar") +``` + +Further information on micropython-packaging can be found here: https://docs.micropython.org/en/latest/reference/packages.html + +### [Custom Health Check Endpoint](#pypiserver) + +**pypiserver** provides a default health endpoint at **/health**. It always returns +**200 Ok** if the service is up. Otherwise, it means that the service is not responsive. + +In addition, **pypiserver** allows users to customize the health endpoint. +Alphanumeric characters, hyphens, forward slashes and underscores are allowed +and the endpoint should not overlap with any existing routes. +Valid examples: **/healthz**, **/health/live-1**, **/api_health**, **/action/health** + +#### [Configure a custom health endpoint by CLI arguments](#pypiserver) + +Run pypiserver with **--health-endpoint** argument: + +```shell + pypi-server run --health-endpoint /action/health +``` + +#### [Configure a custom health endpoint by script](#pypiserver) + +````python + import pypiserver + from pypiserver import bottle + app = pypiserver.app(root="./packages", health_endpoint="/action/health") + bottle.run(app=app, host=" + +```python + import pypiserver + from pypiserver import bottle + app = pypiserver.app(root="./packages", health_endpoint="/action/health") + bottle.run(app=app, host="0.0.0.0", port=8080, server="auto") +```` + +Try **curl http://localhost:8080/action/health** + +## [Sources](#pypiserver) + +To create a copy of the repository, use + +```shell + git clone https://github.com/pypiserver/pypiserver.git + cd pypiserver +``` + +To receive any later changes, in the above folder use: + +```shell + git pull +``` + +## [Known Limitations](#pypiserver) + +**pypiserver** does not implement the full API as seen on [PyPI](https://pypi.org/). It +implements just enough to make **easy_install**, **pip install**, and +**search** work. + +The following limitations are known: + +- Command **pypi -U** that compares uploaded packages with *pypi* to see if + they are outdated, does not respect a http-proxy environment variable + (see [#19](https://github.com/pypiserver/pypiserver/issues/19). +- It accepts documentation uploads but does not save them to + disk (see [#47](https://github.com/pypiserver/pypiserver/issues/47) for a + discussion) +- It does not handle misspelled packages as *pypi-repo* does, + therefore it is suggested to use it with **--extra-index-url** instead + of **--index-url** (see [#38](https://github.com/pypiserver/pypiserver/issues/38%3E)). + +Please use Github's [bugtracker](https://github.com/pypiserver/pypiserver/issues%3E) +for other bugs you find. + +## [Similar Projects](#pypiserver) + +There are lots of other projects, which allow you to run your own +PyPI server. If **pypiserver** doesn't work for you, the following are +among the most popular alternatives: + +- [devpi-server](https://pypi.org/project/devpi/): + a reliable fast pypi.org caching server, part of + the comprehensive [github-style pypi index server and packaging meta tool](https://pypi.org/project/devpi/). + (version: 2.1.4, access date: 8/3/2015) + +- Check this SO question: [How to roll my own pypi](http://stackoverflow.com/questions/1235331/how-to-roll-my-own-pypi) + +### [Unmaintained or archived](#pypiserver) + +These projects were once alternatives to pypiserver but are now either unmaintained or archived. + +- [pip2pi](https://github.com/wolever/pip2pi) + a simple cmd-line tool that builds a PyPI-compatible local folder from pip requirements + +- [flask-pypi-proxy](http://flask-pypi-proxy.readthedocs.org/) + A proxy for PyPI that also enables uploading custom packages. + +## [Related Software](#pypiserver) + +Though not direct alternatives for **pypiserver**'s use as an index +server, the following is a list of related software projects that you +may want to familiarize with: + +- [pypi-uploader](https://pypi.org/project/pypi-uploader/): + A command-line utility to upload packages to your **pypiserver** from pypi without + having to store them locally first. + +- [twine](https://pypi.org/project/twine/): + A command-line utility for interacting with PyPI or **pypiserver**. + +- [warehouse](https://github.com/pypa/warehouse/): + the software that powers [PyPI](https://pypi.org/) itself. It is not generally intended to + be run by end-users. + +# Licensing + +**pypiserver** contains a copy of [bottle](http://bottlepy.org/) which is available under the +MIT license, and the remaining part is distributed under the zlib/libpng license. +See the **LICENSE.txt** file. diff --git a/README.rst b/README.rst deleted file mode 100644 index 39036f2..0000000 --- a/README.rst +++ /dev/null @@ -1,1078 +0,0 @@ -.. -*- mode: rst; coding: utf-8 -*- - -.. image:: docs/__resources__/pypiserver_logo.png - :width: 300 px - :align: center - -============================================================================== -pypiserver - minimal PyPI server for use with pip/easy_install -============================================================================== -|pypi-ver| |test-status| |dependencies| |python-ver| |proj-license| - -:Version: 1.5.2 -:Date: 2023-07-30 -:Source: https://github.com/pypiserver/pypiserver -:PyPI: https://pypi.org/project/pypiserver/ -:Tests: https://github.com/pypiserver/pypiserver/actions -:Maintainers: | Kostis Anagnostopoulos , - | Matthew Planchard , - | Dmitrii Orlov , - | **Someone new?** We are looking for new maintainers! - -:License: zlib/libpng + MIT -:Community: https://pypiserver.zulipchat.com - -Chat with us on `Zulip `_! - -``pypiserver`` is a minimal PyPI_ compatible server for ``pip`` or ``easy_install``. -It is based on bottle_ and serves packages from regular directories. -Wheels, bdists, eggs and accompanying PGP-signatures can be uploaded -either with ``pip``, ``setuptools``, ``twine``, ``pypi-uploader``, or simply copied -with ``scp``. - -.. note:: - The official software powering PyPI_ is Warehouse_. However, Warehouse_ - is fairly specialized to be ``pypi.org``'s own software, and should not - be used in other contexts. In particular, it does not officially support - being used as a custom package index by users wishing to serve their own - packages. - - ``pypiserver`` implements the same interfaces as `PyPI`_, allowing - standard Python packaging tooling such as ``pip`` and ``twine`` to - interact with it as a package index just as they would with PyPI_, while - making it much easier to get a running index server. - -.. contents:: Table of Contents - :backlinks: top - - -Quickstart: Installation and Usage -================================== - -``pypiserver`` works with Python 3.6+ and PyPy3. - -Older Python versions may still work, but they are not tested. - -For legacy Python versions, use ``pypiserver-1.x`` series. Note that these are -not officially supported, and will not receive bugfixes or new features. - -.. Tip:: - The commands below work on a unix-like operating system with a posix shell. - The ``'~'`` character expands to user's home directory. - - If you're using Windows, you'll have to use their "Windows counterparts". - The same is true for the rest of this documentation. - -1. Install ``pypiserver`` with this command:: - - pip install pypiserver # Or: pypiserver[passlib,cache] - mkdir ~/packages # Copy packages into this directory. - - See also `Alternative Installation methods`_. - -2. Copy some packages into your ``~/packages`` folder and then - get your ``pypiserver`` up and running:: - - pypi-server run -p 8080 ~/packages & # Will listen to all IPs. - -3. From the client computer, type this:: - - # Download and install hosted packages. - pip install --extra-index-url http://localhost:8080/simple/ ... - - # or - pip install --extra-index-url http://localhost:8080 ... - - # Search hosted packages. - pip search --index http://localhost:8080 ... - - # Note that pip search does not currently work with the /simple/ endpoint. - - See also `Client-side configurations`_ for avoiding tedious typing. - -4. Enter ``pypi-server -h`` in the cmd-line to print a detailed usage message:: - - start PyPI compatible package server serving packages from PACKAGES_DIRECTORY. If PACKAGES_DIRECTORY is not given on the command line, it uses the default ~/packages. pypiserver scans this directory recursively for packages. It skips packages and directories starting with a dot. Multiple package directories may be specified. - - positional arguments: - {run,update} - run Run pypiserver, serving packages from - PACKAGES_DIRECTORY - update Handle updates of packages managed by pypiserver. By - default, a pip command to update the packages is - printed to stdout for introspection or pipelining. See - the `-x` option for updating packages directly. - - options: - -h, --help show this help message and exit - -v, --verbose Enable verbose logging; repeat for more verbosity. - --log-file FILE Write logging info into this FILE, as well as to - stdout or stderr, if configured. - --log-stream STREAM Log messages to the specified STREAM. Valid values are - stdout, stderr, and none - --log-frmt FORMAT The logging format-string. (see `logging.LogRecord` - class from standard python library) - --hash-algo HASH_ALGO - Any `hashlib` available algorithm to use for - generating fragments on package links. Can be disabled - with one of (0, no, off, false). - --backend {auto,simple-dir,cached-dir} - A backend implementation. Keep the default 'auto' to - automatically determine whether to activate caching or - not - --version show program's version number and exit - - Visit https://github.com/pypiserver/pypiserver for more information - -More details about ``pypi-server run`` --------------------------------------- - -Enter ``pypi-server run -h`` in the cmd-line to print a detailed usage -message *about starting the server*:: - - usage: pypi-server run [-h] [-v] [--log-file FILE] [--log-stream STREAM] [--log-frmt FORMAT] [--hash-algo HASH_ALGO] - [--backend {auto,simple-dir,cached-dir}] [--version] [-p PORT] [-i HOST] [-a AUTHENTICATE] [-P PASSWORD_FILE] [--disable-fallback] - [--fallback-url FALLBACK_URL] [--health-endpoint HEALTH_ENDPOINT] [--server METHOD] [-o] [--welcome HTML_FILE] - [--cache-control AGE] [--log-req-frmt FORMAT] [--log-res-frmt FORMAT] [--log-err-frmt FORMAT] - [package_directory [package_directory ...]] - - positional arguments: - package_directory The directory from which to serve packages. - - optional arguments: - -h, --help show this help message and exit - -v, --verbose Enable verbose logging; repeat for more verbosity. - --log-file FILE Write logging info into this FILE, as well as to stdout or stderr, if configured. - --log-stream STREAM Log messages to the specified STREAM. Valid values are stdout, stderr, and none - --log-frmt FORMAT The logging format-string. (see `logging.LogRecord` class from standard python library) - --hash-algo HASH_ALGO - Any `hashlib` available algorithm to use for generating fragments on package links. Can be disabled with one of (0, no, off, - false). - --backend {auto,simple-dir,cached-dir} - A backend implementation. Keep the default 'auto' to automatically determine whether to activate caching or not - --version show program's version number and exit - -p PORT, --port PORT Listen on port PORT (default: 8080) - -i HOST, -H HOST, --interface HOST, --host HOST - Listen on interface INTERFACE (default: 0.0.0.0) - -a AUTHENTICATE, --authenticate AUTHENTICATE - Comma-separated list of (case-insensitive) actions to authenticate (options: download, list, update; default: update). - - Any actions not specified are not authenticated, so to authenticate downloads and updates, but allow unauthenticated viewing of - the package list, you would use: - - pypi-server -a 'download, update' -P ./my_passwords.htaccess - - To disable authentication, use: - - pypi-server -a . -P . - - See the `-P` option for configuring users and passwords. - - Note that when uploads are not protected, the `register` command is not necessary, but `~/.pypirc` still needs username and - password fields, even if bogus. - -P PASSWORD_FILE, --passwords PASSWORD_FILE - Use an apache htpasswd file PASSWORD_FILE to set usernames and passwords for authentication. - - To allow unauthorized access, use: - - pypi-server -a . -P . - - --disable-fallback Disable the default redirect to PyPI for packages not found in the local index. - --fallback-url FALLBACK_URL - Redirect to FALLBACK_URL for packages not found in the local index. - --health-endpoint HEALTH_ENDPOINT - Configure a custom liveness endpoint. - It always returns 200 Ok if the service is up. - Otherwise, it means that the service is not responsive. - --server METHOD Use METHOD to run the server. Valid values include paste, cherrypy, twisted, gunicorn, gevent, wsgiref, and auto. The default is to - use "auto", which chooses one of paste, cherrypy, twisted, or wsgiref. - -o, --overwrite Allow overwriting existing package files during upload. - --welcome HTML_FILE Use the contents of HTML_FILE as a custom welcome message on the home page. - --cache-control AGE Add "Cache-Control: max-age=AGE" header to package downloads. Pip 6+ requires this for caching. AGE is specified in seconds. - --log-req-frmt FORMAT - A format-string selecting Http-Request properties to log; set to '%s' to see them all. - --log-res-frmt FORMAT - A format-string selecting Http-Response properties to log; set to '%s' to see them all. - --log-err-frmt FORMAT - A format-string selecting Http-Error properties to log; set to '%s' to see them all. - -More details about ``pypi-server update`` ------------------------------------------ - -Enter ``pypi-server update -h`` in the cmd-line to print a detailed usage -message *about updating the managed packages*:: - - usage: pypi-server update [-h] [-v] [--log-file FILE] [--log-stream STREAM] [--log-frmt FORMAT] [--hash-algo HASH_ALGO] - [--backend {auto,simple-dir,cached-dir}] [--version] [-x] [-d DOWNLOAD_DIRECTORY] [-u] [--blacklist-file IGNORELIST_FILE] - [package_directory [package_directory ...]] - - positional arguments: - package_directory The directory from which to serve packages. - - optional arguments: - -h, --help show this help message and exit - -v, --verbose Enable verbose logging; repeat for more verbosity. - --log-file FILE Write logging info into this FILE, as well as to stdout or stderr, if configured. - --log-stream STREAM Log messages to the specified STREAM. Valid values are stdout, stderr, and none - --log-frmt FORMAT The logging format-string. (see `logging.LogRecord` class from standard python library) - --hash-algo HASH_ALGO - Any `hashlib` available algorithm to use for generating fragments on package links. Can be disabled with one of (0, no, off, - false). - --backend {auto,simple-dir,cached-dir} - A backend implementation. Keep the default 'auto' to automatically determine whether to activate caching or not - --version show program's version number and exit - -x, --execute Execute the pip commands rather than printing to stdout - -d DOWNLOAD_DIRECTORY, --download-directory DOWNLOAD_DIRECTORY - Specify a directory where packages updates will be downloaded. The default behavior is to use the directory which contains the - package being updated. - -u, --allow-unstable Allow updating to unstable versions (alpha, beta, rc, dev, etc.) - --blacklist-file IGNORELIST_FILE, --ignorelist-file IGNORELIST_FILE - Don't update packages listed in this file (one package name per line, without versions, '#' comments honored). This can be useful - if you upload private packages into pypiserver, but also keep a mirror of public packages that you regularly update. Attempting to - pull an update of a private package from `pypi.org` might pose a security risk - e.g. a malicious user might publish a higher - version of the private package, containing arbitrary code. - -Client-Side Configurations -========================== - -Always specifying the the pypi url on the command line is a bit -cumbersome. Since ``pypiserver`` redirects ``pip/easy_install`` to the -``pypi.org`` index if it doesn't have a requested package, it is a -good idea to configure them to always use your local pypi index. - -Configuring ``pip`` -------------------- - -For ``pip`` command this can be done by setting the environment variable -``PIP_EXTRA_INDEX_URL`` in your ``.bashr/.profile/.zshrc``:: - - export PIP_EXTRA_INDEX_URL=http://localhost:8080/simple/ - -or by adding the following lines to ``~/.pip/pip.conf``:: - - [global] - extra-index-url = http://localhost:8080/simple/ - -.. Note:: - If you have installed ``pypiserver`` on a remote url without *https* - you will receive an "untrusted" warning from *pip*, urging you to append - the ``--trusted-host`` option. You can also include this option permanently - in your configuration-files or environment variables. - -Configuring ``easy_install`` ----------------------------- - -For ``easy_install`` command you may set the following configuration in -``~/.pydistutils.cfg``:: - - [easy_install] - index_url = http://localhost:8080/simple/ - - -Uploading Packages Remotely -=========================== - -Instead of copying packages directly to the server's folder (i.e. with ``scp``), -you may use python tools for the task, e.g. ``python setup.py upload``. -In that case, ``pypiserver`` is responsible for authenticating the upload-requests. - -.. Note:: - We strongly advise to password-protected your uploads! - - It is possible to disable authentication for uploads (e.g. in intranets). - To avoid lazy security decisions, read help for ``-P`` and ``-a`` options. - -*Apache*-Like Authentication (``htpasswd``) -------------------------------------------- - -#. First make sure you have the *passlib* module installed (note that - ``passlib>=1.6`` is required), which is needed for parsing the Apache - *htpasswd* file specified by the ``-P``, ``--passwords`` option - (see next steps):: - - pip install passlib - -#. Create the Apache *htpasswd* file with at least one user/password pair - with this command (you'll be prompted for a password):: - - htpasswd -sc htpasswd.txt - - .. Tip:: Read this SO question for running `htpasswd` cmd - under *Windows*: - - http://serverfault.com/questions/152950/how-to-create-and-edit-htaccess-and-htpasswd-locally-on-my-computer-and-then-u - - or if you have bogus passwords that you don't care because they are for - an internal service (which is still "bad", from a security perspective...) - you may use this public service: - - http://www.htaccesstools.com/htpasswd-generator/ - - .. Tip:: When accessing pypiserver via the api, alternate authentication - methods are available via the ``auther`` config flag. Any callable - returning a boolean can be passed through to the pypiserver config in - order to provide custom authentication. For example, to configure - pypiserver to authenticate using the `python-pam`_:: - - import pam - pypiserver.default_config(auther=pam.authenticate) - - Please see `Using Ad-hoc authentication providers`_ for more information. - -#. You need to restart the server with the ``-P`` option only once - (but user/password pairs can later be added or updated on the fly):: - - ./pypi-server run -p 8080 -P htpasswd.txt ~/packages & - -Upload with ``setuptools`` --------------------------- - -#. On client-side, edit or create a ``~/.pypirc`` file with a similar content:: - - [distutils] - index-servers = - pypi - local - - [pypi] - username: - password: - - [local] - repository: http://localhost:8080 - username: - password: - -#. Then from within the directory of the python-project you wish to upload, - issue this command:: - - python setup.py sdist upload -r local - -Upload with ``twine`` ---------------------- - -To avoid storing you passwords on disk, in clear text, you may either: - -- use the ``register`` *setuptools*'s command with the ``-r`` option, - like that:: - - python setup.py sdist register -r local upload -r local - -- use `twine`_ library, which - breaks the procedure in two steps. In addition, it supports signing - your files with PGP-Signatures and uploading the generated `.asc` files - to ``pypiserver``:: - - twine upload -r local --sign -identity user_name ./foo-1.zip - - -Using the Docker Image -====================== - -Starting with version 1.2.5, official Docker images will be built for each -push to master, each dev, alpha, or beta release, and each final release. -The most recent full release will always be available under the tag ``latest``, -and the current master branch will always be available under the tag -``unstable``. - -You can always check to see what tags are currently available at our -`Docker Repo`_. - -To run the most recent release of ``pypiserver`` with Docker, simply:: - - docker run pypiserver/pypiserver:latest run - -This starts ``pypiserver`` serving packages from the ``/data/packages`` -directory inside the container, listening on the container port 8080. - -The container takes all the same arguments as the normal ``pypi-server`` -executable, with the exception of the internal container port (``-p``), -which will always be 8080. - -Of course, just running a container isn't that interesting. To map -port 80 on the host to port 8080 on the container:: - - docker run -p 80:8080 pypiserver/pypiserver:latest run - -You can now access your ``pypiserver`` at ``localhost:80`` in a web browser. - -To serve packages from a directory on the host, e.g. ``~/packages``:: - - docker run -p 80:8080 -v ~/packages:/data/packages pypiserver/pypiserver:latest run - -To authenticate against a local ``.htpasswd`` file:: - - docker run -p 80:8080 -v ~/.htpasswd:/data/.htpasswd pypiserver/pypiserver:latest run -P .htpasswd packages - -You can also specify ``pypiserver`` to run as a Docker service using a -composefile. An example composefile is `provided `_. - - -.. _`docker repo`: https://hub.docker.com/r/pypiserver/pypiserver/tags/ - - -Alternative Installation Methods -================================ - -When trying the methods below, first use the following command to check whether -previous versions of ``pypiserver`` already exist, and (optionally) uninstall them:: - - # VERSION-CHECK: Fails if not installed. - pypi-server --version - - # UNINSTALL: Invoke again until it fails. - pip uninstall pypiserver - -Installing the Very Latest Version ----------------------------------- - -In case the latest version in *pypi* is a pre-release, you have to use -*pip*'s `--pre` option. And to update an existing installation combine it -with `--ignore-installed`:: - - pip install pypiserver --pre -I - -You can even install the latest ``pypiserver`` directly from *github* with the -following command, assuming you have *git* installed on your ``PATH``:: - - pip install git+git://github.com/pypiserver/pypiserver.git - - -Recipes -======= - -Managing the Package Directory ------------------------------- - -The ``pypi-server`` command has the ``update`` command that searches for updates of -available packages. It scans the package directory for available -packages and searches on pypi.org for updates. Without further -options ``pypi-server update`` will just print a list of commands which must -be run in order to get the latest version of each package. Output -looks like:: - - $ ./pypi-server update - checking 106 packages for newer version - - .........u.e...........e..u............. - .....e..............................e... - .......................... - - no releases found on pypi for PyXML, Pymacs, mercurial, setuptools - - # update raven from 1.4.3 to 1.4.4 - pip -q install --no-deps --extra-index-url https://pypi.org/simple/ -d /home/ralf/packages/mirror raven==1.4.4 - - # update greenlet from 0.3.3 to 0.3.4 - pip -q install --no-deps --extra-index-url https://pypi.org/simple/ -d /home/ralf/packages/mirror greenlet==0.3.4 - -It first prints for each package a single character after checking the -available versions on pypi. A dot(`.`) means the package is up-to-date, ``'u'`` -means the package can be updated and ``'e'`` means the list of releases on -pypi is empty. After that it shows a *pip* command line which can be used -to update a one package. Either copy and paste that or run -``pypi-server update -x`` in order to really execute those commands. You need -to have *pip* installed for that to work however. - -Specifying an additional ``-u`` option will also allow alpha, beta and -release candidates to be downloaded. Without this option these -releases won't be considered. - -Serving Thousands of Packages ------------------------------ - -By default, ``pypiserver`` scans the entire packages directory each time an -incoming HTTP request occurs. This isn't a problem for a small number of -packages, but causes noticeable slow-downs when serving thousands of packages. - -If you run into this problem, significant speedups can be gained by enabling -pypiserver's directory caching functionality. The only requirement is to -install the ``watchdog`` package, or it can be installed during ``pypiserver`` -installation, by specifying the ``cache`` extras option:: - - pip install pypiserver[cache] - -Additional speedups can be obtained by using your webserver's builtin -caching functionality. For example, if you are using `nginx` as a -reverse-proxy as described below in `Behind a reverse proxy`_, you can -easily enable caching. For example, to allow nginx to cache up to -10 gigabytes of data for up to 1 hour:: - - proxy_cache_path /data/nginx/cache - levels=1:2 - keys_zone=pypiserver_cache:10m - max_size=10g - inactive=60m - use_temp_path=off; - - server { - # ... - location / { - proxy_cache pypiserver_cache; - proxy_pass http://localhost:8080; - } - } - -Using webserver caching is especially helpful if you have high request -volume. Using `nginx` caching, a real-world pypiserver installation was -able to easily support over 1000 package downloads/min at peak load. - -Managing Automated Startup --------------------------- - -There are a variety of options for handling the automated starting of -pypiserver upon system startup. Two of the most common are *systemd* and -*supervisor* for linux systems. For windows creating services with scripts isn't -an easy task without a third party tool such as *NSSM*. - -Running As a ``systemd`` Service -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``systemd`` is installed by default on most modern Linux systems and as such, -it is an excellent option for managing the pypiserver process. An example -config file for ``systemd`` can be seen below:: - - [Unit] - Description=A minimal PyPI server for use with pip/easy_install. - After=network.target - - [Service] - Type=simple - # systemd requires absolute path here too. - PIDFile=/var/run/pypiserver.pid - User=www-data - Group=www-data - - ExecStart=/usr/local/bin/pypi-server run -p 8080 -a update,download --log-file /var/log/pypiserver.log -P /etc/nginx/.htpasswd /var/www/pypi - ExecStop=/bin/kill -TERM $MAINPID - ExecReload=/bin/kill -HUP $MAINPID - Restart=always - - WorkingDirectory=/var/www/pypi - - TimeoutStartSec=3 - RestartSec=5 - - [Install] - WantedBy=multi-user.target - -Adjusting the paths and adding this file as ``pypiserver.service`` into your -``systemd/system`` directory will allow management of the pypiserver process with -``systemctl``, e.g. ``systemctl start pypiserver``. - -More useful information about *systemd* can be found at -https://www.digitalocean.com/community/tutorials/how-to-use-systemctl-to-manage-systemd-services-and-units - -Launching through ``supervisor`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -`supervisor `_ has the benefit of being a pure python -package and as such, it provides excellent cross-platform support for process -management. An example configuration file for ``supervisor`` is given below:: - - [program:pypi] - command=/home/pypi/pypi-venv/bin/pypi-server run -p 7001 -P /home/pypi/.htpasswd /home/pypi/packages - directory=/home/pypi - user=pypi - autostart=true - autorestart=true - stderr_logfile=/var/log/pypiserver.err.log - stdout_logfile=/var/log/pypiserver.out.log - -From there, the process can be managed via ``supervisord`` using ``supervisorctl``. - -Running As a service with ``NSSM`` (Windows) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download NSSM from https://nssm.cc unzip to a desired location such as Program Files. Decide whether you are going -to use win32 or win64, and add that exe to environment PATH. - -Create a start_pypiserver.bat:: - - pypi-server run -p 8080 C:\Path\To\Packages & - -Test the batch file by running it first before creating the service. Make sure you can access -the server remotely, and install packages. If you can, proceed, if not troubleshoot until you can. -This will ensure you know the server works, before adding NSSM into the mix. - -From the command prompt:: - - nssm install pypiserver - -This command will launch a NSSM gui application:: - - Path: C:\Path\To\start_pypiserver.bat - Startup directory: Auto generates when selecting path - Service name: pypiserver - -There are more tabs, but that is the basic setup. If the service needs to be running with a certain -login credentials, make sure you enter those credentials in the logon tab. - -Start the service:: - - nssm start pypiserver - -Other useful commands:: - - nssm --help - nssm stop - nssm restart - nssm status - -For detailed information please visit https://nssm.cc - -Using a Different WSGI Server ------------------------------ - -- The ``bottle`` web-server which supports many WSGI-servers, among others, - ``paste``, ``cherrypy``, ``twisted`` and ``wsgiref`` (part of Python); you select - them using the ``--server`` flag. - -- You may view all supported WSGI servers using the following interactive code:: - - >>> from pypiserver import bottle - >>> list(bottle.server_names.keys()) - ['cgi', 'gunicorn', 'cherrypy', 'eventlet', 'tornado', 'geventSocketIO', - 'rocket', 'diesel', 'twisted', 'wsgiref', 'fapws3', 'bjoern', 'gevent', - 'meinheld', 'auto', 'aiohttp', 'flup', 'gae', 'paste', 'waitress'] - -- If none of the above servers matches your needs, invoke just the - ``pypiserver:app()`` method which returns the internal WSGI-app WITHOUT - starting-up a server - you may then send it to any WSGI server you like. - Read also the `Utilizing the API`_ section. - -- Some examples are given below - you may find more details in `bottle - site `_. - -Apache (``mod_wsgi``) -~~~~~~~~~~~~~~~~~~~~~ - -To use your *Apache2* with ``pypiserver``, prefer to utilize ``mod_wsgi`` as -explained in `bottle's documentation `_. - -.. Note:: - If you choose instead to go with ``mod_proxy``, mind that you may bump into problems - with the prefix-path (see `#155 `_). - -1. Adapt and place the following *Apache* configuration either into top-level scope, - or inside some ```` (contributed by Thomas Waldmann):: - - WSGIScriptAlias / /yoursite/wsgi/pypiserver-wsgi.py - WSGIDaemonProcess pypisrv user=pypisrv group=pypisrv umask=0007 \ - processes=1 threads=5 maximum-requests=500 \ - display-name=wsgi-pypisrv inactivity-timeout=300 - WSGIProcessGroup pypisrv - WSGIPassAuthorization On # Required for authentication (https://github.com/pypiserver/pypiserver/issues/288) - - - Require all granted - - - or if using older ``Apache < 2.4``, substitute the last part with this:: - - - Order deny,allow - Allow from all - - -2. Then create the ``/yoursite/cfg/pypiserver.wsgi`` file and make sure that - the ``user`` and ``group`` of the ``WSGIDaemonProcess`` directive - (``pypisrv:pypisrv`` in the example) have the read permission on it:: - - import pypiserver - - conf = pypiserver.default_config( - root = "/yoursite/packages", - password_file = "/yoursite/htpasswd", ) - application = pypiserver.app(**conf) - - - .. Tip:: - If you have installed ``pypiserver`` in a virtualenv, follow ``mod_wsgi``'s - `instructions `_ - and prepend the python code above with the following:: - - import site - - site.addsitedir('/yoursite/venv/lib/pythonX.X/site-packages') - -.. Note:: - For security reasons, notice that the ``Directory`` directive grants access - to a directory holding the ``wsgi`` start-up script, alone; nothing else. - -.. Note:: - To enable HTTPS support on Apache, configure the directive that contains the - WSGI configuration to use SSL. - -``gunicorn`` -~~~~~~~~~~~~ - -The following command uses ``gunicorn`` to start ``pypiserver``:: - - gunicorn -w4 'pypiserver:app(root="/home/ralf/packages")' - -or when using multiple roots:: - - gunicorn -w4 'pypiserver:app(root=["/home/ralf/packages", "/home/ralf/experimental"])' - -``paste`` -~~~~~~~~~ - -`paste `_ allows to run multiple WSGI applications -under different URL paths. Therefore it is possible to serve different set -of packages on different paths. - -The following example ``paste.ini`` could be used to serve stable and -unstable packages on different paths:: - - [composite:main] - use = egg:Paste#urlmap - /unstable/ = unstable - / = stable - - [app:stable] - use = egg:pypiserver#main - root = ~/stable-packages - - [app:unstable] - use = egg:pypiserver#main - root = ~/stable-packages - ~/unstable-packages - - [server:main] - use = egg:gunicorn#main - host = 0.0.0.0 - port = 9000 - workers = 5 - accesslog = - - -.. Note:: - You need to install some more dependencies for this to work, like:: - - pip install paste pastedeploy gunicorn pypiserver - - The server can then start with:: - - gunicorn_paster paste.ini - - -Behind a Reverse Proxy ----------------------- - -You can run ``pypiserver`` behind a reverse proxy as well. - -Nginx -~~~~~ - -Extend your nginx configuration:: - - upstream pypi { - server pypiserver.example.com:12345 fail_timeout=0; - } - - server { - server_name myproxy.example.com; - - location / { - proxy_set_header Host $host:$server_port; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Real-IP $remote_addr; - proxy_pass http://pypi; - } - } - -As of pypiserver 1.3, you may also use the `X-Forwarded-Host` header in your -reverse proxy config to enable changing the base URL. For example if you -want to host pypiserver under a particular path on your server:: - - upstream pypi { - server localhost:8000; - } - - server { - location /pypi/ { - proxy_set_header X-Forwarded-Host $host:$server_port/pypi; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - proxy_pass http://pypi; - } - } - -Supporting HTTPS -~~~~~~~~~~~~~~~~ - -Using a reverse proxy is the preferred way of getting pypiserver behind -HTTPS. For example, to put pypiserver behind HTTPS on port 443, with -automatic HTTP redirection, using `nginx`:: - - upstream pypi { - server localhost:8000; - } - - server { - listen 80 default_server; - server_name _; - return 301 https://$host$request_uri; - } - - server { - listen 443 ssl; - server_name pypiserver.example.com; - - ssl_certificate /etc/star.example.com.crt; - ssl_certificate_key /etc/star.example.com.key; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers HIGH:!aNULL:!MD5; - - location / { - proxy_set_header Host $host:$server_port; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Real-IP $remote_addr; - proxy_pass http://pypi; - } - } - -Please see `nginx's HTTPS docs for more details `_. - -Getting and keeping your certificates up-to-date can be simplified using, -for example, using `certbot and letsencrypt `_. - -Traefik -~~~~~~~ - -It is also possible to use `Traefik `_ to put pypiserver behind HTTPS on port 443, with -automatic HTTP redirection using Docker Compose. Please see the provided ``_ example for more information. - -Utilizing the API ------------------ - -In order to enable ad-hoc authentication-providers or to use WSGI-servers -not supported by *bottle* out-of-the-box, you needed to launch ``pypiserver`` -via its API. - -- The main entry-point for configuring ``pypiserver`` is the `pypiserver:app() - `_ - function. This function returns the internal WSGI-app that you my then - send to any WSGI-server you like. - -- To get all ``pypiserver:app()`` keywords and their explanations, read the - function `pypiserver:default_config() - `_. - -- Finally, to fire-up a WSGI-server with the configured app, invoke - the ``bottle:run(app, host, port, server)`` function. - Note that ``pypiserver`` ships with it is own copy of *bottle*; to use it, - import it like that: ``from pypiserver import bottle`` - -Using Ad-Hoc Authentication Providers -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``auther`` keyword of ``pypiserver:app()`` function maybe set only using -the API. This can be any callable that returns a boolean when passed -the *username* and the *password* for a given request. - -For example, to authenticate users based on the ``/etc/passwd`` file under Unix, -you may delegate such decisions to the `python-pam`_ library by following -these steps: - -1. Ensure ``python-pam`` module is installed:: - - pip install python-pam - -2. Create a python-script along these lines:: - - $ cat > pypiserver-start.py - import pypiserver - from pypiserver import bottle - import pam - app = pypiserver.app(root='./packages', auther=pam.authenticate) - bottle.run(app=app, host='0.0.0.0', port=80, server='auto') - - [Ctrl+ D] - -3. Invoke the python-script to start-up ``pypiserver``:: - - $ python pypiserver-start.py - - -.. Note:: - The `python-pam`_ module, requires *read* access to ``/etc/shadow`` file; - you may add the user under which ``pypiserver`` runs into the *shadow* - group, with a command like this: ``sudo usermod -a -G shadow pypy-user``. - -Use with MicroPython --------------------- -The MicroPython interpreter for embedded devices can install packages with the -module ``upip.py``. The module uses a specialized json-endpoint to retrieve -package information. This endpoint is supported by ``pypiserver``. - -It can be tested with the UNIX port of ``micropython``:: - - cd micropython - ports/unix/micropython -m tools.upip install -i http://my-server:8080 -p /tmp/mymodules micropython-foobar - -Installing packages from the REPL of an embedded device works in this way: - -.. code-block:: python - - import network - import upip - - sta_if = network.WLAN(network.STA_IF) - sta_if.active(True) - sta_if.connect('', '') - upip.index_urls = ["http://my-server:8080"] - upip.install("micropython-foobar") - -Further information on micropython-packaging can be found here: https://docs.micropython.org/en/latest/reference/packages.html - -Custom Health Check Endpoint ----------------------------- - -``pypiserver`` provides a default health endpoint at ``/health``. It always returns -``200 Ok`` if the service is up. Otherwise, it means that the service is not responsive. - -In addition, ``pypiserver`` allows users to customize the health endpoint. -Alphanumeric characters, hyphens, forward slashes and underscores are allowed -and the endpoint should not overlap with any existing routes. -Valid examples: ``/healthz``, ``/health/live-1``, ``/api_health``, ``/action/health`` - -Configure a custom health endpoint by CLI arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Run pypiserver with ``--health-endpoint`` argument:: - - pypi-server run --health-endpoint /action/health - -Configure a custom health endpoint by script -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - import pypiserver - from pypiserver import bottle - app = pypiserver.app(root="./packages", health_endpoint="/action/health") - bottle.run(app=app, host="0.0.0.0", port=8080, server="auto") - -Try ``curl http://localhost:8080/action/health`` - - -Sources -======= - -To create a copy of the repository, use:: - - git clone https://github.com/pypiserver/pypiserver.git - cd pypiserver - -To receive any later changes, in the above folder use:: - - git pull - - -Known Limitations -================= - -``pypiserver`` does not implement the full API as seen on PyPI_. It -implements just enough to make ``easy_install``, ``pip install``, and -``search`` work. - -The following limitations are known: - -- Command ``pypi -U`` that compares uploaded packages with *pypi* to see if - they are outdated, does not respect a http-proxy environment variable - (see `#19 `_). -- It accepts documentation uploads but does not save them to - disk (see `#47 `_ for a - discussion) -- It does not handle misspelled packages as *pypi-repo* does, - therefore it is suggested to use it with ``--extra-index-url`` instead - of ``--index-url`` (see `#38 `_). - -Please use Github's `bugtracker `_ -for other bugs you find. - - -Similar Projects -================ - -There are lots of other projects, which allow you to run your own -PyPI server. If ``pypiserver`` doesn't work for you, the following are -among the most popular alternatives: - -- `devpi-server `_: - a reliable fast pypi.org caching server, part of - the comprehensive `github-style pypi index server and packaging meta tool - `_. - (version: 2.1.4, access date: 8/3/2015) - -- Check this SO question: `How to roll my own pypi - `_ - - -Unmaintained or archived ------------------------- - -These projects were once alternatives to pypiserver but are now either unmaintained or archived. - -- `pip2pi `_ - a simple cmd-line tool that builds a PyPI-compatible local folder from pip requirements - -- `flask-pypi-proxy `_ - A proxy for PyPI that also enables also uploading custom packages. - - -Related Software -================ - -Though not direct alternatives for ``pypiserver``'s use as an index -server, the following is a list of related software projects that you -may want to familiarize with: - -- `pypi-uploader`_: - A command-line utility to upload packages to your ``pypiserver`` from pypi without - having to store them locally first. - -- `twine`_: - A command-line utility for interacting with PyPI or ``pypiserver``. - -- `warehouse`_: - the software that powers PyPI_ itself. It is not generally intended to - be run by end-users. - -Licensing -========= - -``pypiserver`` contains a copy of bottle_ which is available under the -MIT license, and the remaining part is distributed under the zlib/libpng license. -See the ``LICENSE.txt`` file. - - -.. _bottle: http://bottlepy.org -.. _PyPA: https://www.pypa.io/en/latest/ -.. _PyPI: https://pypi.org -.. _Warehouse: https://github.com/pypa/warehouse/ -.. _twine: https://pypi.org/project/twine/ -.. _pypi-uploader: https://pypi.org/project/pypi-uploader/ -.. _python-pam: https://pypi.org/project/python-pam/ -.. |test-status| image:: https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml/badge.svg - :alt: test status - :scale: 100% - :target: https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml - -.. |pypi-ver| image:: https://img.shields.io/pypi/v/pypiserver.svg - :target: https://pypi.org/project/pypiserver/ - :alt: Latest Version in PyPI - -.. |python-ver| image:: https://img.shields.io/pypi/pyversions/pypiserver.svg - :target: https://pypi.org/project/pypiserver/ - :alt: Supported Python versions - -.. |proj-license| image:: https://img.shields.io/badge/license-BSD%2Bzlib%2Flibpng-blue.svg - :target: https://raw.githubusercontent.com/pypiserver/pypiserver/master/LICENSE.txt - :alt: Project License - -.. |dependencies| image:: https://img.shields.io/requires/github/pypiserver/pypiserver.svg - :target: https://requires.io/github/pypiserver/pypiserver/requirements/ - :alt: Dependencies up-to-date? diff --git a/bin/README.rst b/bin/README.rst index 267ce42..0be879d 100644 --- a/bin/README.rst +++ b/bin/README.rst @@ -14,7 +14,7 @@ Files: Release check-list: =================== -1. Update ``/CHANGES.rst`` (+ Title + Date) & ``/README.rst`` (Date, +1. Update ``/CHANGES.rst`` (+ Title + Date) & ``/README.md`` (Date, not version). 2. Push to GitHub to run all TCs once more. diff --git a/bin/bumpver.py b/bin/bumpver.py index e69d52c..533f933 100755 --- a/bin/bumpver.py +++ b/bin/bumpver.py @@ -49,7 +49,7 @@ VFILE = osp.join(my_dir, "..", "pypiserver", "__init__.py") VFILE_regex_v = re.compile(r'version *= *__version__ *= *"([^"]+)"') VFILE_regex_d = re.compile(r'__updated__ *= *"([^"]+)"') -RFILE = osp.join(my_dir, "..", "README.rst") +RFILE = osp.join(my_dir, "..", "README.md") PYTEST_ARGS = [osp.join("tests", "test_docs.py")] diff --git a/bin/check_readme.sh b/bin/check_readme.sh deleted file mode 100755 index 5da49fc..0000000 --- a/bin/check_readme.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -#-*- coding: utf-8 -*- -# -## Checks that README has no RsT-syntactic errors. -# Since it is used by `setup.py`'s `description` if it has any errors, -# PyPi would fail parsing them, ending up with an ugly landing page, -# when uploaded. - ->&2 echo "+++ Checking README for PyPy...." -set +x ## Enable for debug - -my_dir=`dirname "$0"` -cd $my_dir/.. - -py="" -rst="rst2html" -if [ ! -x "`which $rst 2>/dev/null`" ]; then - ## In WinPython, only a python-script exist in PATH, - # so execute it with python-interpreter. - # - exe="`which rst2html.py 2> /dev/null`" - if [ $? -eq 0 ]; then - py=python - rst="$exe" - else - echo -e "Cannot find 'rst2html'! \n Sphinx installed? `pip show sphinx`" && - exit 1 - fi - - if [ -x "`which cygpath`" ]; then - rst="`cygpath -w $rst`" - fi -fi - -export PYTHONPATH='$my_dir/..' -#python setup.py --long-description > t.rst ## Uncomment to inspect it. -python setup.py --long-description | $py "$rst" --halt=warning > /dev/null && echo OK diff --git a/docs/README.md b/docs/README.md index 42702d3..de63188 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ Welcome to the documentation section of `pypiserver`. ## More information If something is missing in the documentation here, maybe it is covered in -[`README.rst`](../README.rst). +[README.md](../README.md). ## Documentation setup diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 3c322aa..ef0c340 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -9,3 +9,6 @@ tox twine webtest wheel>=0.25.0 +mdformat-gfm +mdformat-frontmatter +mdformat-footnote diff --git a/setup.py b/setup.py index 0b3d74f..c4eb99d 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def get_version(): setup( name="pypiserver", description="A minimal PyPI server for use with pip/easy_install.", - long_description=read_file("README.rst"), + long_description=read_file("README.md"), version=get_version(), packages=["pypiserver"], package_data={"pypiserver": ["welcome.html"]}, diff --git a/tests/test_docs.py b/tests/test_docs.py index 2214740..7fae5d3 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -8,10 +8,12 @@ from pypiserver import version as my_ver @pytest.fixture() def readme(): - return Path(__file__).parents[1].joinpath("README.rst").read_text() + return Path(__file__).parents[1].joinpath("README.md").read_text() def test_READMEversion(readme): - m = re.compile(r"^\s*:Version:\s*(.+)\s*$", re.MULTILINE).search(readme) + m = re.compile( + r"^\|\s*Version\s*\|\s*(\d+\.\d+\.\d+)\s*\|$", re.MULTILINE + ).search(readme) assert m, "Could not find version on README!" assert m.group(1) == my_ver, f"Updated version({m.group(1)}) on README!"