2020-11-16 03:08:58 +01:00
|
|
|
# Run tests
|
|
|
|
|
2021-02-08 03:24:33 +01:00
|
|
|
name: CI
|
2020-11-16 03:08:58 +01:00
|
|
|
|
|
|
|
on:
|
2021-02-08 03:24:33 +01:00
|
|
|
# This will run when any branch or tag is pushed
|
2020-11-16 03:08:58 +01:00
|
|
|
push:
|
|
|
|
# standalone is an old branch containing a fully functional pypiserver
|
|
|
|
# executable, from back in the day before docker & a better pip.
|
|
|
|
branches-ignore:
|
|
|
|
- standalone
|
|
|
|
|
|
|
|
jobs:
|
|
|
|
test-cpython:
|
|
|
|
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
strategy:
|
|
|
|
matrix:
|
|
|
|
python-version: ["3.6", "3.7", "3.8", "3.9"]
|
|
|
|
|
|
|
|
steps:
|
|
|
|
- uses: actions/checkout@v2
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
|
|
uses: actions/setup-python@v2
|
|
|
|
with:
|
|
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install dependencies
|
|
|
|
run: pip install tox
|
|
|
|
- name: Run tests
|
|
|
|
# Create a tox env specification by stripping the dot out of the version
|
|
|
|
# specification and appending it to "py"
|
|
|
|
run: |
|
|
|
|
tox -e "py$(echo ${{ matrix.python-version }} | tr -d .)"
|
|
|
|
|
|
|
|
test-pypy:
|
|
|
|
|
|
|
|
# Run a a separate job so we don't need to mess with conditionally
|
Docker improvements (#365)
* Docker improvements
This addresses much of what was brought up in #359. Specifically, it:
- Significantly improves testing for the Docker image, adding a
`docker/test_docker.py` file using the regular pytest machinery to
set up and run docker images for testing
- Hopefully addresses a variety of permissions issues, by being explicit
about what access pypiserver needs and asking for it, only erroring
if that access is not available
- Requires RX permissions on `/data` (R to read files, X to list files
and to be able to cd into the directory. This is important since
`/data` is the `WORKDIR`)
- Requires RWX permissions on `/data/packages`, so that we can list
packages, write packages, and read packages.
- When running in the default configuration (as root on Linux or
as the pypiserver-named rootish user on Mac), with no volumes
mounted, these requirements are all satisfied
- Volume mounts still must be readable by the pypiserver user (UID
9898) in order for the container to run. However, we now error early
if this is not the case, and direct users to a useful issue.
- If the container is run as a non-root, non-pypiserver user (e.g.
because someone ran `docker run --user=<user_id>`, we try to run
pypiserver as that user). Provided that user has access to the
necessary directories, it should run fine.
- Fixes issues with running help and similar commands
- Updates the Docker image to use `PYPISERVER_PORT` for port
specification, while still falling back to `PORT` for backwards
compatibility
- Moves some docker-related things into a `/docker` directory
- Adds a `Makefile` for building a test fixture package sdist and wheel,
so that test code can call `make mypkg` and not need to worry about it
potentially building multiple times
The only issue #359 raises that's not addressed here is the one of
running pypiserver in the Docker container using some non-default server
for performance. I would like to do some benchmarking before deciding on
what to do there.
2021-02-06 18:28:15 +01:00
|
|
|
# splitting the python version from the build matrix. Also the pypy
|
|
|
|
# tests take freaking forever.
|
2020-11-16 03:08:58 +01:00
|
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
|
|
- uses: actions/checkout@v2
|
|
|
|
- name: Set up Python
|
|
|
|
uses: actions/setup-python@v2
|
|
|
|
with:
|
|
|
|
python-version: pypy3
|
|
|
|
- name: Install dependencies
|
|
|
|
run: pip install tox
|
|
|
|
- name: Run tests
|
|
|
|
run: tox -e pypy3
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
|
|
|
# These checks only need to be done once, not for every python version we s
|
|
|
|
# upport
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
|
|
- uses: actions/checkout@v2
|
|
|
|
- name: Set up Python
|
|
|
|
uses: actions/setup-python@v2
|
|
|
|
with:
|
|
|
|
# Pretty much any python version will do
|
|
|
|
python-version: "3.9"
|
|
|
|
- name: Install dependencies
|
|
|
|
run: pip install -r "requirements/dev.pip"
|
|
|
|
- name: Check types
|
|
|
|
# individual mypy files for now, until we get the rest
|
|
|
|
# of the project typechecking
|
Docker improvements (#365)
* Docker improvements
This addresses much of what was brought up in #359. Specifically, it:
- Significantly improves testing for the Docker image, adding a
`docker/test_docker.py` file using the regular pytest machinery to
set up and run docker images for testing
- Hopefully addresses a variety of permissions issues, by being explicit
about what access pypiserver needs and asking for it, only erroring
if that access is not available
- Requires RX permissions on `/data` (R to read files, X to list files
and to be able to cd into the directory. This is important since
`/data` is the `WORKDIR`)
- Requires RWX permissions on `/data/packages`, so that we can list
packages, write packages, and read packages.
- When running in the default configuration (as root on Linux or
as the pypiserver-named rootish user on Mac), with no volumes
mounted, these requirements are all satisfied
- Volume mounts still must be readable by the pypiserver user (UID
9898) in order for the container to run. However, we now error early
if this is not the case, and direct users to a useful issue.
- If the container is run as a non-root, non-pypiserver user (e.g.
because someone ran `docker run --user=<user_id>`, we try to run
pypiserver as that user). Provided that user has access to the
necessary directories, it should run fine.
- Fixes issues with running help and similar commands
- Updates the Docker image to use `PYPISERVER_PORT` for port
specification, while still falling back to `PORT` for backwards
compatibility
- Moves some docker-related things into a `/docker` directory
- Adds a `Makefile` for building a test fixture package sdist and wheel,
so that test code can call `make mypkg` and not need to worry about it
potentially building multiple times
The only issue #359 raises that's not addressed here is the one of
running pypiserver in the Docker container using some non-default server
for performance. I would like to do some benchmarking before deciding on
what to do there.
2021-02-06 18:28:15 +01:00
|
|
|
run: >-
|
|
|
|
mypy
|
|
|
|
docker/test_docker.py
|
|
|
|
pypiserver/config.py
|
|
|
|
tests/test_init.py
|
2020-11-16 03:08:58 +01:00
|
|
|
- name: Check formatting
|
|
|
|
run: black --diff --check .
|
|
|
|
- name: Validate README
|
|
|
|
run: ./bin/check_readme.sh
|
|
|
|
|
|
|
|
# Full-flow docker tests, again not python version dependent
|
Docker improvements (#365)
* Docker improvements
This addresses much of what was brought up in #359. Specifically, it:
- Significantly improves testing for the Docker image, adding a
`docker/test_docker.py` file using the regular pytest machinery to
set up and run docker images for testing
- Hopefully addresses a variety of permissions issues, by being explicit
about what access pypiserver needs and asking for it, only erroring
if that access is not available
- Requires RX permissions on `/data` (R to read files, X to list files
and to be able to cd into the directory. This is important since
`/data` is the `WORKDIR`)
- Requires RWX permissions on `/data/packages`, so that we can list
packages, write packages, and read packages.
- When running in the default configuration (as root on Linux or
as the pypiserver-named rootish user on Mac), with no volumes
mounted, these requirements are all satisfied
- Volume mounts still must be readable by the pypiserver user (UID
9898) in order for the container to run. However, we now error early
if this is not the case, and direct users to a useful issue.
- If the container is run as a non-root, non-pypiserver user (e.g.
because someone ran `docker run --user=<user_id>`, we try to run
pypiserver as that user). Provided that user has access to the
necessary directories, it should run fine.
- Fixes issues with running help and similar commands
- Updates the Docker image to use `PYPISERVER_PORT` for port
specification, while still falling back to `PORT` for backwards
compatibility
- Moves some docker-related things into a `/docker` directory
- Adds a `Makefile` for building a test fixture package sdist and wheel,
so that test code can call `make mypkg` and not need to worry about it
potentially building multiple times
The only issue #359 raises that's not addressed here is the one of
running pypiserver in the Docker container using some non-default server
for performance. I would like to do some benchmarking before deciding on
what to do there.
2021-02-06 18:28:15 +01:00
|
|
|
# We _could_ test this on MacOS, but it takes forever to get docker
|
|
|
|
# installed. I'm going to say for now probably 99% of people using
|
|
|
|
# the docker image will be doing so from a linux system, e.g. for
|
|
|
|
# a k8s deploy, and I've verified manually that things work on
|
|
|
|
# MacOS, so /shrug.
|
|
|
|
test-docker:
|
|
|
|
runs-on: "ubuntu-latest"
|
2020-11-16 03:08:58 +01:00
|
|
|
steps:
|
|
|
|
- uses: actions/checkout@v2
|
|
|
|
- name: Set up Python
|
|
|
|
uses: actions/setup-python@v2
|
|
|
|
with:
|
|
|
|
# Pretty much any python version will do
|
|
|
|
python-version: "3.9"
|
Docker improvements (#365)
* Docker improvements
This addresses much of what was brought up in #359. Specifically, it:
- Significantly improves testing for the Docker image, adding a
`docker/test_docker.py` file using the regular pytest machinery to
set up and run docker images for testing
- Hopefully addresses a variety of permissions issues, by being explicit
about what access pypiserver needs and asking for it, only erroring
if that access is not available
- Requires RX permissions on `/data` (R to read files, X to list files
and to be able to cd into the directory. This is important since
`/data` is the `WORKDIR`)
- Requires RWX permissions on `/data/packages`, so that we can list
packages, write packages, and read packages.
- When running in the default configuration (as root on Linux or
as the pypiserver-named rootish user on Mac), with no volumes
mounted, these requirements are all satisfied
- Volume mounts still must be readable by the pypiserver user (UID
9898) in order for the container to run. However, we now error early
if this is not the case, and direct users to a useful issue.
- If the container is run as a non-root, non-pypiserver user (e.g.
because someone ran `docker run --user=<user_id>`, we try to run
pypiserver as that user). Provided that user has access to the
necessary directories, it should run fine.
- Fixes issues with running help and similar commands
- Updates the Docker image to use `PYPISERVER_PORT` for port
specification, while still falling back to `PORT` for backwards
compatibility
- Moves some docker-related things into a `/docker` directory
- Adds a `Makefile` for building a test fixture package sdist and wheel,
so that test code can call `make mypkg` and not need to worry about it
potentially building multiple times
The only issue #359 raises that's not addressed here is the one of
running pypiserver in the Docker container using some non-default server
for performance. I would like to do some benchmarking before deciding on
what to do there.
2021-02-06 18:28:15 +01:00
|
|
|
- name: Install test dependencies
|
|
|
|
run: pip install -r "requirements/test.pip"
|
|
|
|
- name: Install package
|
|
|
|
run: pip install -r "requirements/exe.pip"
|
2020-11-16 03:08:58 +01:00
|
|
|
- name: Run tests
|
Docker improvements (#365)
* Docker improvements
This addresses much of what was brought up in #359. Specifically, it:
- Significantly improves testing for the Docker image, adding a
`docker/test_docker.py` file using the regular pytest machinery to
set up and run docker images for testing
- Hopefully addresses a variety of permissions issues, by being explicit
about what access pypiserver needs and asking for it, only erroring
if that access is not available
- Requires RX permissions on `/data` (R to read files, X to list files
and to be able to cd into the directory. This is important since
`/data` is the `WORKDIR`)
- Requires RWX permissions on `/data/packages`, so that we can list
packages, write packages, and read packages.
- When running in the default configuration (as root on Linux or
as the pypiserver-named rootish user on Mac), with no volumes
mounted, these requirements are all satisfied
- Volume mounts still must be readable by the pypiserver user (UID
9898) in order for the container to run. However, we now error early
if this is not the case, and direct users to a useful issue.
- If the container is run as a non-root, non-pypiserver user (e.g.
because someone ran `docker run --user=<user_id>`, we try to run
pypiserver as that user). Provided that user has access to the
necessary directories, it should run fine.
- Fixes issues with running help and similar commands
- Updates the Docker image to use `PYPISERVER_PORT` for port
specification, while still falling back to `PORT` for backwards
compatibility
- Moves some docker-related things into a `/docker` directory
- Adds a `Makefile` for building a test fixture package sdist and wheel,
so that test code can call `make mypkg` and not need to worry about it
potentially building multiple times
The only issue #359 raises that's not addressed here is the one of
running pypiserver in the Docker container using some non-default server
for performance. I would like to do some benchmarking before deciding on
what to do there.
2021-02-06 18:28:15 +01:00
|
|
|
run: "pytest docker/test_docker.py"
|
2021-02-03 03:36:41 +01:00
|
|
|
|
|
|
|
|
|
|
|
tests:
|
|
|
|
runs-on: "ubuntu-latest"
|
|
|
|
needs:
|
|
|
|
- "check"
|
Docker improvements (#365)
* Docker improvements
This addresses much of what was brought up in #359. Specifically, it:
- Significantly improves testing for the Docker image, adding a
`docker/test_docker.py` file using the regular pytest machinery to
set up and run docker images for testing
- Hopefully addresses a variety of permissions issues, by being explicit
about what access pypiserver needs and asking for it, only erroring
if that access is not available
- Requires RX permissions on `/data` (R to read files, X to list files
and to be able to cd into the directory. This is important since
`/data` is the `WORKDIR`)
- Requires RWX permissions on `/data/packages`, so that we can list
packages, write packages, and read packages.
- When running in the default configuration (as root on Linux or
as the pypiserver-named rootish user on Mac), with no volumes
mounted, these requirements are all satisfied
- Volume mounts still must be readable by the pypiserver user (UID
9898) in order for the container to run. However, we now error early
if this is not the case, and direct users to a useful issue.
- If the container is run as a non-root, non-pypiserver user (e.g.
because someone ran `docker run --user=<user_id>`, we try to run
pypiserver as that user). Provided that user has access to the
necessary directories, it should run fine.
- Fixes issues with running help and similar commands
- Updates the Docker image to use `PYPISERVER_PORT` for port
specification, while still falling back to `PORT` for backwards
compatibility
- Moves some docker-related things into a `/docker` directory
- Adds a `Makefile` for building a test fixture package sdist and wheel,
so that test code can call `make mypkg` and not need to worry about it
potentially building multiple times
The only issue #359 raises that's not addressed here is the one of
running pypiserver in the Docker container using some non-default server
for performance. I would like to do some benchmarking before deciding on
what to do there.
2021-02-06 18:28:15 +01:00
|
|
|
- "test-docker"
|
2021-02-03 03:36:41 +01:00
|
|
|
- "test-cpython"
|
|
|
|
- "test-pypy"
|
|
|
|
steps:
|
|
|
|
- name: "Everything is good!"
|
|
|
|
run: "echo true"
|
2021-02-08 03:24:33 +01:00
|
|
|
|
|
|
|
# figure out which docker tags we need to push
|
|
|
|
docker-determine-tags:
|
|
|
|
runs-on: "ubuntu-latest"
|
|
|
|
needs:
|
|
|
|
- "tests"
|
|
|
|
steps:
|
|
|
|
- uses: "actions/checkout@v2"
|
|
|
|
|
|
|
|
- uses: "actions/setup-python@v2"
|
|
|
|
with:
|
|
|
|
python-version: "3.9"
|
|
|
|
|
|
|
|
# This script prints a JSON array of needed docker tags, depending on the
|
|
|
|
# ref. That array is then used to construct the matrix of the
|
|
|
|
# deploy-docker job
|
|
|
|
- name: "Get expected docker tags"
|
|
|
|
id: "tags"
|
|
|
|
run: >-
|
|
|
|
echo "::set-output name=tags::$(bin/ci_helper.py ${{ github.ref }} docker_tags)"
|
|
|
|
|
|
|
|
# This is needed because GH actions will fail on an empty matrix, so
|
|
|
|
# we need to be sure the `if` condition is false on the next job if
|
|
|
|
# the matrix will be empty. The script prints 'true' if the array is
|
|
|
|
# not empty, or 'false' otherwise.
|
|
|
|
- name: "Determine whether any tags are needed"
|
|
|
|
id: "has_tags"
|
|
|
|
run: >-
|
|
|
|
echo "::set-output name=has_tags::$(bin/ci_helper.py ${{ github.ref }} has_tags)"
|
|
|
|
|
|
|
|
outputs:
|
|
|
|
tags: "${{ steps.tags.outputs.tags }}"
|
|
|
|
has_tags: "${{ steps.has_tags.outputs.has_tags }}"
|
|
|
|
|
|
|
|
# Deploy any needed docker tags
|
|
|
|
deploy-docker:
|
|
|
|
runs-on: "ubuntu-latest"
|
|
|
|
needs:
|
|
|
|
- "tests"
|
|
|
|
- "docker-determine-tags"
|
|
|
|
if: "${{ fromJson(needs.docker-determine-tags.outputs.has_tags) }}"
|
|
|
|
strategy:
|
|
|
|
matrix:
|
|
|
|
tag: "${{ fromJson(needs.docker-determine-tags.outputs.tags) }}"
|
|
|
|
steps:
|
|
|
|
- uses: "actions/checkout@v2"
|
|
|
|
|
|
|
|
- name: "Cache Docker layers"
|
|
|
|
uses: "actions/cache@v2"
|
|
|
|
with:
|
|
|
|
path: "/tmp/.buildx-cache"
|
|
|
|
key: "${{ runner.os }}-buildx-${{ github.sha }}"
|
|
|
|
restore-keys: |
|
|
|
|
${{ runner.os }}-buildx-
|
|
|
|
|
|
|
|
- name: "Login to Docker Hub"
|
|
|
|
uses: "docker/login-action@v1"
|
|
|
|
with:
|
|
|
|
username: "${{ secrets.DOCKER_HUB_USER }}"
|
|
|
|
password: "${{ secrets.DOCKER_HUB_TOKEN }}"
|
|
|
|
|
|
|
|
- name: "Set up Docker Buildx"
|
|
|
|
id: "buildx"
|
|
|
|
uses: "docker/setup-buildx-action@v1"
|
|
|
|
|
|
|
|
- name: "Build and push"
|
|
|
|
id: "docker_build"
|
|
|
|
uses: "docker/build-push-action@v2"
|
|
|
|
with:
|
|
|
|
context: "./"
|
|
|
|
file: "./Dockerfile"
|
|
|
|
builder: "${{ steps.buildx.outputs.name }}"
|
|
|
|
push: true
|
|
|
|
tags: "pypiserver/pypiserver:${{ matrix.tag }}"
|
|
|
|
cache-from: "type=local,src=/tmp/.buildx-cache"
|
|
|
|
cache-to: "type=local,dest=/tmp/.buildx-cache"
|
|
|
|
|
|
|
|
- name: "Image digest"
|
|
|
|
run: "echo ${{ steps.docker_build.outputs.digest }}"
|