From f40901bc33817a5ce11f1cc448848bc72269b841 Mon Sep 17 00:00:00 2001 From: Matthew Planchard Date: Mon, 18 Jan 2016 12:02:55 -0600 Subject: [PATCH 1/9] Impl #106: Start impl uploading GPG-sigs. - Added path munging for .asc files. - core: store .asc files, if given. - Testing sig-file parsing. --- pypiserver/_app.py | 13 +++++++++++-- pypiserver/core.py | 11 +++++++++-- tests/test_core.py | 5 +++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index b690575..0aa4e2e 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -144,10 +144,18 @@ def update(): except KeyError: raise HTTPError(400, "Missing 'content' file-field!") - if (not is_valid_pkg_filename(content.raw_filename) or + if (not is_valid_pkg_filename(content.raw_filename) or core.guess_pkgname_and_version(content.raw_filename) is None): raise HTTPError(400, "Bad filename: %s" % content.raw_filename) + try: + gpg_signature = request.files['gpg_signature'] + except KeyError: + gpg_signature = None + + if "/" in content.filename: + raise HTTPError(400, output="bad filename") + if not config.overwrite and core.exists(packages.root, content.raw_filename): log.warn("Cannot upload package(%s) since it already exists! \n" + " You may use `--overwrite` option when starting server to disable this check. ", @@ -155,7 +163,8 @@ def update(): msg = "Package already exists! Start server with `--overwrite` option?" raise HTTPError(409, msg) - core.store(packages.root, content.raw_filename, content.save) + core.store(packages.root, content.filename, content.save, + gpg_signature.filename, gpg_signature.save) return "" diff --git a/pypiserver/core.py b/pypiserver/core.py index a7078dd..a65b35f 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -129,6 +129,7 @@ _archive_suffix_rx = re.compile( r"(\.zip|\.tar\.gz|\.tgz|\.tar\.bz2|-py[23]\.\d-.*|" "\.win-amd64-py[23]\.\d\..*|\.win32-py[23]\.\d\..*|\.egg)$", re.I) + wheel_file_re = re.compile( r"""^(?P(?P.+?)-(?P\d.*?)) ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) @@ -155,6 +156,8 @@ def _guess_pkgname_and_version_wheel(basename): def guess_pkgname_and_version(path): path = os.path.basename(path) + if path.endswith(".asc"): + path = path.rstrip(".asc") if path.endswith(".whl"): return _guess_pkgname_and_version_wheel(path) if not _archive_suffix_rx.search(path): @@ -277,11 +280,15 @@ def exists(root, filename): return os.path.exists(dest_fn) -def store(root, filename, save_method): +def store(root, filename, save_method, + gpg_filename=None, gpg_save_method=None): assert "/" not in filename dest_fn = os.path.join(root, filename) save_method(dest_fn, overwrite=True) # Overwite check elsewhere. - + if gpg_filename is not None: + assert gpg_save_method is not None + gpg_dest_fn = os.path.join(root, gpg_filename) + save_method(gpg_dest_fn, overwrite=True) log.info("Stored package: %s", filename) return True diff --git a/tests/test_core.py b/tests/test_core.py index a95f1da..e270433 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -54,6 +54,11 @@ files = [ ("pkg.zip", 'pkg', ''), ("foo/pkg.zip", 'pkg', ''), ("foo/pkg-1b.zip", 'pkg', '1b'), + ("package-name-0.0.1.alpha.1.win-amd64-py3.2.exe", "package-name", "0.0.1.alpha.1"), + ("some_package-0.1.0-py2.py3-none-any.whl", "some_package", "0.1.0"), + ("some_package-0.1.0-py2.py3-none-any.whl.asc", "some_package", "0.1.0"), + ("some_package-0.1.0.tar.gz.asc", "some_package", "0.1.0"), + ("some_package-0.1.0.tar.gz", "some_package", "0.1.0") ] def _capitalize_ext(fpath): From 2fff9a2c21199e99d15c72c27c2a18adbcde9941 Mon Sep 17 00:00:00 2001 From: Matthew Planchard Date: Mon, 18 Jan 2016 15:49:14 -0600 Subject: [PATCH 2/9] gpg #106: Updated logic for absent gpg info. - core: Removed uneeded assertion. --- pypiserver/_app.py | 7 +++++-- pypiserver/core.py | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 0aa4e2e..dc858ef 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -151,7 +151,10 @@ def update(): try: gpg_signature = request.files['gpg_signature'] except KeyError: - gpg_signature = None + gpg_filename = gpg_save = None + else: + gpg_filename = gpg_signature.filename + gpg_save = gpg_signature.save if "/" in content.filename: raise HTTPError(400, output="bad filename") @@ -164,7 +167,7 @@ def update(): raise HTTPError(409, msg) core.store(packages.root, content.filename, content.save, - gpg_signature.filename, gpg_signature.save) + gpg_filename, gpg_save) return "" diff --git a/pypiserver/core.py b/pypiserver/core.py index a65b35f..0e0ad74 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -286,7 +286,6 @@ def store(root, filename, save_method, dest_fn = os.path.join(root, filename) save_method(dest_fn, overwrite=True) # Overwite check elsewhere. if gpg_filename is not None: - assert gpg_save_method is not None gpg_dest_fn = os.path.join(root, gpg_filename) save_method(gpg_dest_fn, overwrite=True) log.info("Stored package: %s", filename) From 64d0f3aef75b6526f42e13b9a96dc2fbf3e94318 Mon Sep 17 00:00:00 2001 From: Matthew Planchard Date: Tue, 19 Jan 2016 04:55:49 -0600 Subject: [PATCH 3/9] gpg: #106: Check also signature if overwrite. - Re-organized call to core.store - Added `.asc` mimetype --- pypiserver/_app.py | 25 ++++++++++++++++++------- pypiserver/core.py | 1 + 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index dc858ef..40c4c51 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -151,23 +151,34 @@ def update(): try: gpg_signature = request.files['gpg_signature'] except KeyError: - gpg_filename = gpg_save = None - else: - gpg_filename = gpg_signature.filename - gpg_save = gpg_signature.save + gpg_signature = None if "/" in content.filename: raise HTTPError(400, output="bad filename") - if not config.overwrite and core.exists(packages.root, content.raw_filename): + if gpg_signature is not None and "/" in gpg_signature.filename: + raise HTTPError(400, output="bad filename") + + if not config.overwrite and core.exists(packages.root, content.filename): log.warn("Cannot upload package(%s) since it already exists! \n" + " You may use `--overwrite` option when starting server to disable this check. ", content.raw_filename) msg = "Package already exists! Start server with `--overwrite` option?" raise HTTPError(409, msg) - core.store(packages.root, content.filename, content.save, - gpg_filename, gpg_save) + if not config.overwrite and core.exists(packages.root, + gpg_signature.filename): + log.warn("Cannot upload package(%s) because its signature already " + "exists! \n You may use the `--overwrite` option when" + "starting the server to disable this check.") + raise HTTPError(409, outut="signature file already exists") + + if gpg_signature is None: + core.store(packages.root, content.filename, content.save) + else: + core.store(packages.root, content.filename, content.save, + gpg_signature.filename, gpg_signature.save) + return "" diff --git a/pypiserver/core.py b/pypiserver/core.py index 0e0ad74..0898b9f 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -91,6 +91,7 @@ def auth_by_htpasswd_file(htPsswdFile, username, password): mimetypes.add_type("application/octet-stream", ".egg") mimetypes.add_type("application/octet-stream", ".whl") +mimetypes.add_type("text/plain", ".asc") #### Next 2 functions adapted from :mod:`distribute.pkg_resources`. From 7da927fd6e7f0a6c1defc6bc1ff1b660c2230bbe Mon Sep 17 00:00:00 2001 From: Matthew Planchard Date: Tue, 19 Jan 2016 09:02:46 -0600 Subject: [PATCH 4/9] gpg, #106: adjusted from rebase onto master. --- pypiserver/_app.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 40c4c51..56d8937 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -145,20 +145,9 @@ def update(): raise HTTPError(400, "Missing 'content' file-field!") if (not is_valid_pkg_filename(content.raw_filename) or - core.guess_pkgname_and_version(content.raw_filename) is None): + core.guess_pkgname_and_version(content.raw_filename) is None): raise HTTPError(400, "Bad filename: %s" % content.raw_filename) - try: - gpg_signature = request.files['gpg_signature'] - except KeyError: - gpg_signature = None - - if "/" in content.filename: - raise HTTPError(400, output="bad filename") - - if gpg_signature is not None and "/" in gpg_signature.filename: - raise HTTPError(400, output="bad filename") - if not config.overwrite and core.exists(packages.root, content.filename): log.warn("Cannot upload package(%s) since it already exists! \n" + " You may use `--overwrite` option when starting server to disable this check. ", @@ -166,12 +155,26 @@ def update(): msg = "Package already exists! Start server with `--overwrite` option?" raise HTTPError(409, msg) + try: + gpg_signature = request.files['gpg_signature'] + except KeyError: + gpg_signature = None + + if (gpg_signature is not None and + (not is_valid_pkg_filename(gpg_signature.raw_filename) + or core.guess_pkgname_and_version(content.raw_filename) is None)): + raise HTTPError(400, "Bad gpg signature name: %s" % + gpg_signature.raw_filename) + + if not config.overwrite and core.exists(packages.root, gpg_signature.filename): log.warn("Cannot upload package(%s) because its signature already " "exists! \n You may use the `--overwrite` option when" "starting the server to disable this check.") - raise HTTPError(409, outut="signature file already exists") + msg = ("Signature file already exists! Start server with " + "`--overwrite` option?") + raise HTTPError(409, msg) if gpg_signature is None: core.store(packages.root, content.filename, content.save) From abf8785f4dededdca318471b9c5e085d02d035f1 Mon Sep 17 00:00:00 2001 From: Matthew Planchard Date: Tue, 19 Jan 2016 09:11:43 -0600 Subject: [PATCH 5/9] gpg, #106, test: Add TC for parsing all-filenames with `.asc`. --- tests/test_core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index e270433..9984d08 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -55,10 +55,6 @@ files = [ ("foo/pkg.zip", 'pkg', ''), ("foo/pkg-1b.zip", 'pkg', '1b'), ("package-name-0.0.1.alpha.1.win-amd64-py3.2.exe", "package-name", "0.0.1.alpha.1"), - ("some_package-0.1.0-py2.py3-none-any.whl", "some_package", "0.1.0"), - ("some_package-0.1.0-py2.py3-none-any.whl.asc", "some_package", "0.1.0"), - ("some_package-0.1.0.tar.gz.asc", "some_package", "0.1.0"), - ("some_package-0.1.0.tar.gz", "some_package", "0.1.0") ] def _capitalize_ext(fpath): @@ -73,6 +69,12 @@ def test_guess_pkgname_and_version(filename, pkgname, version): assert core.guess_pkgname_and_version(filename) == exp assert core.guess_pkgname_and_version(_capitalize_ext(filename)) == exp +@pytest.mark.parametrize(("filename", "pkgname", "version"), files) +def test_guess_pkgname_and_version_asc(filename, pkgname, version): + exp = (pkgname, version) + filename = '%s.asc' % filename + assert core.guess_pkgname_and_version(filename) == exp + def test_listdir_bad_name(tmpdir): tmpdir.join("foo.whl").ensure() From 72f641595cb0bf6d1ef695f640cbf844fe1d5854 Mon Sep 17 00:00:00 2001 From: Matthew Planchard Date: Tue, 19 Jan 2016 09:32:59 -0600 Subject: [PATCH 6/9] gpg, #106: Fixed _app errors due to missing gpg_signature. --- pypiserver/_app.py | 48 ++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 56d8937..8797a95 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -145,10 +145,10 @@ def update(): raise HTTPError(400, "Missing 'content' file-field!") if (not is_valid_pkg_filename(content.raw_filename) or - core.guess_pkgname_and_version(content.raw_filename) is None): + core.guess_pkgname_and_version(content.raw_filename) is None): raise HTTPError(400, "Bad filename: %s" % content.raw_filename) - if not config.overwrite and core.exists(packages.root, content.filename): + if not config.overwrite and core.exists(packages.root, content.raw_filename): log.warn("Cannot upload package(%s) since it already exists! \n" + " You may use `--overwrite` option when starting server to disable this check. ", content.raw_filename) @@ -156,31 +156,29 @@ def update(): raise HTTPError(409, msg) try: - gpg_signature = request.files['gpg_signature'] + gpg_sig = request.files['gpg_signature'] except KeyError: - gpg_signature = None - - if (gpg_signature is not None and - (not is_valid_pkg_filename(gpg_signature.raw_filename) - or core.guess_pkgname_and_version(content.raw_filename) is None)): - raise HTTPError(400, "Bad gpg signature name: %s" % - gpg_signature.raw_filename) - - - if not config.overwrite and core.exists(packages.root, - gpg_signature.filename): - log.warn("Cannot upload package(%s) because its signature already " - "exists! \n You may use the `--overwrite` option when" - "starting the server to disable this check.") - msg = ("Signature file already exists! Start server with " - "`--overwrite` option?") - raise HTTPError(409, msg) - - if gpg_signature is None: - core.store(packages.root, content.filename, content.save) + gpg_sig = None else: - core.store(packages.root, content.filename, content.save, - gpg_signature.filename, gpg_signature.save) + if (not is_valid_pkg_filename(gpg_sig.raw_filename) or + core.guess_pkgname_and_version(gpg_sig.raw_filename) is None): + raise HTTPError(400, "Bad gpg signature name: %s" % + gpg_sig.raw_filename) + + if not config.overwrite and core.exists(packages.root, + gpg_sig.raw_filename): + log.warn("Cannot upload package(%s) because its signature already " + "exists! \n You may use the `--overwrite` option when" + "starting the server to disable this check.") + msg = ("Signature file already exists! Start server with " + "`--overwrite` option?") + raise HTTPError(409, msg) + + if gpg_sig is None: + core.store(packages.root, content.raw_filename, content.save) + else: + core.store(packages.root, content.raw_filename, content.save, + gpg_sig.raw_filename, gpg_sig.save) return "" From 140796bb6ec250d858292764a008ef198098f2e8 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 19 Jan 2016 18:46:51 +0100 Subject: [PATCH 7/9] gpg, #106, test: Add upload+sig TC. - Rename upload-file vars in /upload action. --- pypiserver/_app.py | 32 ++++++++++++++++---------------- tests/test_app.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 8797a95..79ab7a0 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -140,33 +140,33 @@ def update(): raise HTTPError(400, "Unsupported ':action' field: %s" % action) try: - content = request.files['content'] + req_pkg = request.files['content'] except KeyError: raise HTTPError(400, "Missing 'content' file-field!") - if (not is_valid_pkg_filename(content.raw_filename) or - core.guess_pkgname_and_version(content.raw_filename) is None): - raise HTTPError(400, "Bad filename: %s" % content.raw_filename) + if (not is_valid_pkg_filename(req_pkg.raw_filename) or + core.guess_pkgname_and_version(req_pkg.raw_filename) is None): + raise HTTPError(400, "Bad filename: %s" % req_pkg.raw_filename) - if not config.overwrite and core.exists(packages.root, content.raw_filename): + if not config.overwrite and core.exists(packages.root, req_pkg.raw_filename): log.warn("Cannot upload package(%s) since it already exists! \n" + " You may use `--overwrite` option when starting server to disable this check. ", - content.raw_filename) + req_pkg.raw_filename) msg = "Package already exists! Start server with `--overwrite` option?" raise HTTPError(409, msg) try: - gpg_sig = request.files['gpg_signature'] + req_sig = request.files['gpg_signature'] except KeyError: - gpg_sig = None + req_sig = None else: - if (not is_valid_pkg_filename(gpg_sig.raw_filename) or - core.guess_pkgname_and_version(gpg_sig.raw_filename) is None): + if (not is_valid_pkg_filename(req_sig.raw_filename) or + core.guess_pkgname_and_version(req_sig.raw_filename) is None): raise HTTPError(400, "Bad gpg signature name: %s" % - gpg_sig.raw_filename) + req_sig.raw_filename) if not config.overwrite and core.exists(packages.root, - gpg_sig.raw_filename): + req_sig.raw_filename): log.warn("Cannot upload package(%s) because its signature already " "exists! \n You may use the `--overwrite` option when" "starting the server to disable this check.") @@ -174,11 +174,11 @@ def update(): "`--overwrite` option?") raise HTTPError(409, msg) - if gpg_sig is None: - core.store(packages.root, content.raw_filename, content.save) + if req_sig is None: + core.store(packages.root, req_pkg.raw_filename, req_pkg.save) else: - core.store(packages.root, content.raw_filename, content.save, - gpg_sig.raw_filename, gpg_sig.save) + core.store(packages.root, req_pkg.raw_filename, req_pkg.save, + req_sig.raw_filename, req_sig.save) return "" diff --git a/tests/test_app.py b/tests/test_app.py index 9e4e3e9..3081a4d 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -334,6 +334,20 @@ def test_upload(package, root, testapp): assert len(uploaded_pkgs) == 1 assert uploaded_pkgs[0].lower() == package.lower() +@pytest.mark.parametrize(("package"), [f[0] + for f in test_core.files + if f[1] and '/' not in f[0]]) +def test_upload_with_signature(package, root, testapp): + resp = testapp.post("/", params={':action': 'file_upload'}, + upload_files=[ + ('content', package, b''), + ('gpg_signature', '%s.asc' % package, b'')]) + assert resp.status_int == 200 + uploaded_pkgs = [f.basename for f in root.listdir()] + assert len(uploaded_pkgs) == 2 + assert uploaded_pkgs[0].lower() == package.lower() + assert uploaded_pkgs[1].lower() == '%s.asc' % package.lower() + @pytest.mark.parametrize(("package"), [ f[0] for f in test_core.files if f[1] is None]) From 6dacda5677685ef866b732ad4083e7aa3e3a6ec1 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 19 Jan 2016 20:04:54 +0100 Subject: [PATCH 8/9] gpg(#103), app: Rewritte /update to store pkg/sig in a loop. + Also check that sig-name matches uploaded pkg-name. + Log ignored /upload actions. --- pypiserver/_app.py | 94 ++++++++++++++++++++-------------------------- pypiserver/core.py | 18 ++++----- 2 files changed, 48 insertions(+), 64 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 79ab7a0..a75466f 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -1,10 +1,15 @@ -import os -import zipfile -import mimetypes +from collections import namedtuple import logging +import mimetypes +import os import re +import sys +import zipfile +from . import __version__ from . import core +from .bottle import static_file, redirect, request, response, HTTPError, Bottle, template + try: from io import BytesIO @@ -16,8 +21,6 @@ try: # PY3 except ImportError: # PY2 from urlparse import urljoin -from .bottle import static_file, redirect, request, response, HTTPError, Bottle, template -from . import __version__ log = logging.getLogger(__name__) packages = None @@ -96,6 +99,9 @@ def is_valid_pkg_filename(fname): return _bottle_upload_filename_re.match(fname) is not None +Upload = namedtuple('Upload', 'pkg sig') + + @app.post('/') @auth("update") def update(): @@ -105,9 +111,8 @@ def update(): raise HTTPError(400, "Missing ':action' field!") if action in ("verify", "submit"): - return "" - - if action == "doc_upload": + log.warning("Ignored ':action': %s", action) + elif action == "doc_upload": try: content = request.files['content'] except KeyError: @@ -118,9 +123,7 @@ def update(): zf.getinfo('index.html') except Exception: raise HTTPError(400, "not a zip file") - return "" - - if action == "remove_pkg": + elif action == "remove_pkg": name = request.forms.get("name") version = request.forms.get("version") if not name or not version: @@ -134,52 +137,35 @@ def update(): if found is None: raise HTTPError(404, "%s (%s) not found" % (name, version)) os.unlink(found.fn) - return "" + elif action == "file_upload": + ufiles = Upload._make( + request.files.get(f, None) for f in ('content', 'gpg_signature')) + if not ufiles.pkg: + raise HTTPError(400, "Missing 'content' file-field!") + if (ufiles.sig and + '%s.asc' % ufiles.pkg.raw_filename != ufiles.sig.raw_filename): + raise HTTPError(400, "Unrelated signature %r for package %r!", + ufiles.sig, ufiles.pkg) - if action != "file_upload": + for uf in ufiles: + if not uf: + continue + if (not is_valid_pkg_filename(uf.raw_filename) or + core.guess_pkgname_and_version(uf.raw_filename) is None): + raise HTTPError(400, "Bad filename: %s" % uf.raw_filename) + + if not config.overwrite and core.exists(packages.root, uf.raw_filename): + log.warn("Cannot upload %r since it already exists! \n" + " You may start server with `--overwrite` option. ", + uf.raw_filename) + HTTPError(409, "Package %r already exists!\n" + " You may start server with `--overwrite` option.", + uf.raw_filename) + + core.store(packages.root, uf.raw_filename, uf.save) + else: raise HTTPError(400, "Unsupported ':action' field: %s" % action) - try: - req_pkg = request.files['content'] - except KeyError: - raise HTTPError(400, "Missing 'content' file-field!") - - if (not is_valid_pkg_filename(req_pkg.raw_filename) or - core.guess_pkgname_and_version(req_pkg.raw_filename) is None): - raise HTTPError(400, "Bad filename: %s" % req_pkg.raw_filename) - - if not config.overwrite and core.exists(packages.root, req_pkg.raw_filename): - log.warn("Cannot upload package(%s) since it already exists! \n" + - " You may use `--overwrite` option when starting server to disable this check. ", - req_pkg.raw_filename) - msg = "Package already exists! Start server with `--overwrite` option?" - raise HTTPError(409, msg) - - try: - req_sig = request.files['gpg_signature'] - except KeyError: - req_sig = None - else: - if (not is_valid_pkg_filename(req_sig.raw_filename) or - core.guess_pkgname_and_version(req_sig.raw_filename) is None): - raise HTTPError(400, "Bad gpg signature name: %s" % - req_sig.raw_filename) - - if not config.overwrite and core.exists(packages.root, - req_sig.raw_filename): - log.warn("Cannot upload package(%s) because its signature already " - "exists! \n You may use the `--overwrite` option when" - "starting the server to disable this check.") - msg = ("Signature file already exists! Start server with " - "`--overwrite` option?") - raise HTTPError(409, msg) - - if req_sig is None: - core.store(packages.root, req_pkg.raw_filename, req_pkg.save) - else: - core.store(packages.root, req_pkg.raw_filename, req_pkg.save, - req_sig.raw_filename, req_sig.save) - return "" diff --git a/pypiserver/core.py b/pypiserver/core.py index 0898b9f..ba7c69d 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -1,10 +1,11 @@ #! /usr/bin/env python """minimal PyPI like server for use with pip/easy_install""" +from collections import namedtuple +import functools import hashlib import io import itertools -import functools import logging import mimetypes import os @@ -12,8 +13,10 @@ import re import sys import pkg_resources + from . import Configuration + log = logging.getLogger(__name__) @@ -130,7 +133,6 @@ _archive_suffix_rx = re.compile( r"(\.zip|\.tar\.gz|\.tgz|\.tar\.bz2|-py[23]\.\d-.*|" "\.win-amd64-py[23]\.\d\..*|\.win32-py[23]\.\d\..*|\.egg)$", re.I) - wheel_file_re = re.compile( r"""^(?P(?P.+?)-(?P\d.*?)) ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) @@ -281,16 +283,12 @@ def exists(root, filename): return os.path.exists(dest_fn) -def store(root, filename, save_method, - gpg_filename=None, gpg_save_method=None): +def store(root, filename, save_method): assert "/" not in filename dest_fn = os.path.join(root, filename) - save_method(dest_fn, overwrite=True) # Overwite check elsewhere. - if gpg_filename is not None: - gpg_dest_fn = os.path.join(root, gpg_filename) - save_method(gpg_dest_fn, overwrite=True) - log.info("Stored package: %s", filename) - return True + save_method(dest_fn, overwrite=True) # Overwite check earlier. + log.info("Stored %r.", filename) + def digest_file(fpath, hash_algo): """ From 62fa55f250688f708aed18e43abd57322c999487 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 19 Jan 2016 20:07:00 +0100 Subject: [PATCH 9/9] app: Split /upload action into sub-functions. + pep8-formatted _app.py. --- pypiserver/_app.py | 120 +++++++++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 53 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index a75466f..594549e 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -94,14 +94,73 @@ def root(): ) _bottle_upload_filename_re = re.compile(r'^[a-z0-9_.!+-]+$', re.I) + + def is_valid_pkg_filename(fname): """See https://github.com/pypiserver/pypiserver/issues/102""" return _bottle_upload_filename_re.match(fname) is not None +def doc_upload(): + try: + content = request.files['content'] + except KeyError: + raise HTTPError(400, "Missing 'content' file-field!") + zip_data = content.file.read() + try: + zf = zipfile.ZipFile(BytesIO(zip_data)) + zf.getinfo('index.html') + except Exception: + raise HTTPError(400, "not a zip file") + + +def remove_pkg(): + name = request.forms.get("name") + version = request.forms.get("version") + if not name or not version: + msg = "Missing 'name'/'version' fields: name=%s, version=%s" + raise HTTPError(400, msg % (name, version)) + found = None + for pkg in core.find_packages(packages()): + if pkg.pkgname == name and pkg.version == version: + found = pkg + break + if found is None: + raise HTTPError(404, "%s (%s) not found" % (name, version)) + os.unlink(found.fn) + + Upload = namedtuple('Upload', 'pkg sig') +def file_upload(): + ufiles = Upload._make( + request.files.get(f, None) for f in ('content', 'gpg_signature')) + if not ufiles.pkg: + raise HTTPError(400, "Missing 'content' file-field!") + if (ufiles.sig and + '%s.asc' % ufiles.pkg.raw_filename != ufiles.sig.raw_filename): + raise HTTPError(400, "Unrelated signature %r for package %r!", + ufiles.sig, ufiles.pkg) + + for uf in ufiles: + if not uf: + continue + if (not is_valid_pkg_filename(uf.raw_filename) or + core.guess_pkgname_and_version(uf.raw_filename) is None): + raise HTTPError(400, "Bad filename: %s" % uf.raw_filename) + + if not config.overwrite and core.exists(packages.root, uf.raw_filename): + log.warn("Cannot upload %r since it already exists! \n" + " You may start server with `--overwrite` option. ", + uf.raw_filename) + HTTPError(409, "Package %r already exists!\n" + " You may start server with `--overwrite` option.", + uf.raw_filename) + + core.store(packages.root, uf.raw_filename, uf.save) + + @app.post('/') @auth("update") def update(): @@ -113,56 +172,11 @@ def update(): if action in ("verify", "submit"): log.warning("Ignored ':action': %s", action) elif action == "doc_upload": - try: - content = request.files['content'] - except KeyError: - raise HTTPError(400, "Missing 'content' file-field!") - zip_data = content.file.read() - try: - zf = zipfile.ZipFile(BytesIO(zip_data)) - zf.getinfo('index.html') - except Exception: - raise HTTPError(400, "not a zip file") + doc_upload() elif action == "remove_pkg": - name = request.forms.get("name") - version = request.forms.get("version") - if not name or not version: - msg = "Missing 'name'/'version' fields: name=%s, version=%s" - raise HTTPError(400, msg % (name, version)) - found = None - for pkg in core.find_packages(packages()): - if pkg.pkgname == name and pkg.version == version: - found = pkg - break - if found is None: - raise HTTPError(404, "%s (%s) not found" % (name, version)) - os.unlink(found.fn) + remove_pkg() elif action == "file_upload": - ufiles = Upload._make( - request.files.get(f, None) for f in ('content', 'gpg_signature')) - if not ufiles.pkg: - raise HTTPError(400, "Missing 'content' file-field!") - if (ufiles.sig and - '%s.asc' % ufiles.pkg.raw_filename != ufiles.sig.raw_filename): - raise HTTPError(400, "Unrelated signature %r for package %r!", - ufiles.sig, ufiles.pkg) - - for uf in ufiles: - if not uf: - continue - if (not is_valid_pkg_filename(uf.raw_filename) or - core.guess_pkgname_and_version(uf.raw_filename) is None): - raise HTTPError(400, "Bad filename: %s" % uf.raw_filename) - - if not config.overwrite and core.exists(packages.root, uf.raw_filename): - log.warn("Cannot upload %r since it already exists! \n" - " You may start server with `--overwrite` option. ", - uf.raw_filename) - HTTPError(409, "Package %r already exists!\n" - " You may start server with `--overwrite` option.", - uf.raw_filename) - - core.store(packages.root, uf.raw_filename, uf.save) + file_upload() else: raise HTTPError(400, "Unsupported ':action' field: %s" % action) @@ -213,7 +227,7 @@ def simple(prefix=""): links = [(os.path.basename(f.relfn), urljoin(fp, "../../packages/%s#%s" % (f.relfn_unix, - f.hash(config.hash_algo)))) + f.hash(config.hash_algo)))) for f in files] tmpl = """\ @@ -240,11 +254,11 @@ def list_packages(): fp += "/" files = sorted(core.find_packages(packages()), - key=lambda x: (os.path.dirname(x.relfn), - x.pkgname, - x.parsed_version)) + key=lambda x: (os.path.dirname(x.relfn), + x.pkgname, + x.parsed_version)) links = [(f.relfn_unix, '%s#%s' % (urljoin(fp, f.relfn), - f.hash(config.hash_algo))) + f.hash(config.hash_algo))) for f in files] tmpl = """\