diff --git a/.gitignore b/.gitignore index 25f5d34..7edd45c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,11 +17,14 @@ ID __pycache__/ /build/ /dist/ +/*.egg /MANIFEST /README.html /pypi-server-standalone.py +/.project +/.pydevproject /.tox/ -/pypiserver.egg-info/ +/*.egg-info/ /.standalone/ /.coverage /htmlcov/ @@ -30,3 +33,4 @@ __pycache__/ /develop-eggs/ /eggs/ /parts/ +/.settings/ diff --git a/README.rst b/README.rst index 3b4acfa..5623dd6 100644 --- a/README.rst +++ b/README.rst @@ -88,9 +88,16 @@ pypi-server -h will print a detailed usage message:: -i INTERFACE, --interface INTERFACE listen on interface INTERFACE (default: 0.0.0.0, any interface) + -a (update|download|list), ... --authenticate (update|download|list), ... + comma-separated list of actions to authenticate (requires giving also + the -P option). For example to password-protect package uploads and + downloads while leaving listings public, give: -a update,download. + Note: make sure there is no space around the comma(s); otherwise, an + error will occur. + -P PASSWORD_FILE, --passwords PASSWORD_FILE - use apache htpasswd file PASSWORD_FILE in order to enable password - protected uploads. + use apache htpasswd file PASSWORD_FILE to set usernames & passwords + used for authentication (requires giving the -s option as well). --disable-fallback disable redirect to real PyPI index for packages not found in the @@ -111,7 +118,29 @@ pypi-server -h will print a detailed usage message:: -o, --overwrite allow overwriting existing package files - + + -v + enable INFO logging; repeate for more verbosity. + + --log-file + write logging info into this FILE. + + --log-frmt + the logging format-string. (see `logging.LogRecord` class from standard python library) + [Default: %(asctime)s|%(levelname)s|%(thread)d|%(message)s] + + --log-req-frmt FORMAT + a format-string selecting Http-Request properties to log; set to '%s' to see them all. + [Default: %(bottle.request)s] + + --log-res-frmt FORMAT + a format-string selecting Http-Response properties to log; set to '%s' to see them all. + [Default: %(status)s] + + --log-err-frmt FORMAT + a format-string selecting Http-Error properties to log; set to '%s' to see them all. + [Default: %(body)s: %(exception)s \n%(traceback)s] + pypi-server -h pypi-server --help show this help message @@ -446,6 +475,13 @@ proxypypi (https://pypi.python.org/pypi/proxypypi) Changelog ========= + +1.1.7 (2015-01-10) +------------------ +- support password protected package listings and downloads, + in addition to uploads (use the -a, --authenticate option + to specify which to protect). + 1.1.6 (2014-03-05) ------------------ - remove --index-url cli parameter introduced in 1.1.5 diff --git a/pypiserver/__init__.py b/pypiserver/__init__.py index 657e76e..d1616c5 100644 --- a/pypiserver/__init__.py +++ b/pypiserver/__init__.py @@ -5,8 +5,12 @@ version = __version__ = "1.1.6" def app(root=None, redirect_to_fallback=True, fallback_url=None, + authenticated=[], password_file=None, - overwrite=False): + overwrite=False, + log_req_frmt="%(bottle.request)s", + log_res_frmt="%(status)s", + log_err_frmt="%(body)s: %(exception)s \n%(traceback)s"): import sys, os from pypiserver import core sys.modules.pop("pypiserver._app", None) @@ -22,7 +26,8 @@ def app(root=None, fallback_url = "http://pypi.python.org/simple" _app.configure(root=root, redirect_to_fallback=redirect_to_fallback, fallback_url=fallback_url, - password_file=password_file, overwrite=overwrite) + authenticated=authenticated, password_file=password_file, overwrite=overwrite, + log_req_frmt=log_req_frmt, log_res_frmt=log_res_frmt, log_err_frmt=log_err_frmt) _app.app.module = _app bottle.debug(True) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 59d757b..c72ea54 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -1,4 +1,4 @@ -import sys, os, itertools, zipfile, mimetypes +import sys, os, itertools, zipfile, mimetypes, logging try: from io import BytesIO @@ -10,10 +10,11 @@ if sys.version_info >= (3, 0): else: from urlparse import urljoin -from bottle import static_file, redirect, request, HTTPError, Bottle +from bottle import static_file, redirect, request, response, HTTPError, Bottle from pypiserver import __version__ from pypiserver.core import listdir, find_packages, store, get_prefixes, exists +log = logging.getLogger('pypiserver.http') packages = None @@ -32,13 +33,48 @@ def validate_user(username, password): return config.htpasswdfile.check_password(username, password) +class auth(object): + "decorator to apply authentication if specified for the decorated method & action" + + def __init__(self, action): + self.action = action + + def __call__(self, method): + + def protector(*args, **kwargs): + if self.action in config.authenticated: + if not request.auth or request.auth[1] is None: + raise HTTPError(401, header={"WWW-Authenticate": 'Basic realm="pypi"'}) + if not validate_user(*request.auth): + raise HTTPError(403) + return method(*args, **kwargs) + + return protector + + def configure(root=None, redirect_to_fallback=True, fallback_url=None, + authenticated=[], password_file=None, - overwrite=False): + overwrite=False, + log_req_frmt=None, + log_res_frmt=None, + log_err_frmt=None): global packages + log.info("Starting(%s)", dict(root=root, + redirect_to_fallback=redirect_to_fallback, + fallback_url=fallback_url, + authenticated=authenticated, + password_file=password_file, + overwrite=overwrite, + log_req_frmt=log_req_frmt, + log_res_frmt=log_res_frmt, + log_err_frmt=log_err_frmt)) + + config.authenticated = authenticated + if root is None: root = os.path.expanduser("~/packages") @@ -68,9 +104,33 @@ def configure(root=None, config.htpasswdfile = HtpasswdFile(password_file) config.overwrite = overwrite + config.log_req_frmt = log_req_frmt + config.log_res_frmt = log_res_frmt + config.log_err_frmt = log_err_frmt + app = Bottle() +@app.hook('before_request') +def log_request(): + log.info(config.log_req_frmt, request.environ) + + +@app.hook('after_request') +def log_response(): + log.info(config.log_res_frmt, #vars(response)) ## DOES NOT WORK! + dict( + response=response, + status=response.status, headers=response.headers, + body=response.body, cookies=response.COOKIES, + )) + + +@app.error +def log_error(http_error): + log.info(config.log_err_frmt, vars(http_error)) + + @app.route("/favicon.ico") def favicon(): return HTTPError(404) @@ -109,13 +169,8 @@ easy_install -i %(URL)ssimple/ PACKAGE @app.post('/') +@auth("update") def update(): - if not request.auth or request.auth[1] is None: - raise HTTPError(401, header={"WWW-Authenticate": 'Basic realm="pypi"'}) - - if not validate_user(*request.auth): - raise HTTPError(403) - try: action = request.forms[':action'] except KeyError: @@ -171,11 +226,13 @@ def update(): @app.route("/simple") +@auth("list") def simpleindex_redirect(): return redirect(request.fullpath + "/") @app.route("/simple/") +@auth("list") def simpleindex(): res = ["Simple Index\n"] for x in sorted(get_prefixes(packages())): @@ -186,6 +243,7 @@ def simpleindex(): @app.route("/simple/:prefix") @app.route("/simple/:prefix/") +@auth("list") def simple(prefix=""): fp = request.fullpath if not fp.endswith("/"): @@ -209,6 +267,7 @@ def simple(prefix=""): @app.route('/packages') @app.route('/packages/') +@auth("list") def list_packages(): fp = request.fullpath if not fp.endswith("/"): @@ -225,6 +284,7 @@ def list_packages(): @app.route('/packages/:filename#.*#') +@auth("download") def server_static(filename): entries = find_packages(packages()) for x in entries: diff --git a/pypiserver/core.py b/pypiserver/core.py index 3bd4f72..b0859f4 100755 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -1,7 +1,7 @@ #! /usr/bin/env python """minimal PyPI like server for use with pip/easy_install""" -import os, sys, getopt, re, mimetypes, warnings, itertools +import os, sys, getopt, re, mimetypes, warnings, itertools, logging warnings.filterwarnings("ignore", "Python 2.5 support may be dropped in future versions of Bottle") from pypiserver import bottle, __version__, app @@ -12,12 +12,20 @@ mimetypes.add_type("application/octet-stream", ".egg") mimetypes.add_type("application/octet-stream", ".whl") DEFAULT_SERVER = None +log = logging.getLogger('pypiserver.core') # --- the following two functions were copied from distribute's pkg_resources module component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) replace = {'pre': 'c', 'preview': 'c', '-': 'final-', 'rc': 'c', 'dev': '@'}.get +def init_logging(level=None, format=None, filename=None): + logging.basicConfig(level=level, format=format) + rlog = logging.getLogger() + rlog.setLevel(level) + if filename: + rlog.addHandler(logging.FileHandler(filename)) + def _parse_version_parts(s): for part in component_re.split(s): part = replace(part, part) @@ -164,6 +172,8 @@ def store(root, filename, data): dest_fh = open(dest_fn, "wb") dest_fh.write(data) dest_fh.close() + + log.info("Stored package: %s", filename) return True @@ -184,9 +194,16 @@ pypi-server understands the following options: -i INTERFACE, --interface INTERFACE listen on interface INTERFACE (default: 0.0.0.0, any interface) + -a (update|download|list), ... --authenticate (update|download|list), ... + comma-separated list of actions to authenticate (requires giving also + the -P option). For example to password-protect package uploads and + downloads while leaving listings public, give: -a update,download. + Note: make sure there is no space around the comma(s); otherwise, an + error will occur. + -P PASSWORD_FILE, --passwords PASSWORD_FILE - use apache htpasswd file PASSWORD_FILE in order to enable password - protected uploads. + use apache htpasswd file PASSWORD_FILE to set usernames & passwords + used for authentication (requires giving the -s option as well). --disable-fallback disable redirect to real PyPI index for packages not found in the @@ -208,6 +225,28 @@ pypi-server understands the following options: -o, --overwrite allow overwriting existing package files + -v + enable verbose logging; repeate for more verbosity. + + --log-file + write logging info into this FILE. + + --log-frmt + the logging format-string. (see `logging.LogRecord` class from standard python library) + [Default: %(asctime)s|%(levelname)s|%(thread)d|%(message)s] + + --log-req-frmt FORMAT + a format-string selecting Http-Request properties to log; set to '%s' to see them all. + [Default: %(bottle.request)s] + + --log-res-frmt FORMAT + a format-string selecting Http-Response properties to log; set to '%s' to see them all. + [Default: %(status)s] + + --log-err-frmt FORMAT + a format-string selecting Http-Error properties to log; set to '%s' to see them all. + [Default: %(body)s: %(exception)s \n%(traceback)s] + pypi-server -h pypi-server --help show this help message @@ -236,7 +275,6 @@ The following additional options can be specified with -U: Visit http://pypi.python.org/pypi/pypiserver for more information. """) - def main(argv=None): if argv is None: argv = sys.argv @@ -249,23 +287,36 @@ def main(argv=None): server = DEFAULT_SERVER redirect_to_fallback = True fallback_url = "http://pypi.python.org/simple" + authenticated = [] password_file = None overwrite = False + verbosity = 1 + log_file = None + log_frmt = None + log_req_frmt = None + log_res_frmt = None + log_err_frmt = None update_dry_run = True update_directory = None update_stable_only = True try: - opts, roots = getopt.getopt(argv[1:], "i:p:r:d:P:Uuxoh", [ + opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [ "interface=", "passwords=", + "authenticate=", "port=", "root=", "server=", "fallback-url=", "disable-fallback", "overwrite", + "log-file=", + "log-frmt=", + "log-req-frmt=", + "log-res-frmt=", + "log-err-frmt=", "version", "help" ]) @@ -276,6 +327,13 @@ def main(argv=None): for k, v in opts: if k in ("-p", "--port"): port = int(v) + elif k in ("-a", "--authenticate"): + authenticated = [a.strip() for a in v.strip(',').split(',')] + actions = ("list", "download", "update") + for a in authenticated: + if a not in actions: + errmsg = "Incorrect action '%s' given with option '%s'" % (a, k) + sys.exit(errmsg) elif k in ("-i", "--interface"): host = v elif k in ("-r", "--root"): @@ -304,16 +362,34 @@ def main(argv=None): password_file = v elif k in ("-o", "--overwrite"): overwrite = True + elif k == "--log-file": + log_file = v + elif k == "--log-frmt": + log_frmt = v + elif k == "--log-req-frmt": + log_req_frmt = v + elif k == "--log-res-frmt": + log_res_frmt = v + elif k == "--log-err-frmt": + log_err_frmt = v + elif k == "-v": + verbosity += 1 elif k in ("-h", "--help"): usage() sys.exit(0) + if (password_file or authenticated) and not (password_file and authenticated): + sys.exit("Must give both password file (-P) and actions to authenticate (-a).") + if len(roots) == 0: roots.append(os.path.expanduser("~/packages")) roots = [os.path.abspath(x) for x in roots] - + verbose_levels = [logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET] + log_level = list(zip(verbose_levels, range(verbosity)))[-1][0] + init_logging(level=log_level, filename=log_file, format=log_frmt) + if command == "update": packages = frozenset(itertools.chain(*[listdir(r) for r in roots])) from pypiserver import manage @@ -323,9 +399,11 @@ def main(argv=None): a = app( root=roots, redirect_to_fallback=redirect_to_fallback, + authenticated=authenticated, password_file=password_file, fallback_url=fallback_url, overwrite=overwrite, + log_req_frmt=log_req_frmt, log_res_frmt=log_res_frmt, log_err_frmt=log_err_frmt, ) server = server or "auto" sys.stdout.write("This is pypiserver %s serving %r on http://%s:%s\n\n" % (__version__, ", ".join(roots), host, port)) diff --git a/tests/test_app.py b/tests/test_app.py index 8d95d2a..2918f33 100755 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -3,6 +3,10 @@ from pypiserver import core # do no remove. needed for bottle import pytest, bottle, webtest +## Enable logging to detect any problems with it +## +import logging +core.init_logging(level=logging.NOTSET) @pytest.fixture() def _app(app): diff --git a/tests/test_core.py b/tests/test_core.py index 28e786e..f10fb02 100755 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -3,6 +3,11 @@ import pytest from pypiserver import core +## Enable logging to detect any problems with it +## +import logging +core.init_logging(level=logging.NOTSET) + files = [ ("pytz-2012b.tar.bz2", "pytz", "2012b"), diff --git a/tests/test_main.py b/tests/test_main.py index 087195d..3c9bcdd 100755 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,6 @@ #! /usr/bin/env py.test -import sys, os, pytest +import sys, os, pytest, logging from pypiserver import core @@ -84,3 +84,21 @@ def test_fallback_url_default(main): main([]) assert main.app.module.config.fallback_url == \ "http://pypi.python.org/simple" + +@pytest.fixture +def logfile(tmpdir): + return tmpdir.mkdir("logs").join('test.log') + +def test_logging(main, logfile): + main(["-v", "--log-file", logfile.strpath]) + assert logfile.check(), logfile + +def test_logging_verbosity(main): + main([]) + assert logging.getLogger().level == logging.WARN + main(["-v"]) + assert logging.getLogger().level == logging.INFO + main(["-v", "-v"]) + assert logging.getLogger().level == logging.DEBUG + main(["-v", "-v", "-v"]) + assert logging.getLogger().level == logging.NOTSET