diff --git a/pypi-server b/pypi-server index 266a12a..7758008 100755 --- a/pypi-server +++ b/pypi-server @@ -1,5 +1,5 @@ #! /usr/bin/env python - +## Failback script if installed with `distutils`. if __name__ == "__main__": from pypiserver.core import main main() diff --git a/pypi-server-in.py b/pypi-server-in.py index d0ca1d4..c578aec 100755 --- a/pypi-server-in.py +++ b/pypi-server-in.py @@ -62,9 +62,5 @@ importer.sources = sources sys.meta_path.append(importer) if __name__ == "__main__": - from pypiserver import core - if sys.version_info >= (2, 6): - core.DEFAULT_SERVER = "waitress" - else: - core.bottle.AutoServer.adapters.remove(core.bottle.WaitressServer) - core.main() + from pypiserver import __main__ + __main__.main() diff --git a/pypiserver/__main__.py b/pypiserver/__main__.py index a3fdd84..1fe50bd 100644 --- a/pypiserver/__main__.py +++ b/pypiserver/__main__.py @@ -1,6 +1,301 @@ +#! /usr/bin/env python + +import os +import sys +import getopt +import re +import logging +from pypiserver import __version__ + +DEFAULT_SERVER = "auto" + +log = logging.getLogger('pypiserver.main') + + +def init_logging(level=None, frmt=None, filename=None): + logging.basicConfig(level=level, format=format) + rlog = logging.getLogger() + rlog.setLevel(level) + if filename: + rlog.addHandler(logging.FileHandler(filename)) + + +def usage(): + sys.stdout.write("""pypi-server [OPTIONS] [PACKAGES_DIRECTORY...] + start PyPI compatible package server serving packages from + PACKAGES_DIRECTORY. If PACKAGES_DIRECTORY is not given on the + command line, it uses the default ~/packages. pypiserver scans this + directory recursively for packages. It skips packages and + directories starting with a dot. Multiple package directories can be + specified. + +pypi-server understands the following options: + + -p, --port PORT + listen on port PORT (default: 8080) + + -i, --interface INTERFACE + listen on interface INTERFACE (default: 0.0.0.0, any interface) + + -a, --authenticate (UPDATE|download|list), ... + comma-separated list of (case-insensitive) actions to authenticate + Requires -P option and cannot not be empty unless -P is '.' + For example to password-protect package downloads (in addition to uploads) + while leaving listings public, give: + -P foo/htpasswd.txt -a update,download + To drop all authentications, use: + -P . -a '' + By default, only 'update' is password-protected. + + -P, --passwords PASSWORD_FILE + use apache htpasswd file PASSWORD_FILE to set usernames & passwords + used for authentication of certain actions (see -a option). + Set it explicitly to '.' to allow empty list of actions to authenticate; + then no `register` command is neccessary, but `~/.pypirc` still needs + `username` and `password` fields, even if bogus. + + --disable-fallback + disable redirect to real PyPI index for packages not found in the + local index + + --fallback-url FALLBACK_URL + for packages not found in the local index, this URL will be used to + redirect to (default: http://pypi.python.org/simple) + + --server METHOD + use METHOD to run the server. Valid values include paste, + cherrypy, twisted, gunicorn, gevent, wsgiref, auto. The + default is to use "auto" which chooses one of paste, cherrypy, + twisted or wsgiref. + + -r, --root PACKAGES_DIRECTORY + [deprecated] serve packages from PACKAGES_DIRECTORY + + -o, --overwrite + allow overwriting existing package files + + --welcome HTML_FILE + uses the ASCII contents of HTML_FILE as welcome message response. + + -v + enable verbose logging; repeat for more verbosity. + + --log-file + write logging info into this FILE. + + --log-frmt + the logging format-string. (see `logging.LogRecord` class from standard python library) + [Default: %(asctime)s|%(levelname)s|%(thread)d|%(message)s] + + --log-req-frmt FORMAT + a format-string selecting Http-Request properties to log; set to '%s' to see them all. + [Default: %(bottle.request)s] + + --log-res-frmt FORMAT + a format-string selecting Http-Response properties to log; set to '%s' to see them all. + [Default: %(status)s] + + --log-err-frmt FORMAT + a format-string selecting Http-Error properties to log; set to '%s' to see them all. + [Default: %(body)s: %(exception)s \n%(traceback)s] + + --cache-control AGE + Add "Cache-Control: max-age=AGE, public" header to package downloads. + Pip 6+ needs this for caching. + + +pypi-server -h +pypi-server --help + show this help message + +pypi-server --version + show pypi-server's version + +pypi-server -U [OPTIONS] [PACKAGES_DIRECTORY...] + update packages in PACKAGES_DIRECTORY. This command searches + pypi.python.org for updates and shows a pip command line which + updates the package. + +The following additional options can be specified with -U: + + -x + execute the pip commands instead of only showing them + + -d DOWNLOAD_DIRECTORY + download package updates to this directory. The default is to use + the directory which contains the latest version of the package to + be updated. + + -u + allow updating to unstable version (alpha, beta, rc, dev versions) + +Visit https://pypi.python.org/pypi/pypiserver for more information. +""") + + +def main(argv=None): + if argv is None: + argv = sys.argv + + global packages + + command = "serve" + host = "0.0.0.0" + port = 8080 + server = DEFAULT_SERVER + redirect_to_fallback = True + fallback_url = "http://pypi.python.org/simple" + authenticated = ['update'] + password_file = None + overwrite = False + verbosity = 1 + log_file = None + log_frmt = "g%(asctime)s|%(levelname)s|%(thread)d|%(message)s" + log_req_frmt = "%(bottle.request)s" + log_res_frmt = "%(status)s" + log_err_frmt = "%(body)s: %(exception)s \n%(traceback)s" + welcome_file = None + cache_control = None + + update_dry_run = True + update_directory = None + update_stable_only = True + + try: + opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [ + "interface=", + "passwords=", + "authenticate=", + "port=", + "root=", + "server=", + "fallback-url=", + "disable-fallback", + "overwrite", + "log-file=", + "log-frmt=", + "log-req-frmt=", + "log-res-frmt=", + "log-err-frmt=", + "welcome=", + "cache-control=", + "version", + "help" + ]) + except getopt.GetoptError: + err = sys.exc_info()[1] + sys.exit("usage error: %s" % (err,)) + + for k, v in opts: + if k in ("-p", "--port"): + port = int(v) + elif k in ("-a", "--authenticate"): + authenticated = [a.lower() + for a in re.split("[, ]+", v.strip(" ,")) + if a] + actions = ("list", "download", "update") + for a in authenticated: + if a not in actions: + errmsg = "Action '%s' for option `%s` not one of %s!" % ( + a, k, actions) + sys.exit(errmsg) + elif k in ("-i", "--interface"): + host = v + elif k in ("-r", "--root"): + roots.append(v) + elif k == "--disable-fallback": + redirect_to_fallback = False + elif k == "--fallback-url": + fallback_url = v + elif k == "--server": + server = v + elif k == "--welcome": + welcome_file = v + elif k == "--version": + sys.stdout.write("pypiserver %s\n" % __version__) + sys.exit(0) + elif k == "-U": + command = "update" + elif k == "-x": + update_dry_run = False + elif k == "-u": + update_stable_only = False + elif k == "-d": + update_directory = v + elif k in ("-P", "--passwords"): + password_file = v + elif k in ("-o", "--overwrite"): + overwrite = True + elif k == "--log-file": + log_file = v + elif k == "--log-frmt": + log_frmt = v + elif k == "--log-req-frmt": + log_req_frmt = v + elif k == "--log-res-frmt": + log_res_frmt = v + elif k == "--log-err-frmt": + log_err_frmt = v + elif k == "--cache-control": + cache_control = v + elif k == "-v": + verbosity += 1 + elif k in ("-h", "--help"): + usage() + sys.exit(0) + + if password_file and password_file != '.' and not authenticated: + sys.exit( + "Actions to authenticate (-a) must not be empty, unless password file (-P) is '.'!") + + if len(roots) == 0: + roots.append(os.path.expanduser("~/packages")) + + roots = [os.path.abspath(x) for x in roots] + + verbose_levels = [ + logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET] + log_level = list(zip(verbose_levels, range(verbosity)))[-1][0] + init_logging(level=log_level, filename=log_file, frmt=log_frmt) + + if command == "update": + from pypiserver.manage import update_all_packages + update_all_packages( + roots, update_directory, update_dry_run, stable_only=update_stable_only) + return + + # Fixes #49: + # The gevent server adapter needs to patch some + # modules BEFORE importing bottle! + if server and server.startswith('gevent'): + import gevent.monkey # @UnresolvedImport + gevent.monkey.patch_all() + + from pypiserver.bottle import server_names, run + if server not in server_names: + sys.exit("unknown server %r. choose one of %s" % ( + server, ", ".join(server_names.keys()))) + + from pypiserver import app + a = app( + root=roots, + redirect_to_fallback=redirect_to_fallback, + authenticated=authenticated, + password_file=password_file, + fallback_url=fallback_url, + overwrite=overwrite, + log_req_frmt=log_req_frmt, log_res_frmt=log_res_frmt, log_err_frmt=log_err_frmt, + welcome_file=welcome_file, + cache_control=cache_control, + ) + log.info("This is pypiserver %s serving %r on http://%s:%s\n\n", + __version__, ", ".join(roots), host, port) + run(app=a, host=host, port=port, server=server) + + if __name__ == "__main__": - if __package__ == "": # running as python pypiserver-...whl/pypiserver? - import sys, os - sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) - from pypiserver import core - core.main() +# if __package__ == "": # running as python pypiserver-...whl/pypiserver? +# import sys +# import os +# sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + main() diff --git a/pypiserver/core.py b/pypiserver/core.py index 0624006..0a79bcf 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -2,24 +2,15 @@ """minimal PyPI like server for use with pip/easy_install""" import os -import sys -import getopt import re import mimetypes import warnings -import itertools import logging warnings.filterwarnings("ignore", "Python 2.5 support may be dropped in future versions of Bottle") -from pypiserver import bottle, __version__, app - -sys.modules["bottle"] = bottle -from bottle import run, server_names - mimetypes.add_type("application/octet-stream", ".egg") mimetypes.add_type("application/octet-stream", ".whl") -DEFAULT_SERVER = None log = logging.getLogger('pypiserver.core') # --- the following two functions were copied from distribute's pkg_resources module @@ -27,14 +18,6 @@ component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) replace = {'pre': 'c', 'preview': 'c', '-': 'final-', 'rc': 'c', 'dev': '@'}.get -def init_logging(level=None, format=None, filename=None): - logging.basicConfig(level=level, format=format) - rlog = logging.getLogger() - rlog.setLevel(level) - if filename: - rlog.addHandler(logging.FileHandler(filename)) - - def _parse_version_parts(s): for part in component_re.split(s): part = replace(part, part) @@ -188,269 +171,3 @@ def store(root, filename, data): log.info("Stored package: %s", filename) return True - - -def usage(): - sys.stdout.write("""pypi-server [OPTIONS] [PACKAGES_DIRECTORY...] - start PyPI compatible package server serving packages from - PACKAGES_DIRECTORY. If PACKAGES_DIRECTORY is not given on the - command line, it uses the default ~/packages. pypiserver scans this - directory recursively for packages. It skips packages and - directories starting with a dot. Multiple package directories can be - specified. - -pypi-server understands the following options: - - -p, --port PORT - listen on port PORT (default: 8080) - - -i, --interface INTERFACE - listen on interface INTERFACE (default: 0.0.0.0, any interface) - - -a, --authenticate (UPDATE|download|list), ... - comma-separated list of (case-insensitive) actions to authenticate - Requires -P option and cannot not be empty unless -P is '.' - For example to password-protect package downloads (in addition to uploads) - while leaving listings public, give: - -P foo/htpasswd.txt -a update,download - To drop all authentications, use: - -P . -a '' - By default, only 'update' is password-protected. - - -P, --passwords PASSWORD_FILE - use apache htpasswd file PASSWORD_FILE to set usernames & passwords - used for authentication of certain actions (see -a option). - Set it explicitly to '.' to allow empty list of actions to authenticate; - then no `register` command is neccessary, but `~/.pypirc` still needs - `username` and `password` fields, even if bogus. - - --disable-fallback - disable redirect to real PyPI index for packages not found in the - local index - - --fallback-url FALLBACK_URL - for packages not found in the local index, this URL will be used to - redirect to (default: http://pypi.python.org/simple) - - --server METHOD - use METHOD to run the server. Valid values include paste, - cherrypy, twisted, gunicorn, gevent, wsgiref, auto. The - default is to use "auto" which chooses one of paste, cherrypy, - twisted or wsgiref. - - -r, --root PACKAGES_DIRECTORY - [deprecated] serve packages from PACKAGES_DIRECTORY - - -o, --overwrite - allow overwriting existing package files - - --welcome HTML_FILE - uses the ASCII contents of HTML_FILE as welcome message response. - - -v - enable verbose logging; repeat for more verbosity. - - --log-file - write logging info into this FILE. - - --log-frmt - the logging format-string. (see `logging.LogRecord` class from standard python library) - [Default: %(asctime)s|%(levelname)s|%(thread)d|%(message)s] - - --log-req-frmt FORMAT - a format-string selecting Http-Request properties to log; set to '%s' to see them all. - [Default: %(bottle.request)s] - - --log-res-frmt FORMAT - a format-string selecting Http-Response properties to log; set to '%s' to see them all. - [Default: %(status)s] - - --log-err-frmt FORMAT - a format-string selecting Http-Error properties to log; set to '%s' to see them all. - [Default: %(body)s: %(exception)s \n%(traceback)s] - - --cache-control AGE - Add "Cache-Control: max-age=AGE, public" header to package downloads. - Pip 6+ needs this for caching. - - -pypi-server -h -pypi-server --help - show this help message - -pypi-server --version - show pypi-server's version - -pypi-server -U [OPTIONS] [PACKAGES_DIRECTORY...] - update packages in PACKAGES_DIRECTORY. This command searches - pypi.python.org for updates and shows a pip command line which - updates the package. - -The following additional options can be specified with -U: - - -x - execute the pip commands instead of only showing them - - -d DOWNLOAD_DIRECTORY - download package updates to this directory. The default is to use - the directory which contains the latest version of the package to - be updated. - - -u - allow updating to unstable version (alpha, beta, rc, dev versions) - -Visit https://pypi.python.org/pypi/pypiserver for more information. -""") - - -def main(argv=None): - if argv is None: - argv = sys.argv - - global packages - - command = "serve" - host = "0.0.0.0" - port = 8080 - server = DEFAULT_SERVER - redirect_to_fallback = True - fallback_url = "http://pypi.python.org/simple" - authenticated = ['update'] - password_file = None - overwrite = False - verbosity = 1 - log_file = None - log_frmt = "g%(asctime)s|%(levelname)s|%(thread)d|%(message)s" - log_req_frmt = "%(bottle.request)s" - log_res_frmt = "%(status)s" - log_err_frmt = "%(body)s: %(exception)s \n%(traceback)s" - welcome_file = None - cache_control = None - - update_dry_run = True - update_directory = None - update_stable_only = True - - try: - opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [ - "interface=", - "passwords=", - "authenticate=", - "port=", - "root=", - "server=", - "fallback-url=", - "disable-fallback", - "overwrite", - "log-file=", - "log-frmt=", - "log-req-frmt=", - "log-res-frmt=", - "log-err-frmt=", - "welcome=", - "cache-control=", - "version", - "help" - ]) - except getopt.GetoptError: - err = sys.exc_info()[1] - sys.exit("usage error: %s" % (err,)) - - for k, v in opts: - if k in ("-p", "--port"): - port = int(v) - elif k in ("-a", "--authenticate"): - authenticated = [a.lower() - for a in re.split("[, ]+", v.strip(" ,")) - if a] - actions = ("list", "download", "update") - for a in authenticated: - if a not in actions: - errmsg = "Action '%s' for option `%s` not one of %s!" % (a, k, actions) - sys.exit(errmsg) - elif k in ("-i", "--interface"): - host = v - elif k in ("-r", "--root"): - roots.append(v) - elif k == "--disable-fallback": - redirect_to_fallback = False - elif k == "--fallback-url": - fallback_url = v - elif k == "--server": - if v not in server_names: - sys.exit("unknown server %r. choose one of %s" % ( - v, ", ".join(server_names.keys()))) - server = v - elif k == "--welcome": - welcome_file = v - elif k == "--version": - sys.stdout.write("pypiserver %s\n" % __version__) - sys.exit(0) - elif k == "-U": - command = "update" - elif k == "-x": - update_dry_run = False - elif k == "-u": - update_stable_only = False - elif k == "-d": - update_directory = v - elif k in ("-P", "--passwords"): - password_file = v - elif k in ("-o", "--overwrite"): - overwrite = True - elif k == "--log-file": - log_file = v - elif k == "--log-frmt": - log_frmt = v - elif k == "--log-req-frmt": - log_req_frmt = v - elif k == "--log-res-frmt": - log_res_frmt = v - elif k == "--log-err-frmt": - log_err_frmt = v - elif k == "--cache-control": - cache_control = v - elif k == "-v": - verbosity += 1 - elif k in ("-h", "--help"): - usage() - sys.exit(0) - - if password_file and password_file != '.' and not authenticated: - sys.exit("Actions to authenticate (-a) must not be empty, unless password file (-P) is '.'!") - - if len(roots) == 0: - roots.append(os.path.expanduser("~/packages")) - - roots = [os.path.abspath(x) for x in roots] - - verbose_levels = [logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET] - log_level = list(zip(verbose_levels, range(verbosity)))[-1][0] - init_logging(level=log_level, filename=log_file, format=log_frmt) - - if command == "update": - packages = frozenset(itertools.chain(*[listdir(r) for r in roots])) - from pypiserver import manage - - manage.update(packages, update_directory, update_dry_run, stable_only=update_stable_only) - return - - a = app( - root=roots, - redirect_to_fallback=redirect_to_fallback, - authenticated=authenticated, - password_file=password_file, - fallback_url=fallback_url, - overwrite=overwrite, - log_req_frmt=log_req_frmt, log_res_frmt=log_res_frmt, log_err_frmt=log_err_frmt, - welcome_file=welcome_file, - cache_control=cache_control, - ) - server = server or "auto" - log.info("This is pypiserver %s serving %r on http://%s:%s\n\n", - __version__, ", ".join(roots), host, port) - run(app=a, host=host, port=port, server=server) - - -if __name__ == "__main__": - main(sys.argv) diff --git a/pypiserver/manage.py b/pypiserver/manage.py index 397debb..3dac95a 100644 --- a/pypiserver/manage.py +++ b/pypiserver/manage.py @@ -3,6 +3,7 @@ import os from subprocess import call from pypiserver import core +import itertools if sys.version_info >= (3, 0): from xmlrpc.client import Server @@ -146,3 +147,7 @@ def update(pkgset, destdir=None, dry_run=False, stable_only=True): sys.stdout.write("%s\n\n" % (" ".join(cmd),)) if not dry_run: call(cmd) + +def update_all_packages(roots, destdir=None, dry_run=False, stable_only=True): + packages = frozenset(itertools.chain(*[core.listdir(r) for r in roots])) + update(packages, destdir, dry_run, stable_only) diff --git a/setup.py b/setup.py index 28ee3c9..c5b8f19 100644 --- a/setup.py +++ b/setup.py @@ -4,17 +4,17 @@ import sys, os try: from setuptools import setup - extra = dict(entry_points={ - 'paste.app_factory': ['main=pypiserver:paste_app_factory'], - 'console_scripts': ['pypi-server=pypiserver.core:main'] - }) + extra = {'entry_points': { + 'paste.app_factory': ['main=pypiserver:paste_app_factory'], + 'console_scripts': ['pypi-server=pypiserver.__main__:main'] + }} except ImportError: from distutils.core import setup extra = dict(scripts=["pypi-server"]) if sys.version_info >= (3, 0): exec("def do_exec(co, loc): exec(co, loc)\n") - tests_require = [] + tests_require = [] else: exec("def do_exec(co, loc): exec co in loc\n") tests_require = ['mock'] diff --git a/tests/test_app.py b/tests/test_app.py index 4dd7863..4a1e02b 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,12 +1,12 @@ #! /usr/bin/env py.test -from pypiserver import core # do no remove. needed for bottle -import pytest, bottle, webtest +from pypiserver import __main__, bottle # do no remove. needed for bottle +import pytest, webtest ## Enable logging to detect any problems with it ## import logging -core.init_logging(level=logging.NOTSET) +__main__.init_logging(level=logging.NOTSET) @pytest.fixture() def _app(app): diff --git a/tests/test_core.py b/tests/test_core.py index 6cc4f9c..e365488 100755 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,12 +1,12 @@ #! /usr/bin/env py.test import pytest -from pypiserver import core +from pypiserver import core, __main__ ## Enable logging to detect any problems with it ## import logging -core.init_logging(level=logging.NOTSET) +__main__.init_logging(level=logging.NOTSET) files = [ diff --git a/tests/test_main.py b/tests/test_main.py index 0fe253d..54f1711 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,7 +1,7 @@ #! /usr/bin/env py.test import sys, os, pytest, logging -from pypiserver import core +from pypiserver import core, __main__ try: from unittest import mock except ImportError: @@ -16,7 +16,7 @@ class main_wrapper(object): def __call__(self, argv): sys.stdout.write("Running %s\n" % (argv,)) - core.main(["pypi-server"] + argv) + __main__.main(["pypi-server"] + argv) return self.run_kwargs @@ -35,8 +35,8 @@ def main(request, monkeypatch): main.pkgdir = pkgdir return [] - monkeypatch.setattr(core, "run", run) - monkeypatch.setattr(os, "listdir", listdir) + monkeypatch.setattr("pypiserver.bottle.run", run) + monkeypatch.setattr("os.listdir", listdir) return main