Restructure main to allow for gevent monkeypatch (FIX #49).

+ Update zip and setuptools startup-cripts.
This commit is contained in:
Kostis Anagnostopoulos at STUW025 2015-09-15 19:53:59 +02:00 committed by Kostis Anagnostopoulos
parent f6a3342666
commit 2c1ceb04b5
9 changed files with 322 additions and 309 deletions

@ -1,5 +1,5 @@
#! /usr/bin/env python
## Failback script if installed with `distutils`.
if __name__ == "__main__":
from pypiserver.core import main
main()

@ -62,9 +62,5 @@ importer.sources = sources
sys.meta_path.append(importer)
if __name__ == "__main__":
from pypiserver import core
if sys.version_info >= (2, 6):
core.DEFAULT_SERVER = "waitress"
else:
core.bottle.AutoServer.adapters.remove(core.bottle.WaitressServer)
core.main()
from pypiserver import __main__
__main__.main()

@ -1,6 +1,301 @@
#! /usr/bin/env python
import os
import sys
import getopt
import re
import logging
from pypiserver import __version__
DEFAULT_SERVER = "auto"
log = logging.getLogger('pypiserver.main')
def init_logging(level=None, frmt=None, filename=None):
logging.basicConfig(level=level, format=format)
rlog = logging.getLogger()
rlog.setLevel(level)
if filename:
rlog.addHandler(logging.FileHandler(filename))
def usage():
sys.stdout.write("""pypi-server [OPTIONS] [PACKAGES_DIRECTORY...]
start PyPI compatible package server serving packages from
PACKAGES_DIRECTORY. If PACKAGES_DIRECTORY is not given on the
command line, it uses the default ~/packages. pypiserver scans this
directory recursively for packages. It skips packages and
directories starting with a dot. Multiple package directories can be
specified.
pypi-server understands the following options:
-p, --port PORT
listen on port PORT (default: 8080)
-i, --interface INTERFACE
listen on interface INTERFACE (default: 0.0.0.0, any interface)
-a, --authenticate (UPDATE|download|list), ...
comma-separated list of (case-insensitive) actions to authenticate
Requires -P option and cannot not be empty unless -P is '.'
For example to password-protect package downloads (in addition to uploads)
while leaving listings public, give:
-P foo/htpasswd.txt -a update,download
To drop all authentications, use:
-P . -a ''
By default, only 'update' is password-protected.
-P, --passwords PASSWORD_FILE
use apache htpasswd file PASSWORD_FILE to set usernames & passwords
used for authentication of certain actions (see -a option).
Set it explicitly to '.' to allow empty list of actions to authenticate;
then no `register` command is neccessary, but `~/.pypirc` still needs
`username` and `password` fields, even if bogus.
--disable-fallback
disable redirect to real PyPI index for packages not found in the
local index
--fallback-url FALLBACK_URL
for packages not found in the local index, this URL will be used to
redirect to (default: http://pypi.python.org/simple)
--server METHOD
use METHOD to run the server. Valid values include paste,
cherrypy, twisted, gunicorn, gevent, wsgiref, auto. The
default is to use "auto" which chooses one of paste, cherrypy,
twisted or wsgiref.
-r, --root PACKAGES_DIRECTORY
[deprecated] serve packages from PACKAGES_DIRECTORY
-o, --overwrite
allow overwriting existing package files
--welcome HTML_FILE
uses the ASCII contents of HTML_FILE as welcome message response.
-v
enable verbose logging; repeat for more verbosity.
--log-file <FILE>
write logging info into this FILE.
--log-frmt <FILE>
the logging format-string. (see `logging.LogRecord` class from standard python library)
[Default: %(asctime)s|%(levelname)s|%(thread)d|%(message)s]
--log-req-frmt FORMAT
a format-string selecting Http-Request properties to log; set to '%s' to see them all.
[Default: %(bottle.request)s]
--log-res-frmt FORMAT
a format-string selecting Http-Response properties to log; set to '%s' to see them all.
[Default: %(status)s]
--log-err-frmt FORMAT
a format-string selecting Http-Error properties to log; set to '%s' to see them all.
[Default: %(body)s: %(exception)s \n%(traceback)s]
--cache-control AGE
Add "Cache-Control: max-age=AGE, public" header to package downloads.
Pip 6+ needs this for caching.
pypi-server -h
pypi-server --help
show this help message
pypi-server --version
show pypi-server's version
pypi-server -U [OPTIONS] [PACKAGES_DIRECTORY...]
update packages in PACKAGES_DIRECTORY. This command searches
pypi.python.org for updates and shows a pip command line which
updates the package.
The following additional options can be specified with -U:
-x
execute the pip commands instead of only showing them
-d DOWNLOAD_DIRECTORY
download package updates to this directory. The default is to use
the directory which contains the latest version of the package to
be updated.
-u
allow updating to unstable version (alpha, beta, rc, dev versions)
Visit https://pypi.python.org/pypi/pypiserver for more information.
""")
def main(argv=None):
if argv is None:
argv = sys.argv
global packages
command = "serve"
host = "0.0.0.0"
port = 8080
server = DEFAULT_SERVER
redirect_to_fallback = True
fallback_url = "http://pypi.python.org/simple"
authenticated = ['update']
password_file = None
overwrite = False
verbosity = 1
log_file = None
log_frmt = "g%(asctime)s|%(levelname)s|%(thread)d|%(message)s"
log_req_frmt = "%(bottle.request)s"
log_res_frmt = "%(status)s"
log_err_frmt = "%(body)s: %(exception)s \n%(traceback)s"
welcome_file = None
cache_control = None
update_dry_run = True
update_directory = None
update_stable_only = True
try:
opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [
"interface=",
"passwords=",
"authenticate=",
"port=",
"root=",
"server=",
"fallback-url=",
"disable-fallback",
"overwrite",
"log-file=",
"log-frmt=",
"log-req-frmt=",
"log-res-frmt=",
"log-err-frmt=",
"welcome=",
"cache-control=",
"version",
"help"
])
except getopt.GetoptError:
err = sys.exc_info()[1]
sys.exit("usage error: %s" % (err,))
for k, v in opts:
if k in ("-p", "--port"):
port = int(v)
elif k in ("-a", "--authenticate"):
authenticated = [a.lower()
for a in re.split("[, ]+", v.strip(" ,"))
if a]
actions = ("list", "download", "update")
for a in authenticated:
if a not in actions:
errmsg = "Action '%s' for option `%s` not one of %s!" % (
a, k, actions)
sys.exit(errmsg)
elif k in ("-i", "--interface"):
host = v
elif k in ("-r", "--root"):
roots.append(v)
elif k == "--disable-fallback":
redirect_to_fallback = False
elif k == "--fallback-url":
fallback_url = v
elif k == "--server":
server = v
elif k == "--welcome":
welcome_file = v
elif k == "--version":
sys.stdout.write("pypiserver %s\n" % __version__)
sys.exit(0)
elif k == "-U":
command = "update"
elif k == "-x":
update_dry_run = False
elif k == "-u":
update_stable_only = False
elif k == "-d":
update_directory = v
elif k in ("-P", "--passwords"):
password_file = v
elif k in ("-o", "--overwrite"):
overwrite = True
elif k == "--log-file":
log_file = v
elif k == "--log-frmt":
log_frmt = v
elif k == "--log-req-frmt":
log_req_frmt = v
elif k == "--log-res-frmt":
log_res_frmt = v
elif k == "--log-err-frmt":
log_err_frmt = v
elif k == "--cache-control":
cache_control = v
elif k == "-v":
verbosity += 1
elif k in ("-h", "--help"):
usage()
sys.exit(0)
if password_file and password_file != '.' and not authenticated:
sys.exit(
"Actions to authenticate (-a) must not be empty, unless password file (-P) is '.'!")
if len(roots) == 0:
roots.append(os.path.expanduser("~/packages"))
roots = [os.path.abspath(x) for x in roots]
verbose_levels = [
logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET]
log_level = list(zip(verbose_levels, range(verbosity)))[-1][0]
init_logging(level=log_level, filename=log_file, frmt=log_frmt)
if command == "update":
from pypiserver.manage import update_all_packages
update_all_packages(
roots, update_directory, update_dry_run, stable_only=update_stable_only)
return
# Fixes #49:
# The gevent server adapter needs to patch some
# modules BEFORE importing bottle!
if server and server.startswith('gevent'):
import gevent.monkey # @UnresolvedImport
gevent.monkey.patch_all()
from pypiserver.bottle import server_names, run
if server not in server_names:
sys.exit("unknown server %r. choose one of %s" % (
server, ", ".join(server_names.keys())))
from pypiserver import app
a = app(
root=roots,
redirect_to_fallback=redirect_to_fallback,
authenticated=authenticated,
password_file=password_file,
fallback_url=fallback_url,
overwrite=overwrite,
log_req_frmt=log_req_frmt, log_res_frmt=log_res_frmt, log_err_frmt=log_err_frmt,
welcome_file=welcome_file,
cache_control=cache_control,
)
log.info("This is pypiserver %s serving %r on http://%s:%s\n\n",
__version__, ", ".join(roots), host, port)
run(app=a, host=host, port=port, server=server)
if __name__ == "__main__":
if __package__ == "": # running as python pypiserver-...whl/pypiserver?
import sys, os
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from pypiserver import core
core.main()
# if __package__ == "": # running as python pypiserver-...whl/pypiserver?
# import sys
# import os
# sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
main()

@ -2,24 +2,15 @@
"""minimal PyPI like server for use with pip/easy_install"""
import os
import sys
import getopt
import re
import mimetypes
import warnings
import itertools
import logging
warnings.filterwarnings("ignore", "Python 2.5 support may be dropped in future versions of Bottle")
from pypiserver import bottle, __version__, app
sys.modules["bottle"] = bottle
from bottle import run, server_names
mimetypes.add_type("application/octet-stream", ".egg")
mimetypes.add_type("application/octet-stream", ".whl")
DEFAULT_SERVER = None
log = logging.getLogger('pypiserver.core')
# --- the following two functions were copied from distribute's pkg_resources module
@ -27,14 +18,6 @@ component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE)
replace = {'pre': 'c', 'preview': 'c', '-': 'final-', 'rc': 'c', 'dev': '@'}.get
def init_logging(level=None, format=None, filename=None):
logging.basicConfig(level=level, format=format)
rlog = logging.getLogger()
rlog.setLevel(level)
if filename:
rlog.addHandler(logging.FileHandler(filename))
def _parse_version_parts(s):
for part in component_re.split(s):
part = replace(part, part)
@ -188,269 +171,3 @@ def store(root, filename, data):
log.info("Stored package: %s", filename)
return True
def usage():
sys.stdout.write("""pypi-server [OPTIONS] [PACKAGES_DIRECTORY...]
start PyPI compatible package server serving packages from
PACKAGES_DIRECTORY. If PACKAGES_DIRECTORY is not given on the
command line, it uses the default ~/packages. pypiserver scans this
directory recursively for packages. It skips packages and
directories starting with a dot. Multiple package directories can be
specified.
pypi-server understands the following options:
-p, --port PORT
listen on port PORT (default: 8080)
-i, --interface INTERFACE
listen on interface INTERFACE (default: 0.0.0.0, any interface)
-a, --authenticate (UPDATE|download|list), ...
comma-separated list of (case-insensitive) actions to authenticate
Requires -P option and cannot not be empty unless -P is '.'
For example to password-protect package downloads (in addition to uploads)
while leaving listings public, give:
-P foo/htpasswd.txt -a update,download
To drop all authentications, use:
-P . -a ''
By default, only 'update' is password-protected.
-P, --passwords PASSWORD_FILE
use apache htpasswd file PASSWORD_FILE to set usernames & passwords
used for authentication of certain actions (see -a option).
Set it explicitly to '.' to allow empty list of actions to authenticate;
then no `register` command is neccessary, but `~/.pypirc` still needs
`username` and `password` fields, even if bogus.
--disable-fallback
disable redirect to real PyPI index for packages not found in the
local index
--fallback-url FALLBACK_URL
for packages not found in the local index, this URL will be used to
redirect to (default: http://pypi.python.org/simple)
--server METHOD
use METHOD to run the server. Valid values include paste,
cherrypy, twisted, gunicorn, gevent, wsgiref, auto. The
default is to use "auto" which chooses one of paste, cherrypy,
twisted or wsgiref.
-r, --root PACKAGES_DIRECTORY
[deprecated] serve packages from PACKAGES_DIRECTORY
-o, --overwrite
allow overwriting existing package files
--welcome HTML_FILE
uses the ASCII contents of HTML_FILE as welcome message response.
-v
enable verbose logging; repeat for more verbosity.
--log-file <FILE>
write logging info into this FILE.
--log-frmt <FILE>
the logging format-string. (see `logging.LogRecord` class from standard python library)
[Default: %(asctime)s|%(levelname)s|%(thread)d|%(message)s]
--log-req-frmt FORMAT
a format-string selecting Http-Request properties to log; set to '%s' to see them all.
[Default: %(bottle.request)s]
--log-res-frmt FORMAT
a format-string selecting Http-Response properties to log; set to '%s' to see them all.
[Default: %(status)s]
--log-err-frmt FORMAT
a format-string selecting Http-Error properties to log; set to '%s' to see them all.
[Default: %(body)s: %(exception)s \n%(traceback)s]
--cache-control AGE
Add "Cache-Control: max-age=AGE, public" header to package downloads.
Pip 6+ needs this for caching.
pypi-server -h
pypi-server --help
show this help message
pypi-server --version
show pypi-server's version
pypi-server -U [OPTIONS] [PACKAGES_DIRECTORY...]
update packages in PACKAGES_DIRECTORY. This command searches
pypi.python.org for updates and shows a pip command line which
updates the package.
The following additional options can be specified with -U:
-x
execute the pip commands instead of only showing them
-d DOWNLOAD_DIRECTORY
download package updates to this directory. The default is to use
the directory which contains the latest version of the package to
be updated.
-u
allow updating to unstable version (alpha, beta, rc, dev versions)
Visit https://pypi.python.org/pypi/pypiserver for more information.
""")
def main(argv=None):
if argv is None:
argv = sys.argv
global packages
command = "serve"
host = "0.0.0.0"
port = 8080
server = DEFAULT_SERVER
redirect_to_fallback = True
fallback_url = "http://pypi.python.org/simple"
authenticated = ['update']
password_file = None
overwrite = False
verbosity = 1
log_file = None
log_frmt = "g%(asctime)s|%(levelname)s|%(thread)d|%(message)s"
log_req_frmt = "%(bottle.request)s"
log_res_frmt = "%(status)s"
log_err_frmt = "%(body)s: %(exception)s \n%(traceback)s"
welcome_file = None
cache_control = None
update_dry_run = True
update_directory = None
update_stable_only = True
try:
opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [
"interface=",
"passwords=",
"authenticate=",
"port=",
"root=",
"server=",
"fallback-url=",
"disable-fallback",
"overwrite",
"log-file=",
"log-frmt=",
"log-req-frmt=",
"log-res-frmt=",
"log-err-frmt=",
"welcome=",
"cache-control=",
"version",
"help"
])
except getopt.GetoptError:
err = sys.exc_info()[1]
sys.exit("usage error: %s" % (err,))
for k, v in opts:
if k in ("-p", "--port"):
port = int(v)
elif k in ("-a", "--authenticate"):
authenticated = [a.lower()
for a in re.split("[, ]+", v.strip(" ,"))
if a]
actions = ("list", "download", "update")
for a in authenticated:
if a not in actions:
errmsg = "Action '%s' for option `%s` not one of %s!" % (a, k, actions)
sys.exit(errmsg)
elif k in ("-i", "--interface"):
host = v
elif k in ("-r", "--root"):
roots.append(v)
elif k == "--disable-fallback":
redirect_to_fallback = False
elif k == "--fallback-url":
fallback_url = v
elif k == "--server":
if v not in server_names:
sys.exit("unknown server %r. choose one of %s" % (
v, ", ".join(server_names.keys())))
server = v
elif k == "--welcome":
welcome_file = v
elif k == "--version":
sys.stdout.write("pypiserver %s\n" % __version__)
sys.exit(0)
elif k == "-U":
command = "update"
elif k == "-x":
update_dry_run = False
elif k == "-u":
update_stable_only = False
elif k == "-d":
update_directory = v
elif k in ("-P", "--passwords"):
password_file = v
elif k in ("-o", "--overwrite"):
overwrite = True
elif k == "--log-file":
log_file = v
elif k == "--log-frmt":
log_frmt = v
elif k == "--log-req-frmt":
log_req_frmt = v
elif k == "--log-res-frmt":
log_res_frmt = v
elif k == "--log-err-frmt":
log_err_frmt = v
elif k == "--cache-control":
cache_control = v
elif k == "-v":
verbosity += 1
elif k in ("-h", "--help"):
usage()
sys.exit(0)
if password_file and password_file != '.' and not authenticated:
sys.exit("Actions to authenticate (-a) must not be empty, unless password file (-P) is '.'!")
if len(roots) == 0:
roots.append(os.path.expanduser("~/packages"))
roots = [os.path.abspath(x) for x in roots]
verbose_levels = [logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET]
log_level = list(zip(verbose_levels, range(verbosity)))[-1][0]
init_logging(level=log_level, filename=log_file, format=log_frmt)
if command == "update":
packages = frozenset(itertools.chain(*[listdir(r) for r in roots]))
from pypiserver import manage
manage.update(packages, update_directory, update_dry_run, stable_only=update_stable_only)
return
a = app(
root=roots,
redirect_to_fallback=redirect_to_fallback,
authenticated=authenticated,
password_file=password_file,
fallback_url=fallback_url,
overwrite=overwrite,
log_req_frmt=log_req_frmt, log_res_frmt=log_res_frmt, log_err_frmt=log_err_frmt,
welcome_file=welcome_file,
cache_control=cache_control,
)
server = server or "auto"
log.info("This is pypiserver %s serving %r on http://%s:%s\n\n",
__version__, ", ".join(roots), host, port)
run(app=a, host=host, port=port, server=server)
if __name__ == "__main__":
main(sys.argv)

@ -3,6 +3,7 @@ import os
from subprocess import call
from pypiserver import core
import itertools
if sys.version_info >= (3, 0):
from xmlrpc.client import Server
@ -146,3 +147,7 @@ def update(pkgset, destdir=None, dry_run=False, stable_only=True):
sys.stdout.write("%s\n\n" % (" ".join(cmd),))
if not dry_run:
call(cmd)
def update_all_packages(roots, destdir=None, dry_run=False, stable_only=True):
packages = frozenset(itertools.chain(*[core.listdir(r) for r in roots]))
update(packages, destdir, dry_run, stable_only)

@ -4,10 +4,10 @@ import sys, os
try:
from setuptools import setup
extra = dict(entry_points={
extra = {'entry_points': {
'paste.app_factory': ['main=pypiserver:paste_app_factory'],
'console_scripts': ['pypi-server=pypiserver.core:main']
})
'console_scripts': ['pypi-server=pypiserver.__main__:main']
}}
except ImportError:
from distutils.core import setup
extra = dict(scripts=["pypi-server"])

@ -1,12 +1,12 @@
#! /usr/bin/env py.test
from pypiserver import core # do no remove. needed for bottle
import pytest, bottle, webtest
from pypiserver import __main__, bottle # do no remove. needed for bottle
import pytest, webtest
## Enable logging to detect any problems with it
##
import logging
core.init_logging(level=logging.NOTSET)
__main__.init_logging(level=logging.NOTSET)
@pytest.fixture()
def _app(app):

@ -1,12 +1,12 @@
#! /usr/bin/env py.test
import pytest
from pypiserver import core
from pypiserver import core, __main__
## Enable logging to detect any problems with it
##
import logging
core.init_logging(level=logging.NOTSET)
__main__.init_logging(level=logging.NOTSET)
files = [

@ -1,7 +1,7 @@
#! /usr/bin/env py.test
import sys, os, pytest, logging
from pypiserver import core
from pypiserver import core, __main__
try:
from unittest import mock
except ImportError:
@ -16,7 +16,7 @@ class main_wrapper(object):
def __call__(self, argv):
sys.stdout.write("Running %s\n" % (argv,))
core.main(["pypi-server"] + argv)
__main__.main(["pypi-server"] + argv)
return self.run_kwargs
@ -35,8 +35,8 @@ def main(request, monkeypatch):
main.pkgdir = pkgdir
return []
monkeypatch.setattr(core, "run", run)
monkeypatch.setattr(os, "listdir", listdir)
monkeypatch.setattr("pypiserver.bottle.run", run)
monkeypatch.setattr("os.listdir", listdir)
return main