fnexe.com
/
pypiserver
forked from github.com/pypiserver
1
0
Fork 0
Code Pull Requests Activity

Add an optional in-memory cache to hold package list 

- Precomputes various attributes
- File digest is cached on access
- Cache requires watchdog to be installed
This commit is contained in:
Dustin Spicuzza 2016-01-04 16:42:39 -05:00
parent 62f8626ed6
commit e8f1f149a5
6 changed files with 100 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pypiserver/_app.py
View File

@ -192,7 +192,7 @@ def simple(prefix=""):
return HTTPError(404)
links = [(os.path.basename(f.relfn),
urljoin(fp, "../../packages/%s#%s" % (f.relfn_unix(),
urljoin(fp, "../../packages/%s#%s" % (f.relfn_unix,
f.hash(config.hash_algo))))
for f in files]
@ -224,7 +224,7 @@ def list_packages():
key=lambda x: (os.path.dirname(x.relfn),
x.pkgname,
x.parsed_version))
links = [(f.relfn_unix(), '%s#%s' % (urljoin(fp, f.relfn),
links = [(f.relfn_unix, '%s#%s' % (urljoin(fp, f.relfn),
f.hash(config.hash_algo)))
for f in files]
tmpl = """\
@ -248,7 +248,7 @@ def list_packages():
def server_static(filename):
entries = core.find_packages(packages())
for x in entries:
f = x.relfn_unix()
f = x.relfn_unix
if f == filename:
response = static_file(
filename, root=x.root, mimetype=mimetypes.guess_type(filename)[0])

49
pypiserver/cache.py Normal file
View File

@ -0,0 +1,49 @@
# Dumb cache implementation -- requires watchdog to be installed
# Basically -- cache the results of listdir in memory until something
# gets modified, then invalidate the whole thing
from watchdog.observers import Observer
import threading
class ListdirCache(object):
def __init__(self):
self.cache = {}
self.observer = Observer()
self.observer.start()
self.watched = set()
self.lock = threading.Lock()
def get(self, root, fn):
with self.lock:
try:
return self.cache[root]
except KeyError:
# check to see if we're watching
if root not in self.watched:
self._watch(root)
v = list(fn(root))
self.cache[root] = v
return v
def _watch(self, root):
self.watched.add(root)
self.observer.schedule(_EventHandler(self, root), root, recursive=True)
class _EventHandler(object):
def __init__(self, lcache, root):
self.lcache = lcache
self.root = root
def dispatch(self, event):
'''Called by watchdog observer'''
with self.lcache.lock:
self.lcache.cache.pop(self.root, None)
listdir_cache = ListdirCache()

53
pypiserver/core.py
View File

@ -182,22 +182,38 @@ def is_allowed_path(path_part):
class PkgFile(object):
def __init__(self, **kw):
self.__dict__.update(kw)
__slots__ = ['fn', 'root', '_hash',
'relfn', 'relfn_unix',
'pkgname_norm',
'pkgname',
'version',
'parsed_version',
'replaces']
def __init__(self, pkgname, version, fn=None, root=None, relfn=None, replaces=None):
self.pkgname = pkgname
self.pkgname_norm = normalize_pkgname(pkgname)
self.version = version
self.parsed_version = parse_version(version)
self.fn = fn
self.root = root
self.relfn = relfn
self.relfn_unix = None if relfn is None else relfn.replace("\\", "/")
self.replaces = replaces
def __repr__(self):
return "%s(%s)" % (
self.__class__.__name__,
", ".join(["%s=%r" % (k, v) for k, v in sorted(self.__dict__.items())]))
def relfn_unix(self):
return self.relfn.replace("\\", "/")
", ".join(["%s=%r" % (k, getattr(self, k, v)) for k in sorted(self.__slots__)]))
def hash(self, hash_algo):
return '%s=%.32s' % (hash_algo, digest_file(self.fn, hash_algo))
if not hasattr(self, '_hash'):
self._hash = '%s=%.32s' % (hash_algo, digest_file(self.fn, hash_algo))
return self._hash
def listdir(root):
def _listdir(root):
root = os.path.abspath(root)
for dirpath, dirnames, filenames in os.walk(root):
dirnames[:] = [x for x in dirnames if is_allowed_path(x)]
@ -211,22 +227,30 @@ def listdir(root):
continue
pkgname, version = res
if pkgname:
yield PkgFile(fn=fn, root=root, relfn=fn[len(root) + 1:],
pkgname=pkgname,
yield PkgFile(pkgname=pkgname,
version=version,
parsed_version=parse_version(version))
fn=fn, root=root,
relfn=fn[len(root) + 1:])
try:
from .cache import listdir_cache
def listdir(root):
return listdir_cache.get(root, _listdir)
except ImportError:
listdir = _listdir
def find_packages(pkgs, prefix=""):
prefix = normalize_pkgname(prefix)
for x in pkgs:
if prefix and normalize_pkgname(x.pkgname) != prefix:
if prefix and x.pkgname_norm != prefix:
continue
yield x
def get_prefixes(pkgs):
pkgnames = set()
normalized_pkgnames = set()
eggs = set()
for x in pkgs:
@ -235,11 +259,10 @@ def get_prefixes(pkgs):
eggs.add(x.pkgname)
else:
pkgnames.add(x.pkgname)
normalized_pkgnames = set(map(normalize_pkgname, pkgnames))
normalized_pkgnames.add(x.pkgname_norm)
for x in eggs:
if normalize_pkgname(x) not in normalized_pkgnames:
if x not in normalized_pkgnames:
pkgnames.add(x)
return pkgnames

5
pypiserver/manage.py
View File

@ -84,9 +84,8 @@ def build_releases(pkg, versions):
for x in versions:
parsed_version = core.parse_version(x)
if parsed_version > pkg.parsed_version:
yield core.PkgFile(version=x,
parsed_version=parsed_version,
pkgname=pkg.pkgname,
yield core.PkgFile(pkgname=pkg.pkgname,
version=x,
replaces=pkg)

3
setup.py
View File

@ -36,7 +36,8 @@ setup(name="pypiserver",
'wheel',
],
extras_require={
'passlib': ['passlib']
'passlib': ['passlib'],
'cache': ['watchdog']
},
tests_require=tests_require,
url="https://github.com/pypiserver/pypiserver",

8
tests/test_manage.py
View File

@ -13,8 +13,9 @@ def touch_files(root, files):
def pkgfile_from_path(fn):
pkgname, version = guess_pkgname_and_version(fn)
return PkgFile(root=py.path.local(fn).parts()[1].strpath,
fn=fn, pkgname=pkgname, version=version, parsed_version=parse_version(version))
return PkgFile(pkgname=pkgname, version=version,
root=py.path.local(fn).parts()[1].strpath,
fn=fn)
@pytest.mark.parametrize(
@ -40,7 +41,8 @@ def test_build_releases():
version='0.3.0')
res, = list(build_releases(p, ["0.3.0"]))
assert res.__dict__ == expected
for k, v in expected.items():
assert getattr(res, k) == v
def test_filter_stable_releases():