Add an optional in-memory cache to hold package list
- Precomputes various attributes - File digest is cached on access - Cache requires watchdog to be installed
This commit is contained in:
parent
62f8626ed6
commit
e8f1f149a5
|
@ -192,7 +192,7 @@ def simple(prefix=""):
|
||||||
return HTTPError(404)
|
return HTTPError(404)
|
||||||
|
|
||||||
links = [(os.path.basename(f.relfn),
|
links = [(os.path.basename(f.relfn),
|
||||||
urljoin(fp, "../../packages/%s#%s" % (f.relfn_unix(),
|
urljoin(fp, "../../packages/%s#%s" % (f.relfn_unix,
|
||||||
|
|
||||||
f.hash(config.hash_algo))))
|
f.hash(config.hash_algo))))
|
||||||
for f in files]
|
for f in files]
|
||||||
|
@ -224,7 +224,7 @@ def list_packages():
|
||||||
key=lambda x: (os.path.dirname(x.relfn),
|
key=lambda x: (os.path.dirname(x.relfn),
|
||||||
x.pkgname,
|
x.pkgname,
|
||||||
x.parsed_version))
|
x.parsed_version))
|
||||||
links = [(f.relfn_unix(), '%s#%s' % (urljoin(fp, f.relfn),
|
links = [(f.relfn_unix, '%s#%s' % (urljoin(fp, f.relfn),
|
||||||
f.hash(config.hash_algo)))
|
f.hash(config.hash_algo)))
|
||||||
for f in files]
|
for f in files]
|
||||||
tmpl = """\
|
tmpl = """\
|
||||||
|
@ -248,7 +248,7 @@ def list_packages():
|
||||||
def server_static(filename):
|
def server_static(filename):
|
||||||
entries = core.find_packages(packages())
|
entries = core.find_packages(packages())
|
||||||
for x in entries:
|
for x in entries:
|
||||||
f = x.relfn_unix()
|
f = x.relfn_unix
|
||||||
if f == filename:
|
if f == filename:
|
||||||
response = static_file(
|
response = static_file(
|
||||||
filename, root=x.root, mimetype=mimetypes.guess_type(filename)[0])
|
filename, root=x.root, mimetype=mimetypes.guess_type(filename)[0])
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
|
||||||
|
# Dumb cache implementation -- requires watchdog to be installed
|
||||||
|
|
||||||
|
# Basically -- cache the results of listdir in memory until something
|
||||||
|
# gets modified, then invalidate the whole thing
|
||||||
|
|
||||||
|
|
||||||
|
from watchdog.observers import Observer
|
||||||
|
import threading
|
||||||
|
|
||||||
|
class ListdirCache(object):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.cache = {}
|
||||||
|
self.observer = Observer()
|
||||||
|
self.observer.start()
|
||||||
|
|
||||||
|
self.watched = set()
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
|
||||||
|
def get(self, root, fn):
|
||||||
|
with self.lock:
|
||||||
|
try:
|
||||||
|
return self.cache[root]
|
||||||
|
except KeyError:
|
||||||
|
# check to see if we're watching
|
||||||
|
if root not in self.watched:
|
||||||
|
self._watch(root)
|
||||||
|
|
||||||
|
v = list(fn(root))
|
||||||
|
self.cache[root] = v
|
||||||
|
return v
|
||||||
|
|
||||||
|
def _watch(self, root):
|
||||||
|
self.watched.add(root)
|
||||||
|
self.observer.schedule(_EventHandler(self, root), root, recursive=True)
|
||||||
|
|
||||||
|
class _EventHandler(object):
|
||||||
|
|
||||||
|
def __init__(self, lcache, root):
|
||||||
|
self.lcache = lcache
|
||||||
|
self.root = root
|
||||||
|
|
||||||
|
def dispatch(self, event):
|
||||||
|
'''Called by watchdog observer'''
|
||||||
|
with self.lcache.lock:
|
||||||
|
self.lcache.cache.pop(self.root, None)
|
||||||
|
|
||||||
|
listdir_cache = ListdirCache()
|
|
@ -182,22 +182,38 @@ def is_allowed_path(path_part):
|
||||||
|
|
||||||
|
|
||||||
class PkgFile(object):
|
class PkgFile(object):
|
||||||
def __init__(self, **kw):
|
|
||||||
self.__dict__.update(kw)
|
__slots__ = ['fn', 'root', '_hash',
|
||||||
|
'relfn', 'relfn_unix',
|
||||||
|
'pkgname_norm',
|
||||||
|
'pkgname',
|
||||||
|
'version',
|
||||||
|
'parsed_version',
|
||||||
|
'replaces']
|
||||||
|
|
||||||
|
def __init__(self, pkgname, version, fn=None, root=None, relfn=None, replaces=None):
|
||||||
|
self.pkgname = pkgname
|
||||||
|
self.pkgname_norm = normalize_pkgname(pkgname)
|
||||||
|
self.version = version
|
||||||
|
self.parsed_version = parse_version(version)
|
||||||
|
self.fn = fn
|
||||||
|
self.root = root
|
||||||
|
self.relfn = relfn
|
||||||
|
self.relfn_unix = None if relfn is None else relfn.replace("\\", "/")
|
||||||
|
self.replaces = replaces
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return "%s(%s)" % (
|
return "%s(%s)" % (
|
||||||
self.__class__.__name__,
|
self.__class__.__name__,
|
||||||
", ".join(["%s=%r" % (k, v) for k, v in sorted(self.__dict__.items())]))
|
", ".join(["%s=%r" % (k, getattr(self, k, v)) for k in sorted(self.__slots__)]))
|
||||||
|
|
||||||
def relfn_unix(self):
|
|
||||||
return self.relfn.replace("\\", "/")
|
|
||||||
|
|
||||||
def hash(self, hash_algo):
|
def hash(self, hash_algo):
|
||||||
return '%s=%.32s' % (hash_algo, digest_file(self.fn, hash_algo))
|
if not hasattr(self, '_hash'):
|
||||||
|
self._hash = '%s=%.32s' % (hash_algo, digest_file(self.fn, hash_algo))
|
||||||
|
return self._hash
|
||||||
|
|
||||||
|
|
||||||
|
def _listdir(root):
|
||||||
def listdir(root):
|
|
||||||
root = os.path.abspath(root)
|
root = os.path.abspath(root)
|
||||||
for dirpath, dirnames, filenames in os.walk(root):
|
for dirpath, dirnames, filenames in os.walk(root):
|
||||||
dirnames[:] = [x for x in dirnames if is_allowed_path(x)]
|
dirnames[:] = [x for x in dirnames if is_allowed_path(x)]
|
||||||
|
@ -211,22 +227,30 @@ def listdir(root):
|
||||||
continue
|
continue
|
||||||
pkgname, version = res
|
pkgname, version = res
|
||||||
if pkgname:
|
if pkgname:
|
||||||
yield PkgFile(fn=fn, root=root, relfn=fn[len(root) + 1:],
|
yield PkgFile(pkgname=pkgname,
|
||||||
pkgname=pkgname,
|
|
||||||
version=version,
|
version=version,
|
||||||
parsed_version=parse_version(version))
|
fn=fn, root=root,
|
||||||
|
relfn=fn[len(root) + 1:])
|
||||||
|
|
||||||
|
try:
|
||||||
|
from .cache import listdir_cache
|
||||||
|
|
||||||
|
def listdir(root):
|
||||||
|
return listdir_cache.get(root, _listdir)
|
||||||
|
except ImportError:
|
||||||
|
listdir = _listdir
|
||||||
|
|
||||||
def find_packages(pkgs, prefix=""):
|
def find_packages(pkgs, prefix=""):
|
||||||
prefix = normalize_pkgname(prefix)
|
prefix = normalize_pkgname(prefix)
|
||||||
for x in pkgs:
|
for x in pkgs:
|
||||||
if prefix and normalize_pkgname(x.pkgname) != prefix:
|
if prefix and x.pkgname_norm != prefix:
|
||||||
continue
|
continue
|
||||||
yield x
|
yield x
|
||||||
|
|
||||||
|
|
||||||
def get_prefixes(pkgs):
|
def get_prefixes(pkgs):
|
||||||
pkgnames = set()
|
pkgnames = set()
|
||||||
|
normalized_pkgnames = set()
|
||||||
eggs = set()
|
eggs = set()
|
||||||
|
|
||||||
for x in pkgs:
|
for x in pkgs:
|
||||||
|
@ -235,11 +259,10 @@ def get_prefixes(pkgs):
|
||||||
eggs.add(x.pkgname)
|
eggs.add(x.pkgname)
|
||||||
else:
|
else:
|
||||||
pkgnames.add(x.pkgname)
|
pkgnames.add(x.pkgname)
|
||||||
|
normalized_pkgnames.add(x.pkgname_norm)
|
||||||
normalized_pkgnames = set(map(normalize_pkgname, pkgnames))
|
|
||||||
|
|
||||||
for x in eggs:
|
for x in eggs:
|
||||||
if normalize_pkgname(x) not in normalized_pkgnames:
|
if x not in normalized_pkgnames:
|
||||||
pkgnames.add(x)
|
pkgnames.add(x)
|
||||||
|
|
||||||
return pkgnames
|
return pkgnames
|
||||||
|
|
|
@ -84,9 +84,8 @@ def build_releases(pkg, versions):
|
||||||
for x in versions:
|
for x in versions:
|
||||||
parsed_version = core.parse_version(x)
|
parsed_version = core.parse_version(x)
|
||||||
if parsed_version > pkg.parsed_version:
|
if parsed_version > pkg.parsed_version:
|
||||||
yield core.PkgFile(version=x,
|
yield core.PkgFile(pkgname=pkg.pkgname,
|
||||||
parsed_version=parsed_version,
|
version=x,
|
||||||
pkgname=pkg.pkgname,
|
|
||||||
replaces=pkg)
|
replaces=pkg)
|
||||||
|
|
||||||
|
|
||||||
|
|
3
setup.py
3
setup.py
|
@ -36,7 +36,8 @@ setup(name="pypiserver",
|
||||||
'wheel',
|
'wheel',
|
||||||
],
|
],
|
||||||
extras_require={
|
extras_require={
|
||||||
'passlib': ['passlib']
|
'passlib': ['passlib'],
|
||||||
|
'cache': ['watchdog']
|
||||||
},
|
},
|
||||||
tests_require=tests_require,
|
tests_require=tests_require,
|
||||||
url="https://github.com/pypiserver/pypiserver",
|
url="https://github.com/pypiserver/pypiserver",
|
||||||
|
|
|
@ -13,8 +13,9 @@ def touch_files(root, files):
|
||||||
|
|
||||||
def pkgfile_from_path(fn):
|
def pkgfile_from_path(fn):
|
||||||
pkgname, version = guess_pkgname_and_version(fn)
|
pkgname, version = guess_pkgname_and_version(fn)
|
||||||
return PkgFile(root=py.path.local(fn).parts()[1].strpath,
|
return PkgFile(pkgname=pkgname, version=version,
|
||||||
fn=fn, pkgname=pkgname, version=version, parsed_version=parse_version(version))
|
root=py.path.local(fn).parts()[1].strpath,
|
||||||
|
fn=fn)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
@ -40,7 +41,8 @@ def test_build_releases():
|
||||||
version='0.3.0')
|
version='0.3.0')
|
||||||
|
|
||||||
res, = list(build_releases(p, ["0.3.0"]))
|
res, = list(build_releases(p, ["0.3.0"]))
|
||||||
assert res.__dict__ == expected
|
for k, v in expected.items():
|
||||||
|
assert getattr(res, k) == v
|
||||||
|
|
||||||
|
|
||||||
def test_filter_stable_releases():
|
def test_filter_stable_releases():
|
||||||
|
|
Loading…
Reference in New Issue