forked from github.com/pypiserver
Add an optional in-memory cache to hold package list
- Precomputes various attributes - File digest is cached on access - Cache requires watchdog to be installed
This commit is contained in:
Dustin Spicuzza
parent
62f8626ed6
commit
e8f1f149a5
@ -192,7 +192,7 @@ def simple(prefix=""):
|
||||
return HTTPError(404)
|
||||
|
||||
links = [(os.path.basename(f.relfn),
|
||||
urljoin(fp, "../../packages/%s#%s" % (f.relfn_unix(),
|
||||
urljoin(fp, "../../packages/%s#%s" % (f.relfn_unix,
|
||||
|
||||
f.hash(config.hash_algo))))
|
||||
for f in files]
|
||||
@ -224,7 +224,7 @@ def list_packages():
|
||||
key=lambda x: (os.path.dirname(x.relfn),
|
||||
x.pkgname,
|
||||
x.parsed_version))
|
||||
links = [(f.relfn_unix(), '%s#%s' % (urljoin(fp, f.relfn),
|
||||
links = [(f.relfn_unix, '%s#%s' % (urljoin(fp, f.relfn),
|
||||
f.hash(config.hash_algo)))
|
||||
for f in files]
|
||||
tmpl = """\
|
||||
@ -248,7 +248,7 @@ def list_packages():
|
||||
def server_static(filename):
|
||||
entries = core.find_packages(packages())
|
||||
for x in entries:
|
||||
f = x.relfn_unix()
|
||||
f = x.relfn_unix
|
||||
if f == filename:
|
||||
response = static_file(
|
||||
filename, root=x.root, mimetype=mimetypes.guess_type(filename)[0])
|
||||
|
||||
49
pypiserver/cache.py
Normal file
49
pypiserver/cache.py
Normal file
@ -0,0 +1,49 @@
|
||||
|
||||
# Dumb cache implementation -- requires watchdog to be installed
|
||||
|
||||
# Basically -- cache the results of listdir in memory until something
|
||||
# gets modified, then invalidate the whole thing
|
||||
|
||||
|
||||
from watchdog.observers import Observer
|
||||
import threading
|
||||
|
||||
class ListdirCache(object):
|
||||
|
||||
def __init__(self):
|
||||
self.cache = {}
|
||||
self.observer = Observer()
|
||||
self.observer.start()
|
||||
|
||||
self.watched = set()
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def get(self, root, fn):
|
||||
with self.lock:
|
||||
try:
|
||||
return self.cache[root]
|
||||
except KeyError:
|
||||
# check to see if we're watching
|
||||
if root not in self.watched:
|
||||
self._watch(root)
|
||||
|
||||
v = list(fn(root))
|
||||
self.cache[root] = v
|
||||
return v
|
||||
|
||||
def _watch(self, root):
|
||||
self.watched.add(root)
|
||||
self.observer.schedule(_EventHandler(self, root), root, recursive=True)
|
||||
|
||||
class _EventHandler(object):
|
||||
|
||||
def __init__(self, lcache, root):
|
||||
self.lcache = lcache
|
||||
self.root = root
|
||||
|
||||
def dispatch(self, event):
|
||||
'''Called by watchdog observer'''
|
||||
with self.lcache.lock:
|
||||
self.lcache.cache.pop(self.root, None)
|
||||
|
||||
listdir_cache = ListdirCache()
|
||||
@ -182,22 +182,38 @@ def is_allowed_path(path_part):
|
||||
|
||||
|
||||
class PkgFile(object):
|
||||
def __init__(self, **kw):
|
||||
self.__dict__.update(kw)
|
||||
|
||||
__slots__ = ['fn', 'root', '_hash',
|
||||
'relfn', 'relfn_unix',
|
||||
'pkgname_norm',
|
||||
'pkgname',
|
||||
'version',
|
||||
'parsed_version',
|
||||
'replaces']
|
||||
|
||||
def __init__(self, pkgname, version, fn=None, root=None, relfn=None, replaces=None):
|
||||
self.pkgname = pkgname
|
||||
self.pkgname_norm = normalize_pkgname(pkgname)
|
||||
self.version = version
|
||||
self.parsed_version = parse_version(version)
|
||||
self.fn = fn
|
||||
self.root = root
|
||||
self.relfn = relfn
|
||||
self.relfn_unix = None if relfn is None else relfn.replace("\\", "/")
|
||||
self.replaces = replaces
|
||||
|
||||
def __repr__(self):
|
||||
return "%s(%s)" % (
|
||||
self.__class__.__name__,
|
||||
", ".join(["%s=%r" % (k, v) for k, v in sorted(self.__dict__.items())]))
|
||||
|
||||
def relfn_unix(self):
|
||||
return self.relfn.replace("\\", "/")
|
||||
", ".join(["%s=%r" % (k, getattr(self, k, v)) for k in sorted(self.__slots__)]))
|
||||
|
||||
def hash(self, hash_algo):
|
||||
return '%s=%.32s' % (hash_algo, digest_file(self.fn, hash_algo))
|
||||
if not hasattr(self, '_hash'):
|
||||
self._hash = '%s=%.32s' % (hash_algo, digest_file(self.fn, hash_algo))
|
||||
return self._hash
|
||||
|
||||
|
||||
def listdir(root):
|
||||
def _listdir(root):
|
||||
root = os.path.abspath(root)
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = [x for x in dirnames if is_allowed_path(x)]
|
||||
@ -211,22 +227,30 @@ def listdir(root):
|
||||
continue
|
||||
pkgname, version = res
|
||||
if pkgname:
|
||||
yield PkgFile(fn=fn, root=root, relfn=fn[len(root) + 1:],
|
||||
pkgname=pkgname,
|
||||
yield PkgFile(pkgname=pkgname,
|
||||
version=version,
|
||||
parsed_version=parse_version(version))
|
||||
fn=fn, root=root,
|
||||
relfn=fn[len(root) + 1:])
|
||||
|
||||
try:
|
||||
from .cache import listdir_cache
|
||||
|
||||
def listdir(root):
|
||||
return listdir_cache.get(root, _listdir)
|
||||
except ImportError:
|
||||
listdir = _listdir
|
||||
|
||||
def find_packages(pkgs, prefix=""):
|
||||
prefix = normalize_pkgname(prefix)
|
||||
for x in pkgs:
|
||||
if prefix and normalize_pkgname(x.pkgname) != prefix:
|
||||
if prefix and x.pkgname_norm != prefix:
|
||||
continue
|
||||
yield x
|
||||
|
||||
|
||||
def get_prefixes(pkgs):
|
||||
pkgnames = set()
|
||||
normalized_pkgnames = set()
|
||||
eggs = set()
|
||||
|
||||
for x in pkgs:
|
||||
@ -235,11 +259,10 @@ def get_prefixes(pkgs):
|
||||
eggs.add(x.pkgname)
|
||||
else:
|
||||
pkgnames.add(x.pkgname)
|
||||
|
||||
normalized_pkgnames = set(map(normalize_pkgname, pkgnames))
|
||||
normalized_pkgnames.add(x.pkgname_norm)
|
||||
|
||||
for x in eggs:
|
||||
if normalize_pkgname(x) not in normalized_pkgnames:
|
||||
if x not in normalized_pkgnames:
|
||||
pkgnames.add(x)
|
||||
|
||||
return pkgnames
|
||||
|
||||
@ -84,9 +84,8 @@ def build_releases(pkg, versions):
|
||||
for x in versions:
|
||||
parsed_version = core.parse_version(x)
|
||||
if parsed_version > pkg.parsed_version:
|
||||
yield core.PkgFile(version=x,
|
||||
parsed_version=parsed_version,
|
||||
pkgname=pkg.pkgname,
|
||||
yield core.PkgFile(pkgname=pkg.pkgname,
|
||||
version=x,
|
||||
replaces=pkg)
|
||||
|
||||
|
||||
|
||||
3
setup.py
3
setup.py
@ -36,7 +36,8 @@ setup(name="pypiserver",
|
||||
'wheel',
|
||||
],
|
||||
extras_require={
|
||||
'passlib': ['passlib']
|
||||
'passlib': ['passlib'],
|
||||
'cache': ['watchdog']
|
||||
},
|
||||
tests_require=tests_require,
|
||||
url="https://github.com/pypiserver/pypiserver",
|
||||
|
||||
@ -13,8 +13,9 @@ def touch_files(root, files):
|
||||
|
||||
def pkgfile_from_path(fn):
|
||||
pkgname, version = guess_pkgname_and_version(fn)
|
||||
return PkgFile(root=py.path.local(fn).parts()[1].strpath,
|
||||
fn=fn, pkgname=pkgname, version=version, parsed_version=parse_version(version))
|
||||
return PkgFile(pkgname=pkgname, version=version,
|
||||
root=py.path.local(fn).parts()[1].strpath,
|
||||
fn=fn)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@ -40,7 +41,8 @@ def test_build_releases():
|
||||
version='0.3.0')
|
||||
|
||||
res, = list(build_releases(p, ["0.3.0"]))
|
||||
assert res.__dict__ == expected
|
||||
for k, v in expected.items():
|
||||
assert getattr(res, k) == v
|
||||
|
||||
|
||||
def test_filter_stable_releases():
|
||||
|
||||
Loading…
Reference in New Issue
Block a user