Add the option to specify list of modules we don't want to update (#298)

* Add the option to specify list of modules we don't want to update

Signed-off-by: Peter Slovak <peter.slovak@websupport.sk>

* Fix docs

Signed-off-by: Peter Slovak <peter.slovak@websupport.sk>

* Minimize the number of strip() calls

Co-authored-by: Matthew Planchard <mplanchard@users.noreply.github.com>

* Log an exception when we fail to open/read the package blacklist file

* Abort server startup if we fail to read the blacklist file

Co-authored-by: Matthew Planchard <mplanchard@users.noreply.github.com>

pypiserver/__main__.py

@@ -145,6 +145,15 @@ def usage():
 
     -u
       Allow updating to unstable version (alpha, beta, rc, dev versions).
+      
+    --blacklist-file BLACKLIST_FILE
+      Don't update packages listed in this file (one package name per line,
+      without versions, '#' comments honored). This can be useful if you upload
+      private packages into pypiserver, but also keep a mirror of public
+      packages that you regularly update. Attempting to pull an update of
+      a private package from `pypi.org` might pose a security risk - e.g. a
+      malicious user might publish a higher version of the private package,
+      containing arbitrary code.
 
   Visit https://pypi.org/project/pypiserver/ for more information.
   """)
@@ -163,6 +172,7 @@ def main(argv=None):
     update_dry_run = True
     update_directory = None
     update_stable_only = True
+    update_blacklist_file = None
 
     try:
         opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [
@@ -176,6 +186,7 @@ def main(argv=None):
             "disable-fallback",
             "overwrite",
             "hash-algo=",
+            "blacklist-file=",
             "log-file=",
             "log-frmt=",
             "log-req-frmt=",
@@ -232,6 +243,8 @@ def main(argv=None):
             update_stable_only = False
         elif k == "-d":
             update_directory = v
+        elif k == "--blacklist-file":
+            update_blacklist_file = v
         elif k in ("-P", "--passwords"):
             c.password_file = v
         elif k in ("-o", "--overwrite"):
@@ -274,8 +287,11 @@ def main(argv=None):
 
     if command == "update":
         from pypiserver.manage import update_all_packages
-        update_all_packages(roots, update_directory,
-                dry_run=update_dry_run, stable_only=update_stable_only)
+        update_all_packages(
+            roots, update_directory,
+            dry_run=update_dry_run, stable_only=update_stable_only,
+            blacklist_file=update_blacklist_file
+        )
         return
 
     # Fixes #49:

pypiserver/core.py

@@ -222,7 +222,7 @@ class PkgFile(object):
     def __repr__(self):
         return "%s(%s)" % (
             self.__class__.__name__,
-            ", ".join(["%s=%r" % (k, getattr(self, k))
+            ", ".join(["%s=%r" % (k, getattr(self, k, 'AttributeError'))
                                   for k in sorted(self.__slots__)]))
 
     def fname_and_hash(self, hash_algo):
@@ -256,6 +256,29 @@ def _listdir(root):
                               relfn=fn[len(root) + 1:])
 
 
+def read_lines(filename):
+    """
+    Read the contents of `filename`, stripping empty lines and '#'-comments.
+    Return a list of strings, containing the lines of the file.
+    """
+    lines = []
+
+    try:
+        with open(filename) as f:
+            lines = [
+                line 
+                for line in (ln.strip() for ln in f.readlines()) 
+                if line and not line.startswith('#')
+            ]
+    except Exception:
+        log.error('Failed to read package blacklist file "%s". '
+                  'Aborting server startup, please fix this.'
+                  % filename)
+        raise
+
+    return lines
+
+
 def find_packages(pkgs, prefix=""):
     prefix = normalize_pkgname(prefix)
     for x in pkgs:

pypiserver/manage.py

@@ -200,6 +200,15 @@ def update(pkgset, destdir=None, dry_run=False, stable_only=True):
         update_package(pkg, destdir, dry_run=dry_run)
 
 
-def update_all_packages(roots, destdir=None, dry_run=False, stable_only=True):
-    packages = frozenset(itertools.chain(*[core.listdir(r) for r in roots]))
+def update_all_packages(roots, destdir=None, dry_run=False, stable_only=True, blacklist_file=None):
+    all_packages = itertools.chain(*[core.listdir(r) for r in roots])
+
+    skip_packages = set()
+    if blacklist_file:
+        skip_packages = set(core.read_lines(blacklist_file))
+        print('Skipping update of blacklisted packages (listed in "{}"): {}'
+              .format(blacklist_file, ', '.join(sorted(skip_packages))))
+
+    packages = frozenset([pkg for pkg in all_packages if pkg.pkgname not in skip_packages])
+
     update(packages, destdir, dry_run, stable_only)

tests/test_core.py

@@ -82,6 +82,23 @@ def test_listdir_bad_name(tmpdir):
     res = list(core.listdir(tmpdir.strpath))
     assert res == []
 
+
+def test_read_lines(tmpdir):
+    filename = 'pkg_blacklist'
+    file_contents = (
+        '# Names of private packages that we don\'t want to upgrade\n'
+        '\n'
+        'my_private_pkg \n'
+        ' \t# This is a comment with starting space and tab\n'
+        ' my_other_private_pkg'
+    )
+
+    f = tmpdir.join(filename).ensure()
+    f.write(file_contents)
+
+    assert core.read_lines(f.strpath) == ['my_private_pkg', 'my_other_private_pkg']
+
+
 hashes = (
     # empty-sha256
     ('sha256', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'),

tests/test_main.py

@@ -168,3 +168,12 @@ def test_dot_password_without_auth_list(main, monkeypatch):
 
     main(["-P", ".", "-a", "."])
     assert main.app.module.config.authenticated == []
+
+
+def test_blacklist_file(main):
+    """
+    Test that calling the app with the --blacklist-file argument does not
+    throw a getopt error
+    """
+    blacklist_file = "/root/pkg_blacklist"
+    main(["--blacklist-file", blacklist_file])

tests/test_manage.py

@@ -24,6 +24,7 @@ from pypiserver.manage import (
     filter_latest_pkgs,
     is_stable_version,
     update_package,
+    update_all_packages,
 )
 
 
@@ -178,3 +179,97 @@ def test_update_package_dry_run(monkeypatch):
     pkg = PkgFile('mypkg', '1.0', replaces=PkgFile('mypkg', '0.9'))
     update_package(pkg, '.', dry_run=True)
     assert not manage.call.mock_calls  # pylint: disable=no-member
+
+
+def test_update_all_packages(monkeypatch):
+    """Test calling update_all_packages()"""
+    public_pkg_1 = PkgFile('Flask', '1.0')
+    public_pkg_2 = PkgFile('requests', '1.0')
+    private_pkg_1 = PkgFile('my_private_pkg', '1.0')
+    private_pkg_2 = PkgFile('my_other_private_pkg', '1.0')
+
+    roots_mock = {
+        '/opt/pypi': [
+            public_pkg_1,
+            private_pkg_1,
+        ],
+        '/data/pypi': [
+            public_pkg_2,
+            private_pkg_2
+        ],
+    }
+
+    def core_listdir_mock(directory):
+        return roots_mock.get(directory, [])
+
+    monkeypatch.setattr(manage.core, 'listdir', core_listdir_mock)
+    monkeypatch.setattr(manage.core, 'read_lines', Mock(return_value=[]))
+    monkeypatch.setattr(manage, 'update', Mock(return_value=None))
+
+    destdir = None
+    dry_run = False
+    stable_only = True
+    blacklist_file = None
+
+    update_all_packages(
+        roots=list(roots_mock.keys()),
+        destdir=destdir,
+        dry_run=dry_run,
+        stable_only=stable_only,
+        blacklist_file=blacklist_file,
+    )
+
+    manage.core.read_lines.assert_not_called()   # pylint: disable=no-member
+    manage.update.assert_called_once_with(   # pylint: disable=no-member
+        frozenset([public_pkg_1, public_pkg_2, private_pkg_1, private_pkg_2]),
+        destdir,
+        dry_run,
+        stable_only
+    )
+
+
+def test_update_all_packages_with_blacklist(monkeypatch):
+    """Test calling update_all_packages()"""
+    public_pkg_1 = PkgFile('Flask', '1.0')
+    public_pkg_2 = PkgFile('requests', '1.0')
+    private_pkg_1 = PkgFile('my_private_pkg', '1.0')
+    private_pkg_2 = PkgFile('my_other_private_pkg', '1.0')
+
+    roots_mock = {
+        '/opt/pypi': [
+            public_pkg_1,
+            private_pkg_1,
+        ],
+        '/data/pypi': [
+            public_pkg_2,
+            private_pkg_2
+        ],
+    }
+
+    def core_listdir_mock(directory):
+        return roots_mock.get(directory, [])
+
+    monkeypatch.setattr(manage.core, 'listdir', core_listdir_mock)
+    monkeypatch.setattr(manage.core, 'read_lines', Mock(return_value=['my_private_pkg', 'my_other_private_pkg']))
+    monkeypatch.setattr(manage, 'update', Mock(return_value=None))
+
+    destdir = None
+    dry_run = False
+    stable_only = True
+    blacklist_file = '/root/pkg_blacklist'
+
+    update_all_packages(
+        roots=list(roots_mock.keys()),
+        destdir=destdir,
+        dry_run=dry_run,
+        stable_only=stable_only,
+        blacklist_file=blacklist_file,
+    )
+
+    manage.update.assert_called_once_with(   # pylint: disable=no-member
+        frozenset([public_pkg_1, public_pkg_2]),
+        destdir,
+        dry_run,
+        stable_only
+    )
+    manage.core.read_lines.assert_called_once_with(blacklist_file)   # pylint: disable=no-member