From 569929c95b86159d7661de2a10ff42a02109758d Mon Sep 17 00:00:00 2001 From: swe-jaeyoungpark Date: Tue, 16 Apr 2019 15:46:59 +0900 Subject: [PATCH 1/7] support changing the prefix of the path of the url --- pypiserver/_app.py | 14 +++++--------- pypiserver/core.py | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 357a51f..7a39674 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -80,8 +80,6 @@ def favicon(): @app.route('/') def root(): - fp = request.fullpath - try: numpkgs = len(list(packages())) except: @@ -93,8 +91,8 @@ def root(): URL=request.url, VERSION=__version__, NUMPKGS=numpkgs, - PACKAGES=urljoin(fp, "packages/"), - SIMPLE=urljoin(fp, "simple/") + PACKAGES=urljoin(request.url, "packages/"), + SIMPLE=urljoin(request.url, "simple/") ) _bottle_upload_filename_re = re.compile(r'^[a-z0-9_.!+-]+$', re.I) @@ -197,7 +195,7 @@ def update(): @app.route('/packages') @auth("list") def pep_503_redirects(prefix=None): - return redirect(request.fullpath + "/", 301) + return redirect(request.url + "/", 301) @app.post('/RPC2') @@ -261,9 +259,8 @@ def simple(prefix=""): return redirect("%s/%s/" % (config.fallback_url.rstrip("/"), prefix)) return HTTPError(404, 'Not Found (%s does not exist)\n\n' % normalized) - fp = request.fullpath links = [(os.path.basename(f.relfn), - urljoin(fp, "../../packages/%s" % f.fname_and_hash(config.hash_algo))) + urljoin(request.url, "../../packages/%s" % f.fname_and_hash(config.hash_algo))) for f in files] tmpl = """\ @@ -284,12 +281,11 @@ def simple(prefix=""): @app.route('/packages/') @auth("list") def list_packages(): - fp = request.fullpath files = sorted(core.find_packages(packages()), key=lambda x: (os.path.dirname(x.relfn), x.pkgname, x.parsed_version)) - links = [(f.relfn_unix, urljoin(fp, f.fname_and_hash(config.hash_algo))) + links = [(f.relfn_unix, urljoin(request.url, f.fname_and_hash(config.hash_algo))) for f in files] tmpl = """\ diff --git a/pypiserver/core.py b/pypiserver/core.py index 8a50288..307259a 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -285,7 +285,7 @@ def store(root, filename, save_method): def get_bad_url_redirect_path(request, prefix): """Get the path for a bad root url.""" - p = request.fullpath + p = request.url if p.endswith("/"): p = p[:-1] p = p.rsplit('/', 1)[0] From d154fc09cabd4f8cb78856206c703e545e01419f Mon Sep 17 00:00:00 2001 From: swe-jaeyoungpark Date: Mon, 29 Apr 2019 18:21:41 +0900 Subject: [PATCH 2/7] Fix BaseRequest.urlparts.host and path correctly when the server is behind the reverse proxy --- pypiserver/_app.py | 16 ++++++++++------ pypiserver/bottle.py | 20 ++++++++++++++++---- pypiserver/core.py | 2 +- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 7a39674..12fb590 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -80,6 +80,8 @@ def favicon(): @app.route('/') def root(): + fp = request.fullpath + try: numpkgs = len(list(packages())) except: @@ -88,11 +90,11 @@ def root(): # Ensure template() does not consider `msg` as filename! msg = config.welcome_msg + '\n' return template(msg, - URL=request.url, + URL=request.url.rstrip("/") + '/', VERSION=__version__, NUMPKGS=numpkgs, - PACKAGES=urljoin(request.url, "packages/"), - SIMPLE=urljoin(request.url, "simple/") + PACKAGES=fp.rstrip("/") + "/packages/", + SIMPLE=fp.rstrip("/") + "/simple/" ) _bottle_upload_filename_re = re.compile(r'^[a-z0-9_.!+-]+$', re.I) @@ -195,7 +197,7 @@ def update(): @app.route('/packages') @auth("list") def pep_503_redirects(prefix=None): - return redirect(request.url + "/", 301) + return redirect(request.fullpath + "/", 301) @app.post('/RPC2') @@ -259,8 +261,9 @@ def simple(prefix=""): return redirect("%s/%s/" % (config.fallback_url.rstrip("/"), prefix)) return HTTPError(404, 'Not Found (%s does not exist)\n\n' % normalized) + fp = request.fullpath links = [(os.path.basename(f.relfn), - urljoin(request.url, "../../packages/%s" % f.fname_and_hash(config.hash_algo))) + urljoin(fp, "../../packages/%s" % f.fname_and_hash(config.hash_algo))) for f in files] tmpl = """\ @@ -281,11 +284,12 @@ def simple(prefix=""): @app.route('/packages/') @auth("list") def list_packages(): + fp = request.fullpath files = sorted(core.find_packages(packages()), key=lambda x: (os.path.dirname(x.relfn), x.pkgname, x.parsed_version)) - links = [(f.relfn_unix, urljoin(request.url, f.fname_and_hash(config.hash_algo))) + links = [(f.relfn_unix, urljoin(fp, f.fname_and_hash(config.hash_algo))) for f in files] tmpl = """\ diff --git a/pypiserver/bottle.py b/pypiserver/bottle.py index e4b17d8..2ec1d7b 100644 --- a/pypiserver/bottle.py +++ b/pypiserver/bottle.py @@ -117,7 +117,7 @@ except IOError: if py3k: import http.client as httplib import _thread as thread - from urllib.parse import urljoin, SplitResult as UrlSplitResult + from urllib.parse import urlparse, urljoin, SplitResult as UrlSplitResult from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote urlunquote = functools.partial(urlunquote, encoding='latin1') from http.cookies import SimpleCookie @@ -136,7 +136,7 @@ if py3k: else: # 2.x import httplib import thread - from urlparse import urljoin, SplitResult as UrlSplitResult + from urlparse import urlparse, urljoin, SplitResult as UrlSplitResult from urllib import urlencode, quote as urlquote, unquote as urlunquote from Cookie import SimpleCookie from itertools import imap @@ -1330,20 +1330,32 @@ class BaseRequest(object): http = env.get('HTTP_X_FORWARDED_PROTO') \ or env.get('wsgi.url_scheme', 'http') host = env.get('HTTP_X_FORWARDED_HOST') or env.get('HTTP_HOST') + + path = urlquote(self.scriptpath) + if host: + parsed = urlparse(http + "://" + host) + path = parsed.path.rstrip('/') + '/' + path.lstrip('/') + host = parsed.netloc + if not host: # HTTP 1.1 requires a Host-header. This is for HTTP/1.0 clients. host = env.get('SERVER_NAME', '127.0.0.1') port = env.get('SERVER_PORT') if port and port != ('80' if http == 'http' else '443'): host += ':' + port - path = urlquote(self.fullpath) + return UrlSplitResult(http, host, path, env.get('QUERY_STRING'), '') @property - def fullpath(self): + def scriptpath(self): """ Request path including :attr:`script_name` (if present). """ return urljoin(self.script_name, self.path.lstrip('/')) + @property + def fullpath(self): + """ The path including :attr:`scriptpath`, the prefix of the url """ + return self.urlparts.path + @property def query_string(self): """ The raw :attr:`query` part of the URL (everything in between ``?`` diff --git a/pypiserver/core.py b/pypiserver/core.py index 307259a..8a50288 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -285,7 +285,7 @@ def store(root, filename, save_method): def get_bad_url_redirect_path(request, prefix): """Get the path for a bad root url.""" - p = request.url + p = request.fullpath if p.endswith("/"): p = p[:-1] p = p.rsplit('/', 1)[0] From 8a196ddc907994880e672d2fd1be5e0b97bb316e Mon Sep 17 00:00:00 2001 From: swe-jaeyoungpark Date: Mon, 29 Apr 2019 18:22:35 +0900 Subject: [PATCH 3/7] add test cases with X_FORWARDED_HOST header --- tests/test_app.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_app.py b/tests/test_app.py index 158c088..1754f5b 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -292,6 +292,18 @@ def test_nonroot_root(testpriv): resp.mustcontain("easy_install -i http://nonroot/priv/simple/ PACKAGE") +def test_nonroot_root_with_x_forwarded_host(testapp): + resp = testapp.get("/", headers={"X-Forwarded-Host": "forward.ed/priv/"}) + resp.mustcontain("easy_install -i http://forward.ed/priv/simple/ PACKAGE") + resp.mustcontain("""here""") + + +def test_nonroot_root_with_x_forwarded_host_without_tailing_slash(testapp): + resp = testapp.get("/", headers={"X-Forwarded-Host": "forward.ed/priv/"}) + resp.mustcontain("easy_install -i http://forward.ed/priv/simple/ PACKAGE") + resp.mustcontain("""here""") + + def test_nonroot_simple_index(root, testpriv): root.join("foobar-1.0.zip").write("") resp = testpriv.get("/priv/simple/foobar/") @@ -300,6 +312,14 @@ def test_nonroot_simple_index(root, testpriv): assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#") +def test_nonroot_simple_index_with_x_forwarded_host(root, testapp): + root.join("foobar-1.0.zip").write("") + resp = testapp.get("/simple/foobar/", headers={"X-Forwarded-Host": "forwarded.ed/priv/"}) + links = resp.html("a") + assert len(links) == 1 + assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#") + + def test_nonroot_simple_packages(root, testpriv): root.join("foobar-1.0.zip").write("123") resp = testpriv.get("/priv/packages/") @@ -308,6 +328,14 @@ def test_nonroot_simple_packages(root, testpriv): assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#") +def test_nonroot_simple_packages_with_x_forwarded_host(root, testapp): + root.join("foobar-1.0.zip").write("123") + resp = testapp.get("/packages/", headers={"X-Forwarded-Host": "forwarded/priv/"}) + links = resp.html("a") + assert len(links) == 1 + assert links[0]["href"].startswith("/priv/packages/foobar-1.0.zip#") + + def test_root_no_relative_paths(testpriv): """https://github.com/pypiserver/pypiserver/issues/25""" resp = testpriv.get("/priv/") From ae4dab30cc73d7856a67bb31375c0dd9cb1a8d01 Mon Sep 17 00:00:00 2001 From: swe-jaeyoungpark Date: Thu, 2 May 2019 22:00:22 +0900 Subject: [PATCH 4/7] rollback bottle.py codes --- pypiserver/bottle.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/pypiserver/bottle.py b/pypiserver/bottle.py index 2ec1d7b..e4b17d8 100644 --- a/pypiserver/bottle.py +++ b/pypiserver/bottle.py @@ -117,7 +117,7 @@ except IOError: if py3k: import http.client as httplib import _thread as thread - from urllib.parse import urlparse, urljoin, SplitResult as UrlSplitResult + from urllib.parse import urljoin, SplitResult as UrlSplitResult from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote urlunquote = functools.partial(urlunquote, encoding='latin1') from http.cookies import SimpleCookie @@ -136,7 +136,7 @@ if py3k: else: # 2.x import httplib import thread - from urlparse import urlparse, urljoin, SplitResult as UrlSplitResult + from urlparse import urljoin, SplitResult as UrlSplitResult from urllib import urlencode, quote as urlquote, unquote as urlunquote from Cookie import SimpleCookie from itertools import imap @@ -1330,32 +1330,20 @@ class BaseRequest(object): http = env.get('HTTP_X_FORWARDED_PROTO') \ or env.get('wsgi.url_scheme', 'http') host = env.get('HTTP_X_FORWARDED_HOST') or env.get('HTTP_HOST') - - path = urlquote(self.scriptpath) - if host: - parsed = urlparse(http + "://" + host) - path = parsed.path.rstrip('/') + '/' + path.lstrip('/') - host = parsed.netloc - if not host: # HTTP 1.1 requires a Host-header. This is for HTTP/1.0 clients. host = env.get('SERVER_NAME', '127.0.0.1') port = env.get('SERVER_PORT') if port and port != ('80' if http == 'http' else '443'): host += ':' + port - + path = urlquote(self.fullpath) return UrlSplitResult(http, host, path, env.get('QUERY_STRING'), '') @property - def scriptpath(self): + def fullpath(self): """ Request path including :attr:`script_name` (if present). """ return urljoin(self.script_name, self.path.lstrip('/')) - @property - def fullpath(self): - """ The path including :attr:`scriptpath`, the prefix of the url """ - return self.urlparts.path - @property def query_string(self): """ The raw :attr:`query` part of the URL (everything in between ``?`` From c3965e31a018e7a8af4aafa39fe6bc3fb77165df Mon Sep 17 00:00:00 2001 From: swe-jaeyoungpark Date: Thu, 2 May 2019 23:14:31 +0900 Subject: [PATCH 5/7] fix typos on test_app.py --- tests/test_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_app.py b/tests/test_app.py index 1754f5b..48108e8 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -298,8 +298,8 @@ def test_nonroot_root_with_x_forwarded_host(testapp): resp.mustcontain("""here""") -def test_nonroot_root_with_x_forwarded_host_without_tailing_slash(testapp): - resp = testapp.get("/", headers={"X-Forwarded-Host": "forward.ed/priv/"}) +def test_nonroot_root_with_x_forwarded_host_without_trailing_slash(testapp): + resp = testapp.get("/", headers={"X-Forwarded-Host": "forward.ed/priv"}) resp.mustcontain("easy_install -i http://forward.ed/priv/simple/ PACKAGE") resp.mustcontain("""here""") From a060e99a253af1a201ce5f041fe2fd6b1cd8a897 Mon Sep 17 00:00:00 2001 From: swe-jaeyoungpark Date: Thu, 2 May 2019 23:18:31 +0900 Subject: [PATCH 6/7] implement "supporting changing the prefix of the path of url" again, using before_request hook. --- pypiserver/_app.py | 19 +++++++++++++------ tests/test_core.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pypiserver/_app.py b/pypiserver/_app.py index 12fb590..846dd8a 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -21,9 +21,9 @@ except ImportError: from StringIO import StringIO as BytesIO try: # PY3 - from urllib.parse import urljoin + from urllib.parse import urljoin, urlparse except ImportError: # PY2 - from urlparse import urljoin + from urlparse import urljoin, urlparse log = logging.getLogger(__name__) @@ -59,6 +59,13 @@ def log_request(): log.info(config.log_req_frmt, request.environ) +@app.hook('before_request') +def print_request(): + parsed = urlparse(request.urlparts.scheme + "://" + request.urlparts.netloc) + request.custom_host = parsed.netloc + request.custom_fullpath = parsed.path.rstrip('/') + '/' + request.fullpath.lstrip('/') + + @app.hook('after_request') def log_response(): log.info(config.log_res_frmt, { # vars(response)) ## DOES NOT WORK! @@ -80,7 +87,7 @@ def favicon(): @app.route('/') def root(): - fp = request.fullpath + fp = request.custom_fullpath try: numpkgs = len(list(packages())) @@ -197,7 +204,7 @@ def update(): @app.route('/packages') @auth("list") def pep_503_redirects(prefix=None): - return redirect(request.fullpath + "/", 301) + return redirect(request.custom_fullpath + "/", 301) @app.post('/RPC2') @@ -261,7 +268,7 @@ def simple(prefix=""): return redirect("%s/%s/" % (config.fallback_url.rstrip("/"), prefix)) return HTTPError(404, 'Not Found (%s does not exist)\n\n' % normalized) - fp = request.fullpath + fp = request.custom_fullpath links = [(os.path.basename(f.relfn), urljoin(fp, "../../packages/%s" % f.fname_and_hash(config.hash_algo))) for f in files] @@ -284,7 +291,7 @@ def simple(prefix=""): @app.route('/packages/') @auth("list") def list_packages(): - fp = request.fullpath + fp = request.custom_fullpath files = sorted(core.find_packages(packages()), key=lambda x: (os.path.dirname(x.relfn), x.pkgname, diff --git a/tests/test_core.py b/tests/test_core.py index 2828342..ebefc93 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -96,7 +96,7 @@ def test_hashfile(tmpdir, algo, digest): def test_redirect_prefix_encodes_newlines(): """Ensure raw newlines are url encoded in the generated redirect.""" request = Namespace( - fullpath='/\nSet-Cookie:malicious=1;' + custom_fullpath='/\nSet-Cookie:malicious=1;' ) prefix = '\nSet-Cookie:malicious=1;' newpath = core.get_bad_url_redirect_path(request, prefix) From ad8db21b18f588febbf9fa2d74da9238a3f1bd4b Mon Sep 17 00:00:00 2001 From: swe-jaeyoungpark Date: Thu, 2 May 2019 23:39:00 +0900 Subject: [PATCH 7/7] fix request.fullpath -> request.custom_fullpath on core.py --- pypiserver/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypiserver/core.py b/pypiserver/core.py index 8a50288..2923774 100644 --- a/pypiserver/core.py +++ b/pypiserver/core.py @@ -285,7 +285,7 @@ def store(root, filename, save_method): def get_bad_url_redirect_path(request, prefix): """Get the path for a bad root url.""" - p = request.fullpath + p = request.custom_fullpath if p.endswith("/"): p = p[:-1] p = p.rsplit('/', 1)[0]