working on proxying tarball requests

lib/fs-storage.js

@@ -91,7 +91,7 @@ Storage.prototype.write = function(name, value, cb) {
 }
 
 Storage.prototype.write_json = function(name, value, cb) {
-	write(this.path + '/' + name, JSON.stringify(value), cb);
+	write(this.path + '/' + name, JSON.stringify(value, null, '\t'), cb);
 }
 
 module.exports = Storage;

lib/proxy.js

@@ -1,24 +0,0 @@
-var https = require('https');
-
-module.exports.request = function(req, resp) {
-	https.get({
-		host: 'registry.npmjs.org',
-		path: req.url,
-		ca: require('./npmsslkeys'),
-		headers: {
-			'User-Agent': 'sinopia/0.0.0',
-		},
-	}, function(res) {
-		resp.writeHead(res.statusCode, res.headers);
-		res.on('data', function(d) {
-			resp.write(d);
-		});
-		res.on('end', function() {
-			resp.end();
-		});
-	}).on('error', function(err) {
-		console.error(err);
-		resp.send(500);
-	});
-}
-

lib/st-local.js

@@ -1,8 +1,9 @@
+var fs = require('fs');
 var semver = require('semver');
 var fs_storage = require('./fs-storage');
 var UError = require('./error').UserError;
+var utils = require('./utils');
 var info_file = 'package.json';
-var fs = require('fs');
 
 function Storage(config) {
 	if (!(this instanceof Storage)) return new Storage(config);
@@ -14,9 +15,14 @@ function Storage(config) {
 // returns the minimal package file
 function get_boilerplate(name) {
 	return {
+		// standard things
 		name: name,
 		versions: {},
 		'dist-tags': {},
+
+		// our own object
+		// type: "filename"->"metadata"
+		'_distfiles': {},
 	};
 }
 
@@ -58,8 +64,28 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
 		var change = false;
 		for (var ver in newdata.versions) {
 			if (data.versions[ver] == null) {
+				var verdata = newdata.versions[ver];
+
+				// why does anyone need to keep that in database?
+				delete verdata.readme;
+
 				change = true;
-				data.versions[ver] = newdata.versions[ver];
+				data.versions[ver] = verdata;
+				
+				if (verdata.dist && verdata.dist.tarball) {
+					var url = utils.parse_tarball_url(
+						verdata.dist.__sinopia_orig_tarball || verdata.dist.tarball
+					);
+
+					// we do NOT overwrite any existing records
+					if (url != null && data._distfiles[url.filename] == null) {
+						data._distfiles[url.filename] = {
+							url: verdata.dist.tarball,
+							prefix: url.protocol + '//' + url.host + url.prepath,
+							sha: verdata.dist.shasum,
+						};
+					}
+				}
 			}
 		}
 		for (var tag in newdata['dist-tags']) {
@@ -85,6 +111,9 @@ Storage.prototype.update_versions = function(name, newdata, callback) {
 Storage.prototype.add_version = function(name, version, metadata, tag, callback) {
 	var self = this;
 	self._read_create_package(name, function(err, data) {
+		// why does anyone need to keep that in database?
+		delete metadata.readme;
+
 		if (err) return callback(err);
 
 		if (data.versions[version] != null) {

lib/st-proxy.js

@@ -2,24 +2,40 @@ var request = require('request');
 var UError = require('./error').UserError;
 var URL = require('url');
 
-function Storage(name, config) {
+function Storage(config, mainconfig) {
 	if (!(this instanceof Storage)) return new Storage(config);
 	this.config = config;
-	this.name = name;
+	this.ua = mainconfig.user_agent;
 	this.ca;
 
-	if (URL.parse(this.config.uplinks[this.name].url).hostname === 'registry.npmjs.org') {
+	this.url = URL.parse(this.config.url);
+	if (this.url.hostname === 'registry.npmjs.org') {
 		this.ca = require('./npmsslkeys');
+		if (this.config._autogenerated) {
+			// encrypt all the things!
+			this.url.protocol = 'https';
+			this.config.url = URL.format(this.url);
+		}
 	}
+
+	this.config.url = this.config.url.replace(/\/$/, '');
 	return this;
 }
 
+Storage.prototype.can_fetch_url = function(url) {
+	url = URL.parse(url);
+
+	return url.protocol === this.url.protocol
+	    && url.host === this.url.host
+	    && url.path === this.url.path
+}
+
 Storage.prototype.get_package = function(name, callback) {
 	request({
-		url: this.config.uplinks[this.name].url + '/' + name,
+		url: this.config.url + '/' + name,
 		json: true,
 		headers: {
-			'User-Agent': this.config.user_agent,
+			'User-Agent': this.ua,
 		},
 		ca: this.ca,
 	}, function(err, res, body) {
@@ -37,5 +53,28 @@ Storage.prototype.get_package = function(name, callback) {
 	});
 }
 
+Storage.prototype.get_tarball = function(name, filename, callback) {
+	request({
+		url: this.config.url + '/' + name + '/-/' + filename,
+		headers: {
+			'User-Agent': this.ua,
+		},
+		ca: this.ca,
+		encoding: null,
+	}, function(err, res, body) {
+		if (err) return callback(err);
+		if (res.statusCode === 404) {
+			return callback(new UError({
+				msg: 'file doesn\'t exist on uplink',
+				status: 404,
+			}));
+		}
+		if (!(res.statusCode >= 200 && res.statusCode < 300)) {
+			return callback(new Error('bad status code: ' + res.statusCode));
+		}
+		callback(null, body);
+	});
+}
+
 module.exports = Storage;
 

lib/storage.js

@@ -11,7 +11,7 @@ function Storage(config) {
 	this.config = config;
 	this.uplinks = {};
 	for (var p in config.uplinks) {
-		this.uplinks[p] = new Proxy(p, config);
+		this.uplinks[p] = new Proxy(config.uplinks[p], config);
 	}
 	this.local = new Local(config);
 
@@ -67,22 +67,47 @@ Storage.prototype.add_tarball = function(name, filename, stream, callback) {
 }
 
 Storage.prototype.get_tarball = function(name, filename, callback) {
+	var self = this;
+
 	// if someone requesting tarball, it means that we should already have some
 	// information about it, so fetching package info is unnecessary
 
 	// trying local first
-	this.local.get_tarball(name, filename, function(err, results) {
+	self.local.get_tarball(name, filename, function(err, results) {
 		if (err && err.status !== 404) return callback(err);
 		if (!err && results != null) return callback(err, results);
 
+		var uplink = null;
+		var err404 = err;
 
-return callback(err, results);
-/*TODO:// local reported 404
-		this.local.get_package(name, function(err, info) {
+		// local reported 404
+		self.local.get_package(name, function(err, info) {
 			if (err) return callback(err);
 
-			
-		});*/
+			if (info._distfiles[filename] == null) {
+				return callback(err404);
+			}
+
+			var file = info._distfiles[filename];
+			var uplink = null;
+			for (var p in self.uplinks) {
+				if (self.uplinks[p].can_fetch_url(file.url)) {
+					uplink = self.uplinks[p];
+				}
+			}
+			if (uplink == null) {
+				uplink = new Proxy({
+					url: file.prefix,
+					_autogenerated: true,
+				}, self.config);
+			}
+
+			uplink.get_tarball(name, filename, function(err, res) {
+				if (err) return callback(err);
+
+				return callback(err, res);
+			});
+		});
 	});
 }
 

lib/utils.js

@@ -36,32 +36,49 @@ module.exports.validate_metadata = function(object, name) {
 	return object;
 }
 
+module.exports.parse_tarball_url = function(_url) {
+	var url = URL.parse(_url);
+
+	var path = url.path.replace(/^\//, '').split('/');
+	if (path.length >= 3 && path[path.length-2] === '-') {
+		var filename = path.pop();
+		var pkgpath = '/' + filename; // tarball name
+		pkgpath = '/' + path.pop() + pkgpath; // "-"
+		pkgpath = '/' + path.pop() + pkgpath; // package.name
+	} else {
+		return null;
+	}
+
+	return {
+		protocol: url.protocol,
+		host: url.host,
+		prepath: '/' + path.join('/'),
+		pkgpath: pkgpath,
+		filename: filename,
+	};
+}
+
 module.exports.filter_tarball_urls = function(pkg, req, config) {
 	function filter(_url) {
 		if (!req.headers.host) return _url;
 
-		var url = URL.parse(_url);
+		var url = module.exports.parse_tarball_url(_url);
+		// weird url, just return it
+		if (url == null) return _url;
+
 		if (config.url_prefix != null) {
 			var result = config.url_prefix.replace(/\/$/, '');
 		} else {
 			var result = req.protocol + '://' + req.headers.host;
 		}
 
-		path = url.path.replace(/^\//, '').split('/');
-		if (path.length >= 3 && path[path.length-2] === '-') {
-			result += '/' + path[path.length-3]; // package name
-			result += '/' + path[path.length-2]; // "-"
-			result += '/' + path[path.length-1]; // tarball name
-			return result;
-		} else {
-			// weird url, just return it
-			return _url;
-		}
+		return result + url.pkgpath;
 	}
 
 	for (var ver in pkg.versions) {
 		if (pkg.versions[ver].dist != null
 		&&  pkg.versions[ver].dist.tarball != null) {
+			pkg.versions[ver].dist.__sinopia_orig_tarball = pkg.versions[ver].dist.tarball;
 			pkg.versions[ver].dist.tarball = filter(pkg.versions[ver].dist.tarball);
 		}
 	}