diff --git a/lib/index-api.js b/lib/index-api.js new file mode 100644 index 000000000..f9ea92fcc --- /dev/null +++ b/lib/index-api.js @@ -0,0 +1,303 @@ +var Cookies = require('cookies') +var express = require('express') +var expressJson5 = require('express-json5') +var Error = require('http-errors') +var Middleware = require('./middleware') +var Utils = require('./utils') +var expect_json = Middleware.expect_json +var match = Middleware.match +var media = Middleware.media +var validate_name = Middleware.validate_name + +module.exports = function(config, auth, storage) { + var app = express.Router() + var can = Middleware.allow(config) + + // validate all of these params as a package name + // this might be too harsh, so ask if it causes trouble + app.param('package', validate_name) + app.param('filename', validate_name) + app.param('tag', validate_name) + app.param('version', validate_name) + app.param('revision', validate_name) + + // these can't be safely put into express url for some reason + app.param('_rev', match(/^-rev$/)) + app.param('org_couchdb_user', match(/^org\.couchdb\.user:/)) + app.param('anything', match(/.*/)) + + app.use(auth.auth_middleware()) + app.use(expressJson5({ strict: false, limit: config.max_body_size || '10mb' })) + app.use(Middleware.anti_loop(config)) + + // TODO: anonymous user? + app.get('/:package/:version?', can('access'), function(req, res, next) { + storage.get_package(req.params.package, { req: req }, function(err, info) { + if (err) return next(err) + info = Utils.filter_tarball_urls(info, req, config) + + var version = req.params.version + if (!version) return res.send(info) + + var t = Utils.get_version(info, version) + if (t != null) return res.send(t) + + if (info['dist-tags'] != null) { + if (info['dist-tags'][version] != null) { + version = info['dist-tags'][version] + if ((t = Utils.get_version(info, version)) != null) { + return res.send(t) + } + } + } + + return next( Error[404]('version not found: ' + req.params.version) ) + }) + }) + + app.get('/:package/-/:filename', can('access'), function(req, res, next) { + var stream = storage.get_tarball(req.params.package, req.params.filename) + stream.on('content-length', function(v) { + res.header('Content-Length', v) + }) + stream.on('error', function(err) { + return res.report_error(err) + }) + res.header('Content-Type', 'application/octet-stream') + stream.pipe(res) + }) + + // searching packages + app.get('/-/all/:anything?', function(req, res, next) { + storage.search(req.param.startkey || 0, {req: req}, function(err, result) { + if (err) return next(err) + for (var pkg in result) { + if (!config.allow_access(pkg, req.remote_user)) { + delete result[pkg] + } + } + return res.send(result) + }) + }) + + //app.get('/*', function(req, res) { + // proxy.request(req, res) + //}) + + // placeholder 'cause npm require to be authenticated to publish + // we do not do any real authentication yet + app.post('/_session', Cookies.express(), function(req, res) { + res.cookies.set('AuthSession', String(Math.random()), { + // npmjs.org sets 10h expire + expires: new Date(Date.now() + 10*60*60*1000) + }) + res.send({ ok: true, name: 'somebody', roles: [] }) + }) + + app.get('/-/user/:org_couchdb_user', function(req, res, next) { + res.status(200) + return res.send({ + ok: 'you are authenticated as "' + req.remote_user.name + '"', + }) + }) + + app.put('/-/user/:org_couchdb_user/:_rev?/:revision?', function(req, res, next) { + if (req.remote_user.name != null) { + res.status(201) + return res.send({ + ok: 'you are authenticated as "' + req.remote_user.name + '"', + }) + } else { + if (typeof(req.body.name) !== 'string' || typeof(req.body.password) !== 'string') { + if (typeof(req.body.password_sha)) { + return next( Error[422]("your npm version is outdated\nPlease update to npm@1.4.5 or greater.\nSee https://github.com/rlidwka/sinopia/issues/93 for details.") ) + } else { + return next( Error[422]('user/password is not found in request (npm issue?)') ) + } + } + auth.add_user(req.body.name, req.body.password, function(err) { + if (err) { + if (err.status < 500 && err.message === 'this user already exists') { + // with npm registering is the same as logging in + // so we replace message in case of conflict + return next( Error[409]('bad username/password, access denied') ) + } + return next(err) + } + + res.status(201) + return res.send({ ok: 'user "' + req.body.name + '" created' }) + }) + } + }) + + // tagging a package + app.put('/:package/:tag', can('publish'), media('application/json'), function(req, res, next) { + if (typeof(req.body) !== 'string') return next('route') + + var tags = {} + tags[req.params.tag] = req.body + storage.add_tags(req.params.package, tags, function(err) { + if (err) return next(err) + res.status(201) + return res.send({ ok: 'package tagged' }) + }) + }) + + // publishing a package + app.put('/:package/:_rev?/:revision?', can('publish'), media('application/json'), expect_json, function(req, res, next) { + var name = req.params.package + + if (Object.keys(req.body).length == 1 && Utils.is_object(req.body.users)) { + // 501 status is more meaningful, but npm doesn't show error message for 5xx + return next( Error[404]('npm star|unstar calls are not implemented') ) + } + +debugger + try { + var metadata = Utils.validate_metadata(req.body, name) + } catch(err) { + return next( Error[422]('bad incoming package data') ) + } + + if (req.params._rev) { + storage.change_package(name, metadata, req.params.revision, function(err) { + after_change(err, 'package changed') + }) + } else { + storage.add_package(name, metadata, function(err) { + after_change(err, 'created new package') + }) + } + + function after_change(err, ok_message) { + // old npm behaviour + if (metadata._attachments == null) { + if (err) return next(err) + res.status(201) + return res.send({ ok: ok_message }) + } + + // npm-registry-client 0.3+ embeds tarball into the json upload + // https://github.com/isaacs/npm-registry-client/commit/e9fbeb8b67f249394f735c74ef11fe4720d46ca0 + // issue #31, dealing with it here: + + if (typeof(metadata._attachments) !== 'object' + || Object.keys(metadata._attachments).length !== 1 + || typeof(metadata.versions) !== 'object' + || Object.keys(metadata.versions).length !== 1) { + + // npm is doing something strange again + // if this happens in normal circumstances, report it as a bug + return next( Error[400]('unsupported registry call') ) + } + + if (err && err.status != 409) return next(err) + + // at this point document is either created or existed before + var t1 = Object.keys(metadata._attachments)[0] + create_tarball(t1, metadata._attachments[t1], function(err) { + if (err) return next(err) + + var t2 = Object.keys(metadata.versions)[0] + metadata.versions[t2].readme = metadata.readme != null ? String(metadata.readme) : '' + create_version(t2, metadata.versions[t2], function(err) { + if (err) return next(err) + + add_tags(metadata['dist-tags'], function(err) { + if (err) return next(err) + + res.status(201) + return res.send({ ok: ok_message }) + }) + }) + }) + } + + function create_tarball(filename, data, cb) { + var stream = storage.add_tarball(name, filename) + stream.on('error', function(err) { + cb(err) + }) + stream.on('success', function() { + cb() + }) + + // this is dumb and memory-consuming, but what choices do we have? + stream.end(Buffer(data.data, 'base64')) + stream.done() + } + + function create_version(version, data, cb) { + storage.add_version(name, version, data, null, cb) + } + + function add_tags(tags, cb) { + storage.add_tags(name, tags, cb) + } + }) + + // unpublishing an entire package + app.delete('/:package/-rev/*', can('publish'), function(req, res, next) { + storage.remove_package(req.params.package, function(err) { + if (err) return next(err) + res.status(201) + return res.send({ ok: 'package removed' }) + }) + }) + + // removing a tarball + app.delete('/:package/-/:filename/-rev/:revision', can('publish'), function(req, res, next) { + storage.remove_tarball(req.params.package, req.params.filename, req.params.revision, function(err) { + if (err) return next(err) + res.status(201) + return res.send({ ok: 'tarball removed' }) + }) + }) + + // uploading package tarball + app.put('/:package/-/:filename/*', can('publish'), media('application/octet-stream'), function(req, res, next) { + var name = req.params.package + + var stream = storage.add_tarball(name, req.params.filename) + req.pipe(stream) + + // checking if end event came before closing + var complete = false + req.on('end', function() { + complete = true + stream.done() + }) + req.on('close', function() { + if (!complete) { + stream.abort() + } + }) + + stream.on('error', function(err) { + return res.report_error(err) + }) + stream.on('success', function() { + res.status(201) + return res.send({ + ok: 'tarball uploaded successfully' + }) + }) + }) + + // adding a version + app.put('/:package/:version/-tag/:tag', can('publish'), media('application/json'), expect_json, function(req, res, next) { + var name = req.params.package + var version = req.params.version + var tag = req.params.tag + + storage.add_version(name, version, req.body, tag, function(err) { + if (err) return next(err) + res.status(201) + return res.send({ ok: 'package published' }) + }) + }) + + return app +} + diff --git a/lib/index-web.js b/lib/index-web.js index 8793ec28a..5256371aa 100644 --- a/lib/index-web.js +++ b/lib/index-web.js @@ -1,17 +1,26 @@ -var Cookies = require('cookies') -var express = require('express') -var fs = require('fs') -var marked = require('marked') -var Handlebars = require('handlebars') -var Error = require('http-errors') -var bodyParser = require('body-parser') -var Search = require('./search') -var Middleware = require('./middleware') +var bodyParser = require('body-parser') +var Cookies = require('cookies') +var express = require('express') +var fs = require('fs') +var marked = require('marked') +var Handlebars = require('handlebars') +var Error = require('http-errors') +var Search = require('./search') +var Middleware = require('./middleware') +var match = Middleware.match +var validate_name = Middleware.validate_name module.exports = function(config, auth, storage) { - var app = express() + var app = express.Router() var can = Middleware.allow(config) + // validate all of these params as a package name + // this might be too harsh, so ask if it causes trouble + app.param('package', validate_name) + app.param('filename', validate_name) + app.param('version', validate_name) + app.param('anything', match(/.*/)) + app.use(Cookies.express()) app.use(bodyParser.urlencoded({ extended: false })) app.use(auth.cookie_middleware()) diff --git a/lib/index.js b/lib/index.js index 3fccd959e..73a1eeadd 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,8 +1,6 @@ -var Cookies = require('cookies') var express = require('express') var fs = require('fs') var Error = require('http-errors') -var expressJson5 = require('express-json5') var compression = require('compression') var Auth = require('./auth') var Logger = require('./logger') @@ -10,20 +8,6 @@ var Config = require('./config') var Middleware = require('./middleware') var Cats = require('./status-cats') var Storage = require('./storage') -var Utils = require('./utils') -var expect_json = Middleware.expect_json -var media = Middleware.media -var validate_name = Middleware.validate_name - -function match(regexp) { - return function(req, res, next, value, name) { - if (regexp.exec(value)) { - next() - } else { - next('route') - } - } -} module.exports = function(config_hash) { var config = Config(config_hash) @@ -67,23 +51,7 @@ module.exports = function(config_hash) { next() }) app.use(Cats.middleware) - app.use(auth.auth_middleware()) - app.use(expressJson5({ strict: false, limit: config.max_body_size || '10mb' })) app.use(compression()) - app.use(Middleware.anti_loop(config)) - - // validate all of these params as a package name - // this might be too harsh, so ask if it causes trouble - app.param('package', validate_name) - app.param('filename', validate_name) - app.param('tag', validate_name) - app.param('version', validate_name) - app.param('revision', validate_name) - - // these can't be safely put into express url for some reason - app.param('_rev', match(/^-rev$/)) - app.param('org_couchdb_user', match(/^org\.couchdb\.user:/)) - app.param('anything', match(/.*/)) app.get('/favicon.ico', function(req, res, next) { req.url = '/-/static/favicon.png' @@ -102,273 +70,6 @@ module.exports = function(config_hash) { }) })*/ - // TODO: anonymous user? - app.get('/:package/:version?', can('access'), function(req, res, next) { - storage.get_package(req.params.package, { req: req }, function(err, info) { - if (err) return next(err) - info = Utils.filter_tarball_urls(info, req, config) - - var version = req.params.version - if (!version) return res.send(info) - - var t = Utils.get_version(info, version) - if (t != null) return res.send(t) - - if (info['dist-tags'] != null) { - if (info['dist-tags'][version] != null) { - version = info['dist-tags'][version] - if ((t = Utils.get_version(info, version)) != null) { - return res.send(t) - } - } - } - - return next( Error[404]('version not found: ' + req.params.version) ) - }) - }) - - app.get('/:package/-/:filename', can('access'), function(req, res, next) { - var stream = storage.get_tarball(req.params.package, req.params.filename) - stream.on('content-length', function(v) { - res.header('Content-Length', v) - }) - stream.on('error', function(err) { - return res.report_error(err) - }) - res.header('Content-Type', 'application/octet-stream') - stream.pipe(res) - }) - - // searching packages - app.get('/-/all/:anything?', function(req, res, next) { - storage.search(req.param.startkey || 0, {req: req}, function(err, result) { - if (err) return next(err) - for (var pkg in result) { - if (!config.allow_access(pkg, req.remote_user)) { - delete result[pkg] - } - } - return res.send(result) - }) - }) - - //app.get('/*', function(req, res) { - // proxy.request(req, res) - //}) - - // placeholder 'cause npm require to be authenticated to publish - // we do not do any real authentication yet - app.post('/_session', Cookies.express(), function(req, res) { - res.cookies.set('AuthSession', String(Math.random()), { - // npmjs.org sets 10h expire - expires: new Date(Date.now() + 10*60*60*1000) - }) - res.send({ ok: true, name: 'somebody', roles: [] }) - }) - - app.get('/-/user/:org_couchdb_user', function(req, res, next) { - res.status(200) - return res.send({ - ok: 'you are authenticated as "' + req.remote_user.name + '"', - }) - }) - - app.put('/-/user/:org_couchdb_user/:_rev?/:revision?', function(req, res, next) { - if (req.remote_user.name != null) { - res.status(201) - return res.send({ - ok: 'you are authenticated as "' + req.remote_user.name + '"', - }) - } else { - if (typeof(req.body.name) !== 'string' || typeof(req.body.password) !== 'string') { - if (typeof(req.body.password_sha)) { - return next( Error[422]("your npm version is outdated\nPlease update to npm@1.4.5 or greater.\nSee https://github.com/rlidwka/sinopia/issues/93 for details.") ) - } else { - return next( Error[422]('user/password is not found in request (npm issue?)') ) - } - } - auth.add_user(req.body.name, req.body.password, function(err) { - if (err) { - if (err.status < 500 && err.message === 'this user already exists') { - // with npm registering is the same as logging in - // so we replace message in case of conflict - return next( Error[409]('bad username/password, access denied') ) - } - return next(err) - } - - res.status(201) - return res.send({ ok: 'user "' + req.body.name + '" created' }) - }) - } - }) - - // tagging a package - app.put('/:package/:tag', can('publish'), media('application/json'), function(req, res, next) { - if (typeof(req.body) !== 'string') return next('route') - - var tags = {} - tags[req.params.tag] = req.body - storage.add_tags(req.params.package, tags, function(err) { - if (err) return next(err) - res.status(201) - return res.send({ ok: 'package tagged' }) - }) - }) - - // publishing a package - app.put('/:package/:_rev?/:revision?', can('publish'), media('application/json'), expect_json, function(req, res, next) { - var name = req.params.package - - if (Object.keys(req.body).length == 1 && Utils.is_object(req.body.users)) { - // 501 status is more meaningful, but npm doesn't show error message for 5xx - return next( Error[404]('npm star|unstar calls are not implemented') ) - } - - try { - var metadata = Utils.validate_metadata(req.body, name) - } catch(err) { - return next( Error[422]('bad incoming package data') ) - } - - if (req.params._rev) { - storage.change_package(name, metadata, req.params.revision, function(err) { - after_change(err, 'package changed') - }) - } else { - storage.add_package(name, metadata, function(err) { - after_change(err, 'created new package') - }) - } - - function after_change(err, ok_message) { - // old npm behaviour - if (metadata._attachments == null) { - if (err) return next(err) - res.status(201) - return res.send({ ok: ok_message }) - } - - // npm-registry-client 0.3+ embeds tarball into the json upload - // https://github.com/isaacs/npm-registry-client/commit/e9fbeb8b67f249394f735c74ef11fe4720d46ca0 - // issue #31, dealing with it here: - - if (typeof(metadata._attachments) !== 'object' - || Object.keys(metadata._attachments).length !== 1 - || typeof(metadata.versions) !== 'object' - || Object.keys(metadata.versions).length !== 1) { - - // npm is doing something strange again - // if this happens in normal circumstances, report it as a bug - return next( Error[400]('unsupported registry call') ) - } - - if (err && err.status != 409) return next(err) - - // at this point document is either created or existed before - var t1 = Object.keys(metadata._attachments)[0] - create_tarball(t1, metadata._attachments[t1], function(err) { - if (err) return next(err) - - var t2 = Object.keys(metadata.versions)[0] - metadata.versions[t2].readme = metadata.readme != null ? String(metadata.readme) : '' - create_version(t2, metadata.versions[t2], function(err) { - if (err) return next(err) - - add_tags(metadata['dist-tags'], function(err) { - if (err) return next(err) - - res.status(201) - return res.send({ ok: ok_message }) - }) - }) - }) - } - - function create_tarball(filename, data, cb) { - var stream = storage.add_tarball(name, filename) - stream.on('error', function(err) { - cb(err) - }) - stream.on('success', function() { - cb() - }) - - // this is dumb and memory-consuming, but what choices do we have? - stream.end(Buffer(data.data, 'base64')) - stream.done() - } - - function create_version(version, data, cb) { - storage.add_version(name, version, data, null, cb) - } - - function add_tags(tags, cb) { - storage.add_tags(name, tags, cb) - } - }) - - // unpublishing an entire package - app.delete('/:package/-rev/*', can('publish'), function(req, res, next) { - storage.remove_package(req.params.package, function(err) { - if (err) return next(err) - res.status(201) - return res.send({ ok: 'package removed' }) - }) - }) - - // removing a tarball - app.delete('/:package/-/:filename/-rev/:revision', can('publish'), function(req, res, next) { - storage.remove_tarball(req.params.package, req.params.filename, req.params.revision, function(err) { - if (err) return next(err) - res.status(201) - return res.send({ ok: 'tarball removed' }) - }) - }) - - // uploading package tarball - app.put('/:package/-/:filename/*', can('publish'), media('application/octet-stream'), function(req, res, next) { - var name = req.params.package - - var stream = storage.add_tarball(name, req.params.filename) - req.pipe(stream) - - // checking if end event came before closing - var complete = false - req.on('end', function() { - complete = true - stream.done() - }) - req.on('close', function() { - if (!complete) { - stream.abort() - } - }) - - stream.on('error', function(err) { - return res.report_error(err) - }) - stream.on('success', function() { - res.status(201) - return res.send({ - ok: 'tarball uploaded successfully' - }) - }) - }) - - // adding a version - app.put('/:package/:version/-tag/:tag', can('publish'), media('application/json'), expect_json, function(req, res, next) { - var name = req.params.package - var version = req.params.version - var tag = req.params.tag - - storage.add_version(name, version, req.body, tag, function(err) { - if (err) return next(err) - res.status(201) - return res.send({ ok: 'package published' }) - }) - }) - // hook for tests only if (config._debug) { app.get('/-/_debug', function(req, res) { @@ -384,11 +85,13 @@ module.exports = function(config_hash) { }) } + app.use(require('./index-api')(config, auth, storage)) + if (config.web && config.web.enable) { app.use(require('./index-web')(config, auth, storage)) } else { app.get('/', function(req, res) { - res.send('Web interface is disabled in the config file') + next( Error[404]('web interface is disabled in the config file') ) }) } @@ -397,7 +100,7 @@ module.exports = function(config_hash) { }) app.use(function(err, req, res, next) { - if (err.code === 'ECONNABORT' && res.statusCode === 304) return + if (err.code === 'ECONNABORT' && res.statusCode === 304) return next() if (typeof(res.report_error) !== 'function') { // in case of very early error this middleware may not be loaded before error is generated // fixing that diff --git a/lib/local-storage.js b/lib/local-storage.js index 7002c0f38..0641aa133 100644 --- a/lib/local-storage.js +++ b/lib/local-storage.js @@ -309,6 +309,7 @@ Storage.prototype.remove_tarball = function(name, filename, revision, callback) } Storage.prototype.add_tarball = function(name, filename) { +debugger assert(Utils.validate_name(filename)) var stream = MyStreams.UploadTarballStream() diff --git a/lib/middleware.js b/lib/middleware.js index ca51eec47..bcc4fc3c9 100644 --- a/lib/middleware.js +++ b/lib/middleware.js @@ -3,6 +3,16 @@ var Error = require('http-errors') var utils = require('./utils') var Logger = require('./logger') +module.exports.match = function match(regexp) { + return function(req, res, next, value, name) { + if (regexp.exec(value)) { + next() + } else { + next('route') + } + } +} + module.exports.validate_name = function validate_name(req, res, next, value, name) { if (value.charAt(0) === '-') { // special case in couchdb usually diff --git a/test/unit/validate_all.js b/test/unit/validate_all.js index 09a03c0b9..662d41b6b 100644 --- a/test/unit/validate_all.js +++ b/test/unit/validate_all.js @@ -2,32 +2,38 @@ var assert = require('assert') -describe('index.js app', function() { - var source = require('fs').readFileSync(__dirname + '/../../lib/index.js', 'utf8') +describe('index.js app', test('index.js')) +describe('index-api.js app', test('index-api.js')) +describe('index-web.js app', test('index-web.js')) - var very_scary_regexp = /\n\s*app\.(\w+)\s*\(\s*(("[^"]*")|('[^']*'))\s*,/g - var m - var params = {} +function test(file) { + return function() { + var source = require('fs').readFileSync(__dirname + '/../../lib/' + file, 'utf8') - while ((m = very_scary_regexp.exec(source)) != null) { - if (m[1] === 'set') continue + var very_scary_regexp = /\n\s*app\.(\w+)\s*\(\s*(("[^"]*")|('[^']*'))\s*,/g + var m + var params = {} - var inner = m[2].slice(1, m[2].length-1) - var t + while ((m = very_scary_regexp.exec(source)) != null) { + if (m[1] === 'set') continue - inner.split('/').forEach(function(x) { - if (m[1] === 'param') { - params[x] = 'ok' - } else if (t = x.match(/^:([^?:]*)\??$/)) { - params[t[1]] = params[t[1]] || m[0].trim() - } + var inner = m[2].slice(1, m[2].length-1) + var t + + inner.split('/').forEach(function(x) { + if (m[1] === 'param') { + params[x] = 'ok' + } else if (t = x.match(/^:([^?:]*)\??$/)) { + params[t[1]] = params[t[1]] || m[0].trim() + } + }) + } + + Object.keys(params).forEach(function(param) { + it('should validate ":'+param+'"', function() { + assert.equal(params[param], 'ok') + }) }) } - - Object.keys(params).forEach(function(param) { - it('should validate ":'+param+'"', function() { - assert.equal(params[param], 'ok') - }) - }) -}) +}