github.com
/
verdaccio
mirror of https://github.com/verdaccio/verdaccio.git
1
0
Fork 0
Code Releases Wiki Activity

refactor: load storage as plugin and as fallback use LocalStorage

This commit is contained in:
Juan Picado @jotadeveloper 2017-12-22 21:32:52 +01:00 committed by juanpicado
parent f320d5eeb4
commit d352c6c63f
15 changed files with 116 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
package.json
View File

@ -15,10 +15,10 @@
"verdaccio": "./bin/verdaccio"
},
"dependencies": {
"@verdaccio/file-locking": "^0.0.5",
"@verdaccio/local-storage": "^0.1.0",
"@verdaccio/streams": "^0.0.2",
"@verdaccio/types": "^0.1.0",
"@verdaccio/file-locking": "0.0.5",
"@verdaccio/local-storage": "0.1.0",
"@verdaccio/streams": "0.0.2",
"@verdaccio/types": "0.1.0",
"JSONStream": "^1.1.1",
"apache-md5": "^1.1.2",
"async": "^2.6.0",

23
src/api/index.js
View File

@ -1,22 +1,21 @@
'use strict';
import express from 'express';
import Error from 'http-errors';
import compression from 'compression';
import _ from 'lodash';
import cors from 'cors';
import Storage from '../lib/storage';
import {loadPlugin} from '../lib/plugin-loader';
const express = require('express');
const Error = require('http-errors');
const compression = require('compression');
const Auth = require('../lib/auth');
const Logger = require('../lib/logger');
const Config = require('../lib/config');
const Middleware = require('./web/middleware');
const Cats = require('../lib/status-cats');
const Storage = require('../lib/storage');
const _ = require('lodash');
const cors = require('cors');
const load_plugins = require('../lib/plugin-loader').load_plugins;
module.exports = function(config_hash) {
module.exports = function(configHash) {
// Config
Logger.setup(config_hash.logs);
const config = new Config(config_hash);
Logger.setup(configHash.logs);
const config = new Config(configHash);
const storage = new Storage(config);
const auth = new Auth(config);
const app = express();
@ -86,7 +85,7 @@ module.exports = function(config_hash) {
config: config,
logger: Logger.logger,
};
const plugins = load_plugins(config, config.middlewares, plugin_params, function(plugin) {
const plugins = loadPlugin(config, config.middlewares, plugin_params, function(plugin) {
return plugin.register_middlewares;
});
plugins.forEach(function(plugin) {

3
src/api/web/api.js
View File

@ -1,10 +1,9 @@
'use strict';
import Search from '../../lib/search';
const bodyParser = require('body-parser');
const express = require('express');
const marked = require('marked');
const _ = require('lodash');
const Search = require('../../lib/search');
const Middleware = require('./middleware');
const match = Middleware.match;
const validateName = Middleware.validate_name;

11
src/api/web/index.js
View File

@ -1,14 +1,13 @@
'use strict';
import express from 'express';
import _ from 'lodash';
import fs from 'fs';
import Search from '../../lib/search';
import * as Utils from '../../lib/utils';
const express = require('express');
const Search = require('../../lib/search');
const Middleware = require('./middleware');
const Utils = require('../../lib/utils');
/* eslint new-cap:off */
const router = express.Router();
const _ = require('lodash');
const env = require('../../config/env');
const fs = require('fs');
const template = fs.readFileSync(`${env.DIST_PATH}/index.html`).toString();
const spliceURL = require('../../utils/string').spliceURL;

6
src/lib/auth.js
View File

@ -1,12 +1,10 @@
/* eslint prefer-spread: "off" */
/* eslint prefer-rest-params: "off" */
'use strict';
import {loadPlugin} from '../lib/plugin-loader';
const Crypto = require('crypto');
const Error = require('http-errors');
const Logger = require('./logger');
const load_plugins = require('./plugin-loader').load_plugins;
const pkgJson = require('../../package.json');
const jwt = require('jsonwebtoken');
/**
@ -35,7 +33,7 @@ class Auth {
}
}
this.plugins = load_plugins(config, config.auth, plugin_params, function(p) {
this.plugins = loadPlugin(config, config.auth, plugin_params, function(p) {
return p.authenticate || p.allow_access || p.allow_publish;
});

11
src/lib/local-storage.js
View File

@ -551,7 +551,7 @@ class LocalStorage implements IStorage {
return callback( Utils.ErrorCode.get404() );
}
this._readJSON(storage, callback);
this._readPackage(storage, callback);
}
/**
@ -574,6 +574,7 @@ class LocalStorage implements IStorage {
if (err) {
return cb(err);
}
const listVersions: Array<string> = Object.keys(data.versions);
const versions: Array<string> = Utils.semver_sort(listVersions);
const latest: string = data['dist-tags'] && data['dist-tags'].latest ? data['dist-tags'].latest : versions.pop();
@ -623,7 +624,7 @@ class LocalStorage implements IStorage {
* @return {Object}
*/
_getLocalStorage(packageInfo: string): IPackageStorage {
const path: string = this.__getLocalStoragePath(this.config.getMatchedPackagesSpec(packageInfo).storage);
const path: string = this._getLocalStoragePath(this.config.getMatchedPackagesSpec(packageInfo).storage);
if (_.isString(path) === false) {
this.logger.debug( {name: packageInfo}, 'this package has no storage defined: @{name}' );
@ -638,7 +639,7 @@ class LocalStorage implements IStorage {
* @param {Object} storage
* @param {Function} callback
*/
_readJSON(storage: IPackageStorage, callback: Callback) {
_readPackage(storage: IPackageStorage, callback: Callback) {
storage.readPackage(pkgFileName, (err, result) => {
if (err) {
if (err.code === noSuchFile) {
@ -658,7 +659,7 @@ class LocalStorage implements IStorage {
* @return {String}
* @private
*/
__getLocalStoragePath(path: string): string {
_getLocalStoragePath(path: string): string {
if (_.isNil(path) === false) {
return path;
}
@ -850,4 +851,4 @@ class LocalStorage implements IStorage {
}
}
module.exports = LocalStorage;
export default LocalStorage;

21
src/lib/plugin-loader.js
View File

@ -1,14 +1,13 @@
'use strict';
const Path = require('path');
const logger = require('./logger');
import Path from 'path';
import logger from './logger';
/**
* Requires a module.
* @param {*} path the module's path
* @return {Object}
*/
function try_load(path) {
function tryLoad(path) {
try {
return require(path);
} catch(err) {
@ -30,29 +29,29 @@ function try_load(path) {
* @param {*} sanity_check callback that check the shape that should fulfill the plugin
* @return {Array} list of plugins
*/
function load_plugins(config, plugin_configs, params, sanity_check) {
function loadPlugin(config, plugin_configs, params, sanity_check) {
let plugins = Object.keys(plugin_configs || {}).map(function(p) {
let plugin;
// try local plugins first
plugin = try_load(Path.resolve(__dirname + '/..//plugins', p));
plugin = tryLoad(Path.resolve(__dirname + '/..//plugins', p));
// npm package
if (plugin === null && p.match(/^[^\.\/]/)) {
plugin = try_load(`verdaccio-${p}`);
plugin = tryLoad(`verdaccio-${p}`);
// compatibility for old sinopia plugins
if (!plugin) {
plugin = try_load(`sinopia-${p}`);
plugin = tryLoad(`sinopia-${p}`);
}
}
if (plugin === null) {
plugin = try_load(p);
plugin = tryLoad(p);
}
// relative to config path
if (plugin === null && p.match(/^\.\.?($|\/)/)) {
plugin = try_load(Path.resolve(Path.dirname(config.self_path), p));
plugin = tryLoad(Path.resolve(Path.dirname(config.self_path), p));
}
if (plugin === null) {
@ -78,4 +77,4 @@ function load_plugins(config, plugin_configs, params, sanity_check) {
return plugins;
}
exports.load_plugins = load_plugins;
export {loadPlugin};

4
src/lib/search.js
View File

@ -1,7 +1,5 @@
/* eslint no-invalid-this: "off" */
'use strict';
const lunr = require('lunr');
/**
@ -80,4 +78,4 @@ class Search {
}
}
module.exports = new Search();
export default new Search();

70
src/lib/storage.js
View File

@ -1,19 +1,20 @@
'use strict';
const _ = require('lodash');
const assert = require('assert');
const async = require('async');
const Error = require('http-errors');
const semver = require('semver');
const Stream = require('stream');
import _ from 'lodash';
import assert from 'assert';
import async from 'async';
import Error from 'http-errors';
import semver from 'semver';
import Stream from 'stream';
const Search = require('./search');
const Logger = require('./logger');
const LocalStorage = require('./local-storage');
const MyStreams = require('@verdaccio/streams');
const Proxy = require('./up-storage');
const Utils = require('./utils');
import Search from './search';
import LocalStorage from './local-storage';
import {ReadTarball} from '@verdaccio/streams';
import ProxyStorage from './up-storage';
import * as Utils from './utils';
import {loadPlugin} from '../lib/plugin-loader';
const Logger = require('../lib/logger');
const WHITELIST = ['_rev', 'name', 'versions', 'dist-tags', 'readme', 'time'];
const getDefaultMetadata = (name) => {
return {
@ -36,8 +37,29 @@ class Storage {
constructor(config) {
this.config = config;
this._setupUpLinks(this.config);
this.localStorage = new LocalStorage(config, Logger.logger);
this.logger = Logger.logger.child();
this.localStorage = this._loadStorage();
}
_loadStorage() {
const Storage = this._loadStorePlugin();
if (_.isNil(Storage)) {
return new LocalStorage(this.config, Logger.logger);
} else {
return new Storage(this.config, Logger.logger);
}
}
_loadStorePlugin() {
const plugin_params = {
config: this.config,
logger: this.logger,
};
return _.head(loadPlugin(this.config, this.config.store, plugin_params, function(plugin) {
return plugin.getPackageStorage;
}));
}
/**
@ -236,7 +258,7 @@ class Storage {
* @return {Stream}
*/
get_tarball(name, filename) {
let readStream = new MyStreams.ReadTarball();
let readStream = new ReadTarball();
readStream.abort = function() {};
let self = this;
@ -295,7 +317,7 @@ class Storage {
}
}
if (uplink == null) {
uplink = new Proxy({
uplink = new ProxyStorage({
url: file.url,
cache: true,
_autogenerated: true,
@ -629,7 +651,7 @@ class Storage {
for (let p in config.uplinks) {
if (Object.prototype.hasOwnProperty.call(config.uplinks, p)) {
// instance for each up-link definition
this.uplinks[p] = new Proxy(config.uplinks[p], config);
this.uplinks[p] = new ProxyStorage(config.uplinks[p], config);
this.uplinks[p].upname = p;
}
}
@ -653,12 +675,14 @@ class Storage {
}
// refresh dist-tags
for (let i in up['dist-tags']) {
if (local['dist-tags'][i] !== up['dist-tags'][i]) {
if (!local['dist-tags'][i] || semver.lte(local['dist-tags'][i], up['dist-tags'][i])) {
local['dist-tags'][i] = up['dist-tags'][i];
const distTag = 'dist-tags';
for (let i in up[distTag]) {
if (local[distTag][i] !== up[distTag][i]) {
if (!local[distTag][i] || semver.lte(local[distTag][i], up[distTag][i])) {
local[distTag][i] = up[distTag][i];
}
if (i === 'latest' && local['dist-tags'][i] === up['dist-tags'][i]) {
if (i === 'latest' && local[distTag][i] === up[distTag][i]) {
// if remote has more fresh package, we should borrow its readme
local.readme = up.readme;
}
@ -668,4 +692,4 @@ class Storage {
}
module.exports = Storage;
export default Storage;

2
src/lib/up-storage.js
View File

@ -588,4 +588,4 @@ class ProxyStorage {
}
module.exports = ProxyStorage;
export default ProxyStorage;

6
test/unit/no_proxy.spec.js
View File

@ -1,7 +1,5 @@
'use strict';
let assert = require('assert');
let Storage = require('../../src/lib/up-storage');
import assert from 'assert';
import Storage from '../../src/lib/up-storage';
require('../../src/lib/logger').setup([]);

26
test/unit/plugin_loader.spec.js
View File

@ -1,9 +1,9 @@
'use strict';
import {loadPlugin} from '../../src/lib/plugin-loader';
import assert from 'assert';
import path from 'path';
require('../../src/lib/logger').setup([]);
const assert = require('assert');
const load_plugins = require('../../src/lib/plugin-loader').load_plugins;
const path = require('path');
describe('plugin loader', () => {
@ -15,10 +15,10 @@ describe('plugin loader', () => {
'./unit/partials/test-plugin-storage/verdaccio-plugin': {}
}
}
let p = load_plugins(_config, _config.auth, {}, function (p) {
let plugins = loadPlugin(_config, _config.auth, {}, function (p) {
return p.authenticate || p.allow_access || p.allow_publish;
});
assert(p.length === 1);
assert(plugins.length === 1);
});
test('testing auth plugin invalid plugin', () => {
@ -29,7 +29,7 @@ describe('plugin loader', () => {
}
}
try {
load_plugins(_config, _config.auth, {}, function (p) {
loadPlugin(_config, _config.auth, {}, function (p) {
return p.authenticate || p.allow_access || p.allow_publish;
});
} catch(e) {
@ -45,11 +45,11 @@ describe('plugin loader', () => {
}
}
try {
load_plugins(_config, _config.auth, {}, function (p) {
return p.authenticate || p.allow_access || p.allow_publish;
loadPlugin(_config, _config.auth, {}, function (plugin) {
return plugin.authenticate || plugin.allow_access || plugin.allow_publish;
});
} catch(e) {
assert(e.message === '"./unit/partials/test-plugin-storage/invalid-plugin-sanity" doesn\'t look like a valid plugin');
} catch(err) {
assert(err.message === '"./unit/partials/test-plugin-storage/invalid-plugin-sanity" doesn\'t look like a valid plugin');
}
});
@ -61,8 +61,8 @@ describe('plugin loader', () => {
}
}
try {
load_plugins(_config, _config.auth, {}, function (p) {
return p.authenticate || p.allow_access || p.allow_publish;
loadPlugin(_config, _config.auth, {}, function (plugin) {
return plugin.authenticate || plugin.allow_access || plugin.allow_publish;
});
} catch(e) {
assert(e.message === `"./unit/partials/test-plugin-storage/invalid-package" plugin not found\ntry "npm install verdaccio-./unit/partials/test-plugin-storage/invalid-package"`);

8
test/unit/search.spec.js
View File

@ -1,8 +1,8 @@
'use strict';
let assert = require('assert');
let Search = require('../../src/lib/search');
let Storage = require('../../src/lib/storage');
import assert from 'assert';
import Search from '../../src/lib/search';
import Storage from '../../src/lib/storage';
let config_hash = require('./partials/config');
let Config = require('../../src/lib/config');

10
test/unit/st_merge.spec.js
View File

@ -2,11 +2,11 @@
let assert = require('assert');
let semver_sort = require('../../src/lib/utils').semver_sort;
let merge = require('../../src/lib/storage')._merge_versions;
import Storage from '../../src/lib/storage';
require('../../src/lib/logger').setup([]);
describe('merge versions', () => {
describe('Storage._merge_versions versions', () => {
test('simple', () => {
let pkg = {
@ -14,7 +14,7 @@ describe('merge versions', () => {
'dist-tags': {},
};
merge(pkg, {versions: {a: 2, q: 2}});
Storage._merge_versions(pkg, {versions: {a: 2, q: 2}});
assert.deepEqual(pkg, {
'versions': {a: 1, b: 1, c: 1, q: 2},
@ -28,7 +28,7 @@ describe('merge versions', () => {
'dist-tags': {q: '1.1.1', w: '2.2.2'},
};
merge(pkg, {'dist-tags': {q: '2.2.2', w: '3.3.3', t: '4.4.4'}});
Storage._merge_versions(pkg, {'dist-tags': {q: '2.2.2', w: '3.3.3', t: '4.4.4'}});
assert.deepEqual(pkg, {
'versions': {},
@ -48,7 +48,7 @@ describe('merge versions', () => {
// against our local 1.1.10, which may end up published as 1.1.3 in the
// future
merge(pkg, {'dist-tags':{q:'1.1.2',w:'3.3.3',t:'4.4.4'}})
Storage._merge_versions(pkg, {'dist-tags':{q:'1.1.2',w:'3.3.3',t:'4.4.4'}})
assert.deepEqual(pkg, {
versions: {},

8
yarn.lock
View File

@ -59,14 +59,14 @@
version "8.5.1"
resolved "https://registry.npmjs.org/@types/node/-/node-8.5.1.tgz#4ec3020bcdfe2abffeef9ba3fbf26fca097514b5"
"@verdaccio/file-locking@^0.0.5":
"@verdaccio/file-locking@0.0.5", "@verdaccio/file-locking@^0.0.5":
version "0.0.5"
resolved "https://registry.npmjs.org/@verdaccio/file-locking/-/file-locking-0.0.5.tgz#6172dfa2f7094a1da8e4c14906bfa33836b5713d"
dependencies:
lockfile "1.0.3"
lodash "4.17.4"
"@verdaccio/local-storage@^0.1.0":
"@verdaccio/local-storage@0.1.0":
version "0.1.0"
resolved "https://registry.npmjs.org/@verdaccio/local-storage/-/local-storage-0.1.0.tgz#bab877e5d07bea926c97f251bddf2bfa9edd7026"
dependencies:
@ -77,11 +77,11 @@
lodash "4.17.4"
mkdirp "0.5.1"
"@verdaccio/streams@^0.0.2":
"@verdaccio/streams@0.0.2", "@verdaccio/streams@^0.0.2":
version "0.0.2"
resolved "https://registry.npmjs.org/@verdaccio/streams/-/streams-0.0.2.tgz#72cd65449e657b462a1ca094f663cad9ea872427"
"@verdaccio/types@^0.1.0":
"@verdaccio/types@0.1.0":
version "0.1.0"
resolved "https://registry.npmjs.org/@verdaccio/types/-/types-0.1.0.tgz#2a0a6066bbbb7841d29298463e761147fb1f7f00"