github.com
/
verdaccio
mirror of https://github.com/verdaccio/verdaccio.git
1
0
Fork 0
Code Releases Wiki Activity

refactor: mirror and anti-loop

This commit is contained in:
Juan Picado @jotadeveloper 2018-06-23 01:03:25 +02:00
parent 241945f2bb
commit 690b92a652
No known key found for this signature in database
GPG Key ID: 18AC54485952D158
14 changed files with 150 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/api/middleware.js
View File

@ -6,7 +6,7 @@ import {
validate_package as utilValidatePackage,
isObject,
ErrorCode} from '../lib/utils';
import {HEADERS} from '../lib/constants';
import {API_ERROR, HEADER_TYPE, HEADERS, HTTP_STATUS, TOKEN_BASIC, TOKEN_BEARER} from '../lib/constants';
import {stringToMD5} from '../lib/crypto-utils';
import type {$ResponseExtend, $RequestExtend, $NextFunctionVer, IAuth} from '../../types';
import type {Config} from '@verdaccio/types';
@ -57,9 +57,9 @@ export function validatePackage(req: $RequestExtend, res: $ResponseExtend, next:
export function media(expect: string) {
return function(req: $RequestExtend, res: $ResponseExtend, next: $NextFunctionVer) {
if (req.headers['content-type'] !== expect) {
next( ErrorCode.getCode(415, 'wrong content-type, expect: ' + expect
+ ', got: '+req.headers['content-type']) );
if (req.headers[HEADER_TYPE.CONTENT_TYPE] !== expect) {
next( ErrorCode.getCode(HTTP_STATUS.UNSUPORTED_MEDIA, 'wrong content-type, expect: ' + expect
+ ', got: '+req.headers[HEADER_TYPE.CONTENT_TYPE]) );
} else {
next();
}
@ -89,7 +89,7 @@ export function anti_loop(config: Config) {
for (let i=0; i<arr.length; i++) {
const m = arr[i].match(/\s*(\S+)\s+(\S+)/);
if (m && m[2] === config.server_id) {
return next( ErrorCode.getCode(508, 'loop detected') );
return next( ErrorCode.getCode(HTTP_STATUS.LOOP_DETECTED, 'loop detected') );
}
}
}
@ -123,9 +123,9 @@ export function allow(auth: IAuth) {
}
export function final(body: any, req: $RequestExtend, res: $ResponseExtend, next: $NextFunctionVer) {
if (res.statusCode === 401 && !res.getHeader('WWW-Authenticate')) {
if (res.statusCode === HTTP_STATUS.UNAUTHORIZED && !res.getHeader('WWW-Authenticate')) {
// they say it's required for 401, so...
res.header('WWW-Authenticate', 'Basic, Bearer');
res.header('WWW-Authenticate', `${TOKEN_BASIC}, ${TOKEN_BEARER}`);
}
try {
@ -254,24 +254,24 @@ export function log(req: $RequestExtend, res: $ResponseExtend, next: $NextFuncti
// Middleware
export function errorReportingMiddleware(req: $RequestExtend, res: $ResponseExtend, next: $NextFunctionVer) {
res.report_error = res.report_error || function(err) {
if (err.status && err.status >= 400 && err.status < 600) {
if (err.status && err.status >= HTTP_STATUS.BAD_REQUEST && err.status < 600) {
if (_.isNil(res.headersSent) === false) {
res.status(err.status);
next({error: err.message || 'unknown error'});
next({error: err.message || API_ERROR.UNKNOWN_ERROR});
}
} else {
Logger.logger.error( {err: err}
, 'unexpected error: @{!err.message}\n@{err.stack}');
Logger.logger.error( {err: err}, 'unexpected error: @{!err.message}\n@{err.stack}');
if (!res.status || !res.send) {
Logger.logger.error('this is an error in express.js, please report this');
res.destroy();
} else if (!res.headersSent) {
res.status(500);
next({error: 'internal server error'});
res.status(HTTP_STATUS.INTERNAL_ERROR);
next({error: API_ERROR.INTERNAL_SERVER_ERROR});
} else {
// socket should be already closed
}
}
};
next();
}

18
src/lib/constants.js
View File

@ -9,6 +9,8 @@ export const HEADERS = {
export const HEADER_TYPE = {
CONTENT_ENCODING: 'content-encoding',
CONTENT_TYPE: 'content-type',
CONTENT_LENGTH: 'content-length',
ACCEPT_ENCODING: 'accept-encoding',
};
@ -22,25 +24,33 @@ export const DEFAULT_REGISTRY = 'https://registry.npmjs.org/';
export const HTTP_STATUS = {
INTERNAL_ERROR: 500,
SERVICE_UNAVAILABLE: 503,
OK: 200,
CREATED: 201,
MULTIPLE_CHOICES: 300,
BAD_REQUEST: 400,
UNAUTHORIZED: 401,
FORBIDDEN: 403,
NOT_FOUND: 404,
CONFLICT: 409,
UNSUPORTED_MEDIA: 415,
BAD_DATA: 422,
INTERNAL_ERROR: 500,
SERVICE_UNAVAILABLE: 503,
LOOP_DETECTED: 508,
};
export const PORT_SERVER_1 = '55551';
export const PORT_SERVER_2 = '55552';
export const PORT_SERVER_3 = '55551';
export const PORT_SERVER_3 = '55553';
export const PACKAGE_ERROR = {
export const API_ERROR = {
NO_PACKAGE: 'no such package available',
NOT_ALLOWED: 'not allowed to access package',
INTERNAL_SERVER_ERROR: 'internal server error',
UNKNOWN_ERROR: 'unknown error',
NOT_PACKAGE_UPLINK: 'package doesn\'t exist on uplink',
CONTENT_MISMATCH: 'content length mismatch',
NOT_FILE_UPLINK: 'file doesn\'t exist on uplink',
};
export const DEFAULT_NO_README = 'ERROR: No README data found!';

25
src/lib/storage.js
View File

@ -6,23 +6,24 @@ import async from 'async';
import Stream from 'stream';
import ProxyStorage from './up-storage';
import Search from './search';
import {API_ERROR, HTTP_STATUS} from './constants';
import LocalStorage from './local-storage';
import {ReadTarball} from '@verdaccio/streams';
import {checkPackageLocal, publishPackage, checkPackageRemote, cleanUpLinksRef,
mergeUplinkTimeIntoLocal, generatePackageTemplate} from './storage-utils';
mergeUplinkTimeIntoLocal, generatePackageTemplate} from './storage-utils';
import {setupUpLinks, updateVersionsHiddenUpLink} from './uplink-util';
import {mergeVersions} from './metadata-utils';
import {ErrorCode, normalizeDistTags, validate_metadata, isObject, DIST_TAGS} from './utils';
import type {IStorage, IProxy, IStorageHandler, ProxyList, StringValue} from '../../types';
import type {
Versions,
Package,
Config,
MergeTags,
Version,
DistFile,
Callback,
Logger,
Versions,
Package,
Config,
MergeTags,
Version,
DistFile,
Callback,
Logger,
} from '@verdaccio/types';
import type {IReadTarball, IUploadTarball} from '@verdaccio/streams';
@ -146,7 +147,7 @@ class Storage implements IStorageHandler {
let localStream: any = self.localStorage.getTarball(name, filename);
let isOpen = false;
localStream.on('error', (err) => {
if (isOpen || err.status !== 404) {
if (isOpen || err.status !== HTTP_STATUS.NOT_FOUND) {
return readStream.emit('error', err);
}
@ -273,7 +274,7 @@ class Storage implements IStorageHandler {
*/
getPackage(options: any) {
this.localStorage.getPackageMetadata(options.name, (err, data) => {
if (err && (!err.status || err.status >= 500)) {
if (err && (!err.status || err.status >= HTTP_STATUS.INTERNAL_ERROR)) {
// report internal errors right away
return options.callback(err);
}
@ -478,7 +479,7 @@ class Storage implements IStorageHandler {
}, (err: Error, upLinksErrors: any) => {
assert(!err && Array.isArray(upLinksErrors));
if (!exists) {
return callback( ErrorCode.getNotFound('no such package available')
return callback( ErrorCode.getNotFound(API_ERROR.NO_PACKAGE)
, null
, upLinksErrors );
}

24
src/lib/up-storage.js
View File

@ -8,7 +8,7 @@ import Stream from 'stream';
import URL from 'url';
import {parseInterval, isObject, ErrorCode, buildToken} from './utils';
import {ReadTarball} from '@verdaccio/streams';
import {ERROR_CODE, TOKEN_BASIC, TOKEN_BEARER, HEADERS} from './constants';
import {ERROR_CODE, TOKEN_BASIC, TOKEN_BEARER, HEADERS, HTTP_STATUS, API_ERROR, HEADER_TYPE} from './constants';
import type {
Config,
UpLinkConf,
@ -407,10 +407,10 @@ class ProxyStorage implements IProxy {
if (err) {
return callback(err);
}
if (res.statusCode === 404) {
return callback( ErrorCode.getNotFound('package doesn\'t exist on uplink'));
if (res.statusCode === HTTP_STATUS.NOT_FOUND) {
return callback( ErrorCode.getNotFound(API_ERROR.NOT_PACKAGE_UPLINK));
}
if (!(res.statusCode >= 200 && res.statusCode < 300)) {
if (!(res.statusCode >= HTTP_STATUS.OK && res.statusCode < HTTP_STATUS.MULTIPLE_CHOICES)) {
const error = ErrorCode.getInternalError(`bad status code: ${res.statusCode}`);
// $FlowFixMe
error.remoteStatus = res.statusCode;
@ -440,15 +440,15 @@ class ProxyStorage implements IProxy {
});
readStream.on('response', function(res: any) {
if (res.statusCode === 404) {
return stream.emit('error', ErrorCode.getNotFound('file doesn\'t exist on uplink'));
if (res.statusCode === HTTP_STATUS.NOT_FOUND) {
return stream.emit('error', ErrorCode.getNotFound(API_ERROR.NOT_FILE_UPLINK));
}
if (!(res.statusCode >= 200 && res.statusCode < 300)) {
if (!(res.statusCode >= HTTP_STATUS.OK && res.statusCode < HTTP_STATUS.MULTIPLE_CHOICES)) {
return stream.emit('error', ErrorCode.getInternalError(`bad uplink status code: ${res.statusCode}`));
}
if (res.headers['content-length']) {
expected_length = res.headers['content-length'];
stream.emit('content-length', res.headers['content-length']);
if (res.headers[HEADER_TYPE.CONTENT_LENGTH]) {
expected_length = res.headers[HEADER_TYPE.CONTENT_LENGTH];
stream.emit(HEADER_TYPE.CONTENT_LENGTH, res.headers[HEADER_TYPE.CONTENT_LENGTH]);
}
readStream.pipe(stream);
@ -465,7 +465,7 @@ class ProxyStorage implements IProxy {
current_length += data.length;
}
if (expected_length && current_length != expected_length) {
stream.emit('error', ErrorCode.getInternalError('content length mismatch'));
stream.emit('error', ErrorCode.getInternalError(API_ERROR.CONTENT_MISMATCH));
}
});
return stream;
@ -500,7 +500,7 @@ class ProxyStorage implements IProxy {
// See https://github.com/request/request#requestoptions-callback
// Request library will not decode gzip stream.
let jsonStream;
if (res.headers['content-encoding'] === 'gzip') {
if (res.headers[HEADER_TYPE.CONTENT_ENCODING] === HEADERS.GZIP) {
jsonStream = res.pipe(zlib.createUnzip());
} else {
jsonStream = res;

4
src/lib/utils.js
View File

@ -10,7 +10,7 @@ import _ from 'lodash';
import asciidoctor from 'asciidoctor.js';
import createError from 'http-errors';
import marked from 'marked';
import {HTTP_STATUS} from './constants';
import {HTTP_STATUS, API_ERROR} from './constants';
import type {Package} from '@verdaccio/types';
import type {$Request} from 'express';
@ -359,7 +359,7 @@ const ErrorCode = {
return createError(HTTP_STATUS.SERVICE_UNAVAILABLE, message);
},
getNotFound: (customMessage?: string) => {
return createError(HTTP_STATUS.NOT_FOUND, customMessage || 'no such package available');
return createError(HTTP_STATUS.NOT_FOUND, customMessage || API_ERROR.NO_PACKAGE);
},
getCode: (statusCode: number, customMessage: string) => {
return createError(statusCode, customMessage);

13
test/functional/fixtures/package.js
View File

@ -1,13 +1,16 @@
import {PORT_SERVER_1} from "../../../src/lib/constants";
module.exports = function(name, version = '0.0.0', port = PORT_SERVER_1, domain= `http://localhost:${port}`) {
export const FILE_NAME = 'blahblah';
module.exports = function(name, version = '0.0.0', port = PORT_SERVER_1, domain= `http://localhost:${port}`,
fileName = 'blahblah', readme = 'this is a readme') {
return {
name: name,
version: version,
readme: "this is a readme",
name,
version,
readme,
dist: {
shasum: 'fake',
tarball: `${domain}/${encodeURIComponent(name)}/-/blahblah`,
tarball: `${domain}/${encodeURIComponent(name)}/-/${fileName}`,
},
};
};

7
test/functional/lib/environment.js
View File

@ -8,6 +8,7 @@ import VerdaccioProcess from "../../lib/server_process";
import Server from "../../lib/server";
import ExpressServer from "./simple_server";
import type {IServerBridge} from '../../types';
import {PORT_SERVER_1, PORT_SERVER_2, PORT_SERVER_3} from "../../../src/lib/constants";
const EXPRESS_PORT = 55550;
@ -34,17 +35,17 @@ class FunctionalEnvironment extends NodeEnvironment {
const pathStore = path.join(__dirname, '../store');
const listServers = [
{
port: 55551,
port: PORT_SERVER_1,
config: '/config-1.yaml',
storage: '/test-storage'
},
{
port: 55552,
port: PORT_SERVER_2,
config: '/config-2.yaml',
storage: '/test-storage2'
},
{
port: 55553,
port: PORT_SERVER_3,
config: '/config-3.yaml',
storage: '/test-storage3'
}

14
test/functional/plugins/auth.js
View File

@ -1,4 +1,4 @@
import {HTTP_STATUS, PACKAGE_ERROR} from "../../../src/lib/constants";
import {HTTP_STATUS, API_ERROR} from "../../../src/lib/constants";
export default function(server2) {
// credentials
@ -70,19 +70,19 @@ export default function(server2) {
test(`should fails (404) on access ${UNEXISTING_PKG_NAME}`, () => {
return server2.getPackage(UNEXISTING_PKG_NAME)
.status(HTTP_STATUS.NOT_FOUND)
.body_error(PACKAGE_ERROR.NO_PACKAGE);
.body_error(API_ERROR.NO_PACKAGE);
});
test(`should fails (403) access ${ONLY_ACCESS_BY_USER_2}`, () => {
return server2.getPackage(ONLY_ACCESS_BY_USER_2)
.status(HTTP_STATUS.FORBIDDEN)
.body_error(PACKAGE_ERROR.NOT_ALLOWED);
.body_error(API_ERROR.NOT_ALLOWED);
});
test(`should fails (404) access ${AUTH_PKG_ACCESS_NAME}`, () => {
return server2.getPackage(AUTH_PKG_ACCESS_NAME)
.status(HTTP_STATUS.NOT_FOUND)
.body_error(PACKAGE_ERROR.NO_PACKAGE);
.body_error(API_ERROR.NO_PACKAGE);
});
});
@ -96,19 +96,19 @@ export default function(server2) {
test(`should fails (403) on access ${UNEXISTING_PKG_NAME}`, () => {
return server2.getPackage(UNEXISTING_PKG_NAME)
.status(HTTP_STATUS.FORBIDDEN)
.body_error(PACKAGE_ERROR.NOT_ALLOWED);
.body_error(API_ERROR.NOT_ALLOWED);
});
test(`should fails (403) on access ${DENY_PKG_NAME}`, () => {
return server2.getPackage(DENY_PKG_NAME)
.status(HTTP_STATUS.FORBIDDEN)
.body_error(PACKAGE_ERROR.NOT_ALLOWED);
.body_error(API_ERROR.NOT_ALLOWED);
});
test(`should fails (404) access ${AUTH_PKG_ACCESS_NAME}`, () => {
return server2.getPackage(AUTH_PKG_ACCESS_NAME)
.status(HTTP_STATUS.NOT_FOUND)
.body_error(PACKAGE_ERROR.NO_PACKAGE);
.body_error(API_ERROR.NO_PACKAGE);
});
});

92
test/functional/sanity/mirror.js
View File

@ -1,58 +1,76 @@
import assert from 'assert';
import {readFile} from '../lib/test.utils';
import {HTTP_STATUS} from "../../../src/lib/constants";
import generatePkg from '../fixtures/package';
const getBinary = () => readFile('../fixtures/binary');
export default function (server, server2) {
test('testing anti-loop', () => {
return server2.getPackage('testloop')
.status(404)
.body_error(/no such package/);
})
describe('anti-loop testing', () => {
test('testing anti-loop', () => {
return server2.getPackage('testloop').status(HTTP_STATUS.NOT_FOUND)
.body_error(/no such package/);
});
});
;['fwd'].forEach(function (pkg) {
let prefix = pkg + ': ';
pkg = 'test' + pkg;
describe('mirror', () => {
const pkgFileName = 'blahblah';
const pkgList = ['pkg1', 'pkg2', 'pkg3'];
describe(pkg, () => {
beforeAll(function () {
return server.putPackage(pkg, require('../fixtures/package')(pkg))
.status(201)
.body_ok(/created new package/);
});
pkgList.forEach(function (pkg) {
let prefix = pkg;
pkg = 'test-mirror-' + pkg;
test(prefix + 'creating new package', () => {});
describe(pkg, () => {
describe(`testing mirror for ${pkg}`, () => {
beforeAll(function () {
return server.putVersion(pkg, '0.1.1', require('../fixtures/package')(pkg))
.status(201)
.body_ok(/published/);
return server2.putPackage(pkg, generatePkg(pkg))
.status(HTTP_STATUS.CREATED)
.body_ok(/created new package/);
});
test(prefix + 'uploading new package version', () => {});
test(prefix + 'creating new package', () => {});
test(prefix + 'uploading incomplete tarball', () => {
return server.putTarballIncomplete(pkg, pkg + '.bad', getBinary(), 3000);
});
describe('tarball', () => {
describe(pkg, () => {
beforeAll(function () {
return server.putTarball(pkg, pkg + '.file', getBinary())
.status(201)
.body_ok(/.*/);
return server2.putVersion(pkg, '0.1.1', generatePkg(pkg))
.status(HTTP_STATUS.CREATED)
.body_ok(/published/);
});
test(prefix + 'uploading new tarball', () => {
test(`should ${prefix} uploading new package version`, () => {});
test(`${prefix} uploading incomplete tarball`, () => {
return server2.putTarballIncomplete(pkg, pkg + '.bad', getBinary(), 3000);
});
test(prefix + 'downloading tarball from server1', () => {
return server.getTarball(pkg, pkg + '.file')
.status(200)
.then(function (body) {
assert.deepEqual(body, getBinary());
});
describe('should put a tarball', () => {
beforeAll(function () {
return server2.putTarball(pkg, pkgFileName, getBinary())
.status(HTTP_STATUS.CREATED)
.body_ok(/.*/);
});
test(`should ${prefix} uploading new tarball`, () => {});
test(`should ${prefix} downloading tarball from server2`, () => {
return server2.getTarball(pkg, pkgFileName)
.status(HTTP_STATUS.OK)
.then(function (body) {
expect(body).toEqual(getBinary());
});
});
test('testing mirror server1', () => {
return server.getPackage(pkg).status(HTTP_STATUS.OK);
});
test(`should ${prefix} downloading tarball from server1`, () => {
return server.getTarball(pkg, pkgFileName)
.status(HTTP_STATUS.OK)
.then(function (body) {
expect(body).toEqual(getBinary());
});
});
});
});
});

23
test/functional/scenarios/gh29.js
View File

@ -11,19 +11,18 @@ function readfile(filePath) {
const binary = '../fixtures/binary';
const pkgName = 'testpkg-gh29';
const pkgContent = 'blahblah';
const pkgFileName = 'blahblah';
export default function (server, server2) {
test('downloading non-existent tarball #1 / srv2', () => {
return server2.getTarball(pkgName, pkgContent)
.status(HTTP_STATUS.NOT_FOUND)
.body_error(/no such package/);
describe('pkg-gh29 #1', () => {
test('downloading non-existent tarball #1 / srv2', () => {
return server2.getTarball(pkgName, pkgFileName)
.status(HTTP_STATUS.NOT_FOUND)
.body_error(/no such package/);
});
});
describe('pkg-gh29', () => {
describe('pkg-gh29 #2', () => {
beforeAll(function() {
return server.putPackage(pkgName, require('../fixtures/package')(pkgName))
.status(HTTP_STATUS.CREATED)
@ -33,14 +32,14 @@ export default function (server, server2) {
test('creating new package / srv1', () => {});
test('downloading non-existent tarball #2 / srv2', () => {
return server2.getTarball(pkgName, pkgContent)
return server2.getTarball(pkgName, pkgFileName)
.status(HTTP_STATUS.NOT_FOUND)
.body_error(/no such file available/);
});
describe('tarball', () => {
beforeAll(function() {
return server.putTarball(pkgName, pkgContent, readfile(binary))
return server.putTarball(pkgName, pkgFileName, readfile(binary))
.status(HTTP_STATUS.CREATED)
.body_ok(/.*/);
});
@ -59,7 +58,7 @@ export default function (server, server2) {
test('uploading new package version / srv1', () => {});
test('downloading newly created tarball / srv2', () => {
return server2.getTarball(pkgName, pkgContent)
return server2.getTarball(pkgName, pkgFileName)
.status(HTTP_STATUS.OK)
.then(function(body) {
expect(body).toEqual(readfile(binary));

5
test/functional/store/config-1.yaml
View File

@ -34,12 +34,13 @@ packages:
publish: $all
proxy: server2
'testfwd*':
## mirror.js
'test-mirror-*':
access: $all
publish: $all
proxy: server2
proxy_publish: server2
## mirror.js
'testloop':
access: $all
publish: $all

6
test/functional/store/config-2.yaml
View File

@ -39,7 +39,11 @@ packages:
publish: $all
proxy: server1
'testfwd':
'test-fwd':
access: $all
publish: $all
'test-mirror-fwdw*':
access: $all
publish: $all

3
test/lib/request.js
View File

@ -1,8 +1,8 @@
// @flow
import _ from 'lodash';
import assert from 'assert';
import request from 'request';
import _ from 'lodash';
import type {IRequestPromise} from '../types';
const requestData = Symbol('smart_request_data');
@ -54,7 +54,6 @@ export class PromiseAssert extends Promise<any> implements IRequestPromise{
const selfData = this[requestData];
return injectResponse(this, this.then(function(body) {
// console.log("======>smartRequest body_error://", body);
try {
if (_.isRegExp(expected)) {
assert(body.error.match(expected), body.error + ' doesn\'t match ' + expected);

4
test/unit/api/api.spec.js
View File

@ -8,7 +8,7 @@ import publishMetadata from '../partials/publish-api';
import forbiddenPlace from '../partials/forbidden-place';
import Config from '../../../src/lib/config';
import endPointAPI from '../../../src/api/index';
import {HEADERS} from '../../../src/lib/constants';
import {HEADERS, API_ERROR} from '../../../src/lib/constants';
require('../../../src/lib/logger').setup([]);
const credentials = { name: 'Jota', password: 'secretPass' };
@ -595,7 +595,7 @@ describe('endpoint unit test', () => {
.expect(200)
.expect('Content-Type', 'text/plain; charset=utf-8')
.end(function(err, res) {
expect(res.body.error).toMatch('no such package available');
expect(res.body.error).toMatch(API_ERROR.NO_PACKAGE);
done();
});
});