From 74008305051e1ccf3adc706200f358497b7dba39 Mon Sep 17 00:00:00 2001 From: Juan Picado Date: Sun, 5 May 2024 12:35:17 +0200 Subject: [PATCH] revert #4600 (#4623) --- .changeset/poor-seals-turn.md | 6 ++ packages/core/tarball/package.json | 4 +- .../core/tarball/src/getTarballDetails.ts | 34 ------- packages/core/tarball/src/index.ts | 1 - .../core/tarball/tests/assets/tarball.tar | Bin 3584 -> 0 bytes .../core/tarball/tests/assets/tarball.tgz | Bin 682 -> 0 bytes .../tarball/tests/getTarballDetails.spec.ts | 37 -------- packages/store/src/storage.ts | 35 +------- packages/store/test/storage.spec.ts | 2 - pnpm-lock.yaml | 84 +----------------- 10 files changed, 14 insertions(+), 189 deletions(-) create mode 100644 .changeset/poor-seals-turn.md delete mode 100644 packages/core/tarball/src/getTarballDetails.ts delete mode 100644 packages/core/tarball/tests/assets/tarball.tar delete mode 100644 packages/core/tarball/tests/assets/tarball.tgz delete mode 100644 packages/core/tarball/tests/getTarballDetails.spec.ts diff --git a/.changeset/poor-seals-turn.md b/.changeset/poor-seals-turn.md new file mode 100644 index 000000000..421a9e668 --- /dev/null +++ b/.changeset/poor-seals-turn.md @@ -0,0 +1,6 @@ +--- +'@verdaccio/tarball': patch +'@verdaccio/store': patch +--- + +revert #4600 diff --git a/packages/core/tarball/package.json b/packages/core/tarball/package.json index 284c923c7..0a89df7b9 100644 --- a/packages/core/tarball/package.json +++ b/packages/core/tarball/package.json @@ -37,9 +37,7 @@ "@verdaccio/url": "workspace:12.0.0-next-7.14", "@verdaccio/utils": "workspace:7.0.0-next-7.14", "debug": "4.3.4", - "gunzip-maybe": "^1.4.2", - "lodash": "4.17.21", - "tar-stream": "^3.1.7" + "lodash": "4.17.21" }, "devDependencies": { "@verdaccio/types": "workspace:12.0.0-next.2", diff --git a/packages/core/tarball/src/getTarballDetails.ts b/packages/core/tarball/src/getTarballDetails.ts deleted file mode 100644 index df1841a23..000000000 --- a/packages/core/tarball/src/getTarballDetails.ts +++ /dev/null @@ -1,34 +0,0 @@ -import gunzipMaybe from 'gunzip-maybe'; -import { Readable } from 'stream'; -import * as tarStream from 'tar-stream'; - -export type TarballDetails = { - fileCount: number; - unpackedSize: number; // in bytes -}; - -export async function getTarballDetails(readable: Readable): Promise { - let fileCount = 0; - let unpackedSize = 0; - - const unpack = tarStream.extract(); - - return new Promise((resolve, reject) => { - readable - .pipe(gunzipMaybe()) - .pipe(unpack) - .on('entry', (header, stream, next) => { - fileCount++; - unpackedSize += Number(header.size); - stream.resume(); // important to ensure that "entry" events keep firing - next(); - }) - .on('finish', () => { - resolve({ - fileCount, - unpackedSize, - }); - }) - .on('error', reject); - }); -} diff --git a/packages/core/tarball/src/index.ts b/packages/core/tarball/src/index.ts index 27d742065..29e7e81fe 100644 --- a/packages/core/tarball/src/index.ts +++ b/packages/core/tarball/src/index.ts @@ -5,6 +5,5 @@ export { convertDistVersionToLocalTarballsUrl, } from './convertDistRemoteToLocalTarballUrls'; export { extractTarballFromUrl, getLocalRegistryTarballUri } from './getLocalRegistryTarballUri'; -export { TarballDetails, getTarballDetails } from './getTarballDetails'; export { RequestOptions }; diff --git a/packages/core/tarball/tests/assets/tarball.tar b/packages/core/tarball/tests/assets/tarball.tar deleted file mode 100644 index e15803f3e84681b31d1c0a46eed2d9100efbea26..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3584 zcmeHJQBT`25bkp&{==ycG#yox6lf(7dtuYGH#GK$CK>128WKCQ9R@}H@4Iu7wv>$_ zB&12JRVnU$-+jJ2U#?X;!~3|nah>;D39ZRw(%~nq?PoeU8gOk^-A+41w=Y<7W8~(4IDoSHLd;sBKN*`+q=uch$bg#P!agAz>cH7`0SJw^7TZZ?# z!aZmS(@IOPjD{Kd@4{Y-JpY7VsN6+UmE(o*^8q~Wb%B;N+Ig5QR0(ImLb0?cJbtM( z*`fi-u4#Dht&rYW;!;R)uAE;{YGRHa_>fDrKw>4Q@YH^8D(wTXwo|yKC@ngN*~*X8 z;gc=;%4*Q~6E3Ju%3%}{)#P_LB=82MAHdniHXnAGx+aNlD6&gW`4sEKYgMMjMo0Zj zc+nq%oLBmGOD0IQsT`cuoH$24^Yf-GOj!;#BuiCd#MRt1@;PX&v2;)q@wUsVvak0G zs@mHJQbm6LA^cGBgM*ETIqGLMrv!sa?C9oBp}Dqa3D&1(bY`?&cIGxW6E)q%rgpVm zFC$g&@#gX2P;SP@;obrycIKyZ=BIDGvAK#(0^h6Cr5|l)%-tB z4?9VkjE>XY{C|7JU+?)yldY#kfxQ#x`Rm@0zf(^-8rp11<0MXMYlco*Rq{9w?-i@Rp1w~0e*)7 diff --git a/packages/core/tarball/tests/assets/tarball.tgz b/packages/core/tarball/tests/assets/tarball.tgz deleted file mode 100644 index f25e82149ebf810e0664b222092b512d3cdbf236..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 682 zcmV;b0#*GViwFp(4$Wl*0B2=%bYXI0VQg$^Zf0*TbYXG;?N;A!+At8#GgAKti-!_c z6eY9;si=GDrfF~0y2omg>m1-Vu_OD^wN(7?J3Ao(vR^Ol!^S5>?EAj+-Psp|R4F_~ z`MnKZ4>4gnozl}||CvnB#zAs+J~>cBl z&FN!w&IWTkm>u=DDcYl&qtQOO%H&z^~lg@htlQ$Yo~3 zxy<02<8sJej(UtBw6e$+D_O8hzh)#1aD-2#B3n3|ZOh;Z4Hu{lX(^>BlNK9E^_>AS z7vx5+fK>5p&wiy#g`Q~bl&;PT1J*+7{A#)q7{Jmfrr?Ygh)_G>52VC`C{`w9#!6o { - return path.resolve(__dirname, `assets/${filename}`); -}; - -const getFileBuffer = async (filename: string): Promise => { - return fs.promises.readFile(getFilePath(filename)); -}; - -describe('getTarballDetails', () => { - test('should return stats of tarball (gzipped)', async () => { - const buffer = await getFileBuffer('tarball.tgz'); - const readable = Readable.from(buffer); - const details = await getTarballDetails(readable); - expect(details.fileCount).toBe(2); - expect(details.unpackedSize).toBe(1280); - }); - - test('should return stats of tarball (uncompressed)', async () => { - const buffer = await getFileBuffer('tarball.tar'); - const readable = Readable.from(buffer); - const details = await getTarballDetails(readable); - expect(details.fileCount).toBe(2); - expect(details.unpackedSize).toBe(1280); - }); - - test('should throw an error if the buffer is corrupt', async () => { - const corruptBuffer = Buffer.from('this is not a tarball'); - const readable = Readable.from(corruptBuffer); - await expect(getTarballDetails(readable)).rejects.toThrow(); - }); -}); diff --git a/packages/store/src/storage.ts b/packages/store/src/storage.ts index 48625532a..9f58ea890 100644 --- a/packages/store/src/storage.ts +++ b/packages/store/src/storage.ts @@ -34,11 +34,9 @@ import { } from '@verdaccio/proxy'; import Search from '@verdaccio/search'; import { - TarballDetails, convertDistRemoteToLocalTarballUrls, convertDistVersionToLocalTarballsUrl, extractTarballFromUrl, - getTarballDetails, } from '@verdaccio/tarball'; import { AbbreviatedManifest, @@ -1046,8 +1044,6 @@ class Storage { // at this point document is either created or existed before const [firstAttachmentKey] = Object.keys(_attachments); const buffer = this.getBufferManifest(body._attachments[firstAttachmentKey].data as string); - const readable = Readable.from(buffer); - const tarballStats = await this.getTarballStats(versions[versionToPublish], readable); try { // we check if package exist already locally @@ -1086,7 +1082,7 @@ class Storage { _.isNil(manifest.readme) === false ? String(manifest.readme) : ''; } // addVersion will move the readme from the the published version to the root level - await this.addVersion(name, versionToPublish, versions[versionToPublish], null, tarballStats); + await this.addVersion(name, versionToPublish, versions[versionToPublish], null); } catch (err: any) { logger.error({ err: err.message }, 'updated version has failed: @{err}'); debug('error on create a version for %o with error %o', name, err.message); @@ -1114,6 +1110,7 @@ class Storage { // 3. upload the tarball to the storage try { + const readable = Readable.from(buffer); await this.uploadTarball(name, basename(firstAttachmentKey), readable, { signal: options.signal, }); @@ -1286,8 +1283,7 @@ class Storage { name: string, version: string, metadata: Version, - tag: StringValue, - tarballStats: TarballDetails + tag: StringValue ): Promise { debug(`add version %s package for %s`, version, name); await this.updatePackage(name, async (data: Manifest): Promise => { @@ -1299,12 +1295,6 @@ class Storage { metadata.contributors = normalizeContributors(metadata.contributors as Author[]); debug('%s` contributors normalized', name); - // Update tarball stats - if (metadata.dist) { - metadata.dist.fileCount = tarballStats.fileCount; - metadata.dist.unpackedSize = tarballStats.unpackedSize; - } - // if uploaded tarball has a different shasum, it's very likely that we // have some kind of error if (validatioUtils.isObject(metadata.dist) && _.isString(metadata.dist.tarball)) { @@ -1915,25 +1905,6 @@ class Storage { return cacheManifest; } } - - private async getTarballStats(version: Version, readable: Readable): Promise { - if ( - version.dist == undefined || - version.dist?.fileCount == undefined || - version.dist?.unpackedSize == undefined - ) { - debug('tarball stats not found, calculating'); - try { - return await getTarballDetails(readable); - } catch (err: any) { - logger.error({ err: err.message }, 'getting tarball details has failed: @{err}'); - throw err; - } - } else { - debug('tarball stats found'); - return { fileCount: version.dist.fileCount, unpackedSize: version.dist.unpackedSize }; - } - } } export { Storage }; diff --git a/packages/store/test/storage.spec.ts b/packages/store/test/storage.spec.ts index f21a25299..1e767ad61 100644 --- a/packages/store/test/storage.spec.ts +++ b/packages/store/test/storage.spec.ts @@ -262,12 +262,10 @@ describe('storage', () => { expect(manifestVersion._id).toEqual(`${pkgName}@1.0.1`); expect(manifestVersion.description).toEqual('package generated'); expect(manifestVersion.dist).toEqual({ - fileCount: 4, integrity: 'sha512-6gHiERpiDgtb3hjqpQH5/i7zRmvYi9pmCjQf2ZMy3QEa9wVk9RgdZaPWUt7ZOnWUPFjcr9cmE6dUBf+XoPoH4g==', shasum: '2c03764f651a9f016ca0b7620421457b619151b9', tarball: 'http://localhost:5555/upstream/-/upstream-1.0.1.tgz', - unpackedSize: 543, }); expect(manifestVersion.contributors).toEqual([]); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6f8e6f39..b530ae3f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -731,15 +731,9 @@ importers: debug: specifier: 4.3.4 version: 4.3.4(supports-color@5.5.0) - gunzip-maybe: - specifier: ^1.4.2 - version: 1.4.2 lodash: specifier: 4.17.21 version: 4.17.21 - tar-stream: - specifier: ^3.1.7 - version: 3.1.7 devDependencies: '@verdaccio/types': specifier: workspace:12.0.0-next.2 @@ -13613,10 +13607,6 @@ packages: dequal: 2.0.3 dev: false - /b4a@1.6.6: - resolution: {integrity: sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==} - dev: false - /babel-core@7.0.0-bridge.0(@babel/core@7.23.9): resolution: {integrity: sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==} peerDependencies: @@ -13932,12 +13922,6 @@ packages: resolution: {integrity: sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==} dev: true - /bare-events@2.2.2: - resolution: {integrity: sha512-h7z00dWdG0PYOQEvChhOSWvOfkIKsdZGkWr083FgN/HyoQuebSew/cgirYqh9SCuy/hRvxc5Vy6Fw8xAmYHLkQ==} - requiresBuild: true - dev: false - optional: true - /base16@1.0.0: resolution: {integrity: sha512-pNdYkNPiJUnEhnfXV56+sQy8+AaPcG3POZAUnwr4EeqCUZFz4u2PePbo3e5Gj4ziYPCWGUZT9RHisvJKnwFuBQ==} @@ -14209,12 +14193,6 @@ packages: safe-buffer: 5.2.1 dev: true - /browserify-zlib@0.1.4: - resolution: {integrity: sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==} - dependencies: - pako: 0.2.9 - dev: false - /browserify-zlib@0.2.0: resolution: {integrity: sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==} dependencies: @@ -16821,6 +16799,7 @@ packages: inherits: 2.0.4 readable-stream: 2.3.8 stream-shift: 1.0.1 + dev: true /duplexify@4.1.2: resolution: {integrity: sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==} @@ -18099,10 +18078,6 @@ packages: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} dev: false - /fast-fifo@1.3.2: - resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - dev: false - /fast-glob@2.2.7: resolution: {integrity: sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==} engines: {node: '>=4.0.0'} @@ -19163,18 +19138,6 @@ packages: section-matter: 1.0.0 strip-bom-string: 1.0.0 - /gunzip-maybe@1.4.2: - resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} - hasBin: true - dependencies: - browserify-zlib: 0.1.4 - is-deflate: 1.0.0 - is-gzip: 1.0.0 - peek-stream: 1.1.3 - pumpify: 1.5.1 - through2: 2.0.5 - dev: false - /gzip-size@5.1.1: resolution: {integrity: sha512-FNHi6mmoHvs1mxZAds4PpdCS6QG8B4C1krxJsMutgxl5t3+GlRTzzI3NEkifXx2pVsOvJdOGSmIgDhQ55FwdPA==} engines: {node: '>=6'} @@ -20135,10 +20098,6 @@ packages: /is-decimal@2.0.1: resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} - /is-deflate@1.0.0: - resolution: {integrity: sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==} - dev: false - /is-descriptor@0.1.6: resolution: {integrity: sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==} engines: {node: '>=0.10.0'} @@ -20228,11 +20187,6 @@ packages: dependencies: is-extglob: 2.1.1 - /is-gzip@1.0.0: - resolution: {integrity: sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==} - engines: {node: '>=0.10.0'} - dev: false - /is-hexadecimal@1.0.4: resolution: {integrity: sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==} @@ -23659,10 +23613,6 @@ packages: registry-url: 5.1.0 semver: 6.3.1 - /pako@0.2.9: - resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} - dev: false - /pako@1.0.11: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} dev: true @@ -23886,14 +23836,6 @@ packages: sha.js: 2.4.11 dev: true - /peek-stream@1.1.3: - resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==} - dependencies: - buffer-from: 1.1.2 - duplexify: 3.7.1 - through2: 2.0.5 - dev: false - /pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} dev: true @@ -25134,6 +25076,7 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 + dev: true /pump@3.0.0: resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} @@ -25147,6 +25090,7 @@ packages: duplexify: 3.7.1 inherits: 2.0.4 pump: 2.0.1 + dev: true /punycode@1.3.2: resolution: {integrity: sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==} @@ -25211,10 +25155,6 @@ packages: /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - /queue-tick@1.0.1: - resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} - dev: false - /queue@6.0.2: resolution: {integrity: sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==} dependencies: @@ -27389,15 +27329,6 @@ packages: mixme: 0.5.9 dev: true - /streamx@2.16.1: - resolution: {integrity: sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==} - dependencies: - fast-fifo: 1.3.2 - queue-tick: 1.0.1 - optionalDependencies: - bare-events: 2.2.2 - dev: false - /strict-event-emitter@0.2.8: resolution: {integrity: sha512-KDf/ujU8Zud3YaLtMCcTI4xkZlZVIYxTLr+XIULexP+77EEVWixeXroLUXQXiVtH4XH2W7jr/3PT1v3zBuvc3A==} dependencies: @@ -27948,14 +27879,6 @@ packages: resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} engines: {node: '>=6'} - /tar-stream@3.1.7: - resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} - dependencies: - b4a: 1.6.6 - fast-fifo: 1.3.2 - streamx: 2.16.1 - dev: false - /tar@4.4.19: resolution: {integrity: sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==} engines: {node: '>=4.5'} @@ -28144,6 +28067,7 @@ packages: dependencies: readable-stream: 2.3.8 xtend: 4.0.2 + dev: true /through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}