From 562800ece9426c620f52575ca9b3589a23dae45e Mon Sep 17 00:00:00 2001 From: Egor Kushnarev Date: Mon, 1 Apr 2024 10:42:18 +0300 Subject: [PATCH 1/4] by-hash initial commit --- src/deb/deb-builder.mts | 47 +++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/src/deb/deb-builder.mts b/src/deb/deb-builder.mts index 1464a4b..34c75cf 100644 --- a/src/deb/deb-builder.mts +++ b/src/deb/deb-builder.mts @@ -1,3 +1,4 @@ +import { createHash, randomBytes } from 'crypto'; import { createGzip } from 'zlib'; import * as fs from 'fs'; @@ -23,6 +24,7 @@ const ReleaseFileTemplate = Label: Ubuntu/Debian Architecture: $ARCH Component: $COMPONENT +Acquire-By-Hash: yes Codename: $DISTRIBUTION\n`; function iterateComponents(repo: DebRepo, callback: (distribution: string, component: string, deb: Artifact[]) => void): void { @@ -47,6 +49,24 @@ function iterateDebs(repo: DebRepo, callback: (distribution: string, component: }); } +function sha512(filePath: string): Promise { + return new Promise((resolve, reject) => { + const hash = createHash('sha512'); + fs.createReadStream(filePath) + .once('end', () => resolve(hash.digest('hex'))) + .once('error', () => reject) + .pipe(hash); + }); +} + +async function handleByHash(filePath: string): Promise { + const byHashDir = path.join(path.dirname(filePath), 'by-hash', 'SHA512'); + const byHashFileName = path.resolve(path.join(byHashDir, await sha512(filePath))); + + createDir(byHashDir); + return fs.promises.copyFile(filePath, byHashFileName); +} + export interface Config { out: string, gpgKeyName: string; @@ -102,15 +122,13 @@ export class DebBuilder implements Deployer { .replace('$COMPONENT', component); const distributionPath = path.join(this.distsPath, distribution); - const releaseFilePath = path.join(distributionPath, 'Release'); - const releaseGpgFilePath = path.join(distributionPath, 'Release.gpg'); + const tempReleaseFile = path.join(this.tempPath, `Release-${randomBytes(4).toString('hex')}`); const inReleaseFilePath = path.join(distributionPath, 'InRelease'); - await fs.promises.writeFile(releaseFilePath, releaseContent); + await fs.promises.writeFile(tempReleaseFile, releaseContent); - await execToolToFile('apt-ftparchive', ['release', distributionPath], releaseFilePath, true); - await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '-abs', '-o', releaseGpgFilePath, releaseFilePath]); - await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '--clearsign', '-o', inReleaseFilePath, releaseFilePath]); + await execToolToFile('apt-ftparchive', ['release', distributionPath], tempReleaseFile, true); + await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '--clearsign', '-o', inReleaseFilePath, tempReleaseFile]); } private async dpkgScanpackages(): Promise { @@ -201,7 +219,7 @@ export class DebBuilder implements Deployer { } private async makeRelease(): Promise<{}> { - const compressFile = (filePath: string): Promise => new Promise(resolve => { + const compressFile = (filePath: string): Promise => new Promise(resolve => { const inp = fs.createReadStream(filePath); const out = fs.createWriteStream(`${filePath}.gz`); @@ -209,11 +227,12 @@ export class DebBuilder implements Deployer { inp.pipe(gzip).pipe(out) .on('finish', () => { - resolve(); + resolve(`${filePath}.gz`); }); }); - const compressPromises: Promise[] = []; + const compressPromises: Promise[] = []; + const byHashPromises: Promise[] = []; iterateComponents(this.config.repo, (distribution, component) => { const componentRoot = path.join(this.distsPath, distribution, component); @@ -234,12 +253,18 @@ export class DebBuilder implements Deployer { } fs.writeFileSync(targetPackagesFile, packagesContent); - + byHashPromises.push(handleByHash(targetPackagesFile)); compressPromises.push(compressFile(targetPackagesFile)); }); }); - await Promise.all(compressPromises); + const compressedPackages = await Promise.all(compressPromises); + + compressedPackages.forEach(packagePath => { + byHashPromises.push(handleByHash(packagePath)); + }); + + await Promise.all(byHashPromises); const releasesPromises: Promise[] = []; From b01b1e08da80290dbbb2f0e7d9586a194e379ba7 Mon Sep 17 00:00:00 2001 From: Stanislav Makarov Date: Tue, 2 Apr 2024 19:44:32 +0300 Subject: [PATCH 2/4] refactor(debian): store package indices only by-hash --- src/deb/deb-builder.mts | 171 +++++++++++++++++++++------------------- 1 file changed, 92 insertions(+), 79 deletions(-) diff --git a/src/deb/deb-builder.mts b/src/deb/deb-builder.mts index 34c75cf..35d8cee 100644 --- a/src/deb/deb-builder.mts +++ b/src/deb/deb-builder.mts @@ -1,9 +1,13 @@ -import { createHash, randomBytes } from 'crypto'; -import { createGzip } from 'zlib'; +import * as path from 'node:path'; +import { appendFile, writeFile } from 'node:fs/promises'; +import { createHash, randomBytes } from 'node:crypto'; +import { createWriteStream, readdirSync, readFileSync, renameSync, unlinkSync } from 'node:fs'; +import { PassThrough, Readable } from 'node:stream'; +import { buffer } from 'node:stream/consumers'; +import { createGzip } from 'node:zlib'; +import { pipeline } from 'node:stream/promises'; -import * as fs from 'fs'; import * as ini from 'ini'; -import * as path from 'path'; import * as tar from 'tar'; import type { Artifact, ArtifactProvider } from '../artifact-provider.mjs'; @@ -25,7 +29,8 @@ Label: Ubuntu/Debian Architecture: $ARCH Component: $COMPONENT Acquire-By-Hash: yes -Codename: $DISTRIBUTION\n`; +Codename: $DISTRIBUTION +Date: $DATE\n`; function iterateComponents(repo: DebRepo, callback: (distribution: string, component: string, deb: Artifact[]) => void): void { const distributions = Object.keys(repo); @@ -49,24 +54,6 @@ function iterateDebs(repo: DebRepo, callback: (distribution: string, component: }); } -function sha512(filePath: string): Promise { - return new Promise((resolve, reject) => { - const hash = createHash('sha512'); - fs.createReadStream(filePath) - .once('end', () => resolve(hash.digest('hex'))) - .once('error', () => reject) - .pipe(hash); - }); -} - -async function handleByHash(filePath: string): Promise { - const byHashDir = path.join(path.dirname(filePath), 'by-hash', 'SHA512'); - const byHashFileName = path.resolve(path.join(byHashDir, await sha512(filePath))); - - createDir(byHashDir); - return fs.promises.copyFile(filePath, byHashFileName); -} - export interface Config { out: string, gpgKeyName: string; @@ -114,20 +101,25 @@ export class DebBuilder implements Deployer { return `${this.config.applicationName}-${version}_${arch}.deb`; } - private async makeReleaseFileAndSign(distribution: string, component: string, arch: string): Promise { - const releaseContent = ReleaseFileTemplate + // eslint-disable-next-line max-params + private async makeReleaseFileAndSign(distribution: string, component: string, arch: string, indices: BinaryPackageIndexDescription[]): Promise { + let releaseContent = ReleaseFileTemplate .replace('$ORIGIN', this.config.origin) .replace('$DISTRIBUTION', distribution) .replace('$ARCH', arch) - .replace('$COMPONENT', component); + .replace('$COMPONENT', component) + .replace('$DATE', new Date().toUTCString()); - const distributionPath = path.join(this.distsPath, distribution); - const tempReleaseFile = path.join(this.tempPath, `Release-${randomBytes(4).toString('hex')}`); - const inReleaseFilePath = path.join(distributionPath, 'InRelease'); + releaseContent += 'SHA256:\n'; + for (const { address, size, name } of indices) { + releaseContent += ` ${Buffer.from(address.sha256).toString('hex')}\t${size}\t${name}`; + } - await fs.promises.writeFile(tempReleaseFile, releaseContent); + const tempReleaseFile = path.join(this.tempPath, `Release-${randomBytes(4).toString('hex')}`); + await writeFile(tempReleaseFile, releaseContent); - await execToolToFile('apt-ftparchive', ['release', distributionPath], tempReleaseFile, true); + const distributionPath = path.join(this.distsPath, distribution); + const inReleaseFilePath = path.join(distributionPath, 'InRelease'); await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '--clearsign', '-o', inReleaseFilePath, tempReleaseFile]); } @@ -162,7 +154,7 @@ export class DebBuilder implements Deployer { .pipe(tar.extract({ cwd: whereExtract, strip: 1 }, ['./control'])) // eslint-disable-next-line max-statements .on('finish', () => { - const controlMetaContent = fs.readFileSync(path.join(whereExtract, 'control'), 'utf-8').replaceAll(':', '='); + const controlMetaContent = readFileSync(path.join(whereExtract, 'control'), 'utf-8').replaceAll(':', '='); const controlMeta = ini.parse(controlMetaContent); const arch = controlMeta['Architecture']; const version = controlMeta['Version']; @@ -181,7 +173,7 @@ export class DebBuilder implements Deployer { `binary-${arch}`, `${this.debFileName(version, arch)}.meta`); createDir(path.dirname(targetMetaPath)); - fs.renameSync(path.join(whereExtract, 'control'), targetMetaPath); + renameSync(path.join(whereExtract, 'control'), targetMetaPath); removeDir(whereExtract); @@ -191,7 +183,7 @@ export class DebBuilder implements Deployer { this.config.applicationName, distribution, this.debFileName(version, arch)); - const relativeDebPath = path.relative(this.rootPath, debPath); + const relativeDebPath = path.relative(this.rootPath, debPath).replace(/\\/gu, '/'); const debSize = controlTar.headers['content-range']?.split('/')[1]; const sha1 = controlTar.headers['x-checksum-sha1']; const sha256 = controlTar.headers['x-checksum-sha256']; @@ -203,7 +195,7 @@ export class DebBuilder implements Deployer { const dataToAppend = `Filename: ${relativeDebPath}\nSize: ${debSize}\nSHA1: ${sha1}\nSHA256: ${sha256}\nMD5Sum: ${md5}\n`; - fs.promises.appendFile(targetMetaPath, dataToAppend).then(() => resolve()); + appendFile(targetMetaPath, dataToAppend).then(() => resolve()); const createFileOperation = this.packageCreator(deb.md5, debPath); @@ -219,64 +211,85 @@ export class DebBuilder implements Deployer { } private async makeRelease(): Promise<{}> { - const compressFile = (filePath: string): Promise => new Promise(resolve => { - const inp = fs.createReadStream(filePath); - const out = fs.createWriteStream(`${filePath}.gz`); - - const gzip = createGzip({ level: 9 }); - - inp.pipe(gzip).pipe(out) - .on('finish', () => { - resolve(`${filePath}.gz`); - }); - }); - - const compressPromises: Promise[] = []; - const byHashPromises: Promise[] = []; + const binaryPackageIndices: BinaryPackageIndexDescription[] = []; + const componentPromises: Promise[] = []; iterateComponents(this.config.repo, (distribution, component) => { const componentRoot = path.join(this.distsPath, distribution, component); - const componentsByArch = fs.readdirSync(componentRoot).map(dist => path.join(componentRoot, dist)); - - componentsByArch.forEach(dist => { - const targetPackagesFile = path.join(dist, 'Packages'); - const metaFiles = fs.readdirSync(dist) - .filter(fileName => fileName.endsWith('.meta')) - .map(metaFile => path.join(dist, metaFile)); - - let packagesContent = ''; - - for (const metaFile of metaFiles) { - packagesContent += fs.readFileSync(metaFile); - packagesContent += '\n'; - fs.unlinkSync(metaFile); - } - - fs.writeFileSync(targetPackagesFile, packagesContent); - byHashPromises.push(handleByHash(targetPackagesFile)); - compressPromises.push(compressFile(targetPackagesFile)); - }); - }); - - const compressedPackages = await Promise.all(compressPromises); + componentPromises.push(Promise.all( + readdirSync(componentRoot).map(async binarySubdir => { + const binaryPath = path.join(componentRoot, binarySubdir); + const metaFiles = readdirSync(binaryPath) + .filter(fileName => fileName.endsWith('.meta')) + .map(metaFile => path.join(binaryPath, metaFile)); + + let packagesContent = ''; + + for (const metaFile of metaFiles) { + packagesContent += readFileSync(metaFile); + packagesContent += '\n'; + unlinkSync(metaFile); + } - compressedPackages.forEach(packagePath => { - byHashPromises.push(handleByHash(packagePath)); + const packagesContentDescription = await storeStreamInCas(path.resolve(binaryPath, 'by-hash'), Readable.from(packagesContent)); + binaryPackageIndices.push({ + name: `${component}/${binarySubdir}/Packages.gz`, + size: packagesContentDescription.size, + address: packagesContentDescription.address, + }); + }), + )); }); - - await Promise.all(byHashPromises); + await Promise.all(componentPromises); const releasesPromises: Promise[] = []; - iterateComponents(this.config.repo, (distribution, component) => { const archesSet = this.archesByDistComp.get(`${distribution}/${component}`); if (!archesSet) { throw new Error('No arch was found for distribution'); } - releasesPromises.push(this.makeReleaseFileAndSign(distribution, component, [...archesSet.values()].join(' '))); + releasesPromises.push(this.makeReleaseFileAndSign(distribution, component, [...archesSet.values()].join(' '), binaryPackageIndices)); }); - return Promise.all(releasesPromises); } } + +interface ContentAddress { + sha256: Uint8Array; +} + +interface ContentDescription { + size: number; + address: ContentAddress; +} + +// https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29 +async function storeStreamInCas(root: string, stream: NodeJS.ReadableStream): Promise { + // compress and store in a buffer + const gzip = createGzip({ level: 9 }); + const gzippedContent = new PassThrough(); + await pipeline(stream, gzip, gzippedContent); + const contentBuffer = await buffer(gzippedContent); + + // compute a digest + const sha256Hash = createHash('sha256'); + await pipeline(Readable.from(contentBuffer), sha256Hash); + const sha256 = sha256Hash.read(); + + // store in a file + const sha256Dir = path.join(root, 'SHA256'); + const fileName = path.join(sha256Dir, Buffer.from(sha256).toString('hex')); + createDir(sha256Dir); + await pipeline(Readable.from(contentBuffer), createWriteStream(fileName)); + return { + size: contentBuffer.length, + address: { sha256 }, + }; +} + +interface BinaryPackageIndexDescription { + name: string; + size: number; + address: ContentAddress; +} From 272d980d5d32dcc920d1b15c0afdb3de6f554879 Mon Sep 17 00:00:00 2001 From: Stanislav Makarov Date: Wed, 3 Apr 2024 17:49:57 +0300 Subject: [PATCH 3/4] fix(debian): store both uncompressed and gzipped binary package indices --- src/deb/deb-builder.mts | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/deb/deb-builder.mts b/src/deb/deb-builder.mts index 35d8cee..e07a6f9 100644 --- a/src/deb/deb-builder.mts +++ b/src/deb/deb-builder.mts @@ -112,7 +112,7 @@ export class DebBuilder implements Deployer { releaseContent += 'SHA256:\n'; for (const { address, size, name } of indices) { - releaseContent += ` ${Buffer.from(address.sha256).toString('hex')}\t${size}\t${name}`; + releaseContent += ` ${Buffer.from(address.sha256).toString('hex')}\t${size}\t${name}\n`; } const tempReleaseFile = path.join(this.tempPath, `Release-${randomBytes(4).toString('hex')}`); @@ -231,12 +231,13 @@ export class DebBuilder implements Deployer { unlinkSync(metaFile); } - const packagesContentDescription = await storeStreamInCas(path.resolve(binaryPath, 'by-hash'), Readable.from(packagesContent)); - binaryPackageIndices.push({ - name: `${component}/${binarySubdir}/Packages.gz`, - size: packagesContentDescription.size, - address: packagesContentDescription.address, - }); + const storePackageIndex = async(shortName: string, store: (root: string, stream: NodeJS.ReadableStream) => Promise) => { + const name = `${component}/${binarySubdir}/${shortName}`; + const { size, address } = await store(path.resolve(binaryPath, 'by-hash'), Readable.from(packagesContent)); + binaryPackageIndices.push({ name, size, address }); + }; + await storePackageIndex('Packages', storeStreamInCas); + await storePackageIndex('Packages.gz', storeGzippedStreamInCas); }), )); }); @@ -264,13 +265,16 @@ interface ContentDescription { address: ContentAddress; } +async function storeGzippedStreamInCas(root: string, stream: NodeJS.ReadableStream): Promise { + const gzip = createGzip({ level: 9 }); + const gzippedStream = new PassThrough(); + await pipeline(stream, gzip, gzippedStream); + return storeStreamInCas(root, gzippedStream); +} + // https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29 async function storeStreamInCas(root: string, stream: NodeJS.ReadableStream): Promise { - // compress and store in a buffer - const gzip = createGzip({ level: 9 }); - const gzippedContent = new PassThrough(); - await pipeline(stream, gzip, gzippedContent); - const contentBuffer = await buffer(gzippedContent); + const contentBuffer = await buffer(stream); // compute a digest const sha256Hash = createHash('sha256'); From f356d3e398aa1afec23aab5bac508f064f6be887 Mon Sep 17 00:00:00 2001 From: Stanislav Makarov Date: Wed, 3 Apr 2024 18:07:11 +0300 Subject: [PATCH 4/4] refactor(debian): extract content-addressable storage logic to separate module --- src/deb/cas.mts | 45 ++++++++++++++++++++++++++++++++ src/deb/deb-builder.mts | 57 ++++++++--------------------------------- 2 files changed, 56 insertions(+), 46 deletions(-) create mode 100644 src/deb/cas.mts diff --git a/src/deb/cas.mts b/src/deb/cas.mts new file mode 100644 index 0000000..539e938 --- /dev/null +++ b/src/deb/cas.mts @@ -0,0 +1,45 @@ +import * as path from 'node:path'; +import { PassThrough, Readable } from 'node:stream'; +import { buffer } from 'node:stream/consumers'; +import { createGzip } from 'node:zlib'; +import { createHash } from 'node:crypto'; +import { createWriteStream } from 'node:fs'; +import { pipeline } from 'node:stream/promises'; + +import { createDir } from '../fs.mjs'; + +export interface ContentAddress { + sha256: Uint8Array; +} + +export interface ContentDescription { + size: number; + address: ContentAddress; +} + +export async function storeGzippedStream(root: string, stream: NodeJS.ReadableStream): Promise { + const gzip = createGzip({ level: 9 }); + const gzippedStream = new PassThrough(); + await pipeline(stream, gzip, gzippedStream); + return storeStream(root, gzippedStream); +} + +// https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29 +export async function storeStream(root: string, stream: NodeJS.ReadableStream): Promise { + const contentBuffer = await buffer(stream); + + // compute a digest + const sha256Hash = createHash('sha256'); + await pipeline(Readable.from(contentBuffer), sha256Hash); + const sha256 = sha256Hash.read(); + + // store in a file + const sha256Dir = path.join(root, 'SHA256'); + const fileName = path.join(sha256Dir, Buffer.from(sha256).toString('hex')); + createDir(sha256Dir); + await pipeline(Readable.from(contentBuffer), createWriteStream(fileName)); + return { + size: contentBuffer.length, + address: { sha256 }, + }; +} diff --git a/src/deb/deb-builder.mts b/src/deb/deb-builder.mts index e07a6f9..f2a6c30 100644 --- a/src/deb/deb-builder.mts +++ b/src/deb/deb-builder.mts @@ -1,15 +1,13 @@ import * as path from 'node:path'; import { appendFile, writeFile } from 'node:fs/promises'; -import { createHash, randomBytes } from 'node:crypto'; -import { createWriteStream, readdirSync, readFileSync, renameSync, unlinkSync } from 'node:fs'; -import { PassThrough, Readable } from 'node:stream'; -import { buffer } from 'node:stream/consumers'; -import { createGzip } from 'node:zlib'; -import { pipeline } from 'node:stream/promises'; +import { readdirSync, readFileSync, renameSync, unlinkSync } from 'node:fs'; +import { randomBytes } from 'node:crypto'; +import { Readable } from 'node:stream'; import * as ini from 'ini'; import * as tar from 'tar'; +import * as cas from './cas.mjs'; import type { Artifact, ArtifactProvider } from '../artifact-provider.mjs'; import { createDir, execToolToFile, removeDir } from '../fs.mjs'; import type { Deployer } from '../deployer.mjs'; @@ -231,13 +229,16 @@ export class DebBuilder implements Deployer { unlinkSync(metaFile); } - const storePackageIndex = async(shortName: string, store: (root: string, stream: NodeJS.ReadableStream) => Promise) => { + const storePackageIndex = async( + shortName: string, + store: (root: string, stream: NodeJS.ReadableStream) => Promise, + ) => { const name = `${component}/${binarySubdir}/${shortName}`; const { size, address } = await store(path.resolve(binaryPath, 'by-hash'), Readable.from(packagesContent)); binaryPackageIndices.push({ name, size, address }); }; - await storePackageIndex('Packages', storeStreamInCas); - await storePackageIndex('Packages.gz', storeGzippedStreamInCas); + await storePackageIndex('Packages', cas.storeStream); + await storePackageIndex('Packages.gz', cas.storeGzippedStream); }), )); }); @@ -256,44 +257,8 @@ export class DebBuilder implements Deployer { } } -interface ContentAddress { - sha256: Uint8Array; -} - -interface ContentDescription { - size: number; - address: ContentAddress; -} - -async function storeGzippedStreamInCas(root: string, stream: NodeJS.ReadableStream): Promise { - const gzip = createGzip({ level: 9 }); - const gzippedStream = new PassThrough(); - await pipeline(stream, gzip, gzippedStream); - return storeStreamInCas(root, gzippedStream); -} - -// https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29 -async function storeStreamInCas(root: string, stream: NodeJS.ReadableStream): Promise { - const contentBuffer = await buffer(stream); - - // compute a digest - const sha256Hash = createHash('sha256'); - await pipeline(Readable.from(contentBuffer), sha256Hash); - const sha256 = sha256Hash.read(); - - // store in a file - const sha256Dir = path.join(root, 'SHA256'); - const fileName = path.join(sha256Dir, Buffer.from(sha256).toString('hex')); - createDir(sha256Dir); - await pipeline(Readable.from(contentBuffer), createWriteStream(fileName)); - return { - size: contentBuffer.length, - address: { sha256 }, - }; -} - interface BinaryPackageIndexDescription { name: string; size: number; - address: ContentAddress; + address: cas.ContentAddress; }