diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d2679a695..6773a6896f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1510,14 +1510,14 @@ importers: specifier: ^5.18.1 version: 5.30.0 adm-zip: - specifier: ^0.4.16 - version: 0.4.16 + specifier: ^0.5.16 + version: 0.5.16 chalk: specifier: ^5.3.0 version: 5.3.0 debug: specifier: ^4.1.1 - version: 4.3.7 + version: 4.3.7(supports-color@8.1.1) enquirer: specifier: ^2.3.0 version: 2.4.1 @@ -1553,8 +1553,8 @@ importers: specifier: workspace:^ version: link:../hardhat-test-utils '@types/adm-zip': - specifier: ^0.5.5 - version: 0.5.5 + specifier: ^0.5.7 + version: 0.5.7 '@types/debug': specifier: ^4.1.4 version: 4.1.12 @@ -3469,8 +3469,8 @@ packages: hardhat: ^2.9.9 typechain: ^8.3.2 - '@types/adm-zip@0.5.5': - resolution: {integrity: sha512-YCGstVMjc4LTY5uK9/obvxBya93axZOVOyf2GSUulADzmLhYE45u2nAssCs/fWBs1Ifq5Vat75JTPwd5XZoPJw==} + '@types/adm-zip@0.5.7': + resolution: {integrity: sha512-DNEs/QvmyRLurdQPChqq0Md4zGvPwHerAJYWk9l2jCbD1VPpnzRJorOdiq4zsw09NFbYnhfsoEhWtxIzXpn2yw==} '@types/async-eventemitter@0.2.4': resolution: {integrity: sha512-2Bq61VD01kgLf1XkK2xPtoBcu7fgn/km5JyEX9v0BlG5VQBzA+BlF9umFk+8gR8S4+eK7MgDY2oyVZCu6ar3Jw==} @@ -3888,6 +3888,10 @@ packages: resolution: {integrity: sha512-TFi4HBKSGfIKsK5YCkKaaFG2m4PEDyViZmEwof3MTIgzimHLto6muaHVpbrljdIvIrFZzEq/p4nafOeLcYegrg==} engines: {node: '>=0.3.0'} + adm-zip@0.5.16: + resolution: {integrity: sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==} + engines: {node: '>=12.0'} + aes-js@3.0.0: resolution: {integrity: sha512-H7wUZRn8WpTq9jocdxQ2c8x2sKo9ZVmzfRE13GiNJXfp7NcKYEdvl3vspKjXox6RIG2VtaRe4JFvxG4rqp2Zuw==} @@ -7425,7 +7429,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) espree: 9.6.1 globals: 13.24.0 ignore: 5.3.2 @@ -7726,7 +7730,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -8269,7 +8273,7 @@ snapshots: hardhat: link:packages/hardhat-core typechain: 8.3.2(typescript@5.0.4) - '@types/adm-zip@0.5.5': + '@types/adm-zip@0.5.7': dependencies: '@types/node': 22.10.0 @@ -8479,7 +8483,7 @@ snapshots: '@typescript-eslint/type-utils': 7.7.1(eslint@8.57.0)(typescript@5.5.4) '@typescript-eslint/utils': 7.7.1(eslint@8.57.0)(typescript@5.5.4) '@typescript-eslint/visitor-keys': 7.7.1 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.0 graphemer: 1.4.0 ignore: 5.3.2 @@ -8509,7 +8513,7 @@ snapshots: '@typescript-eslint/types': 7.18.0 '@typescript-eslint/typescript-estree': 7.18.0(typescript@5.5.4) '@typescript-eslint/visitor-keys': 7.18.0 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.0 optionalDependencies: typescript: 5.5.4 @@ -8522,7 +8526,7 @@ snapshots: '@typescript-eslint/types': 7.7.1 '@typescript-eslint/typescript-estree': 7.7.1(typescript@5.5.4) '@typescript-eslint/visitor-keys': 7.7.1 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.0 optionalDependencies: typescript: 5.5.4 @@ -8565,7 +8569,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 7.18.0(typescript@5.5.4) '@typescript-eslint/utils': 7.18.0(eslint@8.57.0)(typescript@5.5.4) - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.0 ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: @@ -8577,7 +8581,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 7.7.1(typescript@5.5.4) '@typescript-eslint/utils': 7.7.1(eslint@8.57.0)(typescript@5.5.4) - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) eslint: 8.57.0 ts-api-utils: 1.3.0(typescript@5.5.4) optionalDependencies: @@ -8625,7 +8629,7 @@ snapshots: dependencies: '@typescript-eslint/types': 7.18.0 '@typescript-eslint/visitor-keys': 7.18.0 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.5 @@ -8640,7 +8644,7 @@ snapshots: dependencies: '@typescript-eslint/types': 7.7.1 '@typescript-eslint/visitor-keys': 7.7.1 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.5 @@ -8765,13 +8769,15 @@ snapshots: adm-zip@0.4.16: {} + adm-zip@0.5.16: {} + aes-js@3.0.0: {} aes-js@4.0.0-beta.5: {} agent-base@6.0.2: dependencies: - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -9343,10 +9349,6 @@ snapshots: dependencies: ms: 2.1.3 - debug@4.3.7: - dependencies: - ms: 2.1.3 - debug@4.3.7(supports-color@8.1.1): dependencies: ms: 2.1.3 @@ -9648,7 +9650,7 @@ snapshots: eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@8.57.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) enhanced-resolve: 5.17.1 eslint: 8.57.0 eslint-module-utils: 2.12.0(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0) @@ -9814,7 +9816,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -10415,7 +10417,7 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.7 + debug: 4.3.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color diff --git a/v-next/hardhat/package.json b/v-next/hardhat/package.json index 7de30ed8aa..bccd5e2a47 100644 --- a/v-next/hardhat/package.json +++ b/v-next/hardhat/package.json @@ -64,7 +64,7 @@ "@eslint-community/eslint-plugin-eslint-comments": "^4.3.0", "@ignored/hardhat-vnext-node-test-reporter": "workspace:^3.0.0-next.15", "@nomicfoundation/hardhat-test-utils": "workspace:^", - "@types/adm-zip": "^0.5.5", + "@types/adm-zip": "^0.5.7", "@types/debug": "^4.1.4", "@types/node": "^20.14.9", "@types/semver": "^7.5.8", @@ -90,7 +90,7 @@ "@ignored/hardhat-vnext-zod-utils": "workspace:^3.0.0-next.15", "@nomicfoundation/solidity-analyzer": "^0.1.0", "@sentry/node": "^5.18.1", - "adm-zip": "^0.4.16", + "adm-zip": "^0.5.16", "chalk": "^5.3.0", "debug": "^4.1.1", "enquirer": "^2.3.0", diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/build-system.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/build-system.ts index 349a39a8db..87195675d5 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/build-system.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/build-system.ts @@ -9,6 +9,7 @@ import type { GetCompilationJobsOptions, CompileBuildInfoOptions, RunCompilationJobOptions, + EmitArtifactsOptions, } from "../../../../types/solidity/build-system.js"; import type { CompilationJob } from "../../../../types/solidity/compilation-job.js"; import type { @@ -24,6 +25,7 @@ import { assertHardhatInvariant } from "@ignored/hardhat-vnext-errors"; import { getAllDirectoriesMatching, getAllFilesMatching, + isDirectory, readJsonFile, remove, writeUtf8File, @@ -44,6 +46,7 @@ import { getContractArtifact, getDuplicatedContractNamesDeclarationFile, } from "./artifacts.js"; +import { Cache } from "./cache.js"; import { CompilationJobImplementation } from "./compilation-job.js"; import { downloadConfiguredCompilers, getCompiler } from "./compiler/index.js"; import { buildDependencyGraph } from "./dependency-graph-building.js"; @@ -67,11 +70,21 @@ export interface SolidityBuildSystemOptions { export class SolidityBuildSystemImplementation implements SolidityBuildSystem { readonly #options: SolidityBuildSystemOptions; + readonly #compilerOutputCache: Cache; + readonly #artifactsCache: Cache; readonly #defaultConcurrenty = Math.max(os.cpus().length - 1, 1); #downloadedCompilers = false; constructor(options: SolidityBuildSystemOptions) { this.#options = options; + this.#compilerOutputCache = new Cache( + options.cachePath, + "hardhat.core.solidity.build-system.compiler-output", + ); + this.#artifactsCache = new Cache( + options.cachePath, + "hardhat.core.solidity.build-system.artifacts", + ); } public async getRootFilePaths(): Promise { @@ -112,11 +125,14 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem { const compilationJobs = [...new Set(compilationJobsPerFile.values())]; - // TODO: Filter the compilation jobs based on the cache - + const runCompilationJobOptions: RunCompilationJobOptions = { + force: options?.force, + quiet: options?.quiet, + }; const results: CompilerOutput[] = await pMap( compilationJobs, - (compilationJob) => this.runCompilationJob(compilationJob), + (compilationJob) => + this.runCompilationJob(compilationJob, runCompilationJobOptions), { concurrency: options?.concurrency ?? this.#defaultConcurrenty, // An error when running the compiler is not a compilation failure, but @@ -136,11 +152,16 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem { if (isSuccessfulBuild) { log("Emitting artifacts of successful build"); + const emitArtifactsOptions: EmitArtifactsOptions = { + force: options?.force, + quiet: options?.quiet, + }; await Promise.all( compilationJobs.map(async (compilationJob, i) => { const artifactsPerFile = await this.emitArtifacts( compilationJob, results[i], + emitArtifactsOptions, ); contractArtifactsGeneratedByCompilationJob.set( @@ -312,6 +333,17 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem { compilationJob: CompilationJob, options?: RunCompilationJobOptions, ): Promise { + const buildId = compilationJob.getBuildId(); + + if (options?.force !== true) { + const cachedCompilerOutput = + await this.#compilerOutputCache.getJson(buildId); + if (cachedCompilerOutput !== undefined) { + log(`Using cached compiler output for build ${buildId}`); + return cachedCompilerOutput; + } + } + await this.#downloadConfiguredCompilers(options?.quiet); let numberOfFiles = 0; @@ -332,7 +364,15 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem { "The long version of the compiler should match the long version of the compilation job", ); - return compiler.compile(compilationJob.getSolcInput()); + const compilerOutput = await compiler.compile( + compilationJob.getSolcInput(), + ); + + if (!this.#hasCompilationErrors(compilerOutput)) { + await this.#compilerOutputCache.setJson(buildId, compilerOutput); + } + + return compilerOutput; } public async remapCompilerError( @@ -369,8 +409,43 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem { public async emitArtifacts( compilationJob: CompilationJob, compilerOutput: CompilerOutput, + options?: EmitArtifactsOptions, ): Promise> { const result = new Map(); + const buildId = compilationJob.getBuildId(); + + if (options?.force !== true) { + const cachedFiles = await this.#artifactsCache.getFiles( + buildId, + this.#options.artifactsPath, + ); + if (cachedFiles !== undefined) { + log(`Using cached artifacts for build ${buildId}`); + for (const filePath of cachedFiles) { + const relativePath = path.relative( + this.#options.artifactsPath, + filePath, + ); + if ( + path.dirname(relativePath) === "build-info" || + path.basename(relativePath) === "artifacts.d.ts" + ) { + continue; + } + if (await isDirectory(filePath)) { + result.set(relativePath, []); + } else { + const publicSourceName = path.dirname(relativePath); + const paths = result.get(publicSourceName) ?? []; + paths.push(filePath); + result.set(publicSourceName, paths); + } + } + return result; + } + } + + const filesToCache: string[] = []; // We emit the artifacts for each root file, first emitting one artifact // for each contract, and then one declaration file for the entire file, @@ -430,11 +505,20 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem { artifactsDeclarationFilePath, artifactsDeclarationFile, ); + + if (paths.length === 0) { + filesToCache.push( + path.join(this.#options.artifactsPath, publicSourceName), + ); + } else { + filesToCache.push(...paths); + } + filesToCache.push(artifactsDeclarationFilePath); } // Once we have emitted all the contract artifacts and its declaration // file, we emit the build info file and its output file. - const buildInfoId = compilationJob.getBuildId(); + const buildInfoId = buildId; const buildInfoPath = path.join( this.#options.artifactsPath, @@ -474,6 +558,14 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem { })(), ]); + filesToCache.push(buildInfoPath, buildInfoOutputPath); + + await this.#artifactsCache.setFiles( + buildId, + this.#options.artifactsPath, + filesToCache, + ); + return result; } diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/cache.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/cache.ts new file mode 100644 index 0000000000..e25981959c --- /dev/null +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/cache.ts @@ -0,0 +1,73 @@ +import path from "node:path"; + +import { assertHardhatInvariant } from "@ignored/hardhat-vnext-errors"; +import { + exists, + readJsonFile, + writeJsonFile, +} from "@ignored/hardhat-vnext-utils/fs"; +import AdmZip from "adm-zip"; + +export class Cache { + readonly #basePath: string; + readonly #namespace: string; + + constructor(basePath: string, namespace: string) { + this.#basePath = basePath; + this.#namespace = namespace; + + console.log(`Using cache at ${this.#basePath}`); + } + + #getPath(key: string): string { + return path.join(this.#basePath, this.#namespace, key); + } + + public async setJson(key: string, value: T): Promise { + const filePath = this.#getPath(key); + await writeJsonFile(filePath, value); + } + + public async getJson(key: string): Promise { + const filePath = this.#getPath(key); + return (await exists(filePath)) ? readJsonFile(filePath) : undefined; + } + + public async setFiles( + key: string, + rootPath: string, + filePaths: string[], + ): Promise { + const zipFilePath = this.#getPath(key); + const zip = new AdmZip(); + for (const filePath of filePaths) { + zip.addLocalFile( + filePath, + path.dirname(path.relative(rootPath, filePath)), + ); + } + const zipFileCreated = await zip.writeZipPromise(zipFilePath, { + overwrite: true, + }); + assertHardhatInvariant( + zipFileCreated, + `Failed to create zip file ${zipFilePath}`, + ); + } + + public async getFiles( + key: string, + rootPath: string, + ): Promise { + const zipFilePath = this.#getPath(key); + if (await exists(zipFilePath)) { + const zip = new AdmZip(zipFilePath); + zip.extractAllTo(rootPath, true); + const filePaths = zip + .getEntries() + .map((entry) => path.join(rootPath, entry.entryName)); + return filePaths; + } + return undefined; + } +} diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity/hook-handlers/hre.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity/hook-handlers/hre.ts index f50592d26e..29dacae291 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity/hook-handlers/hre.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity/hook-handlers/hre.ts @@ -3,6 +3,7 @@ import type { BuildOptions, CompilationJobCreationError, CompileBuildInfoOptions, + EmitArtifactsOptions, FileBuildResult, GetCompilationJobsOptions, RunCompilationJobOptions, @@ -70,9 +71,10 @@ class LazySolidityBuildSystem implements SolidityBuildSystem { public async emitArtifacts( compilationJob: CompilationJob, compilerOutput: CompilerOutput, + options?: EmitArtifactsOptions, ): Promise> { const buildSystem = await this.#getBuildSystem(); - return buildSystem.emitArtifacts(compilationJob, compilerOutput); + return buildSystem.emitArtifacts(compilationJob, compilerOutput, options); } public async cleanupArtifacts(rootFilePaths: string[]): Promise { diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity/tasks/compile.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity/tasks/compile.ts index 0bb657db05..1da60d956a 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity/tasks/compile.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity/tasks/compile.ts @@ -49,9 +49,7 @@ const compileAction: NewTaskActionFunction = async ( } const sucessful = [...results.values()].every( - ({ type }) => - type === FileBuildResultType.CACHE_HIT || - type === FileBuildResultType.BUILD_SUCCESS, + ({ type }) => type === FileBuildResultType.BUILD_SUCCESS, ); if (!sucessful) { diff --git a/v-next/hardhat/src/types/solidity/build-system.ts b/v-next/hardhat/src/types/solidity/build-system.ts index 821622e61e..56bb8a0531 100644 --- a/v-next/hardhat/src/types/solidity/build-system.ts +++ b/v-next/hardhat/src/types/solidity/build-system.ts @@ -62,12 +62,34 @@ export type GetCompilationJobsOptions = Omit< * The options of the `runCompilationJob` method. */ export interface RunCompilationJobOptions { + /** + * If `true`, this option foces the build system to rerun the compilation job, + * even if its output is cached. + */ + force?: boolean; + /** * If `true`, the compilation process doesn't print any output. */ quiet?: boolean; } +/** + * The options of the `emitArtifacts` method. + */ +export interface EmitArtifactsOptions { + /** + * If `true`, this option foces the build system to recreate the artifacts, + * even if they are cached. + */ + force?: boolean; + + /** + * If `true`, the emit process doesn't print any output. + */ + quiet?: boolean; +} + /** * The options of the `compileBuildInfo` method. */ @@ -135,22 +157,11 @@ export type CompilationJobCreationError = * The restult of building a file. */ export enum FileBuildResultType { - CACHE_HIT = "CACHE_HIT", BUILD_SUCCESS = "BUILD_SUCCESS", BUILD_FAILURE = "BUILD_FAILURE", } -export type FileBuildResult = - | CacheHitFileBuildResult - | SuccessfulFileBuildResult - | FailedFileBuildResult; - -export interface CacheHitFileBuildResult { - type: FileBuildResultType.CACHE_HIT; - // TODO: Should we remove this? It is a buildId of an already existing build - // info. - buildId: string; -} +export type FileBuildResult = SuccessfulFileBuildResult | FailedFileBuildResult; export interface SuccessfulFileBuildResult { type: FileBuildResultType.BUILD_SUCCESS; @@ -253,6 +264,7 @@ export interface SolidityBuildSystem { emitArtifacts( compilationJob: CompilationJob, compilerOutput: CompilerOutput, + options?: EmitArtifactsOptions, ): Promise>; /** @@ -265,7 +277,7 @@ export interface SolidityBuildSystem { * This method should only be used after a complete build has succeeded, as * it relies on the build system to have generated all the necessary artifact * files. - + * @param rootFilePaths All the root files of the project. */ cleanupArtifacts(rootFilePaths: string[]): Promise;