Merge pull request #5 from dl-solarity/refactor_compile_task

Refactor compile task

.gitignore

@@ -4,7 +4,6 @@ node_modules
 .eslintcache
 
 # Hardhat files
-cache
 artifacts
 coverage.json
 coverage

src/internal/CircomCircuitsCache.ts → src/cache/CircomCircuitsCache.ts

@@ -2,22 +2,22 @@ import fsExtra from "fs-extra";
 import * as t from "io-ts";
 import { isEqual } from "lodash";
 
-import { FORMAT_VERSION } from "./constants";
-import { Cache, CacheEntry } from "../types/internal/circuits-cache";
-import { CompileOptions } from "../types/compile";
+import { FORMAT_VERSION } from "../constants";
+import { Cache, CacheEntry, CompileFlags } from "../types/compile";
 
-const CompileOptionsCodec = t.type({
+const CompileFlagsCodec = t.type({
+  r1cs: t.boolean,
+  wasm: t.boolean,
   sym: t.boolean,
   json: t.boolean,
   c: t.boolean,
-  quiet: t.boolean,
 });
 
 const CacheEntryCodec = t.type({
   lastModificationDate: t.number,
   contentHash: t.string,
   sourceName: t.string,
-  compileOptions: CompileOptionsCodec,
+  compileFlags: CompileFlagsCodec,
   imports: t.array(t.string),
   versionPragmas: t.array(t.string),
 });
@@ -27,33 +27,33 @@ const CacheCodec = t.type({
   files: t.record(t.string, CacheEntryCodec),
 });
 
-export class CircomCircuitsCache {
-  public static createEmpty(): CircomCircuitsCache {
-    return new CircomCircuitsCache({
+class BaseCircomCircuitsCache {
+  public static createEmpty(): BaseCircomCircuitsCache {
+    return new BaseCircomCircuitsCache({
       _format: FORMAT_VERSION,
       files: {},
     });
   }
 
-  public static async readFromFile(solidityFilesCachePath: string): Promise<CircomCircuitsCache> {
+  public static async readFromFile(circuitsFilesCachePath: string): Promise<BaseCircomCircuitsCache> {
     let cacheRaw: Cache = {
       _format: FORMAT_VERSION,
       files: {},
     };
 
-    if (await fsExtra.pathExists(solidityFilesCachePath)) {
-      cacheRaw = await fsExtra.readJson(solidityFilesCachePath);
+    if (await fsExtra.pathExists(circuitsFilesCachePath)) {
+      cacheRaw = await fsExtra.readJson(circuitsFilesCachePath);
     }
 
     const result = CacheCodec.decode(cacheRaw);
 
     if (result.isRight()) {
-      const solidityFilesCache = new CircomCircuitsCache(result.value);
+      const solidityFilesCache = new BaseCircomCircuitsCache(result.value);
       await solidityFilesCache.removeNonExistingFiles();
       return solidityFilesCache;
     }
 
-    return new CircomCircuitsCache({
+    return new BaseCircomCircuitsCache({
       _format: FORMAT_VERSION,
       files: {},
     });
@@ -93,7 +93,7 @@ export class CircomCircuitsCache {
     delete this._cache.files[file];
   }
 
-  public hasFileChanged(absolutePath: string, contentHash: string, compileOptions: CompileOptions): boolean {
+  public hasFileChanged(absolutePath: string, contentHash: string, compileFlags: CompileFlags): boolean {
     const cacheEntry = this.getEntry(absolutePath);
 
     if (cacheEntry === undefined) {
@@ -104,10 +104,31 @@ export class CircomCircuitsCache {
       return true;
     }
 
-    if (!isEqual(cacheEntry.compileOptions, compileOptions)) {
+    if (!isEqual(cacheEntry.compileFlags, compileFlags)) {
       return true;
     }
 
     return false;
   }
 }
+
+export let CircomCircuitsCache: BaseCircomCircuitsCache | null = null;
+
+export async function createCircuitsCache(circuitsFilesCachePath?: string) {
+  if (CircomCircuitsCache) {
+    return;
+  }
+
+  if (circuitsFilesCachePath) {
+    CircomCircuitsCache = await BaseCircomCircuitsCache.readFromFile(circuitsFilesCachePath);
+  } else {
+    CircomCircuitsCache = BaseCircomCircuitsCache.createEmpty();
+  }
+}
+
+/**
+ * Used only in test environments to ensure test atomicity
+ */
+export function resetCircuitsCache() {
+  CircomCircuitsCache = null;
+}

src/compile/core/CircomCompiler.ts

@@ -0,0 +1,53 @@
+import fs from "fs";
+
+import { HardhatZKitError } from "../../errors";
+import { ICircomCompiler, CompileConfig } from "../../types/compile/core/circom-compiler";
+
+// eslint-disable-next-line
+const { Context, CircomRunner, bindings } = require("@distributedlab/circom2");
+
+export class CircomCompiler implements ICircomCompiler {
+  constructor(private readonly _compiler: typeof Context) {}
+
+  public async compile(config: CompileConfig) {
+    const compilationArgs: string[] = this.getCompilationArgs(config);
+
+    const circomRunner: typeof CircomRunner = new CircomRunner({
+      args: compilationArgs,
+      preopens: { "/": "/" },
+      bindings: {
+        ...bindings,
+        exit(code: number) {
+          throw new HardhatZKitError(`Compilation error. Exit code: ${code}.`);
+        },
+        fs,
+      },
+      quiet: config.quiet,
+    });
+
+    try {
+      await circomRunner.execute(this._compiler);
+    } catch (err) {
+      const parentErr = new Error(undefined, { cause: err });
+
+      if (config.quiet) {
+        throw new HardhatZKitError(
+          "Compilation failed with an unknown error. Consider passing 'quiet=false' flag to see the compilation error.",
+          parentErr,
+        );
+      }
+
+      throw new HardhatZKitError("Compilation failed.", parentErr);
+    }
+  }
+
+  public getCompilationArgs(config: CompileConfig): string[] {
+    const args = [config.circuitFullPath, "-o", config.artifactsFullPath];
+
+    for (const [key, value] of Object.entries(config.compileFlags)) {
+      value && args.push(`--${key}`);
+    }
+
+    return args;
+  }
+}

src/compile/core/CircomCompilerFactory.ts

@@ -0,0 +1,23 @@
+import fs from "fs";
+
+import { CircomCompiler } from "./CircomCompiler";
+import { ICircomCompiler, CompilerVersion } from "../../types/compile";
+import { HardhatZKitError } from "../../errors";
+
+// eslint-disable-next-line
+const { Context } = require("@distributedlab/circom2");
+
+export class CircomCompilerFactory {
+  public static createCircomCompiler(version: CompilerVersion): ICircomCompiler {
+    switch (version) {
+      case "0.2.18":
+        return new CircomCompiler(this._getCircomCompiler("@distributedlab/circom2/circom.wasm"));
+      default:
+        throw new HardhatZKitError(`Unsupported Circom compiler version - ${version}. Please provide another version.`);
+    }
+  }
+
+  private static _getCircomCompiler(compilerPath: string): typeof Context {
+    return fs.readFileSync(require.resolve(compilerPath));
+  }
+}

src/compile/core/CompilationFilesManager.ts

@@ -0,0 +1,200 @@
+import os from "os";
+import path from "path";
+import fsExtra from "fs-extra";
+
+import { HardhatConfig, ProjectPathsConfig } from "hardhat/types";
+import { getAllFilesMatching } from "hardhat/internal/util/fs-utils";
+import { localPathToSourceName } from "hardhat/utils/source-names";
+import { ResolvedFile } from "hardhat/types/builtin-tasks";
+
+import { FileFilterSettings, ZKitConfig } from "../../types/zkit-config";
+import { CompileFlags, CompilationFilesManagerConfig, ResolvedFileWithDependencies } from "../../types/compile";
+import { DependencyGraph, Parser, Resolver } from "../dependencies";
+import { CircomCircuitsCache } from "../../cache/CircomCircuitsCache";
+
+import { getNormalizedFullPath } from "../../utils/path-utils";
+import { MAIN_COMPONENT_REG_EXP } from "../../constants";
+import { HardhatZKitError } from "../../errors";
+
+export class CompilationFilesManager {
+  private readonly _zkitConfig: ZKitConfig;
+  private readonly _projectPaths: ProjectPathsConfig;
+
+  constructor(
+    private readonly _config: CompilationFilesManagerConfig,
+    private readonly _readFile: (absolutePath: string) => Promise<string>,
+    hardhatConfig: HardhatConfig,
+  ) {
+    this._zkitConfig = hardhatConfig.zkit;
+    this._projectPaths = hardhatConfig.paths;
+  }
+
+  public async getResolvedFilesToCompile(
+    compileFlags: CompileFlags,
+    force: boolean,
+  ): Promise<ResolvedFileWithDependencies[]> {
+    const circuitsSourcePaths: string[] = await getAllFilesMatching(this.getCircuitsDirFullPath(), (f) =>
+      f.endsWith(".circom"),
+    );
+
+    const filteredCircuitsSourcePaths: string[] = this._filterSourcePaths(
+      circuitsSourcePaths,
+      this._zkitConfig.compilationSettings,
+    );
+
+    const sourceNames: string[] = await this._getSourceNamesFromSourcePaths(filteredCircuitsSourcePaths);
+
+    const dependencyGraph: DependencyGraph = await this._getDependencyGraph(sourceNames);
+
+    const resolvedFilesToCompile: ResolvedFile[] = this._filterResolvedFiles(
+      dependencyGraph.getResolvedFiles(),
+      sourceNames,
+      true,
+    );
+
+    this._validateResolvedFiles(resolvedFilesToCompile);
+    this._invalidateCacheMissingArtifacts(resolvedFilesToCompile);
+
+    let resolvedFilesWithDependencies: ResolvedFileWithDependencies[] = [];
+
+    for (const file of resolvedFilesToCompile) {
+      resolvedFilesWithDependencies.push({
+        resolvedFile: file,
+        dependencies: dependencyGraph.getTransitiveDependencies(file).map((dep) => dep.dependency),
+      });
+    }
+
+    if (!force) {
+      resolvedFilesWithDependencies = resolvedFilesWithDependencies.filter((file) =>
+        this._needsCompilation(file, compileFlags),
+      );
+    }
+
+    return resolvedFilesWithDependencies;
+  }
+
+  public getCircuitsDirFullPath(): string {
+    return getNormalizedFullPath(this._projectPaths.root, this._zkitConfig.circuitsDir);
+  }
+
+  public getArtifactsDirFullPath(): string {
+    return getNormalizedFullPath(
+      this._projectPaths.root,
+      this._config.artifactsDir ?? this._zkitConfig.compilationSettings.artifactsDir,
+    );
+  }
+
+  public getPtauDirFullPath(): string {
+    const ptauDir = this._config.ptauDir ?? this._zkitConfig.ptauDir;
+
+    if (ptauDir) {
+      return path.isAbsolute(ptauDir) ? ptauDir : getNormalizedFullPath(this._projectPaths.root, ptauDir);
+    } else {
+      return path.join(os.homedir(), ".zkit", ".ptau");
+    }
+  }
+
+  protected _filterSourcePaths(sourcePaths: string[], filterSettings: FileFilterSettings): string[] {
+    const contains = (circuitsRoot: string, pathList: string[], source: any) => {
+      const isSubPath = (parent: string, child: string) => {
+        const parentTokens = parent.split(path.posix.sep).filter((i) => i.length);
+        const childTokens = child.split(path.posix.sep).filter((i) => i.length);
+
+        return parentTokens.every((t, i) => childTokens[i] === t);
+      };
+
+      return pathList.some((p: any) => {
+        return isSubPath(getNormalizedFullPath(circuitsRoot, p), source);
+      });
+    };
+
+    const circuitsRoot = this.getCircuitsDirFullPath();
+
+    return sourcePaths.filter((sourceName: string) => {
+      return (
+        (filterSettings.onlyFiles.length == 0 || contains(circuitsRoot, filterSettings.onlyFiles, sourceName)) &&
+        !contains(circuitsRoot, filterSettings.skipFiles, sourceName)
+      );
+    });
+  }
+
+  protected _filterResolvedFiles(
+    resolvedFiles: ResolvedFile[],
+    sourceNames: string[],
+    withMainComponent: boolean,
+  ): ResolvedFile[] {
+    return resolvedFiles.filter((file: ResolvedFile) => {
+      return (!withMainComponent || this._hasMainComponent(file)) && sourceNames.includes(file.sourceName);
+    });
+  }
+
+  protected async _getSourceNamesFromSourcePaths(sourcePaths: string[]): Promise<string[]> {
+    return Promise.all(sourcePaths.map((p) => localPathToSourceName(this._projectPaths.root, p)));
+  }
+
+  protected async _getDependencyGraph(sourceNames: string[]): Promise<DependencyGraph> {
+    const parser = new Parser();
+    const remappings = this._getRemappings();
+    const resolver = new Resolver(this._projectPaths.root, parser, remappings, this._readFile);
+
+    const resolvedFiles = await Promise.all(sourceNames.map((sn) => resolver.resolveSourceName(sn)));
+
+    return DependencyGraph.createFromResolvedFiles(resolver, resolvedFiles);
+  }
+
+  protected _hasMainComponent(resolvedFile: ResolvedFile): boolean {
+    return new RegExp(MAIN_COMPONENT_REG_EXP).test(resolvedFile.content.rawContent);
+  }
+
+  protected _getRemappings(): Record<string, string> {
+    return {};
+  }
+
+  protected _validateResolvedFiles(resolvedFiles: ResolvedFile[]) {
+    const circuitsNameCount = {} as Record<string, ResolvedFile>;
+
+    resolvedFiles.forEach((file: ResolvedFile) => {
+      const circuitName = path.parse(file.absolutePath).name;
+
+      if (circuitsNameCount[circuitName]) {
+        throw new HardhatZKitError(
+          `Circuit ${file.sourceName} duplicated ${circuitsNameCount[circuitName].sourceName} circuit`,
+        );
+      }
+
+      circuitsNameCount[circuitName] = file;
+    });
+  }
+
+  protected _invalidateCacheMissingArtifacts(resolvedFiles: ResolvedFile[]) {
+    const circuitsDirFullPath = this.getCircuitsDirFullPath();
+    const artifactsDirFullPath = this.getArtifactsDirFullPath();
+
+    for (const file of resolvedFiles) {
+      const cacheEntry = CircomCircuitsCache!.getEntry(file.absolutePath);
+
+      if (cacheEntry === undefined) {
+        continue;
+      }
+
+      if (!fsExtra.existsSync(file.absolutePath.replace(circuitsDirFullPath, artifactsDirFullPath))) {
+        CircomCircuitsCache!.removeEntry(file.absolutePath);
+      }
+    }
+  }
+
+  protected _needsCompilation(
+    resolvedFilesWithDependencies: ResolvedFileWithDependencies,
+    compileFlags: CompileFlags,
+  ): boolean {
+    for (const file of [resolvedFilesWithDependencies.resolvedFile, ...resolvedFilesWithDependencies.dependencies]) {
+      const hasChanged = CircomCircuitsCache!.hasFileChanged(file.absolutePath, file.contentHash, compileFlags);
+
+      if (hasChanged) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+}