fix: Fixing local node

.dockerignore

@@ -39,3 +39,8 @@ contracts/.git
 !bellman-cuda
 !prover/vk_setup_data_generator_server_fri/data/
 !.github/release-please/manifest.json
+
+!etc/env/file_based
+!etc/env/dev.toml
+!etc/env/consensus_secrets.yaml
+!etc/env/consensus_config.yaml

.github/workflows/build-local-node-docker.yml

@@ -56,16 +56,13 @@ jobs:
       - name: init
         run: |
           ci_run git config --global --add safe.directory /usr/src/zksync
-          ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
 
           ci_run zk
-          ci_run zk run yarn
           ci_run cp etc/tokens/{test,localhost}.json
           ci_run zk compiler all
           ci_run zk contract build
-          ci_run zk f yarn run l2-contracts build
 
       - name: update-image
         run: |

docker/local-node/Dockerfile

@@ -1,35 +1,37 @@
 # Image is always built from the server image to reuse the common parts
 # This image is expected to be built locally beforehand (implemented in the `zk` tool)
-FROM matterlabs/server-v2:latest2.0
+ARG BASE_VERSION=latest2.0
+FROM matterlabs/server-v2:${BASE_VERSION}
 
 WORKDIR /
 
 # Install required dependencies
-RUN apt-get update; apt-get install -y make bash git openssl libssl-dev gcc g++ curl pkg-config software-properties-common jq wget
+RUN apt-get update; apt-get install -y make bash git openssl libssl-dev gcc g++ curl pkg-config software-properties-common jq wget vim-tiny
 RUN apt-get install -y curl gnupg libpq5 ca-certificates postgresql-client && rm -rf /var/lib/apt/lists/*
 
 # Install node and yarn
 ENV NODE_MAJOR=18
 RUN mkdir -p /etc/apt/keyrings && \
     wget -c -O - https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
     echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && apt-get install nodejs npm -y && \
-    npm install -g yarn && npm install -g cspell && npm install -g markdown-link-check
+    apt-get update && apt-get install nodejs -y && \
+    npm install -g yarn
 
-# Copy compiler (both solc and zksolc) binaries
-# Obtain `solc` 0.8.12.
-RUN wget https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/solc-linux-amd64-v0.8.12%2Bcommit.f00d7308 \
-    && mv solc-linux-amd64-v0.8.12+commit.f00d7308 /usr/bin/solc \
-    && chmod +x /usr/bin/solc
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
+    rustup install stable
+
+RUN cargo install sqlx-cli --version 0.7.3
 
 # Copy required packages while preserving the folders structure from the repo
 # It's required because these packages use relative paths to the SDK
 # Copy `zk` tool
 COPY infrastructure/zk /infrastructure/zk
 # Copy `local-setup-preparation` tool
 COPY infrastructure/local-setup-preparation /infrastructure/local-setup-preparation
-# Copy migrations
-COPY core/lib/dal/migrations /migrations
 # Copy dev configs
 COPY etc/env /etc/env
 # Copy test configs (required to list rich accounts)
@@ -55,6 +57,16 @@ RUN cd /contracts/l1-contracts && yarn && cd /
 # Same for L2 contracts
 RUN cd /contracts/l2-contracts && yarn && cd /
 
+# Copy the ZK tool binary
+COPY bin/zk /bin/zk
+# Copy package json (which gives us yarn workspace - and makes commands more similar to what we normally run)
+COPY package.json /
+
+# Copy DAL - needed to setup database schemas.
+COPY core/lib/dal core/lib/dal
+COPY prover/prover_dal prover/prover_dal
+
+
 # setup entrypoint script
 COPY ./docker/local-node/entrypoint.sh /usr/bin/
 ENTRYPOINT ["entrypoint.sh"]

docker/local-node/README.md

@@ -0,0 +1,33 @@
+# local-node docker
+
+This docker container is used for 'local-node' - the fast way to bring up the fully working system (similar to what you
+get by running `zk init`).
+
+You can find more instructions (and docker compose files) in [https://github.com/matter-labs/local-setup]
+
+This directory is more focused on 'creating' the image rather than using it.
+
+## Testing & debugging
+
+To build the node locally run:
+
+```shell
+zk docker build local-node --custom-tag "my_custom_local_tag"
+```
+
+Then you can quickly test it locally (assuming that you already have a postgres and reth docker running):
+
+```shell
+docker run --network host --env-file=docker-env.list --entrypoint /bin/bash -it --name my_local_zksync matterlabs/local-node:my_custom_local_tag
+```
+
+The `docker-env.list` should contain environment variables:
+
+```
+DATABASE_PROVER_URL=postgres://postgres:notsecurepassword@localhost/prover_local
+DATABASE_URL=postgres://postgres:notsecurepassword@localhost/zksync_local
+ETH_CLIENT_WEB3_URL=http://127.0.0.1:8545
+```
+
+The command above will start it in the 'bash' mode - where you can look around and modify things. If you want to start
+it with the default entrypoint - simply remove the '--entrypoint /bin/bash' from the command above.

docker/local-node/entrypoint.sh

@@ -1,46 +1,93 @@
 #!/bin/bash
 set -ea
 
+# These 3 env variables must be provided.
+if [ -z "$DATABASE_URL" ]; then
+  echo "ERROR: DATABASE_URL is not set."
+  exit 1
+fi
+
+if [ -z "$DATABASE_PROVER_URL" ]; then
+  echo "ERROR: DATABASE_PROVER_URL is not set."
+  exit 1
+fi
+
+if [ -z "$ETH_CLIENT_WEB3_URL" ]; then
+  echo "ERROR: ETH_CLIENT_WEB3_URL is not set."
+  exit 1
+fi
+
+# Updates the value in the .toml config.
+update_config() {
+    # Assigning arguments to readable variable names
+    local file="$1"
+    local parameter="$2"
+    local new_value="$3"
+    local pattern="^${parameter} =.*$"
+
+    # Check if the parameter exists in the file
+    if grep -q "$pattern" "$file"; then
+        # The parameter exists, so replace its value
+        sed -i "s!$pattern!${parameter} =\"${new_value}\"!" "$file"
+        echo "Update successful for $parameter in $file."
+    else
+        # The parameter does not exist in the file, output error message and return non-zero status
+        echo "Error: '$parameter' not found in $file."
+        return 1  # Return with an error status
+    fi
+}
+
+
 # wait till db service is ready
 until psql ${DATABASE_URL%/*} -c '\q'; do
   >&2 echo "Postgres is unavailable - sleeping"
   sleep 5
 done
 
-# ensure database initialization
-if ! psql $DATABASE_URL -c '\q' 2>/dev/null;
-then
-    echo "Initialing local environment"
-    psql ${DATABASE_URL%/*} -c "create database ${DATABASE_URL##*/}"
-    find /migrations -name "*up.sql" | sort | xargs printf -- ' -f %s' | xargs -t psql $DATABASE_URL
 
-    cd /infrastructure/zk
-    # Compile configs
-    yarn start config compile
+# Normally, the /etc/env and /var/lib/zksync/data should be mapped to volumes
+# so that they are persisted between docker restarts - which would allow even faster starts.
+
+# We use the existance of this init file to decide whether to restart or not.
+INIT_FILE="/var/lib/zksync/data/INIT_COMPLETED.remove_to_reset"
 
-    # Override values for database URL and eth client in the toml config files
-    # so they will be taken into account
-    sed -i 's!^database_url=.*$!database_url="'"$DATABASE_URL"'"!' /etc/env/base/private.toml
-    sed -i 's!^web3_url=.*$!web3_url="'"$ETH_CLIENT_WEB3_URL"'"!' /etc/env/base/eth_client.toml
-    sed -i 's!^path=.*$!path="/var/lib/zksync/data"!' /etc/env/base/database.toml
-    sed -i 's!^state_keeper_db_path=.*$!state_keeper_db_path="/var/lib/zksync/data/state_keeper"!' /etc/env/base/database.toml
-    sed -i 's!^merkle_tree_backup_path=.*$!merkle_tree_backup_path="/var/lib/zksync/data/backups"!' /etc/env/base/database.toml
+if [ -f "$INIT_FILE" ]; then
+    echo "Initialization was done in the past - simply starting server"
+else
+    echo "Initialing local environment"
 
-    # Switch zksolc compiler source from docker to binary
-    sed -i "s!'docker'!'binary'!" /contracts/l2-contracts/hardhat.config.ts
+    mkdir -p /var/lib/zksync/data
 
-    # Compile configs again (with changed values)
-    yarn start config compile
+    update_config "/etc/env/base/private.toml" "database_url" "$DATABASE_URL"
+    update_config "/etc/env/base/private.toml" "database_prover_url" "$DATABASE_PROVER_URL"
+    update_config "/etc/env/base/eth_client.toml" "web3_url" "$ETH_CLIENT_WEB3_URL"
+    # Put database in a special /var/lib directory so that it is persisted between docker runs.
+    update_config "/etc/env/base/database.toml" "path" "/var/lib/zksync/data"
+    update_config "/etc/env/base/database.toml" "state_keeper_db_path" "/var/lib/zksync/data/state_keeper"
+    update_config "/etc/env/base/database.toml" "backup_path" "/var/lib/zksync/data/backups"
 
+    zk config compile
+
+    zk db reset
+
     # Perform initialization
-    yarn start lightweight-init
-    yarn start f yarn --cwd /infrastructure/local-setup-preparation start
 
-    # Return to the root directory
-    cd /
+    zk contract deploy --only-verifier
+    zk f zksync_server --genesis
+    zk run deploy-erc20 dev # (created etc/tokens/localhost)
+    zk contract deploy # (deploys rest of stuff)
+
+    zk contract initialize-validator 
+
+    zk contract deploy-l2
+    zk contract initialize-governance
+
+    zk f yarn --cwd /infrastructure/local-setup-preparation start
+
+    # Create init file.
+    echo "System initialized. Please remove this file if you want to reset the system" > $INIT_FILE
+
 fi
 
 # start server
-source /etc/env/dev.env
-source /etc/env/.init.env
-exec zksync_server
+zk f zksync_server

infrastructure/local-setup-preparation/src/index.ts

@@ -31,7 +31,8 @@ async function depositWithRichAccounts() {
         const contract = new ethers.Contract(process.env.CONTRACTS_DIAMOND_PROXY_ADDR, utils.ZKSYNC_MAIN_ABI, wallet);
 
         const overrides = {
-            value: AMOUNT_TO_DEPOSIT.add(expectedCost)
+            // TODO(EVM-565): expected cost calculation seems to be off, understand why and then remove the second add.
+            value: AMOUNT_TO_DEPOSIT.add(expectedCost).add(expectedCost)
         };
 
         const balance = await wallet.getBalance();

infrastructure/zk/src/contract.ts

@@ -35,30 +35,18 @@ function updateContractsEnv(deployLog: String, envVars: Array<string>) {
 
 export async function initializeValidator(args: any[] = []) {
     await utils.confirmAction();
-
-    const isLocalSetup = process.env.ZKSYNC_LOCAL_SETUP;
-    const baseCommandL1 = isLocalSetup ? `yarn --cwd /contracts/l1-contracts` : `yarn l1-contracts`;
-
-    await utils.spawn(`${baseCommandL1} initialize-validator ${args.join(' ')} | tee initializeValidator.log`);
+    await utils.spawn(`yarn l1-contracts initialize-validator ${args.join(' ')} | tee initializeValidator.log`);
 }
 
 export async function initializeGovernance(args: any[] = []) {
     await utils.confirmAction();
-
-    const isLocalSetup = process.env.ZKSYNC_LOCAL_SETUP;
-    const baseCommandL1 = isLocalSetup ? `yarn --cwd /contracts/l1-contracts` : `yarn l1-contracts`;
-
-    await utils.spawn(`${baseCommandL1} initialize-governance ${args.join(' ')} | tee initializeGovernance.log`);
+    await utils.spawn(`yarn l1-contracts initialize-governance ${args.join(' ')} | tee initializeGovernance.log`);
 }
 
 export async function initializeWethToken(args: any[] = []) {
     await utils.confirmAction();
-
-    const isLocalSetup = process.env.ZKSYNC_LOCAL_SETUP;
-    const baseCommandL1 = isLocalSetup ? `yarn --cwd /contracts/l1-contracts` : `yarn l1-contracts`;
-
     await utils.spawn(
-        `${baseCommandL1} initialize-l2-weth-token instant-call ${args.join(' ')} | tee initializeWeth.log`
+        `yarn l1-contracts initialize-l2-weth-token instant-call ${args.join(' ')} | tee initializeWeth.log`
     );
 }
 
@@ -67,25 +55,22 @@ export async function deployL2(args: any[] = [], includePaymaster?: boolean, inc
 
     const isLocalSetup = process.env.ZKSYNC_LOCAL_SETUP;
 
-    // In the localhost setup scenario we don't have the workspace,
-    // so we have to `--cwd` into the required directory.
-    const baseCommandL2 = isLocalSetup ? `yarn --cwd /contracts/l2-contracts` : `yarn l2-contracts`;
-    const baseCommandL1 = isLocalSetup ? `yarn --cwd /contracts/l1-contracts` : `yarn l1-contracts`;
-
     // Skip compilation for local setup, since we already copied artifacts into the container.
-    await utils.spawn(`${baseCommandL2} build`);
+    if (!isLocalSetup) {
+        await utils.spawn(`yarn l2-contracts build`);
+    }
 
-    await utils.spawn(`${baseCommandL1} initialize-bridges ${args.join(' ')} | tee deployL2.log`);
+    await utils.spawn(`yarn l1-contracts initialize-bridges ${args.join(' ')} | tee deployL2.log`);
 
     if (includePaymaster) {
-        await utils.spawn(`${baseCommandL2} deploy-testnet-paymaster ${args.join(' ')} | tee -a deployL2.log`);
+        await utils.spawn(`yarn l2-contracts deploy-testnet-paymaster ${args.join(' ')} | tee -a deployL2.log`);
     }
 
     if (includeWETH) {
-        await utils.spawn(`${baseCommandL2} deploy-l2-weth ${args.join(' ')} | tee -a deployL2.log`);
+        await utils.spawn(`yarn l2-contracts deploy-l2-weth ${args.join(' ')} | tee -a deployL2.log`);
     }
 
-    await utils.spawn(`${baseCommandL2} deploy-force-deploy-upgrader ${args.join(' ')} | tee -a deployL2.log`);
+    await utils.spawn(`yarn l2-contracts deploy-force-deploy-upgrader ${args.join(' ')} | tee -a deployL2.log`);
 
     const l2DeployLog = fs.readFileSync('deployL2.log').toString();
     const l2DeploymentEnvVars = [
@@ -98,7 +83,7 @@ export async function deployL2(args: any[] = [], includePaymaster?: boolean, inc
     updateContractsEnv(l2DeployLog, l2DeploymentEnvVars);
 
     if (includeWETH) {
-        await utils.spawn(`${baseCommandL1} initialize-weth-bridges ${args.join(' ')} | tee -a deployL1.log`);
+        await utils.spawn(`yarn l1-contracts initialize-weth-bridges ${args.join(' ')} | tee -a deployL1.log`);
     }
 
     const l1DeployLog = fs.readFileSync('deployL1.log').toString();
@@ -108,12 +93,7 @@ export async function deployL2(args: any[] = [], includePaymaster?: boolean, inc
 
 export async function deployL1(args: any[]) {
     await utils.confirmAction();
-
-    // In the localhost setup scenario we don't have the workspace,
-    // so we have to `--cwd` into the required directory.
-    const baseCommand = process.env.ZKSYNC_LOCAL_SETUP ? `yarn --cwd /contracts/l1-contracts` : `yarn l1-contracts`;
-
-    await utils.spawn(`${baseCommand} deploy-no-build ${args.join(' ')} | tee deployL1.log`);
+    await utils.spawn(`yarn l1-contracts deploy-no-build ${args.join(' ')} | tee deployL1.log`);
     const deployLog = fs.readFileSync('deployL1.log').toString();
     const envVars = [
         'CONTRACTS_CREATE2_FACTORY_ADDR',
@@ -179,5 +159,19 @@ command
     .action(redeployL1);
 command.command('deploy [deploy-opts...]').allowUnknownOption(true).description('deploy contracts').action(deployL1);
 command.command('build').description('build contracts').action(build);
-command.command('initialize-validator').description('initialize validator').action(initializeValidator);
+command
+    .command('initialize-validator [init-opts...]')
+    .allowUnknownOption(true)
+    .description('initialize validator')
+    .action(initializeValidator);
 command.command('verify').description('verify L1 contracts').action(verifyL1Contracts);
+command
+    .command('deploy-l2 [deploy-opts...]')
+    .allowUnknownOption(true)
+    .description('deploy l2 contracts')
+    .action(deployL2);
+command
+    .command('initialize-governance [gov-opts...]')
+    .allowUnknownOption(true)
+    .description('initialize governance')
+    .action(initializeGovernance);

infrastructure/zk/src/database.ts

@@ -130,7 +130,10 @@ export async function setupForDal(dalPath: DalPath, dbUrl: string) {
     }
     await utils.spawn(`cargo sqlx database create --database-url ${dbUrl}`);
     await utils.spawn(`cargo sqlx migrate run --database-url ${dbUrl}`);
-    if (dbUrl.startsWith(localDbUrl)) {
+    const isLocalSetup = process.env.ZKSYNC_LOCAL_SETUP;
+
+    if (dbUrl.startsWith(localDbUrl) && !isLocalSetup) {
+        // Dont't do this preparation for local (docker) setup - as it requires full cargo compilation.
         await utils.spawn(
             `cargo sqlx prepare --check --database-url ${dbUrl} -- --tests || cargo sqlx prepare --database-url ${dbUrl} -- --tests`
         );