feat: add run-observability to zk (#1359)

## What ❔ Add a new parameter --run-observability to zk and zk stack commands that allows to run dockprom stack alongside other containers. The command also provisions grafana with Era dashboards.
matter-labs · Mar 5, 2024 · 2b520f6 · 2b520f6
1 parent bc9c2b9
commit 2b520f6
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 19 deletions.
diff --git a/docs/guides/launch.md b/docs/guides/launch.md
@@ -38,6 +38,22 @@ launch:
 zk up
 ```
 
+### Run observability stack
+
+If you want to run [Dockprom](https://github.com/stefanprodan/dockprom/) stack (Prometheus, Grafana) alongside other
+containers - add `--run-observability` parameter during initialisation.
+
+```
+zk init --run-observability
+```
+
+That will also provision Grafana with
+[era-observability](https://github.com/matter-labs/era-observability/tree/main/dashboards) dashboards. You can then
+access it at `http://127.0.0.1:3000/` under credentials `admin/admin`.
+
+> If you don't see any data displayed on the Grafana dashboards - try setting the timeframe to "Last 30 minutes". You
+> will also have to have `jq` installed on your system.
+
 ## (Re)deploy db and contracts
 
 ```

diff --git a/infrastructure/zk/src/down.ts b/infrastructure/zk/src/down.ts
@@ -5,6 +5,9 @@ export async function down() {
     await utils.spawn('docker compose down -v');
     await utils.spawn('docker compose rm -s -f -v');
     await utils.spawn('docker run --rm -v ./volumes:/volumes postgres:14 bash -c "rm -rf /volumes/*"');
+    // cleaning up dockprom
+    // no need to delete the folder - it's going to be deleted on the next start
+    await utils.spawn('[ -d "./target/dockprom" ] && docker compose -f ./target/dockprom/docker-compose.yml down -v');
 }
 
 export const command = new Command('down').description('stop development containers').action(down);
diff --git a/infrastructure/zk/src/hyperchain_wizard.ts b/infrastructure/zk/src/hyperchain_wizard.ts
@@ -44,8 +44,8 @@ export interface BasePromptOptions {
 }
 
 // An init command that allows configuring and spinning up a new hyperchain network.
-async function initHyperchain() {
-    await announced('Initializing hyperchain creation', setupConfiguration());
+async function initHyperchain(runObservability: boolean) {
+    await announced('Initializing hyperchain creation', setupConfiguration(runObservability));
 
     const deployerPrivateKey = process.env.DEPLOYER_PRIVATE_KEY;
     const governorPrivateKey = process.env.GOVERNOR_PRIVATE_KEY;
@@ -55,6 +55,7 @@ async function initHyperchain() {
     const initArgs: InitArgs = {
         skipSubmodulesCheckout: false,
         skipEnvSetup: true,
+        runObservability: runObservability,
         governorPrivateKeyArgs: ['--private-key', governorPrivateKey],
         deployerL2ContractInput: {
             args: ['--private-key', deployerPrivateKey],
@@ -79,7 +80,7 @@ async function initHyperchain() {
     await announced('Start server', startServer());
 }
 
-async function setupConfiguration() {
+async function setupConfiguration(runObservability: boolean) {
     const CONFIGURE = 'Configure new chain';
     const USE_EXISTING = 'Use existing configuration';
     const questions: BasePromptOptions[] = [
@@ -94,7 +95,7 @@ async function setupConfiguration() {
     const results: any = await enquirer.prompt(questions);
 
     if (results.config === CONFIGURE) {
-        await announced('Setting hyperchain configuration', setHyperchainMetadata());
+        await announced('Setting hyperchain configuration', setHyperchainMetadata(runObservability));
         await announced('Validating information and balances to deploy hyperchain', checkReadinessToDeploy());
     } else {
         const envName = await selectHyperchainConfiguration();
@@ -103,7 +104,7 @@ async function setupConfiguration() {
     }
 }
 
-async function setHyperchainMetadata() {
+async function setHyperchainMetadata(runObservability: boolean) {
     const BASE_NETWORKS = [
         BaseNetwork.LOCALHOST,
         BaseNetwork.LOCALHOST_CUSTOM,
@@ -288,7 +289,7 @@ async function setHyperchainMetadata() {
         feeReceiver = undefined;
         feeReceiverAddress = richWallets[3].address;
 
-        await up();
+        await up(runObservability);
         await announced('Ensuring databases are up', db.wait({ server: true, prover: false }));
     }
 
@@ -801,6 +802,7 @@ async function configDemoHyperchain(cmd: Command) {
     const initArgs: InitArgs = {
         skipSubmodulesCheckout: false,
         skipEnvSetup: cmd.skipEnvSetup,
+        runObservability: false,
         governorPrivateKeyArgs: ['--private-key', governorPrivateKey],
         deployerL2ContractInput: {
             args: ['--private-key', deployerPrivateKey],
@@ -815,7 +817,7 @@ async function configDemoHyperchain(cmd: Command) {
     };
 
     if (!cmd.skipEnvSetup) {
-        await up();
+        await up(initArgs.runObservability);
     }
     await init(initArgs);
 
@@ -865,8 +867,11 @@ export const initHyperchainCommand = new Command('stack')
 
 initHyperchainCommand
     .command('init')
+    .option('--run-observability')
     .description('Wizard for hyperchain creation/configuration')
-    .action(initHyperchain);
+    .action(async (cmd: Command) => {
+        await initHyperchain(cmd.runObservability);
+    });
 initHyperchainCommand
     .command('docker-setup')
     .option('--custom-docker-org <value>', 'Custom organization name for the docker images')

diff --git a/infrastructure/zk/src/init.ts b/infrastructure/zk/src/init.ts
@@ -21,17 +21,22 @@ export async function init(initArgs: InitArgs = DEFAULT_ARGS) {
     const {
         skipSubmodulesCheckout,
         skipEnvSetup,
+        runObservability: runObservability,
         testTokens,
         governorPrivateKeyArgs,
         deployerPrivateKeyArgs,
         deployerL2ContractInput
     } = initArgs;
 
+    if (runObservability) {
+        await announced('Pulling observability repos', setupObservability());
+    }
+
     if (!process.env.CI && !skipEnvSetup) {
         await announced('Pulling images', docker.pull());
-        await announced('Checking environment', checkEnv());
+        await announced('Checking environment', checkEnv(runObservability));
         await announced('Checking git hooks', env.gitHooks());
-        await announced('Setting up containers', up());
+        await announced('Setting up containers', up(runObservability));
     }
     if (!skipSubmodulesCheckout) {
         await announced('Checkout system-contracts submodule', submoduleUpdate());
@@ -69,8 +74,8 @@ export async function init(initArgs: InitArgs = DEFAULT_ARGS) {
 
 // A smaller version of `init` that "resets" the localhost environment, for which `init` was already called before.
 // It does less and runs much faster.
-export async function reinit() {
-    await announced('Setting up containers', up());
+export async function reinit(runObservability: boolean) {
+    await announced('Setting up containers', up(runObservability));
     await announced('Compiling JS packages', run.yarn());
     await announced('Compile l2 contracts', compiler.compileAll());
     await announced('Drop postgres db', db.drop({ server: true, prover: true }));
@@ -124,8 +129,29 @@ export async function submoduleUpdate() {
     await utils.exec('git submodule update');
 }
 
-async function checkEnv() {
+// clone dockprom and zksync-era dashboards
+export async function setupObservability() {
+    // clone dockprom, era-observability repos and export era dashboards to dockprom
+    await utils.spawn(
+        `rm -rf ./target/dockprom && git clone git@github.com:stefanprodan/dockprom.git ./target/dockprom \
+            && rm -rf ./target/era-observability && git clone git@github.com:matter-labs/era-observability.git ./target/era-observability \
+            && cp ./target/era-observability/dashboards/* ./target/dockprom/grafana/provisioning/dashboards
+        `
+    );
+    // add scrape configuration to prometheus
+    await utils.spawn(
+        `yq eval '.scrape_configs += [{"job_name": "zksync", "scrape_interval": "5s", "honor_labels": true, "static_configs": [{"targets": ["host.docker.internal:3312"]}]}]' \
+            -i ./target/dockprom/prometheus/prometheus.yml
+        `
+    );
+}
+
+async function checkEnv(runObservability: boolean) {
     const tools = ['node', 'yarn', 'docker', 'cargo'];
+    if (runObservability) {
+        tools.push('yq');
+    }
+
     for (const tool of tools) {
         await utils.exec(`which ${tool}`);
     }
@@ -140,6 +166,7 @@ async function checkEnv() {
 export interface InitArgs {
     skipSubmodulesCheckout: boolean;
     skipEnvSetup: boolean;
+    runObservability: boolean;
     governorPrivateKeyArgs: any[];
     deployerPrivateKeyArgs: any[];
     deployerL2ContractInput: {
@@ -156,6 +183,7 @@ export interface InitArgs {
 const DEFAULT_ARGS: InitArgs = {
     skipSubmodulesCheckout: false,
     skipEnvSetup: false,
+    runObservability: false,
     governorPrivateKeyArgs: [],
     deployerPrivateKeyArgs: [],
     deployerL2ContractInput: { args: [], includePaymaster: true, includeL2WETH: true },
@@ -165,11 +193,13 @@ const DEFAULT_ARGS: InitArgs = {
 export const initCommand = new Command('init')
     .option('--skip-submodules-checkout')
     .option('--skip-env-setup')
+    .option('--run-observability')
     .description('perform zksync network initialization for development')
     .action(async (cmd: Command) => {
         const initArgs: InitArgs = {
             skipSubmodulesCheckout: cmd.skipSubmodulesCheckout,
             skipEnvSetup: cmd.skipEnvSetup,
+            runObservability: cmd.runObservability,
             governorPrivateKeyArgs: [],
             deployerL2ContractInput: { args: [], includePaymaster: true, includeL2WETH: true },
             testTokens: { deploy: true, args: [] },
@@ -179,7 +209,10 @@ export const initCommand = new Command('init')
     });
 export const reinitCommand = new Command('reinit')
     .description('"reinitializes" network. Runs faster than `init`, but requires `init` to be executed prior')
-    .action(reinit);
+    .option('--run-observability')
+    .action(async (cmd) => {
+        await reinit(cmd.runObservability);
+    });
 export const lightweightInitCommand = new Command('lightweight-init')
     .description('perform lightweight zksync network initialization for development')
     .action(lightweightInit);
diff --git a/infrastructure/zk/src/up.ts b/infrastructure/zk/src/up.ts
@@ -30,21 +30,27 @@ function createVolumes() {
     );
 }
 
-export async function up(composeFile?: string) {
+export async function up(runObservability: boolean, composeFile?: string) {
     await down();
     // There is some race on the filesystem, so backoff here
     await utils.sleep(1);
     createVolumes();
+
     if (composeFile) {
-        await utils.spawn(`docker compose -f ${composeFile} up -d geth postgres`);
+        await utils.spawn(`docker compose -f ${composeFile} up -d`);
     } else {
         await utils.spawn('docker compose up -d');
     }
+
+    if (runObservability) {
+        await utils.spawn(`docker compose -f ./target/dockprom/docker-compose.yml up -d`);
+    }
 }
 
 export const command = new Command('up')
     .description('start development containers')
     .option('--docker-file <dockerFile>', 'path to a custom docker file')
+    .option('--run-observability', 'whether to run observability stack')
     .action(async (cmd) => {
-        await up(cmd.dockerFile);
+        await up(cmd.runObservability, cmd.dockerFile);
     });
diff --git a/prover/prover_fri/src/gpu_prover_job_processor.rs b/prover/prover_fri/src/gpu_prover_job_processor.rs
@@ -3,8 +3,9 @@ pub mod gpu_prover {
     use std::{collections::HashMap, sync::Arc, time::Instant};
 
     use anyhow::Context as _;
-    use shivini::gpu_proof_config::GpuProofConfig;
-    use shivini::{gpu_prove_from_external_witness_data, ProverContext};
+    use shivini::{
+        gpu_proof_config::GpuProofConfig, gpu_prove_from_external_witness_data, ProverContext,
+    };
     use tokio::task::JoinHandle;
     use zksync_config::configs::{fri_prover_group::FriProverGroupConfig, FriProverConfig};
     use zksync_dal::{fri_prover_dal::types::SocketAddress, ConnectionPool};