More ANY_EVM fixes (#1566)

* Also include any_evm in filtered events * More fixes for any_evm * Other updates to fix any_evm handling * workflow updates
opensource-observer · May 31, 2024 · 83e4751 · 83e4751
1 parent e6e4c23
commit 83e4751
Show file tree

Hide file tree

Showing 9 changed files with 158 additions and 290 deletions.
diff --git a/.github/workflows/warehouse-publish-docker-containers.yml b/.github/workflows/warehouse-publish-docker-containers.yml
@@ -1,4 +1,4 @@
-name: warehouse-publish-cloudquery-plugins
+name: warehouse-publish-docker-containers
 env:
   X_GITHUB_GRAPHQL_API: ${{ vars.X_GITHUB_GRAPHQL_API }}
   X_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -18,6 +18,8 @@ jobs:
 
     permissions:
       packages: write
+      contents: 'read'
+      id-token: 'write'
 
     steps:
       - name: Checkout code
@@ -37,4 +39,25 @@ jobs:
 
       - name: Package and publish other docker containers
         run: bash .github/scripts/publish-docker-containers.sh
-
+
+      # The remaining steps currently kill a pods in our kubernetes
+      # because we haven't dealt with using strict versions on the cluster yet.
+      # This ensures the pods are up to date. This is a hack for now.
+      - uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_JSON }}'
+          create_credentials_file: true
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v2'
+        with:
+          version: '>= 363.0.0'
+
+      # Get the GKE credentials so we can deploy to the cluster
+      - uses: google-github-actions/get-gke-credentials@db150f2cc60d1716e61922b832eae71d2a45938f
+        with:
+          cluster_name: ${{ secrets.GKE_CLUSTER_NAME }}
+          location: ${{ secrets.GKE_CLUSTER_REGION }}
+
+      - name: Delete pod for dagster
+        run: kubectl delete pods --namespace production-dagster -l app.kubernetes.io/instance=production-dagster,component=user-deployments
diff --git a/docker/images/dagster-dask/Dockerfile b/docker/images/dagster-dask/Dockerfile
@@ -1,5 +1,7 @@
 FROM ubuntu:jammy
 
+ARG GCLOUD_VERSION=478.0.0
+
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
     apt-get install -y software-properties-common && \
@@ -10,6 +12,12 @@ RUN apt-get install -y curl && \
     curl -o get-pip.py https://bootstrap.pypa.io/get-pip.py && \
     python3.12 get-pip.py
 RUN pip3.12 install poetry
+RUN curl -o gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
+    tar xvf gcloud.tar.gz && \
+    bash ./google-cloud-sdk/install.sh && \
+    rm gcloud.tar.gz && true
+ENV PATH $PATH:/google-cloud-sdk/bin
+
 
 ENV DAGSTER_DBT_PARSE_PROJECT_ON_LOAD=1
 ENV DAGSTER_DBT_GENERATE_AND_AUTH_GCP=1

diff --git a/warehouse/dbt/macros/models/contract_invocation_events_with_l1.sql b/warehouse/dbt/macros/models/contract_invocation_events_with_l1.sql
@@ -4,35 +4,23 @@
 
 with blockchain_artifacts as (
   select
-    artifact_source_id,
-    MAX_BY(artifact_type, artifact_rank) as artifact_type
-  from (
-    select
-      LOWER(artifact_source_id) as artifact_source_id,
-      artifact_type,
-      case
-        when artifact_type = 'SAFE' then 5
-        when artifact_type = 'FACTORY' then 4
-        when artifact_type = 'CONTRACT' then 3
-        when artifact_type = 'DEPLOYER' then 2
-        when artifact_type = 'EOA' then 1
-        else 0
-      end as artifact_rank
-    from {{ ref('int_all_artifacts') }}
-    where artifact_source in ("{{ upper_network_name }}", "ANY_EVM")
-  )
-  group by artifact_source_id
+    artifact_id,
+    LOWER(artifact_source_id) as artifact_source_id,
+  from {{ ref('int_all_artifacts') }}
+  where UPPER(artifact_source) in ("{{ upper_network_name }}", "ANY_EVM")
 ),
 
 all_transactions as (
   select -- noqa: ST06
     TIMESTAMP_TRUNC(transactions.block_timestamp, day) as `time`,
-    LOWER(transactions.to_address) as to_name,
-    COALESCE(to_artifacts.artifact_type, "CONTRACT") as to_type,
-    LOWER(transactions.to_address) as to_source_id,
-    LOWER(transactions.from_address) as from_name,
-    COALESCE(from_artifacts.artifact_type, "EOA") as from_type,
-    LOWER(transactions.from_address) as from_source_id,
+    COALESCE(to_artifacts.artifact_id, {{ oso_id("'%s'" % upper_network_name, "transactions.to_address") }}) as to_artifact_id,
+    LOWER(transactions.to_address) as to_artifact_name,
+    "CONTRACT" as to_artifact_type,
+    LOWER(transactions.to_address) as to_artifact_source_id,
+    COALESCE(to_artifacts.artifact_id, {{ oso_id("'%s'" % upper_network_name, "transactions.from_address") }}) as from_artifact_id,
+    LOWER(transactions.from_address) as from_artifact_name,
+    "EOA" as from_artifact_type,
+    LOWER(transactions.from_address) as from_artifact_source_id,
     transactions.receipt_status,
     (
       transactions.receipt_gas_used
@@ -51,72 +39,74 @@ all_transactions as (
 contract_invocations as (
   select
     time,
-    to_name,
-    to_type,
-    to_source_id,
-    from_name,
-    from_type,
-    from_source_id,
+    to_artifact_id,
+    to_artifact_name,
+    to_artifact_type,
+    to_artifact_source_id,
+    from_artifact_id,
+    from_artifact_name,
+    from_artifact_type,
+    from_artifact_source_id,
     "{{ upper_network_name }}" as event_source,
-    "{{ lower_network_name }}" as to_namespace,
-    "{{ lower_network_name }}" as from_namespace,    
     SUM(l2_gas_fee) as total_l2_gas_used,
     COUNT(*) as total_count,
     SUM(case when receipt_status = 1 then 1 else 0 end) as success_count
   from all_transactions
   group by
     time,
-    to_name,
-    to_type,
-    to_source_id,
-    from_name,
-    from_type,
-    from_source_id
+    to_artifact_id,
+    to_artifact_name,
+    to_artifact_type,
+    to_artifact_source_id,
+    from_artifact_id,
+    from_artifact_name,
+    from_artifact_type,
+    from_artifact_source_id
 ),
 
 all_events as (
   select 
     time,    
     'CONTRACT_INVOCATION_DAILY_L2_GAS_USED' as event_type,
     event_source,
-    to_name,
-    to_namespace,
-    to_type,
-    to_source_id,
-    from_name,
-    from_namespace,
-    from_type,
-    from_source_id,
+    to_artifact_id,
+    to_artifact_name,
+    to_artifact_type,
+    to_artifact_source_id,
+    from_artifact_id,
+    from_artifact_name,
+    from_artifact_type,
+    from_artifact_source_id,
     total_l2_gas_used as amount
   from contract_invocations
   union all
   select 
     time,
     'CONTRACT_INVOCATION_DAILY_COUNT' as event_type,
     event_source,
-    to_name,
-    to_namespace,
-    to_type,
-    to_source_id,
-    from_name,
-    from_namespace,
-    from_type,
-    from_source_id,
+    to_artifact_id,
+    to_artifact_name,
+    to_artifact_type,
+    to_artifact_source_id,
+    from_artifact_id,
+    from_artifact_name,
+    from_artifact_type,
+    from_artifact_source_id,
     total_count as amount
   from contract_invocations
   union all
   select 
     time,
     'CONTRACT_INVOCATION_SUCCESS_DAILY_COUNT' as event_type,
     event_source,
-    to_name,
-    to_namespace,
-    to_type,
-    to_source_id,
-    from_name,
-    from_namespace,
-    from_type,
-    from_source_id,
+    to_artifact_id,
+    to_artifact_name,
+    to_artifact_type,
+    to_artifact_source_id,
+    from_artifact_id,
+    from_artifact_name,
+    from_artifact_type,
+    from_artifact_source_id,
     success_count as amount
   from contract_invocations
 )
@@ -125,15 +115,17 @@ select
   time,
   event_type,  
   event_source,
-  to_name,
-  to_namespace,
-  to_type,
-  to_source_id,
-  from_name,
-  from_namespace,
-  from_type,
-  from_source_id,
+  to_artifact_id,
+  to_artifact_name,
+  "{{ lower_network_name }}" as to_artifact_namespace,
+  to_artifact_type,
+  to_artifact_source_id,
+  from_artifact_id,
+  from_artifact_name,
+  "{{ lower_network_name }}" as from_artifact_namespace,
+  from_artifact_type,
+  from_artifact_source_id,
   amount,
-  {{ oso_id('event_source', 'time', 'to_name', 'from_name') }} as event_source_id
+  {{ oso_id('event_source', 'time', 'to_artifact_source_id', 'from_artifact_source_id') }} as event_source_id
 from all_events
 {% endmacro %}
diff --git a/warehouse/dbt/macros/models/filtered_blockchain_events.sql b/warehouse/dbt/macros/models/filtered_blockchain_events.sql
@@ -2,7 +2,7 @@
 with known_addresses as (
   select distinct `artifact_source_id` as `address`
   from {{ ref("int_all_artifacts") }} 
-  where LOWER(artifact_source) = LOWER('{{ artifact_source }}')
+  where UPPER(artifact_source) in (UPPER('{{ artifact_source }}'), 'ANY_EVM')
 ), known_to as (
 select events.* 
 from {{ oso_source(source_name, source_table)}} as events