Adding new sector (DAO Creator Tools)

dbt_project.yml

@@ -220,6 +220,10 @@ models:
       +schema: labels
       +materialized: view
 
+    daos:
+      +schema: daos
+      +materialized: view
+
     addresses:
       +schema: addresses
       +materialized: table
@@ -524,6 +528,26 @@ seeds:
           buyer: string
           amount_raw: string
           currency_contract: string
+    daos:
+      +enabled: true 
+      +schema: test_data
+      daos_addresses_seed:
+        +column_types:
+          blockchain: string
+          dao_creator_tool: string
+          dao: string
+          dao_wallet_address: string
+          created_date: date 
+      daos_transactions_seed:
+        +column_types:
+          blockchain: string
+          dao_creator_tool: string
+          dao: string
+          dao_wallet_address: string
+          value: float
+          block_date: date
+          tx_type: string
+          tx_hash: string
     curvefi:
       ethereum:
         +enabled: true

models/base_sources/gnosis_base_sources.yml

@@ -78,4 +78,99 @@ sources:
             tests:
               - accepted_values:
                   values: ["factory", "base", "dynamic"]
-          - name: created_at
+          - name: created_at
+
+      - name: traces
+        loaded_at_field: block_time
+        description: "A Gnosis trace is a small atomic action that modify the internal state of the Ethereum Virtual Machine. The three main trace types are call, create, and suicide."
+        columns:
+          - name: block_time
+          - name: block_number
+          - name: value
+          - name: gas
+            description: "Amount of gas consumed by the trace"
+          - name: gas_used
+            description: "Number of gas units used by the trace"
+          - name: block_hash
+          - name: success
+            description: "Whether the trace was completed successfully"
+          - name: tx_index
+            description: "Transaction index"
+          - name: subtraces
+            description: "Number of subtraces (i.e, number of calls at a particular level within a transaction)"
+          - name: error
+            description: "Error log"
+          - name: tx_success
+            description: "Whether the transaction was completed sucessfully"
+          - name: tx_hash
+            description: "Primary key of the transaction"
+          - name: from
+            description: "Wallet address that initiated the trace"
+          - name: to
+            description: "Wallet address that received the trace"
+          - name: trace_address
+            description: "All returned traces, gives the exact location in the call trace"
+          - name: type
+            description: "Type of trace (e.g., call, create, suicide)"
+          - name: address
+            description: "Address of the trace creator"
+          - name: code
+            description: "Raw EVM code for the trace"
+          - name: call_type
+            description: "Hexadecimal representations of the trace's call type"
+          - name: input
+            description: "Input data for the trace"
+          - name: output
+            description: "Output data for the trace"
+          - name: refund_address
+            description: "Refund Address"
+
+      - name: logs
+        loaded_at_field: block_time
+        description: "An Ethereum log can be used to describe an event within a smart contract, like a token transfer or a change of ownership."
+        columns:
+          - name: block_time
+          - name: block_number
+          - name: block_hash
+          - name: contract_address
+            description: "Address of the ethereum smart contract generating the log"
+          - name: topic1
+            description: "Topics are 32-byte (256 bit) “words” that are used to describe what’s going on in an event. The first topic usually consists of the signature of the name of the event that occurred, including the types (uint256, string, etc.) of its parameters."
+          - name: topic2
+            description: "Second topic"
+          - name: topic3
+            description: "Third topic"
+          - name: topic4
+            description: "Fourth topic"
+          - name: data
+            description: "Additional data for the log. Data is not searchable (while topics are), but is a lot cheaper and can include large or complicated data like arrays or strings."
+          - name: tx_hash
+          - name: index
+            description: "Log index"
+          - name: tx_index
+
+  # ERC Transfer Tables
+  - name: erc20_gnosis
+    description: "Transfers events for ERC20 tokens."
+    tables:
+      - name: evt_transfer
+        loaded_at_field: evt_block_time
+        description: "Transfers events for ERC20 tokens."
+        freshness:
+          warn_after: { count: 12, period: hour }
+          error_after: { count: 24, period: hour }
+        columns:
+          - name: contract_address
+            description: "ERC20 token contract address"
+          - name: evt_tx_hash
+            description: "Transaction hash of the event"
+          - name: evt_index
+            description: "Event index"
+          - name: evt_block_time
+            description: "Timestamp for block event time in UTC"
+          - name: evt_block_number
+            description: "Event block number"
+          - name: from
+          - name: to
+          - name: value
+            description: "Amount of ERC20 token transferred"

models/daos/addresses/daos_addresses.sql

@@ -0,0 +1,16 @@
+{{ config(
+    alias = 'addresses',
+    materialized = 'view',
+    file_format = 'delta',
+    post_hook='{{ expose_spells(\'["ethereum", "gnosis"]\',
+                                "sector",
+                                "daos",
+                                \'["henrystats"]\') }}')
+}}
+
+
+SELECT * FROM {{ ref('daos_addresses_ethereum') }}
+
+UNION ALL 
+
+SELECT * FROM {{ ref('daos_addresses_gnosis') }}

models/daos/addresses/daos_addresses_schema.yml

@@ -0,0 +1,31 @@
+version: 2
+
+models:
+  - name: daos_addresses
+    meta:
+      blockchain: ethereum, gnosis
+      sector: daos
+      contributors: henrystats 
+    config:
+      tags: [ 'ethereum','gnosis','daos', 'dao creator tools', 'henrystats' ]
+    description: >
+      wallet addresses of daos created using popular dao creator tools 
+    columns:
+      - &blockchain
+        name: blockchain
+        description: "Blockchain which the DAO is deployed"
+      - &dao_creator_tool
+        name: dao_creator_tool
+        description: "Name of creator tool used to create DAO"
+      - &dao
+        name: dao
+        description: "address of the dao deployed"
+      - &dao_wallet_address
+        name: dao_wallet_address
+        desciption: "wallet address used by DAO to receive & transfer funds"
+      - &created_date
+        name: created_date
+        description: "UTC event block date of DAO creation"
+      - &created_block_time
+        name: created_block_time
+        description: "UTC event block time of DAO creation"

models/daos/addresses/ethereum/daos_addresses_ethereum.sql

@@ -0,0 +1,28 @@
+{{config(alias='addresses_ethereum')}}
+
+WITH 
+
+mapping as (
+        SELECT blockchain, dao_creator_tool, dao, dao_wallet_address, created_block_time, created_date
+        FROM {{ ref('daos_addresses_ethereum_aragon') }}
+
+        UNION ALL 
+
+        SELECT blockchain, dao_creator_tool, dao, dao_wallet_address, created_block_time, created_date
+        FROM {{ ref('daos_addresses_ethereum_daohaus') }}
+
+        UNION ALL 
+
+        SELECT blockchain, dao_creator_tool, dao, dao_wallet_address, created_block_time, created_date
+        FROM {{ ref('daos_addresses_ethereum_zodiac') }}
+
+        UNION ALL 
+
+        SELECT blockchain, dao_creator_tool, dao, dao_wallet_address, created_block_time, created_date
+        FROM {{ ref('daos_addresses_ethereum_syndicate') }}
+
+
+)
+
+SELECT * FROM mapping
+

models/daos/addresses/ethereum/daos_addresses_ethereum_aragon.sql

@@ -0,0 +1,68 @@
+{{ config(
+    alias = 'addresses_ethereum_aragon',
+    partition_by = ['created_date'],
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['created_block_time', 'dao_wallet_address', 'blockchain', 'dao', 'dao_creator_tool']
+    )
+}}
+
+{% set project_start_date = '2018-10-27' %}
+
+WITH -- dune query here  https://dune.com/queries/1435748
+
+aragon_daos as ( -- decoded table for aragon on dune that returns the address of daos deployed on ethereum
+        SELECT 
+            evt_block_time as created_block_time, 
+            date_trunc('day', evt_block_time) as created_date, 
+            dao 
+        FROM {{ source('aragon_ethereum', 'DAOFactory_evt_DeployDAO') }}
+        {% if not is_incremental() %}
+        WHERE evt_block_time >= '{{project_start_date}}'
+        {% endif %}
+        {% if is_incremental() %}
+        WHERE evt_block_time >= date_trunc("day", now() - interval '1 week')
+        {% endif %}
+), 
+
+app_ids (app_id) as ( -- aragon apps that allow daos to manage funds 
+        VALUES 
+            (LOWER('0x9ac98dc5f995bf0211ed589ef022719d1487e5cb2bab505676f0d084c07cf89a')), -- agent 
+            (LOWER('0x701a4fd1f5174d12a0f1d9ad2c88d0ad11ab6aad0ac72b7d9ce621815f8016a9')), -- agent 
+            (LOWER('0xf2e5eb0f21694bf4e28f98a980dfc4d6a568b5b3e593cfe9cedfd0aed59d8148')), -- agent 
+            (LOWER('0xbf8491150dafc5dcaee5b861414dca922de09ccffa344964ae167212e8c673ae')), -- finance 
+            (LOWER('0x5c9918c99c4081ca9459c178381be71d9da40e49e151687da55099c49a4237f1')), -- finance 
+            (LOWER('0xa9efdd08ab8a16b35803b9887d721f0b9cf17df8ff66b9e57f23bbe4ae5f18ba')), -- finance 
+            (LOWER('0x7e852e0fcfce6551c13800f1e7476f982525c2b5277ba14b24339c68416336d1')) -- vault 
+), 
+
+get_aragon_wallets as ( -- this is getting the app address that is deployed for daos and used to manage the apps above 
+        SELECT 
+            contract_address as dao, 
+            CONCAT('0x', SUBSTRING(data, 27, 40)) as dao_wallet_address -- app address 
+        FROM 
+        {{ source('ethereum', 'logs') }}
+        {% if not is_incremental() %}
+        WHERE block_time >= '{{project_start_date}}'
+        {% endif %}
+        {% if is_incremental() %}
+        WHERE block_time >= date_trunc("day", now() - interval '1 week')
+        {% endif %}
+        AND topic1 = LOWER('0xd880e726dced8808d727f02dd0e6fdd3a945b24bfee77e13367bcbe61ddbaf47') -- aragon apps deployment event 
+        AND contract_address IN (SELECT dao FROM aragon_daos)
+        AND CONCAT('0x', SUBSTRING(data, 131, 64)) IN (SELECT app_id FROM app_ids) -- app id 
+)
+
+SELECT 
+    'ethereum' as blockchain, 
+    'aragon' as dao_creator_tool, 
+    ad.dao, 
+    gw.dao_wallet_address, 
+    ad.created_block_time,
+    TRY_CAST(ad.created_date as DATE) as created_date
+FROM 
+aragon_daos ad 
+INNER JOIN 
+get_aragon_wallets gw  -- inner join to get the dao address mapped to the app address
+    ON ad.dao = gw.dao 

models/daos/addresses/ethereum/daos_addresses_ethereum_daohaus.sql

@@ -0,0 +1,92 @@
+{{ config(
+    alias = 'addresses_ethereum_daohaus',
+    partition_by = ['created_date'],
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['created_block_time', 'dao_wallet_address', 'blockchain', 'dao', 'dao_creator_tool']
+    )
+}}
+
+{% set moloch_start_date = '2021-01-25' %}
+
+{% set minion_start_date = '2021-09-17' %}
+
+WITH  -- dune query here - https://dune.com/queries/1433790
+
+get_daohaus_molochs as ( -- molochs are daos and this is getting a list of molochs created through daohaus 
+        SELECT 
+            block_time as created_block_time, 
+            TRY_CAST(date_trunc('day', block_time) as DATE) as created_date, 
+            CONCAT('0x', RIGHT(topic2, 40)) as moloch
+        FROM 
+        {{ source('ethereum', 'logs') }}
+        {% if not is_incremental() %}
+        WHERE block_time >= '{{moloch_start_date}}'
+        {% endif %}
+        {% if is_incremental() %}
+        WHERE block_time >= date_trunc("day", now() - interval '1 week')
+        {% endif %}
+        AND topic1 = '0x099e0b09e056ad33e22e4d35de2e837a30ba249f33d912abb7e1e273bbf9d650' -- summon moloch event which is the event emitted when a moloch is created through daohaus 
+        AND contract_address = '0x38064f40b20347d58b326e767791a6f79cdeddce' -- dao haus moloch v2.1 contract address 
+), 
+
+get_minion_creations as ( -- minions are created by molochs to manage funds (this is a gnosis safe that's controlled with zodiac's reality.eth module)
+        SELECT 
+            CONCAT('0x', RIGHT(topic3, 40)) as moloch,  
+            CONCAT('0x', RIGHT(topic2, 40)) as wallet_address
+        FROM 
+        {{ source('ethereum', 'logs') }}
+        {% if not is_incremental() %}
+        WHERE block_time >= '{{minion_start_date}}'
+        {% endif %}
+        {% if is_incremental() %}
+        WHERE block_time >= date_trunc("day", now() - interval '1 week')
+        {% endif %}
+        AND topic1 = '0xbaefe449c0963ab3bd87eb56115a3f8420fbefae45878f063cc59a6cb99d3ae0' -- summon minion event which is emitted when a minion is created through dao haus 
+        AND contract_address IN ('0x594af060c08eea9f559bc668484e50596bcb2cfb', '0xbc37509a283e2bb67fd151c34e72e826c501e108') -- dao haus minion summoner contract addresses 
+), 
+
+get_daohaus_wallets as (
+        SELECT 
+            gm.created_date, 
+            gm.created_block_time, 
+            gm.moloch as dao, 
+            gm.moloch as dao_wallet, 
+            gc.wallet_address as minion_wallet
+        FROM 
+        get_daohaus_molochs gm 
+        LEFT JOIN -- getting minions mapped to molochs (using a left join since not all molochs have a minion)
+        get_minion_creations gc 
+            ON gm.moloch = gc.moloch
+), 
+
+mapped_wallets as (
+        SELECT 
+            'ethereum' as blockchain, 
+            'dao-haus' as dao_creator_tool, 
+            dao, 
+            dao_wallet as dao_wallet_address, 
+            created_block_time, 
+            created_date
+        FROM 
+        get_daohaus_wallets
+
+        UNION -- molochs are wallet addresses as well so using a union here since there'll be duplicates as i'm unioning the moloch addresses & minion addresses 
+
+        SELECT 
+            'ethereum' as blockchain, 
+            'dao-haus' as dao_creator_tool, 
+            dao, 
+            minion_wallet as dao_wallet_address,
+            created_block_time, 
+            created_date
+        FROM 
+        get_daohaus_wallets
+)
+
+SELECT 
+    DISTINCT(mw.*) -- there are still duplicates so I'm using a distinct to filter for the duplicates 
+FROM 
+mapped_wallets mw 
+WHERE dao_wallet_address IS NOT NULL