Solana subproject (#6197)

* Move solana to its own subproject

* Add Solana to GH actions

* Move seeds

* move seed

.github/workflows/commit_manifest.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
     strategy:
       matrix:
-        project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex']
+        project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex', 'solana']
       max-parallel: 1
 
     steps:

.github/workflows/solana.yml

@@ -0,0 +1,19 @@
+name: DBT Solana sector run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dex/**
+      - .github/workflows/solana.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'solana'

.github/workflows/spellbook.yml

@@ -10,6 +10,7 @@ on:
       - 'tokens/**'
       - 'daily_spellbook/**'
       - 'nft/**'
+      - 'solana/**'
 
 concurrency:
   group:  ${{ github.workflow }}-${{ github.ref }}

dbt_project.yml

@@ -110,14 +110,6 @@ models:
       ethereum:
         +schema: addresses_ethereum
 
-    tokens:
-      +schema: tokens
-      +materialized: table
-      +file_format: delta
-
-    solana_utils:
-      +schema: solana_utils
-
     transfers:
       +schema: transfers
       ethereum:

solana/README.md

@@ -0,0 +1,3 @@
+## Solana subproject
+
+This is a DBT subproject for the main lineages of Solana data.

solana/dbt_project.yml

@@ -0,0 +1,67 @@
+#Welcome to your Spellbook!
+
+name: "solana"
+version: "1.0.0"
+config-version: 2
+
+quoting:
+  database: false
+  schema: false
+  identifier: false
+
+# This setting configures which "profile" dbt uses for this project.
+profile: "spellbook-local"
+
+vars:
+  DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}"
+  DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}"
+  DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'day') }}"
+
+# These configurations specify where dbt should look for different types of files.
+# You don't need to change these!
+model-paths: ["models", "../sources"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros", "../macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target" # directory which will store compiled SQL files
+clean-targets: # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+# Configuring tests
+# Full documentation: https://docs.getdbt.com/reference/test-configs
+tests:
+  solana:
+    +store_failures: true  # store failures for all tests
+
+# Configuring seeds
+# Full documentation: https://docs.getdbt.com/reference/seed-configs
+# For configuring individual seeds (e.g. overriding column types) we recommend a yml file nested in the same folder as the seed
+seeds:
+  solana:
+    +schema: test_data
+
+# defining search order of macro invokation
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['trino_utils', 'dbt_utils']
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+models:
+  solana:
+    +post-hook:
+      - sql: "{{ set_trino_session_property(is_materialized(model), 'writer_scaling_min_data_processed', model.config.get('writer_min_size', '500MB')) }}"
+        transaction: true
+      - sql: "{{ set_trino_session_property(is_materialized(model), 'task_scale_writers_enabled', false) }}"
+        transaction: true
+      - sql: "{{ optimize_spell(this, model.config.materialized) }}"
+        transaction: true
+      - sql: "{{ mark_as_spell(this, model.config.materialized) }}"
+        transaction: true
+    +materialized: view
+    +schema: no_schema    # this should be overriden in model specific configs
+    +view_security: invoker

models/jupiter/solana/jupiter_solana_aggregator_swaps.sql → solana/models/_sector/dex/jupiter/jupiter_solana_aggregator_swaps.sql

@@ -15,7 +15,7 @@
                                     \'["ilemi"]\') }}')
 }}
 
-with 
+with
     amms as (
         SELECT
         *
@@ -65,13 +65,13 @@ with
                 ('Jupiter Perps', 'PERPHjGBqRHArX4DySjwM6UJHiR3sWAatqfdBS2qQJu')
             ) as v(amm_name, amm)
     )
-    
+
 
     , jup_messages_logs as (
         --SwapEvent through log messages
-        with 
+        with
             hex_data as (
-                SELECT 
+                SELECT
                     from_base64(split(l.logs, ' ')[3]) as hex_data
                     -- , split(logs, ' ')[3] as base64_data
                     , l.log_index_raw
@@ -80,7 +80,7 @@ with
                     , t.block_slot
                     , t.block_time
                     , t.signer as tx_signer
-                    , case when contains(t.account_keys, 'JUP5pEAZeHdHrLxh5UCwAbpjGwYKKoquCpda2hfP4u8') then 5 
+                    , case when contains(t.account_keys, 'JUP5pEAZeHdHrLxh5UCwAbpjGwYKKoquCpda2hfP4u8') then 5
                         when contains(t.account_keys, 'JUP4Fb2cqiRUcaTHdrPC8h2gNsA2ETXiPDD33WcGuJB') then 4
                         end as jup_version
                 FROM {{ source('solana','transactions') }} t
@@ -95,8 +95,8 @@ with
                 {% endif %}
                 -- and block_time >= now() - interval '7' day --shorten CI
         )
-        
-        SELECT 
+
+        SELECT
             a.amm_name
             , toBase58(bytearray_substring(hex_data,1+8,32)) as amm
             , toBase58(bytearray_substring(hex_data,1+40,32)) as input_mint
@@ -112,7 +112,7 @@ with
         FROM hex_data
         JOIN amms a ON a.amm = toBase58(bytearray_substring(hex_data,1+8,32)) --only include amms that we are tracking.
     )
-    
+
     , jup6_logs as (
         --uses event CPI
         SELECT
@@ -129,7 +129,7 @@ with
             , tx_signer
             , 6 as jup_version
         FROM (
-            SELECT 
+            SELECT
                 *
                 , row_number() over (partition by tx_id order by outer_instruction_index asc, COALESCE(inner_instruction_index,0) asc) as log_index
             FROM {{ source('solana','instruction_calls') }}
@@ -168,22 +168,22 @@ l.amm
 , l.tx_signer
 , l.jup_version
 FROM (
-    SELECT * FROM jup_messages_logs 
+    SELECT * FROM jup_messages_logs
     WHERE input_amount > 0 and output_amount > 0
-    UNION ALL 
+    UNION ALL
     SELECT * FROM jup6_logs
-) l 
+) l
 --tokens
 LEFT JOIN {{ ref('tokens_solana_fungible') }} tk_1 ON tk_1.token_mint_address = l.input_mint
 LEFT JOIN {{ ref('tokens_solana_fungible') }} tk_2 ON tk_2.token_mint_address = l.output_mint
-LEFT JOIN {{ ref('prices_usd_forward_fill') }} p_1 ON p_1.blockchain = 'solana' 
-    AND date_trunc('minute', l.block_time) = p_1.minute 
+LEFT JOIN {{ source('prices','usd_forward_fill') }} p_1 ON p_1.blockchain = 'solana'
+    AND date_trunc('minute', l.block_time) = p_1.minute
     AND l.input_mint = toBase58(p_1.contract_address)
     {% if is_incremental() %}
     AND {{ incremental_predicate('p_1.minute') }}
     {% endif %}
-LEFT JOIN {{ ref('prices_usd_forward_fill') }}  p_2 ON p_2.blockchain = 'solana' 
-    AND date_trunc('minute', l.block_time) = p_2.minute 
+LEFT JOIN {{ source('prices', 'usd_forward_fill') }}  p_2 ON p_2.blockchain = 'solana'
+    AND date_trunc('minute', l.block_time) = p_2.minute
     AND l.output_mint = toBase58(p_2.contract_address)
     {% if is_incremental() %}
     AND {{ incremental_predicate('p_2.minute') }}

models/pumpdotfun/solana/pumpdotfun_solana_trades.sql → solana/models/_sector/dex/pumpdotfun/solana/pumpdotfun_solana_trades.sql

@@ -18,15 +18,15 @@
 
 {% set project_start_date = '2024-01-14' %} --grabbed program deployed at time (account created at)
 
-with 
+with
     bonding_curves as (
         SELECT
             account_arguments[1] as token_mint_address
-            , account_arguments[3] as bonding_curve 
+            , account_arguments[3] as bonding_curve
             , account_arguments[4] as bonding_curve_vault
         FROM {{ source('solana','instruction_calls') }}
         WHERE executing_account = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P'
-            AND bytearray_substring(data,1,8) = 0x181ec828051c0777 --Create https://solscan.io/tx/2Vfq4gS9nq2jvpmZSxVXJ3uHGeheENXetwUus6KnhBzFu23Brqbt5EoNiTLds6jr72yZYGJ9YbMDG1BYKMRe3hSQ 
+            AND bytearray_substring(data,1,8) = 0x181ec828051c0777 --Create https://solscan.io/tx/2Vfq4gS9nq2jvpmZSxVXJ3uHGeheENXetwUus6KnhBzFu23Brqbt5EoNiTLds6jr72yZYGJ9YbMDG1BYKMRe3hSQ
             and tx_success = true
         {% if is_incremental() %}
         AND {{incremental_predicate('block_time')}}
@@ -35,7 +35,7 @@ with
         -- and block_time >= now() - interval '7' day
         {% endif %}
     )
-    
+
     , swaps as (
         SELECT
             to_base58(bytearray_substring(data,1+16,32)) as token_mint_address
@@ -56,7 +56,7 @@ with
             , outer_executing_account
         FROM {{ source('solana','instruction_calls') }}
         WHERE executing_account = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P'
-        AND bytearray_substring(data,1,16) = 0xe445a52e51cb9a1dbddb7fd34ee661ee --SwapEvent 
+        AND bytearray_substring(data,1,16) = 0xe445a52e51cb9a1dbddb7fd34ee661ee --SwapEvent
         and tx_success = true
         {% if is_incremental() %}
         AND {{incremental_predicate('block_time')}}
@@ -69,9 +69,9 @@ with
         --buy https://solscan.io/tx/5782i58ZHCKSANgTi3WXL5vjFAWEVJuPMGAr6gjx2qqcRP6LLNRH2g6pb61tk44H7PC9ohFNCUKQVUAMd5vYeUns
         --sell https://solscan.io/tx/4thHCu9SX166TP2cnjgwJ7mDSSMn5MBe8xLsTYzRLJmE7jPgwQatXh4ehh3At4xvVcgUefFzzsYVBaVwYyS1bA6v
     )
-    
+
     , trades_base as (
-        SELECT 
+        SELECT
             sp.block_time
             , 'pumpdotfun' as project
             , 1 as version
@@ -85,29 +85,29 @@ with
             end as token_pair
             --bought
             , case when is_buy = 1 then COALESCE(tk.symbol, sp.token_mint_address)
-                else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112') 
-                end as token_bought_symbol 
+                else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112')
+                end as token_bought_symbol
             , case when is_buy = 1 then sp.token_mint_address
                 else 'So11111111111111111111111111111111111111112'
                 end as token_bought_mint_address
             , case when is_buy = 1 then token_amount
                 else sol_amount
                 end as token_bought_amount_raw
             , case when is_buy = 1 then token_amount/pow(10,tk.decimals)
-                else sol_amount/pow(10,tk_sol.decimals) 
+                else sol_amount/pow(10,tk_sol.decimals)
                 end as token_bought_amount
             --sold
             , case when is_buy = 0 then COALESCE(tk.symbol, sp.token_mint_address)
-                else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112') 
-                end as token_sold_symbol 
+                else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112')
+                end as token_sold_symbol
             , case when is_buy = 0 then sp.token_mint_address
                 else 'So11111111111111111111111111111111111111112'
                 end as token_sold_mint_address
             , case when is_buy = 0 then token_amount
                 else sol_amount
                 end as token_sold_amount_raw
             , case when is_buy = 0 then token_amount/pow(10,tk.decimals)
-                else sol_amount/pow(10,tk_sol.decimals) 
+                else sol_amount/pow(10,tk_sol.decimals)
                 end as token_sold_amount
             , sp.sol_amount*0.01 as sol_fee_raw
             , sp.sol_amount/pow(10,tk_sol.decimals)*0.01 as sol_fee
@@ -134,10 +134,10 @@ with
         LEFT JOIN {{ ref('tokens_solana_fungible') }} tk_sol ON tk_sol.token_mint_address = 'So11111111111111111111111111111111111111112'
         LEFT JOIN bonding_curves bc ON bc.token_mint_address = sp.token_mint_address
     )
-    
-SELECT    
+
+SELECT
     tb.blockchain
-    , tb.project 
+    , tb.project
     , tb.version
     , CAST(date_trunc('month', tb.block_time) AS DATE) as block_month
     , tb.block_time
@@ -154,7 +154,7 @@ SELECT
     , sol_fee_raw
     , sol_fee
     , (case when tb.token_bought_mint_address = 'So11111111111111111111111111111111111111112' then p_bought.price
-        else p_sold.price 
+        else p_sold.price
         end) * sol_fee as fee_usd --fees are only in sol
     , tb.token_sold_mint_address
     , tb.token_bought_mint_address
@@ -172,16 +172,16 @@ SELECT
     , tb.inner_instruction_index
     , tb.tx_index
 FROM trades_base tb
-LEFT JOIN {{ ref('prices_usd_forward_fill') }} p_bought ON p_bought.blockchain = 'solana' 
-    AND date_trunc('minute', tb.block_time) = p_bought.minute 
+LEFT JOIN {{ source('prices', 'usd_forward_fill') }} p_bought ON p_bought.blockchain = 'solana'
+    AND date_trunc('minute', tb.block_time) = p_bought.minute
     AND token_bought_mint_address = toBase58(p_bought.contract_address)
     {% if is_incremental() %}
     AND {{incremental_predicate('p_bought.minute')}}
     {% else %}
     AND p_bought.minute >= TIMESTAMP '{{project_start_date}}'
     {% endif %}
-LEFT JOIN {{ ref('prices_usd_forward_fill') }} p_sold ON p_sold.blockchain = 'solana' 
-    AND date_trunc('minute', tb.block_time) = p_sold.minute 
+LEFT JOIN {{ source('prices', 'usd_forward_fill') }} p_sold ON p_sold.blockchain = 'solana'
+    AND date_trunc('minute', tb.block_time) = p_sold.minute
     AND token_sold_mint_address = toBase58(p_sold.contract_address)
     {% if is_incremental() %}
     AND {{incremental_predicate('p_sold.minute')}}

solana/package-lock.yml

@@ -0,0 +1,4 @@
+packages:
+- package: dbt-labs/dbt_utils
+  version: 1.2.0
+sha1_hash: d4f259856543b0ef301e0b3b0bbc94ccb6b12a54

solana/packages.yml

@@ -0,0 +1,3 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: 1.2.0

solana/profiles.yml

@@ -0,0 +1,12 @@
+spellbook-local:
+  target: dev
+  outputs:
+    dev:
+      type: trino
+      user: trino
+      password: trino
+      host: trino
+      port: 1234
+      database: hive
+      schema: wizard
+      threads: 1