ELE-489 Support dbt-trino connector (elementary-data#1378)

* ELE-489 Support dbt-trino connector * Force get_adapter_type_and_unique_id to fail, therefore setting warehouse_type to null * Bump internal dbt-data-reliability Package * get_adapter_type_and_unique_id - fix for athena and trino * link_local_elementary_dbt_pkg: bugfix * add (temporarily) new index html * pyproject: reduce dbt-trino lower version limit * Update index.html to v1.0.18 * update dbt package ref --------- Co-authored-by: Itamar Hartstein <haritamar@gmail.com>
kostek-pl · Apr 15, 2024 · a52ba6b · a52ba6b
1 parent 09becbc
commit a52ba6b
Show file tree

Hide file tree

Showing 13 changed files with 704 additions and 488 deletions.
diff --git a/docs/_snippets/cloud/integrations/trino.mdx b/docs/_snippets/cloud/integrations/trino.mdx
@@ -0,0 +1 @@
+Coming soon!
diff --git a/docs/_snippets/install-cli.mdx b/docs/_snippets/install-cli.mdx
@@ -14,6 +14,7 @@ pip install 'elementary-data[bigquery]'
 pip install 'elementary-data[redshift]'
 pip install 'elementary-data[databricks]'
 pip install 'elementary-data[athena]'
+pip install 'elementary-data[trino]'
 ## Postgres doesn't require this step
 ```
 

diff --git a/docs/_snippets/oss/adapters-cards.mdx b/docs/_snippets/oss/adapters-cards.mdx
@@ -246,6 +246,96 @@
       </svg>
     }
   ></Card>
+  <Card
+    title="dbt-trino"
+    icon={
+      <svg
+        height="40"
+        viewBox="0 0 40 40"
+        width="40"
+        xmlns="http://www.w3.org/2000/svg"
+      >
+        <path
+          d="m15.68244,35.3069c0.58332,-0.38677 -0.94473,-0.24094 -1.07154,-0.5009a9.6629,9.6629 0 0 1 -1.81338,-0.32336c-1.07154,-0.27898 -3.67114,-2.02262 -3.94378,-3.68382s0,-4.85047 1.16031,-6.72726a10.90564,10.90564 0 0 1 2.78347,-2.94199s-2.26355,-5.38307 -2.46011,-9.44733s0.12047,-11.34313 2.78347,-11.47628s2.26355,6.64483 1.81338,11.05147a57.06439,57.06439 0 0 0 -0.13315,8.80694a12.25616,12.25616 0 0 1 3.49361,-0.58332a13.39111,13.39111 0 0 1 3.10684,0.29166s0.83694,-6.25172 3.63944,-10.11942s5.70644,-7.90025 7.60859,-6.7653s-0.51358,5.19286 -2.19381,7.30424s-4.59685,5.28163 -5.56695,6.97454a29.54033,29.54033 0 0 0 -1.68023,3.49361a9.51073,9.51073 0 0 1 4.05791,4.00719c1.31248,2.71373 1.43929,6.08053 0.72916,7.5008a5.11677,5.11677 0 0 1 -3.55701,2.5362c-0.84328,0.03804 -4.38128,0.91937 -4.38128,0.91937l-4.37494,-0.31702z"
+          fill="#fff"
+        />
+        <path
+          d="m21.87076,19.82343s2.11138,-5.51622 4.00719,-8.62306s4.43834,-5.44648 5.23724,-4.97094s-1.62951,3.37948 -3.75357,6.22636a70.83593,70.83593 0 0 0 -4.71732,7.84318l-0.77354,-0.47554z"
+          fill="#dd00a1"
+        />
+        <path
+          d="m13.39986,20.77451l0.60235,-0.47554s-0.08243,-5.51622 0,-8.87668s0.25996,-7.55152 -0.86231,-7.50714s-1.89581,3.26535 -1.46465,7.84952a55.46025,55.46025 0 0 0 1.72461,9.00983z"
+          fill="#dd00a1"
+        />
+        <circle r="1.23005" cy="31.38214" cx="11.50406" fill="#f9d8d2" />
+        <circle r="1.23005" cy="31.38214" cx="26.03646" fill="#f9d8d2" />
+        <path
+          d="m20.8119,31.27435a0.11413,0.11413 0 0 0 -0.16485,0.04438s-0.36141,0.63405 -0.89401,0.67209a1.32516,1.32516 0 0 1 -0.95107,-0.44383l0,-0.90669c0.34239,-0.18387 0.89401,-0.75452 0.89401,-0.95107a0.88767,0.88767 0 0 0 -0.95107,-0.69111c-0.63405,0 -1.07788,0.41213 -1.07788,0.75452s0.69745,0.77988 0.89401,0.91303l0,0.91937a1.13495,1.13495 0 0 1 -0.82426,0.44383c-0.50724,0 -0.88767,-0.68477 -0.88767,-0.69111a0.12047,0.12047 0 0 0 -0.16485,-0.05072a0.11413,0.11413 0 0 0 -0.04438,0.15851c0,0.03804 0.45017,0.81792 1.0969,0.81792a1.33784,1.33784 0 0 0 0.95107,-0.45652a1.5661,1.5661 0 0 0 1.00814,0.46286l0.07609,0a1.57878,1.57878 0 0 0 1.08422,-0.79256a0.11413,0.11413 0 0 0 -0.04438,-0.2029z"
+          fill="#10110e"
+        />
+        <path
+          d="m12.79752,21.12957s-1.76266,-4.43834 -2.0987,-6.79066s-0.63405,-6.201 -0.19021,-8.78158a30.37094,30.37094 0 0 1 0.77988,-3.51897s-0.8306,6.01712 -0.51992,8.64843s2.02896,10.44278 2.02896,10.44278z"
+          fill="#e5e5e5"
+        />
+        <path
+          d="m21.40156,19.77271s1.2681,-6.71458 3.11318,-9.35856s4.134,-5.83325 5.29431,-6.64483l1.15397,-0.81158s-4.65392,4.85681 -6.44828,8.88936s-3.11318,7.92561 -3.11318,7.92561z"
+          fill="#e5e5e5"
+        />
+        <path
+          d="m14.00855,27.38129a0.96375,0.96375 0 1 0 0.96375,0.95741a0.96375,0.96375 0 0 0 -0.96375,-0.95741zm0.25362,0.89401a0.2663,0.2663 0 1 1 0.2663,-0.2663a0.2663,0.2663 0 0 1 -0.2663,0.2663z"
+          fill="#10110e"
+        />
+        <path
+          d="m23.48758,27.38129a0.96375,0.96375 0 1 0 0.95741,0.95741a0.96375,0.96375 0 0 0 -0.95741,-0.95741zm0.28532,0.89401a0.2663,0.2663 0 1 1 0.25996,-0.2663a0.25996,0.25996 0 0 1 -0.25996,0.2663z"
+          fill="#10110e"
+        />
+        <ellipse
+          opacity="0.2"
+          ry="6.44194"
+          rx="8.94009"
+          cy="28.70646"
+          cx="18.92877"
+          fill="#8accce"
+        />
+        <path
+          d="m29.75198,25.12408l0,-4.26081a1.55976,1.55976 0 0 0 1.21103,-1.45197a1.68023,1.68023 0 0 0 -3.34144,0a1.5661,1.5661 0 0 0 1.22371,1.45197l0,4.15302l-0.12681,0c-1.2681,-4.4003 -5.01533,-7.13305 -9.91018,-7.13305s-8.66111,2.70739 -9.90384,7.13305l-0.42481,0a1.74363,1.74363 0 0 0 -1.82606,1.63585l0,3.27803a1.74363,1.74363 0 0 0 1.82606,1.64219l0.57698,0a8.24263,8.24263 0 0 0 2.27624,3.22097l-0.81792,0.74184a0.5009,0.5009 0 0 0 -0.05706,0.71013c1.85776,2.23819 4.89486,3.55067 8.32506,3.55067s6.47364,-1.2681 8.3314,-3.55067a0.5009,0.5009 0 0 0 -0.0634,-0.71013l-0.81792,-0.74184a8.35676,8.35676 0 0 0 2.27624,-3.22097l0.63405,0a1.73729,1.73729 0 0 0 1.81972,-1.64219l0,-3.27803a1.64219,1.64219 0 0 0 -1.21103,-1.52806zm-10.94368,-2.5362c4.52711,0 8.20459,2.98003 8.20459,6.63849c0,3.96281 -4.12766,5.73814 -8.20459,5.73814s-8.20459,-1.78802 -8.20459,-5.76984c0,-3.62676 3.65846,-6.63849 8.20459,-6.63849l0,0.0317z"
+          fill="#fff"
+        />
+        <path
+          d="m30.05633,28.83961l-1.21103,0c0,-0.19021 0,-0.38043 0,-0.60235a12.24982,12.24982 0 0 0 -0.24728,-2.40305l0.51992,0a0.85597,0.85597 0 0 1 0.91303,0.81792l0.02536,2.18747z"
+          fill="#515151"
+        />
+        <path
+          d="m29.1433,30.75443l-0.63405,0a9.69461,9.69461 0 0 0 0.2663,-1.38223l1.2681,0l0,0.54528a0.85597,0.85597 0 0 1 -0.90035,0.83694z"
+          fill="#515151"
+        />
+        <path
+          d="m18.8083,38.94634c-3.03709,0 -5.74448,-1.09056 -7.44373,-3.03075l0.84963,-0.7672a11.29875,11.29875 0 0 0 6.59411,1.90215a11.27973,11.27973 0 0 0 6.59411,-1.90215l0.85597,0.7672c-1.70559,1.94019 -4.38762,3.03075 -7.45007,3.03075z"
+          fill="#515151"
+        />
+        <path
+          d="m8.47965,25.83421l0.54528,0a12.45272,12.45272 0 0 0 -0.24094,2.40305a4.98996,4.98996 0 0 0 0,0.60235l-1.18567,0l0,-2.18747a0.80524,0.80524 0 0 1 0.88133,-0.81792z"
+          fill="#515151"
+        />
+        <path d="m18.8083,22.0109a10.98173,10.98173 0 0 0 -3.03709,0.43749l0,-3.30973a10.85492,10.85492 0 0 1 3.03709,-0.40579a10.8676,10.8676 0 0 1 3.04343,0.40579l0,3.3351a10.14478,10.14478 0 0 0 -3.04343,-0.46286z" />
+        <path
+          d="m15.16252,19.32887l0,3.3351c-3.03709,1.12227 -5.1675,3.6331 -5.1675,6.5307c0,3.74089 3.55701,6.28342 8.81328,6.28342s8.81328,-2.5362 8.81328,-6.28342c0,-2.8976 -2.12406,-5.40844 -5.1675,-6.5307l0,-3.3351c3.55701,1.2681 5.77618,4.56515 5.77618,8.94009c0,4.91388 -3.76625,8.24263 -9.42197,8.24263s-9.42197,-3.3351 -9.42197,-8.27434c0,-4.34323 2.21917,-7.62127 5.77618,-8.90839z"
+          fill="#515151"
+        />
+        <path
+          d="m7.56662,29.37221l1.2681,0a10.68372,10.68372 0 0 0 0.27264,1.36955l-0.63405,0a0.85597,0.85597 0 0 1 -0.91303,-0.82426l0.00634,-0.54528z"
+          fill="#515151"
+        />
+        <ellipse
+          ry="0.68477"
+          rx="0.76086"
+          cy="19.4113"
+          cx="29.29547"
+          fill="#515151"
+        />
+      </svg>
+    }
+  ></Card>
   <Card
     title="Ask us"
     href="https://github.com/elementary-data/elementary/issues"

diff --git a/docs/_snippets/profiles/all-profiles.mdx b/docs/_snippets/profiles/all-profiles.mdx
@@ -137,4 +137,25 @@ elementary:
       threads: [number of threads like 8]
 ```
 
+```yml Trino
+## Trino ##
+## By default, edr expects the profile name 'elementary'.      ##
+## Configure the database and schema of elementary models.     ##
+## Check where 'elementary_test_results' is to find it.        ##
+
+elementary:
+  outputs:
+    default:
+      type: trino
+      host: [hostname]
+      port: [port]
+      database: [database name]
+      schema: [schema name] # elementary schema, usually [schema name]_elementary
+      threads: [1 or more]
+      method: [authentication method] # ldap, oauth etc.
+      user: [username]
+      # password: [optional, used with ldap authentication ]
+      # session_properties: [optional, sets Trino session properties used in the connection]
+```
+
 </CodeGroup>
diff --git a/docs/cloud/integrations/dwh/trino.mdx b/docs/cloud/integrations/dwh/trino.mdx
@@ -0,0 +1,6 @@
+---
+title: "Connect to Trino"
+sidebarTitle: "Trino"
+---
+
+<Snippet file="cloud/integrations/trino.mdx" />
diff --git a/docs/data-tests/schema-tests/schema-changes-from-baseline.mdx b/docs/data-tests/schema-tests/schema-changes-from-baseline.mdx
@@ -18,7 +18,7 @@ Supported parameters for the test:
 
 <Info>
   Supported in Databricks with Unity Catalog only. <br/>
-  Not supported in Athena.
+  Not supported in Athena or Trino.
 </Info>
 
 ### Auto-generate baseline schema

diff --git a/docs/data-tests/schema-tests/schema-changes.mdx b/docs/data-tests/schema-tests/schema-changes.mdx
@@ -9,7 +9,7 @@ Data Types note: Snowflake, Redshift and Postgres all has "synonym" data types,
 
 <Info>
   Supported in Databricks with Unity Catalog only. <br/>
-  Not supported in Athena.
+  Not supported in Athena or Trino.
 </Info>
 
 <RequestExample>

diff --git a/docs/mint.json b/docs/mint.json
@@ -263,7 +263,8 @@
         "cloud/integrations/dwh/redshift",
         "cloud/integrations/dwh/databricks",
         "cloud/integrations/dwh/postgres",
-        "cloud/integrations/dwh/athena"
+        "cloud/integrations/dwh/athena",
+        "cloud/integrations/dwh/trino"
       ]
     },
     {

diff --git a/elementary/monitor/data_monitoring/report/index.html b/elementary/monitor/data_monitoring/report/index.html
diff --git a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql
@@ -12,4 +12,8 @@
 
 {% macro bigquery__get_adapter_unique_id() %}
     {{ return(target.project) }}
-{% endmacro %}
+{% endmacro %}
+
+{% macro athena__get_adapter_unique_id() %}
+    {{ return(target.s3_staging_dir) }}
+{% endmacro %}
diff --git a/elementary/monitor/dbt_project/packages.yml b/elementary/monitor/dbt_project/packages.yml
@@ -7,4 +7,4 @@ packages:
   #  NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this
   #  commented, so it will be easy to access)
   - git: https://github.com/elementary-data/dbt-data-reliability.git
-    revision: ec912b0f3678f187372551a89baab2444bcac829
+    revision: 140665a199ccb096b456b9f0239d370c0ff65deb
diff --git a/pyproject.toml b/pyproject.toml
@@ -43,6 +43,7 @@ dbt-postgres = {version = ">=0.20,<2.0.0", optional = true}
 dbt-databricks = {version = ">=0.20,<2.0.0", optional = true}
 dbt-spark = {version = ">=0.20,<2.0.0", optional = true}
 dbt-athena-community = {version = ">=1.6.3,<2.0.0", optional = true}
+dbt-trino = {version = ">=1.5.0,<2.0.0", optional = true}
 
 [tool.poetry.extras]
 snowflake = ["dbt-snowflake"]
@@ -52,6 +53,7 @@ postgres = ["dbt-postgres"]
 databricks = ["dbt-databricks"]
 spark = ["dbt-spark"]
 athena = ["dbt-athena-community"]
+trino = ["dbt-trino"]
 all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark"]
 
 [build-system]

diff --git a/scripts/link_local_elementary_dbt_pkg.py b/scripts/link_local_elementary_dbt_pkg.py
@@ -2,9 +2,9 @@
 import sys
 from pathlib import Path
 
-from elementary.monitor.dbt_project_utils import _DBT_PACKAGE_NAME, _PACKAGES_PATH
+from elementary.monitor.dbt_project_utils import _DBT_PACKAGE_NAME, CLI_DBT_PROJECT_PATH
 
-ELE_DBT_PKG_PATH = Path(_PACKAGES_PATH) / _DBT_PACKAGE_NAME
+ELE_DBT_PKG_PATH = Path(CLI_DBT_PROJECT_PATH) / "dbt_packages" / _DBT_PACKAGE_NAME
 
 
 def main():