Merge pull request #343 from tinybirdco/341-use-case-drop-column-base

Add drop column base use case

Author: joscalv

drop_column/.tinyenv

@@ -0,0 +1,31 @@
+
+# VERSION format is major.minor.patch-post where major, minor, patch and post are integer numbers
+# bump post to deploy to the current live Release, rollback to previous post version is not available
+# bump patch or minor to deploy a new Release and auto-promote it to live. Add TB_AUTO_PROMOTE=0 to create the Release in preview status
+# bump major to deploy a new Release in preview status
+VERSION=0.0.0
+
+
+
+##########
+# OPTIONAL env vars
+# Deploy a new Release in preview status (default is 1)
+# TB_AUTO_PROMOTE=0
+
+# Check if deploy requires backfilling on preview (default is 1)
+# TB_CHECK_BACKFILL_REQUIRED=0
+
+# Force old Releases deletion on promote (default is 0)
+# Setting it to 1 will remove oldest rollback Releases even when some resource is still in use
+# TB_FORCE_REMOVE_OLDEST_ROLLBACK=0
+
+# Don't print CLI version warning message if there's a new available version
+# TB_VERSION_WARNING=0
+
+# Skip regression tests
+# TB_SKIP_REGRESSION=0
+
+# Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
+# OBFUSCATE_REGEX_PATTERN="https://(www\.)?[^/]+||^Follow these instructions =>"
+# OBFUSCATE_PATTERN_SEPARATOR=||
+##########

drop_column/datasources/analytics_events.datasource

@@ -0,0 +1,16 @@
+TOKEN "tracker" APPEND
+
+DESCRIPTION >
+    Analytics events landing data source
+
+SCHEMA >
+    `timestamp` DateTime `json:$.timestamp`,
+    `session_id` String `json:$.session_id`,
+    `action` LowCardinality(String) `json:$.action`,
+    `version` LowCardinality(String) `json:$.version`,
+    `payload` String `json:$.payload`
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(timestamp)"
+ENGINE_SORTING_KEY "timestamp"
+ENGINE_TTL "timestamp + toIntervalDay(60)"

drop_column/datasources/analytics_pages_mv.datasource

@@ -0,0 +1,13 @@
+
+SCHEMA >
+    `date` Date,
+    `device` String,
+    `browser` String,
+    `location` String,
+    `pathname` String,
+    `visits` AggregateFunction(uniq, String),
+    `hits` AggregateFunction(count)
+
+ENGINE "AggregatingMergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(date)"
+ENGINE_SORTING_KEY "date, device, browser, location, pathname"

drop_column/datasources/analytics_sessions_mv.datasource

@@ -0,0 +1,14 @@
+
+SCHEMA >
+    `date` Date,
+    `session_id` String,
+    `device` SimpleAggregateFunction(any, String),
+    `browser` SimpleAggregateFunction(any, String),
+    `location` SimpleAggregateFunction(any, String),
+    `first_hit` SimpleAggregateFunction(min, DateTime),
+    `latest_hit` SimpleAggregateFunction(max, DateTime),
+    `hits` AggregateFunction(count)
+
+ENGINE "AggregatingMergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(date)"
+ENGINE_SORTING_KEY "date, session_id"

drop_column/datasources/analytics_sources_mv.datasource

@@ -0,0 +1,13 @@
+
+SCHEMA >
+    `date` Date,
+    `device` String,
+    `browser` String,
+    `location` String,
+    `referrer` String,
+    `visits` AggregateFunction(uniq, String),
+    `hits` AggregateFunction(count)
+
+ENGINE "AggregatingMergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(date)"
+ENGINE_SORTING_KEY "date, device, browser, location, referrer"

drop_column/datasources/fixtures/.gitkeep

@@ -0,0 +1,65 @@
+DESCRIPTION >
+	Parsed `page_hit` events, implementing `browser` and `device` detection logic.
+
+
+TOKEN "dashboard" READ
+
+NODE parsed_hits
+DESCRIPTION >
+    Parse raw page_hit events
+
+SQL >
+
+    SELECT
+        timestamp,
+        action,
+        version,
+        coalesce(session_id, '0') as session_id,
+        JSONExtractString(payload, 'locale') as locale,
+        JSONExtractString(payload, 'location') as location,
+        JSONExtractString(payload, 'referrer') as referrer,
+        JSONExtractString(payload, 'pathname') as pathname,
+        JSONExtractString(payload, 'href') as href,
+        lower(JSONExtractString(payload, 'user-agent')) as user_agent
+    FROM analytics_events
+    where action = 'page_hit'
+
+
+
+NODE endpoint
+SQL >
+
+    SELECT
+        timestamp,
+        action,
+        version,
+        session_id,
+        location,
+        referrer,
+        pathname,
+        href,
+        case
+            when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot')
+            then 'bot'
+            when match(user_agent, 'android')
+            then 'mobile-android'
+            when match(user_agent, 'ipad|iphone|ipod')
+            then 'mobile-ios'
+            else 'desktop'
+        END as device,
+        case
+            when match(user_agent, 'firefox')
+            then 'firefox'
+            when match(user_agent, 'chrome|crios')
+            then 'chrome'
+            when match(user_agent, 'opera')
+            then 'opera'
+            when match(user_agent, 'msie|trident')
+            then 'ie'
+            when match(user_agent, 'iphone|ipad|safari')
+            then 'safari'
+            else 'Unknown'
+        END as browser
+    FROM parsed_hits
+
+

drop_column/pipes/analytics_hits.pipe

@@ -0,0 +1,21 @@
+NODE analytics_pages_1
+DESCRIPTION >
+    Aggregate by pathname and calculate session and hits
+
+SQL >
+
+    SELECT
+        toDate(timestamp) AS date,
+        device,
+        browser,
+        location,
+        pathname,
+        uniqState(session_id) AS visits,
+        countState() AS hits
+    FROM analytics_hits
+    GROUP BY date, device, browser, location, pathname
+
+TYPE materialized
+DATASOURCE analytics_pages_mv
+
+

drop_column/pipes/analytics_pages.pipe

@@ -0,0 +1,22 @@
+NODE analytics_sessions_1
+DESCRIPTION >
+    Aggregate by session_id and calculate session metrics
+
+SQL >
+
+    SELECT
+        toDate(timestamp) AS date,
+        session_id,
+        anySimpleState(device) AS device,
+        anySimpleState(browser) AS browser,
+        anySimpleState(location) AS location,
+        minSimpleState(timestamp) AS first_hit,
+        maxSimpleState(timestamp) AS latest_hit,
+        countState() AS hits
+    FROM analytics_hits
+    GROUP BY date, session_id
+
+TYPE materialized
+DATASOURCE analytics_sessions_mv
+
+

drop_column/pipes/analytics_sessions.pipe

@@ -0,0 +1,23 @@
+NODE analytics_sources_1
+DESCRIPTION >
+    Aggregate by referral and calculate session and hits
+
+SQL >
+
+    WITH (SELECT domainWithoutWWW(href) FROM analytics_hits LIMIT 1) AS currenct_domain
+    SELECT
+        toDate(timestamp) AS date,
+        device,
+        browser,
+        location,
+        referrer,
+        uniqState(session_id) AS visits,
+        countState() AS hits
+    FROM analytics_hits
+    WHERE domainWithoutWWW(referrer) != currenct_domain
+    GROUP BY date, device, browser, location, referrer
+
+TYPE materialized
+DATASOURCE analytics_sources_mv
+
+