Resolved issue where timezone settings were not valid.

treasure-data · Mar 4, 2024 · bf476f7 · bf476f7
1 parent 9644a3a
commit bf476f7
Show file tree

Hide file tree

Showing 16 changed files with 39 additions and 26 deletions.
diff --git a/scenarios/cdp_campaign_management/README.md b/scenarios/cdp_campaign_management/README.md
@@ -306,8 +306,9 @@ By editing/adding and uploading the value of each utm parameter in this table, t
 This file must be edited for run. The following sample is an example to illustrate the process.
 
 ```yaml
-timezone: JST
 td:
+    timezone: JST
+
     ps:
         - 489726
 

diff --git a/scenarios/cdp_campaign_management/common/gsheet_settings.yaml b/scenarios/cdp_campaign_management/common/gsheet_settings.yaml
@@ -1,4 +1,7 @@
 gsheet:
+    result_connection: gsheet_for_roi
+    sheet_folder: 1fLWGown_zelyAkSPKxwxxqnSgi6riAqp
+    spreadsheet_title: cdp_campaign_management2
     # result_connection: ******
     # sheet_folder: ******
     # spreadsheet_title: cdp_campaign_management
diff --git a/scenarios/cdp_campaign_management/common/user_settings.yaml b/scenarios/cdp_campaign_management/common/user_settings.yaml
@@ -1,5 +1,6 @@
-# timezone:
 td:
+    # timezone:
+
     ps: #required
 
     user_id: #required

diff --git a/scenarios/cdp_campaign_management/common/user_settings_sample.yaml b/scenarios/cdp_campaign_management/common/user_settings_sample.yaml
@@ -1,14 +1,14 @@
-timezone: JST
 td:
+    timezone: JST
     ps:
         - 507568
 
     user_id:
         507568: td_client_id
 
     activations_tables:
-        507568:
-            scan_journey_tables: true
+        # 507568:
+        #     scan_journey_tables: true
 
     clicks_tables:
         507568:

diff --git a/scenarios/cdp_campaign_management/main_incremental_ingest.dig b/scenarios/cdp_campaign_management/main_incremental_ingest.dig
@@ -16,10 +16,14 @@ _export:
         ps_id: ${td.ps}
     _do:
         _export:
+            user_timezone: "${typeof td.timezone === 'undefined' ? 'UTC' : td.timezone}"
             td:
                 database: ${td.base_db_name}_${ps_id}
             user_id: ${td.user_id[ps_id]}
 
+        +show_timezone:
+            echo>: "Timezone: ${user_timezone}"
+
         +prepare_dbs:
             td_ddl>:
             create_databases:

diff --git a/scenarios/cdp_campaign_management/main_initial_ingest.dig b/scenarios/cdp_campaign_management/main_initial_ingest.dig
@@ -1,6 +1,6 @@
 _export:
     !include : common/system_settings.yaml
-    !include : common/user_settings.yaml
+    !include : common/user_settings_sample.yaml
     !include : common/gsheet_settings.yaml
 
 # +check_settings:
@@ -11,10 +11,14 @@ _export:
         ps_id: ${td.ps}
     _do:
         _export:
+            user_timezone: "${typeof td.timezone === 'undefined' ? 'UTC' : td.timezone}"
             td:
                 database: ${td.base_db_name}_${ps_id}
             user_id: ${td.user_id[ps_id]}
 
+        +show_timezone:
+            echo>: "Timezone: ${user_timezone}"
+
         +prepare_dbs:
             td_ddl>:
             create_databases:

diff --git a/scenarios/cdp_campaign_management/queries/ingest_activations.sql b/scenarios/cdp_campaign_management/queries/ingest_activations.sql
@@ -29,7 +29,7 @@ WITH tbl_base_activations AS
         AND CAST(audience_id AS VARCHAR) = '${ps_id}'
         AND TD_TIME_RANGE(time,${time_from},${time_to})
     ) t1
-    JOIN
+    LEFT OUTER JOIN
     (
         SELECT
             CAST(id AS VARCHAR) AS activation_step_id
@@ -46,10 +46,10 @@ SELECT
     ,${user_id}
     ,s1.activation_step_id
     ,syndication_id
+    ,segment_id
+    ,segment_name
     ,activation_type
     ,activation_name
-    -- ,segment_id
-    -- ,segment_name
     ,journey_id
     ,cv_name
     ,utm_source

diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_activations.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_activations.sql
@@ -1,6 +1,6 @@
 SELECT
-    TD_DATE_TRUNC('day', time, '${timezone}') AS time
-    ,TD_TIME_STRING(time, 'd!', '${timezone}') AS date
+    TD_DATE_TRUNC('day', time, '${user_timezone}') AS time
+    ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date
     ,activation_step_id
     ,cv_name
     ,utm_source

diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_activations_info.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_activations_info.sql
@@ -11,8 +11,8 @@ WITH tbl_jobs AS
 ,tbl_act_hst AS
 (
     SELECT
-        TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${timezone}') AS time
-        ,TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.finishedAt'), '${timezone}') AS time_finished
+        TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${user_timezone}') AS time
+        ,TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.finishedAt'), '${user_timezone}') AS time_finished
         ,id AS syndication_id
         ,name AS activation_name
         ,JSON_EXTRACT_SCALAR(exe, '$.workflowId') AS workflow_id
@@ -25,7 +25,7 @@ WITH tbl_jobs AS
         ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.activations}
     CROSS JOIN UNNEST(CAST(JSON_PARSE(executions) AS ARRAY(JSON))) AS t(exe)
     WHERE ps_id = '${ps_id}'
-    AND TD_TIME_RANGE(TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${timezone}'), ${time_from}, ${time_to})
+    AND TD_TIME_RANGE(TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${user_timezone}'), ${time_from}, ${time_to})
 )
 
 SELECT act_hst.*

diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_clicks.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_clicks.sql
@@ -1,6 +1,6 @@
 SELECT
-    TD_DATE_TRUNC('day', time, '${timezone}') AS time
-    ,TD_TIME_STRING(time, 'd!', '${timezone}') AS date
+    TD_DATE_TRUNC('day', time, '${user_timezone}') AS time
+    ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date
     ,activation_step_id
     ,cv_name
     ,utm_source

diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_conversions.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_conversions.sql
@@ -1,6 +1,6 @@
 SELECT
-    TD_DATE_TRUNC('day', time, '${timezone}') AS time
-    ,TD_TIME_STRING(time, 'd!', '${timezone}') AS date
+    TD_DATE_TRUNC('day', time, '${user_timezone}') AS time
+    ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date
     ,cv_name
     ,COUNT(1) AS cnt
     ,SUM(val) AS val

diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_mta.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_mta.sql
@@ -35,8 +35,8 @@ FROM
     ON act.time = clk.time AND act.activation_step_id = clk.activation_step_id
     FULL OUTER JOIN (
         SELECT
-            TD_DATE_TRUNC('day',time, '${timezone}') AS time
-            ,TD_TIME_STRING(time, 'd!', '${timezone}') AS date
+            TD_DATE_TRUNC('day',time, '${user_timezone}') AS time
+            ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date
             ,activation_step_id
             ,type
             ,utm_source

diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_mta_conversion_journeys.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_mta_conversion_journeys.sql
@@ -35,8 +35,8 @@ FROM
     ON act.time = clk.time AND act.activation_step_id = clk.activation_step_id
     FULL OUTER JOIN (
         SELECT
-            TD_DATE_TRUNC('day',time, '${timezone}') AS time
-            ,TD_TIME_STRING(time, 'd!', '${timezone}') AS date
+            TD_DATE_TRUNC('day',time, '${user_timezone}') AS time
+            ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date
             ,activation_step_id
             ,type
             ,utm_source

diff --git a/scenarios/cdp_campaign_management/queries/ingest_journeys.sql b/scenarios/cdp_campaign_management/queries/ingest_journeys.sql
@@ -1,5 +1,5 @@
 SELECT
-    TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${timezone}') AS time
+    TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${user_timezone}') AS time
     ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.audienceId') AS audience_id
     ,id
     ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.name') AS name
@@ -14,7 +14,7 @@ FROM ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.entities}
 WHERE type = 'journey'
 AND JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.audienceId') = '${ps_id}'
 AND TD_TIME_RANGE(
-    TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${timezone}')
+    TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${user_timezone}')
     ,${time_from}
     ,${time_to}
 )
diff --git a/scenarios/cdp_campaign_management/queries/ingest_master_activations.sql b/scenarios/cdp_campaign_management/queries/ingest_master_activations.sql
@@ -31,7 +31,7 @@ WITH tbl_act AS (
             ,idx-1 AS stage_no
             ,TD_TIME_PARSE(
                 JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt')
-                ,'${timezone}'
+                ,'${user_timezone}'
             ) AS time
             ,JSON_EXTRACT_SCALAR(elm, '$.name') AS stage_name
             ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.audienceId') AS ps_id

diff --git a/scenarios/cdp_campaign_management/write_to_gsheet.dig b/scenarios/cdp_campaign_management/write_to_gsheet.dig
@@ -59,7 +59,7 @@
 
     +export_mta:
         td>:
-        query: SELECT TD_TIME_STRING(time,'d!','${timezone}') AS date,* FROM ${td.tables.mta_conversion_journeys} ORDER BY cv_name, ${user_id}, cv_id, position LIMIT 1000
+        query: SELECT TD_TIME_STRING(time,'d!','${user_timezone}') AS date,* FROM ${td.tables.mta_conversion_journeys} ORDER BY cv_name, ${user_id}, cv_id, position LIMIT 1000
         result_connection: ${gsheet.result_connection}
         result_settings:
             spreadsheet_folder: ${gsheet.sheet_folder}