Skip to content

Commit

Permalink
Move events.location_ids DDL into flowdb scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
jc-harrison committed Jun 14, 2023
1 parent ab1ef5e commit cbec098
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 8 deletions.
15 changes: 15 additions & 0 deletions flowdb/bin/build/0020_schema_events.sql
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ cause a great toll to the ingestion process. IDs are not
mandatory, but a number of features created by `flowmachine`
require this field.
This schema also includes a table 'location_ids', for
recording all location IDs that appear in the CDR
with the earliest and latest date that each location ID
appears. The data ingestion pipeline defined in FlowETL
includes a task that updates the 'location_ids' table when
new CDR records are ingested.
-----------------------------------------------------------
*/
CREATE SCHEMA IF NOT EXISTS events;
Expand Down Expand Up @@ -150,3 +157,11 @@ CREATE SCHEMA IF NOT EXISTS events;
country_code NUMERIC

);

CREATE TABLE IF NOT EXISTS events.location_ids (
location_id TEXT,
cdr_type TEXT,
first_active_date DATE,
last_active_date DATE,
PRIMARY KEY(location_id, cdr_type)
);
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,6 @@
UpdateLocationIDsTableOperator = fixed_sql_operator(
class_name="UpdateLocationIDsTableOperator",
sql="""
CREATE TABLE IF NOT EXISTS events.location_ids (
location_id TEXT,
cdr_type TEXT,
first_active_date DATE,
last_active_date DATE,
PRIMARY KEY(location_id, cdr_type)
);
INSERT INTO events.location_ids (location_id, cdr_type, first_active_date, last_active_date)
SELECT location_id,
'{{ params.cdr_type }}' AS cdr_type,
Expand Down

0 comments on commit cbec098

Please sign in to comment.