Merge pull request #144 from dekart-xyz/gh-140-data-source-ui

Gh 140 data source UI
dekart-xyz · Oct 16, 2023 · 6ada8a1 · 6ada8a1
2 parents d4c9647 + 1597fc4
commit 6ada8a1
Show file tree

Hide file tree

Showing 66 changed files with 7,325 additions and 1,096 deletions.
diff --git a/cypress/e2e/athena/spec.cy.js b/cypress/e2e/athena/spec.cy.js
@@ -5,7 +5,8 @@ describe('basic query flow', () => {
   it('should make simple athena query and get ready status', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.athena_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_athena_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')
@@ -17,7 +18,8 @@ describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.athena_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_athena_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`button:contains("${copy.execute}")`).should('be.disabled')

diff --git a/cypress/e2e/bq/basicFlow.cy.js b/cypress/e2e/bq/basicFlow.cy.js
@@ -5,7 +5,8 @@ describe('basic query flow', () => {
   it('should make simple bigquery query and get ready status', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.bigquery_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_sql_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

diff --git a/cypress/e2e/bq/cancelQuery.cy.js b/cypress/e2e/bq/cancelQuery.cy.js
@@ -5,7 +5,8 @@ describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.bigquery_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_sql_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`button:contains("${copy.cancel}")`).should('be.visible')

diff --git a/cypress/e2e/bq/scripts.cy.js b/cypress/e2e/bq/scripts.cy.js
@@ -5,7 +5,8 @@ describe('support bq scripts', () => {
   it('retrieve result for bigquery script', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.bigquery_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.bigquery_script, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get('div:contains("1 rows")', { timeout: 20000 }).should('be.visible')

diff --git a/cypress/e2e/bq/updateDataset.cy.js b/cypress/e2e/bq/updateDataset.cy.js
@@ -8,14 +8,16 @@ describe('update dataset', () => {
     cy.get(`button:contains("${copy.create_report}")`).click()
 
     // first query
-    cy.get(`button:contains("${copy.bigquery_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('SELECT primary_type, district, latitude, longitude, date from `bigquery-public-data.chicago_crime.crime` limit 1', { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')
 
     // second query
     cy.get('#dekart-report-page-tabs button.ant-tabs-nav-add:first').click({ force: true })
-    cy.get(`button:contains("${copy.bigquery_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('SELECT primary_type, district, latitude, longitude, date from `bigquery-public-data.chicago_crime.crime` limit 2', { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

diff --git a/cypress/e2e/snowflake/cancelQuery.cy.js b/cypress/e2e/snowflake/cancelQuery.cy.js
@@ -5,7 +5,8 @@ describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.snowflake_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`button:contains("${copy.cancel}")`).should('be.visible')

diff --git a/cypress/e2e/snowflake/emptyResult.cy.js b/cypress/e2e/snowflake/emptyResult.cy.js
@@ -5,7 +5,8 @@ describe('empty path', () => {
   it('should complete query returning empty result', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.snowflake_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('SELECT ROUND(uniform(-90::float, 90::float, random()), 6) AS lat, ROUND(uniform(-180::float, 180::float, random()), 6) AS lon FROM TABLE(GENERATOR(ROWCOUNT => 0))', { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

diff --git a/cypress/e2e/snowflake/fork.cy.js b/cypress/e2e/snowflake/fork.cy.js
@@ -24,8 +24,8 @@ describe('fork', () => {
     let originalColor
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.snowflake_query}")`).should('be.visible')
-    cy.get(`button:contains("${copy.snowflake_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type('select 0 as lat, 0 as lon', { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get('div:contains("1 rows")', { timeout: 20000 }).should('be.visible')

diff --git a/cypress/e2e/snowflake/happyPath.cy.js b/cypress/e2e/snowflake/happyPath.cy.js
@@ -5,7 +5,8 @@ describe('happy path', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.snowflake_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')

diff --git a/cypress/e2e/snowflake/spec.cy.js b/cypress/e2e/snowflake/spec.cy.js
@@ -5,7 +5,8 @@ describe('basic query flow', () => {
   it('should make simple snowflake query and get ready status', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.snowflake_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`span:contains("${copy.ready}")`, { timeout: 20000 }).should('be.visible')
@@ -18,7 +19,8 @@ describe('cancelling query', () => {
   it('should cancels query', () => {
     cy.visit('/')
     cy.get(`button:contains("${copy.create_report}")`).click()
-    cy.get(`button:contains("${copy.snowflake_query}")`).click()
+    cy.get('button:contains("Add data from...")').click()
+    cy.get('span:contains("SQL query")').click()
     cy.get('textarea').type(copy.simple_snowflake_query, { force: true })
     cy.get(`button:contains("${copy.execute}")`).click()
     cy.get(`button:contains("${copy.cancel}")`).should('be.visible')

diff --git a/migrations/000014_create_connections_table.up.sql b/migrations/000014_create_connections_table.up.sql
@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS connections (
+  id uuid NOT NULL,
+  connection_name text NOT NULL,
+  author_email text default 'UNKNOWN_EMAIL',
+  bigquery_project_id text default NULL,
+  cloud_storage_bucket text default NULL,
+  archived boolean default false,
+  updated_at timestamptz DEFAULT CURRENT_TIMESTAMP,
+  created_at timestamptz DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY(id)
+);
diff --git a/migrations/000015_dataset_connection_id.up.sql b/migrations/000015_dataset_connection_id.up.sql
@@ -0,0 +1,2 @@
+ALTER TABLE datasets ADD COLUMN connection_id uuid;
+ALTER TABLE datasets ADD CONSTRAINT fk_source FOREIGN KEY(connection_id) REFERENCES connections;
diff --git a/proto/dekart.proto b/proto/dekart.proto
@@ -13,7 +13,8 @@ service Dekart {
     // datasets
     rpc CreateDataset(CreateDatasetRequest) returns (CreateDatasetResponse) {}
     rpc RemoveDataset(RemoveDatasetRequest) returns (RemoveDatasetResponse) {}
-    rpc UpdateDataset(UpdateDatasetRequest) returns (UpdateDatasetResponse) {}
+    rpc UpdateDatasetName(UpdateDatasetNameRequest) returns (UpdateDatasetNameResponse) {}
+    rpc UpdateDatasetConnection(UpdateDatasetConnectionRequest) returns (UpdateDatasetConnectionResponse) {}
 
     // files
     rpc CreateFile(CreateFileRequest) returns (CreateFileResponse) {}
@@ -28,9 +29,72 @@ service Dekart {
     // streams
     rpc GetReportStream(ReportStreamRequest) returns (stream ReportStreamResponse) {}
     rpc GetReportListStream(ReportListRequest) returns (stream ReportListResponse) {}
+    rpc GetUserStream(GetUserStreamRequest) returns (stream GetUserStreamResponse) {}
 
     //statistics
     rpc GetUsage(GetUsageRequest) returns (GetUsageResponse) {}
+
+    //connections
+    rpc CreateConnection(CreateConnectionRequest) returns (CreateConnectionResponse) {}
+    rpc UpdateConnection(UpdateConnectionRequest) returns (UpdateConnectionResponse) {}
+    rpc ArchiveConnection(ArchiveConnectionRequest) returns (ArchiveConnectionResponse) {}
+    rpc GetConnectionList(GetConnectionListRequest) returns (GetConnectionListResponse) {}
+    rpc TestConnection(TestConnectionRequest) returns (TestConnectionResponse) {}
+}
+
+message GetConnectionListRequest {
+}
+
+message GetConnectionListResponse {
+    repeated Connection connections = 1;
+}
+
+message GetUserStreamRequest {
+    StreamOptions stream_options = 1;
+}
+message GetUserStreamResponse {
+    StreamOptions stream_options = 1;
+    int64 connection_update = 2;
+}
+
+message TestConnectionRequest {
+    Connection connection = 1;
+}
+
+message TestConnectionResponse {
+    bool success = 1;
+    string error = 2;
+}
+
+message ArchiveConnectionRequest {
+    string connection_id = 1;
+}
+
+message ArchiveConnectionResponse {
+}
+
+message UpdateConnectionRequest {
+    Connection connection = 1;
+}
+
+message UpdateConnectionResponse {
+    Connection connection = 1;
+}
+
+message CreateConnectionRequest {
+    string connection_name = 1;
+}
+
+message CreateConnectionResponse {
+    Connection connection = 1;
+}
+
+message Connection {
+    string id = 1;
+    string connection_name = 2;
+    string bigquery_project_id = 3;
+    string cloud_storage_bucket = 4;
+    bool is_default = 5;
 }
 
 message GetUsageRequest {}
@@ -78,6 +142,8 @@ message GetEnvResponse {
             TYPE_REQUIRE_IAP = 8;
             TYPE_DISABLE_USAGE_STATS = 9;
             TYPE_REQUIRE_GOOGLE_OAUTH = 10;
+            TYPE_BIGQUERY_PROJECT_ID = 11;
+            TYPE_CLOUD_STORAGE_BUCKET = 12;
         }
         Type type = 1;
         string value = 2;
@@ -137,6 +203,7 @@ message Dataset {
     int64 updated_at = 5;
     string file_id = 6;
     string name = 7;
+    string connection_id = 8;
 }
 
 message Query {
@@ -170,7 +237,6 @@ message Query {
     }
     QuerySource query_source = 13;
     string query_source_id = 14;
-
 }
 
 message File {
@@ -214,12 +280,20 @@ message CancelQueryRequest {
 message CancelQueryResponse {
 }
 
-message UpdateDatasetRequest {
+message UpdateDatasetNameRequest {
     string dataset_id = 1;
     string name = 2;
 }
 
-message UpdateDatasetResponse {
+message UpdateDatasetNameResponse {
+}
+
+message UpdateDatasetConnectionRequest {
+    string dataset_id = 1;
+    string connection_id = 2;
+}
+
+message UpdateDatasetConnectionResponse {
 }
 
 message CreateDatasetRequest {
@@ -256,6 +330,7 @@ message ReportStreamResponse {
     StreamOptions stream_options = 3;
     repeated Dataset datasets = 4;
     repeated File files = 5;
+    repeated Connection connections = 6;
 }
 
 message ForkReportRequest {

diff --git a/src/client/App.js b/src/client/App.js
@@ -16,6 +16,7 @@ import { getUsage } from './actions/usage'
 import { AuthState, RedirectState as DekartRedirectState } from '../proto/dekart_pb'
 import { getEnv } from './actions/env'
 import { setRedirectState } from './actions/redirectState'
+import { subscribeUserStream, unsubscribeUserStream } from './actions/user'
 
 // RedirectState reads states passed in the URL from the server
 function RedirectState () {
@@ -98,6 +99,12 @@ export default function App () {
       dispatch(getUsage())
     }
   }, [env, usage, dispatch, status])
+  useEffect(() => {
+    dispatch(subscribeUserStream())
+    return () => {
+      dispatch(unsubscribeUserStream())
+    }
+  }, [dispatch])
   return (
     <Router>
       <RedirectState />