Merge pull request #180 from dekart-xyz/oss-bigquery-temp-storage

Bigquery temp storage and google project suggest

migrations/000019_dw_job_id.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE queries
+ADD COLUMN dw_job_id VARCHAR(255) NULL DEFAULT NULL;

proto/dekart.proto

@@ -37,13 +37,21 @@ service Dekart {
 
     //connections
     rpc CreateConnection(CreateConnectionRequest) returns (CreateConnectionResponse) {}
+    rpc GetGcpProjectList(GetGcpProjectListRequest) returns (GetGcpProjectListResponse) {}
     rpc UpdateConnection(UpdateConnectionRequest) returns (UpdateConnectionResponse) {}
     rpc ArchiveConnection(ArchiveConnectionRequest) returns (ArchiveConnectionResponse) {}
     rpc GetConnectionList(GetConnectionListRequest) returns (GetConnectionListResponse) {}
     rpc TestConnection(TestConnectionRequest) returns (TestConnectionResponse) {}
     rpc SetDefaultConnection(SetDefaultConnectionRequest) returns (SetDefaultConnectionResponse) {}
 }
 
+message GetGcpProjectListRequest {
+}
+
+message GetGcpProjectListResponse {
+    repeated string projects = 1;
+}
+
 message SetDefaultConnectionRequest {
     string connection_id = 1;
 }
@@ -117,6 +125,8 @@ message Connection {
     string author_email = 6;
     int64 created_at = 7;
     int64 updated_at = 8;
+    int64 dataset_count = 9;
+    bool can_store_files = 10;
 }
 
 message GetUsageRequest {}

src/client/ConnectionModal.js

@@ -8,12 +8,16 @@ import { useEffect } from 'react'
 import { archiveConnection, closeConnectionDialog, connectionChanged, saveConnection, testConnection } from './actions/connection'
 import { CheckCircleTwoTone, ExclamationCircleTwoTone } from '@ant-design/icons'
 import Tooltip from 'antd/es/tooltip'
+import AutoComplete from 'antd/es/auto-complete'
+import Alert from 'antd/es/alert'
 
 function Footer ({ form }) {
   const { dialog, test } = useSelector(state => state.connection)
   const { tested, testing, error: testError, success: testSuccess } = test
   const { id, loading } = dialog
 
+  const connection = useSelector(state => state.connection.list.find(s => s.id === id))
+
   const dispatch = useDispatch()
 
   return (
@@ -33,21 +37,22 @@ function Footer ({ form }) {
       <div className={styles.spacer} />
       <Button
         id='saveConnection'
-        type={tested && testSuccess ? 'primary' : 'default'} disabled={!tested || loading} onClick={() => {
+        type={tested && testSuccess ? 'primary' : 'default'} disabled={!tested || loading} danger={connection?.datasetCount > 0} onClick={() => {
           dispatch(saveConnection(id, form.getFieldsValue()))
         }}
+        title={connection?.datasetCount > 0 ? 'This connection is used in dataset. Modifying it may lead to errors in reports. Consider creating new connection and archiving this one' : 'Save connection'}
       >
         Save
       </Button>
-      <Button danger onClick={() => dispatch(archiveConnection(id))}>
+      <Button onClick={() => dispatch(archiveConnection(id))}>
         Archive
       </Button>
     </div>
   )
 }
 
 export default function ConnectionModal () {
-  const { dialog } = useSelector(state => state.connection)
+  const { dialog, projects } = useSelector(state => state.connection)
   const { visible, id, loading } = dialog
 
   const env = useSelector(state => state.env)
@@ -88,15 +93,28 @@ export default function ConnectionModal () {
             if (changedValues.bigqueryProjectId || changedValues.cloudStorageBucket) {
               dispatch(connectionChanged(allValues))
             }
+            if (changedValues.bigqueryProjectId && !allValues.connectionName) {
+              form.setFieldsValue({ connectionName: changedValues.bigqueryProjectId })
+            }
           }}
         >
-          <Form.Item label='Connection Name' name='connectionName' required>
-            <Input />
-          </Form.Item>
+          {connection?.datasetCount > 0 ? <div className={styles.datasetsCountAlert}><Alert message={<>Danger zone! This connection is used in {connection.datasetCount} dataset{connection.datasetCount > 1 ? 's' : ''}.</>} description="Modifying it may lead to queries and results not being found in reports. Consider creating new connection and archiving this one. Modifying name is safe." type="error" /></div> : null}
           <Form.Item label='Google Cloud project ID' extra='used to bill BigQuery jobs' required name='bigqueryProjectId'>
-            <Input readOnly={BIGQUERY_PROJECT_ID} />
+            {
+              projects.length > 0 && !BIGQUERY_PROJECT_ID
+                ? (
+                  <AutoComplete
+                    options={projects.map(project => ({ value: project, label: project }))}
+                    filterOption={(inputValue, option) => option.value.toUpperCase().indexOf(inputValue.toUpperCase()) !== -1}
+                  />
+                  )
+                : <Input readOnly={BIGQUERY_PROJECT_ID} />
+              }
+          </Form.Item>
+          <Form.Item label='Connection Name' required name='connectionName'>
+            <Input />
           </Form.Item>
-          <Form.Item label='Google Cloud Storage bucket' extra='where queries, files and query results stored' required name='cloudStorageBucket'>
+          <Form.Item label='Optional Google Cloud Storage bucket' extra='to store files and result cache ' name='cloudStorageBucket'>
             <Input placeholder='my-company-storage-bucket' readOnly={CLOUD_STORAGE_BUCKET} />
           </Form.Item>
         </Form>

src/client/ConnectionModal.module.css

@@ -15,4 +15,8 @@
     width: 32px;
     height: 32px;
     font-size: 18px;
+}
+
+.datasetsCountAlert {
+    margin-bottom: 20px;
 }

src/client/Dataset.js

@@ -17,9 +17,21 @@ function DatasetTypeSelector ({ dataset }) {
   const dispatch = useDispatch()
 
   const userDefinedConnection = useSelector(state => state.connection.userDefined)
+  const connectionList = useSelector(state => state.connection.list)
 
   const env = useSelector(state => state.env)
   const { ALLOW_FILE_UPLOAD } = env.variables
+  let allowFileUpload = ALLOW_FILE_UPLOAD
+  let disabledFileUploadNote = 'File upload is disabled in Dekart configuration'
+  if (allowFileUpload && userDefinedConnection) {
+    // check if selected connection supports file upload
+    const connection = connectionList.find(c => c.id === dataset.connectionId)
+    allowFileUpload = connection?.canStoreFiles
+    if (!allowFileUpload) {
+      disabledFileUploadNote = 'Selected connection does not support file upload'
+    }
+  }
+
 
   return (
     <div className={styles.datasetTypeSelector}>
@@ -35,8 +47,8 @@ function DatasetTypeSelector ({ dataset }) {
             {
               label: 'File upload',
               icon: <UploadOutlined />,
-              title: !ALLOW_FILE_UPLOAD ? 'File upload is disabled in Dekart configuration' : null,
-              disabled: !ALLOW_FILE_UPLOAD,
+              title: !allowFileUpload ? disabledFileUploadNote : null,
+              disabled: !allowFileUpload,
               key: 'file'
             }
           ],

src/client/File.js

@@ -34,6 +34,7 @@ function getStorageName (env) {
       storageName = 'S3'
       break
     case 'GCS':
+    case 'USER': // only GCS is supported for user storage
       storageName = 'Cloud Storage'
       break
     default:

src/client/actions/connection.js

@@ -1,4 +1,4 @@
-import { ArchiveConnectionRequest, CreateConnectionRequest, GetConnectionListRequest, Connection, TestConnectionRequest, UpdateConnectionRequest, SetDefaultConnectionRequest } from '../../proto/dekart_pb'
+import { ArchiveConnectionRequest, CreateConnectionRequest, GetConnectionListRequest, Connection, TestConnectionRequest, UpdateConnectionRequest, SetDefaultConnectionRequest, GetGcpProjectListRequest } from '../../proto/dekart_pb'
 import { Dekart } from '../../proto/dekart_pb_service'
 import { updateDatasetConnection } from './dataset'
 import { grpcCall } from './grpc'
@@ -11,8 +11,33 @@ export function closeConnectionDialog () {
   return { type: closeConnectionDialog.name }
 }
 
+export function projectListUpdate (projectsList) {
+  return { type: projectListUpdate.name, projectsList }
+}
+
+export function getProjectList () {
+  return async (dispatch) => {
+    dispatch({ type: getProjectList.name })
+    const request = new GetGcpProjectListRequest()
+    const res = await new Promise((resolve, reject) => {
+      dispatch(grpcCall(Dekart.GetGcpProjectList, request, resolve, (err) => {
+        resolve({ projectsList: [] })
+        if (err.code === 7) {
+          // insufficient permissions for scopes
+          return
+        }
+        return err
+      }))
+    })
+    dispatch(projectListUpdate(res.projectsList))
+  }
+}
+
 export function editConnection (id) {
-  return { type: editConnection.name, id }
+  return async (dispatch) => {
+    dispatch({ type: editConnection.name, id })
+    dispatch(getProjectList())
+  }
 }
 
 export function selectConnection (id) {
@@ -49,21 +74,11 @@ export function archiveConnection (id) {
   }
 }
 
-function getDefaultName (connectionsList, suffix = 0) {
-  const name = suffix ? `BigQuery (${suffix})` : 'BigQuery'
-  if (connectionsList.find(c => c.connectionName === name)) {
-    return getDefaultName(connectionsList, suffix + 1)
-  }
-  return name
-}
-
 export function newConnection (datasetId) {
   return async (dispatch, getState) => {
     dispatch({ type: newConnection.name })
 
-    const connectionName = getDefaultName(getState().connection.list)
     const request = new CreateConnectionRequest()
-    request.setConnectionName(connectionName)
 
     const res = await new Promise((resolve) => {
       dispatch(grpcCall(Dekart.CreateConnection, request, resolve))
@@ -72,6 +87,7 @@ export function newConnection (datasetId) {
       dispatch(updateDatasetConnection(datasetId, res.connection.id))
     }
     dispatch(connectionCreated(res.connection))
+    dispatch(getProjectList())
   }
 }
 

src/client/reducers/connectionReducer.js

@@ -1,5 +1,5 @@
 import { combineReducers } from 'redux'
-import { closeConnectionDialog, connectionChanged, connectionCreated, connectionListUpdate, connectionSaved, editConnection, newConnection, saveConnection, testConnection, testConnectionResponse } from '../actions/connection'
+import { closeConnectionDialog, connectionChanged, connectionCreated, connectionListUpdate, connectionSaved, editConnection, newConnection, projectListUpdate, saveConnection, testConnection, testConnectionResponse } from '../actions/connection'
 import { setEnv } from '../actions/env'
 
 function dialog (state = {
@@ -114,10 +114,20 @@ function userDefined (state = false, action) {
   }
 }
 
+function projects (state = [], action) {
+  switch (action.type) {
+    case projectListUpdate.name:
+      return action.projectsList
+    default:
+      return state
+  }
+}
+
 export default combineReducers({
   dialog,
   test,
   list,
   userDefined,
-  listLoaded
+  listLoaded,
+  projects
 })