Merge pull request #150 from dekart-xyz/fix-storage-connection-test

Fix storage connection test
dekart-xyz · Dec 1, 2023 · 95f97b6 · 95f97b6
2 parents 706f376 + ff56552
commit 95f97b6
Show file tree

Hide file tree

Showing 4 changed files with 121 additions and 9 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -1,6 +1,6 @@
 name: e2e_tests
 
-on: [workflow_call]    
+on: [workflow_call]
 
 env:
   DEKART_POSTGRES_PASSWORD: dekart
@@ -41,6 +41,65 @@ jobs:
         with:
           path: ${{ env.IMAGE_PATH }}
           key: ${{ env.IMAGE_CACHE_KEY }}
+  google-oauth:
+    runs-on: ubuntu-latest
+    needs: build
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: ${{ env.DEKART_POSTGRES_PASSWORD }}
+          POSTGRES_USER: ${{ env.DEKART_POSTGRES_USER }}
+          POSTGRES_DB: ${{ env.DEKART_POSTGRES_DB }}
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Create google credentials
+        run: |
+          echo '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}' > $GOOGLE_APPLICATION_CREDENTIALS
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Load Docker image from cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.IMAGE_PATH }}
+          key: ${{ env.IMAGE_CACHE_KEY }}
+          fail-on-cache-miss: true
+      - name: Load Docker image
+        run: |
+          docker load --input ${{ env.IMAGE_PATH }}
+          docker image ls -a
+      - name: E2E test
+        run: |
+          docker run -i \
+            --network="host" \
+            -v /tmp/cypress/videos:/dekart/cypress/videos/ \
+          	-v /tmp/cypress/screenshots:/dekart/cypress/screenshots/ \
+            -e DEKART_POSTGRES_USER=$DEKART_POSTGRES_USER \
+            -e DEKART_POSTGRES_PASSWORD=$DEKART_POSTGRES_PASSWORD \
+            -e DEKART_POSTGRES_PORT=$DEKART_POSTGRES_PORT \
+            -e DEKART_POSTGRES_HOST=localhost \
+            -e DEKART_ALLOW_FILE_UPLOAD=1 \
+            -e DEKART_REQUIRE_GOOGLE_OAUTH=1 \
+            -e DEKART_GOOGLE_OAUTH_CLIENT_ID=${{ secrets.GOOGLE_OAUTH_CLIENT_ID }} \
+            -e DEKART_GOOGLE_OAUTH_SECRET=${{ secrets.GOOGLE_OAUTH_SECRET }} \
+            -e DEKART_DEV_REFRESH_TOKEN=${{ secrets.DEV_REFRESH_TOKEN }} \
+            -e TEST_SPEC=cypress/e2e/google-oauth \
+            -e CYPRESS_CI=1 \
+            e2etest:latest
+      - name: Upload cypress artifacts
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: google-oauth
+          path: /tmp/cypress
   bigquery:
     runs-on: ubuntu-latest
     needs: build
@@ -55,7 +114,7 @@ jobs:
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
-          --health-retries 5    
+          --health-retries 5
         ports:
           - 5432:5432
     steps:
@@ -75,7 +134,7 @@ jobs:
       - name: Load Docker image
         run: |
           docker load --input ${{ env.IMAGE_PATH }}
-          docker image ls -a        
+          docker image ls -a
       - name: E2E test BigQuery
         run: |
           docker run -i \
@@ -114,7 +173,7 @@ jobs:
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
-          --health-retries 5    
+          --health-retries 5
         ports:
           - 5432:5432
     steps:
@@ -131,7 +190,7 @@ jobs:
       - name: Load Docker image
         run: |
           docker load --input ${{ env.IMAGE_PATH }}
-          docker image ls -a        
+          docker image ls -a
       - name: E2E test Athena
         run: |
           docker run -i \
@@ -176,7 +235,7 @@ jobs:
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
-          --health-retries 5    
+          --health-retries 5
         ports:
           - 5432:5432
     steps:
@@ -193,7 +252,7 @@ jobs:
       - name: Load Docker image
         run: |
           docker load --input ${{ env.IMAGE_PATH }}
-          docker image ls -a        
+          docker image ls -a
       - name: E2E test Snowflake
         run: |
           docker run -i \

diff --git a/Makefile b/Makefile
@@ -40,7 +40,26 @@ gotest:
 	docker buildx build -f ./Dockerfile --target gotest .
 
 e2e: bq athena snowflake
-
+
+google-oauth:
+	docker buildx build --tag ${DEKART_DOCKER_E2E_TAG} -o type=image -f ./Dockerfile --target e2etest .
+	docker run -it --rm \
+	-v $$(pwd)/cypress/videos:/dekart/cypress/videos/ \
+	-v $$(pwd)/cypress/screenshots:/dekart/cypress/screenshots/ \
+	-e DEKART_POSTGRES_DB=${DEKART_POSTGRES_DB} \
+	-e DEKART_POSTGRES_USER=${DEKART_POSTGRES_USER} \
+	-e DEKART_POSTGRES_PASSWORD=${DEKART_POSTGRES_PASSWORD} \
+	-e DEKART_POSTGRES_PORT=${DEKART_POSTGRES_PORT} \
+	-e DEKART_POSTGRES_HOST=host.docker.internal \
+	-e DEKART_ALLOW_FILE_UPLOAD=1 \
+	-e DEKART_REQUIRE_GOOGLE_OAUTH=1 \
+	-e DEKART_GOOGLE_OAUTH_CLIENT_ID=${DEKART_GOOGLE_OAUTH_CLIENT_ID} \
+	-e DEKART_GOOGLE_OAUTH_SECRET=${DEKART_GOOGLE_OAUTH_SECRET} \
+	-e DEKART_DEV_REFRESH_TOKEN=${DEKART_DEV_REFRESH_TOKEN} \
+	-e TEST_SPEC=cypress/e2e/google-oauth \
+	-e CYPRESS_CI=1 \
+	${DEKART_DOCKER_E2E_TAG}
+
 bq:
 	docker buildx build --tag ${DEKART_DOCKER_E2E_TAG} -o type=image -f ./Dockerfile --target e2etest .
 	docker run -it --rm \

diff --git a/cypress/e2e/google-oauth/testConnection.cy.js b/cypress/e2e/google-oauth/testConnection.cy.js
@@ -0,0 +1,22 @@
+/* eslint-disable no-undef */
+import copy from '../../fixtures/copy.json'
+
+describe('basic query flow', () => {
+  it('should make simple bigquery query and get ready status', () => {
+    cy.visit('/')
+    cy.get(`button:contains("${copy.create_report}")`).click()
+
+    // create connection
+    cy.get('#dekart-connection-select').click()
+    cy.get('div.ant-select-item-option-content:contains("New")').click()
+    const randomConnectionName = `test-${Math.floor(Math.random() * 1000000)}`
+    cy.get('div.ant-modal-title').should('contain', 'Edit Connection')
+    cy.get('input#connectionName').clear()
+    cy.get('input#connectionName').type(randomConnectionName)
+    cy.get('input#bigqueryProjectId').type('dekart-dev')
+    cy.get('input#cloudStorageBucket').type('dekart-dev')
+    cy.get('button:contains("Test Connection")').click()
+    cy.get('button#saveConnection').should('be.enabled')
+    cy.get('button#saveConnection').click()
+  })
+})
diff --git a/src/server/storage/storage.go b/src/server/storage/storage.go
@@ -17,6 +17,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/sync/errgroup"
+	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 )
 
@@ -75,13 +76,24 @@ func TestConnection(ctx context.Context, connection *proto.Connection) (*proto.T
 		log.Fatal().Err(err).Send()
 	}
 	bucket := client.Bucket(connection.CloudStorageBucket)
-	_, err = bucket.Attrs(ctx)
+
+	// Try to list the objects in the bucket
+	it := bucket.Objects(ctx, nil)
+	_, err = it.Next()
 	if err != nil {
+		if err == iterator.Done {
+			// The bucket is empty, but we have the necessary permission
+			return &proto.TestConnectionResponse{
+				Success: true,
+			}, nil
+		}
+		// We got an error, so we don't have the necessary permission
 		return &proto.TestConnectionResponse{
 			Success: false,
 			Error:   err.Error(),
 		}, nil
 	}
+
 	return &proto.TestConnectionResponse{
 		Success: true,
 	}, nil