From e9c1daa19f362276c23191aa98438bb231f52992 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Thu, 26 Aug 2021 19:32:01 +0800 Subject: [PATCH 01/10] Checkpoint: Removing the workspace volume on Kube works! --- airbyte-commons/src/main/resources/log4j2.xml | 20 +++++++-------- .../io/airbyte/config/helpers/CloudLogs.java | 12 ++++++++- .../config/helpers/LogClientSingleton.java | 25 ++++++++++++++++++- .../io/airbyte/config/helpers/S3Logs.java | 20 +++++++++++++-- .../airbyte/scheduler/app/SchedulerApp.java | 2 +- .../java/io/airbyte/server/ServerApp.java | 2 +- .../workers/DefaultNormalizationWorker.java | 9 ++++++- .../temporal/TemporalAttemptExecution.java | 13 +++++++--- kube/resources/scheduler.yaml | 5 ---- kube/resources/server.yaml | 5 ---- kube/resources/volume-workspace.yaml | 12 --------- 11 files changed, 82 insertions(+), 43 deletions(-) delete mode 100644 kube/resources/volume-workspace.yaml diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml index 6e1dd1a1a29785..f92796e64a5463 100644 --- a/airbyte-commons/src/main/resources/log4j2.xml +++ b/airbyte-commons/src/main/resources/log4j2.xml @@ -47,19 +47,19 @@ 2) Routes don't support routing log output to more than Route --> - + - + - + gcpStorageBucket="${gcp-storage-bucket}" gcpStorageBlobNamePrefix="job-logging${ctx:job_log_path_cloud}"> @@ -90,18 +90,18 @@ - + - + - + gcpStorageBucket="${gcp-storage-bucket}" gcpStorageBlobNamePrefix="app-logging${ctx:cloud_workspace_app_root}"> diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/CloudLogs.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/CloudLogs.java index f4958f92ea286e..5a3603153cf955 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/CloudLogs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/CloudLogs.java @@ -59,11 +59,16 @@ public interface CloudLogs { * @return true if no cloud logging configuration is set; */ static boolean hasEmptyConfigs(LogConfigs configs) { - return !hasS3Configuration(configs) && !hasGcpConfiguration(configs); + return !hasMinioConfiguration(configs) && !hasS3Configuration(configs) && !hasGcpConfiguration(configs); } static CloudLogs createCloudLogClient(LogConfigs configs) { // check if the configs exists, and pick a client. + if (hasMinioConfiguration(configs)) { + LOGGER.info("Creating Minio Log Client"); + return new S3Logs(); + } + if (hasS3Configuration(configs)) { LOGGER.info("Creating AWS Log Client"); return new S3Logs(); @@ -77,6 +82,11 @@ static CloudLogs createCloudLogClient(LogConfigs configs) { throw new RuntimeException("Error no cloud credentials configured.."); } + private static boolean hasMinioConfiguration(LogConfigs configs) { + return !configs.getS3LogBucket().isBlank() && !configs.getAwsAccessKey().isBlank() + && !configs.getAwsSecretAccessKey().isBlank() && !configs.getS3MinioEndpoint().isBlank(); + } + private static boolean hasS3Configuration(LogConfigs configs) { return !configs.getAwsAccessKey().isBlank() && !configs.getAwsSecretAccessKey().isBlank() && diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java index bc1599d916971e..71eebc4ac378da 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java @@ -28,6 +28,7 @@ import io.airbyte.commons.io.IOs; import io.airbyte.config.Configs; import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.EnvConfigs; import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -54,7 +55,10 @@ public class LogClientSingleton { // Any changes to the following values must also be propagated to the log4j2.xml in main/resources. public static String WORKSPACE_MDC_KEY = "workspace_app_root"; + public static String CLOUD_WORKSPACE_MDC_KEY = "cloud_workspace_app_root"; + public static String JOB_LOG_PATH_MDC_KEY = "job_log_path"; + public static String CLOUD_JOB_LOG_PATH_MDC_KEY = "cloud_job_log_path"; // S3/Minio public static String S3_LOG_BUCKET = "S3_LOG_BUCKET"; @@ -138,7 +142,26 @@ public static void deleteLogs(Configs configs, String logPath) { } public static void setJobMdc(Path path) { - MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + if (shouldUseLocalLogs(new EnvConfigs())) { + LOGGER.info("Setting docker job mdc"); + MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + } else { + LOGGER.info("Setting kube job mdc"); + MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + } + } + + public static void setWorkspaceMdc(Path path) { + LOGGER.info("Worker env: {}", new EnvConfigs().getWorkerEnvironment()); + LOGGER.info("env is docker: {}", new EnvConfigs().getWorkerEnvironment().equals(WorkerEnvironment.DOCKER)); + LOGGER.info("empty cloud log config: {}", CloudLogs.hasEmptyConfigs(new LogConfigDelegator(new EnvConfigs()))); + if (shouldUseLocalLogs(new EnvConfigs())) { + LOGGER.info("Setting docker workspace mdc"); + MDC.put(LogClientSingleton.WORKSPACE_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + } else { + LOGGER.info("Setting kube workspace mdc"); + MDC.put(LogClientSingleton.CLOUD_WORKSPACE_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + } } private static boolean shouldUseLocalLogs(Configs configs) { diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java index c9f36d6808a291..63b269751267a9 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java @@ -58,8 +58,15 @@ public class S3Logs implements CloudLogs { private static void assertValidS3Configuration(LogConfigs configs) { Preconditions.checkNotNull(configs.getAwsAccessKey()); Preconditions.checkNotNull(configs.getAwsSecretAccessKey()); - Preconditions.checkNotNull(configs.getS3LogBucketRegion()); Preconditions.checkNotNull(configs.getS3LogBucket()); + + if (configs.getS3LogBucketRegion().isBlank()) { + Preconditions.checkNotNull(configs.getS3MinioEndpoint(), "Either S3 region or endpoint needs to be configured."); + } + + if (configs.getS3MinioEndpoint().isBlank()) { + Preconditions.checkNotNull(configs.getS3LogBucketRegion(), "Either S3 region or endpoint needs to be configured."); + } } @Override @@ -151,14 +158,23 @@ public void deleteLogs(LogConfigs configs, String logPath) { private static void createS3ClientIfNotExist(LogConfigs configs) { if (S3 == null) { assertValidS3Configuration(configs); + + var builder = S3Client.builder(); + + // The V2 S3 client requires region. var s3Region = configs.getS3LogBucketRegion(); - var builder = S3Client.builder().region(Region.of(s3Region)); + if (!s3Region.isBlank()) { + builder.region(Region.of(s3Region)); + } + // The minio client is still on the older client, which uses endpoints. This is + // incompatible with regions. var minioEndpoint = configs.getS3MinioEndpoint(); if (!minioEndpoint.isBlank()) { try { var minioUri = new URI(minioEndpoint); builder.endpointOverride(minioUri); + builder.region(Region.US_EAST_1); } catch (URISyntaxException e) { throw new RuntimeException("Error creating S3 log client to Minio", e); } diff --git a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java index 32c79ed12cabee..49e8c96cab694a 100644 --- a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java +++ b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/SchedulerApp.java @@ -196,7 +196,7 @@ public static void main(String[] args) throws IOException, InterruptedException final Configs configs = new EnvConfigs(); - MDC.put(LogClientSingleton.WORKSPACE_MDC_KEY, LogClientSingleton.getSchedulerLogsRoot(configs).toString()); + LogClientSingleton.setWorkspaceMdc(LogClientSingleton.getSchedulerLogsRoot(configs)); final Path workspaceRoot = configs.getWorkspaceRoot(); LOGGER.info("workspaceRoot = " + workspaceRoot); diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 2ecff8af146728..f7ee4b5375a8ea 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -170,7 +170,7 @@ private static void createWorkspaceIfNoneExists(final ConfigRepository configRep public static ServerRunnable getServer(ServerFactory apiFactory) throws Exception { final Configs configs = new EnvConfigs(); - MDC.put(LogClientSingleton.WORKSPACE_MDC_KEY, LogClientSingleton.getServerLogsRoot(configs).toString()); + LogClientSingleton.setWorkspaceMdc(LogClientSingleton.getSchedulerLogsRoot(configs)); LOGGER.info("Creating config repository..."); final Database configDatabase = new ConfigsDatabaseInstance( diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java index 29c05e233e28df..3f0efa8b9118f3 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java @@ -24,6 +24,8 @@ package io.airbyte.workers; +import io.airbyte.config.Configs.WorkerEnvironment; +import io.airbyte.config.EnvConfigs; import io.airbyte.config.NormalizationInput; import io.airbyte.workers.normalization.NormalizationRunner; import java.nio.file.Files; @@ -60,7 +62,12 @@ public Void run(NormalizationInput input, Path jobRoot) throws WorkerException { try (normalizationRunner) { LOGGER.info("Running normalization."); normalizationRunner.start(); - final Path normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize")); + + Path normalizationRoot = null; + if (new EnvConfigs().getWorkerEnvironment().equals(WorkerEnvironment.DOCKER)) { + normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize")); + } + if (!normalizationRunner.normalize(jobId, attempt, normalizationRoot, input.getDestinationConfiguration(), input.getCatalog(), input.getResourceRequirements())) { throw new WorkerException("Normalization Failed."); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index f08dda23cbcd35..f809c135baddcf 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -28,6 +28,7 @@ import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.commons.io.IOs; +import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.EnvConfigs; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.scheduler.models.JobRunConfig; @@ -108,11 +109,15 @@ public OUTPUT get() { mdcSetter.accept(jobRoot); LOGGER.info("Executing worker wrapper. Airbyte version: {}", new EnvConfigs().getAirbyteVersionOrWarning()); - jobRootDirCreator.accept(jobRoot); - final String workflowId = workflowIdProvider.get(); - final Path workflowIdFile = jobRoot.getParent().resolve(WORKFLOW_ID_FILENAME); - IOs.writeFile(workflowIdFile, workflowId); + if (new EnvConfigs().getWorkerEnvironment().equals(WorkerEnvironment.DOCKER)) { + LOGGER.debug("Creating local workspace directory.."); + jobRootDirCreator.accept(jobRoot); + + final String workflowId = workflowIdProvider.get(); + final Path workflowIdFile = jobRoot.getParent().resolve(WORKFLOW_ID_FILENAME); + IOs.writeFile(workflowIdFile, workflowId); + } final Worker worker = workerSupplier.get(); final CompletableFuture outputFuture = new CompletableFuture<>(); diff --git a/kube/resources/scheduler.yaml b/kube/resources/scheduler.yaml index d21d0835d7d840..61b98af81abdd0 100644 --- a/kube/resources/scheduler.yaml +++ b/kube/resources/scheduler.yaml @@ -199,15 +199,10 @@ spec: volumeMounts: - name: airbyte-volume-configs mountPath: /configs - - name: airbyte-volume-workspace - mountPath: /workspace - name: gcs-log-creds-volume mountPath: /secrets/gcs-log-creds readOnly: true volumes: - - name: airbyte-volume-workspace - persistentVolumeClaim: - claimName: airbyte-volume-workspace - name: airbyte-volume-configs persistentVolumeClaim: claimName: airbyte-volume-configs diff --git a/kube/resources/server.yaml b/kube/resources/server.yaml index 4bac02ed2c4d39..152174c1d9bf65 100644 --- a/kube/resources/server.yaml +++ b/kube/resources/server.yaml @@ -158,15 +158,10 @@ spec: volumeMounts: - name: airbyte-volume-configs mountPath: /configs - - name: airbyte-volume-workspace - mountPath: /workspace - name: gcs-log-creds-volume mountPath: /secrets/gcs-log-creds readOnly: true volumes: - - name: airbyte-volume-workspace - persistentVolumeClaim: - claimName: airbyte-volume-workspace - name: airbyte-volume-configs persistentVolumeClaim: claimName: airbyte-volume-configs diff --git a/kube/resources/volume-workspace.yaml b/kube/resources/volume-workspace.yaml deleted file mode 100644 index 6094deef2b824c..00000000000000 --- a/kube/resources/volume-workspace.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: airbyte-volume-workspace - labels: - airbyte: volume-workspace -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Mi From 847e75a0a93d09d200857edcb78aa2ce14e8c983 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Thu, 26 Aug 2021 19:40:46 +0800 Subject: [PATCH 02/10] Add commets. --- .../io/airbyte/config/helpers/LogClientSingleton.java | 3 --- .../src/main/java/io/airbyte/config/helpers/S3Logs.java | 8 ++++---- .../io/airbyte/workers/DefaultNormalizationWorker.java | 1 + .../workers/temporal/TemporalAttemptExecution.java | 1 + 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java index 71eebc4ac378da..9fceb9fa8a6449 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java @@ -152,9 +152,6 @@ public static void setJobMdc(Path path) { } public static void setWorkspaceMdc(Path path) { - LOGGER.info("Worker env: {}", new EnvConfigs().getWorkerEnvironment()); - LOGGER.info("env is docker: {}", new EnvConfigs().getWorkerEnvironment().equals(WorkerEnvironment.DOCKER)); - LOGGER.info("empty cloud log config: {}", CloudLogs.hasEmptyConfigs(new LogConfigDelegator(new EnvConfigs()))); if (shouldUseLocalLogs(new EnvConfigs())) { LOGGER.info("Setting docker workspace mdc"); MDC.put(LogClientSingleton.WORKSPACE_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java index 63b269751267a9..f21d96796cca7b 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/S3Logs.java @@ -60,6 +60,7 @@ private static void assertValidS3Configuration(LogConfigs configs) { Preconditions.checkNotNull(configs.getAwsSecretAccessKey()); Preconditions.checkNotNull(configs.getS3LogBucket()); + // When region is set, endpoint cannot be set and vice versa. if (configs.getS3LogBucketRegion().isBlank()) { Preconditions.checkNotNull(configs.getS3MinioEndpoint(), "Either S3 region or endpoint needs to be configured."); } @@ -161,20 +162,19 @@ private static void createS3ClientIfNotExist(LogConfigs configs) { var builder = S3Client.builder(); - // The V2 S3 client requires region. + // Pure S3 Client var s3Region = configs.getS3LogBucketRegion(); if (!s3Region.isBlank()) { builder.region(Region.of(s3Region)); } - // The minio client is still on the older client, which uses endpoints. This is - // incompatible with regions. + // The Minio S3 client. var minioEndpoint = configs.getS3MinioEndpoint(); if (!minioEndpoint.isBlank()) { try { var minioUri = new URI(minioEndpoint); builder.endpointOverride(minioUri); - builder.region(Region.US_EAST_1); + builder.region(Region.US_EAST_1); // Although this is not used, the S3 client will error out if this is not set. Set a stub value. } catch (URISyntaxException e) { throw new RuntimeException("Error creating S3 log client to Minio", e); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java index 3f0efa8b9118f3..fe481954c34db9 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultNormalizationWorker.java @@ -64,6 +64,7 @@ public Void run(NormalizationInput input, Path jobRoot) throws WorkerException { normalizationRunner.start(); Path normalizationRoot = null; + // There are no shared volumes on Kube; only create this for Docker. if (new EnvConfigs().getWorkerEnvironment().equals(WorkerEnvironment.DOCKER)) { normalizationRoot = Files.createDirectories(jobRoot.resolve("normalize")); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index f809c135baddcf..99d8e30b4e4fc5 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -110,6 +110,7 @@ public OUTPUT get() { LOGGER.info("Executing worker wrapper. Airbyte version: {}", new EnvConfigs().getAirbyteVersionOrWarning()); + // There are no shared volumes on Kube; only do this for Docker. if (new EnvConfigs().getWorkerEnvironment().equals(WorkerEnvironment.DOCKER)) { LOGGER.debug("Creating local workspace directory.."); jobRootDirCreator.accept(jobRoot); From 9e0f5f8c55df80fdc828132b86675943af6dec09 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Thu, 26 Aug 2021 22:49:19 +0800 Subject: [PATCH 03/10] Fix test. --- .../src/test/java/io/airbyte/config/helpers/CloudLogsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java index 9e0280a4923274..ba2ed3373475b5 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java @@ -34,6 +34,7 @@ public class CloudLogsTest { @Test public void createCloudLogClientTestAws() { var configs = Mockito.mock(LogConfigs.class); + Mockito.when(configs.getS3MinioEndpoint()).thenReturn(""); Mockito.when(configs.getAwsAccessKey()).thenReturn("access-key"); Mockito.when(configs.getAwsSecretAccessKey()).thenReturn("access-key-secret"); Mockito.when(configs.getS3LogBucket()).thenReturn("test-bucket"); From d754f6b22e45bb7ffa63db956dd26d7768cecdd8 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 1 Sep 2021 17:55:29 +0800 Subject: [PATCH 04/10] Workspace MDC should respect the previous configuration. --- .../java/io/airbyte/config/helpers/LogClientSingleton.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java index 9fceb9fa8a6449..be54877df661c1 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java @@ -154,10 +154,10 @@ public static void setJobMdc(Path path) { public static void setWorkspaceMdc(Path path) { if (shouldUseLocalLogs(new EnvConfigs())) { LOGGER.info("Setting docker workspace mdc"); - MDC.put(LogClientSingleton.WORKSPACE_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + MDC.put(LogClientSingleton.WORKSPACE_MDC_KEY, path.toString()); } else { LOGGER.info("Setting kube workspace mdc"); - MDC.put(LogClientSingleton.CLOUD_WORKSPACE_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + MDC.put(LogClientSingleton.CLOUD_WORKSPACE_MDC_KEY, path.toString()); } } From 59e852042337e8e1a885fce84ba0123a42707dab Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 1 Sep 2021 18:21:33 +0800 Subject: [PATCH 05/10] Fix kubernetes acceptance test. --- .../java/io/airbyte/test/acceptance/AcceptanceTests.java | 2 -- kube/resources/kustomization.yaml | 1 - 2 files changed, 3 deletions(-) diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java index 5fb9c70da8a82d..ecd95bf5331c47 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java @@ -754,8 +754,6 @@ public void testCheckpointing() throws Exception { @Test @Order(14) - @DisabledIfEnvironmentVariable(named = "KUBE", - matches = "true") public void testRedactionOfSensitiveRequestBodies() throws Exception { // check that the source password is not present in the logs final List serverLogLines = java.nio.file.Files.readAllLines( diff --git a/kube/resources/kustomization.yaml b/kube/resources/kustomization.yaml index 934f54b26adb24..f6e422853a6938 100644 --- a/kube/resources/kustomization.yaml +++ b/kube/resources/kustomization.yaml @@ -12,5 +12,4 @@ resources: - temporal.yaml - volume-configs.yaml - volume-db.yaml - - volume-workspace.yaml - webapp.yaml From eb95c5d770e75ce40a15e804bf494fcb2666ee1c Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 1 Sep 2021 18:23:03 +0800 Subject: [PATCH 06/10] Fix gcp log configuration error. --- airbyte-commons/src/main/resources/log4j2.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml index f92796e64a5463..b7e7eafe2433f0 100644 --- a/airbyte-commons/src/main/resources/log4j2.xml +++ b/airbyte-commons/src/main/resources/log4j2.xml @@ -59,7 +59,7 @@ s3Bucket="${s3-bucket}" s3Path="job-logging${ctx:cloud_job_log_path}" s3Region="${s3-region}" s3AwsKey="${s3-aws-key}" s3AwsSecret="${s3-aws-secret}" s3ServiceEndpoint="${s3-minio-endpoint}" s3PathStyleAccess="${s3-path-style-access}" - gcpStorageBucket="${gcp-storage-bucket}" gcpStorageBlobNamePrefix="job-logging${ctx:job_log_path_cloud}"> + gcpStorageBucket="${gcp-storage-bucket}" gcpStorageBlobNamePrefix="job-logging${ctx:cloud_job_log_path}"> From 7ba8f21b96626ad2ae43d4c79b0a25c7ac433baa Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 1 Sep 2021 19:24:22 +0800 Subject: [PATCH 07/10] Print out docker images. --- .github/workflows/gradle.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index cb8440ce1afaa9..1983129cae3cb0 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -252,7 +252,9 @@ jobs: # - name: Get Docker Space # run: docker run --rm busybox df -h - # make sure these always run before pushing platform docker images + - name: Check documentation + run: docker images | head -n 10 + - name: Run End-to-End Acceptance Tests run: ./tools/bin/acceptance_test.sh From 5a7e4d8ac765b2ce5362c1f9b36e0d000099e04d Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 1 Sep 2021 19:27:52 +0800 Subject: [PATCH 08/10] Correct error! --- airbyte-server/src/main/java/io/airbyte/server/ServerApp.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index f7ee4b5375a8ea..9ab9215016a2d3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -170,7 +170,7 @@ private static void createWorkspaceIfNoneExists(final ConfigRepository configRep public static ServerRunnable getServer(ServerFactory apiFactory) throws Exception { final Configs configs = new EnvConfigs(); - LogClientSingleton.setWorkspaceMdc(LogClientSingleton.getSchedulerLogsRoot(configs)); + LogClientSingleton.setWorkspaceMdc(LogClientSingleton.getServerLogsRoot(configs)); LOGGER.info("Creating config repository..."); final Database configDatabase = new ConfigsDatabaseInstance( From 56651916f3c603f3a13609d0ef2546fececf350f Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 1 Sep 2021 21:41:25 +0800 Subject: [PATCH 09/10] Remove debug statement. --- .github/workflows/gradle.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 1983129cae3cb0..b3babb14afab92 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -252,9 +252,6 @@ jobs: # - name: Get Docker Space # run: docker run --rm busybox df -h - - name: Check documentation - run: docker images | head -n 10 - - name: Run End-to-End Acceptance Tests run: ./tools/bin/acceptance_test.sh From 50b17a4487c3789de4a5401e365d93771c6d8ec2 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 1 Sep 2021 21:56:27 +0800 Subject: [PATCH 10/10] Add test for Minio client. --- .../io/airbyte/config/helpers/CloudLogsTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java index ba2ed3373475b5..2e9a077ebb8fb8 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java @@ -31,6 +31,18 @@ public class CloudLogsTest { + @Test + public void createCloudLogClientTestMinio() { + var configs = Mockito.mock(LogConfigs.class); + Mockito.when(configs.getS3MinioEndpoint()).thenReturn("minio-endpoint"); + Mockito.when(configs.getAwsAccessKey()).thenReturn("access-key"); + Mockito.when(configs.getAwsSecretAccessKey()).thenReturn("access-key-secret"); + Mockito.when(configs.getS3LogBucket()).thenReturn("test-bucket"); + Mockito.when(configs.getS3LogBucketRegion()).thenReturn(""); + + assertEquals(S3Logs.class, CloudLogs.createCloudLogClient(configs).getClass()); + } + @Test public void createCloudLogClientTestAws() { var configs = Mockito.mock(LogConfigs.class);