Job Sync fails with StackOverflow error: an exception occurred processing Appender SecretMaskRewrite

[ragyabraham]

Helm Chart Version

0.87.4

What step the error happened?

During the Sync

Relevant information

Currently running airbyte on GKE using the airbyte helm chart. However, when trying to sync a datasource we get the following error in the worker pod. This causes the sync to timeout

2024-06-03T07:36:33.798595266Z Activity Executor taskQueue="SYNC", namespace="default": 2 ERROR An exception occurred processing Appender SecretMaskRewrite org.apache.logging.log4j.core.appender.AppenderLoggingException: java.lang.StackOverflowError
	at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:164)
	at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133)
	at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
	at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:88)
	at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:705)
	at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:663)
	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:639)
	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:575)
	at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:92)
	at org.apache.logging.log4j.core.Logger.log(Logger.java:169)
	at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2906)
	at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2859)
	at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2841)
	at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2648)
	at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:2415)
	at org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:183)
	at io.airbyte.workers.temporal.sync.ReplicationActivityImpl.lambda$replicateV2$3(ReplicationActivityImpl.java:174)
	at io.airbyte.commons.temporal.HeartbeatUtils.withBackgroundHeartbeat(HeartbeatUtils.java:57)
	at io.airbyte.workers.temporal.sync.ReplicationActivityImpl.replicateV2(ReplicationActivityImpl.java:170)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base/java.lang.reflect.Method.invoke(Method.java:580)
	at io.temporal.internal.activity.RootActivityInboundCallsInterceptor$POJOActivityInboundCallsInterceptor.executeActivity(RootActivityInboundCallsInterceptor.java:64)
	at io.temporal.internal.activity.RootActivityInboundCallsInterceptor.execute(RootActivityInboundCallsInterceptor.java:43)
	at io.temporal.internal.activity.ActivityTaskExecutors$BaseActivityTaskExecutor.execute(ActivityTaskExecutors.java:107)
	at io.temporal.internal.activity.ActivityTaskHandlerImpl.handle(ActivityTaskHandlerImpl.java:124)
	at io.temporal.internal.worker.ActivityWorker$TaskHandlerImpl.handleActivity(ActivityWorker.java:278)
	at io.temporal.internal.worker.ActivityWorker$TaskHandlerImpl.handle(ActivityWorker.java:243)
	at io.temporal.internal.worker.ActivityWorker$TaskHandlerImpl.handle(ActivityWorker.java:216)
	at io.temporal.internal.worker.PollTaskExecutor.lambda$process$0(PollTaskExecutor.java:105)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.StackOverflowError
	at java.base/java.util.regex.Pattern.lambda$negate$7(Pattern.java:5818)
	at java.base/java.util.regex.Pattern$CharProperty.match(Pattern.java:4109)
	at java.base/java.util.regex.Pattern$Branch.match(Pattern.java:4914)
	at java.base/java.util.regex.Pattern$GroupHead.match(Pattern.java:4969)
	at java.base/java.util.regex.Pattern$Loop.match(Pattern.java:5078)
	at java.base/java.util.regex.Pattern$GroupTail.match(Pattern.java:5000)
	at java.base/java.util.regex.Pattern$BranchConn.match(Pattern.java:4878)
	at java.base/java.util.regex.Pattern$CharProperty.match(Pattern.java:4110)
	at java.base/java.util.regex.Pattern$Branch.match(Pattern.java:4914)
	at java.base/java.util.regex.Pattern$GroupHead.match(Pattern.java:4969)
	at java.base/java.util.regex.Pattern$Loop.match(Pattern.java:5078)
	at java.base/java.util.regex.Pattern$GroupTail.match(Pattern.java:5000)
	at java.base/java.util.regex.Pattern$BranchConn.match(Pattern.java:4878)
	at java.base/java.util.regex.Pattern$CharProperty.match(Pattern.java:4110)
	at java.base/java.util.regex.Pattern$Branch.match(Pattern.java:4914)
	at java.base/java.util.regex.Pattern$GroupHead.match(Pattern.java:4969)
	at java.base/java.util.regex.Pattern$Loop.match(Pattern.java:5078)
	at java.base/java.util.regex.Pattern$GroupTail.match(Pattern.java:5000)
	at java.base/java.util.regex.Pattern$BranchConn.match(Pattern.java:4878)
	at java.base/java.util.regex.Pattern$CharProperty.match(Pattern.java:4110)
	at java.base/java.util.regex.Pattern$Branch.match(Pattern.java:4914)
	at java.base/java.util.regex.Pattern$GroupHead.match(Pattern.java:4969)
	at java.base/java.util.regex.Pattern$Loop.match(Pattern.java:5078)
	at java.base/java.util.regex.Pattern$GroupTail.match(Pattern.java:5000)
	at java.base/java.util.regex.Pattern$BranchConn.match(Pattern.java:4878)
	at java.base/java.util.regex.Pattern$CharProperty.match(Pattern.java:4110)
	at java.base/java.util.regex.Pattern$Branch.match(Pattern.java:4914)
	at java.base/java.util.regex.Pattern$GroupHead.match(Pattern.java:4969)
	at java.base/java.util.regex.Pattern$Loop.match(Pattern.java:5078)
	at java.base/java.util.regex.Pattern$GroupTail.match(Pattern.java:5000)
	at java.base/java.util.regex.Pattern$BranchConn.match(Pattern.java:4878)
	at java.base/java.util.regex.Pattern$CharProperty.match(Pattern.java:4110)
	at java.base/java.util.regex.Pattern$Branch.match(Pattern.java:4914)
	
	

2024-06-03 07:36:33 platform > Cloud storage job log path: /workspace/6/1/logs.log
2024-06-03 07:36:33 platform > Executing worker wrapper. Airbyte version: 0.60.1
2024-06-03 07:36:33 platform > Attempt 0 to save workflow id for cancellation
2024-06-03 07:36:33 platform > Creating orchestrator-repl-job-6-attempt-1 for attempt number: 1
2024-06-03 07:36:33 platform > There are currently running pods for the connection: [orchestrator-repl-job-6-attempt-0]. Killing these pods to enforce one execution at a time.
2024-06-03 07:36:33 platform > Attempting to delete pods: [orchestrator-repl-job-6-attempt-0]
2024-06-03 07:36:33 platform > Waiting for deletion...
2024-06-03 07:36:34 platform > Successfully deleted all running pods for the connection!
2024-06-03 07:36:35 platform > Waiting for pod to be running...

Relevant log output

No response

[marcosmarxm]

@ragyabraham, could you give more details about your deployment? Like resources of your cluster and the number and type of connectors you're using?

[ragyabraham]

Hi, @marcosmarxm. The deployment is currently very simple. We are just trying to get it to a state where it works and syncs data.

At the moment, everything is running on a single cluster that has a single node pool with a limit of 6 nodes. Airbyte is deployed with a vanilla helm chart, we have not changed any of the values as of yet. We are only running a single connector, BigQuery.

[marcosmarxm]

@ragyabraham did you configured the secret manager or it this actually a defautl chart value run? I didn't see any other user complain using the default values.

[ragyabraham]

@marcosmarxm This is a complete vanilla implementation. I have not changed any values at all. The only thing that's 'changed' is that it's being deployed in a specific namespace.
I think what would help is if you could explain to me what operation is failing/what service it's struggling to reach. That would give me some insight into what I need to do to fix it. Because, at the moment, this error is straight-up gibberish to me :)

Also, I should add that this error only occurs on the sync after airbyte has pulled the schema from BQ. Which tells me it can communicate with BQ. So not sure why it fails on the sync step

[klahdenk]

I'm also experiencing this issue using the almost default docker-compose setup v0.63.0. Only real difference to the default is that we have AWS secrets manager enabled for secret storage.

[ragyabraham]

I'm also experiencing this issue using the almost default docker-compose setup v0.63.0. Only real difference to the default is that we have AWS secrets manager enabled for secret storage.

I tried switching to Google Secret Manger as the secret store, following the instructions in this doc. However, gives me the below error

2024-06-19 00:04:35 ERROR i.a.s.e.UncaughtExceptionHandler(handle):31 - Uncaught exception
java.lang.RuntimeException: com.google.gson.stream.MalformedJsonException: Use JsonReader.setLenient(true) to accept malformed JSON at line 1 column 1 path $
	at io.airbyte.server.apis.ApiHelper.execute(ApiHelper.java:46) ~[io.airbyte-airbyte-server-0.60.1.jar:?]
	at io.airbyte.server.apis.SourceApiController.createSource(SourceApiController.java:94) ~[io.airbyte-airbyte-server-0.60.1.jar:?]
	at io.airbyte.server.apis.$SourceApiController$Definition$Exec.dispatch(Unknown Source) ~[io.airbyte-airbyte-server-0.60.1.jar:?]
	at io.micronaut.context.AbstractExecutableMethodsDefinition$DispatchedExecutableMethod.invokeUnsafe(AbstractExecutableMethodsDefinition.java:461) ~[micronaut-inject-4.4.8.jar:4.4.8]
	at io.micronaut.context.DefaultBeanContext$BeanContextUnsafeExecutionHandle.invokeUnsafe(DefaultBeanContext.java:4232) ~[micronaut-inject-4.4.8.jar:4.4.8]
	at io.micronaut.web.router.AbstractRouteMatch.execute(AbstractRouteMatch.java:271) ~[micronaut-router-4.4.8.jar:4.4.8]
	at io.micronaut.http.server.RouteExecutor.executeRouteAndConvertBody(RouteExecutor.java:488) ~[micronaut-http-server-4.4.8.jar:4.4.8]
	at io.micronaut.http.server.RouteExecutor.lambda$callRoute$6(RouteExecutor.java:465) ~[micronaut-http-server-4.4.8.jar:4.4.8]
	at io.micronaut.core.execution.ExecutionFlow.lambda$async$1(ExecutionFlow.java:87) ~[micronaut-core-4.4.8.jar:4.4.8]
	at io.micronaut.core.propagation.PropagatedContext.lambda$wrap$3(PropagatedContext.java:211) ~[micronaut-core-4.4.8.jar:4.4.8]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: com.google.gson.stream.MalformedJsonException: Use JsonReader.setLenient(true) to accept malformed JSON at line 1 column 1 path $
	at com.google.gson.stream.JsonReader.syntaxError(JsonReader.java:1659) ~[gson-2.10.1.jar:?]
	at com.google.gson.stream.JsonReader.checkLenient(JsonReader.java:1465) ~[gson-2.10.1.jar:?]
	at com.google.gson.stream.JsonReader.doPeek(JsonReader.java:602) ~[gson-2.10.1.jar:?]
	at com.google.gson.stream.JsonReader.peek(JsonReader.java:433) ~[gson-2.10.1.jar:?]
	at com.google.api.client.json.gson.GsonParser.nextToken(GsonParser.java:149) ~[google-http-client-gson-1.44.1.jar:?]
	at com.google.api.client.json.JsonParser.startParsing(JsonParser.java:213) ~[google-http-client-1.44.1.jar:1.44.1]
	at com.google.api.client.json.JsonParser.parse(JsonParser.java:358) ~[google-http-client-1.44.1.jar:1.44.1]
	at com.google.api.client.json.JsonParser.parse(JsonParser.java:335) ~[google-http-client-1.44.1.jar:1.44.1]
	at com.google.api.client.json.JsonObjectParser.parseAndClose(JsonObjectParser.java:79) ~[google-http-client-1.44.1.jar:1.44.1]
	at com.google.api.client.json.JsonObjectParser.parseAndClose(JsonObjectParser.java:73) ~[google-http-client-1.44.1.jar:1.44.1]
	at com.google.auth.oauth2.GoogleCredentials.fromStream(GoogleCredentials.java:189) ~[google-auth-library-oauth2-http-1.23.0.jar:1.23.0]
	at com.google.auth.oauth2.ServiceAccountCredentials.fromStream(ServiceAccountCredentials.java:469) ~[google-auth-library-oauth2-http-1.23.0.jar:1.23.0]
	at com.google.auth.oauth2.ServiceAccountCredentials.fromStream(ServiceAccountCredentials.java:452) ~[google-auth-library-oauth2-http-1.23.0.jar:1.23.0]
	at io.airbyte.config.secrets.persistence.GoogleSecretManagerServiceClient.createClient(GoogleSecretManagerPersistence.kt:138) ~[io.airbyte.airbyte-config-config-secrets-0.60.1.jar:?]
	at io.airbyte.config.secrets.persistence.GoogleSecretManagerPersistence.writeWithExpiry(GoogleSecretManagerPersistence.kt:96) ~[io.airbyte.airbyte-config-config-secrets-0.60.1.jar:?]
	at io.airbyte.config.secrets.SecretsRepositoryWriter.splitSecretConfig(SecretsRepositoryWriter.kt:144) ~[io.airbyte.airbyte-config-config-secrets-0.60.1.jar:?]
	at io.airbyte.config.secrets.SecretsRepositoryWriter.splitSecretConfig$default(SecretsRepositoryWriter.kt:128) ~[io.airbyte.airbyte-config-config-secrets-0.60.1.jar:?]
	at io.airbyte.config.secrets.SecretsRepositoryWriter.statefulSplitSecrets(SecretsRepositoryWriter.kt:58) ~[io.airbyte.airbyte-config-config-secrets-0.60.1.jar:?]
	at io.airbyte.data.services.impls.jooq.SourceServiceJooqImpl.writeSourceConnectionWithSecrets(SourceServiceJooqImpl.java:723) ~[io.airbyte-airbyte-data-0.60.1.jar:?]
	at io.airbyte.commons.server.handlers.SourceHandler.persistSourceConnection(SourceHandler.java:492) ~[io.airbyte-airbyte-commons-server-0.60.1.jar:?]
	at io.airbyte.commons.server.handlers.SourceHandler.createSource(SourceHandler.java:177) ~[io.airbyte-airbyte-commons-server-0.60.1.jar:?]
	at io.airbyte.commons.server.handlers.SourceHandler.createSourceWithOptionalSecret(SourceHandler.java:140) ~[io.airbyte-airbyte-commons-server-0.60.1.jar:?]
	at io.airbyte.server.apis.SourceApiController.lambda$createSource$4(SourceApiController.java:94) ~[io.airbyte-airbyte-server-0.60.1.jar:?]
	at io.airbyte.server.apis.ApiHelper.execute(ApiHelper.java:28) ~[io.airbyte-airbyte-server-0.60.1.jar:?]