From afbe09c62647bca8eb9aa51f5ab63a13396800f7 Mon Sep 17 00:00:00 2001 From: losalex <90795544+losalex@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:26:21 -0800 Subject: [PATCH] feat: Add support for batching configuration (#977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add support for batching configuration * Add space * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Remove redundant HTML appender tag * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Add failing dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .readme-partials.yaml | 18 +++- README.md | 18 +++- pom.xml | 10 ++- .../logback/LogbackBatchingSettings.java | 87 +++++++++++++++++++ .../logging/logback/LoggingAppender.java | 32 ++++++- .../logback/LoggingAppenderLogbackTest.java | 57 ++++++++++++ .../google/cloud/logging/logback/logback.xml | 57 ++++++++++++ 7 files changed, 273 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java create mode 100644 src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java create mode 100644 src/test/java/com/google/cloud/logging/logback/logback.xml diff --git a/.readme-partials.yaml b/.readme-partials.yaml index 3a9925d33..209e8a391 100644 --- a/.readme-partials.yaml +++ b/.readme-partials.yaml @@ -17,7 +17,7 @@ custom_content: | application.log - WARNING + WARN SYNC @@ -43,7 +43,21 @@ custom_content: | - true + true + + + + 100 + 1000 + 500 + 10000 + 100000 + Ignore + diff --git a/README.md b/README.md index b77c30efd..5aabe0ea6 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ See [Logback filters](https://logback.qos.ch/manual/filters.html#thresholdFilter application.log - WARNING + WARN SYNC @@ -114,7 +114,21 @@ See [Logback filters](https://logback.qos.ch/manual/filters.html#thresholdFilter - true + true + + + + 100 + 1000 + 500 + 10000 + 100000 + Ignore + diff --git a/pom.xml b/pom.xml index bfea39f4e..166bb603a 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,15 @@ com.google.protobuf protobuf-java - + + + com.google.api + gax + + + org.threeten + threetenbp + diff --git a/src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java b/src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java new file mode 100644 index 000000000..48e8b7b9a --- /dev/null +++ b/src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java @@ -0,0 +1,87 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.logging.logback; + +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController.LimitExceededBehavior; +import org.threeten.bp.Duration; + +/** + * This class is used only to provide batch settings configuration in logback.xml since {@link + * com.google.api.gax.batching.BatchingSettings} cannot be used as is with logback configuration + * described in https://logback.qos.ch/manual/configuration.html. All data members below are simply + * copy of {@link com.google.api.gax.batching.BatchingSettings} class, so it could be used with + * logback.xml. + */ +public class LogbackBatchingSettings { + private Long elementCountThreshold = null; + private Long requestByteThreshold = null; + private Long delayThreshold = null; + private Long maxOutstandingElementCount = null; + private Long maxOutstandingRequestBytes = null; + private LimitExceededBehavior limitExceededBehavior = null; + + public void setElementCountThreshold(Long value) { + elementCountThreshold = value; + } + + public void setRequestByteThreshold(Long value) { + requestByteThreshold = value; + } + + public void setDelayThreshold(Long value) { + delayThreshold = value; + } + + public void setMaxOutstandingElementCount(Long value) { + maxOutstandingElementCount = value; + } + + public void setMaxOutstandingRequestBytes(Long value) { + maxOutstandingRequestBytes = value; + } + + public void setLimitExceededBehavior(LimitExceededBehavior value) { + limitExceededBehavior = value; + } + + public BatchingSettings build() { + BatchingSettings.Builder settings = BatchingSettings.newBuilder(); + if (elementCountThreshold != null) { + settings.setElementCountThreshold(elementCountThreshold); + } + if (requestByteThreshold != null) { + settings.setRequestByteThreshold(requestByteThreshold); + } + if (delayThreshold != null) { + settings.setDelayThreshold(Duration.ofMillis(delayThreshold)); + } + if (maxOutstandingElementCount != null + || maxOutstandingRequestBytes != null + || limitExceededBehavior != null) { + FlowControlSettings.Builder flowControlSettings = FlowControlSettings.newBuilder(); + flowControlSettings.setMaxOutstandingElementCount(maxOutstandingElementCount); + flowControlSettings.setMaxOutstandingRequestBytes(maxOutstandingRequestBytes); + if (limitExceededBehavior != null) { + flowControlSettings.setLimitExceededBehavior(limitExceededBehavior); + } + settings.setFlowControlSettings(flowControlSettings.build()); + } + return settings.build(); + } +} diff --git a/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java b/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java index 48274898a..3d88bceb0 100644 --- a/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java +++ b/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java @@ -63,7 +63,7 @@ * <log>application.log</log> * * <!-- Optional: defaults to {@code "ERROR"} --> - * <flushLevel>WARNING</flushLevel> + * <flushLevel>WARN</flushLevel> * * <!-- Optional: defaults to {@code ASYNC} --> * <writeSynchronicity>SYNC</writeSynchronicity> @@ -91,6 +91,20 @@ * * <!-- Optional: specifies if a batch's valid entries should be written even if some other entry failed due to an error. Defaults to {@code true} --> * <partialSuccess>true</partialSuccess> + * + * <!-- Optional: In the asynchronous mode the call(s) to Logging API takes place asynchronously and few calls to `write()` + * method may be batched together to compose a single call to Logging API. In order to control the batching settings, + * the `logbackBatchingSettings` section can be used as shown below. + * See [BatchingSettings](https://cloud.google.com/java/docs/reference/gax/latest/com.google.api.gax.batching.BatchingSettings) + * for more info regarding parameters shown below --> + * <logbackBatchingSettings> + * <elementCountThreshold>100</elementCountThreshold> + * <requestByteThreshold>1000</requestByteThreshold> + * <delayThreshold>500</delayThreshold> + * <maxOutstandingElementCount>10000</maxOutstandingElementCount> + * <maxOutstandingRequestBytes>100000</maxOutstandingRequestBytes> + * <limitExceededBehavior>Ignore</limitExceededBehavior> + * </logbackBatchingSettings> * </appender> * */ @@ -131,6 +145,7 @@ public class LoggingAppender extends UnsynchronizedAppenderBase { private Synchronicity writeSyncFlag = Synchronicity.ASYNC; private final Set enhancerClassNames = new HashSet<>(); private final Set loggingEventEnhancerClassNames = new HashSet<>(); + private LogbackBatchingSettings logbackBatchingSettings = null; /** * Sets a threshold for log severity level to flush all log entries that were batched so far. @@ -224,6 +239,19 @@ public void setRedirectToStdout(boolean flag) { redirectToStdout = flag; } + /** + * Sets the {@link LogbackBatchingSettings} to be used for the asynchronous mode call(s) to + * Logging API + * + *

Default to {@code null}. + * + * @param batchingSettings the {@link LogbackBatchingSettings} to be used for asynchronous mode + * call(s) to Logging API + */ + public void setLogbackBatchingSettings(LogbackBatchingSettings batchingSettings) { + logbackBatchingSettings = batchingSettings; + } + /** * Sets the flag indicating if a batch's valid entries should be written even if some other entry * failed due to an error. @@ -430,6 +458,8 @@ protected LoggingOptions getLoggingOptions() { } // opt-out metadata auto-population to control it in the appender code builder.setAutoPopulateMetadata(false); + builder.setBatchingSettings( + this.logbackBatchingSettings != null ? this.logbackBatchingSettings.build() : null); loggingOptions = builder.build(); } return loggingOptions; diff --git a/src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java b/src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java new file mode 100644 index 000000000..7691549e1 --- /dev/null +++ b/src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.logging.logback; + +import static com.google.common.truth.Truth.assertThat; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import com.google.api.gax.batching.FlowController.LimitExceededBehavior; +import com.google.cloud.logging.LoggingOptions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LoggingAppenderLogbackTest { + @Test + public void testLoggingOptionsFromLogbackXMLFileConfig() throws JoranException { + LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator jc = new JoranConfigurator(); + jc.setContext(context); + context.reset(); + jc.doConfigure("src/test/java/com/google/cloud/logging/logback/logback.xml"); + Logger logger = LoggerFactory.getLogger(LoggingAppenderLogbackTest.class); + assertThat(logger.getName()) + .isEqualTo("com.google.cloud.logging.logback.LoggingAppenderLogbackTest"); + LoggingAppender appender = (LoggingAppender) context.getLogger("ROOT").getAppender("CLOUD"); + LoggingOptions options = appender.getLoggingOptions(); + assertThat(options.getAutoPopulateMetadata()).isEqualTo(false); + assertThat(options.getBatchingSettings().getDelayThreshold().toMillis()).isEqualTo(500); + assertThat(options.getBatchingSettings().getElementCountThreshold()).isEqualTo(100); + assertThat(options.getBatchingSettings().getIsEnabled()).isEqualTo(true); + assertThat(options.getBatchingSettings().getRequestByteThreshold()).isEqualTo(1000); + assertThat(options.getBatchingSettings().getFlowControlSettings().getLimitExceededBehavior()) + .isEqualTo(LimitExceededBehavior.Ignore); + assertThat( + options.getBatchingSettings().getFlowControlSettings().getMaxOutstandingElementCount()) + .isEqualTo(10000); + assertThat( + options.getBatchingSettings().getFlowControlSettings().getMaxOutstandingRequestBytes()) + .isEqualTo(100000); + } +} diff --git a/src/test/java/com/google/cloud/logging/logback/logback.xml b/src/test/java/com/google/cloud/logging/logback/logback.xml new file mode 100644 index 000000000..d0ec77843 --- /dev/null +++ b/src/test/java/com/google/cloud/logging/logback/logback.xml @@ -0,0 +1,57 @@ + + + + + INFO + + + + application.log + + + WARN + + + SYNC + + + false + + + true + + + global + + + src/test/java/com/google/cloud/logging/logback/dummy-credentials.json + + + String + + + + + + true + + + + 100 + 1000 + 500 + 10000 + 100000 + Ignore + + + + + + +