Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
5528: [Backport 0.23] Don't update exporter position to smaller value r=Zelldon a=Zelldon ## Description Backports #5462 <!-- Please explain the changes you made here. --> ## Related issues <!-- Which issues are closed by this PR or are related --> closes #5294 ## Definition of Done _Not all items need to be done depending on the issue and the pull request._ Code changes: * [ ] The changes are backwards compatibility with previous versions * [ ] If it fixes a bug then PRs are created to [backport](https://github.com/zeebe-io/zeebe/compare/stable/0.24...develop?expand=1&template=backport_template.md&title=[Backport%200.24]) the fix to the last two minor versions Testing: * [ ] There are unit/integration tests that verify all acceptance criterias of the issue * [ ] New tests are written to ensure backwards compatibility with further versions * [ ] The behavior is tested manually * [ ] The impact of the changes is verified by a benchmark Documentation: * [ ] The documentation is updated (e.g. BPMN reference, configuration, examples, get-started guides, etc.) * [ ] New content is added to the [release announcement](https://drive.google.com/drive/u/0/folders/1DTIeswnEEq-NggJ25rm2BsDjcCQpDape) Co-authored-by: Christopher Zell <zelldon91@googlemail.com>
- Loading branch information
Showing
6 changed files
with
599 additions
and
167 deletions.
There are no files selected for viewing
156 changes: 156 additions & 0 deletions
156
broker/src/main/java/io/zeebe/broker/exporter/stream/ExporterContainer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| /* | ||
| * Copyright Camunda Services GmbH and/or licensed to Camunda Services GmbH under | ||
| * one or more contributor license agreements. See the NOTICE file distributed | ||
| * with this work for additional information regarding copyright ownership. | ||
| * Licensed under the Zeebe Community License 1.0. You may not use this file | ||
| * except in compliance with the Zeebe Community License 1.0. | ||
| */ | ||
| package io.zeebe.broker.exporter.stream; | ||
|
|
||
| import io.zeebe.broker.Loggers; | ||
| import io.zeebe.broker.exporter.context.ExporterContext; | ||
| import io.zeebe.broker.exporter.repo.ExporterDescriptor; | ||
| import io.zeebe.engine.processor.TypedRecord; | ||
| import io.zeebe.exporter.api.Exporter; | ||
| import io.zeebe.exporter.api.context.Context; | ||
| import io.zeebe.exporter.api.context.Controller; | ||
| import io.zeebe.protocol.impl.record.RecordMetadata; | ||
| import io.zeebe.protocol.record.Record; | ||
| import io.zeebe.util.sched.ActorControl; | ||
| import java.time.Duration; | ||
| import org.slf4j.Logger; | ||
|
|
||
| final class ExporterContainer implements Controller { | ||
|
|
||
| private static final Logger LOG = Loggers.EXPORTER_LOGGER; | ||
|
|
||
| private static final String SKIP_POSITION_UPDATE_ERROR_MESSAGE = | ||
| "Failed to update exporter position when skipping filtered record, can be skipped, but may indicate an issue if it occurs often"; | ||
|
|
||
| private final ExporterContext context; | ||
| private final Exporter exporter; | ||
| private long position; | ||
| private long lastUnacknowledgedPosition; | ||
| private ExportersState exportersState; | ||
| private ActorControl actor; | ||
|
|
||
| ExporterContainer(final ExporterDescriptor descriptor) { | ||
| context = | ||
| new ExporterContext( | ||
| Loggers.getExporterLogger(descriptor.getId()), descriptor.getConfiguration()); | ||
|
|
||
| exporter = descriptor.newInstance(); | ||
| } | ||
|
|
||
| void initContainer(final ActorControl actor, final ExportersState state) { | ||
| this.actor = actor; | ||
| exportersState = state; | ||
| } | ||
|
|
||
| void initPosition() { | ||
| position = exportersState.getPosition(getId()); | ||
| lastUnacknowledgedPosition = position; | ||
| if (position == ExportersState.VALUE_NOT_FOUND) { | ||
| exportersState.setPosition(getId(), -1L); | ||
| } | ||
| } | ||
|
|
||
| void openExporter() { | ||
| LOG.debug("Open exporter with id '{}'", getId()); | ||
| exporter.open(this); | ||
| } | ||
|
|
||
| public ExporterContext getContext() { | ||
| return context; | ||
| } | ||
|
|
||
| public Exporter getExporter() { | ||
| return exporter; | ||
| } | ||
|
|
||
| public long getPosition() { | ||
| return position; | ||
| } | ||
|
|
||
| long getLastUnacknowledgedPosition() { | ||
| return lastUnacknowledgedPosition; | ||
| } | ||
|
|
||
| /** | ||
| * Updates the exporter's position if it is up to date - that is, if it's last acknowledged | ||
| * position is greater than or equal to its last unacknowledged position. This is safe to do when | ||
| * skipping records as it means we passed no record to this exporter between both. | ||
| * | ||
| * @param eventPosition the new, up to date position | ||
| */ | ||
| void updatePositionOnSkipIfUpToDate(final long eventPosition) { | ||
| if (position >= lastUnacknowledgedPosition && position < eventPosition) { | ||
| try { | ||
| updateExporterLastExportedRecordPosition(eventPosition); | ||
| } catch (final Exception e) { | ||
| LOG.warn(SKIP_POSITION_UPDATE_ERROR_MESSAGE, e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void updateExporterLastExportedRecordPosition(final long eventPosition) { | ||
| if (position < eventPosition) { | ||
| exportersState.setPosition(getId(), eventPosition); | ||
| position = eventPosition; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void updateLastExportedRecordPosition(final long position) { | ||
| actor.run(() -> updateExporterLastExportedRecordPosition(position)); | ||
| } | ||
|
|
||
| @Override | ||
| public void scheduleTask(final Duration delay, final Runnable task) { | ||
| actor.runDelayed(delay, task); | ||
| } | ||
|
|
||
| public String getId() { | ||
| return context.getConfiguration().getId(); | ||
| } | ||
|
|
||
| private boolean acceptRecord(final RecordMetadata metadata) { | ||
| final Context.RecordFilter filter = context.getFilter(); | ||
| return filter.acceptType(metadata.getRecordType()) | ||
| && filter.acceptValue(metadata.getValueType()); | ||
| } | ||
|
|
||
| void configureExporter() throws Exception { | ||
| LOG.debug("Configure exporter with id '{}'", getId()); | ||
| exporter.configure(context); | ||
| } | ||
|
|
||
| boolean exportRecord(final RecordMetadata rawMetadata, final TypedRecord typedEvent) { | ||
| try { | ||
| if (position < typedEvent.getPosition()) { | ||
| if (acceptRecord(rawMetadata)) { | ||
| export(typedEvent); | ||
| } else { | ||
| updatePositionOnSkipIfUpToDate(typedEvent.getPosition()); | ||
| } | ||
| } | ||
| return true; | ||
| } catch (final Exception ex) { | ||
| context.getLogger().error("Error on exporting record with key {}", typedEvent.getKey(), ex); | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| private void export(final Record<?> record) { | ||
| exporter.export(record); | ||
| lastUnacknowledgedPosition = record.getPosition(); | ||
| } | ||
|
|
||
| public void close() { | ||
| try { | ||
| exporter.close(); | ||
| } catch (final Exception e) { | ||
| context.getLogger().error("Error on close", e); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.