From 667c9c6f18a517ffda77c74810dabdaab99a990e Mon Sep 17 00:00:00 2001 From: Yufei Cai Date: Fri, 11 Dec 2020 11:29:10 +0100 Subject: [PATCH 1/7] [#914] Add Ditto acknowledgement label "search-persisted". Signed-off-by: Yufei Cai --- .../acks/AbstractCommandAckRequestSetter.java | 25 ++++++++++++++++--- .../base/acks/DittoAcknowledgementLabel.java | 10 +++++++- .../AcknowledgementForwarderActorStarter.java | 5 ++-- .../ThingModifyCommandAckRequestSetter.java | 5 +++- .../ThingLiveCommandAckRequestSetterTest.java | 3 ++- ...hingModifyCommandAckRequestSetterTest.java | 8 +++--- 6 files changed, 45 insertions(+), 11 deletions(-) diff --git a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java index 7a700bbeea..cabef0b274 100644 --- a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java +++ b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java @@ -54,12 +54,31 @@ public abstract class AbstractCommandAckRequestSetter negatedDittoAcknowledgementLabels; + /** + * Create a command acknowledgment request setter that sets one Ditto acknowledgement label implicitly and + * filters out other Ditto acknowledgment5 labels. + * + * @param implicitAcknowledgementLabel the label to set if the header 'requested-acks' is absent. + */ protected AbstractCommandAckRequestSetter(final AcknowledgementLabel implicitAcknowledgementLabel) { - negatedDittoAcknowledgementLabels = Collections.unmodifiableSet( - Arrays.stream(DittoAcknowledgementLabel.values()) + this(implicitAcknowledgementLabel, + Collections.unmodifiableSet(Arrays.stream(DittoAcknowledgementLabel.values()) .filter(v -> !implicitAcknowledgementLabel.equals(v)) - .collect(Collectors.toSet())); + .collect(Collectors.toSet())) + ); + } + + /** + * Create a command acknowledgement request setter. + * + * @param implicitAcknowledgementLabel the label to set if the header 'requested-acks' is absent. + * @param negatedDittoAcknowledgementLabels the labels to filter out if present. + * @since 1.6.0 + */ + protected AbstractCommandAckRequestSetter(final AcknowledgementLabel implicitAcknowledgementLabel, + final Set negatedDittoAcknowledgementLabels) { this.implicitAcknowledgementLabel = implicitAcknowledgementLabel; + this.negatedDittoAcknowledgementLabels = negatedDittoAcknowledgementLabels; } @Override diff --git a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java index 01ccab03d9..e7d18d616b 100644 --- a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java +++ b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java @@ -40,6 +40,14 @@ public final class DittoAcknowledgementLabel implements AcknowledgementLabel { */ public static final DittoAcknowledgementLabel LIVE_RESPONSE = new DittoAcknowledgementLabel("live-response"); + /** + * Label for Acknowledgements indicating that a change to an entity (e. g. a thing) has successfully been reflected + * in the search index. + * + * @since 1.6.0 + */ + public static final DittoAcknowledgementLabel SEARCH_PERSISTED = new DittoAcknowledgementLabel("search-persisted"); + private final AcknowledgementLabel delegate; private DittoAcknowledgementLabel(final CharSequence labelValue) { @@ -52,7 +60,7 @@ private DittoAcknowledgementLabel(final CharSequence labelValue) { * @return an array containing the Ditto acknowledgement labels, in the order they're declared. */ public static AcknowledgementLabel[] values() { - return new AcknowledgementLabel[]{TWIN_PERSISTED, LIVE_RESPONSE}; + return new AcknowledgementLabel[]{TWIN_PERSISTED, LIVE_RESPONSE, SEARCH_PERSISTED}; } /** diff --git a/services/models/acks/src/main/java/org/eclipse/ditto/services/models/acks/AcknowledgementForwarderActorStarter.java b/services/models/acks/src/main/java/org/eclipse/ditto/services/models/acks/AcknowledgementForwarderActorStarter.java index 7bac9a352f..bd9c3ff89f 100644 --- a/services/models/acks/src/main/java/org/eclipse/ditto/services/models/acks/AcknowledgementForwarderActorStarter.java +++ b/services/models/acks/src/main/java/org/eclipse/ditto/services/models/acks/AcknowledgementForwarderActorStarter.java @@ -212,7 +212,7 @@ private Acknowledgement getNack(final AcknowledgementLabel label, dittoRuntimeException.toJson()); } - static boolean isNotBuiltIn(final AcknowledgementRequest request) { + static boolean isNotTwinPersistedOrLiveResponse(final AcknowledgementRequest request) { return isNotLiveResponse(request) && isNotTwinPersisted(request); } @@ -231,7 +231,8 @@ static boolean isLiveSignal(final Signal signal) { static boolean hasEffectiveAckRequests(final Signal signal, final Set ackRequests) { final boolean isLiveSignal = isLiveSignal(signal); if (signal instanceof ThingEvent && !isLiveSignal) { - return ackRequests.stream().anyMatch(AcknowledgementForwarderActorStarter::isNotBuiltIn); + return ackRequests.stream() + .anyMatch(AcknowledgementForwarderActorStarter::isNotTwinPersistedOrLiveResponse); } else if (signal instanceof MessageCommand || (isLiveSignal && signal instanceof ThingCommand)) { return ackRequests.stream().anyMatch(AcknowledgementForwarderActorStarter::isNotTwinPersisted); } else { diff --git a/signals/commands/things/src/main/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetter.java b/signals/commands/things/src/main/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetter.java index 59776396d1..8ecf567f1b 100644 --- a/signals/commands/things/src/main/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetter.java +++ b/signals/commands/things/src/main/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetter.java @@ -14,6 +14,8 @@ import static org.eclipse.ditto.model.base.common.ConditionChecker.checkNotNull; +import java.util.Collections; + import javax.annotation.concurrent.Immutable; import org.eclipse.ditto.model.base.acks.AbstractCommandAckRequestSetter; @@ -37,7 +39,8 @@ public final class ThingModifyCommandAckRequestSetter extends AbstractCommandAck private static final ThingModifyCommandAckRequestSetter INSTANCE = new ThingModifyCommandAckRequestSetter(); private ThingModifyCommandAckRequestSetter() { - super(DittoAcknowledgementLabel.TWIN_PERSISTED); + // The Ditto acknowledgement label "search-persisted" is tolerated but not set by default. + super(DittoAcknowledgementLabel.TWIN_PERSISTED, Collections.singleton(DittoAcknowledgementLabel.LIVE_RESPONSE)); } /** diff --git a/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingLiveCommandAckRequestSetterTest.java b/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingLiveCommandAckRequestSetterTest.java index 52db758add..28d9b8fba8 100644 --- a/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingLiveCommandAckRequestSetterTest.java +++ b/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingLiveCommandAckRequestSetterTest.java @@ -87,7 +87,8 @@ public void filterOutOtherBuiltInDittoAcknowledgementLabels() { final DittoHeaders dittoHeaders = DittoHeaders.newBuilder() .channel("live") .acknowledgementRequest(AcknowledgementRequest.of(DittoAcknowledgementLabel.TWIN_PERSISTED), - AcknowledgementRequest.of(DittoAcknowledgementLabel.LIVE_RESPONSE)) + AcknowledgementRequest.of(DittoAcknowledgementLabel.LIVE_RESPONSE), + AcknowledgementRequest.of(DittoAcknowledgementLabel.SEARCH_PERSISTED)) .randomCorrelationId() .build(); final CreateThing command = CreateThing.of(Thing.newBuilder().build(), null, dittoHeaders); diff --git a/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetterTest.java b/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetterTest.java index 26fed5ed6e..2d5be72d6e 100644 --- a/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetterTest.java +++ b/signals/commands/things/src/test/java/org/eclipse/ditto/signals/commands/things/acks/ThingModifyCommandAckRequestSetterTest.java @@ -72,15 +72,17 @@ public void addPersistedAckLabelByDefault() { } @Test - public void filterOutOtherBuiltInDittoAcknowledgementLabels() { + public void filterOutLiveResponseLabel() { final DittoHeaders dittoHeaders = DittoHeaders.newBuilder() .acknowledgementRequest(AcknowledgementRequest.of(DittoAcknowledgementLabel.TWIN_PERSISTED), - AcknowledgementRequest.of(DittoAcknowledgementLabel.LIVE_RESPONSE)) + AcknowledgementRequest.of(DittoAcknowledgementLabel.LIVE_RESPONSE), + AcknowledgementRequest.of(DittoAcknowledgementLabel.SEARCH_PERSISTED)) .randomCorrelationId() .build(); final CreateThing command = CreateThing.of(Thing.newBuilder().build(), null, dittoHeaders); final DittoHeaders expectedHeaders = dittoHeaders.toBuilder() - .acknowledgementRequest(AcknowledgementRequest.of(DittoAcknowledgementLabel.TWIN_PERSISTED)) + .acknowledgementRequest(AcknowledgementRequest.of(DittoAcknowledgementLabel.TWIN_PERSISTED), + AcknowledgementRequest.of(DittoAcknowledgementLabel.SEARCH_PERSISTED)) .responseRequired(true) .build(); final CreateThing expected = CreateThing.of(Thing.newBuilder().build(), null, expectedHeaders); From 35856a6b7de4911ce2faa9a4bf73952612a89ff9 Mon Sep 17 00:00:00 2001 From: Yufei Cai Date: Sat, 12 Dec 2020 18:55:47 +0100 Subject: [PATCH 2/7] [#914] send search-persisted acknowledgements. Duplicate-key errors are considered successes because they may happen during upsert. Signed-off-by: Yufei Cai --- .../write/mapping/EnforcedThingMapper.java | 11 ++- .../persistence/write/model/Metadata.java | 80 +++++++++++++++++-- .../streaming/BulkWriteResultAckFlow.java | 29 +++++-- .../write/streaming/ChangeQueueActor.java | 2 +- .../write/streaming/EnforcementFlow.java | 3 +- .../streaming/TestSearchUpdaterStream.java | 3 +- .../model/AbstractWithActorSystemTest.java | 35 ++++++++ .../persistence/write/model/MetadataTest.java | 17 +++- .../write/model/ThingDeleteModelTest.java | 9 ++- .../write/model/ThingWriteModelTest.java | 9 ++- .../streaming/BulkWriteResultAckFlowTest.java | 53 ++++++++++-- .../updater/actors/NewEventForwarder.java | 3 +- .../updater/actors/ThingUpdater.java | 21 ++++- 13 files changed, 242 insertions(+), 33 deletions(-) create mode 100644 services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/AbstractWithActorSystemTest.java diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/mapping/EnforcedThingMapper.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/mapping/EnforcedThingMapper.java index 5d3b8833eb..cd205e2358 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/mapping/EnforcedThingMapper.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/mapping/EnforcedThingMapper.java @@ -24,6 +24,8 @@ import java.util.Map; import java.util.stream.Stream; +import javax.annotation.Nullable; + import org.bson.BsonArray; import org.bson.BsonString; import org.bson.BsonValue; @@ -77,7 +79,7 @@ public static ThingWriteModel toWriteModel(final JsonObject thing, final Enforcer enforcer, final long policyRevision) { - return toWriteModel(thing, enforcer, policyRevision, -1); + return toWriteModel(thing, enforcer, policyRevision, -1, null); } /** @@ -87,13 +89,15 @@ public static ThingWriteModel toWriteModel(final JsonObject thing, * @param enforcer the policy- or ACL-enforcer of the Thing. * @param policyRevision revision of the policy for an policy enforcer, or any number for an ACL enforcer. * @param maxArraySize only arrays smaller than this are indexed. + * @param oldMetadata the meatadata that triggered the search update, possibly containing sender information. * @return BSON document to write into the search index. * @throws org.eclipse.ditto.json.JsonMissingFieldException if Thing ID or revision is missing. */ public static ThingWriteModel toWriteModel(final JsonObject thing, final Enforcer enforcer, final long policyRevision, - final int maxArraySize) { + final int maxArraySize, + @Nullable final Metadata oldMetadata) { final String extractedThing = thing.getValueOrThrow(Thing.JsonFields.ID); final ThingId thingId = ThingId.of(extractedThing); @@ -117,7 +121,8 @@ public static ThingWriteModel toWriteModel(final JsonObject thing, .append(FIELD_SORTING, thingCopyForSorting) .append(FIELD_INTERNAL, flattenedValues); - return ThingWriteModel.of(metadata, thingDocument); + final Metadata metadataToRetain = oldMetadata == null ? metadata : oldMetadata.prependSenders(metadata); + return ThingWriteModel.of(metadataToRetain, thingDocument); } private static BsonArray getGlobalRead(final Enforcer enforcer) { diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java index a0ca16c3f6..1aa16935ad 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java @@ -13,15 +13,24 @@ package org.eclipse.ditto.services.thingsearch.persistence.write.model; import java.time.Instant; +import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; +import org.eclipse.ditto.model.base.acks.DittoAcknowledgementLabel; +import org.eclipse.ditto.model.base.common.HttpStatusCode; +import org.eclipse.ditto.model.base.headers.DittoHeaders; import org.eclipse.ditto.model.policies.PolicyId; import org.eclipse.ditto.model.things.ThingId; import org.eclipse.ditto.services.models.thingsearch.commands.sudo.UpdateThingResponse; +import org.eclipse.ditto.signals.acks.base.Acknowledgement; + +import akka.actor.ActorRef; /** * Data class holding information about a "thingEntities" database record. @@ -35,17 +44,24 @@ public final class Metadata { @Nullable private final Long policyRevision; @Nullable final Instant modified; + /** + * Using Scala list to have immutability and constant-time prepend at the same time. + */ + private final List senders; + private Metadata(final ThingId thingId, final long thingRevision, @Nullable final PolicyId policyId, @Nullable final Long policyRevision, - @Nullable final Instant modified) { + @Nullable final Instant modified, + final List senders) { this.thingId = thingId; this.thingRevision = thingRevision; this.policyId = policyId; this.policyRevision = policyRevision; this.modified = modified; + this.senders = senders; } /** @@ -62,7 +78,26 @@ public static Metadata of(final ThingId thingId, @Nullable final PolicyId policyId, @Nullable final Long policyRevision) { - return new Metadata(thingId, thingRevision, policyId, policyRevision, null); + return new Metadata(thingId, thingRevision, policyId, policyRevision, null, List.of()); + } + + /** + * Create an Metadata object retaining the original sender of an event. + * + * @param thingId the Thing ID. + * @param thingRevision the Thing revision. + * @param policyId the Policy ID if the Thing has one. + * @param policyRevision the Policy revision if the Thing has a policy, or null if it does not. + * @param sender the sender + * @return the new Metadata object. + */ + public static Metadata of(final ThingId thingId, + final long thingRevision, + @Nullable final PolicyId policyId, + @Nullable final Long policyRevision, + final ActorRef sender) { + + return new Metadata(thingId, thingRevision, policyId, policyRevision, null, List.of(sender)); } /** @@ -81,7 +116,7 @@ public static Metadata of(final ThingId thingId, @Nullable final Long policyRevision, @Nullable final Instant modified) { - return new Metadata(thingId, thingRevision, policyId, policyRevision, modified); + return new Metadata(thingId, thingRevision, policyId, policyRevision, modified, List.of()); } /** @@ -151,6 +186,39 @@ public Optional getModified() { return Optional.ofNullable(modified); } + /** + * Prepend new senders to the senders stored in this object. + * + * @param newMetadata a previous metadata record. + * @return the new metadata with concatenated senders. + */ + public Metadata prependSenders(final Metadata newMetadata) { + final List newSenders = + Stream.concat(newMetadata.senders.stream(), senders.stream()).collect(Collectors.toList()); + return new Metadata(newMetadata.thingId, newMetadata.thingRevision, newMetadata.policyId, + newMetadata.policyRevision, newMetadata.modified, newSenders); + } + + /** + * Send negative acknowledgements to senders. + */ + public void sendNAck() { + send(Acknowledgement.of(DittoAcknowledgementLabel.SEARCH_PERSISTED, thingId, + HttpStatusCode.INTERNAL_SERVER_ERROR, DittoHeaders.empty())); + } + + /** + * Send positive acknowledgements to senders. + */ + public void sendAck() { + send(Acknowledgement.of(DittoAcknowledgementLabel.SEARCH_PERSISTED, thingId, HttpStatusCode.NO_CONTENT, + DittoHeaders.empty())); + } + + private void send(final Acknowledgement ack) { + senders.forEach(sender -> sender.tell(ack, ActorRef.noSender())); + } + @Override public boolean equals(final Object o) { if (this == o) { @@ -164,12 +232,13 @@ public boolean equals(final Object o) { Objects.equals(policyRevision, that.policyRevision) && Objects.equals(thingId, that.thingId) && Objects.equals(policyId, that.policyId) && - Objects.equals(modified, that.modified); + Objects.equals(modified, that.modified) && + Objects.equals(senders, that.senders); } @Override public int hashCode() { - return Objects.hash(thingId, thingRevision, policyId, policyRevision, modified); + return Objects.hash(thingId, thingRevision, policyId, policyRevision, modified, senders); } @Override @@ -180,6 +249,7 @@ public String toString() { ", policyId=" + policyId + ", policyRevision=" + policyRevision + ", modified=" + modified + + ", senders=" + senders + "]"; } diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java index 319264d5c0..528a4f78f8 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java @@ -15,6 +15,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -31,6 +32,7 @@ import org.eclipse.ditto.services.utils.metrics.instruments.counter.Counter; import org.eclipse.ditto.signals.base.ShardedMessageEnvelope; +import com.mongodb.ErrorCategory; import com.mongodb.bulk.BulkWriteError; import com.mongodb.bulk.BulkWriteResult; @@ -65,30 +67,44 @@ Flow start() { private Iterable checkBulkWriteResult(final WriteResultAndErrors writeResultAndErrors) { if (wasNotAcknowledged(writeResultAndErrors)) { // All failed. - acknowledgeFailures(getAllThings(writeResultAndErrors)); + acknowledgeFailures(getAllMetadata(writeResultAndErrors)); return Collections.singleton(logResult("NotAcknowledged", writeResultAndErrors, false)); } else { final Optional consistencyError = checkForConsistencyError(writeResultAndErrors); if (consistencyError.isPresent()) { // write result is not consistent; there is a bug with Ditto or with its environment - acknowledgeFailures(getAllThings(writeResultAndErrors)); + acknowledgeFailures(getAllMetadata(writeResultAndErrors)); return Collections.singleton(consistencyError.get()); } else { final List errors = writeResultAndErrors.getBulkWriteErrors(); final List logEntries = new ArrayList<>(errors.size() + 1); - final List failedThings = new ArrayList<>(errors.size()); + final List failedMetadata = new ArrayList<>(errors.size()); logEntries.add(logResult("Acknowledged", writeResultAndErrors, errors.isEmpty())); + final BitSet failedIndices = new BitSet(writeResultAndErrors.getWriteModels().size()); for (final BulkWriteError error : errors) { final Metadata metadata = writeResultAndErrors.getWriteModels().get(error.getIndex()).getMetadata(); logEntries.add(String.format("UpdateFailed for %s due to %s", metadata, error)); - failedThings.add(metadata); + if (error.getCategory() != ErrorCategory.DUPLICATE_KEY) { + failedIndices.set(error.getIndex()); + failedMetadata.add(metadata); + // duplicate key error is considered success + } } - acknowledgeFailures(failedThings); + acknowledgeFailures(failedMetadata); + acknowledgeSuccesses(failedIndices, writeResultAndErrors.getWriteModels()); return logEntries; } } } + private void acknowledgeSuccesses(final BitSet failedIndices, final List writeModels) { + for (int i = 0; i < writeModels.size(); ++i) { + if (!failedIndices.get(i)) { + writeModels.get(i).getMetadata().sendAck(); + } + } + } + private void acknowledgeFailures(final List things) { errorsCounter.increment(things.size()); acknowledge(things, BulkWriteResultAckFlow::createFailureResponse); @@ -102,6 +118,7 @@ private void acknowledge(final List things, final ShardedMessageEnvelope envelope = ShardedMessageEnvelope.of(response.getEntityId(), response.getType(), response.toJson(), response.getDittoHeaders()); + metadata.sendNAck(); updaterShard.tell(envelope, ActorRef.noSender()); } } @@ -145,7 +162,7 @@ private static boolean areAllIndexesWithinBounds(final List bulk return bulkWriteErrors.stream().mapToInt(BulkWriteError::getIndex).allMatch(i -> 0 <= i && i < requested); } - private static List getAllThings(final WriteResultAndErrors writeResultAndErrors) { + private static List getAllMetadata(final WriteResultAndErrors writeResultAndErrors) { return writeResultAndErrors.getWriteModels() .stream() .map(AbstractWriteModel::getMetadata) diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java index ad50b484b3..2553d85e55 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java @@ -74,7 +74,7 @@ public Receive createReceive() { * @param metadata a description of the change. */ private void enqueue(final Metadata metadata) { - cache.put(metadata.getThingId(), metadata); + cache.merge(metadata.getThingId(), metadata, Metadata::prependSenders); } /** diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java index d0e75abb8d..b94ee486fd 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java @@ -227,7 +227,8 @@ private Source computeWriteModel(final Metadata met try { return EnforcedThingMapper.toWriteModel(thing, entry.getValueOrThrow(), entry.getRevision(), - maxArraySize); + maxArraySize, + metadata); } catch (final JsonRuntimeException e) { log.error(e.getMessage(), e); return ThingDeleteModel.of(metadata); diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/TestSearchUpdaterStream.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/TestSearchUpdaterStream.java index 3378d4d339..a36f8d5d8c 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/TestSearchUpdaterStream.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/TestSearchUpdaterStream.java @@ -71,7 +71,8 @@ public Source write(final Thing thing, final long policyRevision) { final JsonObject thingJson = thing.toJson(FieldType.all()); - final AbstractWriteModel writeModel = EnforcedThingMapper.toWriteModel(thingJson, enforcer, policyRevision, -1); + final AbstractWriteModel writeModel = EnforcedThingMapper.toWriteModel(thingJson, enforcer, policyRevision, -1, + null); return Source.single(Source.single(writeModel)) .via(mongoSearchUpdaterFlow.start(1, 1, Duration.ZERO)); diff --git a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/AbstractWithActorSystemTest.java b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/AbstractWithActorSystemTest.java new file mode 100644 index 0000000000..adf002b57d --- /dev/null +++ b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/AbstractWithActorSystemTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.ditto.services.thingsearch.persistence.write.model; + +import javax.annotation.Nullable; + +import org.junit.After; + +import akka.actor.ActorSystem; +import akka.testkit.javadsl.TestKit; + +/** + * For tests that may create an actor system. + */ +abstract class AbstractWithActorSystemTest { + + @Nullable protected ActorSystem system; + + @After + public void shutdown() { + if (system != null) { + TestKit.shutdownActorSystem(system); + } + } +} diff --git a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/MetadataTest.java b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/MetadataTest.java index d761ba62a6..1fc2aef7c5 100644 --- a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/MetadataTest.java +++ b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/MetadataTest.java @@ -12,32 +12,45 @@ */ package org.eclipse.ditto.services.thingsearch.persistence.write.model; +import static org.mutabilitydetector.unittesting.AllowedReason.assumingFields; import static org.mutabilitydetector.unittesting.AllowedReason.provided; import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf; import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable; +import javax.annotation.Nullable; + import org.eclipse.ditto.model.policies.PolicyId; import org.eclipse.ditto.model.things.ThingId; +import org.junit.After; import org.junit.Test; +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.testkit.TestProbe; +import akka.testkit.javadsl.TestKit; import nl.jqno.equalsverifier.EqualsVerifier; /** * Unit test for {@link Metadata}. */ -public final class MetadataTest { +public final class MetadataTest extends AbstractWithActorSystemTest { @Test public void assertImmutability() { assertInstancesOf(Metadata.class, areImmutable(), - provided(ThingId.class, PolicyId.class).isAlsoImmutable()); + provided(ThingId.class, PolicyId.class).areAlsoImmutable(), + assumingFields("senders").areSafelyCopiedUnmodifiableCollectionsWithImmutableElements()); } @Test public void testHashCodeAndEquals() { + system = ActorSystem.create(); + final TestProbe probe1 = TestProbe.apply(system); + final TestProbe probe2 = TestProbe.apply(system); EqualsVerifier.forClass(Metadata.class) .usingGetClass() + .withPrefabValues(ActorRef.class, probe1.ref(), probe2.ref()) .verify(); } diff --git a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingDeleteModelTest.java b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingDeleteModelTest.java index 468d656d75..7d0c9425ff 100644 --- a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingDeleteModelTest.java +++ b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingDeleteModelTest.java @@ -17,12 +17,15 @@ import org.junit.Test; +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.testkit.TestProbe; import nl.jqno.equalsverifier.EqualsVerifier; /** * Unit test for {@link ThingDeleteModel}. */ -public final class ThingDeleteModelTest { +public final class ThingDeleteModelTest extends AbstractWithActorSystemTest { @Test public void assertImmutability() { @@ -31,8 +34,12 @@ public void assertImmutability() { @Test public void testHashCodeAndEquals() { + system = ActorSystem.create(); + final TestProbe probe1 = TestProbe.apply(system); + final TestProbe probe2 = TestProbe.apply(system); EqualsVerifier.forClass(ThingDeleteModel.class) .usingGetClass() + .withPrefabValues(ActorRef.class, probe1.ref(), probe2.ref()) .verify(); } diff --git a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingWriteModelTest.java b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingWriteModelTest.java index 55a3e364a2..8aa2d079dd 100644 --- a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingWriteModelTest.java +++ b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/ThingWriteModelTest.java @@ -14,17 +14,24 @@ import org.junit.Test; +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.testkit.TestProbe; import nl.jqno.equalsverifier.EqualsVerifier; /** * Unit test for {@link ThingWriteModel}. */ -public final class ThingWriteModelTest { +public final class ThingWriteModelTest extends AbstractWithActorSystemTest { @Test public void testHashCodeAndEquals() { + system = ActorSystem.create(); + final TestProbe probe1 = TestProbe.apply(system); + final TestProbe probe2 = TestProbe.apply(system); EqualsVerifier.forClass(ThingWriteModel.class) .usingGetClass() + .withPrefabValues(ActorRef.class, probe1.ref(), probe2.ref()) .verify(); } diff --git a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlowTest.java b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlowTest.java index 4f2eba569c..bf9a127499 100644 --- a/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlowTest.java +++ b/services/thingsearch/persistence/src/test/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlowTest.java @@ -16,10 +16,13 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.bson.BsonDocument; import org.bson.BsonString; import org.bson.Document; +import org.eclipse.ditto.model.base.common.HttpStatusCode; import org.eclipse.ditto.model.policies.PolicyId; import org.eclipse.ditto.model.things.ThingId; import org.eclipse.ditto.services.models.thingsearch.commands.sudo.UpdateThingResponse; @@ -28,6 +31,7 @@ import org.eclipse.ditto.services.thingsearch.persistence.write.model.ThingDeleteModel; import org.eclipse.ditto.services.thingsearch.persistence.write.model.ThingWriteModel; import org.eclipse.ditto.services.thingsearch.persistence.write.model.WriteResultAndErrors; +import org.eclipse.ditto.signals.acks.base.Acknowledgement; import org.eclipse.ditto.signals.base.ShardedMessageEnvelope; import org.junit.After; import org.junit.Test; @@ -92,11 +96,9 @@ public void partialSuccess() { // THEN: the non-duplicate-key error triggers a failure acknowledgement actorSystem.log().info(message); - for (int i = 3; i < 5; ++i) { - assertThat(expectUpdateThingResponse(writeModels.get(i).getMetadata().getThingId())) - .describedAs("response is failure") - .returns(false, UpdateThingResponse::isSuccess); - } + assertThat(expectUpdateThingResponse(writeModels.get(4).getMetadata().getThingId())) + .describedAs("response is failure") + .returns(false, UpdateThingResponse::isSuccess); assertThat(message).contains("Acknowledged: PartialSuccess"); } @@ -145,6 +147,35 @@ public void errorIndexOutOfBoundError() { assertThat(message).contains("ConsistencyError[indexOutOfBound]"); } + @Test + public void acknowledgements() { + final List probes = + IntStream.range(0, 5).mapToObj(i -> TestProbe.apply(actorSystem)).collect(Collectors.toList()); + final List writeModels = generateWriteModels(probes); + final BulkWriteResult result = BulkWriteResult.acknowledged(1, 2, 1, 2, List.of()); + final List updateFailure = List.of( + new BulkWriteError(11000, "E11000 duplicate key error", new BsonDocument(), 3), + new BulkWriteError(50, "E50 operation timed out", new BsonDocument(), 4) + ); + + // WHEN: BulkWriteResultAckFlow receives partial update success with errors, one of which is not duplicate key + final WriteResultAndErrors resultAndErrors = WriteResultAndErrors.failure(writeModels, + new MongoBulkWriteException(result, updateFailure, null, new ServerAddress())); + runBulkWriteResultAckFlowAndGetFirstLogEntry(resultAndErrors); + + // THEN: only the non-duplicate-key sender receives negative acknowledgement + assertThat(probes.get(0).expectMsgClass(Acknowledgement.class).getStatusCode()) + .isEqualTo(HttpStatusCode.NO_CONTENT); + assertThat(probes.get(1).expectMsgClass(Acknowledgement.class).getStatusCode()) + .isEqualTo(HttpStatusCode.NO_CONTENT); + assertThat(probes.get(2).expectMsgClass(Acknowledgement.class).getStatusCode()) + .isEqualTo(HttpStatusCode.NO_CONTENT); + assertThat(probes.get(3).expectMsgClass(Acknowledgement.class).getStatusCode()) + .isEqualTo(HttpStatusCode.NO_CONTENT); + assertThat(probes.get(4).expectMsgClass(Acknowledgement.class).getStatusCode()) + .isEqualTo(HttpStatusCode.INTERNAL_SERVER_ERROR); + } + private String runBulkWriteResultAckFlowAndGetFirstLogEntry(final WriteResultAndErrors writeResultAndErrors) { return Source.single(writeResultAndErrors) .via(underTest.start()) @@ -153,15 +184,21 @@ private String runBulkWriteResultAckFlowAndGetFirstLogEntry(final WriteResultAnd .join(); } - private static List generate5WriteModels() { - final int howMany = 5; + private List generate5WriteModels() { + return generateWriteModels( + IntStream.range(0, 5).mapToObj(i -> TestProbe.apply(actorSystem)).collect(Collectors.toList())); + } + + private List generateWriteModels(final List probes) { + final int howMany = probes.size(); final List writeModels = new ArrayList<>(howMany); for (int i = 0; i < howMany; ++i) { final ThingId thingId = ThingId.of("thing", String.valueOf(i)); final long thingRevision = i * 10; final PolicyId policyId = i % 4 < 2 ? null : PolicyId.of("policy", String.valueOf(i)); final long policyRevision = i * 100; - final Metadata metadata = Metadata.of(thingId, thingRevision, policyId, policyRevision); + final Metadata metadata = + Metadata.of(thingId, thingRevision, policyId, policyRevision, probes.get(i).ref()); if (i % 2 == 0) { writeModels.add(ThingDeleteModel.of(metadata)); } else { diff --git a/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java b/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java index 06595aa91d..2cf3db124a 100644 --- a/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java +++ b/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java @@ -133,7 +133,8 @@ private void updateSubscriptions(final ShardRegion.ClusterShardingStats stats) { private void processThingEvent(final ThingEvent thingEvent) { log.withCorrelationId(thingEvent) .debug("Forwarding incoming ThingEvent for thingId '{}'", thingEvent.getThingEntityId()); - namespaceBlockingBehavior.block(thingEvent).thenAccept(m -> shardRegion.tell(m, getSelf())); + final ActorRef sender = getSender(); + namespaceBlockingBehavior.block(thingEvent).thenAccept(m -> shardRegion.tell(m, sender())); } private static Collection getActiveShardIds(final ShardRegion.ClusterShardingStats stats) { diff --git a/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/ThingUpdater.java b/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/ThingUpdater.java index 6f680f9513..afafa19f9c 100644 --- a/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/ThingUpdater.java +++ b/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/ThingUpdater.java @@ -21,6 +21,8 @@ import javax.annotation.Nullable; +import org.eclipse.ditto.model.base.acks.AcknowledgementRequest; +import org.eclipse.ditto.model.base.acks.DittoAcknowledgementLabel; import org.eclipse.ditto.model.policies.PolicyId; import org.eclipse.ditto.model.things.ThingId; import org.eclipse.ditto.services.base.actors.ShutdownBehaviour; @@ -50,6 +52,9 @@ */ final class ThingUpdater extends AbstractActor { + private static final AcknowledgementRequest SEARCH_PERSISTED_REQUEST = + AcknowledgementRequest.of(DittoAcknowledgementLabel.SEARCH_PERSISTED); + private final DittoDiagnosticLoggingAdapter log; private final ThingId thingId; private final ShutdownBehaviour shutdownBehaviour; @@ -113,6 +118,14 @@ private Metadata exportMetadata() { return Metadata.of(thingId, thingRevision, policyId, policyRevision); } + private Metadata exportMetadataWithSender(final boolean shouldAcknowledge, final ActorRef sender) { + if (shouldAcknowledge) { + return Metadata.of(thingId, thingRevision, policyId, policyRevision, sender); + } else { + return exportMetadata(); + } + } + /** * Push metadata of this updater to the queue of thing-changes to be streamed into the persistence. */ @@ -175,19 +188,21 @@ private void processPolicyReferenceTag(final PolicyReferenceTag policyReferenceT acknowledge(policyReferenceTag); } - private void processThingEvent(final ThingEvent thingEvent) { + private void processThingEvent(final ThingEvent thingEvent) { log.withCorrelationId(thingEvent); log.debug("Received new thing event for thing id <{}> with revision <{}>.", thingId, thingEvent.getRevision()); + final boolean shouldAcknowledge = + thingEvent.getDittoHeaders().getAcknowledgementRequests().contains(SEARCH_PERSISTED_REQUEST); // check if the revision is valid (thingEvent.revision = 1 + sequenceNumber) - if (thingEvent.getRevision() <= thingRevision) { + if (thingEvent.getRevision() <= thingRevision && !shouldAcknowledge) { log.debug("Dropped thing event for thing id <{}> with revision <{}> because it was older than or " + "equal to the current sequence number <{}> of the update actor.", thingId, thingEvent.getRevision(), thingRevision); } else { log.debug("Applying thing event <{}>.", thingEvent); thingRevision = thingEvent.getRevision(); - enqueueMetadata(); + enqueueMetadata(exportMetadataWithSender(shouldAcknowledge, getSender())); } } From 3969c60bac8c82bb53e89ad9e1fc5f4ec809bc16 Mon Sep 17 00:00:00 2001 From: Yufei Cai Date: Sun, 13 Dec 2020 16:09:27 +0100 Subject: [PATCH 3/7] [#914] document "search-persisted". Signed-off-by: Yufei Cai --- .../src/main/resources/pages/ditto/basic-acknowledgements.md | 3 +++ documentation/src/main/resources/pages/ditto/basic-search.md | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/documentation/src/main/resources/pages/ditto/basic-acknowledgements.md b/documentation/src/main/resources/pages/ditto/basic-acknowledgements.md index 495e6e5ccd..c1858167f7 100644 --- a/documentation/src/main/resources/pages/ditto/basic-acknowledgements.md +++ b/documentation/src/main/resources/pages/ditto/basic-acknowledgements.md @@ -47,6 +47,9 @@ It is ignored for commands in the live channel. * **live-response**: For acknowledgement requests of live commands and live messages. It is fulfilled when a subscriber of the live command or message sends a corresponding response. It is ignored for commands in the twin channel. +* **search-persisted**: For acknowledgement requests of twin modifying commands. +It is fulfilled when a modifying command has successfully updated the search index of the digital twin. +It is ignored for commands in the live channel. ### Custom acknowledgement labels In addition to the [built-in](#built-in-acknowledgement-labels) acknowledgement requests, diff --git a/documentation/src/main/resources/pages/ditto/basic-search.md b/documentation/src/main/resources/pages/ditto/basic-search.md index 11a3054dac..e45008bbdb 100644 --- a/documentation/src/main/resources/pages/ditto/basic-search.md +++ b/documentation/src/main/resources/pages/ditto/basic-search.md @@ -35,6 +35,10 @@ That means that when a thing is updated and the API (e.g. the HTTP endpoint) ret will not reflect that change in that instant. The change will most likely be reflected in the search index within 1-2 seconds. In rare cases the duration until consistency is reached again might be higher. +If it is important to know when a twin modification is reflected in the search index, request the +[built-in acknowledgement](#built-in-acknowledgement-labels) `search-persisted` in the corresponding command. +Search index update is successful if the status code of `search-persisted` in the command response is 204 "no content". +Status codes at or above 400 indicate failed search index update due to client or server errors. ## Search queries From 4992d826373cfb1f857c3d95c45c2cd11f26dfe3 Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Tue, 15 Dec 2020 11:56:38 +0100 Subject: [PATCH 4/7] [#914] review: * some minor simplifications * solved some deprecation warnings using deprecated Akka APIs * adjusted since version to 2.0.0 * updated mutability-detector.version Signed-off-by: Thomas Jaeckle --- bom/pom.xml | 2 +- legal/3rd-party-dependencies/test.txt | 2 +- .../acks/AbstractCommandAckRequestSetter.java | 2 +- .../base/acks/DittoAcknowledgementLabel.java | 2 +- .../persistence/write/model/Metadata.java | 6 +---- .../streaming/BulkWriteResultAckFlow.java | 21 ++++------------- .../write/streaming/ChangeQueueActor.java | 23 +++++++++---------- .../write/streaming/EnforcementFlow.java | 8 +++---- .../updater/actors/NewEventForwarder.java | 6 ++--- 9 files changed, 28 insertions(+), 44 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 962e6f1775..2d62afc1f5 100755 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -88,7 +88,7 @@ 4.13.1 3.12.0 - 0.10.2 + 0.10.4 3.1.10 3.1.0 1.2.3 diff --git a/legal/3rd-party-dependencies/test.txt b/legal/3rd-party-dependencies/test.txt index 13bfd4480a..a752bbd140 100644 --- a/legal/3rd-party-dependencies/test.txt +++ b/legal/3rd-party-dependencies/test.txt @@ -23,7 +23,7 @@ org.hamcrest:hamcrest-library:jar:1.3:test org.json:json:jar:20090211:test org.json:json:jar:20180130:test org.mockito:mockito-core:jar:3.1.0:test -org.mutabilitydetector:MutabilityDetector:jar:0.10.2:test +org.mutabilitydetector:MutabilityDetector:jar:0.10.4:test org.objenesis:objenesis:jar:2.6:test org.openjdk.jmh:jmh-core:jar:1.21:test org.openjdk.jmh:jmh-generator-annprocess:jar:1.21:test diff --git a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java index cabef0b274..dc62056f2b 100644 --- a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java +++ b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/AbstractCommandAckRequestSetter.java @@ -73,7 +73,7 @@ protected AbstractCommandAckRequestSetter(final AcknowledgementLabel implicitAck * * @param implicitAcknowledgementLabel the label to set if the header 'requested-acks' is absent. * @param negatedDittoAcknowledgementLabels the labels to filter out if present. - * @since 1.6.0 + * @since 2.0.0 */ protected AbstractCommandAckRequestSetter(final AcknowledgementLabel implicitAcknowledgementLabel, final Set negatedDittoAcknowledgementLabels) { diff --git a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java index e7d18d616b..8127845c12 100644 --- a/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java +++ b/model/base/src/main/java/org/eclipse/ditto/model/base/acks/DittoAcknowledgementLabel.java @@ -44,7 +44,7 @@ public final class DittoAcknowledgementLabel implements AcknowledgementLabel { * Label for Acknowledgements indicating that a change to an entity (e. g. a thing) has successfully been reflected * in the search index. * - * @since 1.6.0 + * @since 2.0.0 */ public static final DittoAcknowledgementLabel SEARCH_PERSISTED = new DittoAcknowledgementLabel("search-persisted"); diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java index 1aa16935ad..2d31a14652 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/model/Metadata.java @@ -43,10 +43,6 @@ public final class Metadata { @Nullable private final PolicyId policyId; @Nullable private final Long policyRevision; @Nullable final Instant modified; - - /** - * Using Scala list to have immutability and constant-time prepend at the same time. - */ private final List senders; private Metadata(final ThingId thingId, @@ -61,7 +57,7 @@ private Metadata(final ThingId thingId, this.policyId = policyId; this.policyRevision = policyRevision; this.modified = modified; - this.senders = senders; + this.senders = senders; // does not need to be made unmodifiable as there is no getter returning that to the "outside world" } /** diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java index 528a4f78f8..5f631996b1 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/BulkWriteResultAckFlow.java @@ -19,7 +19,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; import org.eclipse.ditto.model.base.headers.DittoHeaders; @@ -105,16 +104,10 @@ private void acknowledgeSuccesses(final BitSet failedIndices, final List things) { - errorsCounter.increment(things.size()); - acknowledge(things, BulkWriteResultAckFlow::createFailureResponse); - } - - private void acknowledge(final List things, - final Function responseCreator) { - - for (final Metadata metadata : things) { - final UpdateThingResponse response = responseCreator.apply(metadata); + private void acknowledgeFailures(final List metadataList) { + errorsCounter.increment(metadataList.size()); + for (final Metadata metadata : metadataList) { + final UpdateThingResponse response = createFailureResponse(metadata); final ShardedMessageEnvelope envelope = ShardedMessageEnvelope.of(response.getEntityId(), response.getType(), response.toJson(), response.getDittoHeaders()); @@ -124,16 +117,12 @@ private void acknowledge(final List things, } private static UpdateThingResponse createFailureResponse(final Metadata metadata) { - return createResponse(metadata, false); - } - - private static UpdateThingResponse createResponse(final Metadata metadata, final boolean isSuccess) { return UpdateThingResponse.of( metadata.getThingId(), metadata.getThingRevision(), metadata.getPolicyId().map(PolicyId::of).orElse(null), metadata.getPolicyId().flatMap(policyId -> metadata.getPolicyRevision()).orElse(null), - isSuccess, + false, DittoHeaders.empty() ); } diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java index 2553d85e55..f20cb2bb7d 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/ChangeQueueActor.java @@ -25,7 +25,7 @@ import akka.actor.Props; import akka.japi.function.Function; import akka.japi.pf.ReceiveBuilder; -import akka.pattern.PatternsCS; +import akka.pattern.Patterns; import akka.stream.Attributes; import akka.stream.DelayOverflowStrategy; import akka.stream.javadsl.Source; @@ -100,17 +100,16 @@ private void dump(final Control dump) { } private static Function, NotUsed>> askSelf(final ActorRef self) { - return message -> - Source.fromSourceCompletionStage( - PatternsCS.ask(self, message, ASK_SELF_TIMEOUT) - .handle((result, error) -> { - if (result instanceof Map) { - return Source.single((Map) result); - } else { - return Source.empty(); - } - })) - .mapMaterializedValue(whatever -> NotUsed.getInstance()); + return message -> Source.completionStageSource( + Patterns.ask(self, message, ASK_SELF_TIMEOUT) + .handle((result, error) -> { + if (result instanceof Map) { + return Source.single((Map) result); + } else { + return Source.empty(); + } + })) + .mapMaterializedValue(whatever -> NotUsed.getInstance()); } private enum Control { diff --git a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java index b94ee486fd..b025fa7d0c 100644 --- a/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java +++ b/services/thingsearch/persistence/src/main/java/org/eclipse/ditto/services/thingsearch/persistence/write/streaming/EnforcementFlow.java @@ -199,7 +199,7 @@ private Source sudoRetrieveThing(final Thing return Source.single((SudoRetrieveThingResponse) response); } else { if (error != null) { - log.error("Failed " + command, error); + log.error("Failed command <{}>", command, error); } else if (!(response instanceof ThingNotAccessibleException)) { log.error("Unexpected response for <{}>: <{}>", command, response); } @@ -207,7 +207,7 @@ private Source sudoRetrieveThing(final Thing } }); - return Source.fromSourceCompletionStage(responseFuture) + return Source.completionStageSource(responseFuture) .viaMat(Flow.create(), Keep.none()); } @@ -267,7 +267,7 @@ private Source, NotUsed> getEnforcer(final Metadata metadata, fi private Source, NotUsed> readCachedEnforcer(final Metadata metadata, final EntityIdWithResourceType policyId, final int iteration) { - final Source, ?> lazySource = Source.lazily(() -> { + final Source, ?> lazySource = Source.lazySource(() -> { final CompletionStage, NotUsed>> enforcerFuture = policyEnforcerCache.get(policyId) .thenApply(optionalEnforcerEntry -> { if (shouldReloadCache(optionalEnforcerEntry.orElse(null), metadata, iteration)) { @@ -285,7 +285,7 @@ private Source, NotUsed> readCachedEnforcer(final Metadata metad return ENFORCER_NONEXISTENT; }); - return Source.fromSourceCompletionStage(enforcerFuture); + return Source.completionStageSource(enforcerFuture); }); return lazySource.viaMat(Flow.create(), Keep.none()); diff --git a/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java b/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java index 2cf3db124a..8b42300d69 100644 --- a/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java +++ b/services/thingsearch/updater-actors/src/main/java/org/eclipse/ditto/services/thingsearch/updater/actors/NewEventForwarder.java @@ -36,7 +36,7 @@ import akka.actor.Props; import akka.cluster.sharding.ShardRegion; import akka.japi.pf.ReceiveBuilder; -import scala.concurrent.duration.FiniteDuration; +import scala.concurrent.duration.Duration; /** * This Actor forwards thing events belonging to inactive shard regions. @@ -74,7 +74,7 @@ private NewEventForwarder(final DistributedSub thingEventSub, namespaceBlockingBehavior = BlockNamespaceBehavior.of(blockedNamespaces); getClusterShardingStats = new ShardRegion.GetClusterShardingStats( - FiniteDuration.create(updaterConfig.getShardingStatePollInterval().toMillis(), TimeUnit.MILLISECONDS)); + Duration.create(updaterConfig.getShardingStatePollInterval().toMillis(), TimeUnit.MILLISECONDS)); shardRegionExtractor = ShardRegionExtractor.of(clusterConfig.getNumberOfShards(), getContext().getSystem()); @@ -134,7 +134,7 @@ private void processThingEvent(final ThingEvent thingEvent) { log.withCorrelationId(thingEvent) .debug("Forwarding incoming ThingEvent for thingId '{}'", thingEvent.getThingEntityId()); final ActorRef sender = getSender(); - namespaceBlockingBehavior.block(thingEvent).thenAccept(m -> shardRegion.tell(m, sender())); + namespaceBlockingBehavior.block(thingEvent).thenAccept(m -> shardRegion.tell(m, sender)); } private static Collection getActiveShardIds(final ShardRegion.ClusterShardingStats stats) { From 0a7d9133d915c84eaaec5a40d117f38d10aaca5f Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 21 Dec 2020 09:05:56 +0100 Subject: [PATCH 5/7] #748 added commit hook for checking that license header's year for added files is the file's creation year Signed-off-by: Thomas Jaeckle --- .github/workflows/license-check.yml | 45 ++++++++++++++++++++++++++++ .github/workflows/maven.yml | 10 +++++++ .github/workflows/push-dockerhub.yml | 10 +++++++ 3 files changed, 65 insertions(+) create mode 100644 .github/workflows/license-check.yml diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml new file mode 100644 index 0000000000..26d8a14c92 --- /dev/null +++ b/.github/workflows/license-check.yml @@ -0,0 +1,45 @@ +# Copyright (c) 2020 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 +name: license-check + +on: + # Run build for any PR + pull_request: + +jobs: + check-license-header-year: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: jitterbit/get-changed-files@v1 + id: the-files + - name: Printing added files + run: | + echo "Added:" + echo "${{ steps.the-files.outputs.added }}" + - name: Ensure license year for added files is the file's creation year + shell: bash + run: | + included_file_endings=".*\.(java|xml|yml)" + current_year=$(date +'%Y') + missing_counter=0 + for file in ${{ steps.the-files.outputs.added }}; do + if [[ $file =~ $included_file_endings ]]; then + file_creation_year=$(git log --format=%aD $file | tail -1 | awk '{print $4}') + if grep -q "Copyright (c) $file_creation_year Contributors to the Eclipse Foundation" $file; then + printf "\xE2\x9C\x94 $file\n" + else + printf "\xE2\x9D\x8C $file\n\tcopyright header with file creation year '$file_creation_year' is missing in added file\n" + missing_counter=$(expr $missing_counter + 1) + fi + fi + done + exit $missing_counter \ No newline at end of file diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bc80d721b5..7612eb4ebf 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,3 +1,13 @@ +# Copyright (c) 2020 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 name: build on: diff --git a/.github/workflows/push-dockerhub.yml b/.github/workflows/push-dockerhub.yml index 3864e08df8..75499df6b9 100644 --- a/.github/workflows/push-dockerhub.yml +++ b/.github/workflows/push-dockerhub.yml @@ -1,3 +1,13 @@ +# Copyright (c) 2020 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 name: push-dockerhub on: From 3cb62a59cf11cf3781532a73820f5b779e97516c Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 21 Dec 2020 13:42:07 +0100 Subject: [PATCH 6/7] update swagger-ui used in Ditto sandbox + docker deployment + k8s deployment * re-introduce Google based authentication for Sandbox * pre-authenticate with "ditto:ditto" in sandbox + docker Signed-off-by: Thomas Jaeckle --- deployment/docker/docker-compose.yml | 2 +- deployment/docker/sandbox/docker-compose.yml | 2 +- deployment/docker/sandbox/html/index.html | 6 +---- deployment/docker/swagger3-index.html | 17 +++++------- deployment/kubernetes/swagger/swagger.yaml | 2 +- .../src/main/resources/http-api-doc.html | 7 +++-- .../main/resources/openapi/ditto-api-1.yml | 22 ++++++++++++---- .../main/resources/openapi/ditto-api-2.yml | 22 ++++++++++++---- .../resources/openapi/sources/api-1-index.yml | 25 +++++++----------- .../resources/openapi/sources/api-2-index.yml | 26 +++++++------------ .../openapi/sources/security/bearerAuth.yml | 5 ++++ .../openapi/sources/security/google.yml | 10 +++++++ .../openapi/sources/security/nginxBasic.yml | 4 +++ 13 files changed, 89 insertions(+), 61 deletions(-) create mode 100644 documentation/src/main/resources/openapi/sources/security/bearerAuth.yml create mode 100644 documentation/src/main/resources/openapi/sources/security/google.yml create mode 100644 documentation/src/main/resources/openapi/sources/security/nginxBasic.yml diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index bf3479b9ac..2a02c7f019 100755 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -142,7 +142,7 @@ services: command: java -Dditto.gateway.authentication.devops.password=foobar -jar starter.jar swagger-ui: - image: docker.io/swaggerapi/swagger-ui:v3.20.5 + image: docker.io/swaggerapi/swagger-ui:v3.38.0 volumes: - ../../documentation/src/main/resources/openapi:/usr/share/nginx/html/openapi:ro - ../../documentation/src/main/resources/images:/usr/share/nginx/html/images:ro diff --git a/deployment/docker/sandbox/docker-compose.yml b/deployment/docker/sandbox/docker-compose.yml index 780ecb0027..a5de9ad65f 100644 --- a/deployment/docker/sandbox/docker-compose.yml +++ b/deployment/docker/sandbox/docker-compose.yml @@ -121,7 +121,7 @@ services: - OPENJ9_JAVA_OPTIONS=-XX:+ExitOnOutOfMemoryError -Xtune:virtualized -Xss512k -XX:MaxRAMPercentage=80 -Dakka.coordinated-shutdown.exit-jvm=on -Dakka.cluster.shutdown-after-unsuccessful-join-seed-nodes=120s swagger-ui: - image: docker.io/swaggerapi/swagger-ui:v3.20.5 + image: docker.io/swaggerapi/swagger-ui:v3.38.0 volumes: - ../../../documentation/src/main/resources/openapi:/usr/share/nginx/html/openapi:ro - ../../../documentation/src/main/resources/images:/usr/share/nginx/html/images:ro diff --git a/deployment/docker/sandbox/html/index.html b/deployment/docker/sandbox/html/index.html index 4423492f00..4b3b82bc82 100644 --- a/deployment/docker/sandbox/html/index.html +++ b/deployment/docker/sandbox/html/index.html @@ -115,11 +115,7 @@

-

Authentication

When using the HTTP API you can authenticate with your Google account in order to use - an user ID belonging to only you.
- WARNING: This currently won't work due to a missing feature - in swagger-ui 3.x: swagger-ui #4084 and - due to a missing feature in Ditto that Google 'accept_token' are not supported: - Ditto #114. + an user ID belonging to only you.

Alternatively (also in order to connect to the WebSocket API) you can use one of the pre-defined user/password diff --git a/deployment/docker/swagger3-index.html b/deployment/docker/swagger3-index.html index 65e915ec2d..7c74434618 100644 --- a/deployment/docker/swagger3-index.html +++ b/deployment/docker/swagger3-index.html @@ -44,13 +44,6 @@ diff --git a/deployment/kubernetes/swagger/swagger.yaml b/deployment/kubernetes/swagger/swagger.yaml index 0205a7d5d1..62d7defcbc 100644 --- a/deployment/kubernetes/swagger/swagger.yaml +++ b/deployment/kubernetes/swagger/swagger.yaml @@ -23,7 +23,7 @@ spec: spec: containers: - name: swagger-ui - image: docker.io/swaggerapi/swagger-ui:3.17.4 # old version v2.2.9 + image: docker.io/swaggerapi/swagger-ui:3.38.0 volumeMounts: - name: swagger-ui-api mountPath: /usr/share/nginx/html/openapi diff --git a/documentation/src/main/resources/http-api-doc.html b/documentation/src/main/resources/http-api-doc.html index ae240e55aa..766968c25a 100644 --- a/documentation/src/main/resources/http-api-doc.html +++ b/documentation/src/main/resources/http-api-doc.html @@ -14,7 +14,7 @@ // Build a system window.ui = SwaggerUIBundle({ urls: [ - {url: "openapi/ditto-api-1.yml", name: "Ditto API v1- deprecated"}, + {url: "openapi/ditto-api-1.yml", name:"Ditto API v1- deprecated"}, {url: "openapi/ditto-api-2.yml", name:"Ditto API v2"} ], "urls.primaryName": "Ditto API v2", @@ -29,7 +29,10 @@ plugins: [ SwaggerUIBundle.plugins.DownloadUrl ], - layout: "StandaloneLayout" + layout: "StandaloneLayout", + onComplete: function() { + ui.preauthorizeBasic("NginxBasic", "ditto", "ditto"); + } }); } diff --git a/documentation/src/main/resources/openapi/ditto-api-1.yml b/documentation/src/main/resources/openapi/ditto-api-1.yml index 7a1b5261a9..e0635468fb 100644 --- a/documentation/src/main/resources/openapi/ditto-api-1.yml +++ b/documentation/src/main/resources/openapi/ditto-api-1.yml @@ -24,8 +24,10 @@ tags: - name: Messages description: Talk with your Things security: - - basicAuth: [] - - bearerAuth: [] + - Google: + - openid + - NginxBasic: [] + - Bearer: [] paths: /things: get: @@ -2491,15 +2493,25 @@ components: * for an list, the JSON value is the list: `[ 1,2,3 ]` required: true securitySchemes: - basicAuth: + NginxBasic: type: http description: Eclipse Ditto sandbox demo user (demo1 ... demo9) + password (demo) scheme: basic - bearerAuth: + Bearer: type: http scheme: bearer bearerFormat: JWT - description: 'A JSON Web Token issued by a supported OAuth 2.0 Identity Provider, e.g. a Google "id_token"' + description: A JSON Web Token issued by a supported OAuth 2.0 Identity Provider. + Google: + type: oauth2 + description: Allow Eclipse Ditto to perform OAuth2.0 based authentication accessing your Google-ID. + x-tokenName: id_token + flows: + authorizationCode: + authorizationUrl: 'https://accounts.google.com/o/oauth2/v2/auth' + tokenUrl: /oauth2/google/token + scopes: + openid: Access your Google-ID schemas: Error: properties: diff --git a/documentation/src/main/resources/openapi/ditto-api-2.yml b/documentation/src/main/resources/openapi/ditto-api-2.yml index 4bf045651f..86e21e3000 100644 --- a/documentation/src/main/resources/openapi/ditto-api-2.yml +++ b/documentation/src/main/resources/openapi/ditto-api-2.yml @@ -28,8 +28,10 @@ tags: - name: CloudEvents description: Process CloudEvents in Ditto security: - - basicAuth: [] - - bearerAuth: [] + - Google: + - openid + - NginxBasic: [] + - Bearer: [] paths: /things: get: @@ -5844,12 +5846,22 @@ components: type: string description: An auth subject that can be used to provide access for a caller (e.g. in subject entries of policies). securitySchemes: - basicAuth: + NginxBasic: type: http description: Eclipse Ditto sandbox demo user (demo1 ... demo9) + password (demo) scheme: basic - bearerAuth: + Bearer: type: http scheme: bearer bearerFormat: JWT - description: 'A JSON Web Token issued by a supported OAuth 2.0 Identity Provider, e.g. a Google "id_token"' + description: A JSON Web Token issued by a supported OAuth 2.0 Identity Provider. + Google: + type: oauth2 + description: Allow Eclipse Ditto to perform OAuth2.0 based authentication accessing your Google-ID. + x-tokenName: id_token + flows: + authorizationCode: + authorizationUrl: 'https://accounts.google.com/o/oauth2/v2/auth' + tokenUrl: /oauth2/google/token + scopes: + openid: Access your Google-ID diff --git a/documentation/src/main/resources/openapi/sources/api-1-index.yml b/documentation/src/main/resources/openapi/sources/api-1-index.yml index 8e81b1b8ec..233cfefd7f 100644 --- a/documentation/src/main/resources/openapi/sources/api-1-index.yml +++ b/documentation/src/main/resources/openapi/sources/api-1-index.yml @@ -34,10 +34,10 @@ tags: - name: Messages description: Talk with your Things security: -# - Google: -# - openid - - basicAuth: [] - - bearerAuth: [] + - Google: + - openid + - NginxBasic: [] + - Bearer: [] paths: /things: get: @@ -2539,17 +2539,12 @@ components: * for an list, the JSON value is the list: `[ 1,2,3 ]` required: true securitySchemes: - basicAuth: - type: http - description: |- - Eclipse Ditto sandbox demo user (demo1 ... demo9) + password (demo) - scheme: basic - bearerAuth: - type: http - scheme: bearer - bearerFormat: JWT - description: |- - A JSON Web Token issued by a supported OAuth 2.0 Identity Provider, e.g. a Google "id_token" + NginxBasic: + $ref: './security/nginxBasic.yml' + Bearer: + $ref: './security/bearerAuth.yml' + Google: + $ref: './security/google.yml' schemas: Error: properties: diff --git a/documentation/src/main/resources/openapi/sources/api-2-index.yml b/documentation/src/main/resources/openapi/sources/api-2-index.yml index 57c110ae6f..93517f3eed 100644 --- a/documentation/src/main/resources/openapi/sources/api-2-index.yml +++ b/documentation/src/main/resources/openapi/sources/api-2-index.yml @@ -41,10 +41,10 @@ tags: description: Process CloudEvents in Ditto security: - # - Google: - # - openid - - basicAuth: [ ] - - bearerAuth: [ ] + - Google: + - openid + - NginxBasic: [ ] + - Bearer: [ ] paths: @@ -257,15 +257,9 @@ components: $ref: "./schemas/whoamiSubject.yml" securitySchemes: - basicAuth: - type: http - description: |- - Eclipse Ditto sandbox demo user (demo1 ... demo9) + password (demo) - scheme: basic - bearerAuth: - type: http - scheme: bearer - bearerFormat: JWT - description: |- - A JSON Web Token issued by a supported OAuth 2.0 Identity Provider, e.g. a Google "id_token" - + NginxBasic: + $ref: './security/nginxBasic.yml' + Bearer: + $ref: './security/bearerAuth.yml' + Google: + $ref: './security/google.yml' diff --git a/documentation/src/main/resources/openapi/sources/security/bearerAuth.yml b/documentation/src/main/resources/openapi/sources/security/bearerAuth.yml new file mode 100644 index 0000000000..ec0e6b7f12 --- /dev/null +++ b/documentation/src/main/resources/openapi/sources/security/bearerAuth.yml @@ -0,0 +1,5 @@ +type: http +scheme: bearer +bearerFormat: JWT +description: |- + A JSON Web Token issued by a supported OAuth 2.0 Identity Provider. \ No newline at end of file diff --git a/documentation/src/main/resources/openapi/sources/security/google.yml b/documentation/src/main/resources/openapi/sources/security/google.yml new file mode 100644 index 0000000000..3ee92d0330 --- /dev/null +++ b/documentation/src/main/resources/openapi/sources/security/google.yml @@ -0,0 +1,10 @@ +type: oauth2 +description: |- + Allow Eclipse Ditto to perform OAuth2.0 based authentication accessing your Google-ID. +x-tokenName: id_token +flows: + authorizationCode: + authorizationUrl: 'https://accounts.google.com/o/oauth2/v2/auth' + tokenUrl: /oauth2/google/token + scopes: + openid: Access your Google-ID \ No newline at end of file diff --git a/documentation/src/main/resources/openapi/sources/security/nginxBasic.yml b/documentation/src/main/resources/openapi/sources/security/nginxBasic.yml new file mode 100644 index 0000000000..0d34639612 --- /dev/null +++ b/documentation/src/main/resources/openapi/sources/security/nginxBasic.yml @@ -0,0 +1,4 @@ +type: http +description: |- + Eclipse Ditto sandbox demo user (demo1 ... demo9) + password (demo) +scheme: basic \ No newline at end of file From e5f0071007bc10060191ddeb7aa99319635ce1ae Mon Sep 17 00:00:00 2001 From: Thomas Jaeckle Date: Mon, 21 Dec 2020 13:47:34 +0100 Subject: [PATCH 7/7] added missing copyright headers Signed-off-by: Thomas Jaeckle --- .../resources/openapi/sources/security/bearerAuth.yml | 10 ++++++++++ .../main/resources/openapi/sources/security/google.yml | 10 ++++++++++ .../resources/openapi/sources/security/nginxBasic.yml | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/documentation/src/main/resources/openapi/sources/security/bearerAuth.yml b/documentation/src/main/resources/openapi/sources/security/bearerAuth.yml index ec0e6b7f12..bf54a0c80a 100644 --- a/documentation/src/main/resources/openapi/sources/security/bearerAuth.yml +++ b/documentation/src/main/resources/openapi/sources/security/bearerAuth.yml @@ -1,3 +1,13 @@ +# Copyright (c) 2020 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 type: http scheme: bearer bearerFormat: JWT diff --git a/documentation/src/main/resources/openapi/sources/security/google.yml b/documentation/src/main/resources/openapi/sources/security/google.yml index 3ee92d0330..4e4000304c 100644 --- a/documentation/src/main/resources/openapi/sources/security/google.yml +++ b/documentation/src/main/resources/openapi/sources/security/google.yml @@ -1,3 +1,13 @@ +# Copyright (c) 2020 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 type: oauth2 description: |- Allow Eclipse Ditto to perform OAuth2.0 based authentication accessing your Google-ID. diff --git a/documentation/src/main/resources/openapi/sources/security/nginxBasic.yml b/documentation/src/main/resources/openapi/sources/security/nginxBasic.yml index 0d34639612..d5442a6f39 100644 --- a/documentation/src/main/resources/openapi/sources/security/nginxBasic.yml +++ b/documentation/src/main/resources/openapi/sources/security/nginxBasic.yml @@ -1,3 +1,13 @@ +# Copyright (c) 2020 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0 +# +# SPDX-License-Identifier: EPL-2.0 type: http description: |- Eclipse Ditto sandbox demo user (demo1 ... demo9) + password (demo)