configure remember-entities-store = "eventsourced"

* enable "remember-entities" by default as "file access" is not required any longer with that config
* adjusted ping config to wait for the remember-entities being recovered first

Co-authored-by: Stanchev Aleksandar <aleksandar.stanchev@bosch.io>
Signed-off-by: Thomas Jaeckle <thomas.jaeckle@bosch.io>

connectivity/service/src/main/resources/connectivity-dev.conf

@@ -55,8 +55,7 @@ akka {
     }
 
     sharding {
-      # not needed when running locally in dev mode:
-      remember-entities = off
+      remember-entities = on
     }
 
     seed-nodes = [

connectivity/service/src/main/resources/connectivity.conf

@@ -716,7 +716,7 @@ ditto {
       journal-tag = ${?RECONNECT_JOURNAL_TAG}
       # initial delay for pinging / reconnecting the connections after the PersistencePingActor has been started.
       # have a little delay so that connection actors restored with "remember-entities" are not tried to be restored twice
-      initial-delay = 2s
+      initial-delay = 5s
       initial-delay = ${?RECONNECT_INITIAL_DELAY}
       # interval for trying to reconnect all started connections.
       interval = 10m
@@ -960,9 +960,23 @@ akka {
       # When this is set to 'on' the active entity actors will automatically be restarted
       # upon Shard restart. i.e. if the Shard is started on a different ShardRegion
       # due to rebalance or crash.
-      remember-entities = off
+      remember-entities = on
       remember-entities = ${?AKKA_CLUSTER_SHARDING_REMEMBER_ENTITIES}
 
+      # When 'remember-entities' is enabled and the state store mode is ddata this controls
+      # how the remembered entities and shards are stored. Possible values are "eventsourced" and "ddata"
+      # Default is ddata for backwards compatibility.
+      remember-entities-store = "eventsourced"
+      remember-entities-store = ${?AKKA_CLUSTER_SHARDING_REMEMBER_ENTITIES_STORE}
+
+      # The shard saves persistent snapshots after this number of persistent events.
+      # Snapshots are used to reduce recovery times.
+      snapshot-after = 200 # default: 1000
+      snapshot-after = ${?AKKA_CLUSTER_SHARDING_SNAPSHOT_AFTER}
+
+      journal-plugin-id = "akka-contrib-mongodb-persistence-connection-remember-journal"
+      snapshot-plugin-id = "akka-contrib-mongodb-persistence-connection-remember-snapshots"
+
       # increase default timeout because otherwise shards are beeing stopped and restarted if the ShardCoordinator
       # could not load initial entity ids from the "remember entities store" within this timeout. This is required
       # since remember-entities is set to "on".
@@ -1060,6 +1074,34 @@ akka-contrib-mongodb-persistence-connection-snapshots {
   }
 }
 
+akka-contrib-mongodb-persistence-connection-remember-journal {
+  class = "akka.contrib.persistence.mongodb.MongoJournal"
+
+  circuit-breaker {
+    max-failures = 5 # if an exception during persisting an event/snapshot occurs this often -- a successful write resets the counter
+    max-failures = ${?BREAKER_MAXTRIES}
+    call-timeout = 5s # MongoDB Timeouts causing the circuitBreaker to open
+    call-timeout = ${?BREAKER_TIMEOUT}
+    reset-timeout = 3s # after this time in "Open" state, the cicuitBreaker is "Half-opened" again
+    reset-timeout = ${?BREAKER_RESET}
+  }
+
+  overrides {
+    journal-collection = "connection_remember_journal"
+    journal-index = "connection_remember_journal_index"
+    realtime-collection = "connection_remember_realtime"
+    metadata-collection = "connection_remember_metadata"
+  }
+}
+
+akka-contrib-mongodb-persistence-connection-remember-snapshots {
+  class = "akka.contrib.persistence.mongodb.MongoSnapshots"
+  overrides {
+    snaps-collection = "connection_remember_snaps"
+    snaps-index = "connection_remember_snaps_index"
+  }
+}
+
 connection-persistence-dispatcher {
   type = Dispatcher
   executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedForkJoinExecutorServiceConfigurator"