Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
Merge pull request #18264 from akka/wip-18190-leveldb-EventsByPersist…
…enceId-patriknw +per #18190 leveldb impl of EventsByPersistenceId query
- Loading branch information
Showing
16 changed files
with
484 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
####################################################### | ||
# Akka Persistence Query Reference Configuration File # | ||
####################################################### | ||
|
||
# This is the reference config file that contains all the default settings. | ||
# Make your edits in your application.conf in order to override these settings. | ||
|
||
akka.persistence.query { | ||
journal { | ||
leveldb { | ||
class = "akka.persistence.query.journal.leveldb.LeveldbReadJournal" | ||
|
||
# Absolute path to the write journal plugin configuration entry that this query journal | ||
# will connect to. That must be a LeveldbJournal or SharedLeveldbJournal. | ||
# If undefined (or "") it will connect to the default journal as specified by the | ||
# akka.persistence.journal.plugin property. | ||
write-plugin = "" | ||
|
||
# Look for more data with this interval. The query journal is also notified by | ||
# the write journal when something is changed and thereby updated quickly, but | ||
# when there are a lot of changes it falls back to periodic queries to avoid | ||
# overloading the system with many small queries. | ||
refresh-interval = 3s | ||
|
||
# How many events to fetch in one query and keep buffered until they | ||
# are delivered downstreams. | ||
max-buffer-size = 100 | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
...sistence-query/src/main/scala/akka/persistence/query/journal/leveldb/DeliveryBuffer.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
/* | ||
* Copyright (C) 2009-2015 Typesafe Inc. <http://www.typesafe.com> | ||
*/ | ||
package akka.persistence.query.journal.leveldb | ||
|
||
import akka.stream.actor.ActorPublisher | ||
|
||
/** | ||
* INTERNAL API | ||
*/ | ||
private[akka] trait DeliveryBuffer[T] { _: ActorPublisher[T] ⇒ | ||
|
||
var buf = Vector.empty[T] | ||
|
||
def deliverBuf(): Unit = | ||
if (buf.nonEmpty && totalDemand > 0) { | ||
if (buf.size == 1) { | ||
// optimize for this common case | ||
onNext(buf.head) | ||
buf = Vector.empty | ||
} else if (totalDemand <= Int.MaxValue) { | ||
val (use, keep) = buf.splitAt(totalDemand.toInt) | ||
buf = keep | ||
use foreach onNext | ||
} else { | ||
buf foreach onNext | ||
buf = Vector.empty | ||
} | ||
} | ||
|
||
} |
132 changes: 132 additions & 0 deletions
132
...rc/main/scala/akka/persistence/query/journal/leveldb/EventsByPersistenceIdPublisher.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,132 @@ | ||
/** | ||
* Copyright (C) 2015 Typesafe Inc. <http://www.typesafe.com> | ||
*/ | ||
package akka.persistence.query.journal.leveldb | ||
|
||
import scala.concurrent.duration._ | ||
import akka.actor.ActorLogging | ||
import akka.actor.ActorRef | ||
import akka.actor.Props | ||
import akka.persistence.JournalProtocol._ | ||
import akka.persistence.Persistence | ||
import akka.stream.actor.ActorPublisher | ||
import akka.stream.actor.ActorPublisherMessage.Cancel | ||
import akka.stream.actor.ActorPublisherMessage.Request | ||
import akka.persistence.journal.leveldb.LeveldbJournal | ||
import akka.persistence.query.EventEnvelope | ||
|
||
/** | ||
* INTERNAL API | ||
*/ | ||
private[akka] object EventsByPersistenceIdPublisher { | ||
def props(persistenceId: String, fromSequenceNr: Long, toSequenceNr: Long, refreshInterval: Option[FiniteDuration], | ||
maxBufSize: Int, writeJournalPluginId: String): Props = | ||
Props(new EventsByPersistenceIdPublisher(persistenceId, fromSequenceNr, toSequenceNr, refreshInterval, | ||
maxBufSize, writeJournalPluginId)) | ||
|
||
private case object Continue | ||
} | ||
|
||
class EventsByPersistenceIdPublisher(persistenceId: String, fromSequenceNr: Long, toSequenceNr: Long, | ||
refreshInterval: Option[FiniteDuration], | ||
maxBufSize: Int, writeJournalPluginId: String) | ||
extends ActorPublisher[EventEnvelope] with DeliveryBuffer[EventEnvelope] with ActorLogging { | ||
import EventsByPersistenceIdPublisher._ | ||
|
||
val journal: ActorRef = Persistence(context.system).journalFor(writeJournalPluginId) | ||
|
||
var currSeqNo = fromSequenceNr | ||
|
||
val tickTask = refreshInterval.map { interval ⇒ | ||
import context.dispatcher | ||
context.system.scheduler.schedule(interval, interval, self, Continue) | ||
} | ||
|
||
def nonLiveQuery: Boolean = refreshInterval.isEmpty | ||
|
||
override def postStop(): Unit = { | ||
tickTask.foreach(_.cancel()) | ||
} | ||
|
||
def receive = init | ||
|
||
def init: Receive = { | ||
case _: Request ⇒ | ||
journal ! LeveldbJournal.SubscribePersistenceId(persistenceId) | ||
replay() | ||
case Continue ⇒ // skip, wait for first Request | ||
case Cancel ⇒ context.stop(self) | ||
} | ||
|
||
def idle: Receive = { | ||
case Continue | _: LeveldbJournal.ChangedPersistenceId ⇒ | ||
if (timeForReplay) | ||
replay() | ||
|
||
case _: Request ⇒ | ||
deliverBuf() | ||
if (nonLiveQuery) { | ||
if (buf.isEmpty) | ||
onCompleteThenStop() | ||
else | ||
self ! Continue | ||
} | ||
|
||
case Cancel ⇒ | ||
context.stop(self) | ||
|
||
} | ||
|
||
def timeForReplay: Boolean = | ||
buf.isEmpty || buf.size <= maxBufSize / 2 | ||
|
||
def replay(): Unit = { | ||
val limit = maxBufSize - buf.size | ||
log.debug("request replay for persistenceId [{}] from [{}] to [{}] limit [{}]", persistenceId, currSeqNo, toSequenceNr, limit) | ||
journal ! ReplayMessages(currSeqNo, toSequenceNr, limit, persistenceId, self) | ||
context.become(replaying(limit)) | ||
} | ||
|
||
def replaying(limit: Int): Receive = { | ||
var replayCount = 0 | ||
|
||
{ | ||
case ReplayedMessage(p) ⇒ | ||
buf :+= EventEnvelope( | ||
offset = p.sequenceNr, | ||
persistenceId = persistenceId, | ||
sequenceNr = p.sequenceNr, | ||
event = p.payload) | ||
currSeqNo = p.sequenceNr + 1 | ||
replayCount += 1 | ||
deliverBuf() | ||
|
||
case _: RecoverySuccess ⇒ | ||
log.debug("replay completed for persistenceId [{}], currSeqNo [{}], replayCount [{}]", persistenceId, currSeqNo, replayCount) | ||
deliverBuf() | ||
if (buf.isEmpty && currSeqNo > toSequenceNr) | ||
onCompleteThenStop() | ||
else if (nonLiveQuery) { | ||
if (buf.isEmpty && replayCount < limit) | ||
onCompleteThenStop() | ||
else | ||
self ! Continue // more to fetch | ||
} | ||
context.become(idle) | ||
|
||
case ReplayMessagesFailure(cause) ⇒ | ||
log.debug("replay failed for persistenceId [{}], due to [{}]", persistenceId, cause.getMessage) | ||
deliverBuf() | ||
onErrorThenStop(cause) | ||
|
||
case _: Request ⇒ | ||
deliverBuf() | ||
|
||
case Continue | _: LeveldbJournal.ChangedPersistenceId ⇒ // skip during replay | ||
|
||
case Cancel ⇒ | ||
context.stop(self) | ||
} | ||
} | ||
|
||
} |
51 changes: 51 additions & 0 deletions
51
...ence-query/src/main/scala/akka/persistence/query/journal/leveldb/LeveldbReadJournal.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
/** | ||
* Copyright (C) 2015 Typesafe Inc. <http://www.typesafe.com> | ||
*/ | ||
package akka.persistence.query.journal.leveldb | ||
|
||
import scala.concurrent.duration._ | ||
import akka.actor.ExtendedActorSystem | ||
import akka.persistence.query.EventsByPersistenceId | ||
import akka.persistence.query.Hint | ||
import akka.persistence.query.Query | ||
import akka.persistence.query.scaladsl | ||
import akka.serialization.SerializationExtension | ||
import akka.stream.scaladsl.Source | ||
import scala.concurrent.duration.FiniteDuration | ||
import akka.persistence.query.NoRefresh | ||
import akka.persistence.query.RefreshInterval | ||
import com.typesafe.config.Config | ||
import akka.persistence.query.EventEnvelope | ||
|
||
object LeveldbReadJournal { | ||
final val Identifier = "akka.persistence.query.journal.leveldb" | ||
} | ||
|
||
class LeveldbReadJournal(system: ExtendedActorSystem, config: Config) extends scaladsl.ReadJournal { | ||
|
||
private val serialization = SerializationExtension(system) | ||
private val defaulRefreshInterval: Option[FiniteDuration] = | ||
Some(config.getDuration("refresh-interval", MILLISECONDS).millis) | ||
private val writeJournalPluginId: String = config.getString("write-plugin") | ||
private val maxBufSize: Int = config.getInt("max-buffer-size") | ||
|
||
override def query[T, M](q: Query[T, M], hints: Hint*): Source[T, M] = q match { | ||
case EventsByPersistenceId(pid, from, to) ⇒ eventsByPersistenceId(pid, from, to, hints) | ||
case unknown ⇒ unsupportedQueryType(unknown) | ||
} | ||
|
||
def eventsByPersistenceId(persistenceId: String, fromSeqNr: Long, toSeqNr: Long, hints: Seq[Hint]): Source[EventEnvelope, Unit] = { | ||
Source.actorPublisher[EventEnvelope](EventsByPersistenceIdPublisher.props(persistenceId, fromSeqNr, toSeqNr, | ||
refreshInterval(hints), maxBufSize, writeJournalPluginId)).mapMaterializedValue(_ ⇒ ()) | ||
} | ||
|
||
private def refreshInterval(hints: Seq[Hint]): Option[FiniteDuration] = | ||
if (hints.contains(NoRefresh)) | ||
None | ||
else | ||
hints.collectFirst { case RefreshInterval(interval) ⇒ interval }.orElse(defaulRefreshInterval) | ||
|
||
private def unsupportedQueryType[M, T](unknown: Query[T, M]): Nothing = | ||
throw new IllegalArgumentException(s"${getClass.getSimpleName} does not implement the ${unknown.getClass.getName} query type!") | ||
} | ||
|
Oops, something went wrong.