Added sp25 for demo purpose

.gitattributes

@@ -0,0 +1 @@
+*.avro binary

.gitignore

@@ -3,7 +3,6 @@ build/
 .ipynb_checkpoints
 *.iml
 .DS_Store
-*.avro
 target/
 *.jfr
 *.log

CHANGELOG.adoc

@@ -2,16 +2,17 @@
 = Changelog
 
 == Major changes in 3.0
-Roboquant version 3.0 has several changes, several of them are breaking changes. One key reason was to clean-up and simplify areas that had grown in complexity in the past few years and were a hird to maintain and extend.
+Roboquant version 3.0 has several changes, several of them are breaking changes. One key reason was to clean-up and simplify areas that had grown in complexity in the past few years and were a hard to maintain and extend.
 
-- Signals are simplified and now use a double value for the rating rather than an enum. This is better suited for more advanced ML type of algos.
+A non-extensive list of the major changes:
 
-- SimBroker order execution simulation simplified.
+- Signals are simplified and now use a double value for the rating rather than an enum. This is better suited for more advanced ML type of algos.
 
 - Removed the `Lifecycle` interface for all components (Stratyegy, Policy, Broker).
 
-- Order handling is simplified. For example order-ids are handout by the broker (during the place order method), so not assigned when creating the order. This is more inline how
-most brokers work.
+- Order handling is simplified. For example, order-ids are handout by the broker (during the place order method). So they are no longer assigned when creating the order. This is more inline how most brokers work.
+
+- SimBroker order execution is simplified. Every order type has its own executor that is responsible for the whole execution
 
 - Improved the `Broker` interface. Only the `broker.sync()` call now provides access to the `account` object.
 
@@ -26,13 +27,15 @@ fun track(event: Event, account: Account, signals: List<Signal>, orders: List<Or
 +
 Also more clear separation of concerns since multi-run logging logic is now handled by a seperate class.
 
-- Simplified hearth-beat generation on an event channel (now done at consumer level via a time-out). This avoids a lot of possible errors and complications when nesting feeds.
+- Hearth-beat generation for live feeds now done at consumer level via a time-out. This avoids a lot of possible errors and complications when nesting live feeds.
 
-- Better naming convention for many classes/attributes. Main one is event `actions` are now called `items`.
+- Better naming convention for many classes/attributes. Two main ones are:
+    * Event `actions` are now called `items`.
+    * `Instruction` is now the interface for `Order`, `Cancellation` and `Modification`
 
-- For the time being removed the machine leanring module. Rethinking what frameworks to use for future ML algo trading.
+- For the time being removed the machine learning module. Rethinking what frameworks to use for future ML algo trading.
 
-- Removed Binance module. It was using an old unsupported API and no available in my country, so difficult to test moving to a newer API.
+- Removed `roboquant-binance` module. It was using an old unsupported API. And since they no longer operate in my country, difficult to test when moving to the newer API.
 
 - You can now start a run using the `org.roboquant.run` function. So no need to instantiate a Roboquant class anymore. The `run` also returns the latest `account` object. The run method also displays a progress bar if desired.
 +
@@ -49,10 +52,10 @@ roboquant.run(feed)
 val account = run(feed, strategy)
 ----
 
-- Use the new Alpaca API based on published OpenAPI specification.
+- Use the new Alpaca library based on published OpenAPI specification.
 
 - Simplified AvroFeed (have to see if thsi is not too simple)
 
 - Updated almost all dependencies. Only not yet moved to Kotlin 2.0 since Jupyter Notebooks are not yet supported.
 
-- Removed most of `Summary` funcitonality. This was reletive a lot of maintenance and now replaced with a simpler toString() implementation.
+- Removed most of `Summary` funcitonality. This was reletive a lot of maintenance and now replaced with a much simpler toString() implementation.

roboquant-avro/pom.xml

@@ -38,6 +38,12 @@
                 <artifactId>kotlin-maven-plugin</artifactId>
             </plugin>
         </plugins>
+        <resources>
+            <resource>
+                <directory>${basedir}/src/main/resources</directory>
+                <filtering>false</filtering>
+            </resource>
+        </resources>
     </build>
 
     <dependencies>

roboquant-avro/src/main/kotlin/org/roboquant/avro/AvroFeed.kt

@@ -30,13 +30,14 @@ import org.apache.avro.generic.GenericDatumWriter
 import org.apache.avro.generic.GenericRecord
 import org.apache.avro.io.DatumWriter
 import org.apache.avro.util.Utf8
-import org.roboquant.common.Asset
-import org.roboquant.common.Logging
-import org.roboquant.common.Timeframe
-import org.roboquant.common.compareTo
+import org.roboquant.common.*
 import org.roboquant.feeds.*
+import java.io.File
+import java.io.InputStream
 import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 import java.time.Instant
 import java.util.*
 
@@ -60,17 +61,22 @@ internal const val SCHEMA = """{
  *
  * The internal resolution is nanoseconds and stored as a single Long value
  *
- * @property path the path where the Avro file can be found
+ * @property file the Avro file
  * @property template template to use to convert the stored symbols into assets
  *
  * @constructor Create new Avro Feed
  */
-class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMPLATE")) : Feed {
+class AvroFeed(private val file: File, private val template: Asset = Asset("TEMPLATE")) : Feed {
 
     /**
      * Instantiate an Avro Feed based on the Avro file at [path]
      */
-    constructor(path: String) : this(Path.of(path))
+    constructor(path: String) : this(File(path))
+
+    /**
+     * Instantiate an Avro Feed based on the Avro file at [path]
+     */
+    constructor(path: Path) : this(path.toFile())
 
     private val logger = Logging.getLogger(AvroFeed::class)
 
@@ -80,14 +86,14 @@ class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMP
 
 
     init {
-        logger.info { "New AvroFeed path=$path exist=${exists()}" }
+        logger.info { "New AvroFeed file=$file exist=${exists()}" }
     }
 
 
-    fun exists(): Boolean = Files.exists(path)
+    fun exists(): Boolean = file.exists()
 
     private fun getReader(): DataFileReader<GenericRecord> {
-        return DataFileReader(path.toFile(), GenericDatumReader())
+        return DataFileReader(file, GenericDatumReader())
     }
 
     /**
@@ -104,7 +110,7 @@ class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMP
         val cache = mutableMapOf<String, Asset>()
         getReader().use {
             val name = it.schema.fullName
-            assert(name == "org.roboquant.avro.schema.PriceItemV2") { "invalid avro schema $name"}
+            assert(name == "org.roboquant.avro.schema.PriceItemV2") { "invalid avro schema $name" }
             if (timeframe.isFinite()) position(it, timeframe.start)
             while (it.hasNext()) {
                 val rec = it.next()
@@ -141,15 +147,15 @@ class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMP
         if (key != null) r.seek(index.getValue(key))
     }
 
-    private fun createIndex() : TreeMap<Instant, Long> {
+    private fun createIndex(): TreeMap<Instant, Long> {
         val index = TreeMap<Instant, Long>()
         getReader().use {
             while (it.hasNext()) {
                 val position = it.tell()
                 val t = it.next().get(0) as Utf8
                 it.seek(position)
                 if (it.hasNext()) {
-                    index.putIfAbsent(Instant.parse(t),position)
+                    index.putIfAbsent(Instant.parse(t), position)
                     it.nextBlock()
                 }
             }
@@ -158,7 +164,7 @@ class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMP
         return index
     }
 
-    private fun calcTimeframe() : Timeframe {
+    private fun calcTimeframe(): Timeframe {
         if (index.isEmpty()) return Timeframe.EMPTY
         val start = index.firstKey()
         getReader().use {
@@ -187,17 +193,17 @@ class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMP
         compress: Boolean = true,
         timeframe: Timeframe = Timeframe.INFINITE,
         append: Boolean = false,
-        syncInterval: Int = DataFileConstants.DEFAULT_SYNC_INTERVAL
+        syncInterval: Int = DataFileConstants.DEFAULT_SYNC_INTERVAL,
+        assetFilter: AssetFilter = AssetFilter.all()
     ) = runBlocking {
 
         val channel = EventChannel(timeframe = timeframe)
         val schema = Schema.Parser().parse(SCHEMA)
         val datumWriter: DatumWriter<GenericRecord> = GenericDatumWriter(schema)
         val dataFileWriter = DataFileWriter(datumWriter)
-        val file = path.toFile()
 
         if (append) {
-            require(exists()) {"File $file doesn't exist yet, cannot append"}
+            require(exists()) { "File $file doesn't exist yet, cannot append" }
             dataFileWriter.appendTo(file)
         } else {
             if (exists()) logger.info { "Overwriting existing Avro file $file" }
@@ -222,6 +228,7 @@ class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMP
                 val event = channel.receive()
 
                 for (action in event.items.filterIsInstance<PriceItem>()) {
+                    if (!assetFilter.filter(action.asset, event.time)) continue
 
                     val asset = action.asset
                     record.put(0, event.time.toString())
@@ -253,6 +260,41 @@ class AvroFeed(private val path: Path, private val template: Asset = Asset("TEMP
         }
     }
 
+    /**
+     * Standard set of Avro feeds that come with roboquant and will be downloaded the first time when invoked. They are
+     * stored at <User.Home>/.roboquant and reused from there later on.
+     */
+    companion object {
+
+        /**
+         * Get an AvroFeed containing end-of-day [PriceBar] data for the 25 largest US stocks. This feed
+         * contains a few years of public data.
+         *
+         * This is sample data and should NOT be relies on for real back testing.
+         */
+        fun sp25(): AvroFeed {
+            val path = copy("/sp25.avro")
+            return AvroFeed(path)
+        }
+
+        /**
+         * Copy file from jar to local filesystem
+         */
+        private fun copy(fileName: String): Path {
+            val path = Paths.get(Config.home.toString(), fileName)
+            if (Files.notExists(path)) {
+                val stream = AvroFeed::class.java.getResourceAsStream(fileName)
+                stream.use { inputStream: InputStream? ->
+                    Files.copy(
+                        inputStream!!, path, StandardCopyOption.REPLACE_EXISTING
+                    )
+                }
+                require(Files.exists(path))
+            }
+            return path
+        }
+
+    }
 
 
 }

roboquant-avro/src/test/kotlin/org/roboquant/avro/AvroFeedTest.kt

@@ -23,7 +23,7 @@ import org.junit.jupiter.api.assertDoesNotThrow
 import org.junit.jupiter.api.io.TempDir
 import org.roboquant.common.*
 import org.roboquant.feeds.*
-import org.roboquant.feeds.random.RandomWalkFeed
+import org.roboquant.feeds.random.RandomWalk
 import org.roboquant.feeds.util.AssetSerializer.deserialize
 import org.roboquant.feeds.util.AssetSerializer.serialize
 import java.io.File
@@ -112,18 +112,16 @@ internal class AvroFeedTest {
     fun append() {
         val now = Instant.now()
         val past = Timeframe(now - 2.years, now - 1.years)
-        val feed = RandomWalkFeed(past, 1.days)
+        val feed = RandomWalk(past, 1.days)
         val fileName = File(folder, "test2.avro").path
         val feed2 = AvroFeed(fileName)
         feed2.record(feed, compress = true)
 
         val past2 = Timeframe(now - 1.years, now)
-        val feed3 = RandomWalkFeed(past2, 1.days)
+        val feed3 = RandomWalk(past2, 1.days)
         feed2.record(feed3, append = true)
     }
 
-
-
     @Test
     fun assetSerialization() {
         val asset1 = Asset("XYZ")
@@ -137,7 +135,15 @@ internal class AvroFeedTest {
         assertEquals("XYZ\u001FBOND\u001FEUR\u001FAEB\u001F123", str3)
         val asset4 = str3.deserialize()
         assertEquals(asset3, asset4)
+    }
 
+    @Test
+    fun sp25() {
+        assertDoesNotThrow {
+            val feed = AvroFeed.sp25()
+            assertTrue(feed.exists())
+            assertEquals(Timeframe.parse("2020-01-02T21:00:00Z", "2023-12-29T21:00:00Z", inclusive = true), feed.timeframe)
+        }
     }
 
 }