diff --git a/CHANGELOG.md b/CHANGELOG.md index 24159bb2..5ad0ed96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## v2.0.0 - 2025-11-20 + +### Breaking Changes + +- **Removed `redisNamespace`**: The `redisNamespace` parameter has been replaced by the new `namespace` concept. Use the `namespace` parameter to achieve the same functionality. +- **Legacy Event Processor behavior change**: Events without a returned status are now treated as successfully processed (instead of erroneous). This prevents multiple processing attempts for the same event. + +## Added + +- **Namespace support for multi-service setups**: Introduced namespaces to support scenarios where multiple microservices share the same database (HDI Container). See [documentation](https://cap-js-community.github.io/event-queue/configure-event/#namespaces). +- **Enhanced CAP outbox service return values**: CAP outbox services can now return more detailed information. In addition to the event status, you can now update the `startAfter` and `error` fields of an event. See [documentation](https://cap-js-community.github.io/event-queue/configure-event/#namespaces). +- **Configurable reprocessing for 'Open' event status**: With the event configuration property `When an event status of` `Open` is returned, you can now configure when such events should be reprocessed. This allows you to mark event processing as invalid without increasing the attempt counter. +- **CDS namespace support in service names**: Service names now support CDS namespaces, e.g., `cds.env.requires["cds.xt.DeploymentService"]`. +- **Support for `cds.queued`**: Added support for the `cds.queued` property. +- [Admin Service] allow to publish events via Admin Service to all or a defined list of tenants + ## v1.11.1 - 2025-11-13 ### Added diff --git a/docs/Gemfile b/docs/Gemfile index 4b65f4bc..d2dd01aa 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -1,8 +1,13 @@ source "https://rubygems.org" +# https://pages.github.com/versions/ + +# https://github.com/ruby/ruby +ruby "~> 3.4.7" + # NOTE: this fixes the relevant jekyll version # https://github.com/github/pages-gem/releases -gem "github-pages", "~> 231", group: :jekyll_plugins +gem "github-pages", "~> 232", group: :jekyll_plugins # https://github.com/just-the-docs/just-the-docs/releases -gem "just-the-docs", "~> 0.8.1" \ No newline at end of file +gem "just-the-docs", "~> 0.10.1" \ No newline at end of file diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 0c911f26..f2a6542e 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -1,45 +1,66 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.1.7.8) - concurrent-ruby (~> 1.0, >= 1.0.2) + activesupport (8.1.1) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) minitest (>= 5.1) - tzinfo (~> 2.0) - zeitwerk (~> 2.3) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) - base64 (0.2.0) + base64 (0.3.0) + bigdecimal (3.3.1) coffee-script (2.4.1) coffee-script-source execjs coffee-script-source (1.12.2) colorator (1.1.0) - commonmarker (0.23.10) - concurrent-ruby (1.3.4) - dnsruby (1.72.2) + commonmarker (0.23.12) + concurrent-ruby (1.3.5) + connection_pool (2.5.4) + csv (3.3.5) + dnsruby (1.73.1) + base64 (>= 0.2) + logger (~> 1.6) simpleidn (~> 0.2.1) + drb (2.2.3) em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) - ethon (0.16.0) + ethon (0.15.0) ffi (>= 1.15.0) eventmachine (1.2.7) - execjs (2.9.1) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-net_http (3.0.2) - ffi (1.17.0) + execjs (2.10.0) + faraday (2.14.0) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.2) + net-http (~> 0.5) + ffi (1.17.2-aarch64-linux-gnu) + ffi (1.17.2-aarch64-linux-musl) + ffi (1.17.2-arm-linux-gnu) + ffi (1.17.2-arm-linux-musl) + ffi (1.17.2-arm64-darwin) + ffi (1.17.2-x86_64-darwin) + ffi (1.17.2-x86_64-linux-gnu) + ffi (1.17.2-x86_64-linux-musl) forwardable-extended (2.6.0) gemoji (4.1.0) - github-pages (231) + github-pages (232) github-pages-health-check (= 1.18.2) - jekyll (= 3.9.5) + jekyll (= 3.10.0) jekyll-avatar (= 0.8.0) jekyll-coffeescript (= 1.2.2) - jekyll-commonmark-ghpages (= 0.4.0) + jekyll-commonmark-ghpages (= 0.5.1) jekyll-default-layout (= 0.1.5) jekyll-feed (= 0.17.0) jekyll-gist (= 1.5.0) @@ -76,9 +97,10 @@ GEM liquid (= 4.0.4) mercenary (~> 0.3) minima (= 2.5.1) - nokogiri (>= 1.13.6, < 2.0) + nokogiri (>= 1.16.2, < 2.0) rouge (= 3.30.0) terminal-table (~> 1.4) + webrick (~> 1.8) github-pages-health-check (1.18.2) addressable (~> 2.3) dnsruby (~> 1.60) @@ -89,11 +111,12 @@ GEM activesupport (>= 2) nokogiri (>= 1.4) http_parser.rb (0.8.0) - i18n (1.14.6) + i18n (1.14.7) concurrent-ruby (~> 1.0) - jekyll (3.9.5) + jekyll (3.10.0) addressable (~> 2.4) colorator (~> 1.0) + csv (~> 3.0) em-websocket (~> 0.5) i18n (>= 0.7, < 2) jekyll-sass-converter (~> 1.0) @@ -104,6 +127,7 @@ GEM pathutil (~> 0.9) rouge (>= 1.7, < 4) safe_yaml (~> 1.0) + webrick (>= 1.0) jekyll-avatar (0.8.0) jekyll (>= 3.0, < 5.0) jekyll-coffeescript (1.2.2) @@ -111,9 +135,9 @@ GEM coffee-script-source (~> 1.12) jekyll-commonmark (1.4.0) commonmarker (~> 0.22) - jekyll-commonmark-ghpages (0.4.0) - commonmarker (~> 0.23.7) - jekyll (~> 3.9.0) + jekyll-commonmark-ghpages (0.5.1) + commonmarker (>= 0.23.7, < 1.1.0) + jekyll (>= 3.9, < 4.0) jekyll-commonmark (~> 1.4.0) rouge (>= 2.0, < 5.0) jekyll-default-layout (0.1.5) @@ -199,7 +223,8 @@ GEM gemoji (>= 3, < 5) html-pipeline (~> 2.2) jekyll (>= 3.0, < 5.0) - just-the-docs (0.8.2) + json (2.16.0) + just-the-docs (0.10.1) jekyll (>= 3.8.5) jekyll-include-cache jekyll-seo-tag (>= 2.0) @@ -212,15 +237,30 @@ GEM listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) + logger (1.7.0) mercenary (0.3.6) - mini_portile2 (2.8.7) minima (2.5.1) jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - minitest (5.25.1) - nokogiri (1.13.10) - mini_portile2 (~> 2.8.0) + minitest (5.26.1) + net-http (0.7.0) + uri + nokogiri (1.18.10-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.10-aarch64-linux-musl) + racc (~> 1.4) + nokogiri (1.18.10-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.10-arm-linux-musl) + racc (~> 1.4) + nokogiri (1.18.10-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-linux-musl) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) @@ -229,39 +269,50 @@ GEM forwardable-extended (~> 2.6) public_suffix (5.1.1) racc (1.8.1) - rake (13.2.1) + rake (13.3.1) rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) - rexml (3.3.8) + rexml (3.4.4) rouge (3.30.0) - ruby2_keywords (0.0.5) - rubyzip (2.3.2) + rubyzip (2.4.1) safe_yaml (1.0.5) sass (3.7.4) sass-listen (~> 4.0.0) sass-listen (4.0.0) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.9.2) + sawyer (0.9.3) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) + securerandom (0.4.1) simpleidn (0.2.3) terminal-table (1.8.0) unicode-display_width (~> 1.1, >= 1.1.1) - typhoeus (1.4.1) - ethon (>= 0.9.0) + typhoeus (1.5.0) + ethon (>= 0.9.0, < 0.16.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (1.8.0) - zeitwerk (2.6.18) + uri (1.1.1) + webrick (1.9.1) PLATFORMS - ruby + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + x86_64-darwin + x86_64-linux-gnu + x86_64-linux-musl DEPENDENCIES - github-pages (~> 231) - just-the-docs (~> 0.8.1) + github-pages (~> 232) + just-the-docs (~> 0.10.1) + +RUBY VERSION + ruby 3.4.7p58 BUNDLED WITH - 1.17.2 + 2.7.2 diff --git a/docs/configure-event/index.md b/docs/configure-event/index.md index 68902ea8..f03a9658 100644 --- a/docs/configure-event/index.md +++ b/docs/configure-event/index.md @@ -17,7 +17,7 @@ nav_order: 4 # Where to Define Events? -Events don't need to be defined if the CAP Outbox approach is used as they are auto detected. However, if you want to +Events don't need to be defined if the CAP Queue approach is used as they are auto detected. However, if you want to customize the event configuration like retry attempts, transaction mode, parallel processing, etc., you can configure this via `cds.env`. The legacy approach of explictly define Events and implementing Event-Queue classes is still supported @@ -26,7 +26,7 @@ and decribed here, but not recommended to use. {% include warning.html message=" Before event-queue version 1.10.0, it was necessary to implement EventQueue classes to take full advantage of features such as periodic events, clustering, hooks for exceeded events, and more. Since version 1.10.0, all these features are -also available for CAP services using [event-queue as an outbox](/event-queue/use-as-cap-outbox/). Therefore, it is +also available for CAP services using [event-queue as CAP Queue](/event-queue/use-as-cap-outbox/). Therefore, it is strongly recommended to use CAP services instead of EventQueue classes. " %} @@ -45,10 +45,11 @@ they should be processed. | impl | Path of the implementation class associated with the event. | - | | type | Specifies the type of the event. | - | | subType | Specifies the subtype of the event, further categorizing the event type. | - | +| namespace | Default namespace in which event is published | default | | load | Indicates the load of the event, affecting the processing concurrency. | 1 | | retryAttempts | Number of retry attempts for failed events. Set to `-1` for infinite retries. | 3 | | processAfterCommit | Indicates whether an event is processed immediately after the transaction, in which the event was written, has been committed. | true | -| propagateHeaders | Specifies which headers from the original CDS context should be forwarded to the outbox call. Provide an array of header names to propagate. | [] | +| propagateHeaders | Specifies which headers from the original CDS context should be forwarded to the queue call. Provide an array of header names to propagate. | [] | | parallelEventProcessing | Number of events of the same type and subType that can be processed in parallel. The maximum limit is 10. | 1 | | transactionMode | Specifies the transaction mode for the event. For allowed values, refer to [Transaction Handling](/event-queue/transaction-handling/#transaction-modes). | isolated | | selectMaxChunkSize | Number of events selected in a single batch. Set `checkForNextChunk` to `true` if you want to check for more available events after processing a batch. | 100 | @@ -78,6 +79,7 @@ instance is overloaded. | impl | Path of the implementation class associated with the event. | - | | type | Specifies the type of the periodic event. | - | | subType | Specifies the subtype of the periodic event, further categorizing the event type. | - | +| namespace | Default namespace in which event is published | default | | load | Indicates the load of the event, affecting the processing concurrency. | 1 | | transactionMode | Specifies the transaction mode for the periodic event. For allowed values, refer to [Transaction Handling](/event-queue/transaction-handling/#transaction-modes). | isolated | | interval | The interval, in seconds, at which the periodic event should be triggered. | - | @@ -190,6 +192,133 @@ maintains stability and avoids overloading. | `0 0 L * *` | Runs at midnight on the last day of every month. | | `0 6,18 * * *` | Runs at 6:00 AM and 6:00 PM every day. | +# Namespaces + +The event-queue supports namespaces to logically partition events within the same database table. This feature is +particularly useful in architectures where multiple, distinct microservices operate on the same database schema but have +different application logic and responsibilities. By using namespaces, you can ensure that events published by one +service are only processed by the intended service(s), preventing conflicts and unintended behavior. + +## Use Case + +Imagine two microservices, Service-A and Service-B, both connected to the same database. Service-A is responsible for +order management, while Service-B handles inventory updates. Both services use the event queue for asynchronous processing. + +Without namespaces, an “order created” event published by Service-A would be visible to Service-B. Service-B might try +to process this event, leading to errors because it lacks the necessary logic to handle order events. Namespaces solve +this by creating separate, logical channels for events, ensuring each microservice only interacts with relevant events. + +## How It Works + +The namespace mechanism is based on two key configuration parameters: + +- **namespace**: When an event is published, it is tagged with a namespace. This configuration sets the default namespace + for all events published by the application. +- **processingNamespaces**: This configuration defines a list of namespaces that the current application instance will + listen to. The event-queue scheduler will only select events for processing if their namespace is included in this list. + +By default, all events are published to the "default" namespace, and all applications process events from the "default" +namespace, ensuring backward compatibility for existing projects. + +## Example Scenario + +Let’s apply this to our use case with Service-A (Orders) and Service-B (Inventory). + +### Configuration for Service-A (package.json): + +```json +{ + "cds": { + "eventQueue": { + "namespace": "orders", + "processingNamespaces": ["orders"] + } + } +} +``` + +- Service-A will publish all its events to the orders namespace. +- It will only process events that belong to the orders namespace. + +### Configuration for Service-B (package.json): + +```json +{ + "cds": { + "eventQueue": { + "namespace": "inventory", + "processingNamespaces": ["inventory"] + } + } +} +``` + +- Service-B will publish all its events to the inventory namespace. +- It will only process events that belong to the inventory namespace. + +With this setup, order events from Service-A will be completely isolated from Service-B, and vice versa, even though they +share the same physical event queue table. + +## Processing Events from Multiple Namespaces + +An application can also be configured to process events from multiple namespaces. For instance, a central monitoring or +logging service might need to process events from all other services. + +### Configuration for a Logging-Service (package.json): + +```json +{ + "cds": { + "eventQueue": { + "processingNamespaces": ["orders", "inventory", "default"] + } + } +} +``` + +This service will now pick up events from the orders, inventory, and default namespaces, while still ignoring events from +any other unlisted namespaces. + +## Advanced Namespace Control & Precedence + +While the global namespace configuration is straightforward, you can achieve more granular control by defining namespaces +at the service level or even dynamically for a single event emission. + +The namespace for an event is determined based on the following order of precedence, from highest to lowest: + +1. Dynamic Header (`x-eventqueue-namespace`): A namespace specified in the headers during an event emit/send call. +2. Service/Event Definition: A namespace defined for a specific service or event in your `cds.env.requires.` configuration. +3. Global Configuration: The default namespace set in the top-level `cds.env.eventQueue` configuration. + +### Configuration at the Service or Event Level + +You can assign a specific namespace to a CAP service’s or even to a particular event within that service. This is +useful when a service consistently publishes events to a namespace different from the global default. + +In this example, the `StandardService` defaults to publishing events to `namespaceA`. However, the specific event +`timeBucketAction` is configured to publish to `namespaceC`. + +```json +{ + "cds": { + "requires": { + "StandardService": { + "impl": "...", + "queued": { + "kind": "persistent-queue", + "namespace": "namespaceA", + "events": { + "timeBucketAction": { + "namespace": "namespaceC" + } + } + } + } + } + } +} +``` + # Runtime Configuration Changes In certain scenarios, it may be necessary to change configurations during runtime. The event-queue has two main diff --git a/docs/diff-to-outbox/index.md b/docs/diff-to-outbox/index.md index 08e404cd..35ed04d4 100644 --- a/docs/diff-to-outbox/index.md +++ b/docs/diff-to-outbox/index.md @@ -17,10 +17,10 @@ nav_order: 2 In the following chapter, the event queue will be compared and differentiated with other SAP products. -## CAP persistent outbox +## CAP persistent queue -The event-queue package and the CAP persistent outbox share several features, but the event-queue package provides -additional functionalities. A key distinction is that the CAP outbox is designed exclusively for outboxing HTTP requests +The event-queue package and the CAP persistent queue share several features, but the event-queue package provides +additional functionalities. A key distinction is that the CAP Queue is designed exclusively for queueing HTTP requests and should not be used for asynchronous processing of business data. Another important difference is that the event-queue includes load balancing and periodic event scheduling, offering greater flexibility and efficiency in event processing. diff --git a/docs/implement-event/index.md b/docs/implement-event/index.md index d2007ffe..6e9d1709 100644 --- a/docs/implement-event/index.md +++ b/docs/implement-event/index.md @@ -20,7 +20,7 @@ nav_order: 12 {% include warning.html message=" Before event-queue version 1.10.0, it was necessary to implement EventQueue classes to take full advantage of features such as periodic events, clustering, hooks for exceeded events, and more. Since version 1.10.0, all these features are -also available for CAP services using [event-queue as an outbox](/event-queue/use-as-cap-outbox/). Therefore, it is strongly recommended to use CAP +also available for CAP services using [event-queue as CAP Queue](/event-queue/use-as-cap-outbox/). Therefore, it is strongly recommended to use CAP services instead of EventQueue classes. " %} diff --git a/docs/index.md b/docs/index.md index b37fe81b..7e30bf8e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,7 +27,7 @@ processing. - [Load balancing](/event-queue/load-balancing) and concurrency control for event processing across app instances, protecting the application from load spikes. -- [Use as CAP outbox](/event-queue/use-as-cap-outbox) with full support for all event-queue features. +- [Use as CAP Queue](/event-queue/use-as-cap-outbox) with full support for all event-queue features. - [Periodic events](/event-queue/configure-event/#periodic-events) using cron patterns with load management for optimal job execution. - [Managed transactions](/event-queue/transaction-handling) for reliable event processing. diff --git a/docs/publish-event/index.md b/docs/publish-event/index.md index 4ca19bd8..7a6de621 100644 --- a/docs/publish-event/index.md +++ b/docs/publish-event/index.md @@ -18,7 +18,7 @@ nav_order: 11 {% include warning.html message=" Before event-queue version 1.10.0, it was necessary to implement EventQueue classes to take full advantage of features such as periodic events, clustering, hooks for exceeded events, and more. Since version 1.10.0, all these features are -also available for CAP services using [event-queue as an outbox](/event-queue/use-as-cap-outbox/). Therefore, it is strongly recommended to use CAP +also available for CAP services using [event-queue as CAP Queue](/event-queue/use-as-cap-outbox/). Therefore, it is strongly recommended to use CAP services instead of EventQueue classes. " %} diff --git a/docs/setup/index.md b/docs/setup/index.md index 488ade69..76fd2ded 100644 --- a/docs/setup/index.md +++ b/docs/setup/index.md @@ -23,18 +23,18 @@ nav_order: 3 - Run `npm add @cap-js-community/event-queue` in `@sap/cds` project - Initialize the event queue as CAP-Plugin or manually in your server.js -## Using CDS Outbox +## Using CAP Queue {% include warning.html message=" Before event-queue version 1.10.0, it was necessary to implement EventQueue classes to take full advantage of features such as periodic events, clustering, hooks for exceeded events, and more. Since version 1.10.0, all these features are -also available for CAP services using [event-queue as an outbox](/event-queue/use-as-cap-outbox/). Therefore, it is strongly recommended to use CAP +also available for CAP services using [event-queue as CAP Queue](/event-queue/use-as-cap-outbox/). Therefore, it is strongly recommended to use CAP services instead of EventQueue classes. " %} -The simplest way to utilize the event-queue is by allowing it to manage the CDS outbox and outbox services via the -outbox method in conjunction with the event-queue. To accomplish this, the event-queue needs to be set up as a CDS -outbox. Refer to the following guides +The simplest way to utilize the event-queue is by allowing it to manage the CAP Queue and queue services via the +queue method in conjunction with the event-queue. To accomplish this, the event-queue needs to be set up as a CDS +queue. Refer to the following guides on [how to configure the event-queue](/event-queue/use-as-cap-outbox/#how-to-enable-the-event-queue-as-outbox-mechanism-for-cap) and [how to implement a CDS service](/event-queue/use-as-cap-outbox/#example-of-a-custom-outboxed-service).. @@ -72,15 +72,16 @@ The table includes the parameter name, a description of its purpose, and the def | runInterval [ms] | The interval in milliseconds at which the runner runs. | 25 _ 60 _ 1000 | yes | | updatePeriodicEvents | Whether or not to update periodic events. | true | no | | thresholdLoggingEventProcessing [ms] | Threshold after how many milliseconds the processing of a event or periodic event is logged for observability. | 50 | yes | -| useAsCAPQueue/useAsCAPOutbox | Uses the event-queue as the [outbox](https://cap.cloud.sap/docs/node.js/outbox) of CAP. Outbox calls are stored and processed in the event-queue instead of the outbox of CAP. | false | no | +| useAsCAPQueue/useAsCAPOutbox | Uses the event-queue as the [queue](https://cap.cloud.sap/docs/node.js/queue) of CAP. Queued calls are stored and processed in the event-queue instead of the queue of CAP. | false | no | | userId | User id for all created cds contexts. This influences the value for updated managed database fields like createdBy and modifiedBy. | false | yes | | cleanupLocksAndEventsForDev | Deletes all semantic locks and sets all events that are in progress to error during server start. This is used to clean up leftovers from server crashes or restarts during processing. | true | no | -| insertEventsBeforeCommit | If enabled, this feature allows events (including those for outboxed services) to be inserted in bulk using the before commit handler. This is performed to improve performance by mass inserting events instead of single insert operations. This can be disabled by the parameter `skipInsertEventsBeforeCommit` in the function publishEvent. | true | yes | +| insertEventsBeforeCommit | If enabled, this feature allows events (including those for queued services) to be inserted in bulk using the before commit handler. This is performed to improve performance by mass inserting events instead of single insert operations. This can be disabled by the parameter `skipInsertEventsBeforeCommit` in the function publishEvent. | true | yes | | enableTelemetry | If enabled, OpenTelemetry traces for all event-queue activities are written. An OpenTelemetry exporter must be configured. | true | yes | | cronTimezone | Determines whether to apply the central `cronTimezone` setting for scheduling events. If set to `true` and the property `utc` is not enabled for the given event, it will use the defined `cronTimezone`. If set to `false`, the event will use UTC or the server's local time, based on the `utc` setting. | null | yes | | randomOffsetPeriodicEvents | Specifies the default maximum random offset (in seconds) applied to all periodic events to help stagger their start times and reduce simultaneous execution spikes. This setting is especially useful in multi-tenant environments where many events may be triggered at the same time. The actual offset for each event will be a random number between `0` and this value. Can be overridden per event using the `randomOffset` property. | null | yes | | disableRedis | Whether or not to disable Redis. | false | no | -| redisNamespace | Prefix applied to all Redis interactions. Use this when multiple microservices share a Redis instance to prevent naming conflicts. | null | no | +| namespace | The property determines the default namespace in which events are published if no other namespace is configured for the event. Check out the namespace section for more details. | default | no | +| processingNamespaces | The property determines which namespaces are processed in this application. Check out the namespace section for more details. | \[default\] | no | | redisOptions | The option is provided to customize settings when creating Redis clients. The object is spread at the root level for creating a client and within the `default` options for cluster clients. | {} | no | | crashOnRedisUnavailable | If enabled, the application will crash if Redis is unavailable during the connection check. | false | false | | disableProcessingOfSuspendedTenants | Disables the processing of suspended tenants. A tenant is considered suspended if XSUAA returns `404` for the given tenant. | true | true | diff --git a/docs/unit-testing/index.md b/docs/unit-testing/index.md index 88f4b8c0..0b54bf31 100644 --- a/docs/unit-testing/index.md +++ b/docs/unit-testing/index.md @@ -16,7 +16,7 @@ nav_order: 10 # Overview -This topic provides an overview of how to write unit tests for event processors and outboxed CAP services. +This topic provides an overview of how to write unit tests for event processors and queued CAP services. # Setup @@ -116,10 +116,10 @@ it("should process an event", async () => { }); ``` -# Testing CAP outboxed services +# Testing CAP queued services -The event-queue can be used as a CDS outbox. The following examples demonstrates how to test the publishing of events -processing of outboxed CAP services as well as the processing of those events. +The event-queue can be used as a CDS Queue. The following examples demonstrates how to test the publishing of events +processing of queued CAP services as well as the processing of those events. ## Testing if an Event was Correctly Published @@ -127,13 +127,13 @@ processing of outboxed CAP services as well as the processing of those events. const cds = require("@sap/cds"); const { processEventQueue } = require("@cap-js-community/event-queue"); -it("should process an outboxed service", async () => { - // Arrange connect to service and outbox it +it("should process an queued service", async () => { + // Arrange connect to service and queue it const service = await cds.connect.to("NotificationService"); - const outboxedService = cds.outboxed(service); + const queuedService = cds.queued(service); - // Act - call the outboxed service - await outboxedService.send("sendFiori", { + // Act - call the queued service + await queuedService.send("sendFiori", { to: "to", subject: "subject", body: "body", @@ -158,11 +158,11 @@ it("should process an outboxed service", async () => { const cds = require("@sap/cds"); const { processEventQueue } = require("@cap-js-community/event-queue"); -it("should process an outboxed service", async () => { - // Arrange connect to service and outbox it +it("should process an queued service", async () => { + // Arrange connect to service and queue it const service = await cds.connect.to("NotificationService"); - const outboxedService = cds.outboxed(service); - await outboxedService.send("sendFiori", { + const queuedService = cds.queued(service); + await queuedService.send("sendFiori", { to: "to", subject: "subject", body: "body", diff --git a/docs/use-as-cap-outbox/index.md b/docs/use-as-cap-outbox/index.md index 80014d48..4266b0dd 100644 --- a/docs/use-as-cap-outbox/index.md +++ b/docs/use-as-cap-outbox/index.md @@ -1,6 +1,6 @@ --- layout: default -title: CAP Outbox +title: CAP Queue nav_order: 5 --- @@ -8,7 +8,7 @@ nav_order: 5 {: .no_toc} -# CAP Outbox +# CAP Queue @@ -17,16 +17,16 @@ nav_order: 5 -The event-queue can be used to replace the CAP outbox solution to achieve a unified and streamlined architecture for -asynchronous processing. If this feature is activated, the event-queue replaces the outbox implementation of CAP with +The event-queue can be used to replace the CAP queue solution to achieve a unified and streamlined architecture for +asynchronous processing. If this feature is activated, the event-queue replaces the queue implementation of CAP with its own implementation during the bootstrap process. This allows leveraging the features of the event-queue, such as -transaction modes, load balancing, and others, with outboxed CDS services. +transaction modes, load balancing, and others, with queued CDS services. -## How to enable the event-queue as outbox mechanism for CAP +## How to enable the event-queue as queue mechanism for CAP -The initialization parameter `useAsCAPQueue` enables the event-queue to act as a CAP outbox. To set this parameter, +The initialization parameter `useAsCAPQueue` enables the event-queue to act as a CAP queue. To set this parameter, refer to the [setup](/event-queue/setup/#initialization-parameters) part of the documentation. This is the only -configuration needed to enable the event-queue as a CAP outbox. +configuration needed to enable the event-queue as a CAP queue. ```json { @@ -36,20 +36,20 @@ configuration needed to enable the event-queue as a CAP outbox. } ``` -## How to Configure an Outboxed Service +## How to Configure a Queued Service -Services can be outboxed without any additional configuration. In this scenario, the service is outboxed using the -default parameters of the CAP outbox and the event-queue. Currently, the CAP outbox implementation supports the +Services can be queued without any additional configuration. In this scenario, the service is queued using the +default parameters of the CAP queue and the event-queue. Currently, the CAP queue implementation supports the following parameters, which are mapped to the corresponding configuration parameters of the event-queue: -| CAP outbox | event-queue | CAP default | +| CAP queue | event-queue | CAP default | | ----------- | ----------------------- | ----------------- | | chunkSize | selectMaxChunkSize | 100 | | maxAttempts | retryAttempts | 20 | | parallel | parallelEventProcessing | yes (mapped to 5) | | - | useEventQueueUser | false | -The `parallel` parameter is treated specially. The CAP outbox supports `true` or `false` as values for this parameter. +The `parallel` parameter is treated specially. The CAP queue supports `true` or `false` as values for this parameter. Since the event-queue allows specifying concurrency with a number, `parallel=true` is mapped to `parallelEventProcessing=5`, and `parallel=false` is mapped to `parallelEventProcessing=1`. For full flexibility, the configuration prioritizes the `parallelEventProcessing` parameter over `parallel`. @@ -60,22 +60,22 @@ This influences actions such as updating managed database fields like modifiedBy. The default value for this parameter is false. All supported parameters available for [EventQueueProcessors](/event-queue/configure-event/#parameters) are also -available for CAP outboxed services. This means +available for CAP queued services. This means that you can use the same configuration settings and options that you would use with EventQueueProcessors when -configuring CAP outboxed services, ensuring consistent behavior and flexibility across both use cases. +configuring CAP queued services, ensuring consistent behavior and flexibility across both use cases. Parameters are managed via the `cds.require` section, not through the config yml file as with other events. For details on maintaining the `cds.requires` section, refer to the [CAP documentation](https://cap.cloud.sap/docs/node.js/core-services#required-services). Below is an example of -using the event-queue to outbox the `@cap-js/audit-logging` service: +using the event-queue to queue the `@cap-js/audit-logging` service: ```json { "cds": { "requires": { "audit-log": { - "outbox": { - "kind": "persistent-outbox", + "queued": { + "kind": "persistent-queue", "transactionMode": "alwaysRollback", "maxAttempts": 5, "checkForNextChunk": false, @@ -87,17 +87,17 @@ using the event-queue to outbox the `@cap-js/audit-logging` service: } ``` -The parameters in the outbox section of a service are passed as configuration to the event-queue. -The `persistent-outbox` kind allows the event-queue to persist events instead of executing them in memory, mirroring the -behavior of the [CAP outbox](https://cap.cloud.sap/docs/node.js/outbox). The parameters `transactionMode`, +The parameters in the queued section of a service are passed as configuration to the event-queue. +The `persistent-queue` kind allows the event-queue to persist events instead of executing them in memory, mirroring the +behavior of the [CAP queue](https://cap.cloud.sap/docs/node.js/queue). The parameters `transactionMode`, `checkForNextChunk`, and `parallelEventProcessing` are exclusive to the event-queue. -### Periodic Actions/Events in Outboxed Services +### Periodic Actions/Events in Queued Services The event-queue supports to periodically schedule actions of a CAP service based on a cron expression or defined interval. -Every event/action in CAP Service can have a different periodicity. The periodicity is defined in the `outbox` section +Every event/action in CAP Service can have a different periodicity. The periodicity is defined in the `queued` section of the service configuration. In the example below the `syncTasks` action is scheduled every 15 minutes and the `masterDataSync` action is scheduled every day at 3:00 AM. @@ -105,8 +105,8 @@ action is scheduled every day at 3:00 AM. ```json { "my-service": { - "outbox": { - "kind": "persistent-outbox", + "queued": { + "kind": "persistent-queue", "events": { "syncTasks": { "cron": "*/15 * * * *" }, "masterDataSync": { "cron": "0 3 * * *" } @@ -126,8 +126,8 @@ default configuration for the service. ```json { "my-service": { - "outbox": { - "kind": "persistent-outbox", + "queued": { + "kind": "persistent-queue", "maxAttempts": 1, "events": { "importantTask": { "maxAttempts": 20 } } } @@ -135,11 +135,11 @@ default configuration for the service. } ``` -## Example of a Custom Outboxed Service +## Example of a Custom Queued Service ### Internal Service with `cds.Service` -The implementation below demonstrates a basic `cds.Service` that can be outboxed. If you want to configure outboxing +The implementation below demonstrates a basic `cds.Service` that can be queued. If you want to configure queuing via `cds.env.requires`, the service needs to inherit from `cds.Service`. ```js @@ -155,7 +155,7 @@ module.exports = class TaskService extends cds.Service { }; ``` -Outboxing can be enabled via configuration using `cds.env.requires`, for example, through `package.json`. +Queueing can be enabled via configuration using `cds.env.requires`, for example, through `package.json`. ```json { @@ -163,8 +163,8 @@ Outboxing can be enabled via configuration using `cds.env.requires`, for example "requires": { "task-service": { "impl": "./srv/PATH_SERVICE/taskService.js", - "outbox": { - "kind": "persistent-outbox", + "queued": { + "kind": "persistent-queue", "transactionMode": "alwaysRollback", "maxAttempts": 5, "parallelEventProcessing": 5 @@ -177,22 +177,22 @@ Outboxing can be enabled via configuration using `cds.env.requires`, for example ### Application Service with `cds.ApplicationService` -In contrast, `cds.ApplicationService`, which is served based on protocols like odata-v4, cannot be outboxed via -configuration (`cds.env.requires`). Nevertheless, outboxing can be performed manually as shown in the example below: +In contrast, `cds.ApplicationService`, which is served based on protocols like odata-v4, cannot be queued via +configuration (`cds.env.requires`). Nevertheless, queuing can be performed manually as shown in the example below: ```js const service = await cds.connect.to("task-service"); -const outboxedService = cds.outboxed(service, { - kind: "persistent-outbox", +const queuedService = cds.queued(service, { + kind: "persistent-queue", transactionMode: "alwaysRollback", }); -await outboxedService.send("process", { +await queuedService.send("process", { ID: 1, comment: "done", }); ``` -## How to cluster multiple outbox events +## How to cluster multiple queue events This functionality allows multiple service calls to be processed in a single batch, reducing redundant executions. Instead of invoking a service action separately for each event, clustering groups similar events together and processes @@ -260,16 +260,16 @@ this.on("eventQueueRetriesExceeded.sendMail", (req) => { }); ``` -## How to Delay Outboxed Service Calls +## How to Delay Queued Service Calls The event queue has a feature that enables the publication of delayed events. This feature is also applicable to CAP -outboxed services. +queued services. To implement this feature, include the `x-eventqueue-startAfter` header attribute during the send or emit process. ```js -const outboxedService = await cds.connect.to("task-service"); -await outboxedService.send( +const queuedService = await cds.connect.to("task-service"); +await queuedService.send( "process", { ID: 1, @@ -280,12 +280,12 @@ await outboxedService.send( ); ``` -## Additional parameters Outboxed Service Calls +## Additional parameters Queued Service Calls Similar to delaying published events, it is also possible to provide other parameters when publishing events. All event publication properties can be found [here](/event-queue/publish-event/#function-parameters). -## Error Handling in a Custom Outboxed Service +## Error Handling in a Custom Queued Service If the custom service is invoked via `service.send()`, errors can be raised with `req.reject()` and `req.error()`. The `reject` and `error` functions cannot be used if the service call is made via `service.emit()`. Refer to the example @@ -306,14 +306,14 @@ class TaskService extends cds.Service { } ``` -Errors raised in a custom outboxed service are thrown and will be logged from the event queue. The event entry will be +Errors raised in a custom queued service are thrown and will be logged from the event queue. The event entry will be marked as an error and will be retried based on the event configuration. ## Event-Queue properties The event queue properties that are available for the native event queue processor (refer to [this documentation](/event-queue/implement-event/#minimal-implementation-for-ad-hoc-events)) are -also accessible for outboxed services utilizing the event queue. These properties can be accessed via the cds request. +also accessible for queued services utilizing the event queue. These properties can be accessed via the cds request. The following properties are available: - processor: instance of event-queue processor @@ -332,13 +332,80 @@ class TaskService extends cds.Service { } ``` -## How to return a custom status? +## Returning a Custom Status and Event Properties from a Service Handler -It's possible to return a custom status for an event. The allowed status values are -explained [here](/event-queue/status-handling/). +Service handlers can return not just a processing **status**, but also event properties like **startAfter** and +**error**. This enables fine-grained management of the event lifecycle, including custom retry schedules and detailed +error reporting. -```js -this.on("returnPlainStatus", (req) => { +### Supported Return Properties + +For all object or array return structures, the return value is **only interpreted** if each object contains **only the allowed properties**: + +- **`status`**: A numerical value indicating the processing result for an event entry. The allowed status values are + explained [here](/event-queue/status-handling/). +- **`startAfter`**: A `Date` object that specifies the exact time after which a failed or pending event should be retried. This allows for custom backoff strategies. +- **`error`**: An `Error` object or a string to attach a custom error message to the event log, which is useful for diagnostics. + +### Return Formats + +You can return this information in several ways, depending on whether you are setting the outcome for an entire batch or +for individual entries within it. + +**1. Plain Status Value** + +For simple cases, returning a single number sets the status for all event entries in the current batch. + +```javascript +this.on("plainStatus", (req) => { + // Sets the status to Done for all entries in the batch return EventProcessingStatus.Done; }); ``` + +**2. Single Object for All Entries** + +To apply the same outcome to all entries in a batch, return a single object containing `status`, `startAfter`, and/or `error`. + +```javascript +this.on("myEvent", (req) => { + return { + status: EventProcessingStatus.Error, + startAfter: new Date(Date.now() + 60000), + error: new Error("Temporary failure, will retry."), + }; +}); +``` + +**3. Array of Tuples for Per-Entry Control** + +To set a different outcome for each event entry, return an array of tuples. Each tuple should contain the entry `ID` and +an object with its specific `status`, `startAfter`, and/or `error`. This is only useful if event clustering is used as +described [here](#how-to-cluster-multiple-queue-events). + +```javascript +this.on("myBatchEvent", (req) => { + return req.eventQueue.queueEntries.map(({ ID }, index) => [ + ID, + { + status: index % 2 === 0 ? EventProcessingStatus.Done : EventProcessingStatus.Error, + error: index % 2 === 0 ? undefined : new Error("Odd entry failed."), + }, + ]); +}); +``` + +**4. Array of Objects with IDs** + +As an alternative to tuples, you can return an array of objects. Each object must include the `ID` of the event entry +along with its specific outcome properties. + +```javascript +this.on("myBatchEvent", (req) => { + return req.eventQueue.queueEntries.map(({ ID }, index) => ({ + ID, + status: index % 2 === 0 ? EventProcessingStatus.Done : EventProcessingStatus.Error, + startAfter: index % 2 === 0 ? undefined : new Date(Date.now() + 30000), + })); +}); +``` diff --git a/src/EventQueueProcessorBase.js b/src/EventQueueProcessorBase.js index b209c3a4..079ecf47 100644 --- a/src/EventQueueProcessorBase.js +++ b/src/EventQueueProcessorBase.js @@ -296,7 +296,7 @@ class EventQueueProcessorBase { const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap; const errorHandler = (error) => { queueEntries.forEach((queueEntry) => - this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap) + this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, { statusMap }) ); this.logger.error( "The supplied status tuple doesn't have the required structure. Setting all entries to error.", @@ -317,7 +317,7 @@ class EventQueueProcessorBase { try { queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) => - this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap) + this.#determineAndAddEventStatusToMap(id, processingStatus, { statusMap }) ); } catch (error) { errorHandler(error); @@ -344,11 +344,15 @@ class EventQueueProcessorBase { } } - #determineAndAddEventStatusToMap(id, statusOrUpdateData, statusMap = this.__statusMap) { + #determineAndAddEventStatusToMap(id, statusOrUpdateData, { statusMap = this.__statusMap, error } = {}) { if (typeof statusOrUpdateData === "number") { statusOrUpdateData = { status: statusOrUpdateData }; } + if (error) { + statusOrUpdateData.error = error; + } + if (!statusMap[id]) { statusMap[id] = statusOrUpdateData; return; @@ -376,9 +380,17 @@ class EventQueueProcessorBase { } ); queueEntries.forEach((queueEntry) => - this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error) + this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, { error }) + ); + return Object.fromEntries( + queueEntries.map((queueEntry) => [ + queueEntry.ID, + { + status: EventProcessingStatus.Error, + error, + }, + ]) ); - return Object.fromEntries(queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Error])); } handleErrorDuringPeriodicEventProcessing(error, queueEntry) { @@ -389,11 +401,12 @@ class EventQueueProcessorBase { }); } - async setPeriodicEventStatus(queueEntryIds, status) { + async setPeriodicEventStatus(queueEntryIds, status, error) { await this.tx.run( UPDATE.entity(this.#config.tableNameEventQueue) .set({ status: status, + ...(error && { error: this.#error2String(error) }), }) .where({ ID: queueEntryIds, diff --git a/src/processEventQueue.js b/src/processEventQueue.js index 94b8244a..8d2ed584 100644 --- a/src/processEventQueue.js +++ b/src/processEventQueue.js @@ -154,6 +154,7 @@ const processPeriodicEvent = async (context, eventTypeInstance) => { } let status = EventProcessingStatus.Done; + let error; await executeInNewTransaction( eventTypeInstance.context, `eventQueue-periodic-process-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`, @@ -166,6 +167,7 @@ const processPeriodicEvent = async (context, eventTypeInstance) => { eventTypeInstance.startPerformanceTracerPeriodicEvents(); await eventTypeInstance.processPeriodicEvent(tx.context, queueEntry.ID, queueEntry); } catch (err) { + error = err; status = EventProcessingStatus.Error; eventTypeInstance.handleErrorDuringPeriodicEventProcessing(err, queueEntry); await tx.rollback(); @@ -190,7 +192,7 @@ const processPeriodicEvent = async (context, eventTypeInstance) => { async (tx) => { await trace(eventTypeInstance.context, "periodic-event-set-status", async () => { eventTypeInstance.processEventContext = tx.context; - await eventTypeInstance.setPeriodicEventStatus(queueEntry.ID, status); + await eventTypeInstance.setPeriodicEventStatus(queueEntry.ID, status, error); }); } ); diff --git a/test-integration/integration-main.test.js b/test-integration/integration-main.test.js index 069653dd..76f9de7a 100644 --- a/test-integration/integration-main.test.js +++ b/test-integration/integration-main.test.js @@ -136,7 +136,12 @@ describe("integration-main", () => { await eventQueue.processEventQueue(context, event.type, event.subType); expect(loggerMock.callsLengths().error).toEqual(1); expect(loggerMock.calls().error[0][1]).toMatchInlineSnapshot(`[Error: error during processing]`); - await testHelper.selectEventQueueAndExpectError(tx); + const [eventDb] = await testHelper.selectEventQueueAndReturn(tx, { additionalColumns: ["error"] }); + expect(eventDb.status).toEqual(EventProcessingStatus.Error); + expect(JSON.parse(eventDb.error)).toMatchObject({ + name: "Error", + message: "error during processing", + }); expect(dbCounts).toMatchSnapshot(); });