From 222c6132cdb8b3cb9592fb3fa58e85aaa8598c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 15:10:38 +0200 Subject: [PATCH 01/68] Initial implementation --- .github/ISSUE_TEMPLATE/bug.yaml | 83 + .github/ISSUE_TEMPLATE/documentation.yaml | 32 + .github/ISSUE_TEMPLATE/feature.yaml | 40 + .github/workflows/analysis.yml | 23 + .github/workflows/linter.yml | 17 + .github/workflows/tests.yml | 28 + .gitignore | 4 + CHANGELOG.md | 21 + CODE_OF_CONDUCT.md | 76 + CONTRIBUTING.md | 101 + Dockerfile.php-8.3 | 25 + Dockerfile.php-8.4 | 25 + Dockerfile.php-8.5 | 25 + LICENSE.md | 20 + README.md | 314 ++ composer.json | 55 + composer.lock | 4103 +++++++++++++++++++++ docker-compose.yml | 36 + docs/migration.md | 193 + phpstan.neon | 19 + phpunit.xml | 19 + pint.json | 3 + src/Feed/Adapter.php | 142 + src/Feed/Adapter/Http.php | 121 + src/Feed/Adapter/Memory.php | 113 + src/Feed/Adapter/None.php | 39 + src/Feed/Adapter/Pool.php | 56 + src/Feed/Adapter/Redis.php | 97 + src/Feed/Consumer.php | 232 ++ src/Feed/Cursor.php | 87 + src/Feed/Cursor/Cache.php | 66 + src/Feed/Cursor/Memory.php | 39 + src/Feed/Cursor/Pool.php | 54 + src/Feed/Cursor/Redis.php | 72 + src/Feed/Event.php | 146 + src/Feed/Exception.php | 14 + src/Feed/Exception/Invalid.php | 18 + src/Feed/Exception/Transport.php | 21 + src/Feed/Exception/Unsupported.php | 18 + src/Feed/Feed.php | 213 ++ src/Feed/Id.php | 85 + src/Feed/Protocol.php | 199 + tests/Feed/E2E/RedisTest.php | 243 ++ tests/Feed/Unit/ConsumerTest.php | 323 ++ tests/Feed/Unit/CursorTest.php | 121 + tests/Feed/Unit/EventTest.php | 164 + tests/Feed/Unit/FeedTest.php | 282 ++ tests/Feed/Unit/HttpAdapterTest.php | 223 ++ tests/Feed/Unit/IdTest.php | 82 + tests/Feed/Unit/ProtocolTest.php | 150 + tests/Feed/Unit/RoundTripTest.php | 212 ++ tests/Feed/Unit/Support/FailingCursor.php | 40 + tests/Feed/Unit/Support/FakeTransport.php | 87 + tests/Feed/Unit/Support/FeedServer.php | 58 + 54 files changed, 9079 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug.yaml create mode 100644 .github/ISSUE_TEMPLATE/documentation.yaml create mode 100644 .github/ISSUE_TEMPLATE/feature.yaml create mode 100644 .github/workflows/analysis.yml create mode 100644 .github/workflows/linter.yml create mode 100644 .github/workflows/tests.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile.php-8.3 create mode 100644 Dockerfile.php-8.4 create mode 100644 Dockerfile.php-8.5 create mode 100755 LICENSE.md create mode 100644 README.md create mode 100644 composer.json create mode 100644 composer.lock create mode 100644 docker-compose.yml create mode 100644 docs/migration.md create mode 100644 phpstan.neon create mode 100644 phpunit.xml create mode 100644 pint.json create mode 100644 src/Feed/Adapter.php create mode 100644 src/Feed/Adapter/Http.php create mode 100644 src/Feed/Adapter/Memory.php create mode 100644 src/Feed/Adapter/None.php create mode 100644 src/Feed/Adapter/Pool.php create mode 100644 src/Feed/Adapter/Redis.php create mode 100644 src/Feed/Consumer.php create mode 100644 src/Feed/Cursor.php create mode 100644 src/Feed/Cursor/Cache.php create mode 100644 src/Feed/Cursor/Memory.php create mode 100644 src/Feed/Cursor/Pool.php create mode 100644 src/Feed/Cursor/Redis.php create mode 100644 src/Feed/Event.php create mode 100644 src/Feed/Exception.php create mode 100644 src/Feed/Exception/Invalid.php create mode 100644 src/Feed/Exception/Transport.php create mode 100644 src/Feed/Exception/Unsupported.php create mode 100644 src/Feed/Feed.php create mode 100644 src/Feed/Id.php create mode 100644 src/Feed/Protocol.php create mode 100644 tests/Feed/E2E/RedisTest.php create mode 100644 tests/Feed/Unit/ConsumerTest.php create mode 100644 tests/Feed/Unit/CursorTest.php create mode 100644 tests/Feed/Unit/EventTest.php create mode 100644 tests/Feed/Unit/FeedTest.php create mode 100644 tests/Feed/Unit/HttpAdapterTest.php create mode 100644 tests/Feed/Unit/IdTest.php create mode 100644 tests/Feed/Unit/ProtocolTest.php create mode 100644 tests/Feed/Unit/RoundTripTest.php create mode 100644 tests/Feed/Unit/Support/FailingCursor.php create mode 100644 tests/Feed/Unit/Support/FakeTransport.php create mode 100644 tests/Feed/Unit/Support/FeedServer.php diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml new file mode 100644 index 0000000..147e920 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -0,0 +1,83 @@ +name: "πŸ› Bug Report" +description: "Submit a bug report to help us improve" +title: "πŸ› Bug Report: " +labels: [bug] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out our bug report form πŸ™ + - type: textarea + id: steps-to-reproduce + validations: + required: true + attributes: + label: "πŸ‘Ÿ Reproduction steps" + description: "How do you trigger this bug? Please walk us through it step by step." + placeholder: "When I ..." + - type: textarea + id: expected-behavior + validations: + required: true + attributes: + label: "πŸ‘ Expected behavior" + description: "What did you think would happen?" + placeholder: "It should ..." + - type: textarea + id: actual-behavior + validations: + required: true + attributes: + label: "πŸ‘Ž Actual Behavior" + description: "What did actually happen? Add screenshots, if applicable." + placeholder: "It actually ..." + - type: dropdown + id: utopia-version + attributes: + label: "🎲 Utopia Abuse version" + description: "What version of Utopia Abuse are you running?" + options: + - Version 0.6.x + - Version 0.5.x + - Version 0.4.x + - Version 0.3.x + - Version 0.2.x + - Version 0.1.x + - Different version (specify in environment) + validations: + required: true + - type: dropdown + id: operating-system + attributes: + label: "πŸ’» Operating system" + description: "What OS is your server / device running on?" + options: + - Linux + - MacOS + - Windows + - Something else + validations: + required: true + - type: textarea + id: enviromnemt + validations: + required: false + attributes: + label: "🧱 Your Environment" + description: "Is your environment customized in any way?" + placeholder: "I use Cloudflare for ..." + - type: checkboxes + id: no-duplicate-issues + attributes: + label: "πŸ‘€ Have you spent some time to check if this issue has been raised before?" + description: "Have you Googled for a similar issue or checked our older issues for a similar bug?" + options: + - label: "I checked and didn't find similar issue" + required: true + - type: checkboxes + id: read-code-of-conduct + attributes: + label: "🏒 Have you read the Code of Conduct?" + options: + - label: "I have read the [Code of Conduct](https://github.com/appwrite/appwrite/blob/HEAD/CODE_OF_CONDUCT.md)" + required: true diff --git a/.github/ISSUE_TEMPLATE/documentation.yaml b/.github/ISSUE_TEMPLATE/documentation.yaml new file mode 100644 index 0000000..b6189b0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yaml @@ -0,0 +1,32 @@ +name: "πŸ“š Documentation" +description: "Report an issue related to documentation" +title: "πŸ“š Documentation: " +labels: [documentation] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to make our documentation better πŸ™ + - type: textarea + id: issue-description + validations: + required: true + attributes: + label: "πŸ’­ Description" + description: "A clear and concise description of what the issue is." + placeholder: "Documentation should not ..." + - type: checkboxes + id: no-duplicate-issues + attributes: + label: "πŸ‘€ Have you spent some time to check if this issue has been raised before?" + description: "Have you Googled for a similar issue or checked our older issues for a similar bug?" + options: + - label: "I checked and didn't find similar issue" + required: true + - type: checkboxes + id: read-code-of-conduct + attributes: + label: "🏒 Have you read the Code of Conduct?" + options: + - label: "I have read the [Code of Conduct](https://github.com/appwrite/appwrite/blob/HEAD/CODE_OF_CONDUCT.md)" + required: true diff --git a/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml new file mode 100644 index 0000000..630af44 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -0,0 +1,40 @@ +name: πŸš€ Feature +description: "Submit a proposal for a new feature" +title: "πŸš€ Feature: " +labels: [feature] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out our feature request form πŸ™ + - type: textarea + id: feature-description + validations: + required: true + attributes: + label: "πŸ”– Feature description" + description: "A clear and concise description of what the feature is." + placeholder: "You should add ..." + - type: textarea + id: pitch + validations: + required: true + attributes: + label: "🎀 Pitch" + description: "Please explain why this feature should be implemented and how it would be used. Add examples, if applicable." + placeholder: "In my use-case, ..." + - type: checkboxes + id: no-duplicate-issues + attributes: + label: "πŸ‘€ Have you spent some time to check if this issue has been raised before?" + description: "Have you Googled for a similar issue or checked our older issues for a similar bug?" + options: + - label: "I checked and didn't find similar issue" + required: true + - type: checkboxes + id: read-code-of-conduct + attributes: + label: "🏒 Have you read the Code of Conduct?" + options: + - label: "I have read the [Code of Conduct](https://github.com/appwrite/appwrite/blob/HEAD/CODE_OF_CONDUCT.md)" + required: true diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml new file mode 100644 index 0000000..e57c462 --- /dev/null +++ b/.github/workflows/analysis.yml @@ -0,0 +1,23 @@ +name: "Static analysis" + +on: [pull_request] + +jobs: + check: + name: PHPStan + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Run against the test image rather than a bare composer container: the + # Redis stream calls only resolve where ext-redis is actually installed. + - name: Build + run: | + export PHP_VERSION=8.4 + docker compose build tests + docker compose up -d tests + + - name: Run PHPStan + run: docker compose exec -T tests composer check diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..1dbd22b --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,17 @@ +name: "Linter" + +on: [pull_request] + +jobs: + lint: + name: Linter + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Linter + run: | + docker run --rm -v $PWD:/app composer sh -c \ + "composer install --profile --ignore-platform-reqs && composer lint" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..461b5f8 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,28 @@ +name: "Tests" + +on: [pull_request] + +jobs: + tests: + name: Tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + php-versions: ['8.3', '8.4', '8.5'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build + run: | + export PHP_VERSION=${{ matrix.php-versions }} + docker compose build + docker compose up -d + + - name: Unit tests + run: docker compose exec -T tests vendor/bin/phpunit --testsuite unit + + - name: E2E tests + run: docker compose exec -T tests vendor/bin/phpunit --testsuite e2e diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6bf176b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/vendor/ +/.idea/ +.phpunit.result.cache +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..08eaf70 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +## 0.1.0 + +Initial release. + +Extracted from the pull-based feed built across `appwrite-labs/cloud` (producer) +and `appwrite-labs/edge` (consumer), where the event model, the pull loop, the +cursor bookkeeping and the HTTP contract joining the two had each been written +twice. See [docs/migration.md](docs/migration.md) for how those map onto this +library. + +- `Feed` β€” append, read and long-poll an ordered event log +- `Event` β€” a CloudEvent, with a strict decode that a feed's ids can be paged from +- `Adapter\Redis`, `Adapter\Pool` β€” Redis streams, directly or over a pool +- `Adapter\Http` β€” another service's feed, read over the wire +- `Adapter\Memory`, `Adapter\None` β€” for tests, and for no backend configured +- `Consumer` β€” the pull loop, with at-least-once semantics and a durable position +- `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory` β€” where that position lives +- `Protocol` β€” the http-feeds wire contract, shared by producer and consumer +- `Id` β€” feed positions, and the arithmetic for paging past one diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..2dec654 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity, expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at team@appwrite.io. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..29eb794 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,101 @@ +# Contributing + +We would ❀️ for you to contribute to Utopia-php and help make it better! We want contributing to Utopia-php to be fun, enjoyable, and educational for anyone and everyone. All contributions are welcome, including issues, new docs as well as updates and tweaks, blog posts, workshops, and more. + +## How to Start? + +If you are worried or don’t know where to start, check out our next section explaining what kind of help we could use and where can you get involved. You can reach out with questions to [Eldad Fux (@eldadfux)](https://twitter.com/eldadfux) or anyone from the [Appwrite team on Discord](https://discord.gg/GSeTUeA). You can also submit an issue, and a maintainer can guide you! + +## Code of Conduct + +Help us keep Utopia-php open and inclusive. Please read and follow our [Code of Conduct](/CODE_OF_CONDUCT.md). + +## Submit a Pull Request πŸš€ + +Branch naming convention is as following + +`TYPE-ISSUE_ID-DESCRIPTION` + +example: + +``` +doc-548-submit-a-pull-request-section-to-contribution-guide +``` + +When `TYPE` can be: + +- **feat** - is a new feature +- **doc** - documentation only changes +- **cicd** - changes related to CI/CD system +- **fix** - a bug fix +- **refactor** - code change that neither fixes a bug nor adds a feature + +**All PRs must include a commit message with the changes description!** + +For the initial start, fork the project and use git clone command to download the repository to your computer. A standard procedure for working on an issue would be to: + +1. `git pull`, before creating a new branch, pull the changes from upstream. Your master needs to be up to date. + +``` +$ git pull +``` + +2. Create new branch from `master` like: `doc-548-submit-a-pull-request-section-to-contribution-guide`
+ +``` +$ git checkout -b [name_of_your_new_branch] +``` + +3. Work - commit - repeat ( be sure to be in your branch ) + +4. Push changes to GitHub + +``` +$ git push origin [name_of_your_new_branch] +``` + +5. Submit your changes for review + If you go to your repository on GitHub, you'll see a `Compare & pull request` button. Click on that button. +6. Start a Pull Request + Now submit the pull request and click on `Create pull request`. +7. Get a code review approval/reject +8. After approval, merge your PR +9. GitHub will automatically delete the branch after the merge is done. (they can still be restored). + +## Introducing New Features + +We would πŸ’– you to contribute to Utopia-php, but we would also like to make sure Utopia-php is as great as possible and loyal to its vision and mission statement πŸ™. + +For us to find the right balance, please open an issue explaining your ideas before introducing a new pull request. + +This will allow the Utopia-php community to have sufficient discussion about the new feature value and how it fits in the product roadmap and vision. + +This is also important for the Utopia-php lead developers to be able to give technical input and different emphasis regarding the feature design and architecture. Some bigger features might need to go through our [RFC process](https://github.com/appwrite/rfc). + +## Other Ways to Help + +Pull requests are great, but there are many other areas where you can help Utopia-php. + +### Blogging & Speaking + +Blogging, speaking about, or creating tutorials about one of Utopia-php’s many features is great way to contribute and help our project grow. + +### Presenting at Meetups + +Presenting at meetups and conferences about your Utopia-php projects. Your unique challenges and successes in building things with Utopia-php can provide great speaking material. We’d love to review your talk abstract/CFP, so get in touch with us if you’d like some help! + +### Sending Feedbacks & Reporting Bugs + +Sending feedback is a great way for us to understand your different use cases of Utopia-php better. If you had any issues, bugs, or want to share about your experience, feel free to do so on our GitHub issues page or at our [Discord channel](https://discord.gg/GSeTUeA). + +### Submitting New Ideas + +If you think Utopia-php could use a new feature, please open an issue on our GitHub repository, stating as much information as you can think about your new idea and it's implications. We would also use this issue to gather more information, get more feedback from the community, and have a proper discussion about the new feature. + +### Improving Documentation + +Submitting documentation updates, enhancements, designs, or bug fixes. Spelling or grammar fixes will be very much appreciated. + +### Helping Someone + +Searching for Utopia-php, GitHub or StackOverflow and helping someone else who needs help. You can also help by teaching others how to contribute to Utopia-php's repo! diff --git a/Dockerfile.php-8.3 b/Dockerfile.php-8.3 new file mode 100644 index 0000000..0493147 --- /dev/null +++ b/Dockerfile.php-8.3 @@ -0,0 +1,25 @@ +FROM composer:2.7 as step0 + +WORKDIR /src/ + +COPY composer.lock /src/ +COPY composer.json /src/ + +RUN composer install --ignore-platform-reqs --optimize-autoloader \ + --no-plugins --no-scripts --prefer-dist + +FROM appwrite/utopia-base:php-8.3-1.0.0 as final + +LABEL maintainer="team@appwrite.io" + +WORKDIR /code + +COPY --from=step0 /src/vendor /code/vendor + +# Add Source Code +COPY ./src /code/src +COPY ./tests /code/tests +COPY ./phpunit.xml /code/phpunit.xml +COPY ./phpstan.neon /code/phpstan.neon + +CMD [ "tail", "-f", "/dev/null" ] diff --git a/Dockerfile.php-8.4 b/Dockerfile.php-8.4 new file mode 100644 index 0000000..08df930 --- /dev/null +++ b/Dockerfile.php-8.4 @@ -0,0 +1,25 @@ +FROM composer:2.7 as step0 + +WORKDIR /src/ + +COPY composer.lock /src/ +COPY composer.json /src/ + +RUN composer install --ignore-platform-reqs --optimize-autoloader \ + --no-plugins --no-scripts --prefer-dist + +FROM appwrite/utopia-base:php-8.4-1.0.0 as final + +LABEL maintainer="team@appwrite.io" + +WORKDIR /code + +COPY --from=step0 /src/vendor /code/vendor + +# Add Source Code +COPY ./src /code/src +COPY ./tests /code/tests +COPY ./phpunit.xml /code/phpunit.xml +COPY ./phpstan.neon /code/phpstan.neon + +CMD [ "tail", "-f", "/dev/null" ] diff --git a/Dockerfile.php-8.5 b/Dockerfile.php-8.5 new file mode 100644 index 0000000..e582fd7 --- /dev/null +++ b/Dockerfile.php-8.5 @@ -0,0 +1,25 @@ +FROM composer:2.7 as step0 + +WORKDIR /src/ + +COPY composer.lock /src/ +COPY composer.json /src/ + +RUN composer install --ignore-platform-reqs --optimize-autoloader \ + --no-plugins --no-scripts --prefer-dist + +FROM appwrite/utopia-base:php-8.5-1.0.0 as final + +LABEL maintainer="team@appwrite.io" + +WORKDIR /code + +COPY --from=step0 /src/vendor /code/vendor + +# Add Source Code +COPY ./src /code/src +COPY ./tests /code/tests +COPY ./phpunit.xml /code/phpunit.xml +COPY ./phpstan.neon /code/phpstan.neon + +CMD [ "tail", "-f", "/dev/null" ] diff --git a/LICENSE.md b/LICENSE.md new file mode 100755 index 0000000..27feb08 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Eldad Fux + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0152c47 --- /dev/null +++ b/README.md @@ -0,0 +1,314 @@ +# Utopia Feed + +[![Build Status](https://github.com/utopia-php/feed/actions/workflows/tests.yml/badge.svg)](https://github.com/utopia-php/feed/actions) +![Total Downloads](https://img.shields.io/packagist/dt/utopia-php/feed.svg) +[![Discord](https://img.shields.io/discord/564160730845151244)](https://appwrite.io/discord) + +Utopia Feed is a simple and lite library for moving events between services with +**pull-based HTTP event feeds** ([http-feeds.org](https://www.http-feeds.org/)), +instead of pushing them to every service that needs them. This library is aiming +to be as simple and easy to learn and use. This library is maintained by the +[Appwrite team](https://appwrite.io). + +Although this library is part of the [Utopia +Framework](https://github.com/utopia-php/framework) project, it is dependency +light and can be used as standalone with any other PHP project or framework. + +## Why pull instead of push + +A service that pushes an event to its consumers has to reach all of them at the +moment it happens. Any consumer that is down, redeploying, rate limited or +simply new misses the event, and there is nothing in the system that will ever +tell it. The producer also grows an outbound call per consumer, has to hold a +retry queue per consumer, and has to be told when a consumer is added. + +A feed inverts that. The producer appends to an ordered log and forgets about +it. Each consumer asks *"what has happened since the last thing I saw?"*, +quoting the id of that event. A consumer that was down catches up on its next +poll. A consumer that is added later starts from whatever is still retained. The +producer keeps no per-consumer state at all, so nothing about it changes when +consumers come and go. + +``` + append GET /feeds/edge?lastEventId=... + producer ───────────────▢ feed ◀───────────────────────────── consumer A + (log) ◀───────────────────────────── consumer B + consumer C ← added later, + catches up on its own +``` + +The trade is **at-least-once delivery**: consumers retry, replay, and restart +from positions they have already passed, so every event has to be safe to +process twice. Retention is bounded, so a consumer that falls a long way behind +resumes from the oldest retained event rather than failing. That makes a feed a +poor fit for events whose effect depends on seeing every one of them (a balance +built out of deltas) and a good fit for events that describe a state to converge +on β€” a cache tag to drop, a record to refresh, a config to reload. + +## Features + +- **Ordered, resumable log** β€” consumers page by event id, and hold their own position +- **CloudEvents** β€” events are [CloudEvents](https://cloudevents.io/), as http-feeds requires +- **Adapters** β€” Redis streams, a pooled Redis, in-memory, or another service's feed over HTTP +- **Long polling** β€” subscribe in near real time without hammering the producer +- **Cursors** β€” positions in a Utopia cache, in Redis, or in memory +- **Consumer** β€” the pull loop, the position bookkeeping and the at-least-once semantics, written once + +## Getting started + +Install using composer: + +```bash +composer require utopia-php/feed +``` + +### Producing + +```php +use Utopia\Feed\Adapter\Redis as RedisAdapter; +use Utopia\Feed\Feed; + +$redis = new Redis(); +$redis->connect('redis', 6379); + +$feed = new Feed( + new RedisAdapter($redis, 'edge'), + source: 'urn:appwrite:cloud:fra', +); + +$feed->append( + type: 'io.appwrite.edge.invalidate-rule', + data: ['tags' => ['domain' => 'example.com']], + subject: 'example.com', +); +``` + +`append()` returns the event's id, which is its position in the feed. + +Give a feed a typed vocabulary by subclassing it, so callers cannot invent an +event type or misspell a payload key: + +```php +class EdgeFeed extends Feed +{ + public const string NAME = 'edge'; + + public function invalidateRule(string $domain): string + { + if ($domain === '') { + throw new \InvalidArgumentException('Rule invalidation requires a domain'); + } + + return $this->append( + 'io.appwrite.edge.invalidate-rule', + ['tags' => ['domain' => $domain]], + $domain, + ); + } +} +``` + +### Serving a feed over HTTP + +`Protocol` holds the wire contract β€” the query parameters, the response body and +the caching rules β€” so the endpoint and its consumers cannot drift apart. It +deals in arrays rather than requests and responses, so it fits whichever HTTP +layer the producer is written in: + +```php +use Utopia\Feed\Feed; +use Utopia\Feed\Protocol; + +// GET /v1/feeds/:feedId +$limit = (int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH); + +$events = $feed->poll( + $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, + $limit, + (int) $request->getParam(Protocol::PARAM_TIMEOUT, 0), +); + +$response + ->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)) + ->json(Protocol::encode($events)); +``` + +`cacheControl()` marks a full batch immutable β€” the same query returns the same +events forever β€” and a short batch `no-store`, because it is the live end of the +feed and will grow. It defaults to `private`, since a feed is usually served +behind authorization and `public` would let a shared cache hand one consumer's +events to a requester that never presented a credential. + +### Consuming + +A `Consumer` reads from where it last got to, hands each new event to a handler, +and records how far it got: + +```php +use Utopia\Feed\Consumer; +use Utopia\Feed\Cursor\Cache as CacheCursor; +use Utopia\Feed\Event; + +$consumer = new Consumer( + feed: $feed, + name: 'cache-invalidator', + cursor: new CacheCursor($cache, 'edge'), +); + +$handled = $consumer->consume(function (Event $event) use ($router) { + $router->invalidate($event->getData('tags', [])); +}); +``` + +Consuming **another service's** feed is the same code with a different adapter: + +```php +use Utopia\Feed\Adapter\Http; +use Utopia\Fetch\Client; + +$client = (new Client())->addHeader('x-appwrite-jwt', $token); + +$feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); +``` + +Nothing above the adapter knows the events are arriving over the network, +including the long polling β€” `Http` hands the wait to the producer, so a poll is +one held request rather than a client-side loop. + +Call `consume()` on a timer, or give the consumer a `timeout` and loop: + +```php +// Returns as soon as an event arrives, or after 20s with nothing. +$consumer = new Consumer($feed, 'cache-invalidator', $cursor, timeout: 20_000); + +while (true) { + $consumer->consume($handler); +} +``` + +## Adapters + +| Adapter | Use for | Append | Read | +| --- | --- | --- | --- | +| `Adapter\Redis` | Producing a feed on a Redis stream | βœ… | βœ… | +| `Adapter\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | βœ… | +| `Adapter\Http` | Consuming another service's feed | ❌ | βœ… | +| `Adapter\Memory` | Tests, and single-process development | βœ… | βœ… | +| `Adapter\None` | No backend configured | ❌ | ❌ | + +`Adapter\Pool` is what most services producing a feed want: a long poll holds +its connection for the whole timeout, so reading through a shared client would +block every other user of it. + +`Adapter\None` throws on every operation rather than doing nothing, so a +misconfigured service fails at the point of use instead of silently dropping +events β€” which only shows up much later, somewhere else. `Adapter\Memory` +implements the same id and retention semantics as `Adapter\Redis`, including the +awkward parts like resuming from a trimmed position, so code tested against it +behaves the same when it is swapped out. + +## Cursors + +http-feeds puts the position on the consumer rather than the producer, which is +what makes adding a consumer free. A cursor is just somewhere to write a string: + +| Cursor | Use for | +| --- | --- | +| `Cursor\Cache` | A consumer with a [Utopia cache](https://github.com/utopia-php/cache) β€” the usual choice for one reading a remote feed | +| `Cursor\Redis` | A consumer running inside the producer, with no store of its own | +| `Cursor\Pool` | The same, over a pooled connection | +| `Cursor\Memory` | Tests, or a consumer that should replay from the beginning on every restart | + +The store is allowed to be lossy. A lost position is not a lost event β€” a +consumer with no position resumes from the oldest retained event β€” so the +consequence is redundant work, not a gap. That is why a cache is a reasonable +place to put one, and it is also why a `Consumer` treats a store that is down as +a warning rather than a failure: it keeps its position in memory and carries on, +and only a restart before the store recovers replays anything. Pass +`onWarning()` to hear about it. + +## Delivery semantics + +**A handler must be safe to run twice on the same event.** There are three +independent reasons, and no arrangement of this library removes any of them: + +1. A handler can succeed and the position then fail to save. +2. A run interrupted partway resumes from the last event that succeeded. +3. A consumer whose position was lost restarts from the oldest retained event. + +**A handler rejects an event by throwing.** That stops the run at that event and +leaves the position before it, so the next run starts there and tries again. +Everything already handled in that run stays handled β€” progress is committed +before the failure is re-raised. A handler that keeps failing on one event +therefore blocks everything behind it, which is the intended behaviour: a feed +is ordered, and stepping over a failure would deliver later events on top of +state that was never updated. + +**A consumer with no recorded position starts at the oldest retained event, +never at the tip.** Starting at the tip would drop whatever is already in the +feed, and for a consumer being deployed for the first time that is not a +hypothetical backlog β€” it is exactly the events it was meant to catch up on. +This is what makes a staged rollout safe: ship the producer first, let events +accumulate, then ship the consumer, and nothing in between is lost. + +## Rolling out a feed + +Replacing push delivery with a feed is a two-release change, and the order +matters: + +1. **Release the producer.** It appends events; nothing reads them yet. +2. **Release the consumers.** Each drains the backlog from its first poll, + because a consumer with no position starts at the oldest retained event. +3. **Only then remove the push path.** Until every consumer is polling, removing + it means nothing is delivered. + +While step 2 is in progress, consumers that have not shipped yet will get a 404 +from a producer that does not serve the feed β€” normal, not a fault. That status +is on the exception, so it can be told apart from a real failure: + +```php +use Utopia\Feed\Exception\Transport; + +try { + $consumer->consume($handler); +} catch (Transport $error) { + if ($error->getCode() !== 404) { + throw $error; + } + + // Producer does not serve the feed yet; retry quietly until it does. +} +``` + +## Tests + +Unit tests need nothing but composer: + +```bash +composer install +composer test +``` + +The E2E suite runs against a real Redis, which is where the assumptions about +stream ids and `MAXLEN` trimming are actually confirmed: + +```bash +docker compose up -d +docker compose exec tests composer test:e2e +``` + +Static analysis runs at PHPStan level max. Run it inside the container, where +`ext-redis` is installed: + +```bash +docker compose exec tests composer check +``` + +## System requirements + +Utopia Framework requires PHP 8.3 or later. We recommend using the latest PHP +version whenever possible. + +## Copyright and license + +The MIT License (MIT) [http://www.opensource.org/licenses/mit-license.php](http://www.opensource.org/licenses/mit-license.php) diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..e8efe02 --- /dev/null +++ b/composer.json @@ -0,0 +1,55 @@ +{ + "name": "utopia-php/feed", + "description": "A simple library for publishing and consuming pull-based HTTP event feeds", + "type": "library", + "keywords": [ + "php", + "framework", + "upf", + "utopia", + "feed", + "http-feeds", + "cloudevents", + "events", + "streams" + ], + "license": "MIT", + "minimum-stability": "stable", + "scripts": { + "test": "vendor/bin/phpunit --testsuite unit", + "test:e2e": "vendor/bin/phpunit --testsuite e2e", + "check": "vendor/bin/phpstan analyse --memory-limit=1G", + "lint": "vendor/bin/pint --test", + "format": "vendor/bin/pint" + }, + "autoload": { + "psr-4": { + "Utopia\\Feed\\": "src/Feed" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Feed" + } + }, + "require": { + "php": ">=8.3", + "ext-json": "*", + "ext-redis": "*", + "utopia-php/cache": "3.*", + "utopia-php/fetch": "^1.1", + "utopia-php/pools": "1.*" + }, + "require-dev": { + "laravel/pint": "^1.21", + "phpstan/phpstan": "^2.2", + "phpunit/phpunit": "^9.6", + "swoole/ide-helper": "^6.0" + }, + "config": { + "allow-plugins": { + "php-http/discovery": true, + "tbachert/spi": true + } + } +} diff --git a/composer.lock b/composer.lock new file mode 100644 index 0000000..fa56c6b --- /dev/null +++ b/composer.lock @@ -0,0 +1,4103 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "4966f1f4c743766e6702072707dbf2b2", + "packages": [ + { + "name": "brick/math", + "version": "0.18.0", + "source": { + "type": "git", + "url": "https://github.com/brick/math.git", + "reference": "82944324d1c1bdb2c2618e89978d4e2ad78d69ad" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/brick/math/zipball/82944324d1c1bdb2c2618e89978d4e2ad78d69ad", + "reference": "82944324d1c1bdb2c2618e89978d4e2ad78d69ad", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "phpstan/phpstan": "2.1.22", + "phpunit/phpunit": "^11.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Brick\\Math\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Arbitrary-precision arithmetic library", + "keywords": [ + "Arbitrary-precision", + "BigInteger", + "BigRational", + "arithmetic", + "bigdecimal", + "bignum", + "bignumber", + "brick", + "decimal", + "integer", + "math", + "mathematics", + "rational" + ], + "support": { + "issues": "https://github.com/brick/math/issues", + "source": "https://github.com/brick/math/tree/0.18.0" + }, + "funding": [ + { + "url": "https://github.com/BenMorel", + "type": "github" + } + ], + "time": "2026-06-14T18:21:03+00:00" + }, + { + "name": "composer/semver", + "version": "3.4.4", + "source": { + "type": "git", + "url": "https://github.com/composer/semver.git", + "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/composer/semver/zipball/198166618906cb2de69b95d7d47e5fa8aa1b2b95", + "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95", + "shasum": "" + }, + "require": { + "php": "^5.3.2 || ^7.0 || ^8.0" + }, + "require-dev": { + "phpstan/phpstan": "^1.11", + "symfony/phpunit-bridge": "^3 || ^7" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Composer\\Semver\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nils Adermann", + "email": "naderman@naderman.de", + "homepage": "http://www.naderman.de" + }, + { + "name": "Jordi Boggiano", + "email": "j.boggiano@seld.be", + "homepage": "http://seld.be" + }, + { + "name": "Rob Bast", + "email": "rob.bast@gmail.com", + "homepage": "http://robbast.nl" + } + ], + "description": "Semver library that offers utilities, version constraint parsing and validation.", + "keywords": [ + "semantic", + "semver", + "validation", + "versioning" + ], + "support": { + "irc": "ircs://irc.libera.chat:6697/composer", + "issues": "https://github.com/composer/semver/issues", + "source": "https://github.com/composer/semver/tree/3.4.4" + }, + "funding": [ + { + "url": "https://packagist.com", + "type": "custom" + }, + { + "url": "https://github.com/composer", + "type": "github" + } + ], + "time": "2025-08-20T19:15:30+00:00" + }, + { + "name": "google/protobuf", + "version": "v5.35.1", + "source": { + "type": "git", + "url": "https://github.com/protocolbuffers/protobuf-php.git", + "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/55bb4a7d6739b5af0927b96213c1371a3afb7cfb", + "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb", + "shasum": "" + }, + "require": { + "php": ">=8.2.0" + }, + "require-dev": { + "phpunit/phpunit": ">=11.5.0 <12.0.0" + }, + "suggest": { + "ext-bcmath": "Need to support JSON deserialization" + }, + "type": "library", + "autoload": { + "psr-4": { + "Google\\Protobuf\\": "src/Google/Protobuf", + "GPBMetadata\\Google\\Protobuf\\": "src/GPBMetadata/Google/Protobuf" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "description": "proto library for PHP", + "homepage": "https://developers.google.com/protocol-buffers/", + "keywords": [ + "proto" + ], + "support": { + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v5.35.1" + }, + "time": "2026-06-11T21:19:23+00:00" + }, + { + "name": "nyholm/psr7", + "version": "1.8.2", + "source": { + "type": "git", + "url": "https://github.com/Nyholm/psr7.git", + "reference": "a71f2b11690f4b24d099d6b16690a90ae14fc6f3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Nyholm/psr7/zipball/a71f2b11690f4b24d099d6b16690a90ae14fc6f3", + "reference": "a71f2b11690f4b24d099d6b16690a90ae14fc6f3", + "shasum": "" + }, + "require": { + "php": ">=7.2", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.1 || ^2.0" + }, + "provide": { + "php-http/message-factory-implementation": "1.0", + "psr/http-factory-implementation": "1.0", + "psr/http-message-implementation": "1.0" + }, + "require-dev": { + "http-interop/http-factory-tests": "^0.9", + "php-http/message-factory": "^1.0", + "php-http/psr7-integration-tests": "^1.0", + "phpunit/phpunit": "^7.5 || ^8.5 || ^9.4", + "symfony/error-handler": "^4.4" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.8-dev" + } + }, + "autoload": { + "psr-4": { + "Nyholm\\Psr7\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com" + }, + { + "name": "Martijn van der Ven", + "email": "martijn@vanderven.se" + } + ], + "description": "A fast PHP7 implementation of PSR-7", + "homepage": "https://tnyholm.se", + "keywords": [ + "psr-17", + "psr-7" + ], + "support": { + "issues": "https://github.com/Nyholm/psr7/issues", + "source": "https://github.com/Nyholm/psr7/tree/1.8.2" + }, + "funding": [ + { + "url": "https://github.com/Zegnat", + "type": "github" + }, + { + "url": "https://github.com/nyholm", + "type": "github" + } + ], + "time": "2024-09-09T07:06:30+00:00" + }, + { + "name": "nyholm/psr7-server", + "version": "1.1.0", + "source": { + "type": "git", + "url": "https://github.com/Nyholm/psr7-server.git", + "reference": "4335801d851f554ca43fa6e7d2602141538854dc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Nyholm/psr7-server/zipball/4335801d851f554ca43fa6e7d2602141538854dc", + "reference": "4335801d851f554ca43fa6e7d2602141538854dc", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.0 || ^2.0" + }, + "require-dev": { + "nyholm/nsa": "^1.1", + "nyholm/psr7": "^1.3", + "phpunit/phpunit": "^7.0 || ^8.5 || ^9.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Nyholm\\Psr7Server\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com" + }, + { + "name": "Martijn van der Ven", + "email": "martijn@vanderven.se" + } + ], + "description": "Helper classes to handle PSR-7 server requests", + "homepage": "http://tnyholm.se", + "keywords": [ + "psr-17", + "psr-7" + ], + "support": { + "issues": "https://github.com/Nyholm/psr7-server/issues", + "source": "https://github.com/Nyholm/psr7-server/tree/1.1.0" + }, + "funding": [ + { + "url": "https://github.com/Zegnat", + "type": "github" + }, + { + "url": "https://github.com/nyholm", + "type": "github" + } + ], + "time": "2023-11-08T09:30:43+00:00" + }, + { + "name": "open-telemetry/api", + "version": "1.10.0", + "source": { + "type": "git", + "url": "https://github.com/opentelemetry-php/api.git", + "reference": "7c029c4a6fd457094a20569bf98f93d95e9a7559" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/7c029c4a6fd457094a20569bf98f93d95e9a7559", + "reference": "7c029c4a6fd457094a20569bf98f93d95e9a7559", + "shasum": "" + }, + "require": { + "open-telemetry/context": "^1.4", + "php": "^8.1", + "psr/log": "^1.1|^2.0|^3.0", + "symfony/polyfill-php82": "^1.26" + }, + "conflict": { + "open-telemetry/sdk": "<=1.11" + }, + "type": "library", + "extra": { + "spi": { + "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ + "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" + ] + }, + "branch-alias": { + "dev-main": "1.8.x-dev" + } + }, + "autoload": { + "files": [ + "Trace/functions.php" + ], + "psr-4": { + "OpenTelemetry\\API\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "opentelemetry-php contributors", + "homepage": "https://github.com/open-telemetry/opentelemetry-php/graphs/contributors" + } + ], + "description": "API for OpenTelemetry PHP.", + "keywords": [ + "Metrics", + "api", + "apm", + "logging", + "opentelemetry", + "otel", + "tracing" + ], + "support": { + "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", + "docs": "https://opentelemetry.io/docs/languages/php", + "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", + "source": "https://github.com/open-telemetry/opentelemetry-php" + }, + "time": "2026-07-06T12:28:04+00:00" + }, + { + "name": "open-telemetry/context", + "version": "1.5.0", + "source": { + "type": "git", + "url": "https://github.com/opentelemetry-php/context.git", + "reference": "3c414b246e0dabb7d6145404e6a5e4536ca18d07" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/3c414b246e0dabb7d6145404e6a5e4536ca18d07", + "reference": "3c414b246e0dabb7d6145404e6a5e4536ca18d07", + "shasum": "" + }, + "require": { + "php": "^8.1", + "symfony/polyfill-php82": "^1.26" + }, + "suggest": { + "ext-ffi": "To allow context switching in Fibers" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.0.x-dev" + } + }, + "autoload": { + "files": [ + "fiber/initialize_fiber_handler.php" + ], + "psr-4": { + "OpenTelemetry\\Context\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "opentelemetry-php contributors", + "homepage": "https://github.com/open-telemetry/opentelemetry-php/graphs/contributors" + } + ], + "description": "Context implementation for OpenTelemetry PHP.", + "keywords": [ + "Context", + "opentelemetry", + "otel" + ], + "support": { + "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", + "docs": "https://opentelemetry.io/docs/languages/php", + "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", + "source": "https://github.com/open-telemetry/opentelemetry-php" + }, + "time": "2025-10-19T06:44:33+00:00" + }, + { + "name": "open-telemetry/exporter-otlp", + "version": "1.4.0", + "source": { + "type": "git", + "url": "https://github.com/opentelemetry-php/exporter-otlp.git", + "reference": "283a0d66522f2adc6d8d7debfd7686be91c282be" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/283a0d66522f2adc6d8d7debfd7686be91c282be", + "reference": "283a0d66522f2adc6d8d7debfd7686be91c282be", + "shasum": "" + }, + "require": { + "open-telemetry/api": "^1.0", + "open-telemetry/gen-otlp-protobuf": "^1.1", + "open-telemetry/sdk": "^1.0", + "php": "^8.1", + "php-http/discovery": "^1.14" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.0.x-dev" + } + }, + "autoload": { + "files": [ + "_register.php" + ], + "psr-4": { + "OpenTelemetry\\Contrib\\Otlp\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "opentelemetry-php contributors", + "homepage": "https://github.com/open-telemetry/opentelemetry-php/graphs/contributors" + } + ], + "description": "OTLP exporter for OpenTelemetry.", + "keywords": [ + "Metrics", + "exporter", + "gRPC", + "http", + "opentelemetry", + "otel", + "otlp", + "tracing" + ], + "support": { + "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", + "docs": "https://opentelemetry.io/docs/languages/php", + "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", + "source": "https://github.com/open-telemetry/opentelemetry-php" + }, + "time": "2026-02-05T09:44:52+00:00" + }, + { + "name": "open-telemetry/gen-otlp-protobuf", + "version": "1.10.0", + "source": { + "type": "git", + "url": "https://github.com/opentelemetry-php/gen-otlp-protobuf.git", + "reference": "66f04d0e448ad333033bfc7baae1aa56330be088" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/66f04d0e448ad333033bfc7baae1aa56330be088", + "reference": "66f04d0e448ad333033bfc7baae1aa56330be088", + "shasum": "" + }, + "require": { + "google/protobuf": "^3.22 || ^4.0 || ^5.0", + "php": "^8.0" + }, + "suggest": { + "ext-protobuf": "For better performance, when dealing with the protobuf format" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Opentelemetry\\Proto\\": "Opentelemetry/Proto/", + "GPBMetadata\\Opentelemetry\\": "GPBMetadata/Opentelemetry/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "opentelemetry-php contributors", + "homepage": "https://github.com/open-telemetry/opentelemetry-php/graphs/contributors" + } + ], + "description": "PHP protobuf files for communication with OpenTelemetry OTLP collectors/servers.", + "keywords": [ + "Metrics", + "apm", + "gRPC", + "logging", + "opentelemetry", + "otel", + "otlp", + "protobuf", + "tracing" + ], + "support": { + "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", + "docs": "https://opentelemetry.io/docs/languages/php", + "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", + "source": "https://github.com/open-telemetry/opentelemetry-php" + }, + "time": "2026-06-17T12:06:32+00:00" + }, + { + "name": "open-telemetry/sdk", + "version": "1.15.0", + "source": { + "type": "git", + "url": "https://github.com/opentelemetry-php/sdk.git", + "reference": "77e1aa73850154abb86937d52a70883edc3b4547" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/77e1aa73850154abb86937d52a70883edc3b4547", + "reference": "77e1aa73850154abb86937d52a70883edc3b4547", + "shasum": "" + }, + "require": { + "ext-json": "*", + "nyholm/psr7-server": "^1.1", + "open-telemetry/api": "^1.8", + "open-telemetry/context": "^1.4", + "open-telemetry/sem-conv": "^1.38.0", + "php": "^8.1", + "php-http/discovery": "^1.14", + "psr/http-client": "^1.0", + "psr/http-client-implementation": "^1.0", + "psr/http-factory-implementation": "^1.0", + "psr/http-message": "^1.0.1|^2.0", + "psr/log": "^1.1|^2.0|^3.0", + "ramsey/uuid": "^3.0 || ^4.0", + "symfony/polyfill-mbstring": "^1.23", + "symfony/polyfill-php82": "^1.26", + "tbachert/spi": "^1.0.5" + }, + "suggest": { + "ext-gmp": "To support unlimited number of synchronous metric readers", + "ext-mbstring": "To increase performance of string operations", + "open-telemetry/sdk-configuration": "File-based OpenTelemetry SDK configuration" + }, + "type": "library", + "extra": { + "spi": { + "OpenTelemetry\\API\\Configuration\\ConfigEnv\\EnvComponentLoader": [ + "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderHttpConfig", + "OpenTelemetry\\API\\Instrumentation\\Configuration\\General\\ConfigEnv\\EnvComponentLoaderPeerConfig", + "OpenTelemetry\\SDK\\ConfigEnv\\Trace\\SpanSuppressionStrategySemConv", + "OpenTelemetry\\SDK\\ConfigEnv\\Trace\\SpanSuppressionStrategySpanKind", + "OpenTelemetry\\SDK\\ConfigEnv\\Distribution\\DistributionConfigurationSdk" + ], + "OpenTelemetry\\SDK\\Common\\Configuration\\Resolver\\ResolverInterface": [ + "OpenTelemetry\\SDK\\Common\\Configuration\\Resolver\\SdkConfigurationResolver" + ], + "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ + "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" + ] + }, + "branch-alias": { + "dev-main": "1.14.x-dev" + } + }, + "autoload": { + "files": [ + "Common/Util/functions.php", + "Logs/Exporter/_register.php", + "Metrics/MetricExporter/_register.php", + "Propagation/_register.php", + "Trace/SpanExporter/_register.php", + "Common/Dev/Compatibility/_load.php", + "_autoload.php" + ], + "psr-4": { + "OpenTelemetry\\SDK\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "opentelemetry-php contributors", + "homepage": "https://github.com/open-telemetry/opentelemetry-php/graphs/contributors" + } + ], + "description": "SDK for OpenTelemetry PHP.", + "keywords": [ + "Metrics", + "apm", + "logging", + "opentelemetry", + "otel", + "sdk", + "tracing" + ], + "support": { + "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", + "docs": "https://opentelemetry.io/docs/languages/php", + "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", + "source": "https://github.com/open-telemetry/opentelemetry-php" + }, + "time": "2026-07-14T13:09:54+00:00" + }, + { + "name": "open-telemetry/sem-conv", + "version": "1.38.0", + "source": { + "type": "git", + "url": "https://github.com/opentelemetry-php/sem-conv.git", + "reference": "e613bc640a407def4991b8a936a9b27edd9a3240" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/e613bc640a407def4991b8a936a9b27edd9a3240", + "reference": "e613bc640a407def4991b8a936a9b27edd9a3240", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "OpenTelemetry\\SemConv\\": "." + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "opentelemetry-php contributors", + "homepage": "https://github.com/open-telemetry/opentelemetry-php/graphs/contributors" + } + ], + "description": "Semantic conventions for OpenTelemetry PHP.", + "keywords": [ + "Metrics", + "apm", + "logging", + "opentelemetry", + "otel", + "semantic conventions", + "semconv", + "tracing" + ], + "support": { + "chat": "https://app.slack.com/client/T08PSQ7BQ/C01NFPCV44V", + "docs": "https://opentelemetry.io/docs/languages/php", + "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", + "source": "https://github.com/open-telemetry/opentelemetry-php" + }, + "time": "2026-01-21T04:14:03+00:00" + }, + { + "name": "php-http/discovery", + "version": "1.20.0", + "source": { + "type": "git", + "url": "https://github.com/php-http/discovery.git", + "reference": "82fe4c73ef3363caed49ff8dd1539ba06044910d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-http/discovery/zipball/82fe4c73ef3363caed49ff8dd1539ba06044910d", + "reference": "82fe4c73ef3363caed49ff8dd1539ba06044910d", + "shasum": "" + }, + "require": { + "composer-plugin-api": "^1.0|^2.0", + "php": "^7.1 || ^8.0" + }, + "conflict": { + "nyholm/psr7": "<1.0", + "zendframework/zend-diactoros": "*" + }, + "provide": { + "php-http/async-client-implementation": "*", + "php-http/client-implementation": "*", + "psr/http-client-implementation": "*", + "psr/http-factory-implementation": "*", + "psr/http-message-implementation": "*" + }, + "require-dev": { + "composer/composer": "^1.0.2|^2.0", + "graham-campbell/phpspec-skip-example-extension": "^5.0", + "php-http/httplug": "^1.0 || ^2.0", + "php-http/message-factory": "^1.0", + "phpspec/phpspec": "^5.1 || ^6.1 || ^7.3", + "sebastian/comparator": "^3.0.5 || ^4.0.8", + "symfony/phpunit-bridge": "^6.4.4 || ^7.0.1" + }, + "type": "composer-plugin", + "extra": { + "class": "Http\\Discovery\\Composer\\Plugin", + "plugin-optional": true + }, + "autoload": { + "psr-4": { + "Http\\Discovery\\": "src/" + }, + "exclude-from-classmap": [ + "src/Composer/Plugin.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "MΓ‘rk SΓ‘gi-KazΓ‘r", + "email": "mark.sagikazar@gmail.com" + } + ], + "description": "Finds and installs PSR-7, PSR-17, PSR-18 and HTTPlug implementations", + "homepage": "http://php-http.org", + "keywords": [ + "adapter", + "client", + "discovery", + "factory", + "http", + "message", + "psr17", + "psr7" + ], + "support": { + "issues": "https://github.com/php-http/discovery/issues", + "source": "https://github.com/php-http/discovery/tree/1.20.0" + }, + "time": "2024-10-02T11:20:13+00:00" + }, + { + "name": "psr/container", + "version": "2.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/container.git", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "shasum": "" + }, + "require": { + "php": ">=7.4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Container\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common Container Interface (PHP FIG PSR-11)", + "homepage": "https://github.com/php-fig/container", + "keywords": [ + "PSR-11", + "container", + "container-interface", + "container-interop", + "psr" + ], + "support": { + "issues": "https://github.com/php-fig/container/issues", + "source": "https://github.com/php-fig/container/tree/2.0.2" + }, + "time": "2021-11-05T16:47:00+00:00" + }, + { + "name": "psr/http-client", + "version": "1.0.3", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-client.git", + "reference": "bb5906edc1c324c9a05aa0873d40117941e5fa90" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-client/zipball/bb5906edc1c324c9a05aa0873d40117941e5fa90", + "reference": "bb5906edc1c324c9a05aa0873d40117941e5fa90", + "shasum": "" + }, + "require": { + "php": "^7.0 || ^8.0", + "psr/http-message": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Client\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP clients", + "homepage": "https://github.com/php-fig/http-client", + "keywords": [ + "http", + "http-client", + "psr", + "psr-18" + ], + "support": { + "source": "https://github.com/php-fig/http-client" + }, + "time": "2023-09-23T14:17:50+00:00" + }, + { + "name": "psr/http-factory", + "version": "1.1.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-factory.git", + "reference": "2b4765fddfe3b508ac62f829e852b1501d3f6e8a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-factory/zipball/2b4765fddfe3b508ac62f829e852b1501d3f6e8a", + "reference": "2b4765fddfe3b508ac62f829e852b1501d3f6e8a", + "shasum": "" + }, + "require": { + "php": ">=7.1", + "psr/http-message": "^1.0 || ^2.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "PSR-17: Common interfaces for PSR-7 HTTP message factories", + "keywords": [ + "factory", + "http", + "message", + "psr", + "psr-17", + "psr-7", + "request", + "response" + ], + "support": { + "source": "https://github.com/php-fig/http-factory" + }, + "time": "2024-04-15T12:06:14+00:00" + }, + { + "name": "psr/http-message", + "version": "2.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-message.git", + "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-message/zipball/402d35bcb92c70c026d1a6a9883f06b2ead23d71", + "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP messages", + "homepage": "https://github.com/php-fig/http-message", + "keywords": [ + "http", + "http-message", + "psr", + "psr-7", + "request", + "response" + ], + "support": { + "source": "https://github.com/php-fig/http-message/tree/2.0" + }, + "time": "2023-04-04T09:54:51+00:00" + }, + { + "name": "psr/log", + "version": "3.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/log.git", + "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/log/zipball/f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", + "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", + "shasum": "" + }, + "require": { + "php": ">=8.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Log\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for logging libraries", + "homepage": "https://github.com/php-fig/log", + "keywords": [ + "log", + "psr", + "psr-3" + ], + "support": { + "source": "https://github.com/php-fig/log/tree/3.0.2" + }, + "time": "2024-09-11T13:17:53+00:00" + }, + { + "name": "ramsey/collection", + "version": "2.1.1", + "source": { + "type": "git", + "url": "https://github.com/ramsey/collection.git", + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ramsey/collection/zipball/344572933ad0181accbf4ba763e85a0306a8c5e2", + "reference": "344572933ad0181accbf4ba763e85a0306a8c5e2", + "shasum": "" + }, + "require": { + "php": "^8.1" + }, + "require-dev": { + "captainhook/plugin-composer": "^5.3", + "ergebnis/composer-normalize": "^2.45", + "fakerphp/faker": "^1.24", + "hamcrest/hamcrest-php": "^2.0", + "jangregor/phpstan-prophecy": "^2.1", + "mockery/mockery": "^1.6", + "php-parallel-lint/php-console-highlighter": "^1.0", + "php-parallel-lint/php-parallel-lint": "^1.4", + "phpspec/prophecy-phpunit": "^2.3", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^10.5", + "ramsey/coding-standard": "^2.3", + "ramsey/conventional-commits": "^1.6", + "roave/security-advisories": "dev-latest" + }, + "type": "library", + "extra": { + "captainhook": { + "force-install": true + }, + "ramsey/conventional-commits": { + "configFile": "conventional-commits.json" + } + }, + "autoload": { + "psr-4": { + "Ramsey\\Collection\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ben Ramsey", + "email": "ben@benramsey.com", + "homepage": "https://benramsey.com" + } + ], + "description": "A PHP library for representing and manipulating collections.", + "keywords": [ + "array", + "collection", + "hash", + "map", + "queue", + "set" + ], + "support": { + "issues": "https://github.com/ramsey/collection/issues", + "source": "https://github.com/ramsey/collection/tree/2.1.1" + }, + "time": "2025-03-22T05:38:12+00:00" + }, + { + "name": "ramsey/uuid", + "version": "4.9.3", + "source": { + "type": "git", + "url": "https://github.com/ramsey/uuid.git", + "reference": "1df15849d00943a67d677dc9cfd80795f038c9f8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ramsey/uuid/zipball/1df15849d00943a67d677dc9cfd80795f038c9f8", + "reference": "1df15849d00943a67d677dc9cfd80795f038c9f8", + "shasum": "" + }, + "require": { + "brick/math": ">=0.8.16 <=0.18", + "php": "^8.0", + "ramsey/collection": "^1.2 || ^2.0" + }, + "replace": { + "rhumsaa/uuid": "self.version" + }, + "require-dev": { + "captainhook/captainhook": "^5.25", + "captainhook/plugin-composer": "^5.3", + "dealerdirect/phpcodesniffer-composer-installer": "^1.0", + "ergebnis/composer-normalize": "^2.47", + "mockery/mockery": "^1.6", + "paragonie/random-lib": "^2", + "php-mock/php-mock": "^2.6", + "php-mock/php-mock-mockery": "^1.5", + "php-parallel-lint/php-parallel-lint": "^1.4.0", + "phpbench/phpbench": "^1.2.14", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^9.6", + "slevomat/coding-standard": "^8.18", + "squizlabs/php_codesniffer": "^3.13" + }, + "suggest": { + "ext-bcmath": "Enables faster math with arbitrary-precision integers using BCMath.", + "ext-gmp": "Enables faster math with arbitrary-precision integers using GMP.", + "ext-uuid": "Enables the use of PeclUuidTimeGenerator and PeclUuidRandomGenerator.", + "paragonie/random-lib": "Provides RandomLib for use with the RandomLibAdapter", + "ramsey/uuid-doctrine": "Allows the use of Ramsey\\Uuid\\Uuid as Doctrine field type." + }, + "type": "library", + "extra": { + "captainhook": { + "force-install": true + } + }, + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "Ramsey\\Uuid\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A PHP library for generating and working with universally unique identifiers (UUIDs).", + "keywords": [ + "guid", + "identifier", + "uuid" + ], + "support": { + "issues": "https://github.com/ramsey/uuid/issues", + "source": "https://github.com/ramsey/uuid/tree/4.9.3" + }, + "time": "2026-06-18T03:57:49+00:00" + }, + { + "name": "symfony/deprecation-contracts", + "version": "v3.7.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/deprecation-contracts.git", + "reference": "f3202fa1b5097b0af062dc978b32ecf63404e31d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/f3202fa1b5097b0af062dc978b32ecf63404e31d", + "reference": "f3202fa1b5097b0af062dc978b32ecf63404e31d", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.7-dev" + } + }, + "autoload": { + "files": [ + "function.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "A generic function and convention to trigger deprecation notices", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.7.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-06-05T06:23:12+00:00" + }, + { + "name": "symfony/http-client", + "version": "v7.4.15", + "source": { + "type": "git", + "url": "https://github.com/symfony/http-client.git", + "reference": "817bb83ef06717f67ab72a3f03e3b63fbe138de1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/http-client/zipball/817bb83ef06717f67ab72a3f03e3b63fbe138de1", + "reference": "817bb83ef06717f67ab72a3f03e3b63fbe138de1", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "psr/log": "^1|^2|^3", + "symfony/deprecation-contracts": "^2.5|^3", + "symfony/http-client-contracts": "~3.4.4|^3.5.2", + "symfony/polyfill-php83": "^1.29", + "symfony/service-contracts": "^2.5|^3" + }, + "conflict": { + "amphp/amp": "<2.5", + "amphp/socket": "<1.1", + "php-http/discovery": "<1.15", + "symfony/http-foundation": "<6.4" + }, + "provide": { + "php-http/async-client-implementation": "*", + "php-http/client-implementation": "*", + "psr/http-client-implementation": "1.0", + "symfony/http-client-implementation": "3.0" + }, + "require-dev": { + "amphp/http-client": "^4.2.1|^5.0", + "amphp/http-tunnel": "^1.0|^2.0", + "guzzlehttp/promises": "^1.4|^2.0", + "nyholm/psr7": "^1.0", + "php-http/httplug": "^1.0|^2.0", + "psr/http-client": "^1.0", + "symfony/amphp-http-client-meta": "^1.0|^2.0", + "symfony/cache": "^6.4|^7.0|^8.0", + "symfony/dependency-injection": "^6.4|^7.0|^8.0", + "symfony/http-kernel": "^6.4|^7.0|^8.0", + "symfony/messenger": "^6.4|^7.0|^8.0", + "symfony/process": "^6.4|^7.0|^8.0", + "symfony/rate-limiter": "^6.4|^7.0|^8.0", + "symfony/stopwatch": "^6.4|^7.0|^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\HttpClient\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides powerful methods to fetch HTTP resources synchronously or asynchronously", + "homepage": "https://symfony.com", + "keywords": [ + "http" + ], + "support": { + "source": "https://github.com/symfony/http-client/tree/v7.4.15" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-07-29T07:12:33+00:00" + }, + { + "name": "symfony/http-client-contracts", + "version": "v3.7.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/http-client-contracts.git", + "reference": "41fc42d276aeff21192465331ebbab7d83a743c0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/41fc42d276aeff21192465331ebbab7d83a743c0", + "reference": "41fc42d276aeff21192465331ebbab7d83a743c0", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.7-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\HttpClient\\": "" + }, + "exclude-from-classmap": [ + "/Test/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to HTTP clients", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "support": { + "source": "https://github.com/symfony/http-client-contracts/tree/v3.7.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-06-05T06:23:12+00:00" + }, + { + "name": "symfony/polyfill-mbstring", + "version": "v1.38.2", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-mbstring.git", + "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "shasum": "" + }, + "require": { + "ext-iconv": "*", + "php": ">=7.2" + }, + "provide": { + "ext-mbstring": "*" + }, + "suggest": { + "ext-mbstring": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for the Mbstring extension", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "mbstring", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-05-27T06:59:30+00:00" + }, + { + "name": "symfony/polyfill-php82", + "version": "v1.38.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php82.git", + "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", + "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php82\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.2+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php82/tree/v1.38.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-05-26T12:45:58+00:00" + }, + { + "name": "symfony/polyfill-php83", + "version": "v1.41.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php83.git", + "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/5ea99087fb99c273a9b9236ed4c31e78b16103c6", + "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php83\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.3+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php83/tree/v1.41.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-07-01T12:47:55+00:00" + }, + { + "name": "symfony/service-contracts", + "version": "v3.7.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/service-contracts.git", + "reference": "c0a284bab1ed8aa0417e3d69250ab437739563a0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/c0a284bab1ed8aa0417e3d69250ab437739563a0", + "reference": "c0a284bab1ed8aa0417e3d69250ab437739563a0", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "psr/container": "^1.1|^2.0", + "symfony/deprecation-contracts": "^2.5|^3" + }, + "conflict": { + "ext-psr": "<1.1|>=2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.7-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\Service\\": "" + }, + "exclude-from-classmap": [ + "/Test/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to writing services", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "support": { + "source": "https://github.com/symfony/service-contracts/tree/v3.7.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-06-16T09:55:08+00:00" + }, + { + "name": "tbachert/spi", + "version": "v1.0.5", + "source": { + "type": "git", + "url": "https://github.com/Nevay/spi.git", + "reference": "e7078767866d0a9e0f91d3f9d42a832df5e39002" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Nevay/spi/zipball/e7078767866d0a9e0f91d3f9d42a832df5e39002", + "reference": "e7078767866d0a9e0f91d3f9d42a832df5e39002", + "shasum": "" + }, + "require": { + "composer-plugin-api": "^2.0", + "composer/semver": "^1.0 || ^2.0 || ^3.0", + "php": "^8.1" + }, + "require-dev": { + "composer/composer": "^2.0", + "infection/infection": "^0.27.9", + "phpunit/phpunit": "^10.5", + "psalm/phar": "^5.18" + }, + "type": "composer-plugin", + "extra": { + "class": "Nevay\\SPI\\Composer\\Plugin", + "branch-alias": { + "dev-main": "1.0.x-dev" + }, + "plugin-optional": true + }, + "autoload": { + "psr-4": { + "Nevay\\SPI\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Service provider loading facility", + "keywords": [ + "service provider" + ], + "support": { + "issues": "https://github.com/Nevay/spi/issues", + "source": "https://github.com/Nevay/spi/tree/v1.0.5" + }, + "time": "2025-06-29T15:42:06+00:00" + }, + { + "name": "utopia-php/cache", + "version": "3.4.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/cache.git", + "reference": "5c292d4df156f8b008f29fa6b033834b906bf328" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/5c292d4df156f8b008f29fa6b033834b906bf328", + "reference": "5c292d4df156f8b008f29fa6b033834b906bf328", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-memcached": "*", + "ext-redis": "*", + "php": ">=8.3", + "utopia-php/circuit-breaker": "0.3.*", + "utopia-php/pools": "1.*", + "utopia-php/telemetry": "*" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "^1.12", + "phpunit/phpunit": "^9.3", + "swoole/ide-helper": "^6.0", + "vimeo/psalm": "4.13.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Cache\\": "src/Cache" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple cache library to manage application cache storing, loading and purging", + "keywords": [ + "cache", + "framework", + "php", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/cache/issues", + "source": "https://github.com/utopia-php/cache/tree/3.4.0" + }, + "time": "2026-07-01T15:44:35+00:00" + }, + { + "name": "utopia-php/circuit-breaker", + "version": "0.3.1", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/circuit-breaker.git", + "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/circuit-breaker/zipball/db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", + "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", + "shasum": "" + }, + "require": { + "php": ">=8.2" + }, + "require-dev": { + "laravel/pint": "^1.29", + "phpstan/phpstan": "^2.1", + "phpunit/phpunit": "^10.0", + "utopia-php/telemetry": "^0.4" + }, + "suggest": { + "ext-opentelemetry": "Required by utopia-php/telemetry when using OpenTelemetry metrics.", + "ext-protobuf": "Required by utopia-php/telemetry when using OpenTelemetry metrics.", + "ext-redis": "Required when using Utopia\\CircuitBreaker\\Adapter\\Redis with the phpredis extension.", + "ext-swoole": "Required when using Utopia\\CircuitBreaker\\Adapter\\SwooleTable.", + "utopia-php/telemetry": "Required when passing telemetry adapters or running the local telemetry demo." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\CircuitBreaker\\": "src/CircuitBreaker" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "description": "Light & simple Circuit Breaker for PHP to prevent cascading failures in distributed systems.", + "keywords": [ + "circuit-breaker", + "fault-tolerance", + "framework", + "php", + "resilience", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/circuit-breaker/issues", + "source": "https://github.com/utopia-php/circuit-breaker/tree/0.3.1" + }, + "time": "2026-05-29T12:12:23+00:00" + }, + { + "name": "utopia-php/fetch", + "version": "1.1.2", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "64f2b3a789480f1deb102ce684dac4217d8e98d5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/64f2b3a789480f1deb102ce684dac4217d8e98d5", + "reference": "64f2b3a789480f1deb102ce684dac4217d8e98d5", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5", + "swoole/ide-helper": "^6.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/1.1.2" + }, + "time": "2026-04-29T11:19:19+00:00" + }, + { + "name": "utopia-php/pools", + "version": "1.1.1", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/pools.git", + "reference": "2c2ee1570a0b16b630671bfa72da5bc9ccf3ac21" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/2c2ee1570a0b16b630671bfa72da5bc9ccf3ac21", + "reference": "2c2ee1570a0b16b630671bfa72da5bc9ccf3ac21", + "shasum": "" + }, + "require": { + "php": ">=8.4", + "utopia-php/telemetry": "^0.4" + }, + "require-dev": { + "swoole/ide-helper": "6.*" + }, + "suggest": { + "ext-mongodb": "Needed to support MongoDB database pools", + "ext-pdo": "Needed to support MariaDB, MySQL or SQLite database pools", + "ext-redis": "Needed to support Redis cache pools", + "ext-swoole": "Needed to support Swoole based pool adapter" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Pools\\": "src/Pools" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "description": "A simple library to manage connection pools", + "keywords": [ + "framework", + "php", + "pools", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/pools/issues", + "source": "https://github.com/utopia-php/pools/tree/1.1.1" + }, + "time": "2026-07-27T15:27:26+00:00" + }, + { + "name": "utopia-php/telemetry", + "version": "0.4.5", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/telemetry.git", + "reference": "139943bffcd4f6dd8fb9ed247f946a1d151b006a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/139943bffcd4f6dd8fb9ed247f946a1d151b006a", + "reference": "139943bffcd4f6dd8fb9ed247f946a1d151b006a", + "shasum": "" + }, + "require": { + "ext-protobuf": "*", + "nyholm/psr7": "1.*", + "open-telemetry/exporter-otlp": "1.*", + "open-telemetry/sdk": "1.*", + "php": ">=8.0", + "symfony/http-client": "7.*" + }, + "require-dev": { + "swoole/ide-helper": "6.*" + }, + "suggest": { + "ext-sockets": "Required for the Swoole transport implementation", + "ext-swoole": "Required for the Swoole transport implementation" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Telemetry\\": "src/Telemetry" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A lite & fast telemetry library, with adapters for OpenTelemetry", + "keywords": [ + "framework", + "php", + "upf" + ], + "support": { + "issues": "https://github.com/utopia-php/telemetry/issues", + "source": "https://github.com/utopia-php/telemetry/tree/0.4.5" + }, + "time": "2026-07-08T11:07:25+00:00" + } + ], + "packages-dev": [ + { + "name": "doctrine/instantiator", + "version": "2.1.0", + "source": { + "type": "git", + "url": "https://github.com/doctrine/instantiator.git", + "reference": "23da848e1a2308728fe5fdddabf4be17ff9720c7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/23da848e1a2308728fe5fdddabf4be17ff9720c7", + "reference": "23da848e1a2308728fe5fdddabf4be17ff9720c7", + "shasum": "" + }, + "require": { + "php": "^8.4" + }, + "require-dev": { + "doctrine/coding-standard": "^14", + "ext-pdo": "*", + "ext-phar": "*", + "phpbench/phpbench": "^1.2", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^10.5.58" + }, + "type": "library", + "autoload": { + "psr-4": { + "Doctrine\\Instantiator\\": "src/Doctrine/Instantiator/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Marco Pivetta", + "email": "ocramius@gmail.com", + "homepage": "https://ocramius.github.io/" + } + ], + "description": "A small, lightweight utility to instantiate objects in PHP without invoking their constructors", + "homepage": "https://www.doctrine-project.org/projects/instantiator.html", + "keywords": [ + "constructor", + "instantiate" + ], + "support": { + "issues": "https://github.com/doctrine/instantiator/issues", + "source": "https://github.com/doctrine/instantiator/tree/2.1.0" + }, + "funding": [ + { + "url": "https://www.doctrine-project.org/sponsorship.html", + "type": "custom" + }, + { + "url": "https://www.patreon.com/phpdoctrine", + "type": "patreon" + }, + { + "url": "https://tidelift.com/funding/github/packagist/doctrine%2Finstantiator", + "type": "tidelift" + } + ], + "time": "2026-01-05T06:47:08+00:00" + }, + { + "name": "laravel/pint", + "version": "v1.30.0", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "72a0540d1aa10b6c146bda2a22f3ae003123c0ea" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/72a0540d1aa10b6c146bda2a22f3ae003123c0ea", + "reference": "72a0540d1aa10b6c146bda2a22f3ae003123c0ea", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.2.0" + }, + "require-dev": { + "composer/semver": "^3.4.4", + "friendsofphp/php-cs-fixer": "^3.95.17", + "illuminate/view": "^12.64.0", + "larastan/larastan": "^3.10.0", + "laravel-zero/framework": "^12.1.0", + "laravel/agent-detector": "^2.0.2", + "laravel/prompts": "^0.3.21", + "mockery/mockery": "^1.6.12", + "nunomaduro/termwind": "^2.4.0", + "pestphp/pest": "^3.8.7" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "dev", + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2026-07-28T20:48:56+00:00" + }, + { + "name": "myclabs/deep-copy", + "version": "1.13.4", + "source": { + "type": "git", + "url": "https://github.com/myclabs/DeepCopy.git", + "reference": "07d290f0c47959fd5eed98c95ee5602db07e0b6a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/07d290f0c47959fd5eed98c95ee5602db07e0b6a", + "reference": "07d290f0c47959fd5eed98c95ee5602db07e0b6a", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0" + }, + "conflict": { + "doctrine/collections": "<1.6.8", + "doctrine/common": "<2.13.3 || >=3 <3.2.2" + }, + "require-dev": { + "doctrine/collections": "^1.6.8", + "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", + "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" + }, + "type": "library", + "autoload": { + "files": [ + "src/DeepCopy/deep_copy.php" + ], + "psr-4": { + "DeepCopy\\": "src/DeepCopy/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Create deep copies (clones) of your objects", + "keywords": [ + "clone", + "copy", + "duplicate", + "object", + "object graph" + ], + "support": { + "issues": "https://github.com/myclabs/DeepCopy/issues", + "source": "https://github.com/myclabs/DeepCopy/tree/1.13.4" + }, + "funding": [ + { + "url": "https://tidelift.com/funding/github/packagist/myclabs/deep-copy", + "type": "tidelift" + } + ], + "time": "2025-08-01T08:46:24+00:00" + }, + { + "name": "nikic/php-parser", + "version": "v5.8.0", + "source": { + "type": "git", + "url": "https://github.com/nikic/PHP-Parser.git", + "reference": "044a6a392ff8ad0d61f14370a5fbbd0a0107152f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/044a6a392ff8ad0d61f14370a5fbbd0a0107152f", + "reference": "044a6a392ff8ad0d61f14370a5fbbd0a0107152f", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-tokenizer": "*", + "php": ">=7.4" + }, + "require-dev": { + "ircmaxell/php-yacc": "^0.0.7", + "phpunit/phpunit": "^9.0" + }, + "bin": [ + "bin/php-parse" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.x-dev" + } + }, + "autoload": { + "psr-4": { + "PhpParser\\": "lib/PhpParser" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Nikita Popov" + } + ], + "description": "A PHP parser written in PHP", + "keywords": [ + "parser", + "php" + ], + "support": { + "issues": "https://github.com/nikic/PHP-Parser/issues", + "source": "https://github.com/nikic/PHP-Parser/tree/v5.8.0" + }, + "time": "2026-07-04T14:30:18+00:00" + }, + { + "name": "phar-io/manifest", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/phar-io/manifest.git", + "reference": "54750ef60c58e43759730615a392c31c80e23176" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/54750ef60c58e43759730615a392c31c80e23176", + "reference": "54750ef60c58e43759730615a392c31c80e23176", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-libxml": "*", + "ext-phar": "*", + "ext-xmlwriter": "*", + "phar-io/version": "^3.0.1", + "php": "^7.2 || ^8.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + }, + { + "name": "Sebastian Heuer", + "email": "sebastian@phpeople.de", + "role": "Developer" + }, + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "Developer" + } + ], + "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", + "support": { + "issues": "https://github.com/phar-io/manifest/issues", + "source": "https://github.com/phar-io/manifest/tree/2.0.4" + }, + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2024-03-03T12:33:53+00:00" + }, + { + "name": "phar-io/version", + "version": "3.2.1", + "source": { + "type": "git", + "url": "https://github.com/phar-io/version.git", + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phar-io/version/zipball/4f7fd7836c6f332bb2933569e566a0d6c4cbed74", + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + }, + { + "name": "Sebastian Heuer", + "email": "sebastian@phpeople.de", + "role": "Developer" + }, + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "Developer" + } + ], + "description": "Library for handling version information and constraints", + "support": { + "issues": "https://github.com/phar-io/version/issues", + "source": "https://github.com/phar-io/version/tree/3.2.1" + }, + "time": "2022-02-21T01:04:05+00:00" + }, + { + "name": "phpstan/phpstan", + "version": "2.2.6", + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/a6e9b5a9420f6109c091e87d82683bd1a80b87ed", + "reference": "a6e9b5a9420f6109c091e87d82683bd1a80b87ed", + "shasum": "" + }, + "require": { + "php": "^7.4|^8.0" + }, + "conflict": { + "phpstan/phpstan-shim": "*" + }, + "bin": [ + "phpstan", + "phpstan.phar" + ], + "type": "library", + "autoload": { + "files": [ + "bootstrap.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "OndΕ™ej Mirtes" + }, + { + "name": "Markus Staab" + }, + { + "name": "Vincent Langlet" + } + ], + "description": "PHPStan - PHP Static Analysis Tool", + "keywords": [ + "dev", + "static analysis" + ], + "support": { + "docs": "https://phpstan.org/user-guide/getting-started", + "forum": "https://github.com/phpstan/phpstan/discussions", + "issues": "https://github.com/phpstan/phpstan/issues", + "security": "https://github.com/phpstan/phpstan/security/policy", + "source": "https://github.com/phpstan/phpstan-src" + }, + "funding": [ + { + "url": "https://github.com/ondrejmirtes", + "type": "github" + }, + { + "url": "https://github.com/phpstan", + "type": "github" + } + ], + "time": "2026-07-26T21:22:49+00:00" + }, + { + "name": "phpunit/php-code-coverage", + "version": "9.2.32", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-code-coverage.git", + "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/85402a822d1ecf1db1096959413d35e1c37cf1a5", + "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-libxml": "*", + "ext-xmlwriter": "*", + "nikic/php-parser": "^4.19.1 || ^5.1.0", + "php": ">=7.3", + "phpunit/php-file-iterator": "^3.0.6", + "phpunit/php-text-template": "^2.0.4", + "sebastian/code-unit-reverse-lookup": "^2.0.3", + "sebastian/complexity": "^2.0.3", + "sebastian/environment": "^5.1.5", + "sebastian/lines-of-code": "^1.0.4", + "sebastian/version": "^3.0.2", + "theseer/tokenizer": "^1.2.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.6" + }, + "suggest": { + "ext-pcov": "PHP extension that provides line coverage", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "9.2.x-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", + "homepage": "https://github.com/sebastianbergmann/php-code-coverage", + "keywords": [ + "coverage", + "testing", + "xunit" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", + "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.32" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-08-22T04:23:01+00:00" + }, + { + "name": "phpunit/php-file-iterator", + "version": "3.0.6", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-file-iterator.git", + "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", + "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "FilterIterator implementation that filters files based on a list of suffixes.", + "homepage": "https://github.com/sebastianbergmann/php-file-iterator/", + "keywords": [ + "filesystem", + "iterator" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-file-iterator/issues", + "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/3.0.6" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2021-12-02T12:48:52+00:00" + }, + { + "name": "phpunit/php-invoker", + "version": "3.1.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-invoker.git", + "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/5a10147d0aaf65b58940a0b72f71c9ac0423cc67", + "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "ext-pcntl": "*", + "phpunit/phpunit": "^9.3" + }, + "suggest": { + "ext-pcntl": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.1-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Invoke callables with a timeout", + "homepage": "https://github.com/sebastianbergmann/php-invoker/", + "keywords": [ + "process" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-invoker/issues", + "source": "https://github.com/sebastianbergmann/php-invoker/tree/3.1.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-09-28T05:58:55+00:00" + }, + { + "name": "phpunit/php-text-template", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-text-template.git", + "reference": "5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28", + "reference": "5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Simple template engine.", + "homepage": "https://github.com/sebastianbergmann/php-text-template/", + "keywords": [ + "template" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-text-template/issues", + "source": "https://github.com/sebastianbergmann/php-text-template/tree/2.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T05:33:50+00:00" + }, + { + "name": "phpunit/php-timer", + "version": "5.0.3", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-timer.git", + "reference": "5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2", + "reference": "5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Utility class for timing", + "homepage": "https://github.com/sebastianbergmann/php-timer/", + "keywords": [ + "timer" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-timer/issues", + "source": "https://github.com/sebastianbergmann/php-timer/tree/5.0.3" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:16:10+00:00" + }, + { + "name": "phpunit/phpunit", + "version": "9.6.35", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/phpunit.git", + "reference": "0edba2f3a0c48df3553cb9b640810b30df60302b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0edba2f3a0c48df3553cb9b640810b30df60302b", + "reference": "0edba2f3a0c48df3553cb9b640810b30df60302b", + "shasum": "" + }, + "require": { + "doctrine/instantiator": "^1.5.0 || ^2", + "ext-dom": "*", + "ext-filter": "*", + "ext-json": "*", + "ext-libxml": "*", + "ext-mbstring": "*", + "ext-xmlwriter": "*", + "myclabs/deep-copy": "^1.13.4", + "phar-io/manifest": "^2.0.4", + "phar-io/version": "^3.2.1", + "php": ">=7.3", + "phpunit/php-code-coverage": "^9.2.32", + "phpunit/php-file-iterator": "^3.0.6", + "phpunit/php-invoker": "^3.1.1", + "phpunit/php-text-template": "^2.0.4", + "phpunit/php-timer": "^5.0.3", + "sebastian/cli-parser": "^1.0.2", + "sebastian/code-unit": "^1.0.8", + "sebastian/comparator": "^4.0.10", + "sebastian/diff": "^4.0.6", + "sebastian/environment": "^5.1.5", + "sebastian/exporter": "^4.0.8", + "sebastian/global-state": "^5.0.8", + "sebastian/object-enumerator": "^4.0.4", + "sebastian/resource-operations": "^3.0.4", + "sebastian/type": "^3.2.1", + "sebastian/version": "^3.0.2" + }, + "suggest": { + "ext-soap": "To be able to generate mocks based on WSDL files", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" + }, + "bin": [ + "phpunit" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.6-dev" + } + }, + "autoload": { + "files": [ + "src/Framework/Assert/Functions.php" + ], + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "The PHP Unit Testing framework.", + "homepage": "https://phpunit.de/", + "keywords": [ + "phpunit", + "testing", + "xunit" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/phpunit/issues", + "security": "https://github.com/sebastianbergmann/phpunit/security/policy", + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.35" + }, + "funding": [ + { + "url": "https://phpunit.de/sponsoring.html", + "type": "other" + } + ], + "time": "2026-07-06T14:48:07+00:00" + }, + { + "name": "sebastian/cli-parser", + "version": "1.0.2", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/cli-parser.git", + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/2b56bea83a09de3ac06bb18b92f068e60cc6f50b", + "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for parsing CLI options", + "homepage": "https://github.com/sebastianbergmann/cli-parser", + "support": { + "issues": "https://github.com/sebastianbergmann/cli-parser/issues", + "source": "https://github.com/sebastianbergmann/cli-parser/tree/1.0.2" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-02T06:27:43+00:00" + }, + { + "name": "sebastian/code-unit", + "version": "1.0.8", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/code-unit.git", + "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/1fc9f64c0927627ef78ba436c9b17d967e68e120", + "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Collection of value objects that represent the PHP code units", + "homepage": "https://github.com/sebastianbergmann/code-unit", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit/issues", + "source": "https://github.com/sebastianbergmann/code-unit/tree/1.0.8" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:08:54+00:00" + }, + { + "name": "sebastian/code-unit-reverse-lookup", + "version": "2.0.3", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", + "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", + "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Looks up which function or method a line of code belongs to", + "homepage": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/issues", + "source": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/tree/2.0.3" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-09-28T05:30:19+00:00" + }, + { + "name": "sebastian/comparator", + "version": "4.0.10", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/comparator.git", + "reference": "e4df00b9b3571187db2831ae9aada2c6efbd715d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/e4df00b9b3571187db2831ae9aada2c6efbd715d", + "reference": "e4df00b9b3571187db2831ae9aada2c6efbd715d", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/diff": "^4.0", + "sebastian/exporter": "^4.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Volker Dusch", + "email": "github@wallbash.com" + }, + { + "name": "Bernhard Schussek", + "email": "bschussek@2bepublished.at" + } + ], + "description": "Provides the functionality to compare PHP values for equality", + "homepage": "https://github.com/sebastianbergmann/comparator", + "keywords": [ + "comparator", + "compare", + "equality" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/comparator/issues", + "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.10" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/comparator", + "type": "tidelift" + } + ], + "time": "2026-01-24T09:22:56+00:00" + }, + { + "name": "sebastian/complexity", + "version": "2.0.3", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/complexity.git", + "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/25f207c40d62b8b7aa32f5ab026c53561964053a", + "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a", + "shasum": "" + }, + "require": { + "nikic/php-parser": "^4.18 || ^5.0", + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for calculating the complexity of PHP code units", + "homepage": "https://github.com/sebastianbergmann/complexity", + "support": { + "issues": "https://github.com/sebastianbergmann/complexity/issues", + "source": "https://github.com/sebastianbergmann/complexity/tree/2.0.3" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-12-22T06:19:30+00:00" + }, + { + "name": "sebastian/diff", + "version": "4.0.6", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/diff.git", + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/ba01945089c3a293b01ba9badc29ad55b106b0bc", + "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "symfony/process": "^4.2 || ^5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Kore Nordmann", + "email": "mail@kore-nordmann.de" + } + ], + "description": "Diff implementation", + "homepage": "https://github.com/sebastianbergmann/diff", + "keywords": [ + "diff", + "udiff", + "unidiff", + "unified diff" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/diff/issues", + "source": "https://github.com/sebastianbergmann/diff/tree/4.0.6" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-02T06:30:58+00:00" + }, + { + "name": "sebastian/environment", + "version": "5.1.5", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/environment.git", + "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", + "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "suggest": { + "ext-posix": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.1-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Provides functionality to handle HHVM/PHP environments", + "homepage": "http://www.github.com/sebastianbergmann/environment", + "keywords": [ + "Xdebug", + "environment", + "hhvm" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/environment/issues", + "source": "https://github.com/sebastianbergmann/environment/tree/5.1.5" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:03:51+00:00" + }, + { + "name": "sebastian/exporter", + "version": "4.0.8", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/exporter.git", + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/14c6ba52f95a36c3d27c835d65efc7123c446e8c", + "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/recursion-context": "^4.0" + }, + "require-dev": { + "ext-mbstring": "*", + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Volker Dusch", + "email": "github@wallbash.com" + }, + { + "name": "Adam Harvey", + "email": "aharvey@php.net" + }, + { + "name": "Bernhard Schussek", + "email": "bschussek@gmail.com" + } + ], + "description": "Provides the functionality to export PHP variables for visualization", + "homepage": "https://www.github.com/sebastianbergmann/exporter", + "keywords": [ + "export", + "exporter" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/exporter/issues", + "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.8" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", + "type": "tidelift" + } + ], + "time": "2025-09-24T06:03:27+00:00" + }, + { + "name": "sebastian/global-state", + "version": "5.0.8", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/global-state.git", + "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/b6781316bdcd28260904e7cc18ec983d0d2ef4f6", + "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/object-reflector": "^2.0", + "sebastian/recursion-context": "^4.0" + }, + "require-dev": { + "ext-dom": "*", + "phpunit/phpunit": "^9.3" + }, + "suggest": { + "ext-uopz": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Snapshotting of global state", + "homepage": "http://www.github.com/sebastianbergmann/global-state", + "keywords": [ + "global state" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/global-state/issues", + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.8" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/global-state", + "type": "tidelift" + } + ], + "time": "2025-08-10T07:10:35+00:00" + }, + { + "name": "sebastian/lines-of-code", + "version": "1.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/lines-of-code.git", + "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/e1e4a170560925c26d424b6a03aed157e7dcc5c5", + "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5", + "shasum": "" + }, + "require": { + "nikic/php-parser": "^4.18 || ^5.0", + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for counting the lines of code in PHP source code", + "homepage": "https://github.com/sebastianbergmann/lines-of-code", + "support": { + "issues": "https://github.com/sebastianbergmann/lines-of-code/issues", + "source": "https://github.com/sebastianbergmann/lines-of-code/tree/1.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-12-22T06:20:34+00:00" + }, + { + "name": "sebastian/object-enumerator", + "version": "4.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/object-enumerator.git", + "reference": "5c9eeac41b290a3712d88851518825ad78f45c71" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/5c9eeac41b290a3712d88851518825ad78f45c71", + "reference": "5c9eeac41b290a3712d88851518825ad78f45c71", + "shasum": "" + }, + "require": { + "php": ">=7.3", + "sebastian/object-reflector": "^2.0", + "sebastian/recursion-context": "^4.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Traverses array structures and object graphs to enumerate all referenced objects", + "homepage": "https://github.com/sebastianbergmann/object-enumerator/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-enumerator/issues", + "source": "https://github.com/sebastianbergmann/object-enumerator/tree/4.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:12:34+00:00" + }, + { + "name": "sebastian/object-reflector", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/object-reflector.git", + "reference": "b4f479ebdbf63ac605d183ece17d8d7fe49c15c7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/b4f479ebdbf63ac605d183ece17d8d7fe49c15c7", + "reference": "b4f479ebdbf63ac605d183ece17d8d7fe49c15c7", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Allows reflection of object attributes, including inherited and non-public ones", + "homepage": "https://github.com/sebastianbergmann/object-reflector/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-reflector/issues", + "source": "https://github.com/sebastianbergmann/object-reflector/tree/2.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-10-26T13:14:26+00:00" + }, + { + "name": "sebastian/recursion-context", + "version": "4.0.6", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/recursion-context.git", + "reference": "539c6691e0623af6dc6f9c20384c120f963465a0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/539c6691e0623af6dc6f9c20384c120f963465a0", + "reference": "539c6691e0623af6dc6f9c20384c120f963465a0", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Adam Harvey", + "email": "aharvey@php.net" + } + ], + "description": "Provides functionality to recursively process PHP variables", + "homepage": "https://github.com/sebastianbergmann/recursion-context", + "support": { + "issues": "https://github.com/sebastianbergmann/recursion-context/issues", + "source": "https://github.com/sebastianbergmann/recursion-context/tree/4.0.6" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/recursion-context", + "type": "tidelift" + } + ], + "time": "2025-08-10T06:57:39+00:00" + }, + { + "name": "sebastian/resource-operations", + "version": "3.0.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/resource-operations.git", + "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/05d5692a7993ecccd56a03e40cd7e5b09b1d404e", + "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Provides a list of PHP built-in functions that operate on resources", + "homepage": "https://www.github.com/sebastianbergmann/resource-operations", + "support": { + "source": "https://github.com/sebastianbergmann/resource-operations/tree/3.0.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-14T16:00:52+00:00" + }, + { + "name": "sebastian/type", + "version": "3.2.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/type.git", + "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", + "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.2-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Collection of value objects that represent the types of the PHP type system", + "homepage": "https://github.com/sebastianbergmann/type", + "support": { + "issues": "https://github.com/sebastianbergmann/type/issues", + "source": "https://github.com/sebastianbergmann/type/tree/3.2.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:13:03+00:00" + }, + { + "name": "sebastian/version", + "version": "3.0.2", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/version.git", + "reference": "c6c1022351a901512170118436c764e473f6de8c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/version/zipball/c6c1022351a901512170118436c764e473f6de8c", + "reference": "c6c1022351a901512170118436c764e473f6de8c", + "shasum": "" + }, + "require": { + "php": ">=7.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library that helps with managing the version number of Git-hosted PHP projects", + "homepage": "https://github.com/sebastianbergmann/version", + "support": { + "issues": "https://github.com/sebastianbergmann/version/issues", + "source": "https://github.com/sebastianbergmann/version/tree/3.0.2" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2020-09-28T06:39:44+00:00" + }, + { + "name": "swoole/ide-helper", + "version": "6.2.2", + "source": { + "type": "git", + "url": "https://github.com/swoole/ide-helper.git", + "reference": "ad91a281e135f498426fb2e5297376581a4e2dca" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/swoole/ide-helper/zipball/ad91a281e135f498426fb2e5297376581a4e2dca", + "reference": "ad91a281e135f498426fb2e5297376581a4e2dca", + "shasum": "" + }, + "type": "library", + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Team Swoole", + "email": "team@swoole.com" + } + ], + "description": "IDE help files for Swoole.", + "support": { + "issues": "https://github.com/swoole/ide-helper/issues", + "source": "https://github.com/swoole/ide-helper/tree/6.2.2" + }, + "time": "2026-07-25T08:15:29+00:00" + }, + { + "name": "theseer/tokenizer", + "version": "1.3.1", + "source": { + "type": "git", + "url": "https://github.com/theseer/tokenizer.git", + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b7489ce515e168639d17feec34b8847c326b0b3c", + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-tokenizer": "*", + "ext-xmlwriter": "*", + "php": "^7.2 || ^8.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + } + ], + "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", + "support": { + "issues": "https://github.com/theseer/tokenizer/issues", + "source": "https://github.com/theseer/tokenizer/tree/1.3.1" + }, + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2025-11-17T20:03:58+00:00" + } + ], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": {}, + "prefer-stable": false, + "prefer-lowest": false, + "platform": { + "php": ">=8.3", + "ext-json": "*", + "ext-redis": "*" + }, + "platform-dev": {}, + "plugin-api-version": "2.9.0" +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b4c0d1e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +name: feed + +services: + tests: + container_name: feed-tests + image: feed-tests:${PHP_VERSION:-8.4} + build: + context: . + dockerfile: Dockerfile.php-${PHP_VERSION:-8.4} + environment: + - REDIS_HOST=redis + - REDIS_PORT=6379 + networks: + - feed + volumes: + - ./phpunit.xml:/code/phpunit.xml + - ./phpstan.neon:/code/phpstan.neon + - ./src:/code/src + - ./tests:/code/tests + depends_on: + redis: + condition: service_healthy + + redis: + image: redis:7.2-alpine + container_name: feed-redis + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 20 + networks: + - feed + +networks: + feed: diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 0000000..8c8ad93 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,193 @@ +# Migrating an existing feed onto this library + +This library was extracted from two implementations of the same idea that had +grown up on either side of one feed: a producer in `appwrite-labs/cloud` and a +consumer in `appwrite-labs/edge`. Between them they had two event models, two +pull loops, two cursor stores and two copies of the HTTP contract that joined +them β€” the duplication that matters most, because the two halves drifting apart +is a wire incompatibility rather than a local bug. + +This is what maps onto what. + +## Producer (cloud) + +| Was | Now | +| --- | --- | +| `Feed::append()` / `read()` / `poll()` | `Feed`, on `Adapter\Pool` | +| `Feed::after()` β€” stream id arithmetic | `Id::after()` | +| `Feed::getCursor()` / `saveCursor()` | `Cursor\Pool` | +| `Consumer::consume()` | `Consumer::consume()` | +| Query params and response shape in `Http\Feeds\Get` | `Protocol` | +| `Cache-Control` rules in `Http\Feeds\Get` | `Protocol::cacheControl()` | +| `EdgeFeed` | Stays β€” subclass `Feed` | +| `FastlyConsumer` | Stays β€” becomes a handler | +| `Response\Model\FeedEvent` | Stays β€” it is an SDK response model | + +`EdgeFeed` stays in cloud because the tag names in its payloads are a contract +with the edge, not a general-purpose feed concern. It keeps its typed method per +invalidatable resource for the same reason as before β€” a caller-supplied tag +array with a typo in it produces an event that silently invalidates nothing β€” +and now only has to define the vocabulary: + +```php +class EdgeFeed extends Feed +{ + public const string NAME = 'edge'; + + public const string EVENT_INVALIDATE_RULE = 'io.appwrite.edge.invalidate-rule'; + + public function __construct(?Pool $pool, string $source, int $maxSize = 100_000) + { + parent::__construct( + $pool === null ? new None(self::NAME) : new Adapter\Pool($pool, self::NAME, $maxSize), + $source, + ); + } + + public function invalidateRule(string $domain, bool $isAppwriteNetwork = false): string + { + if ($domain === '') { + throw new \InvalidArgumentException('Rule invalidation requires a domain'); + } + + return $this->append( + self::EVENT_INVALIDATE_RULE, + ['tags' => ['domain' => $domain], 'isAppwriteNetwork' => $isAppwriteNetwork], + $domain, + ); + } + + // ...one method per invalidatable resource, as before +} +``` + +The nullable pool becomes `Adapter\None`, which throws on use with the same +intent as the old `pool()` guard: a feed with no backend must fail loudly rather +than drop events. + +`Http\Feeds\Get` keeps its route, auth and SDK metadata, and hands the wire +details to `Protocol`: + +```php +$events = $feed->poll($lastEventId === '' ? null : $lastEventId, $limit, $timeout); + +$response->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)); +$response->dynamic(new Document(Protocol::encode($events)), Response::MODEL_FEED_EVENT_LIST); +``` + +`FastlyConsumer` stops extending `Consumer` and becomes a handler passed to one. +Its Fastly-specific parts β€” the purge URL, the surrogate key format, the +credential β€” stay in cloud; the pull loop does not: + +```php +$consumer = new Consumer($edgeFeed, FastlyConsumer::NAME, new Cursor\Pool($pool, EdgeFeed::NAME)); + +$purged = 0; +$seen = []; + +$consumer->consume(function (Event $event) use (&$purged, &$seen): void { + $url = $this->purgeUrl($event); + if ($url === null || isset($seen[$url])) { + return; + } + + $this->send($url); + $seen[$url] = true; + $purged++; +}); +``` + +`purgeUrl()` reads a typed `Event` instead of an array: + +```php +if ($event->type !== EdgeFeed::EVENT_INVALIDATE_RULE) { + return null; +} + +$tags = $event->getData('tags', []); +$domain = \is_array($tags) ? ($tags['domain'] ?? '') : ''; +``` + +## Consumer (edge) + +| Was | Now | +| --- | --- | +| `Feed\Consumer` | `Consumer`, plus a handler | +| `Feed\Cursor` | `Cursor\Cache` | +| `Feed\Event` | `Event` | +| `Feed\Event::FEED` and the type constants | Stay β€” they name cloud's feed and its events | +| `Manager::fetchFeed()` | `Adapter\Http` | +| `Consumer::TIMEOUT_MARGIN` | `Protocol::TIMEOUT_MARGIN` | +| `Feed\Poller` | Stays β€” Swoole interval scheduling | +| `Router\Invalidator` | Stays β€” it purges edge caches | + +The whole of `Feed\Consumer`, `Feed\Cursor` and `Feed\Event` is replaced by +construction: + +```php +$client = (new Client())->addHeader('x-appwrite-jwt', $token); + +$consumer = new Consumer( + feed: new Feed(new Http($client, $endpoint . '/manager/feeds', 'edge')), + name: $region, + cursor: new Cursor\Cache($cache, 'edge'), + batch: 500, + timeout: $timeout, +); + +$consumer->onWarning(fn (\Throwable $error, string $context) => + Console::warning("[feed] Could not {$context} the {$region} cursor: {$error->getMessage()}")); +``` + +The edge's old `Event::from()` normalized tags on the way in so that +`tags === []` was a reliable answer to "is there anything to do?". That belongs +with the invalidator that defines what a usable tag is, so it moves into the +handler: + +```php +$consumer->consume(function (Event $event) use ($invalidator): void { + $tags = $event->getData('tags', []); + $tags = \is_array($tags) ? Invalidator::normalize($tags) : []; + + if ($tags === []) { + Span::add('feed.consume.skipped_type', $event->type); + return; + } + + $invalidator->invalidate($tags); +}); +``` + +`Poller` keeps its Swoole `WaitGroup` fan-out and its once-per-region 404 +reporting. Only the exception it catches changes, from `PlatformException` to +`Utopia\Feed\Exception\Transport` β€” the status is still on `getCode()`, so the +`!== 404` check is unchanged. + +## Behaviour that is deliberately identical + +These were load-bearing in the original implementations and are preserved: + +- **A consumer with no position starts at the oldest retained event**, not at + the tip. This is what makes the staged rollout in the README safe. +- **A run commits the events handled before a failure**, then re-raises it. The + failed event is retried on the next run, and everything behind it waits. +- **A cursor store that is down is a warning, not a failure.** The position is + mirrored in memory, so the consumer keeps working and only a restart replays. +- **The `feed::cursor:` key format**, so consumers keep their + positions across the migration instead of replaying the retained feed. +- **`-` event ids**, so positions already handed out stay valid. +- **The margin a consumer allows its HTTP client over the long-poll timeout**, + without which every quiet tick surfaces as a transport failure. + +## Behaviour that changed + +- **`source` is stamped at append rather than at read.** Previously every event + read from a feed was labelled with the reading region's source, whoever + actually produced it. Storing it at append costs nothing and keeps it correct + for a feed that is replicated or read back somewhere else. +- **A batch containing an event with no id no longer throws away the valid + events in front of it.** `Protocol::decode()` returns the usable prefix, and + throws once the broken event reaches the head of a batch β€” where the feed + stops visibly rather than quietly losing events. +- **`Cache-Control` is computed from the batch**, and `public` is opt-in rather + than a decision baked into one endpoint. diff --git a/phpstan.neon b/phpstan.neon new file mode 100644 index 0000000..7718dd8 --- /dev/null +++ b/phpstan.neon @@ -0,0 +1,19 @@ +parameters: + level: max + paths: + - src + - tests + treatPhpDocTypesAsCertain: false + + # Stream commands landed in ext-redis 5. Analysis run without the + # extension loaded falls back to bundled stubs that predate them, so these + # only resolve where ext-redis is actually installed β€” which is the case in + # CI, and not the case on a machine that only runs the unit suite. + reportUnmatchedIgnoredErrors: false + ignoreErrors: + - + message: '#Call to an undefined method Redis\|RedisCluster::(xAdd|xRange)\(\)#' + path: src/Feed/Adapter/Redis.php + - + message: '#Call to an undefined method Redis::(xAdd|xRange|xLen)\(\)#' + path: tests/Feed/E2E/RedisTest.php diff --git a/phpunit.xml b/phpunit.xml new file mode 100644 index 0000000..4a09dc2 --- /dev/null +++ b/phpunit.xml @@ -0,0 +1,19 @@ + + + + ./tests/Feed/Unit + + + ./tests/Feed/E2E + + + diff --git a/pint.json b/pint.json new file mode 100644 index 0000000..ea5e72c --- /dev/null +++ b/pint.json @@ -0,0 +1,3 @@ +{ + "preset": "psr12" +} diff --git a/src/Feed/Adapter.php b/src/Feed/Adapter.php new file mode 100644 index 0000000..0dd922e --- /dev/null +++ b/src/Feed/Adapter.php @@ -0,0 +1,142 @@ +name; + } + + /** + * Append an event and return the id the backend assigned it. + * + * Any id already on $event is ignored: positions are the backend's to + * allocate, since only it can guarantee they are ordered. + * + * @throws Exception When the event cannot be appended. Never silently, and + * never partially β€” a caller that gets an id back can tell every + * consumer will see the event. + */ + abstract public function append(Event $event): string; + + /** + * Read up to $limit events strictly after $lastEventId, oldest first, or + * from the oldest retained event when it is null. + * + * An empty result means the consumer is caught up, not that the feed is + * empty. + * + * @param int $timeout Milliseconds to wait for an event before giving up, + * honoured only when {@see pollable()} is true; {@see Feed::poll()} + * handles the wait for every other adapter. + * @return list + * @throws Invalid When $lastEventId is not a feed position. + * @throws Exception When the backend cannot be read. + */ + abstract public function read(?string $lastEventId, int $limit, int $timeout = 0): array; + + /** + * Whether the backend blocks until an event arrives on its own. + * + * False here rather than abstract because polling in a loop works against + * anything; an adapter only overrides it when the backend can do better, + * and {@see Feed::poll()} then hands the wait over instead of sleeping. + */ + public function pollable(): bool + { + return false; + } + + /** + * The backend fields an event is stored as. + * + * `data` is JSON so the payload can nest; everything else is a flat string + * because those are the fields a backend may want to index or filter on. + * The id is not among them β€” it is the key the entry is stored under. + * + * @return array + * @throws Invalid When the payload cannot be encoded. + */ + protected static function encode(Event $event): array + { + try { + $data = \json_encode($event->data, JSON_THROW_ON_ERROR); + } catch (\JsonException $error) { + throw new Invalid('Feed event data must be JSON encodable: ' . $error->getMessage(), previous: $error); + } + + return [ + 'type' => $event->type, + 'source' => $event->source, + 'subject' => $event->subject, + 'time' => $event->time, + 'data' => $data, + ]; + } + + /** + * Rebuild an event from what {@see encode()} stored. + * + * Undecodable payloads become an empty array rather than an error: the + * event still happened, its id is still a valid position, and refusing to + * return it would wedge every consumer behind it forever. + * + * @param array $fields + */ + protected static function decode(string $id, array $fields): Event + { + $data = \json_decode(self::field($fields, 'data'), true); + + return new Event( + id: $id, + type: self::field($fields, 'type'), + data: \is_array($data) ? $data : [], + source: self::field($fields, 'source'), + subject: self::field($fields, 'subject'), + time: self::field($fields, 'time'), + ); + } + + /** + * @param array $fields + */ + private static function field(array $fields, string $key): string + { + $value = $fields[$key] ?? ''; + + return \is_scalar($value) ? (string) $value : ''; + } +} diff --git a/src/Feed/Adapter/Http.php b/src/Feed/Adapter/Http.php new file mode 100644 index 0000000..a658f0d --- /dev/null +++ b/src/Feed/Adapter/Http.php @@ -0,0 +1,121 @@ +addHeader('x-appwrite-jwt', $token) + * ->setMaxRetries(0); // The consumer's own retry is the next poll + * + * $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); + * ``` + */ +class Http extends Adapter +{ + /** + * @param Client $client Configured with whatever credentials the producer + * requires. Retries are best left off: a failed read leaves the + * cursor where it was, so the next poll is already the retry, and + * retrying inside a long poll multiplies the time a tick can take. + * @param string $endpoint Base URL the producer serves its feeds under. + * The feed name is appended to it, so + * `https://cloud.example.com/v1/feeds` reads + * `https://cloud.example.com/v1/feeds/edge`. + * @param string $name Feed name, as the producer knows it. + */ + public function __construct( + protected readonly Client $client, + protected readonly string $endpoint, + string $name, + ) { + parent::__construct($name); + } + + /** + * The URL this adapter reads. + */ + public function getUrl(): string + { + return \rtrim($this->endpoint, '/') . '/' . \rawurlencode($this->name); + } + + /** + * @throws Unsupported Always. A consumer cannot append to a feed it does + * not own; call the producer's own API instead. + */ + public function append(Event $event): string + { + throw new Unsupported("The {$this->name} feed is read over HTTP and cannot be appended to"); + } + + public function read(?string $lastEventId, int $limit, int $timeout = 0): array + { + $url = $this->getUrl(); + + try { + $response = $this->client->fetch( + url: $url, + method: Client::METHOD_GET, + query: Protocol::query($lastEventId, $limit, $timeout), + // The producer is expected to answer within its own timeout; + // the margin only stops the client cutting off a poll that is + // legitimately waiting one out. + timeoutMs: $timeout > 0 ? $timeout + Protocol::TIMEOUT_MARGIN : null, + ); + } catch (\Throwable $error) { + throw new Transport("Failed to read the {$this->name} feed at {$url}: {$error->getMessage()}", previous: $error); + } + + $status = $response->getStatusCode(); + + if ($status >= 400) { + // Carried as the exception code so a caller can act on it β€” most + // usefully to treat a 404 as "this producer does not serve the + // feed yet", which is the normal state while a feed is being + // rolled out across services and not something to alert on. + throw new Transport( + "Reading the {$this->name} feed at {$url} failed with status {$status}", + $status, + ); + } + + try { + $body = $response->json(); + } catch (\Throwable $error) { + throw new Transport("The {$this->name} feed at {$url} returned a body that is not JSON: {$error->getMessage()}", previous: $error); + } + + return Protocol::decode($body); + } + + /** + * The producer does the waiting. + */ + public function pollable(): bool + { + return true; + } +} diff --git a/src/Feed/Adapter/Memory.php b/src/Feed/Adapter/Memory.php new file mode 100644 index 0000000..92e8d2b --- /dev/null +++ b/src/Feed/Adapter/Memory.php @@ -0,0 +1,113 @@ + */ + private array $events = []; + + /** + * Last millisecond an event was appended in, with the sequence number + * reached within it. Tracked so several appends inside the same + * millisecond still get ordered ids, the way `XADD` does. + */ + private int $timestamp = 0; + + private int $sequence = -1; + + public function __construct(string $name, protected readonly int $maxSize = 100_000) + { + parent::__construct($name); + } + + public function append(Event $event): string + { + $now = (int) \floor(\microtime(true) * 1000); + + if ($now > $this->timestamp) { + $this->timestamp = $now; + $this->sequence = 0; + } else { + // Also covers a clock that stepped backwards: ids must never go + // backwards, so the sequence keeps climbing under the old + // millisecond rather than the timestamp following the clock down. + $this->sequence++; + } + + $id = Id::encode($this->timestamp, $this->sequence); + + // Stored through the same encode/decode a real backend goes through, + // rather than holding the object. Otherwise this adapter would accept + // payloads that cannot be serialized and hand back values that survived + // a round trip they would not survive in production β€” which is the one + // way a stand-in like this actively causes harm. + $this->events[] = self::decode($id, self::encode($event)); + + if (\count($this->events) > $this->maxSize) { + $this->events = \array_slice($this->events, -$this->maxSize); + } + + return $id; + } + + public function read(?string $lastEventId, int $limit, int $timeout = 0): array + { + // Validates the position even when nothing will be returned, so a + // malformed cursor fails the same way it does on every other adapter + // instead of only once the feed has events in it. + $after = $lastEventId === null ? null : Id::decode($lastEventId); + + $events = []; + + foreach ($this->events as $event) { + if ($after !== null && Id::decode($event->id) <= $after) { + continue; + } + + $events[] = $event; + + if (\count($events) >= $limit) { + break; + } + } + + return $events; + } + + /** + * How many events are currently retained. Test affordance β€” a feed has no + * length a consumer is allowed to care about. + */ + public function count(): int + { + return \count($this->events); + } + + /** + * Drop every event, without resetting the id counter: positions already + * handed out must not be reissued, or a consumer holding one would skip + * whatever is appended next. + */ + public function flush(): void + { + $this->events = []; + } +} diff --git a/src/Feed/Adapter/None.php b/src/Feed/Adapter/None.php new file mode 100644 index 0000000..30eb07e --- /dev/null +++ b/src/Feed/Adapter/None.php @@ -0,0 +1,39 @@ +name} feed"); + } + + public function read(?string $lastEventId, int $limit, int $timeout = 0): array + { + throw new Unsupported("No feed backend is configured for the {$this->name} feed"); + } +} diff --git a/src/Feed/Adapter/Pool.php b/src/Feed/Adapter/Pool.php new file mode 100644 index 0000000..c2f4bee --- /dev/null +++ b/src/Feed/Adapter/Pool.php @@ -0,0 +1,56 @@ + $pool + * @param string $name Feed name; the stream is stored at `feed:`. + * @param int $maxSize Approximate cap on retained events. + */ + public function __construct( + protected readonly UtopiaPool $pool, + string $name, + protected readonly int $maxSize = 100_000, + ) { + parent::__construct($name); + } + + public function append(Event $event): string + { + return $this->pool->use(fn (\Redis|\RedisCluster $redis): string => $this->adapter($redis)->append($event)); + } + + public function read(?string $lastEventId, int $limit, int $timeout = 0): array + { + return $this->pool->use( + fn (\Redis|\RedisCluster $redis): array => $this->adapter($redis)->read($lastEventId, $limit, $timeout) + ); + } + + /** + * The connection is only borrowed for the length of one call, so the + * adapter wrapping it is built per call too. It holds no state beyond the + * connection, which makes that free. + */ + private function adapter(\Redis|\RedisCluster $redis): Redis + { + return new Redis($redis, $this->name, $this->maxSize); + } +} diff --git a/src/Feed/Adapter/Redis.php b/src/Feed/Adapter/Redis.php new file mode 100644 index 0000000..b072ad9 --- /dev/null +++ b/src/Feed/Adapter/Redis.php @@ -0,0 +1,97 @@ +`. + * @param int $maxSize Approximate cap on retained events. The default + * holds a long weekend of a busy feed, which is the window that + * matters: a consumer down for longer than its feed's retention + * resumes from the oldest event it can, rather than from where it + * left off. + */ + public function __construct( + protected readonly \Redis|\RedisCluster $redis, + string $name, + protected readonly int $maxSize = 100_000, + ) { + parent::__construct($name); + } + + /** + * The key the stream lives at. Namespaced so a feed can share a Redis with + * whatever else the service keeps there. + */ + public function getKey(): string + { + return 'feed:' . $this->name; + } + + public function append(Event $event): string + { + try { + $id = $this->redis->xAdd($this->getKey(), '*', self::encode($event), $this->maxSize, true); + } catch (\RedisException $error) { + throw new Transport("Failed to append to the {$this->name} feed: {$error->getMessage()}", previous: $error); + } + + if (!\is_string($id) || $id === '') { + throw new Transport("Failed to append to the {$this->name} feed"); + } + + return $id; + } + + public function read(?string $lastEventId, int $limit, int $timeout = 0): array + { + $start = $lastEventId === null ? '-' : Id::after($lastEventId); + + try { + $entries = $this->redis->xRange($this->getKey(), $start, '+', $limit); + } catch (\RedisException $error) { + throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); + } + + if (!\is_array($entries)) { + return []; + } + + $events = []; + + /** @var mixed $fields */ + foreach ($entries as $id => $fields) { + if (!\is_array($fields)) { + continue; + } + + $events[] = self::decode((string) $id, $fields); + } + + return $events; + } +} diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php new file mode 100644 index 0000000..4578882 --- /dev/null +++ b/src/Feed/Consumer.php @@ -0,0 +1,232 @@ +consume(function (Event $event) use ($cache) { + * $cache->purge($event->getData('tag')); + * }); + * ``` + * + * ## What the handler must tolerate + * + * Delivery is at-least-once, so a handler will see the same event more than + * once and must be safe to repeat. There are three separate reasons, and no + * arrangement of this class removes any of them: a handler can succeed and the + * position fail to save; a batch can be interrupted partway and replay from + * the last event that succeeded; and a consumer whose position is lost + * restarts from the oldest retained event. + * + * A handler rejects an event by throwing. That stops the run at that event and + * leaves the position before it, so the next run starts there and tries again. + * Everything already handled in that run stays handled β€” progress is committed + * before the failure is re-raised β€” which means a handler that fails on one + * event does not undo the batch, but does block everything behind it until it + * stops failing. That is the intended behaviour: a feed is ordered, and + * stepping over a failure would deliver later events on top of state that was + * never updated. + * + * ## Starting position + * + * A consumer with no recorded position starts at the oldest retained event, + * never at the tip. Starting at the tip would drop whatever is already in the + * feed, and for a consumer being deployed for the first time that is not a + * hypothetical backlog β€” it is everything that happened between the producer + * shipping and the consumer shipping, which during a staged rollout is exactly + * the events that were meant to be caught up on. + */ +class Consumer +{ + /** + * Events per run. Small enough that a backlog drains in bounded steps + * instead of one long pass that fails near the end and repeats most of + * itself. + */ + public const int BATCH = 100; + + /** + * The position, mirrored in memory. + * + * A run therefore reads the store once, on its first pass, and a store + * that becomes unavailable afterwards costs nothing β€” the consumer keeps + * making progress and only replays if it restarts before the store + * recovers. + */ + private ?string $position = null; + + private bool $restored = false; + + /** @var (callable(\Throwable, string): void)|null */ + private $onWarning = null; + + /** + * @param Feed $feed Feed to read. + * @param string $name This consumer's name, which its position is stored + * under. Distinct per logical consumer, and stable across restarts + * and replicas β€” two processes sharing a name share a position and + * will each skip what the other handled. + * @param Cursor $cursor Where to keep the position. + * @param int $batch Events per run. + * @param int $timeout Milliseconds to wait for an event when the feed is + * caught up. Zero returns immediately, which is what a consumer + * driven by an external timer wants; a non-zero value suits a + * consumer looping on its own, where it replaces a sleep with a + * wait that ends the moment an event arrives. + * @throws Exception\Invalid When $name is empty. + */ + public function __construct( + protected readonly Feed $feed, + protected readonly string $name, + protected readonly Cursor $cursor, + protected readonly int $batch = self::BATCH, + protected readonly int $timeout = 0, + ) { + if ($name === '') { + throw new Exception\Invalid('Feed consumer requires a name'); + } + } + + public function getName(): string + { + return $this->name; + } + + public function getFeed(): Feed + { + return $this->feed; + } + + /** + * Report failures that were survived rather than raised β€” currently, a + * position that could not be loaded or saved. + * + * These are not fatal: the consumer carries on with its in-memory + * position and the only cost is a replay after a restart. They are still + * worth knowing about, because a store that has been failing quietly for a + * week is a replay of the entire retained feed waiting to happen. + * + * @param (callable(\Throwable, string): void)|null $callback Receives the + * error and a short context string. + */ + public function onWarning(?callable $callback): self + { + $this->onWarning = $callback; + + return $this; + } + + /** + * Hand every event not yet seen to $handler, oldest first, and return how + * many it accepted. + * + * @param callable(Event): void $handler Throws to reject an event, which + * stops the run and leaves the position before it. + * @return int Events handled. Zero means the consumer is caught up. + * @throws Exception When the feed cannot be read. The position stays where + * it was, so the next run retries the same events. + * @throws \Throwable Whatever the handler threw, after the events before + * it have been committed. + */ + public function consume(callable $handler): int + { + $events = $this->feed->poll($this->position(), $this->batch, $this->timeout); + + if ($events === []) { + return 0; + } + + $handled = 0; + $processed = null; + $failure = null; + + foreach ($events as $event) { + try { + $handler($event); + } catch (\Throwable $error) { + $failure = $error; + break; + } + + $processed = $event->id; + $handled++; + } + + if ($processed !== null) { + $this->advance($processed); + } + + if ($failure !== null) { + throw $failure; + } + + return $handled; + } + + /** + * Where this consumer has got to, or null if it has not started. + */ + public function position(): ?string + { + if ($this->restored) { + return $this->position; + } + + $this->restored = true; + + try { + $this->position = $this->cursor->load($this->name); + } catch (\Throwable $error) { + // Falls through to null, which restarts from the oldest retained + // event. Wasteful β€” it replays events already handled β€” but the + // alternatives are worse: guessing at a position risks skipping, + // and refusing to run means an outage in the cursor store becomes + // an outage in whatever the feed drives. + $this->warn($error, 'load'); + } + + return $this->position; + } + + /** + * Drop the position and start again from the oldest retained event on the + * next run. Every event still in the feed will be handled again. + * + * @throws Exception When the store cannot be written. + */ + public function reset(): void + { + $this->cursor->reset($this->name); + + $this->position = null; + $this->restored = true; + } + + private function advance(string $eventId): void + { + $this->position = $eventId; + + try { + $this->cursor->save($this->name, $eventId); + } catch (\Throwable $error) { + // In-memory position already moved, so this process does not + // repeat itself; only a restart before the store recovers replays. + $this->warn($error, 'save'); + } + } + + private function warn(\Throwable $error, string $context): void + { + if ($this->onWarning !== null) { + ($this->onWarning)($error, $context); + } + } +} diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php new file mode 100644 index 0000000..4dbc13b --- /dev/null +++ b/src/Feed/Cursor.php @@ -0,0 +1,87 @@ +feed; + } + + /** + * The last position $consumer recorded, or null if it has never recorded + * one β€” which a caller should read as "start from the beginning of what is + * retained", never as "start from now". + * + * @throws Exception When the store cannot be read. + */ + abstract public function load(string $consumer): ?string; + + /** + * Record a position. + * + * Only ever call this once the events up to $eventId have been handled. A + * position saved ahead of the work it stands for turns a crash into + * silently skipped events, which is the one failure this design cannot + * recover from β€” the events are still in the feed, but nothing will ever + * read them again. + * + * @throws Exception When the store cannot be written. + */ + abstract public function save(string $consumer, string $eventId): void; + + /** + * Forget a consumer's position, so its next read starts from the oldest + * retained event. + * + * @throws Exception When the store cannot be written. + */ + abstract public function reset(string $consumer): void; + + /** + * @throws Invalid When $consumer is empty. + */ + protected function key(string $consumer): string + { + if ($consumer === '') { + throw new Invalid('Cursor requires a consumer name'); + } + + return 'feed:' . $this->feed . ':cursor:' . $consumer; + } +} diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php new file mode 100644 index 0000000..93ae7c6 --- /dev/null +++ b/src/Feed/Cursor/Cache.php @@ -0,0 +1,66 @@ +cache->load($this->key($consumer), $this->ttl); + + return \is_string($cursor) && $cursor !== '' ? $cursor : null; + } + + public function save(string $consumer, string $eventId): void + { + if ($eventId === '') { + return; + } + + $this->cache->save($this->key($consumer), $eventId); + } + + public function reset(string $consumer): void + { + $this->cache->purge($this->key($consumer)); + } +} diff --git a/src/Feed/Cursor/Memory.php b/src/Feed/Cursor/Memory.php new file mode 100644 index 0000000..cc0c1d4 --- /dev/null +++ b/src/Feed/Cursor/Memory.php @@ -0,0 +1,39 @@ + */ + private array $cursors = []; + + public function load(string $consumer): ?string + { + return $this->cursors[$this->key($consumer)] ?? null; + } + + public function save(string $consumer, string $eventId): void + { + if ($eventId === '') { + return; + } + + $this->cursors[$this->key($consumer)] = $eventId; + } + + public function reset(string $consumer): void + { + unset($this->cursors[$this->key($consumer)]); + } +} diff --git a/src/Feed/Cursor/Pool.php b/src/Feed/Cursor/Pool.php new file mode 100644 index 0000000..c4fa67e --- /dev/null +++ b/src/Feed/Cursor/Pool.php @@ -0,0 +1,54 @@ + $pool + */ + public function __construct( + protected readonly UtopiaPool $pool, + string $feed, + ) { + parent::__construct($feed); + } + + public function load(string $consumer): ?string + { + return $this->pool->use(fn (\Redis|\RedisCluster $redis): ?string => $this->cursor($redis)->load($consumer)); + } + + public function save(string $consumer, string $eventId): void + { + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($consumer, $eventId): void { + $this->cursor($redis)->save($consumer, $eventId); + }); + } + + public function reset(string $consumer): void + { + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($consumer): void { + $this->cursor($redis)->reset($consumer); + }); + } + + private function cursor(\Redis|\RedisCluster $redis): Redis + { + return new Redis($redis, $this->feed); + } +} diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php new file mode 100644 index 0000000..11137cd --- /dev/null +++ b/src/Feed/Cursor/Redis.php @@ -0,0 +1,72 @@ +key($consumer); + + try { + /** @var mixed $cursor */ + $cursor = $this->redis->get($key); + } catch (\RedisException $error) { + throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } + + return \is_string($cursor) && $cursor !== '' ? $cursor : null; + } + + public function save(string $consumer, string $eventId): void + { + if ($eventId === '') { + return; + } + + $key = $this->key($consumer); + + try { + // Deliberately no expiry. Unlike a cache, this is the only copy, + // and a position that quietly expired would replay the whole + // retained feed the next time the consumer restarted. + $this->redis->set($key, $eventId); + } catch (\RedisException $error) { + throw new Transport("Failed to save the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } + } + + public function reset(string $consumer): void + { + $key = $this->key($consumer); + + try { + $this->redis->del($key); + } catch (\RedisException $error) { + throw new Transport("Failed to reset the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } + } +} diff --git a/src/Feed/Event.php b/src/Feed/Event.php new file mode 100644 index 0000000..d37f38c --- /dev/null +++ b/src/Feed/Event.php @@ -0,0 +1,146 @@ + $data Payload. Must survive a JSON + * round-trip, which is also why the keys are not narrowed to + * strings: a payload is allowed to be a JSON array, and that decodes + * with integer keys. Producers normally send a map. + * @param string $source Who produced the event, as a URI reference + * (`urn:appwrite:cloud:fra`). Stamped by {@see Feed} on append. + * @param string $subject The single business object the event is about, + * when it has one, so a consumer can filter without decoding + * `$data`. + * @param string $time RFC 3339 timestamp. Stamped on append. + */ + public function __construct( + public string $id, + public string $type, + public array $data = [], + public string $source = '', + public string $subject = '', + public string $time = '', + ) { + } + + /** + * Decode an event received from a producer. + * + * @param array $event + * @throws Invalid When the event carries no id. + * The id is the one field a consumer cannot proceed without: it is + * the cursor position, so accepting an event without one would mean + * losing the place in the feed. Every other field is defaulted, + * because a consumer that only reads `data` should not be stopped + * by a producer that omits `subject`. + */ + public static function fromArray(array $event): self + { + $id = $event['id'] ?? ''; + if (!\is_string($id) || $id === '') { + throw new Invalid('Feed event is missing an id'); + } + + $data = $event['data'] ?? []; + + return new self( + id: $id, + type: self::string($event, 'type'), + data: \is_array($data) ? $data : [], + source: self::string($event, 'source'), + subject: self::string($event, 'subject'), + time: self::string($event, 'time'), + ); + } + + /** + * The event as a CloudEvent, ready to be serialized to JSON. + * + * @return array + */ + public function toArray(): array + { + return [ + 'specversion' => self::SPEC_VERSION, + 'id' => $this->id, + 'type' => $this->type, + 'source' => $this->source, + 'time' => $this->time, + 'subject' => $this->subject, + 'datacontenttype' => self::CONTENT_TYPE, + 'data' => $this->data, + ]; + } + + /** + * Read one key out of the payload. + * + * Handlers are looking at data some other service wrote, so this exists to + * keep them from having to re-check `isset` and the type on every access. + */ + public function getData(string $key, mixed $default = null): mixed + { + return $this->data[$key] ?? $default; + } + + /** + * The same event at a new position, used by adapters to attach the id the + * backend assigned on append. + */ + public function withId(string $id): self + { + return new self( + id: $id, + type: $this->type, + data: $this->data, + source: $this->source, + subject: $this->subject, + time: $this->time, + ); + } + + /** + * Now, in the RFC 3339 form http-feeds asks for: UTC, milliseconds, `Z`. + */ + public static function now(): string + { + return (new \DateTimeImmutable('now', new \DateTimeZone('UTC')))->format('Y-m-d\TH:i:s.v\Z'); + } + + /** + * @param array $event + */ + private static function string(array $event, string $key): string + { + $value = $event[$key] ?? null; + + return \is_string($value) ? $value : ''; + } +} diff --git a/src/Feed/Exception.php b/src/Feed/Exception.php new file mode 100644 index 0000000..0a69927 --- /dev/null +++ b/src/Feed/Exception.php @@ -0,0 +1,14 @@ +append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']]); + * + * foreach ($feed->read($lastEventId) as $event) { + * // ... + * } + * ``` + * + * Subclass to give a feed a typed vocabulary β€” one method per thing that can + * happen, rather than callers assembling event types and payloads by hand. + */ +class Feed +{ + /** + * Most events a single read may return. A cap belongs here rather than on + * the caller: `limit` arrives from a consumer over the network, and an + * unbounded read is a way to hold a producer's worker open. + */ + public const int MAX_BATCH = 1000; + + /** + * Longest a long poll may hold a request open, in milliseconds. Kept + * under the 60s that proxies and load balancers commonly cut idle + * responses off at, so a poll ends by returning empty rather than by + * having the connection dropped underneath it. + */ + public const int MAX_TIMEOUT = 30_000; + + /** + * Microseconds between reads while long polling on a backend that cannot + * block on its own. Half a second bounds delivery latency at roughly that, + * while keeping a quiet feed at two reads a second per consumer. + */ + protected const int POLL_INTERVAL = 500_000; + + /** + * @param Adapter $adapter Where the events live. + * @param string $source Who is producing them, as a URI reference + * (`urn:appwrite:cloud:fra`). Stamped onto every event this + * instance appends, so a consumer merging feeds from several + * producers can tell which one an event came from. Irrelevant when + * the feed is only being read. + */ + public function __construct( + protected readonly Adapter $adapter, + protected readonly string $source = '', + ) { + } + + public function getAdapter(): Adapter + { + return $this->adapter; + } + + public function getName(): string + { + return $this->adapter->getName(); + } + + public function getSource(): string + { + return $this->source; + } + + /** + * Append an event and return its position in the feed. + * + * @param string $type What happened, in reverse-DNS notation. + * @param array $data Payload, JSON encodable. + * @param string $subject The one business object this is about, if there + * is one. + * @throws Exception\Invalid When $type is empty or $data cannot be + * encoded. + * @throws Exception When the backend rejects the append. + */ + public function append(string $type, array $data = [], string $subject = ''): string + { + if ($type === '') { + throw new Exception\Invalid('Feed event type is required'); + } + + return $this->publish(new Event( + id: '', + type: $type, + data: $data, + subject: $subject, + )); + } + + /** + * Append a prepared event, stamping it with this feed's source and the + * current time. + * + * Both are stamped here rather than accepted from the caller because they + * describe the append itself. Recording the source at append rather than + * at read also keeps it correct for a feed that is replicated or read back + * from somewhere other than where it was written. + * + * @throws Exception\Invalid When the event has no type or its data cannot + * be encoded. + * @throws Exception When the backend rejects the append. + */ + public function publish(Event $event): string + { + if ($event->type === '') { + throw new Exception\Invalid('Feed event type is required'); + } + + return $this->adapter->append(new Event( + id: '', + type: $event->type, + data: $event->data, + source: $this->source, + subject: $event->subject, + time: $event->time !== '' ? $event->time : Event::now(), + )); + } + + /** + * Read the events after $lastEventId, oldest first, or from the oldest + * retained event when it is null. + * + * Returns immediately, with an empty list when the consumer is caught up. + * + * @return list + * @throws Exception\Invalid When $lastEventId is not a feed position. + * @throws Exception When the backend cannot be read. + */ + public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): array + { + return $this->adapter->read($lastEventId, self::limit($limit)); + } + + /** + * {@see read()}, but when there is nothing new yet, wait up to $timeout + * milliseconds for something to arrive before answering. + * + * This is how a consumer subscribes in near real time without hammering + * the producer: poll in a loop with a timeout, and each call either + * returns as soon as an event is appended or costs one request per + * timeout while the feed is quiet. A timeout of 0 makes this a plain read. + * + * The batch may still come back empty β€” the timeout elapsing is a normal + * outcome, not a failure. + * + * Where the backend cannot block on its own this waits by re-reading on an + * interval, which under Swoole yields the worker only if coroutine hooks + * are enabled. Without them it holds the worker for the duration, so run + * it with hooks on or keep the timeout at 0. + * + * @return list + * @throws Exception\Invalid When $lastEventId is not a feed position. + * @throws Exception When the backend cannot be read. + */ + public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, int $timeout = 0): array + { + $limit = self::limit($limit); + $timeout = \max(0, \min($timeout, self::MAX_TIMEOUT)); + + if ($this->adapter->pollable()) { + return $this->adapter->read($lastEventId, $limit, $timeout); + } + + $deadline = \microtime(true) + $timeout / 1000; + + while (true) { + $events = $this->adapter->read($lastEventId, $limit); + + if ($events !== [] || \microtime(true) >= $deadline) { + return $events; + } + + \usleep(self::POLL_INTERVAL); + } + } + + /** + * Clamp rather than reject: `limit` is a hint about how much work a + * consumer wants in one go, and failing a read because it asked for too + * much would stall a feed over something the producer can simply decide. + */ + private static function limit(int $limit): int + { + return \max(1, \min($limit, self::MAX_BATCH)); + } +} diff --git a/src/Feed/Id.php b/src/Feed/Id.php new file mode 100644 index 0000000..5ad55cd --- /dev/null +++ b/src/Feed/Id.php @@ -0,0 +1,85 @@ +-`, + * where `seq` disambiguates events appended within the same millisecond. + * + * The format is part of the wire contract, not a Redis implementation detail: + * an id produced by one adapter has to be a valid position for another, so + * that a feed can move between backends without invalidating the positions + * consumers already hold. + */ +final class Id +{ + private const string PATTERN = '/^(\d+)-(\d+)$/'; + + /** + * Whether $id is a feed position. + */ + public static function isValid(string $id): bool + { + return \preg_match(self::PATTERN, $id) === 1; + } + + /** + * Build an id from its parts. + */ + public static function encode(int $timestamp, int $sequence): string + { + return $timestamp . '-' . $sequence; + } + + /** + * Split an id into its millisecond timestamp and sequence number. + * + * @return array{int, int} + * @throws Invalid When $id is not a feed position. + */ + public static function decode(string $id): array + { + if (\preg_match(self::PATTERN, $id, $matches) !== 1) { + throw new Invalid('Invalid feed event id: ' . $id); + } + + return [(int) $matches[1], (int) $matches[2]]; + } + + /** + * The exclusive successor of an id: the smallest position strictly after + * it. + * + * Computed rather than relying on Redis' `(`-prefixed exclusive ranges, so + * reads work against anything speaking the Redis 5 stream API β€” including + * the several proxies and compatible servers that never implemented the + * newer syntax. + * + * @throws Invalid When $id is not a feed position. + */ + public static function after(string $id): string + { + [$timestamp, $sequence] = self::decode($id); + + return self::encode($timestamp, $sequence + 1); + } + + /** + * Compare two positions the way `<=>` would, so ids sort by age rather + * than by string order (`10-0` is after `9-0`, not before it). + * + * @throws Invalid When either id is not a feed position. + */ + public static function compare(string $a, string $b): int + { + return self::decode($a) <=> self::decode($b); + } +} diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php new file mode 100644 index 0000000..1d14a7b --- /dev/null +++ b/src/Feed/Protocol.php @@ -0,0 +1,199 @@ +poll( + * $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, + * (int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH), + * (int) $request->getParam(Protocol::PARAM_TIMEOUT, 0), + * ); + * + * $response + * ->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)) + * ->json(Protocol::encode($events)); + * ``` + */ +final class Protocol +{ + /** + * Position to read from. Omitted or empty means the oldest retained + * event β€” not the newest. Starting at the tip would silently drop whatever + * is already in the feed, and on a fresh consumer that is not a + * hypothetical backlog: nothing is recorded until the first event arrives, + * so the first event is precisely the one that would be skipped. + */ + public const string PARAM_LAST_EVENT_ID = 'lastEventId'; + + /** + * Most events to return. + */ + public const string PARAM_LIMIT = 'limit'; + + /** + * Milliseconds to hold the request open waiting for an event before + * answering with an empty batch. Zero returns immediately. + */ + public const string PARAM_TIMEOUT = 'timeout'; + + public const string KEY_EVENTS = 'events'; + + public const string KEY_TOTAL = 'total'; + + /** + * A full batch is settled history β€” the same query returns the same events + * forever β€” so it may be cached indefinitely. + */ + public const string CACHE_IMMUTABLE = 'max-age=31536000'; + + /** + * A short batch is the live end of the feed. Re-asking the same question a + * second later legitimately returns more, so it must not be cached at all. + */ + public const string CACHE_NONE = 'no-store'; + + /** + * Extra milliseconds a consumer allows its HTTP client on top of the long + * poll timeout it asked for. + * + * Without it the client's own deadline races the server's: a poll that + * correctly waits out its full timeout gets cancelled a hair early and + * surfaces as a transport failure on every quiet tick, burying the + * failures that matter. + */ + public const int TIMEOUT_MARGIN = 10_000; + + /** + * The query string for a read. + * + * Parameters at their default are left out rather than sent explicitly, so + * a consumer and a producer that disagree on a default resolve it the + * producer's way β€” and so the URL of a first read is stable enough to be + * cached and logged as one thing. + * + * @return array + */ + public static function query(?string $lastEventId = null, int $limit = 0, int $timeout = 0): array + { + $query = []; + + if ($lastEventId !== null && $lastEventId !== '') { + $query[self::PARAM_LAST_EVENT_ID] = $lastEventId; + } + + if ($limit > 0) { + $query[self::PARAM_LIMIT] = $limit; + } + + if ($timeout > 0) { + $query[self::PARAM_TIMEOUT] = $timeout; + } + + return $query; + } + + /** + * The response body for a batch. + * + * @param list $events + * @return array{total: int, events: list>} + */ + public static function encode(array $events): array + { + return [ + self::KEY_TOTAL => \count($events), + self::KEY_EVENTS => \array_map(static fn (Event $event): array => $event->toArray(), $events), + ]; + } + + /** + * Read a batch out of a response body. + * + * Stops at the first event that cannot be decoded and returns the ones + * before it, rather than dropping it and carrying on. An event with no id + * has no position, so a consumer cannot record having passed it; skipping + * it would mean every event after it is acknowledged under a cursor that + * never advanced past the gap, and the next restart would replay them all. + * + * Returning the prefix keeps the events that *are* usable moving: the + * consumer applies them, advances to the last one, and meets the broken + * event at the head of the next batch β€” where, with no prefix left to + * salvage, this throws and the feed visibly stops instead of quietly + * losing events. + * + * @return list + * @throws Invalid When the payload is not a batch, or when the very first + * event in it cannot be decoded. + */ + public static function decode(mixed $payload): array + { + if (!\is_array($payload)) { + throw new Invalid('Expected a feed batch, got ' . \get_debug_type($payload)); + } + + $raw = $payload[self::KEY_EVENTS] ?? []; + if (!\is_array($raw)) { + throw new Invalid('Feed batch has a malformed "' . self::KEY_EVENTS . '" field'); + } + + $events = []; + + /** @var mixed $event */ + foreach ($raw as $event) { + if (!\is_array($event)) { + if ($events === []) { + throw new Invalid('Feed batch contains an entry that is not an event'); + } + + break; + } + + try { + $events[] = Event::fromArray($event); + } catch (Invalid $error) { + if ($events === []) { + throw $error; + } + + break; + } + } + + return $events; + } + + /** + * What a producer should send as `Cache-Control` for a batch. + * + * @param int $count Events being returned. + * @param int $limit Events that were asked for. + * @param bool $public Whether a shared cache may store the batch. Off by + * default: feeds are usually served behind authorization, and + * `public` there would let a CDN hand one consumer's events to a + * requester that never presented a credential. Only turn it on for + * a feed whose events are safe for anyone who can reach the URL. + */ + public static function cacheControl(int $count, int $limit, bool $public = false): string + { + if ($count < $limit || $count === 0) { + return self::CACHE_NONE; + } + + return ($public ? 'public, ' : 'private, ') . self::CACHE_IMMUTABLE; + } +} diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php new file mode 100644 index 0000000..2c4b0ff --- /dev/null +++ b/tests/Feed/E2E/RedisTest.php @@ -0,0 +1,243 @@ +redis = new \Redis(); + $this->redis->connect((string) (\getenv('REDIS_HOST') ?: 'redis'), (int) (\getenv('REDIS_PORT') ?: 6379)); + + // A fresh feed per test: these assert on positions, and a shared + // stream would leak them between tests. + $this->name = 'test-' . \bin2hex(\random_bytes(8)); + } + + protected function tearDown(): void + { + $this->redis->del('feed:' . $this->name); + + foreach ((array) $this->redis->keys('feed:' . $this->name . ':cursor:*') as $key) { + if (\is_string($key)) { + $this->redis->del($key); + } + } + + $this->redis->close(); + } + + private function feed(int $maxSize = 100_000): Feed + { + return new Feed(new RedisAdapter($this->redis, $this->name, $maxSize), 'urn:test:e2e'); + } + + public function testAppendsAndReadsBack(): void + { + $feed = $this->feed(); + + $id = $feed->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + + $events = $feed->read(); + + $this->assertCount(1, $events); + $this->assertSame($id, $events[0]->id); + $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); + $this->assertSame('example.com', $events[0]->subject); + $this->assertSame('urn:test:e2e', $events[0]->source); + $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); + $this->assertNotSame('', $events[0]->time); + } + + public function testStreamIdsMatchTheFormatPositionsAreParsedWith(): void + { + $id = $this->feed()->append('test'); + + $this->assertTrue(Id::isValid($id), "Redis returned an id this library cannot page from: {$id}"); + } + + public function testIdsIncreaseAcrossRapidAppends(): void + { + $feed = $this->feed(); + + $ids = []; + for ($i = 0; $i < 100; $i++) { + $ids[] = $feed->append('test'); + } + + $this->assertSame($ids, \array_unique($ids)); + + for ($i = 1; $i < \count($ids); $i++) { + $this->assertSame(1, Id::compare($ids[$i], $ids[$i - 1])); + } + } + + /** + * The reason positions are advanced arithmetically instead of with Redis' + * `(`-exclusive range syntax: this has to hold on every server and proxy + * that speaks the Redis 5 stream API, not only the ones that added it. + */ + public function testReadsStrictlyAfterAPosition(): void + { + $feed = $this->feed(); + + $first = $feed->append('a'); + $second = $feed->append('b'); + + $events = $feed->read($first); + + $this->assertCount(1, $events); + $this->assertSame($second, $events[0]->id); + $this->assertSame([], $feed->read($second)); + } + + public function testNestedPayloadsSurviveTheRoundTrip(): void + { + $data = [ + 'tags' => ['domain' => 'example.com', 'project' => 'p1'], + 'flags' => ['isAppwriteNetwork' => true], + 'count' => 42, + 'unicode' => 'ΓΌnΓ―cΓΈde βœ“', + ]; + + $this->feed()->append('test', $data); + + $this->assertSame($data, $this->feed()->read()[0]->data); + } + + public function testHonoursTheLimit(): void + { + $feed = $this->feed(); + + foreach (\range(1, 10) as $i) { + $feed->append('test'); + } + + $this->assertCount(3, $feed->read(null, 3)); + } + + public function testRejectsAPositionThatIsNotAFeedId(): void + { + $this->expectException(Invalid::class); + + $this->feed()->read('not-a-position'); + } + + /** + * Trimming is approximate, so this asserts the property a consumer relies + * on β€” that a position below the horizon still reads β€” rather than an + * exact retained count. + */ + public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void + { + $feed = $this->feed(maxSize: 10); + + $first = $feed->append('first'); + + foreach (\range(1, 500) as $i) { + $feed->append('event-' . $i); + } + + $events = $feed->read($first); + + $this->assertNotEmpty($events, 'A consumer that fell behind must still get what is retained'); + $this->assertLessThan(500, $this->redis->xLen('feed:' . $this->name), 'The feed must be trimmed'); + } + + public function testLongPollingReturnsAsSoonAsTheFeedHasSomething(): void + { + $feed = $this->feed(); + $feed->append('a'); + + $started = \microtime(true); + $events = $feed->poll(null, 10, 3000); + + $this->assertCount(1, $events); + $this->assertLessThan(1, \microtime(true) - $started); + } + + public function testLongPollingGivesUpAtTheTimeout(): void + { + $started = \microtime(true); + $events = $this->feed()->poll(null, 10, 700); + + $this->assertSame([], $events); + $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started); + } + + public function testConsumesThroughAPersistedCursor(): void + { + $feed = $this->feed(); + $cursor = new RedisCursor($this->redis, $this->name); + + $feed->append('a'); + $last = $feed->append('b'); + + $seen = []; + $handler = function (Event $event) use (&$seen): void { + $seen[] = $event->type; + }; + + $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume($handler)); + $this->assertSame($last, $cursor->load('invalidator')); + + // A second Consumer stands in for a restart: it has no in-memory + // position, so it has to pick the stored one up to avoid replaying. + $this->assertSame(0, (new Consumer($feed, 'invalidator', $cursor))->consume($handler)); + $this->assertSame(['a', 'b'], $seen); + } + + public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void + { + $feed = $this->feed(); + $cursor = new RedisCursor($this->redis, $this->name); + + $feed->append('a'); + + $this->assertSame(1, (new Consumer($feed, 'one', $cursor))->consume(fn (Event $e) => null)); + $this->assertSame(1, (new Consumer($feed, 'two', $cursor))->consume(fn (Event $e) => null)); + } + + public function testResetReplaysTheRetainedFeed(): void + { + $feed = $this->feed(); + $cursor = new RedisCursor($this->redis, $this->name); + + $feed->append('a'); + $feed->append('b'); + + $consumer = new Consumer($feed, 'invalidator', $cursor); + $consumer->consume(fn (Event $e) => null); + $consumer->reset(); + + $this->assertNull($cursor->load('invalidator')); + $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume(fn (Event $e) => null)); + } + + public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void + { + (new RedisCursor($this->redis, $this->name))->save('invalidator', '1-0'); + + $this->assertSame('1-0', $this->redis->get('feed:' . $this->name . ':cursor:invalidator')); + } +} diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php new file mode 100644 index 0000000..dcde876 --- /dev/null +++ b/tests/Feed/Unit/ConsumerTest.php @@ -0,0 +1,323 @@ +adapter = new MemoryAdapter('edge'); + $this->feed = new Feed($this->adapter, 'urn:test'); + $this->cursor = new MemoryCursor('edge'); + } + + private function consumer(?Cursor $cursor = null, int $batch = Consumer::BATCH): Consumer + { + return new Consumer($this->feed, 'invalidator', $cursor ?? $this->cursor, $batch); + } + + /** + * @param-out int $count + * @return list + */ + private function drain(Consumer $consumer, ?int &$count = null): array + { + $seen = []; + $count = $consumer->consume(function (Event $event) use (&$seen): void { + $seen[] = $event->type; + }); + + return $seen; + } + + public function testHandlesEachEventAndAdvancesPastTheLastOne(): void + { + $this->feed->append('a'); + $last = $this->feed->append('b'); + + $consumer = $this->consumer(); + + $this->assertSame(['a', 'b'], $this->drain($consumer, $count)); + $this->assertSame(2, $count); + $this->assertSame($last, $this->cursor->load('invalidator')); + $this->assertSame($last, $consumer->position()); + } + + public function testCaughtUpConsumerDoesNothing(): void + { + $this->feed->append('a'); + + $consumer = $this->consumer(); + $consumer->consume(fn (Event $event) => null); + + $this->assertSame(0, $consumer->consume(fn (Event $event) => null)); + } + + public function testResumesFromTheStoredPosition(): void + { + $first = $this->feed->append('a'); + $this->feed->append('b'); + + $this->cursor->save('invalidator', $first); + + $this->assertSame(['b'], $this->drain($this->consumer())); + } + + /** + * A consumer that has never run starts at the oldest retained event, not + * at the tip β€” otherwise the first event a feed ever carries is the one + * event that is guaranteed to be dropped. + */ + public function testAConsumerWithNoPositionStartsAtTheOldestEventNotTheTip(): void + { + $this->feed->append('a'); + $this->feed->append('b'); + + $this->assertSame(['a', 'b'], $this->drain($this->consumer())); + } + + public function testReadsTheStoreOnceAndThenTracksThePositionInMemory(): void + { + $this->feed->append('a'); + + $cursor = new class ('edge') extends MemoryCursor { + public int $loads = 0; + + public function load(string $consumer): ?string + { + $this->loads++; + + return parent::load($consumer); + } + }; + + $consumer = $this->consumer($cursor); + + $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (Event $event) => null); + + $this->assertSame(1, $cursor->loads); + } + + public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void + { + $first = $this->feed->append('a'); + $this->feed->append('b'); + $this->feed->append('c'); + + $consumer = $this->consumer(); + $seen = []; + + try { + $consumer->consume(function (Event $event) use (&$seen): void { + if ($event->type === 'b') { + throw new \RuntimeException('nope'); + } + + $seen[] = $event->type; + }); + $this->fail('The handler failure should have been re-raised'); + } catch (\RuntimeException $error) { + $this->assertSame('nope', $error->getMessage()); + } + + $this->assertSame(['a'], $seen); + $this->assertSame($first, $this->cursor->load('invalidator'), 'Progress before the failure is committed'); + } + + public function testRetriesTheFailedEventOnTheNextRun(): void + { + $this->feed->append('a'); + $this->feed->append('b'); + + $consumer = $this->consumer(); + $attempts = 0; + + try { + $consumer->consume(function (Event $event) use (&$attempts): void { + if ($event->type === 'b') { + $attempts++; + throw new \RuntimeException('nope'); + } + }); + } catch (\RuntimeException) { + // Expected. + } + + $this->assertSame(['b'], $this->drain($consumer), 'The failed event comes back'); + $this->assertSame(1, $attempts); + } + + /** + * A failure on the very first event of a run commits nothing, so a store + * that was already empty stays empty rather than being written a position + * that stands for no completed work. + */ + public function testAFailureOnTheFirstEventCommitsNothing(): void + { + $this->feed->append('a'); + + try { + $this->consumer()->consume(fn (Event $event) => throw new \RuntimeException('nope')); + } catch (\RuntimeException) { + // Expected. + } + + $this->assertNull($this->cursor->load('invalidator')); + } + + public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void + { + $this->feed->append('a'); + $this->feed->append('b'); + $this->feed->append('c'); + + $this->assertSame(3, $this->consumer()->consume(fn (Event $event) => null)); + } + + public function testDrainsABacklogInBatches(): void + { + foreach (\range(1, 10) as $i) { + $this->feed->append('event-' . $i); + } + + $consumer = $this->consumer(batch: 4); + + $this->assertSame(4, $consumer->consume(fn (Event $event) => null)); + $this->assertSame(4, $consumer->consume(fn (Event $event) => null)); + $this->assertSame(2, $consumer->consume(fn (Event $event) => null)); + $this->assertSame(0, $consumer->consume(fn (Event $event) => null)); + } + + /** + * The store failing must not stop the work: the position is mirrored in + * memory, so the run carries on and only a restart before the store + * recovers replays anything. + */ + public function testKeepsWorkingWhenThePositionCannotBeLoaded(): void + { + $this->feed->append('a'); + + $consumer = $this->consumer(new FailingCursor('edge', onLoad: true)); + $warnings = []; + $consumer->onWarning(function (\Throwable $error, string $context) use (&$warnings): void { + $warnings[] = $context; + }); + + $this->assertSame(['a'], $this->drain($consumer)); + $this->assertSame(['load'], $warnings); + } + + public function testKeepsWorkingWhenThePositionCannotBeSaved(): void + { + $this->feed->append('a'); + $this->feed->append('b'); + + $consumer = $this->consumer(new FailingCursor('edge', onSave: true)); + $warnings = []; + $consumer->onWarning(function (\Throwable $error, string $context) use (&$warnings): void { + $warnings[] = $context; + }); + + $this->assertSame(['a', 'b'], $this->drain($consumer)); + $this->assertSame(['save'], $warnings); + + $this->feed->append('c'); + + $this->assertSame(['c'], $this->drain($consumer), 'The in-memory position still moved'); + } + + public function testSurvivesAFailingStoreWithNoWarningHandler(): void + { + $this->feed->append('a'); + + $this->assertSame(['a'], $this->drain($this->consumer(new FailingCursor('edge', onLoad: true, onSave: true)))); + } + + public function testResetReplaysEverythingStillRetained(): void + { + $this->feed->append('a'); + $this->feed->append('b'); + + $consumer = $this->consumer(); + $consumer->consume(fn (Event $event) => null); + + $consumer->reset(); + + $this->assertNull($consumer->position()); + $this->assertNull($this->cursor->load('invalidator')); + $this->assertSame(['a', 'b'], $this->drain($consumer)); + } + + public function testConsumersOfTheSameFeedTrackSeparatePositions(): void + { + $this->feed->append('a'); + + $one = new Consumer($this->feed, 'one', $this->cursor); + $two = new Consumer($this->feed, 'two', $this->cursor); + + $this->assertSame(1, $one->consume(fn (Event $event) => null)); + $this->assertSame(1, $two->consume(fn (Event $event) => null), 'The second consumer has its own position'); + $this->assertSame(0, $one->consume(fn (Event $event) => null)); + } + + public function testPositionIsNullBeforeTheFirstRun(): void + { + $this->assertNull($this->consumer()->position()); + } + + public function testRejectsAnEmptyConsumerName(): void + { + $this->expectException(Invalid::class); + + new Consumer($this->feed, '', $this->cursor); + } + + public function testExposesWhatItIsConsuming(): void + { + $consumer = $this->consumer(); + + $this->assertSame('invalidator', $consumer->getName()); + $this->assertSame($this->feed, $consumer->getFeed()); + } + + /** + * The failure a consumer must not turn into a gap: if the read itself + * fails, nothing is handled and nothing is committed. + */ + public function testAFailedReadLeavesThePositionAlone(): void + { + $first = $this->feed->append('a'); + $this->feed->append('b'); + $this->cursor->save('invalidator', $first); + + $consumer = new Consumer(new Feed(new \Utopia\Feed\Adapter\None('edge')), 'invalidator', $this->cursor); + + $this->expectException(\Utopia\Feed\Exception\Unsupported::class); + + try { + $consumer->consume(fn (Event $event) => null); + } finally { + $this->assertSame($first, $this->cursor->load('invalidator')); + } + } +} diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php new file mode 100644 index 0000000..5b536ca --- /dev/null +++ b/tests/Feed/Unit/CursorTest.php @@ -0,0 +1,121 @@ + + */ + public static function stores(): array + { + return [ + 'memory' => [new Memory('edge')], + 'cache' => [new Cache(new UtopiaCache(new CacheMemory()), 'edge')], + ]; + } + + /** + * @dataProvider stores + */ + public function testAnUnknownConsumerHasNoPosition(Cursor $cursor): void + { + $this->assertNull($cursor->load('never-run')); + } + + /** + * @dataProvider stores + */ + public function testRoundTripsAPosition(Cursor $cursor): void + { + $cursor->save('invalidator', '1690000000000-0'); + + $this->assertSame('1690000000000-0', $cursor->load('invalidator')); + } + + /** + * @dataProvider stores + */ + public function testOverwritesAPosition(Cursor $cursor): void + { + $cursor->save('invalidator', '1-0'); + $cursor->save('invalidator', '2-0'); + + $this->assertSame('2-0', $cursor->load('invalidator')); + } + + /** + * @dataProvider stores + */ + public function testConsumersDoNotShareAPosition(Cursor $cursor): void + { + $cursor->save('one', '1-0'); + $cursor->save('two', '2-0'); + + $this->assertSame('1-0', $cursor->load('one')); + $this->assertSame('2-0', $cursor->load('two')); + } + + /** + * @dataProvider stores + */ + public function testResetForgetsAPosition(Cursor $cursor): void + { + $cursor->save('invalidator', '1-0'); + $cursor->reset('invalidator'); + + $this->assertNull($cursor->load('invalidator')); + } + + /** + * @dataProvider stores + */ + public function testSavingAnEmptyPositionIsIgnored(Cursor $cursor): void + { + $cursor->save('invalidator', '1-0'); + $cursor->save('invalidator', ''); + + $this->assertSame('1-0', $cursor->load('invalidator'), 'An empty position must not erase a real one'); + } + + /** + * @dataProvider stores + */ + public function testRejectsAnEmptyConsumerName(Cursor $cursor): void + { + $this->expectException(Invalid::class); + + $cursor->load(''); + } + + public function testFeedsDoNotShareAPosition(): void + { + $cache = new UtopiaCache(new CacheMemory()); + + (new Cache($cache, 'edge'))->save('invalidator', '1-0'); + + $this->assertNull((new Cache($cache, 'other'))->load('invalidator')); + } + + public function testRejectsAnEmptyFeedName(): void + { + $this->expectException(Invalid::class); + + new Memory(''); + } + + public function testExposesTheFeedItTracks(): void + { + $this->assertSame('edge', (new Memory('edge'))->getFeed()); + } +} diff --git a/tests/Feed/Unit/EventTest.php b/tests/Feed/Unit/EventTest.php new file mode 100644 index 0000000..b3e3861 --- /dev/null +++ b/tests/Feed/Unit/EventTest.php @@ -0,0 +1,164 @@ + '1.0', + 'id' => '1690000000000-0', + 'type' => 'io.appwrite.edge.invalidate-rule', + 'source' => 'urn:appwrite:cloud:fra', + 'time' => '2026-07-29T12:00:00.000Z', + 'subject' => 'preview.example.com', + 'datacontenttype' => 'application/json', + 'data' => ['tags' => ['domain' => 'preview.example.com']], + ]); + + $this->assertSame('1690000000000-0', $event->id); + $this->assertSame('io.appwrite.edge.invalidate-rule', $event->type); + $this->assertSame('urn:appwrite:cloud:fra', $event->source); + $this->assertSame('2026-07-29T12:00:00.000Z', $event->time); + $this->assertSame('preview.example.com', $event->subject); + $this->assertSame(['tags' => ['domain' => 'preview.example.com']], $event->data); + } + + public function testRejectsAnEventWithoutAnId(): void + { + $this->expectException(Invalid::class); + + Event::fromArray(['type' => 'io.appwrite.edge.invalidate']); + } + + /** + * @return array + */ + public static function unusableIds(): array + { + return [ + 'empty' => [''], + 'null' => [null], + 'numeric' => [42], + 'array' => [[]], + ]; + } + + /** + * @dataProvider unusableIds + */ + public function testRejectsAnIdThatIsNotANonEmptyString(mixed $id): void + { + $this->expectException(Invalid::class); + + Event::fromArray(['id' => $id, 'type' => 'test']); + } + + public function testDefaultsEveryFieldExceptTheId(): void + { + $event = Event::fromArray(['id' => '1-0']); + + $this->assertSame('1-0', $event->id); + $this->assertSame('', $event->type); + $this->assertSame('', $event->source); + $this->assertSame('', $event->subject); + $this->assertSame('', $event->time); + $this->assertSame([], $event->data); + } + + /** + * A producer that adds a field must not break a consumer written before + * it, which is what makes a feed safe to evolve. + */ + public function testIgnoresUnknownFields(): void + { + $event = Event::fromArray([ + 'id' => '1-0', + 'type' => 'test', + 'dataschema' => 'https://example.com/schema.json', + 'somethingnew' => ['a' => 'b'], + ]); + + $this->assertSame('1-0', $event->id); + $this->assertSame('test', $event->type); + } + + public function testCoercesAMalformedFieldToItsDefaultRatherThanFailing(): void + { + $event = Event::fromArray([ + 'id' => '1-0', + 'type' => ['not', 'a', 'string'], + 'data' => 'not an array', + ]); + + $this->assertSame('', $event->type); + $this->assertSame([], $event->data); + } + + public function testRoundTripsThroughItsArrayForm(): void + { + $event = new Event( + id: '1-0', + type: 'io.appwrite.edge.invalidate', + data: ['tags' => ['project' => 'proj-1']], + source: 'urn:appwrite:cloud:fra', + subject: 'proj-1', + time: '2026-07-29T12:00:00.000Z', + ); + + $this->assertEquals($event, Event::fromArray($event->toArray())); + } + + public function testSurvivesAJsonRoundTrip(): void + { + $event = new Event(id: '1-0', type: 'test', data: ['nested' => ['a' => 1]]); + + $encoded = \json_encode($event->toArray()); + $this->assertIsString($encoded); + + $decoded = Event::fromArray((array) \json_decode($encoded, true)); + + $this->assertSame(['nested' => ['a' => 1]], $decoded->data); + } + + public function testAlwaysReportsTheSpecVersionAndContentType(): void + { + $encoded = (new Event(id: '1-0', type: 'test'))->toArray(); + + $this->assertSame('1.0', $encoded['specversion']); + $this->assertSame('application/json', $encoded['datacontenttype']); + } + + public function testReadsPayloadKeysWithADefault(): void + { + $event = new Event(id: '1-0', type: 'test', data: ['tags' => ['a' => 'b']]); + + $this->assertSame(['a' => 'b'], $event->getData('tags')); + $this->assertNull($event->getData('missing')); + $this->assertSame('fallback', $event->getData('missing', 'fallback')); + } + + public function testWithIdLeavesEverythingElseAlone(): void + { + $event = new Event(id: '', type: 'test', data: ['a' => 'b'], subject: 's'); + $stamped = $event->withId('7-0'); + + $this->assertSame('7-0', $stamped->id); + $this->assertSame('test', $stamped->type); + $this->assertSame(['a' => 'b'], $stamped->data); + $this->assertSame('s', $stamped->subject); + $this->assertSame('', $event->id, 'The original must not be mutated'); + } + + public function testNowIsRfc3339WithMilliseconds(): void + { + $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', Event::now()); + } +} diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php new file mode 100644 index 0000000..c920083 --- /dev/null +++ b/tests/Feed/Unit/FeedTest.php @@ -0,0 +1,282 @@ +adapter = new Memory('edge'); + $this->feed = new Feed($this->adapter, 'urn:appwrite:cloud:fra'); + } + + public function testAppendReturnsAPosition(): void + { + $id = $this->feed->append('io.appwrite.edge.invalidate', ['tags' => ['project' => 'p1']]); + + $this->assertTrue(Id::isValid($id)); + } + + public function testReadsBackWhatWasAppended(): void + { + $this->feed->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + + $events = $this->feed->read(); + + $this->assertCount(1, $events); + $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); + $this->assertSame('example.com', $events[0]->subject); + $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); + } + + public function testStampsTheSourceAndTimeOnAppend(): void + { + $this->feed->append('test'); + + $event = $this->feed->read()[0]; + + $this->assertSame('urn:appwrite:cloud:fra', $event->source); + $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', $event->time); + } + + /** + * Recording it at append rather than at read keeps it correct for a feed + * read back somewhere other than where it was written. + */ + public function testKeepsTheSourceOfTheProducerThatAppended(): void + { + (new Feed($this->adapter, 'urn:appwrite:cloud:fra'))->append('test'); + (new Feed($this->adapter, 'urn:appwrite:cloud:nyc'))->append('test'); + + $events = (new Feed($this->adapter, 'urn:appwrite:cloud:syd'))->read(); + + $this->assertSame('urn:appwrite:cloud:fra', $events[0]->source); + $this->assertSame('urn:appwrite:cloud:nyc', $events[1]->source); + } + + public function testEventsComeBackOldestFirst(): void + { + foreach (['a', 'b', 'c'] as $type) { + $this->feed->append($type); + } + + $this->assertSame(['a', 'b', 'c'], \array_map(fn (Event $e): string => $e->type, $this->feed->read())); + } + + public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void + { + $ids = []; + for ($i = 0; $i < 50; $i++) { + $ids[] = $this->feed->append('test'); + } + + $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); + + for ($i = 1; $i < \count($ids); $i++) { + $this->assertSame(1, Id::compare($ids[$i], $ids[$i - 1]), 'Positions must increase'); + } + } + + public function testReadsStrictlyAfterTheGivenPosition(): void + { + $first = $this->feed->append('a'); + $this->feed->append('b'); + + $events = $this->feed->read($first); + + $this->assertCount(1, $events); + $this->assertSame('b', $events[0]->type); + } + + public function testReadFromTheLastEventIsEmpty(): void + { + $this->feed->append('a'); + $last = $this->feed->append('b'); + + $this->assertSame([], $this->feed->read($last)); + } + + public function testNullPositionReadsFromTheOldestRetainedEvent(): void + { + $this->feed->append('a'); + $this->feed->append('b'); + + $this->assertCount(2, $this->feed->read(null)); + } + + public function testHonoursTheLimit(): void + { + foreach (\range(1, 10) as $i) { + $this->feed->append('test'); + } + + $this->assertCount(3, $this->feed->read(null, 3)); + } + + /** + * `limit` arrives from a consumer, so it is clamped rather than rejected β€” + * failing the read would stall a feed over something the producer can just + * decide. + */ + public function testClampsTheLimitToTheMaximum(): void + { + $this->feed->append('test'); + + $this->assertCount(1, $this->feed->read(null, Feed::MAX_BATCH * 10)); + $this->assertCount(1, $this->feed->read(null, 0)); + $this->assertCount(1, $this->feed->read(null, -5)); + } + + public function testRejectsAPositionThatIsNotAFeedId(): void + { + $this->expectException(Invalid::class); + + $this->feed->read('not-a-position'); + } + + public function testRejectsAnEmptyEventType(): void + { + $this->expectException(Invalid::class); + + $this->feed->append(''); + } + + public function testRejectsAPayloadThatCannotBeEncoded(): void + { + $this->expectException(Invalid::class); + + $this->feed->append('test', ['resource' => \fopen('php://memory', 'r')]); + } + + public function testPublishStampsAPreparedEvent(): void + { + $id = $this->feed->publish(new Event(id: 'ignored', type: 'test', data: ['a' => 'b'], subject: 's')); + + $event = $this->feed->read()[0]; + + $this->assertSame($id, $event->id); + $this->assertNotSame('ignored', $event->id, 'The backend assigns the position, not the caller'); + $this->assertSame('urn:appwrite:cloud:fra', $event->source); + $this->assertSame(['a' => 'b'], $event->data); + } + + public function testPublishKeepsATimeTheCallerSet(): void + { + $this->feed->publish(new Event(id: '', type: 'test', time: '2020-01-01T00:00:00.000Z')); + + $this->assertSame('2020-01-01T00:00:00.000Z', $this->feed->read()[0]->time); + } + + public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void + { + $this->feed->append('test'); + + $started = \microtime(true); + $events = $this->feed->poll(null, 10, 2000); + + $this->assertCount(1, $events); + $this->assertLessThan(1, \microtime(true) - $started); + } + + public function testPollGivesUpAtTheTimeoutWithAnEmptyBatch(): void + { + $started = \microtime(true); + $events = $this->feed->poll(null, 10, 600); + $elapsed = \microtime(true) - $started; + + $this->assertSame([], $events); + $this->assertGreaterThanOrEqual(0.4, $elapsed, 'Must actually wait'); + $this->assertLessThan(3.0, $elapsed, 'Must not wait far past the timeout'); + } + + public function testPollWithoutATimeoutIsAPlainRead(): void + { + $started = \microtime(true); + + $this->assertSame([], $this->feed->poll()); + $this->assertLessThan(0.4, \microtime(true) - $started); + } + + public function testRetentionIsBoundedAndTrimsTheOldest(): void + { + $feed = new Feed(new Memory('small', maxSize: 3)); + + foreach (['a', 'b', 'c', 'd', 'e'] as $type) { + $feed->append($type); + } + + $this->assertSame(['c', 'd', 'e'], \array_map(fn (Event $e): string => $e->type, $feed->read())); + } + + /** + * The one case a caller has to design for: a consumer that fell behind the + * trim horizon gets what is left, not an error and not a gap it can detect. + */ + public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void + { + $feed = new Feed($adapter = new Memory('small', maxSize: 2)); + + $first = $feed->append('a'); + $feed->append('b'); + $feed->append('c'); + + $this->assertSame(2, $adapter->count()); + $this->assertSame(['b', 'c'], \array_map(fn (Event $e): string => $e->type, $feed->read($first))); + } + + public function testExposesItsIdentity(): void + { + $this->assertSame('edge', $this->feed->getName()); + $this->assertSame('urn:appwrite:cloud:fra', $this->feed->getSource()); + $this->assertSame($this->adapter, $this->feed->getAdapter()); + } + + public function testAnUnconfiguredBackendFailsLoudlyRatherThanDroppingEvents(): void + { + $feed = new Feed(new None('edge')); + + $this->expectException(Unsupported::class); + + $feed->append('test'); + } + + public function testAnUnconfiguredBackendCannotBeRead(): void + { + $feed = new Feed(new None('edge')); + + $this->expectException(Unsupported::class); + + $feed->read(); + } + + public function testRejectsAnEmptyFeedName(): void + { + $this->expectException(Invalid::class); + + new Memory(''); + } + + public function testFlushingMemoryDoesNotReissuePositions(): void + { + $before = $this->feed->append('a'); + $this->adapter->flush(); + $after = $this->feed->append('b'); + + $this->assertSame(1, Id::compare($after, $before), 'A reissued position would make a consumer skip events'); + } +} diff --git a/tests/Feed/Unit/HttpAdapterTest.php b/tests/Feed/Unit/HttpAdapterTest.php new file mode 100644 index 0000000..43c6682 --- /dev/null +++ b/tests/Feed/Unit/HttpAdapterTest.php @@ -0,0 +1,223 @@ + $responses + * @return array{Feed, FakeTransport} + */ + private function feed(array $responses): array + { + $transport = new FakeTransport($responses); + $adapter = new Http(new Client($transport), 'https://cloud.example.com/v1/feeds', 'edge'); + + return [new Feed($adapter), $transport]; + } + + public function testReadsAFeedOverHttp(): void + { + [$feed] = $this->feed([FakeTransport::ok(Protocol::encode([ + new Event(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']]), + new Event(id: '1-1', type: 'io.appwrite.edge.invalidate'), + ]))]); + + $events = $feed->read(); + + $this->assertCount(2, $events); + $this->assertSame('1-0', $events[0]->id); + $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); + } + + public function testAppendsTheFeedNameToTheEndpoint(): void + { + [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + + $feed->read(); + + $this->assertStringStartsWith('https://cloud.example.com/v1/feeds/edge', $transport->lastRequest()['url']); + } + + public function testEncodesAFeedNameThatNeedsIt(): void + { + $adapter = new Http(new Client(new FakeTransport([])), 'https://cloud.example.com/v1/feeds/', 'a b/c'); + + $this->assertSame('https://cloud.example.com/v1/feeds/a%20b%2Fc', $adapter->getUrl()); + } + + public function testSendsThePositionAndLimit(): void + { + [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + + $feed->read('1-0', 250); + + $url = $transport->lastRequest()['url']; + + $this->assertStringContainsString('lastEventId=1-0', $url); + $this->assertStringContainsString('limit=250', $url); + } + + public function testSendsNoParametersOnAFirstFullRead(): void + { + [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + + $feed->read(null, Feed::MAX_BATCH); + + $this->assertStringNotContainsString('lastEventId', $transport->lastRequest()['url']); + } + + /** + * The producer does the waiting, so a poll is one request rather than a + * client-side loop. + */ + public function testDelegatesLongPollingToTheProducer(): void + { + [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + + $started = \microtime(true); + $feed->poll(null, 100, 5000); + + $this->assertLessThan(1, \microtime(true) - $started, 'Must not wait client-side'); + $this->assertCount(1, $transport->requests, 'Must not poll in a loop'); + $this->assertStringContainsString('timeout=5000', $transport->lastRequest()['url']); + } + + /** + * Without the margin the client's deadline races the producer's, and a + * poll that correctly waits out its timeout surfaces as a failure on every + * quiet tick. + */ + public function testAllowsTheClientLongerThanTheLongPollTimeout(): void + { + [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + + $feed->poll(null, 100, 5000); + + $this->assertSame(5000 + Protocol::TIMEOUT_MARGIN, $transport->lastRequest()['timeout']); + } + + public function testUsesTheClientDefaultTimeoutWhenNotLongPolling(): void + { + $transport = new FakeTransport([FakeTransport::ok([])]); + $client = (new Client($transport))->setTimeout(1234); + + (new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')))->read(); + + $this->assertSame(1234, $transport->lastRequest()['timeout']); + } + + /** + * The status is carried on the exception so a consumer can tell a producer + * that does not serve the feed yet β€” normal during a staged rollout β€” from + * one that is broken. + */ + public function testCarriesTheStatusOfARejectedRead(): void + { + [$feed] = $this->feed([FakeTransport::status(404)]); + + try { + $feed->read(); + $this->fail('A 404 should have been raised'); + } catch (Transport $error) { + $this->assertSame(404, $error->getCode()); + } + } + + public function testRaisesServerErrors(): void + { + [$feed] = $this->feed([FakeTransport::status(503)]); + + try { + $feed->read(); + $this->fail('A 503 should have been raised'); + } catch (Transport $error) { + $this->assertSame(503, $error->getCode()); + } + } + + public function testWrapsATransportFailure(): void + { + [$feed] = $this->feed([new \RuntimeException('Connection refused')]); + + $this->expectException(Transport::class); + $this->expectExceptionMessageMatches('/Connection refused/'); + + $feed->read(); + } + + public function testWrapsABodyThatIsNotJson(): void + { + [$feed] = $this->feed([FakeTransport::raw('502 Bad Gateway')]); + + $this->expectException(Transport::class); + + $feed->read(); + } + + public function testRejectsABodyThatIsNotABatch(): void + { + [$feed] = $this->feed([FakeTransport::raw('"a string"')]); + + $this->expectException(Invalid::class); + + $feed->read(); + } + + public function testCannotAppendToAFeedItDoesNotOwn(): void + { + [$feed] = $this->feed([FakeTransport::ok([])]); + + $this->expectException(Unsupported::class); + + $feed->append('io.appwrite.edge.invalidate'); + } + + /** + * The point of the adapter: a remote feed is consumed with exactly the + * code a local one is. + */ + public function testConsumesARemoteFeedThroughTheSameConsumer(): void + { + [$feed, $transport] = $this->feed([ + FakeTransport::ok(Protocol::encode([ + new Event(id: '1-0', type: 'a'), + new Event(id: '1-1', type: 'b'), + ])), + FakeTransport::ok(Protocol::encode([new Event(id: '1-2', type: 'c')])), + FakeTransport::ok(Protocol::encode([])), + ]); + + $cursor = new MemoryCursor('edge'); + $consumer = new Consumer($feed, 'invalidator', $cursor); + + $seen = []; + $handler = function (Event $event) use (&$seen): void { + $seen[] = $event->type; + }; + + $this->assertSame(2, $consumer->consume($handler)); + $this->assertSame('1-1', $cursor->load('invalidator')); + + $this->assertSame(1, $consumer->consume($handler)); + $this->assertSame(0, $consumer->consume($handler)); + + $this->assertSame(['a', 'b', 'c'], $seen); + $this->assertStringContainsString('lastEventId=1-1', $transport->requests[1]['url']); + } +} diff --git a/tests/Feed/Unit/IdTest.php b/tests/Feed/Unit/IdTest.php new file mode 100644 index 0000000..fa69b56 --- /dev/null +++ b/tests/Feed/Unit/IdTest.php @@ -0,0 +1,82 @@ + + */ + public static function ids(): array + { + return [ + 'well formed' => ['1690000000000-0', true], + 'high sequence' => ['1690000000000-42', true], + 'zero' => ['0-0', true], + 'empty' => ['', false], + 'no sequence' => ['1690000000000', false], + 'not numeric' => ['abc-0', false], + 'negative' => ['-1-0', false], + 'trailing dash' => ['1690000000000-', false], + 'exclusive syntax' => ['(1690000000000-0', false], + 'range token' => ['-', false], + ]; + } + + /** + * @dataProvider ids + */ + public function testValidatesIds(string $id, bool $valid): void + { + $this->assertSame($valid, Id::isValid($id)); + } + + public function testAfterIsTheNextSequenceInTheSameMillisecond(): void + { + $this->assertSame('1690000000000-1', Id::after('1690000000000-0')); + $this->assertSame('1690000000000-43', Id::after('1690000000000-42')); + } + + public function testAfterRejectsAnIdThatIsNotAPosition(): void + { + $this->expectException(Invalid::class); + + Id::after('not-an-id'); + } + + public function testDecodeSplitsAnIdIntoItsParts(): void + { + $this->assertSame([1690000000000, 7], Id::decode('1690000000000-7')); + } + + public function testEncodeAndDecodeAreInverses(): void + { + $this->assertSame([12, 34], Id::decode(Id::encode(12, 34))); + } + + /** + * The reason ids are compared by parts rather than as strings: `10-0` + * sorts before `9-0` lexically, which would make a consumer treat a newer + * event as one it had already passed. + */ + public function testComparesNumericallyNotLexically(): void + { + $this->assertSame(1, Id::compare('10-0', '9-0')); + $this->assertSame(-1, Id::compare('9-0', '10-0')); + $this->assertSame(0, Id::compare('10-0', '10-0')); + $this->assertSame(1, Id::compare('10-2', '10-1')); + } + + public function testCompareRejectsAnIdThatIsNotAPosition(): void + { + $this->expectException(Invalid::class); + + Id::compare('1-0', 'nope'); + } +} diff --git a/tests/Feed/Unit/ProtocolTest.php b/tests/Feed/Unit/ProtocolTest.php new file mode 100644 index 0000000..3173f36 --- /dev/null +++ b/tests/Feed/Unit/ProtocolTest.php @@ -0,0 +1,150 @@ +assertSame([], Protocol::query()); + $this->assertSame([], Protocol::query(null, 0, 0)); + $this->assertSame([], Protocol::query('', 0, 0)); + } + + public function testQueryCarriesTheParametersThatWereSet(): void + { + $this->assertSame([ + 'lastEventId' => '1-0', + 'limit' => 500, + 'timeout' => 20000, + ], Protocol::query('1-0', 500, 20000)); + } + + public function testEncodesABatch(): void + { + $payload = Protocol::encode([ + new Event(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), + new Event(id: '1-1', type: 'b'), + ]); + + $this->assertSame(2, $payload['total']); + $this->assertCount(2, $payload['events']); + $this->assertSame('1-0', $payload['events'][0]['id']); + $this->assertSame(['x' => 1], $payload['events'][0]['data']); + $this->assertSame('1.0', $payload['events'][0]['specversion']); + } + + public function testEncodesAnEmptyBatch(): void + { + $this->assertSame(['total' => 0, 'events' => []], Protocol::encode([])); + } + + public function testDecodesWhatItEncoded(): void + { + $events = [ + new Event(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), + new Event(id: '1-1', type: 'b'), + ]; + + $this->assertEquals($events, Protocol::decode(Protocol::encode($events))); + } + + public function testDecodesAnEmptyBatch(): void + { + $this->assertSame([], Protocol::decode(['total' => 0, 'events' => []])); + $this->assertSame([], Protocol::decode([])); + } + + public function testRejectsAPayloadThatIsNotABatch(): void + { + $this->expectException(Invalid::class); + + Protocol::decode('not a batch'); + } + + public function testRejectsAMalformedEventsField(): void + { + $this->expectException(Invalid::class); + + Protocol::decode(['events' => 'nope']); + } + + /** + * An event with no id has no position, so a consumer cannot record having + * passed it. Returning the usable prefix lets those events be handled and + * the position advance to the last of them; the broken event is then at + * the head of the next batch, where it stops the feed loudly. + */ + public function testKeepsTheEventsBeforeAnUndecodableOne(): void + { + $events = Protocol::decode([ + 'events' => [ + ['id' => '1-0', 'type' => 'a'], + ['id' => '1-1', 'type' => 'b'], + ['type' => 'no id'], + ['id' => '1-3', 'type' => 'd'], + ], + ]); + + $this->assertCount(2, $events); + $this->assertSame(['a', 'b'], \array_map(fn (Event $e): string => $e->type, $events)); + } + + public function testFailsWhenTheFirstEventIsUndecodable(): void + { + $this->expectException(Invalid::class); + + Protocol::decode(['events' => [['type' => 'no id'], ['id' => '1-1', 'type' => 'b']]]); + } + + public function testFailsWhenTheFirstEntryIsNotAnEvent(): void + { + $this->expectException(Invalid::class); + + Protocol::decode(['events' => ['a string']]); + } + + public function testKeepsTheEventsBeforeAnEntryThatIsNotAnEvent(): void + { + $events = Protocol::decode(['events' => [['id' => '1-0', 'type' => 'a'], 'a string']]); + + $this->assertCount(1, $events); + } + + /** + * A full batch is settled history, so it may be cached forever. Anything + * short is the live end of the feed and will grow. + */ + public function testAFullBatchIsCacheable(): void + { + $this->assertSame('private, max-age=31536000', Protocol::cacheControl(100, 100)); + } + + public function testAPartialBatchIsNotCacheable(): void + { + $this->assertSame('no-store', Protocol::cacheControl(99, 100)); + $this->assertSame('no-store', Protocol::cacheControl(0, 100)); + } + + /** + * A batch of zero out of zero is not history β€” it is a caught-up consumer, + * and caching it would pin the consumer at that position forever. + */ + public function testAnEmptyBatchIsNeverCacheable(): void + { + $this->assertSame('no-store', Protocol::cacheControl(0, 0)); + } + + public function testSharedCachingIsOptIn(): void + { + $this->assertStringStartsWith('private, ', Protocol::cacheControl(10, 10)); + $this->assertStringStartsWith('public, ', Protocol::cacheControl(10, 10, public: true)); + } +} diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php new file mode 100644 index 0000000..d36254b --- /dev/null +++ b/tests/Feed/Unit/RoundTripTest.php @@ -0,0 +1,212 @@ +producer = new Feed(new MemoryAdapter('edge'), 'urn:appwrite:cloud:fra'); + $this->server = new FeedServer($this->producer); + + $this->consumerFeed = new Feed( + new Http(new Client($this->server), 'https://cloud.example.com/v1/feeds', 'edge') + ); + + $this->cursor = new CacheCursor(new UtopiaCache(new CacheMemory()), 'edge'); + } + + private function consumer(string $name = 'invalidator', int $batch = Consumer::BATCH): Consumer + { + return new Consumer($this->consumerFeed, $name, $this->cursor, $batch); + } + + public function testAnEventSurvivesTheWholeTrip(): void + { + $this->producer->append( + 'io.appwrite.edge.invalidate-rule', + ['tags' => ['domain' => 'example.com'], 'isAppwriteNetwork' => true], + 'example.com', + ); + + $received = null; + $this->consumer()->consume(function (Event $event) use (&$received): void { + $received = $event; + }); + + $this->assertInstanceOf(Event::class, $received); + $this->assertSame('io.appwrite.edge.invalidate-rule', $received->type); + $this->assertSame('urn:appwrite:cloud:fra', $received->source); + $this->assertSame('example.com', $received->subject); + $this->assertSame(['domain' => 'example.com'], $received->getData('tags')); + $this->assertTrue($received->getData('isAppwriteNetwork')); + } + + public function testTheConsumerOnlyEverSeesEachEventOnce(): void + { + foreach (\range(1, 5) as $i) { + $this->producer->append('event-' . $i); + } + + $consumer = $this->consumer(); + $seen = []; + $handler = function (Event $event) use (&$seen): void { + $seen[] = $event->type; + }; + + $consumer->consume($handler); + $consumer->consume($handler); + + $this->producer->append('event-6'); + $consumer->consume($handler); + + $this->assertSame( + ['event-1', 'event-2', 'event-3', 'event-4', 'event-5', 'event-6'], + $seen, + ); + } + + /** + * The rollout property: a consumer shipped after the producer catches up on + * everything that accumulated in between, rather than starting at the tip. + */ + public function testAConsumerShippedLateDrainsTheBacklog(): void + { + foreach (\range(1, 3) as $i) { + $this->producer->append('missed-' . $i); + } + + $seen = []; + $handled = $this->consumer()->consume(function (Event $event) use (&$seen): void { + $seen[] = $event->type; + }); + + $this->assertSame(3, $handled); + $this->assertSame(['missed-1', 'missed-2', 'missed-3'], $seen); + } + + /** + * The restart property: a new Consumer with no in-memory state picks the + * stored position up rather than replaying. + */ + public function testARestartedConsumerResumesWhereItLeftOff(): void + { + $this->producer->append('a'); + $this->producer->append('b'); + + $this->consumer()->consume(fn (Event $event) => null); + + $this->producer->append('c'); + + $seen = []; + $this->consumer()->consume(function (Event $event) use (&$seen): void { + $seen[] = $event->type; + }); + + $this->assertSame(['c'], $seen); + } + + public function testAFailedEventBlocksTheOnesBehindItUntilItSucceeds(): void + { + $this->producer->append('a'); + $this->producer->append('poison'); + $this->producer->append('c'); + + $consumer = $this->consumer(); + $seen = []; + $attempts = 0; + + // Fails the first time it sees the poison event and succeeds after, + // standing in for a dependency that was briefly unavailable. + $handler = function (Event $event) use (&$seen, &$attempts): void { + if ($event->type === 'poison') { + $attempts++; + + if ($attempts === 1) { + throw new \RuntimeException('not yet'); + } + } + + $seen[] = $event->type; + }; + + try { + $consumer->consume($handler); + $this->fail('The handler failure should have been re-raised'); + } catch (\RuntimeException) { + // Expected. + } + + $this->assertSame(['a'], $seen, 'Nothing behind the failure is delivered'); + + $consumer->consume($handler); + + $this->assertSame(['a', 'poison', 'c'], $seen, 'Order is preserved and nothing is skipped'); + $this->assertSame(2, $attempts, 'The failed event is retried, not dropped'); + } + + public function testTheProducerCachesFullBatchesAndNothingElse(): void + { + foreach (\range(1, 5) as $i) { + $this->producer->append('event-' . $i); + } + + $consumer = $this->consumer(batch: 2); + + $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (Event $event) => null); + + $this->assertSame([ + 'private, max-age=31536000', // 2 of 2 β€” settled history + 'private, max-age=31536000', // 2 of 2 β€” settled history + 'no-store', // 1 of 2 β€” the live end, will grow + 'no-store', // 0 of 2 β€” caught up + ], $this->server->cacheControl); + } + + public function testTwoConsumersOfOneProducerAreIndependent(): void + { + $this->producer->append('a'); + + $one = $this->consumer('one'); + $two = $this->consumer('two'); + + $this->assertSame(1, $one->consume(fn (Event $event) => null)); + + $this->producer->append('b'); + + $this->assertSame(2, $two->consume(fn (Event $event) => null), 'The second consumer starts from the beginning'); + $this->assertSame(1, $one->consume(fn (Event $event) => null), 'The first only sees what is new to it'); + } +} diff --git a/tests/Feed/Unit/Support/FailingCursor.php b/tests/Feed/Unit/Support/FailingCursor.php new file mode 100644 index 0000000..7a3f450 --- /dev/null +++ b/tests/Feed/Unit/Support/FailingCursor.php @@ -0,0 +1,40 @@ +onLoad) { + throw new Transport('Cursor store is unavailable'); + } + + return parent::load($consumer); + } + + public function save(string $consumer, string $eventId): void + { + if ($this->onSave) { + throw new Transport('Cursor store is unavailable'); + } + + parent::save($consumer, $eventId); + } +} diff --git a/tests/Feed/Unit/Support/FakeTransport.php b/tests/Feed/Unit/Support/FakeTransport.php new file mode 100644 index 0000000..55575c3 --- /dev/null +++ b/tests/Feed/Unit/Support/FakeTransport.php @@ -0,0 +1,87 @@ +, timeout: int}> */ + public array $requests = []; + + /** @var list */ + private array $responses; + + /** + * @param list $responses Answered in order; the last + * one repeats once the script runs out. + */ + public function __construct(array $responses) + { + $this->responses = $responses; + } + + /** + * @param array $body + */ + public static function ok(array $body, int $statusCode = 200): Response + { + return new Response($statusCode, (string) \json_encode($body), []); + } + + public static function status(int $statusCode, string $body = '{}'): Response + { + return new Response($statusCode, $body, []); + } + + public static function raw(string $body): Response + { + return new Response(200, $body, []); + } + + public function send( + string $url, + string $method, + mixed $body, + array $headers, + RequestOptions $options, + ?callable $chunkCallback = null + ): Response { + $this->requests[] = [ + 'url' => $url, + 'method' => $method, + 'headers' => $headers, + 'timeout' => $options->getTimeout(), + ]; + + $response = \count($this->responses) > 1 ? \array_shift($this->responses) : ($this->responses[0] ?? null); + + if ($response instanceof \Throwable) { + throw $response; + } + + return $response ?? self::ok(['total' => 0, 'events' => []]); + } + + /** + * @return array{url: string, method: string, headers: array, timeout: int} + */ + public function lastRequest(): array + { + $request = \end($this->requests); + + if ($request === false) { + throw new \RuntimeException('No request was made'); + } + + return $request; + } +} diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php new file mode 100644 index 0000000..9ad9c9a --- /dev/null +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -0,0 +1,58 @@ + */ + public array $cacheControl = []; + + public function __construct(private readonly Feed $feed) + { + } + + public function send( + string $url, + string $method, + mixed $body, + array $headers, + RequestOptions $options, + ?callable $chunkCallback = null + ): Response { + $query = []; + \parse_str((string) \parse_url($url, PHP_URL_QUERY), $query); + + $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; + $limit = (int) ($query[Protocol::PARAM_LIMIT] ?? Feed::MAX_BATCH); + $timeout = (int) ($query[Protocol::PARAM_TIMEOUT] ?? 0); + + $events = $this->feed->poll( + \is_string($lastEventId) && $lastEventId !== '' ? $lastEventId : null, + $limit, + $timeout, + ); + + $this->cacheControl[] = $cacheControl = Protocol::cacheControl(\count($events), $limit); + + return new Response( + 200, + (string) \json_encode(Protocol::encode($events)), + ['cache-control' => $cacheControl], + ); + } +} From 20cf3cb1809365d098b355d6e23832d7a33ceca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 15:19:08 +0200 Subject: [PATCH 02/68] Switch fetch library to client --- .github/workflows/analysis.yml | 1 - .github/workflows/tests.yml | 10 +- CHANGELOG.md | 8 +- Dockerfile | 37 ++++++ Dockerfile.php-8.3 | 25 ---- Dockerfile.php-8.4 | 25 ---- Dockerfile.php-8.5 | 25 ---- README.md | 30 ++++- composer.json | 9 +- composer.lock | 137 +++++++++++++++++++--- docker-compose.yml | 10 +- docs/migration.md | 4 +- src/Feed/Adapter/Http.php | 83 +++++++++---- tests/Feed/Unit/HttpAdapterTest.php | 119 +++++++++++++------ tests/Feed/Unit/RoundTripTest.php | 5 +- tests/Feed/Unit/Support/FakeClient.php | 103 ++++++++++++++++ tests/Feed/Unit/Support/FakeTransport.php | 89 ++++++-------- tests/Feed/Unit/Support/FeedServer.php | 41 +++---- tests/Feed/Unit/Support/Recorder.php | 59 ++++++++++ 19 files changed, 578 insertions(+), 242 deletions(-) create mode 100644 Dockerfile delete mode 100644 Dockerfile.php-8.3 delete mode 100644 Dockerfile.php-8.4 delete mode 100644 Dockerfile.php-8.5 create mode 100644 tests/Feed/Unit/Support/FakeClient.php create mode 100644 tests/Feed/Unit/Support/Recorder.php diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index e57c462..0c6399e 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -15,7 +15,6 @@ jobs: # Redis stream calls only resolve where ext-redis is actually installed. - name: Build run: | - export PHP_VERSION=8.4 docker compose build tests docker compose up -d tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 461b5f8..ee80c4c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,12 +4,17 @@ on: [pull_request] jobs: tests: - name: Tests + name: "PHP ${{ matrix.php-versions }}" runs-on: ubuntu-latest strategy: fail-fast: false matrix: - php-versions: ['8.3', '8.4', '8.5'] + # Add a PHP version here and nowhere else β€” the image is built from one + # parameterized Dockerfile. + php-versions: ['8.5'] + + env: + PHP_VERSION: ${{ matrix.php-versions }} steps: - name: Checkout repository @@ -17,7 +22,6 @@ jobs: - name: Build run: | - export PHP_VERSION=${{ matrix.php-versions }} docker compose build docker compose up -d diff --git a/CHANGELOG.md b/CHANGELOG.md index 08eaf70..67d6179 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,15 @@ library. - `Feed` β€” append, read and long-poll an ordered event log - `Event` β€” a CloudEvent, with a strict decode that a feed's ids can be paged from - `Adapter\Redis`, `Adapter\Pool` β€” Redis streams, directly or over a pool -- `Adapter\Http` β€” another service's feed, read over the wire +- `Adapter\Http` β€” another service's feed, read over the wire with + [utopia-php/client](https://github.com/utopia-php/client); takes any of its + adapters, so a pooled or Swoole coroutine transport drops straight in - `Adapter\Memory`, `Adapter\None` β€” for tests, and for no backend configured - `Consumer` β€” the pull loop, with at-least-once semantics and a durable position - `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory` β€” where that position lives - `Protocol` β€” the http-feeds wire contract, shared by producer and consumer - `Id` β€” feed positions, and the arithmetic for paging past one + +Requires PHP 8.5. CI builds one parameterized image per version in the +`php-versions` matrix of `.github/workflows/tests.yml`, which is the only place +versions are listed. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b02856f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# One image for every PHP version the library is tested against. Adding a +# version is a single entry in the `php-versions` matrix in +# .github/workflows/tests.yml β€” there is nothing to add here. +# +# PHP_VERSION=8.6 docker compose build +ARG PHP_VERSION=8.5 + +FROM composer:2.7 AS vendor + +WORKDIR /src/ + +COPY composer.lock composer.json /src/ + +RUN composer install --ignore-platform-reqs --optimize-autoloader \ + --no-plugins --no-scripts --prefer-dist + +FROM appwrite/utopia-base:php-${PHP_VERSION}-1.0.0 AS final + +LABEL maintainer="team@appwrite.io" + +WORKDIR /code + +COPY --from=vendor /src/vendor /code/vendor + +# Composer itself, for the `test`, `check` and `lint` scripts. The base image +# ships PHP but not composer. +COPY --from=composer:2.7 /usr/bin/composer /usr/bin/composer + +COPY ./composer.json /code/composer.json +COPY ./composer.lock /code/composer.lock +COPY ./phpunit.xml /code/phpunit.xml +COPY ./phpstan.neon /code/phpstan.neon +COPY ./pint.json /code/pint.json +COPY ./src /code/src +COPY ./tests /code/tests + +CMD [ "tail", "-f", "/dev/null" ] diff --git a/Dockerfile.php-8.3 b/Dockerfile.php-8.3 deleted file mode 100644 index 0493147..0000000 --- a/Dockerfile.php-8.3 +++ /dev/null @@ -1,25 +0,0 @@ -FROM composer:2.7 as step0 - -WORKDIR /src/ - -COPY composer.lock /src/ -COPY composer.json /src/ - -RUN composer install --ignore-platform-reqs --optimize-autoloader \ - --no-plugins --no-scripts --prefer-dist - -FROM appwrite/utopia-base:php-8.3-1.0.0 as final - -LABEL maintainer="team@appwrite.io" - -WORKDIR /code - -COPY --from=step0 /src/vendor /code/vendor - -# Add Source Code -COPY ./src /code/src -COPY ./tests /code/tests -COPY ./phpunit.xml /code/phpunit.xml -COPY ./phpstan.neon /code/phpstan.neon - -CMD [ "tail", "-f", "/dev/null" ] diff --git a/Dockerfile.php-8.4 b/Dockerfile.php-8.4 deleted file mode 100644 index 08df930..0000000 --- a/Dockerfile.php-8.4 +++ /dev/null @@ -1,25 +0,0 @@ -FROM composer:2.7 as step0 - -WORKDIR /src/ - -COPY composer.lock /src/ -COPY composer.json /src/ - -RUN composer install --ignore-platform-reqs --optimize-autoloader \ - --no-plugins --no-scripts --prefer-dist - -FROM appwrite/utopia-base:php-8.4-1.0.0 as final - -LABEL maintainer="team@appwrite.io" - -WORKDIR /code - -COPY --from=step0 /src/vendor /code/vendor - -# Add Source Code -COPY ./src /code/src -COPY ./tests /code/tests -COPY ./phpunit.xml /code/phpunit.xml -COPY ./phpstan.neon /code/phpstan.neon - -CMD [ "tail", "-f", "/dev/null" ] diff --git a/Dockerfile.php-8.5 b/Dockerfile.php-8.5 deleted file mode 100644 index e582fd7..0000000 --- a/Dockerfile.php-8.5 +++ /dev/null @@ -1,25 +0,0 @@ -FROM composer:2.7 as step0 - -WORKDIR /src/ - -COPY composer.lock /src/ -COPY composer.json /src/ - -RUN composer install --ignore-platform-reqs --optimize-autoloader \ - --no-plugins --no-scripts --prefer-dist - -FROM appwrite/utopia-base:php-8.5-1.0.0 as final - -LABEL maintainer="team@appwrite.io" - -WORKDIR /code - -COPY --from=step0 /src/vendor /code/vendor - -# Add Source Code -COPY ./src /code/src -COPY ./tests /code/tests -COPY ./phpunit.xml /code/phpunit.xml -COPY ./phpstan.neon /code/phpstan.neon - -CMD [ "tail", "-f", "/dev/null" ] diff --git a/README.md b/README.md index 0152c47..8d50529 100644 --- a/README.md +++ b/README.md @@ -163,10 +163,13 @@ $handled = $consumer->consume(function (Event $event) use ($router) { Consuming **another service's** feed is the same code with a different adapter: ```php +use Utopia\Client; +use Utopia\Client\Adapter\Curl\Client as Curl; use Utopia\Feed\Adapter\Http; -use Utopia\Fetch\Client; -$client = (new Client())->addHeader('x-appwrite-jwt', $token); +$client = (new Client(new Curl())) + ->withHeaders(['x-appwrite-jwt' => $token]) + ->withConnectionReuse(); $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); ``` @@ -175,6 +178,12 @@ Nothing above the adapter knows the events are arriving over the network, including the long polling β€” `Http` hands the wait to the producer, so a poll is one held request rather than a client-side loop. +`Http` takes any [`utopia-php/client`](https://github.com/utopia-php/client) +adapter, so a `Pool` or a Swoole coroutine transport drops straight in. Leave the +`Retry` decorator off, though: a failed read leaves the cursor where it was, so +the next poll is already the retry, and retrying inside a long poll only +multiplies how long a single tick can take. + Call `consume()` on a timer, or give the consumer a `timeout` and loop: ```php @@ -192,7 +201,7 @@ while (true) { | --- | --- | --- | --- | | `Adapter\Redis` | Producing a feed on a Redis stream | βœ… | βœ… | | `Adapter\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | βœ… | -| `Adapter\Http` | Consuming another service's feed | ❌ | βœ… | +| `Adapter\Http` | Consuming another service's feed, over [utopia-php/client](https://github.com/utopia-php/client) | ❌ | βœ… | | `Adapter\Memory` | Tests, and single-process development | βœ… | βœ… | | `Adapter\None` | No backend configured | ❌ | ❌ | @@ -304,10 +313,21 @@ Static analysis runs at PHPStan level max. Run it inside the container, where docker compose exec tests composer check ``` +The image is built from one parameterized `Dockerfile`, so testing against +another PHP version needs no new file: + +```bash +PHP_VERSION=8.6 docker compose build +PHP_VERSION=8.6 docker compose up -d +``` + +To add that version to CI, add it to the `php-versions` matrix in +`.github/workflows/tests.yml` β€” that is the only place versions are listed. + ## System requirements -Utopia Framework requires PHP 8.3 or later. We recommend using the latest PHP -version whenever possible. +Utopia Feed requires PHP 8.5 or later. We recommend using the latest PHP version +whenever possible. ## Copyright and license diff --git a/composer.json b/composer.json index e8efe02..14faa85 100644 --- a/composer.json +++ b/composer.json @@ -33,12 +33,15 @@ } }, "require": { - "php": ">=8.3", + "php": ">=8.5", "ext-json": "*", "ext-redis": "*", + "psr/http-client": "^1.0", + "psr/http-message": "^2.0", "utopia-php/cache": "3.*", - "utopia-php/fetch": "^1.1", - "utopia-php/pools": "1.*" + "utopia-php/client": "^0.2", + "utopia-php/pools": "1.*", + "utopia-php/psr7": "^0.2" }, "require-dev": { "laravel/pint": "^1.21", diff --git a/composer.lock b/composer.lock index fa56c6b..c367587 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4966f1f4c743766e6702072707dbf2b2", + "content-hash": "52ee952ff068b03efb088193a317a726", "packages": [ { "name": "brick/math", @@ -1994,44 +1994,61 @@ "time": "2026-05-29T12:12:23+00:00" }, { - "name": "utopia-php/fetch", - "version": "1.1.2", + "name": "utopia-php/client", + "version": "0.2.3", "source": { "type": "git", - "url": "https://github.com/utopia-php/fetch.git", - "reference": "64f2b3a789480f1deb102ce684dac4217d8e98d5" + "url": "https://github.com/utopia-php/client.git", + "reference": "377dc1f79441ac1584f25dba57904ee1cf0f626b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/64f2b3a789480f1deb102ce684dac4217d8e98d5", - "reference": "64f2b3a789480f1deb102ce684dac4217d8e98d5", + "url": "https://api.github.com/repos/utopia-php/client/zipball/377dc1f79441ac1584f25dba57904ee1cf0f626b", + "reference": "377dc1f79441ac1584f25dba57904ee1cf0f626b", "shasum": "" }, "require": { - "php": ">=8.1" + "php": ">=8.5", + "psr/http-client": "^1.0", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.1 || ^2.0", + "utopia-php/pools": "^1.0", + "utopia-php/psr7": "^0.2", + "utopia-php/span": "^1.1 || ^3.0 || ^4.0" }, "require-dev": { - "laravel/pint": "^1.5.0", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^9.5", "swoole/ide-helper": "^6.0" }, + "suggest": { + "ext-curl": "Required to use the cURL HTTP client adapter.", + "ext-simplexml": "Required to decode XML responses with Response::xml().", + "ext-swoole": "Required to use the Swoole coroutine HTTP client adapter." + }, "type": "library", "autoload": { "psr-4": { - "Utopia\\Fetch\\": "src/" + "Utopia\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], - "description": "A simple library that provides an interface for making HTTP Requests.", + "description": "A lightweight PSR-18 HTTP client with cURL and Swoole coroutine backends", + "keywords": [ + "client", + "curl", + "http", + "php", + "psr-18", + "swoole", + "utopia" + ], "support": { - "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/1.1.2" + "issues": "https://github.com/utopia-php/client/issues", + "source": "https://github.com/utopia-php/client/tree/0.2.3" }, - "time": "2026-04-29T11:19:19+00:00" + "time": "2026-07-13T15:29:09+00:00" }, { "name": "utopia-php/pools", @@ -2089,6 +2106,92 @@ }, "time": "2026-07-27T15:27:26+00:00" }, + { + "name": "utopia-php/psr7", + "version": "0.2.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/psr7.git", + "reference": "115753c36194d53abe5587d383810724ac3a782d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/psr7/zipball/115753c36194d53abe5587d383810724ac3a782d", + "reference": "115753c36194d53abe5587d383810724ac3a782d", + "shasum": "" + }, + "require": { + "php": ">=8.4", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.1 || ^2.0" + }, + "suggest": { + "ext-simplexml": "Required to decode XML responses with Response::xml()." + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Psr7\\": "src/Psr7/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PSR-7 HTTP message implementations and PSR-17 factories for Utopia", + "keywords": [ + "http", + "php", + "psr-17", + "psr-7", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/psr7/issues", + "source": "https://github.com/utopia-php/psr7/tree/0.2.0" + }, + "time": "2026-07-06T12:40:23+00:00" + }, + { + "name": "utopia-php/span", + "version": "4.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/span.git", + "reference": "7969d920044c1e106d9eede414e2d6e8df206ff5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/span/zipball/7969d920044c1e106d9eede414e2d6e8df206ff5", + "reference": "7969d920044c1e106d9eede414e2d6e8df206ff5", + "shasum": "" + }, + "require": { + "php": ">=8.3" + }, + "require-dev": { + "swoole/ide-helper": "^5.0" + }, + "suggest": { + "ext-swoole": "Required for coroutine-based storage" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Span\\": "src/Span/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Simple span tracing library for PHP with coroutine support", + "support": { + "issues": "https://github.com/utopia-php/span/issues", + "source": "https://github.com/utopia-php/span/tree/4.1.0" + }, + "time": "2026-07-24T08:46:29+00:00" + }, { "name": "utopia-php/telemetry", "version": "0.4.5", @@ -4094,7 +4197,7 @@ "prefer-stable": false, "prefer-lowest": false, "platform": { - "php": ">=8.3", + "php": ">=8.5", "ext-json": "*", "ext-redis": "*" }, diff --git a/docker-compose.yml b/docker-compose.yml index b4c0d1e..aadde99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,17 @@ name: feed +# Pick the PHP version with an environment variable rather than a per-version +# file, so testing against a new release is `PHP_VERSION=8.6 docker compose up` +# and adding it to CI is one line in .github/workflows/tests.yml. services: tests: container_name: feed-tests - image: feed-tests:${PHP_VERSION:-8.4} + image: feed-tests:${PHP_VERSION:-8.5} build: context: . - dockerfile: Dockerfile.php-${PHP_VERSION:-8.4} + dockerfile: Dockerfile + args: + PHP_VERSION: ${PHP_VERSION:-8.5} environment: - REDIS_HOST=redis - REDIS_PORT=6379 @@ -15,6 +20,7 @@ services: volumes: - ./phpunit.xml:/code/phpunit.xml - ./phpstan.neon:/code/phpstan.neon + - ./pint.json:/code/pint.json - ./src:/code/src - ./tests:/code/tests depends_on: diff --git a/docs/migration.md b/docs/migration.md index 8c8ad93..9df39ee 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -116,7 +116,7 @@ $domain = \is_array($tags) ? ($tags['domain'] ?? '') : ''; | `Feed\Cursor` | `Cursor\Cache` | | `Feed\Event` | `Event` | | `Feed\Event::FEED` and the type constants | Stay β€” they name cloud's feed and its events | -| `Manager::fetchFeed()` | `Adapter\Http` | +| `Manager::fetchFeed()` | `Adapter\Http`, over `utopia-php/client` | | `Consumer::TIMEOUT_MARGIN` | `Protocol::TIMEOUT_MARGIN` | | `Feed\Poller` | Stays β€” Swoole interval scheduling | | `Router\Invalidator` | Stays β€” it purges edge caches | @@ -125,7 +125,7 @@ The whole of `Feed\Consumer`, `Feed\Cursor` and `Feed\Event` is replaced by construction: ```php -$client = (new Client())->addHeader('x-appwrite-jwt', $token); +$client = (new Client(new Curl()))->withHeaders(['x-appwrite-jwt' => $token]); $consumer = new Consumer( feed: new Feed(new Http($client, $endpoint . '/manager/feeds', 'edge')), diff --git a/src/Feed/Adapter/Http.php b/src/Feed/Adapter/Http.php index a658f0d..8429973 100644 --- a/src/Feed/Adapter/Http.php +++ b/src/Feed/Adapter/Http.php @@ -4,13 +4,18 @@ namespace Utopia\Feed\Adapter; +use Psr\Http\Client\ClientExceptionInterface; +use Utopia\Client\Adapter as ClientAdapter; use Utopia\Feed\Adapter; use Utopia\Feed\Event; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; use Utopia\Feed\Protocol; -use Utopia\Fetch\Client; +use Utopia\Psr7\ContentType; +use Utopia\Psr7\Header; +use Utopia\Psr7\Method; +use Utopia\Psr7\Request\Factory as RequestFactory; /** * Someone else's feed, read over HTTP. @@ -26,20 +31,32 @@ * producer answers the moment an event exists. * * ```php - * $client = (new Client()) - * ->addHeader('x-appwrite-jwt', $token) - * ->setMaxRetries(0); // The consumer's own retry is the next poll + * use Utopia\Client; + * use Utopia\Client\Adapter\Curl\Client as Curl; + * + * $client = (new Client(new Curl())) + * ->withHeaders(['x-appwrite-jwt' => $token]) + * ->withConnectionReuse(); * * $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); * ``` + * + * @see https://github.com/utopia-php/client */ class Http extends Adapter { + private readonly RequestFactory $requests; + /** - * @param Client $client Configured with whatever credentials the producer - * requires. Retries are best left off: a failed read leaves the - * cursor where it was, so the next poll is already the retry, and - * retrying inside a long poll multiplies the time a tick can take. + * @param ClientAdapter $client Configured with whatever credentials the + * producer requires. Typed as the client's own adapter interface + * rather than plain PSR-18, because a read needs to set its own + * deadline β€” which also means a `Retry` or `Pool` decorator can be + * passed here, since those implement it too. + * + * Retries are best left off. A failed read leaves the cursor where + * it was, so the next poll is already the retry; retrying inside a + * long poll only multiplies how long a single tick can take. * @param string $endpoint Base URL the producer serves its feeds under. * The feed name is appended to it, so * `https://cloud.example.com/v1/feeds` reads @@ -47,11 +64,14 @@ class Http extends Adapter * @param string $name Feed name, as the producer knows it. */ public function __construct( - protected readonly Client $client, + protected readonly ClientAdapter $client, protected readonly string $endpoint, string $name, + ?RequestFactory $requests = null, ) { parent::__construct($name); + + $this->requests = $requests ?? new RequestFactory(); } /** @@ -75,17 +95,19 @@ public function read(?string $lastEventId, int $limit, int $timeout = 0): array { $url = $this->getUrl(); + $request = $this->requests->query( + Method::GET, + $url, + Protocol::query($lastEventId, $limit, $timeout), + [Header::ACCEPT => ContentType::JSON], + ); + try { - $response = $this->client->fetch( - url: $url, - method: Client::METHOD_GET, - query: Protocol::query($lastEventId, $limit, $timeout), - // The producer is expected to answer within its own timeout; - // the margin only stops the client cutting off a poll that is - // legitimately waiting one out. - timeoutMs: $timeout > 0 ? $timeout + Protocol::TIMEOUT_MARGIN : null, - ); - } catch (\Throwable $error) { + $response = $this->client($timeout)->sendRequest($request); + } catch (ClientExceptionInterface $error) { + // PSR-18 reserves exceptions for failures that produced no usable + // response, so anything landing here is a transport problem rather + // than something the producer said. throw new Transport("Failed to read the {$this->name} feed at {$url}: {$error->getMessage()}", previous: $error); } @@ -103,8 +125,8 @@ public function read(?string $lastEventId, int $limit, int $timeout = 0): array } try { - $body = $response->json(); - } catch (\Throwable $error) { + $body = \json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + } catch (\JsonException $error) { throw new Transport("The {$this->name} feed at {$url} returned a body that is not JSON: {$error->getMessage()}", previous: $error); } @@ -118,4 +140,23 @@ public function pollable(): bool { return true; } + + /** + * The client to read with, given how long the producer has been asked to + * hold the request. + * + * A long poll needs a deadline past the one it asked for. Without the + * margin the client's deadline races the producer's, and a poll that + * correctly waits out its full timeout gets cancelled a hair early and + * surfaces as a transport failure on every quiet tick β€” burying the + * failures that matter. A plain read keeps whatever the caller configured. + */ + private function client(int $timeout): ClientAdapter + { + if ($timeout <= 0) { + return $this->client; + } + + return $this->client->withTimeout(($timeout + Protocol::TIMEOUT_MARGIN) / 1000); + } } diff --git a/tests/Feed/Unit/HttpAdapterTest.php b/tests/Feed/Unit/HttpAdapterTest.php index 43c6682..a8280d0 100644 --- a/tests/Feed/Unit/HttpAdapterTest.php +++ b/tests/Feed/Unit/HttpAdapterTest.php @@ -5,6 +5,8 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; +use Psr\Http\Message\ResponseInterface; +use Utopia\Client; use Utopia\Feed\Adapter\Http; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Memory as MemoryCursor; @@ -14,26 +16,25 @@ use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; use Utopia\Feed\Protocol; -use Utopia\Fetch\Client; use Utopia\Tests\Unit\Support\FakeTransport; class HttpAdapterTest extends TestCase { /** - * @param list<\Utopia\Fetch\Response|\Throwable> $responses + * @param list $responses * @return array{Feed, FakeTransport} */ - private function feed(array $responses): array + private function feed(array $responses = []): array { - $transport = new FakeTransport($responses); - $adapter = new Http(new Client($transport), 'https://cloud.example.com/v1/feeds', 'edge'); + $transport = FakeTransport::of($responses); + $adapter = new Http($transport, 'https://cloud.example.com/v1/feeds', 'edge'); return [new Feed($adapter), $transport]; } public function testReadsAFeedOverHttp(): void { - [$feed] = $this->feed([FakeTransport::ok(Protocol::encode([ + [$feed] = $this->feed([FakeTransport::json(Protocol::encode([ new Event(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']]), new Event(id: '1-1', type: 'io.appwrite.edge.invalidate'), ]))]); @@ -47,39 +48,57 @@ public function testReadsAFeedOverHttp(): void public function testAppendsTheFeedNameToTheEndpoint(): void { - [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + [$feed, $transport] = $this->feed(); $feed->read(); - $this->assertStringStartsWith('https://cloud.example.com/v1/feeds/edge', $transport->lastRequest()['url']); + $this->assertStringStartsWith('https://cloud.example.com/v1/feeds/edge', $transport->recorder->last()['uri']); } public function testEncodesAFeedNameThatNeedsIt(): void { - $adapter = new Http(new Client(new FakeTransport([])), 'https://cloud.example.com/v1/feeds/', 'a b/c'); + $adapter = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds/', 'a b/c'); $this->assertSame('https://cloud.example.com/v1/feeds/a%20b%2Fc', $adapter->getUrl()); } + public function testReadsWithGet(): void + { + [$feed, $transport] = $this->feed(); + + $feed->read(); + + $this->assertSame('GET', $transport->recorder->last()['method']); + } + + public function testAsksForJson(): void + { + [$feed, $transport] = $this->feed(); + + $feed->read(); + + $this->assertSame('application/json', $transport->recorder->last()['headers']['Accept'] ?? null); + } + public function testSendsThePositionAndLimit(): void { - [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + [$feed, $transport] = $this->feed(); $feed->read('1-0', 250); - $url = $transport->lastRequest()['url']; + $uri = $transport->recorder->last()['uri']; - $this->assertStringContainsString('lastEventId=1-0', $url); - $this->assertStringContainsString('limit=250', $url); + $this->assertStringContainsString('lastEventId=1-0', $uri); + $this->assertStringContainsString('limit=250', $uri); } public function testSendsNoParametersOnAFirstFullRead(): void { - [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + [$feed, $transport] = $this->feed(); $feed->read(null, Feed::MAX_BATCH); - $this->assertStringNotContainsString('lastEventId', $transport->lastRequest()['url']); + $this->assertStringNotContainsString('lastEventId', $transport->recorder->last()['uri']); } /** @@ -88,14 +107,14 @@ public function testSendsNoParametersOnAFirstFullRead(): void */ public function testDelegatesLongPollingToTheProducer(): void { - [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + [$feed, $transport] = $this->feed(); $started = \microtime(true); $feed->poll(null, 100, 5000); $this->assertLessThan(1, \microtime(true) - $started, 'Must not wait client-side'); - $this->assertCount(1, $transport->requests, 'Must not poll in a loop'); - $this->assertStringContainsString('timeout=5000', $transport->lastRequest()['url']); + $this->assertCount(1, $transport->recorder->requests, 'Must not poll in a loop'); + $this->assertStringContainsString('timeout=5000', $transport->recorder->last()['uri']); } /** @@ -105,21 +124,23 @@ public function testDelegatesLongPollingToTheProducer(): void */ public function testAllowsTheClientLongerThanTheLongPollTimeout(): void { - [$feed, $transport] = $this->feed([FakeTransport::ok([])]); + [$feed, $transport] = $this->feed(); $feed->poll(null, 100, 5000); - $this->assertSame(5000 + Protocol::TIMEOUT_MARGIN, $transport->lastRequest()['timeout']); + // Seconds, which is what the client takes; the protocol margin is in + // milliseconds, like the timeout the producer is given. + $this->assertSame(15.0, $transport->recorder->last()['timeout']); + $this->assertSame((float) ((5000 + Protocol::TIMEOUT_MARGIN) / 1000), $transport->recorder->last()['timeout']); } - public function testUsesTheClientDefaultTimeoutWhenNotLongPolling(): void + public function testLeavesTheConfiguredTimeoutAloneWhenNotLongPolling(): void { - $transport = new FakeTransport([FakeTransport::ok([])]); - $client = (new Client($transport))->setTimeout(1234); + [$feed, $transport] = $this->feed(); - (new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')))->read(); + $feed->read(); - $this->assertSame(1234, $transport->lastRequest()['timeout']); + $this->assertNull($transport->recorder->last()['timeout'], 'A plain read must not override the client'); } /** @@ -129,7 +150,7 @@ public function testUsesTheClientDefaultTimeoutWhenNotLongPolling(): void */ public function testCarriesTheStatusOfARejectedRead(): void { - [$feed] = $this->feed([FakeTransport::status(404)]); + [$feed] = $this->feed([FakeTransport::json([], 404)]); try { $feed->read(); @@ -141,7 +162,7 @@ public function testCarriesTheStatusOfARejectedRead(): void public function testRaisesServerErrors(): void { - [$feed] = $this->feed([FakeTransport::status(503)]); + [$feed] = $this->feed([FakeTransport::json([], 503)]); try { $feed->read(); @@ -151,9 +172,23 @@ public function testRaisesServerErrors(): void } } + /** + * PSR-18 returns 4xx and 5xx rather than throwing, so the adapter has to + * check the status itself β€” a producer error must not read as an empty + * batch, which the consumer would take for "caught up". + */ + public function testAnErrorStatusIsNotMistakenForAnEmptyBatch(): void + { + [$feed] = $this->feed([FakeTransport::json(['total' => 0, 'events' => []], 500)]); + + $this->expectException(Transport::class); + + $feed->read(); + } + public function testWrapsATransportFailure(): void { - [$feed] = $this->feed([new \RuntimeException('Connection refused')]); + [$feed] = $this->feed([FakeTransport::offline()]); $this->expectException(Transport::class); $this->expectExceptionMessageMatches('/Connection refused/'); @@ -181,13 +216,31 @@ public function testRejectsABodyThatIsNotABatch(): void public function testCannotAppendToAFeedItDoesNotOwn(): void { - [$feed] = $this->feed([FakeTransport::ok([])]); + [$feed] = $this->feed(); $this->expectException(Unsupported::class); $feed->append('io.appwrite.edge.invalidate'); } + /** + * Anything implementing the client's adapter interface works, including + * the client itself wrapping a transport β€” which is how this is actually + * built in a service. + */ + public function testWorksThroughTheClientItself(): void + { + $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new Event(id: '1-0', type: 'a')]))]); + + $client = (new Client($transport))->withHeaders(['x-appwrite-jwt' => 'token']); + $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); + + $events = $feed->read(); + + $this->assertCount(1, $events); + $this->assertSame('token', $transport->recorder->last()['headers']['x-appwrite-jwt'] ?? null); + } + /** * The point of the adapter: a remote feed is consumed with exactly the * code a local one is. @@ -195,12 +248,12 @@ public function testCannotAppendToAFeedItDoesNotOwn(): void public function testConsumesARemoteFeedThroughTheSameConsumer(): void { [$feed, $transport] = $this->feed([ - FakeTransport::ok(Protocol::encode([ + FakeTransport::json(Protocol::encode([ new Event(id: '1-0', type: 'a'), new Event(id: '1-1', type: 'b'), ])), - FakeTransport::ok(Protocol::encode([new Event(id: '1-2', type: 'c')])), - FakeTransport::ok(Protocol::encode([])), + FakeTransport::json(Protocol::encode([new Event(id: '1-2', type: 'c')])), + FakeTransport::json(Protocol::encode([])), ]); $cursor = new MemoryCursor('edge'); @@ -218,6 +271,6 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void $this->assertSame(0, $consumer->consume($handler)); $this->assertSame(['a', 'b', 'c'], $seen); - $this->assertStringContainsString('lastEventId=1-1', $transport->requests[1]['url']); + $this->assertStringContainsString('lastEventId=1-1', $transport->recorder->uris()[1]); } } diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index d36254b..e135c3b 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -14,7 +14,6 @@ use Utopia\Feed\Event; use Utopia\Feed\Feed; use Utopia\Feed\Protocol; -use Utopia\Fetch\Client; use Utopia\Tests\Unit\Support\FeedServer; /** @@ -39,7 +38,7 @@ protected function setUp(): void $this->server = new FeedServer($this->producer); $this->consumerFeed = new Feed( - new Http(new Client($this->server), 'https://cloud.example.com/v1/feeds', 'edge') + new Http($this->server, 'https://cloud.example.com/v1/feeds', 'edge') ); $this->cursor = new CacheCursor(new UtopiaCache(new CacheMemory()), 'edge'); @@ -192,7 +191,7 @@ public function testTheProducerCachesFullBatchesAndNothingElse(): void 'private, max-age=31536000', // 2 of 2 β€” settled history 'no-store', // 1 of 2 β€” the live end, will grow 'no-store', // 0 of 2 β€” caught up - ], $this->server->cacheControl); + ], $this->server->recorder->cacheControl()); } public function testTwoConsumersOfOneProducerAreIndependent(): void diff --git a/tests/Feed/Unit/Support/FakeClient.php b/tests/Feed/Unit/Support/FakeClient.php new file mode 100644 index 0000000..688a56a --- /dev/null +++ b/tests/Feed/Unit/Support/FakeClient.php @@ -0,0 +1,103 @@ +respond($request); + + /** @var array $headers */ + $headers = \array_map( + static fn (array $values): string => \implode(', ', $values), + $request->getHeaders(), + ); + + $this->recorder->requests[] = [ + 'uri' => (string) $request->getUri(), + 'method' => $request->getMethod(), + 'headers' => $headers, + 'timeout' => $this->timeout, + 'status' => $response->getStatusCode(), + 'cacheControl' => $response->getHeaderLine(Header::CACHE_CONTROL), + ]; + + return $response; + } + + /** + * @throws \Throwable To simulate a transport failure. + */ + abstract protected function respond(RequestInterface $request): ResponseInterface; + + public function withTimeout(float $seconds): static + { + $clone = clone $this; + $clone->timeout = $seconds; + + return $clone; + } + + public function withConnectTimeout(float $seconds): static + { + return clone $this; + } + + public function withSslVerification(bool $enabled = true): static + { + return clone $this; + } + + public function withCustomCA(string $path): static + { + return clone $this; + } + + public function withCertificate(string $certPath, string $keyPath, ?string $passphrase = null): static + { + return clone $this; + } + + public function withMinTlsVersion(Tls $version): static + { + return clone $this; + } + + public function withConnectionReuse(bool $enabled = true): static + { + return clone $this; + } + + /** + * Feeds are read buffered β€” a batch is bounded by `limit`, so there is + * nothing to stream. + */ + public function stream(RequestInterface $request, callable $sink): ResponseInterface + { + throw new \LogicException('A feed is never read as a stream'); + } +} diff --git a/tests/Feed/Unit/Support/FakeTransport.php b/tests/Feed/Unit/Support/FakeTransport.php index 55575c3..c5de32d 100644 --- a/tests/Feed/Unit/Support/FakeTransport.php +++ b/tests/Feed/Unit/Support/FakeTransport.php @@ -4,84 +4,69 @@ namespace Utopia\Tests\Unit\Support; -use Utopia\Fetch\Adapter; -use Utopia\Fetch\Options\Request as RequestOptions; -use Utopia\Fetch\Response; +use Psr\Http\Client\NetworkExceptionInterface; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; +use Utopia\Psr7\ContentType; +use Utopia\Psr7\Header; +use Utopia\Psr7\Response; +use Utopia\Psr7\Stream; /** - * A fetch adapter that answers from a script instead of a network, and records - * what it was asked, so the HTTP feed adapter can be tested without a server. + * A client that answers from a script, for driving the HTTP feed adapter + * through responses a real producer would be awkward to provoke. */ -class FakeTransport implements Adapter +class FakeTransport extends FakeClient { - /** @var list, timeout: int}> */ - public array $requests = []; - - /** @var list */ - private array $responses; - /** - * @param list $responses Answered in order; the last - * one repeats once the script runs out. + * @param list $responses Answered in order; + * the last one repeats once the script runs out. */ - public function __construct(array $responses) + public static function of(array $responses): self { - $this->responses = $responses; + $transport = new self(); + $transport->recorder->responses = $responses; + + return $transport; } /** * @param array $body */ - public static function ok(array $body, int $statusCode = 200): Response + public static function json(array $body, int $statusCode = 200): ResponseInterface { - return new Response($statusCode, (string) \json_encode($body), []); + return self::raw((string) \json_encode($body), $statusCode); } - public static function status(int $statusCode, string $body = '{}'): Response + public static function raw(string $body, int $statusCode = 200): ResponseInterface { - return new Response($statusCode, $body, []); + return (new Response($statusCode, body: new Stream\Factory()->createStream($body))) + ->withHeader(Header::CONTENT_TYPE, ContentType::JSON); } - public static function raw(string $body): Response + /** + * A transport failure, which PSR-18 requires be thrown rather than returned. + */ + public static function offline(string $message = 'Connection refused'): \Throwable { - return new Response(200, $body, []); + return new class ($message) extends \RuntimeException implements NetworkExceptionInterface { + public function getRequest(): RequestInterface + { + throw new \LogicException('Not needed for this test'); + } + }; } - public function send( - string $url, - string $method, - mixed $body, - array $headers, - RequestOptions $options, - ?callable $chunkCallback = null - ): Response { - $this->requests[] = [ - 'url' => $url, - 'method' => $method, - 'headers' => $headers, - 'timeout' => $options->getTimeout(), - ]; + protected function respond(RequestInterface $request): ResponseInterface + { + $responses = &$this->recorder->responses; - $response = \count($this->responses) > 1 ? \array_shift($this->responses) : ($this->responses[0] ?? null); + $response = \count($responses) > 1 ? \array_shift($responses) : ($responses[0] ?? null); if ($response instanceof \Throwable) { throw $response; } - return $response ?? self::ok(['total' => 0, 'events' => []]); - } - - /** - * @return array{url: string, method: string, headers: array, timeout: int} - */ - public function lastRequest(): array - { - $request = \end($this->requests); - - if ($request === false) { - throw new \RuntimeException('No request was made'); - } - - return $request; + return $response ?? self::json(['total' => 0, 'events' => []]); } } diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php index 9ad9c9a..1098d41 100644 --- a/tests/Feed/Unit/Support/FeedServer.php +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -4,38 +4,33 @@ namespace Utopia\Tests\Unit\Support; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; use Utopia\Feed\Feed; use Utopia\Feed\Protocol; -use Utopia\Fetch\Adapter; -use Utopia\Fetch\Options\Request as RequestOptions; -use Utopia\Fetch\Response; +use Utopia\Psr7\ContentType; +use Utopia\Psr7\Header; +use Utopia\Psr7\Response; +use Utopia\Psr7\Stream; /** - * A producer's feed endpoint, as a fetch adapter. + * A producer's feed endpoint, as a client. * * Serves a real {@see Feed} through {@see Protocol} exactly as an HTTP route * would, so a consumer reading it exercises the whole contract β€” parameters, * body and caching β€” rather than a fixture written to match the consumer. */ -class FeedServer implements Adapter +class FeedServer extends FakeClient { - /** @var list */ - public array $cacheControl = []; - - public function __construct(private readonly Feed $feed) + public function __construct(private readonly Feed $feed, Recorder $recorder = new Recorder()) { + parent::__construct($recorder); } - public function send( - string $url, - string $method, - mixed $body, - array $headers, - RequestOptions $options, - ?callable $chunkCallback = null - ): Response { + protected function respond(RequestInterface $request): ResponseInterface + { $query = []; - \parse_str((string) \parse_url($url, PHP_URL_QUERY), $query); + \parse_str($request->getUri()->getQuery(), $query); $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; $limit = (int) ($query[Protocol::PARAM_LIMIT] ?? Feed::MAX_BATCH); @@ -47,12 +42,10 @@ public function send( $timeout, ); - $this->cacheControl[] = $cacheControl = Protocol::cacheControl(\count($events), $limit); + $body = (string) \json_encode(Protocol::encode($events)); - return new Response( - 200, - (string) \json_encode(Protocol::encode($events)), - ['cache-control' => $cacheControl], - ); + return (new Response(200, body: new Stream\Factory()->createStream($body))) + ->withHeader(Header::CONTENT_TYPE, ContentType::JSON) + ->withHeader(Header::CACHE_CONTROL, Protocol::cacheControl(\count($events), $limit)); } } diff --git a/tests/Feed/Unit/Support/Recorder.php b/tests/Feed/Unit/Support/Recorder.php new file mode 100644 index 0000000..198bdaf --- /dev/null +++ b/tests/Feed/Unit/Support/Recorder.php @@ -0,0 +1,59 @@ +, timeout: float|null, status: int, cacheControl: string}> */ + public array $requests = []; + + /** + * Answers, in order. The last one repeats once the script runs out, so a + * test only scripts the responses it cares about. + * + * @var list + */ + public array $responses = []; + + /** + * @return array{uri: string, method: string, headers: array, timeout: float|null, status: int, cacheControl: string} + */ + public function last(): array + { + $request = \end($this->requests); + + if ($request === false) { + throw new \RuntimeException('No request was made'); + } + + return $request; + } + + /** + * @return list + */ + public function uris(): array + { + return \array_map(static fn (array $request): string => $request['uri'], $this->requests); + } + + /** + * @return list + */ + public function cacheControl(): array + { + return \array_map(static fn (array $request): string => $request['cacheControl'], $this->requests); + } +} From daf09e9b15bdcca68a0a362204258f3f5233725d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 15:21:52 +0200 Subject: [PATCH 03/68] Improve compatibility with CloudEvent library --- README.md | 20 ++++++++++++++++++++ tests/Feed/Unit/EventTest.php | 31 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/README.md b/README.md index 8d50529..d8f814e 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,26 @@ a warning rather than a failure: it keeps its position in memory and carries on, and only a restart before the store recovers replays anything. Pass `onWarning()` to hear about it. +## Interoperability + +`Event::toArray()` emits every CloudEvents v1.0 attribute, always populated and +never null, so the wire form is portable to stricter CloudEvents readers without +a conversion step. [`utopia-php/cloudevents`](https://github.com/utopia-php/cloudevents) +consumes it directly, and round-trips back: + +```php +$cloudEvent = CloudEvent::fromArray($event->toArray()); // works, and validate()s +$event = Event::fromArray($cloudEvent->toArray()); // identical event back +``` + +That is why this library models its own event rather than depending on one. A +feed consumer reads events from a producer it does not control, so it has to be +strict about the single field it cannot proceed without β€” `id`, which is its +position in the feed β€” and tolerant about everything else, including attributes +a future producer adds or a `specversion` it has never heard of. A general +CloudEvents type has no reason to make that trade, and the two rules point in +opposite directions. + ## Delivery semantics **A handler must be safe to run twice on the same event.** There are three diff --git a/tests/Feed/Unit/EventTest.php b/tests/Feed/Unit/EventTest.php index b3e3861..3c28a14 100644 --- a/tests/Feed/Unit/EventTest.php +++ b/tests/Feed/Unit/EventTest.php @@ -136,6 +136,37 @@ public function testAlwaysReportsTheSpecVersionAndContentType(): void $this->assertSame('application/json', $encoded['datacontenttype']); } + /** + * Every CloudEvents attribute is emitted, and none of them are null, even + * on an event that set almost nothing. + * + * This is what makes the wire form portable: a stricter CloudEvents reader + * β€” including `utopia-php/cloudevents`, whose `fromArray()` reads + * `specversion`, `source`, `id` and `time` without defaulting them β€” can + * consume it directly. Dropping an empty field here, or letting one be + * null, would break those readers without breaking any test that only + * round-trips through this class. + */ + public function testTheWireFormIsPortableToStricterCloudEventsReaders(): void + { + $encoded = (new Event(id: '1-0', type: 'test'))->toArray(); + + $this->assertSame([ + 'specversion', + 'id', + 'type', + 'source', + 'time', + 'subject', + 'datacontenttype', + 'data', + ], \array_keys($encoded)); + + foreach ($encoded as $field => $value) { + $this->assertNotNull($value, "The {$field} attribute must never be null on the wire"); + } + } + public function testReadsPayloadKeysWithADefault(): void { $event = new Event(id: '1-0', type: 'test', data: ['tags' => ['a' => 'b']]); From 4d3148d64888c0fe07ddeb1ec6a191048157aa45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 15:45:16 +0200 Subject: [PATCH 04/68] Replace event class with CloudEvent library --- CHANGELOG.md | 10 +- README.md | 55 +++++--- composer.json | 3 +- composer.lock | 58 ++++++++- docs/migration.md | 16 ++- src/Feed/Adapter.php | 95 ++++++++++---- src/Feed/Adapter/Http.php | 4 +- src/Feed/Adapter/Memory.php | 6 +- src/Feed/Adapter/None.php | 4 +- src/Feed/Adapter/Pool.php | 4 +- src/Feed/Adapter/Redis.php | 4 +- src/Feed/Consumer.php | 8 +- src/Feed/Event.php | 146 --------------------- src/Feed/Feed.php | 37 +++--- src/Feed/Protocol.php | 62 ++++++--- tests/Feed/E2E/RedisTest.php | 41 +++++- tests/Feed/Unit/ConsumerTest.php | 40 +++--- tests/Feed/Unit/EventTest.php | 195 ---------------------------- tests/Feed/Unit/FeedTest.php | 92 ++++++++++++- tests/Feed/Unit/HttpAdapterTest.php | 16 +-- tests/Feed/Unit/ProtocolTest.php | 75 +++++++++-- tests/Feed/Unit/RoundTripTest.php | 36 ++--- 22 files changed, 493 insertions(+), 514 deletions(-) delete mode 100644 src/Feed/Event.php delete mode 100644 tests/Feed/Unit/EventTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 67d6179..3a499c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,10 @@ twice. See [docs/migration.md](docs/migration.md) for how those map onto this library. - `Feed` β€” append, read and long-poll an ordered event log -- `Event` β€” a CloudEvent, with a strict decode that a feed's ids can be paged from +- Events are [utopia-php/cloudevents](https://github.com/utopia-php/cloudevents) + `CloudEvent` objects β€” this library defines no event type of its own, so a feed + event is accepted anywhere a `CloudEvent` is, and `dataschema` and extension + attributes survive an append and a read - `Adapter\Redis`, `Adapter\Pool` β€” Redis streams, directly or over a pool - `Adapter\Http` β€” another service's feed, read over the wire with [utopia-php/client](https://github.com/utopia-php/client); takes any of its @@ -19,7 +22,10 @@ library. - `Adapter\Memory`, `Adapter\None` β€” for tests, and for no backend configured - `Consumer` β€” the pull loop, with at-least-once semantics and a durable position - `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory` β€” where that position lives -- `Protocol` β€” the http-feeds wire contract, shared by producer and consumer +- `Protocol` β€” the http-feeds wire contract, shared by producer and consumer, and + the one place the feed's decode policy lives: strict about `id` because it is + the consumer's position, tolerant of everything else so a consumer older than + the producer keeps working - `Id` β€” feed positions, and the arithmetic for paging past one Requires PHP 8.5. CI builds one parameterized image per version in the diff --git a/README.md b/README.md index d8f814e..a287931 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ on β€” a cache tag to drop, a record to refresh, a config to reload. ## Features - **Ordered, resumable log** β€” consumers page by event id, and hold their own position -- **CloudEvents** β€” events are [CloudEvents](https://cloudevents.io/), as http-feeds requires +- **CloudEvents** β€” events *are* [`utopia-php/cloudevents`](https://github.com/utopia-php/cloudevents) events, as http-feeds requires - **Adapters** β€” Redis streams, a pooled Redis, in-memory, or another service's feed over HTTP - **Long polling** β€” subscribe in near real time without hammering the producer - **Cursors** β€” positions in a Utopia cache, in Redis, or in memory @@ -145,9 +145,9 @@ A `Consumer` reads from where it last got to, hands each new event to a handler, and records how far it got: ```php +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Cache as CacheCursor; -use Utopia\Feed\Event; $consumer = new Consumer( feed: $feed, @@ -155,8 +155,8 @@ $consumer = new Consumer( cursor: new CacheCursor($cache, 'edge'), ); -$handled = $consumer->consume(function (Event $event) use ($router) { - $router->invalidate($event->getData('tags', [])); +$handled = $consumer->consume(function (CloudEvent $event) use ($router) { + $router->invalidate($event->data['tags'] ?? []); }); ``` @@ -236,25 +236,44 @@ a warning rather than a failure: it keeps its position in memory and carries on, and only a restart before the store recovers replays anything. Pass `onWarning()` to hear about it. -## Interoperability +## Events -`Event::toArray()` emits every CloudEvents v1.0 attribute, always populated and -never null, so the wire form is portable to stricter CloudEvents readers without -a conversion step. [`utopia-php/cloudevents`](https://github.com/utopia-php/cloudevents) -consumes it directly, and round-trips back: +There is no event type in this library. Events **are** +[`Utopia\CloudEvents\CloudEvent`](https://github.com/utopia-php/cloudevents) +objects, so anything already typed against one takes a feed event directly, and +everything a CloudEvent carries β€” `dataschema`, extension attributes such as a +`traceparent` β€” survives an append and a read untouched: ```php -$cloudEvent = CloudEvent::fromArray($event->toArray()); // works, and validate()s -$event = Event::fromArray($cloudEvent->toArray()); // identical event back +use Utopia\CloudEvents\CloudEvent; + +$consumer->consume(function (CloudEvent $event) { + $tags = $event->data['tags'] ?? []; + $trace = $event->getExtension('traceparent'); +}); ``` -That is why this library models its own event rather than depending on one. A -feed consumer reads events from a producer it does not control, so it has to be -strict about the single field it cannot proceed without β€” `id`, which is its -position in the feed β€” and tolerant about everything else, including attributes -a future producer adds or a `specversion` it has never heard of. A general -CloudEvents type has no reason to make that trade, and the two rules point in -opposite directions. +`data` is unrestricted, as the JSON event format requires β€” a map, a list, a +string, a number or null are all valid payloads and all round-trip as +themselves. `subject` is nullable, so an event with no subject reads back as +`null` rather than `''`. + +The one thing this library decides for itself is how a batch is decoded, and it +is deliberately not `CloudEvent::fromArray()`'s default: + +- **Strict about `id`.** For a feed the id *is* the consumer's position, so an + event without one cannot be recorded as passed. The spec makes `id` required + too; `Protocol` enforces exactly that one attribute rather than calling + `validate()`, which would also demand a `source` a feed has no use for. +- **Tolerant about everything else.** Decoding runs with `lenient: true` and + `allowUnknownSpecversion: true`, so a producer that adds an attribute, omits + an optional one, or moves the spec forward does not stop a consumer that + predates it. A feed is read by consumers older than the producer *by design*, + and that is what makes a staged rollout safe. + +An entry that is not a CloudEvent at all β€” no `specversion`, no `type` β€” is not +tolerated, because that is a producer sending something other than what the feed +is specified to carry. ## Delivery semantics diff --git a/composer.json b/composer.json index 14faa85..ef80187 100644 --- a/composer.json +++ b/composer.json @@ -41,7 +41,8 @@ "utopia-php/cache": "3.*", "utopia-php/client": "^0.2", "utopia-php/pools": "1.*", - "utopia-php/psr7": "^0.2" + "utopia-php/psr7": "^0.2", + "utopia-php/cloudevents": "dev-feat-cloudevents-g2" }, "require-dev": { "laravel/pint": "^1.21", diff --git a/composer.lock b/composer.lock index c367587..bd6060c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "52ee952ff068b03efb088193a317a726", + "content-hash": "47e874a6ca9e914a88ab31627d42e31b", "packages": [ { "name": "brick/math", @@ -2050,6 +2050,58 @@ }, "time": "2026-07-13T15:29:09+00:00" }, + { + "name": "utopia-php/cloudevents", + "version": "dev-feat-cloudevents-g2", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/cloudevents.git", + "reference": "9cbc8dfe6b4ad5273b4ed1729f5a35112f20660d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/cloudevents/zipball/9cbc8dfe6b4ad5273b4ed1729f5a35112f20660d", + "reference": "9cbc8dfe6b4ad5273b4ed1729f5a35112f20660d", + "shasum": "" + }, + "require": { + "php": ">=8.3" + }, + "require-dev": { + "laravel/pint": "1.25.*", + "phpstan/phpstan": "2.0.*", + "phpunit/phpunit": "12.4.*" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\CloudEvents\\": "src/CloudEvents" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + } + ], + "description": "Lite & fast micro PHP CloudEvents implementation that is **easy to use**.", + "keywords": [ + "cloudevents", + "framework", + "php", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/cloudevents/issues", + "source": "https://github.com/utopia-php/cloudevents/tree/feat-cloudevents-g2" + }, + "time": "2026-07-29T13:37:00+00:00" + }, { "name": "utopia-php/pools", "version": "1.1.1", @@ -4193,7 +4245,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/cloudevents": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docs/migration.md b/docs/migration.md index 9df39ee..42a02b6 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -85,7 +85,7 @@ $consumer = new Consumer($edgeFeed, FastlyConsumer::NAME, new Cursor\Pool($pool, $purged = 0; $seen = []; -$consumer->consume(function (Event $event) use (&$purged, &$seen): void { +$consumer->consume(function (CloudEvent $event) use (&$purged, &$seen): void { $url = $this->purgeUrl($event); if ($url === null || isset($seen[$url])) { return; @@ -97,14 +97,14 @@ $consumer->consume(function (Event $event) use (&$purged, &$seen): void { }); ``` -`purgeUrl()` reads a typed `Event` instead of an array: +`purgeUrl()` reads a typed `CloudEvent` instead of an array: ```php if ($event->type !== EdgeFeed::EVENT_INVALIDATE_RULE) { return null; } -$tags = $event->getData('tags', []); +$tags = $event->data['tags'] ?? []; $domain = \is_array($tags) ? ($tags['domain'] ?? '') : ''; ``` @@ -114,7 +114,7 @@ $domain = \is_array($tags) ? ($tags['domain'] ?? '') : ''; | --- | --- | | `Feed\Consumer` | `Consumer`, plus a handler | | `Feed\Cursor` | `Cursor\Cache` | -| `Feed\Event` | `Event` | +| `Feed\Event` | `Utopia\CloudEvents\CloudEvent` β€” this library has no event type of its own | | `Feed\Event::FEED` and the type constants | Stay β€” they name cloud's feed and its events | | `Manager::fetchFeed()` | `Adapter\Http`, over `utopia-php/client` | | `Consumer::TIMEOUT_MARGIN` | `Protocol::TIMEOUT_MARGIN` | @@ -145,8 +145,8 @@ with the invalidator that defines what a usable tag is, so it moves into the handler: ```php -$consumer->consume(function (Event $event) use ($invalidator): void { - $tags = $event->getData('tags', []); +$consumer->consume(function (CloudEvent $event) use ($invalidator): void { + $tags = $event->data['tags'] ?? []; $tags = \is_array($tags) ? Invalidator::normalize($tags) : []; if ($tags === []) { @@ -191,3 +191,7 @@ These were load-bearing in the original implementations and are preserved: stops visibly rather than quietly losing events. - **`Cache-Control` is computed from the batch**, and `public` is opt-in rather than a decision baked into one endpoint. +- **Events are `Utopia\CloudEvents\CloudEvent`**, not a bespoke type. `subject` + is nullable, so an event without one reads back as `null` rather than `''`; + `data` is unrestricted, so a list or scalar payload round-trips as itself; and + `dataschema` and extension attributes now survive an append and a read. diff --git a/src/Feed/Adapter.php b/src/Feed/Adapter.php index 0dd922e..c64a723 100644 --- a/src/Feed/Adapter.php +++ b/src/Feed/Adapter.php @@ -4,6 +4,8 @@ namespace Utopia\Feed; +use Utopia\CloudEvents\CloudEvent; +use Utopia\CloudEvents\Exception as CloudEventsException; use Utopia\Feed\Exception\Invalid; /** @@ -50,7 +52,7 @@ public function getName(): string * never partially β€” a caller that gets an id back can tell every * consumer will see the event. */ - abstract public function append(Event $event): string; + abstract public function append(CloudEvent $event): string; /** * Read up to $limit events strictly after $lastEventId, oldest first, or @@ -62,7 +64,7 @@ abstract public function append(Event $event): string; * @param int $timeout Milliseconds to wait for an event before giving up, * honoured only when {@see pollable()} is true; {@see Feed::poll()} * handles the wait for every other adapter. - * @return list + * @return list * @throws Invalid When $lastEventId is not a feed position. * @throws Exception When the backend cannot be read. */ @@ -83,51 +85,88 @@ public function pollable(): bool /** * The backend fields an event is stored as. * - * `data` is JSON so the payload can nest; everything else is a flat string - * because those are the fields a backend may want to index or filter on. - * The id is not among them β€” it is the key the entry is stored under. + * `data` and `extensions` are JSON so they can hold what CloudEvents lets + * them hold; every other attribute is a flat string, because those are the + * ones a backend may want to index or filter on. The id is not among them + * β€” it is the key the entry is stored under. + * + * Extensions are stored rather than dropped: a producer that attaches one + * β€” a `traceparent`, say β€” means it to reach the consumer, and losing it + * on the way through the backend would be invisible at both ends. * * @return array * @throws Invalid When the payload cannot be encoded. */ - protected static function encode(Event $event): array + protected static function encode(CloudEvent $event): array { - try { - $data = \json_encode($event->data, JSON_THROW_ON_ERROR); - } catch (\JsonException $error) { - throw new Invalid('Feed event data must be JSON encodable: ' . $error->getMessage(), previous: $error); - } - return [ 'type' => $event->type, 'source' => $event->source, - 'subject' => $event->subject, + // CloudEvents models an absent subject as null. A backend field is + // a string, so it is normalized here rather than stored as a null + // that would read back as "" on one backend and break on another. + 'subject' => $event->subject ?? '', 'time' => $event->time, - 'data' => $data, + 'dataschema' => $event->dataschema ?? '', + 'data' => self::json($event->data, 'data'), + 'extensions' => self::json($event->getExtensions(), 'extensions'), ]; } /** * Rebuild an event from what {@see encode()} stored. * - * Undecodable payloads become an empty array rather than an error: the - * event still happened, its id is still a valid position, and refusing to - * return it would wedge every consumer behind it forever. + * Decoded leniently, for the same reason {@see Protocol::decode()} is: the + * event happened, its id is a valid position, and refusing to return it + * over one malformed attribute would wedge every consumer behind it. * * @param array $fields + * @throws Invalid When the stored entry cannot be read as an event at all. */ - protected static function decode(string $id, array $fields): Event + protected static function decode(string $id, array $fields): CloudEvent { - $data = \json_decode(self::field($fields, 'data'), true); - - return new Event( - id: $id, - type: self::field($fields, 'type'), - data: \is_array($data) ? $data : [], - source: self::field($fields, 'source'), - subject: self::field($fields, 'subject'), - time: self::field($fields, 'time'), - ); + $extensions = \json_decode(self::field($fields, 'extensions'), true); + + $event = [ + 'specversion' => CloudEvent::SPECVERSION, + 'id' => $id, + 'type' => self::field($fields, 'type'), + 'source' => self::field($fields, 'source'), + 'time' => self::field($fields, 'time'), + 'data' => \json_decode(self::field($fields, 'data'), true), + ...(\is_array($extensions) ? $extensions : []), + ]; + + // The inverse of the normalization in encode(): these two are nullable + // on a CloudEvent, and a backend field cannot hold a null, so an empty + // stored field means the attribute was absent. Passing the empty string + // through instead would turn "no subject" into "a subject that is the + // empty string" on every round trip through a backend. + foreach (['subject', 'dataschema'] as $optional) { + $value = self::field($fields, $optional); + + if ($value !== '') { + $event[$optional] = $value; + } + } + + try { + return CloudEvent::fromArray($event, lenient: true); + } catch (CloudEventsException $error) { + throw new Invalid("Feed entry {$id} could not be read as an event: {$error->getMessage()}", previous: $error); + } + } + + /** + * @throws Invalid When the value cannot be encoded. + */ + private static function json(mixed $value, string $attribute): string + { + try { + return \json_encode($value, JSON_THROW_ON_ERROR); + } catch (\JsonException $error) { + throw new Invalid("Feed event {$attribute} must be JSON encodable: {$error->getMessage()}", previous: $error); + } } /** diff --git a/src/Feed/Adapter/Http.php b/src/Feed/Adapter/Http.php index 8429973..4cb5420 100644 --- a/src/Feed/Adapter/Http.php +++ b/src/Feed/Adapter/Http.php @@ -7,7 +7,7 @@ use Psr\Http\Client\ClientExceptionInterface; use Utopia\Client\Adapter as ClientAdapter; use Utopia\Feed\Adapter; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; @@ -86,7 +86,7 @@ public function getUrl(): string * @throws Unsupported Always. A consumer cannot append to a feed it does * not own; call the producer's own API instead. */ - public function append(Event $event): string + public function append(CloudEvent $event): string { throw new Unsupported("The {$this->name} feed is read over HTTP and cannot be appended to"); } diff --git a/src/Feed/Adapter/Memory.php b/src/Feed/Adapter/Memory.php index 92e8d2b..db4d944 100644 --- a/src/Feed/Adapter/Memory.php +++ b/src/Feed/Adapter/Memory.php @@ -5,7 +5,7 @@ namespace Utopia\Feed\Adapter; use Utopia\Feed\Adapter; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Id; /** @@ -21,7 +21,7 @@ */ class Memory extends Adapter { - /** @var list */ + /** @var list */ private array $events = []; /** @@ -38,7 +38,7 @@ public function __construct(string $name, protected readonly int $maxSize = 100_ parent::__construct($name); } - public function append(Event $event): string + public function append(CloudEvent $event): string { $now = (int) \floor(\microtime(true) * 1000); diff --git a/src/Feed/Adapter/None.php b/src/Feed/Adapter/None.php index 30eb07e..438fdc5 100644 --- a/src/Feed/Adapter/None.php +++ b/src/Feed/Adapter/None.php @@ -5,7 +5,7 @@ namespace Utopia\Feed\Adapter; use Utopia\Feed\Adapter; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Unsupported; /** @@ -27,7 +27,7 @@ public function __construct(string $name = 'none') parent::__construct($name); } - public function append(Event $event): string + public function append(CloudEvent $event): string { throw new Unsupported("No feed backend is configured for the {$this->name} feed"); } diff --git a/src/Feed/Adapter/Pool.php b/src/Feed/Adapter/Pool.php index c2f4bee..5ca023b 100644 --- a/src/Feed/Adapter/Pool.php +++ b/src/Feed/Adapter/Pool.php @@ -5,7 +5,7 @@ namespace Utopia\Feed\Adapter; use Utopia\Feed\Adapter; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Pools\Pool as UtopiaPool; /** @@ -32,7 +32,7 @@ public function __construct( parent::__construct($name); } - public function append(Event $event): string + public function append(CloudEvent $event): string { return $this->pool->use(fn (\Redis|\RedisCluster $redis): string => $this->adapter($redis)->append($event)); } diff --git a/src/Feed/Adapter/Redis.php b/src/Feed/Adapter/Redis.php index b072ad9..8a0f58e 100644 --- a/src/Feed/Adapter/Redis.php +++ b/src/Feed/Adapter/Redis.php @@ -5,7 +5,7 @@ namespace Utopia\Feed\Adapter; use Utopia\Feed\Adapter; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; @@ -52,7 +52,7 @@ public function getKey(): string return 'feed:' . $this->name; } - public function append(Event $event): string + public function append(CloudEvent $event): string { try { $id = $this->redis->xAdd($this->getKey(), '*', self::encode($event), $this->maxSize, true); diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 4578882..f498849 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -4,6 +4,8 @@ namespace Utopia\Feed; +use Utopia\CloudEvents\CloudEvent; + /** * Reads a feed from where it last got to, hands each new event to a handler, * and records how far it got. @@ -12,8 +14,8 @@ * $consumer = new Consumer($feed, 'cache-invalidator', new Cursor\Cache($cache, 'edge')); * * // On a timer, or in a loop with a long-poll timeout: - * $consumer->consume(function (Event $event) use ($cache) { - * $cache->purge($event->getData('tag')); + * $consumer->consume(function (CloudEvent $event) use ($cache) { + * $cache->purge($event->data['tag'] ?? ''); * }); * ``` * @@ -128,7 +130,7 @@ public function onWarning(?callable $callback): self * Hand every event not yet seen to $handler, oldest first, and return how * many it accepted. * - * @param callable(Event): void $handler Throws to reject an event, which + * @param callable(CloudEvent): void $handler Throws to reject an event, which * stops the run and leaves the position before it. * @return int Events handled. Zero means the consumer is caught up. * @throws Exception When the feed cannot be read. The position stays where diff --git a/src/Feed/Event.php b/src/Feed/Event.php deleted file mode 100644 index d37f38c..0000000 --- a/src/Feed/Event.php +++ /dev/null @@ -1,146 +0,0 @@ - $data Payload. Must survive a JSON - * round-trip, which is also why the keys are not narrowed to - * strings: a payload is allowed to be a JSON array, and that decodes - * with integer keys. Producers normally send a map. - * @param string $source Who produced the event, as a URI reference - * (`urn:appwrite:cloud:fra`). Stamped by {@see Feed} on append. - * @param string $subject The single business object the event is about, - * when it has one, so a consumer can filter without decoding - * `$data`. - * @param string $time RFC 3339 timestamp. Stamped on append. - */ - public function __construct( - public string $id, - public string $type, - public array $data = [], - public string $source = '', - public string $subject = '', - public string $time = '', - ) { - } - - /** - * Decode an event received from a producer. - * - * @param array $event - * @throws Invalid When the event carries no id. - * The id is the one field a consumer cannot proceed without: it is - * the cursor position, so accepting an event without one would mean - * losing the place in the feed. Every other field is defaulted, - * because a consumer that only reads `data` should not be stopped - * by a producer that omits `subject`. - */ - public static function fromArray(array $event): self - { - $id = $event['id'] ?? ''; - if (!\is_string($id) || $id === '') { - throw new Invalid('Feed event is missing an id'); - } - - $data = $event['data'] ?? []; - - return new self( - id: $id, - type: self::string($event, 'type'), - data: \is_array($data) ? $data : [], - source: self::string($event, 'source'), - subject: self::string($event, 'subject'), - time: self::string($event, 'time'), - ); - } - - /** - * The event as a CloudEvent, ready to be serialized to JSON. - * - * @return array - */ - public function toArray(): array - { - return [ - 'specversion' => self::SPEC_VERSION, - 'id' => $this->id, - 'type' => $this->type, - 'source' => $this->source, - 'time' => $this->time, - 'subject' => $this->subject, - 'datacontenttype' => self::CONTENT_TYPE, - 'data' => $this->data, - ]; - } - - /** - * Read one key out of the payload. - * - * Handlers are looking at data some other service wrote, so this exists to - * keep them from having to re-check `isset` and the type on every access. - */ - public function getData(string $key, mixed $default = null): mixed - { - return $this->data[$key] ?? $default; - } - - /** - * The same event at a new position, used by adapters to attach the id the - * backend assigned on append. - */ - public function withId(string $id): self - { - return new self( - id: $id, - type: $this->type, - data: $this->data, - source: $this->source, - subject: $this->subject, - time: $this->time, - ); - } - - /** - * Now, in the RFC 3339 form http-feeds asks for: UTC, milliseconds, `Z`. - */ - public static function now(): string - { - return (new \DateTimeImmutable('now', new \DateTimeZone('UTC')))->format('Y-m-d\TH:i:s.v\Z'); - } - - /** - * @param array $event - */ - private static function string(array $event, string $key): string - { - $value = $event[$key] ?? null; - - return \is_string($value) ? $value : ''; - } -} diff --git a/src/Feed/Feed.php b/src/Feed/Feed.php index 2508ad6..5ade3d7 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Feed.php @@ -4,6 +4,8 @@ namespace Utopia\Feed; +use Utopia\CloudEvents\CloudEvent; + /** * An append-only, strongly ordered sequence of events that consumers pull. * @@ -93,24 +95,25 @@ public function getSource(): string * Append an event and return its position in the feed. * * @param string $type What happened, in reverse-DNS notation. - * @param array $data Payload, JSON encodable. + * @param mixed $data Payload, JSON encodable. Usually a map, but the JSON + * event format leaves it unrestricted, so a list or a scalar is + * equally valid. * @param string $subject The one business object this is about, if there - * is one. + * is one. Empty means none, which is how CloudEvents models it. * @throws Exception\Invalid When $type is empty or $data cannot be * encoded. * @throws Exception When the backend rejects the append. */ - public function append(string $type, array $data = [], string $subject = ''): string + public function append(string $type, mixed $data = [], string $subject = ''): string { if ($type === '') { throw new Exception\Invalid('Feed event type is required'); } - return $this->publish(new Event( - id: '', + return $this->publish(new CloudEvent( type: $type, + subject: $subject === '' ? null : $subject, data: $data, - subject: $subject, )); } @@ -127,20 +130,20 @@ public function append(string $type, array $data = [], string $subject = ''): st * be encoded. * @throws Exception When the backend rejects the append. */ - public function publish(Event $event): string + public function publish(CloudEvent $event): string { if ($event->type === '') { throw new Exception\Invalid('Feed event type is required'); } - return $this->adapter->append(new Event( - id: '', - type: $event->type, - data: $event->data, - source: $this->source, - subject: $event->subject, - time: $event->time !== '' ? $event->time : Event::now(), - )); + // Stamped with the withers rather than rebuilt, so anything this + // library does not model itself β€” a dataschema, an extension attribute + // such as a traceparent β€” survives the append untouched. + return $this->adapter->append( + $event + ->withSource($this->source) + ->withTime($event->time !== '' ? $event->time : null) + ); } /** @@ -149,7 +152,7 @@ public function publish(Event $event): string * * Returns immediately, with an empty list when the consumer is caught up. * - * @return list + * @return list * @throws Exception\Invalid When $lastEventId is not a feed position. * @throws Exception When the backend cannot be read. */ @@ -175,7 +178,7 @@ public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): * are enabled. Without them it holds the worker for the duration, so run * it with hooks on or keep the timeout at 0. * - * @return list + * @return list * @throws Exception\Invalid When $lastEventId is not a feed position. * @throws Exception When the backend cannot be read. */ diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 1d14a7b..ec05720 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -4,6 +4,8 @@ namespace Utopia\Feed; +use Utopia\CloudEvents\CloudEvent; +use Utopia\CloudEvents\Exception as CloudEventsException; use Utopia\Feed\Exception\Invalid; /** @@ -110,25 +112,37 @@ public static function query(?string $lastEventId = null, int $limit = 0, int $t /** * The response body for a batch. * - * @param list $events + * @param list $events * @return array{total: int, events: list>} */ public static function encode(array $events): array { return [ self::KEY_TOTAL => \count($events), - self::KEY_EVENTS => \array_map(static fn (Event $event): array => $event->toArray(), $events), + self::KEY_EVENTS => \array_map(static fn (CloudEvent $event): array => $event->toArray(), $events), ]; } /** * Read a batch out of a response body. * + * Decoded leniently, and tolerating a `specversion` this consumer has + * never seen: a feed is read by consumers older than the producer *by + * design*, so a producer that adds an attribute, omits an optional one, or + * moves the spec forward must not stop a consumer that predates it. That + * is what makes a staged rollout safe, and it is the one place this + * library needs a reader more forgiving than a general CloudEvents one. + * + * The exception is `id`, which is enforced here and nowhere else in the + * spec's terms: for a feed the id *is* the consumer's position, so an event + * without one cannot be recorded as passed. {@see CloudEvent::validate()} + * requires `source` too, which a feed has no need of, so this checks the + * one attribute it actually depends on rather than calling it. + * * Stops at the first event that cannot be decoded and returns the ones - * before it, rather than dropping it and carrying on. An event with no id - * has no position, so a consumer cannot record having passed it; skipping - * it would mean every event after it is acknowledged under a cursor that - * never advanced past the gap, and the next restart would replay them all. + * before it, rather than dropping it and carrying on. Skipping it would + * mean every event after it is acknowledged under a cursor that never + * advanced past the gap, and the next restart would replay them all. * * Returning the prefix keeps the events that *are* usable moving: the * consumer applies them, advances to the last one, and meets the broken @@ -136,7 +150,7 @@ public static function encode(array $events): array * salvage, this throws and the feed visibly stops instead of quietly * losing events. * - * @return list + * @return list * @throws Invalid When the payload is not a batch, or when the very first * event in it cannot be decoded. */ @@ -155,19 +169,17 @@ public static function decode(mixed $payload): array /** @var mixed $event */ foreach ($raw as $event) { - if (!\is_array($event)) { - if ($events === []) { + try { + if (!\is_array($event)) { throw new Invalid('Feed batch contains an entry that is not an event'); } - break; - } - - try { - $events[] = Event::fromArray($event); - } catch (Invalid $error) { + $events[] = self::event($event); + } catch (Invalid | CloudEventsException $error) { if ($events === []) { - throw $error; + throw $error instanceof Invalid + ? $error + : new Invalid('Feed batch contains an event that cannot be read: ' . $error->getMessage(), previous: $error); } break; @@ -177,6 +189,24 @@ public static function decode(mixed $payload): array return $events; } + /** + * Decode one event, enforcing the only attribute a feed cannot do without. + * + * @param array $raw + * @throws Invalid When the event carries no usable id. + * @throws CloudEventsException When it is not a CloudEvent at all. + */ + private static function event(array $raw): CloudEvent + { + $event = CloudEvent::fromArray($raw, lenient: true, allowUnknownSpecversion: true); + + if ($event->id === '') { + throw new Invalid('Feed event is missing an id'); + } + + return $event; + } + /** * What a producer should send as `Cache-Control` for a batch. * diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index 2c4b0ff..dbd9878 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -8,7 +8,7 @@ use Utopia\Feed\Adapter\Redis as RedisAdapter; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Redis as RedisCursor; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Feed; use Utopia\Feed\Id; @@ -111,6 +111,35 @@ public function testReadsStrictlyAfterAPosition(): void $this->assertSame([], $feed->read($second)); } + public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void + { + $this->feed()->publish(new CloudEvent( + id: '', + type: 'test', + dataschema: 'https://example.com/schema.json', + extensions: ['traceparent' => '00-abc-def-01'], + )); + + $event = $this->feed()->read()[0]; + + $this->assertSame('https://example.com/schema.json', $event->dataschema); + $this->assertSame('00-abc-def-01', $event->getExtension('traceparent')); + } + + public function testAnAbsentSubjectStaysAbsent(): void + { + $this->feed()->append('test'); + + $this->assertNull($this->feed()->read()[0]->subject); + } + + public function testAScalarPayloadSurvivesTheRoundTrip(): void + { + $this->feed()->append('test', 'a string'); + + $this->assertSame('a string', $this->feed()->read()[0]->data); + } + public function testNestedPayloadsSurviveTheRoundTrip(): void { $data = [ @@ -194,7 +223,7 @@ public function testConsumesThroughAPersistedCursor(): void $last = $feed->append('b'); $seen = []; - $handler = function (Event $event) use (&$seen): void { + $handler = function (CloudEvent $event) use (&$seen): void { $seen[] = $event->type; }; @@ -214,8 +243,8 @@ public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void $feed->append('a'); - $this->assertSame(1, (new Consumer($feed, 'one', $cursor))->consume(fn (Event $e) => null)); - $this->assertSame(1, (new Consumer($feed, 'two', $cursor))->consume(fn (Event $e) => null)); + $this->assertSame(1, (new Consumer($feed, 'one', $cursor))->consume(fn (CloudEvent $e) => null)); + $this->assertSame(1, (new Consumer($feed, 'two', $cursor))->consume(fn (CloudEvent $e) => null)); } public function testResetReplaysTheRetainedFeed(): void @@ -227,11 +256,11 @@ public function testResetReplaysTheRetainedFeed(): void $feed->append('b'); $consumer = new Consumer($feed, 'invalidator', $cursor); - $consumer->consume(fn (Event $e) => null); + $consumer->consume(fn (CloudEvent $e) => null); $consumer->reset(); $this->assertNull($cursor->load('invalidator')); - $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume(fn (Event $e) => null)); + $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume(fn (CloudEvent $e) => null)); } public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index dcde876..1bdb86a 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -9,7 +9,7 @@ use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; use Utopia\Feed\Cursor\Memory as MemoryCursor; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Feed; use Utopia\Tests\Unit\Support\FailingCursor; @@ -41,7 +41,7 @@ private function consumer(?Cursor $cursor = null, int $batch = Consumer::BATCH): private function drain(Consumer $consumer, ?int &$count = null): array { $seen = []; - $count = $consumer->consume(function (Event $event) use (&$seen): void { + $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { $seen[] = $event->type; }); @@ -66,9 +66,9 @@ public function testCaughtUpConsumerDoesNothing(): void $this->feed->append('a'); $consumer = $this->consumer(); - $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); - $this->assertSame(0, $consumer->consume(fn (Event $event) => null)); + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); } public function testResumesFromTheStoredPosition(): void @@ -111,9 +111,9 @@ public function load(string $consumer): ?string $consumer = $this->consumer($cursor); - $consumer->consume(fn (Event $event) => null); - $consumer->consume(fn (Event $event) => null); - $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); $this->assertSame(1, $cursor->loads); } @@ -128,7 +128,7 @@ public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void $seen = []; try { - $consumer->consume(function (Event $event) use (&$seen): void { + $consumer->consume(function (CloudEvent $event) use (&$seen): void { if ($event->type === 'b') { throw new \RuntimeException('nope'); } @@ -153,7 +153,7 @@ public function testRetriesTheFailedEventOnTheNextRun(): void $attempts = 0; try { - $consumer->consume(function (Event $event) use (&$attempts): void { + $consumer->consume(function (CloudEvent $event) use (&$attempts): void { if ($event->type === 'b') { $attempts++; throw new \RuntimeException('nope'); @@ -177,7 +177,7 @@ public function testAFailureOnTheFirstEventCommitsNothing(): void $this->feed->append('a'); try { - $this->consumer()->consume(fn (Event $event) => throw new \RuntimeException('nope')); + $this->consumer()->consume(fn (CloudEvent $event) => throw new \RuntimeException('nope')); } catch (\RuntimeException) { // Expected. } @@ -191,7 +191,7 @@ public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void $this->feed->append('b'); $this->feed->append('c'); - $this->assertSame(3, $this->consumer()->consume(fn (Event $event) => null)); + $this->assertSame(3, $this->consumer()->consume(fn (CloudEvent $event) => null)); } public function testDrainsABacklogInBatches(): void @@ -202,10 +202,10 @@ public function testDrainsABacklogInBatches(): void $consumer = $this->consumer(batch: 4); - $this->assertSame(4, $consumer->consume(fn (Event $event) => null)); - $this->assertSame(4, $consumer->consume(fn (Event $event) => null)); - $this->assertSame(2, $consumer->consume(fn (Event $event) => null)); - $this->assertSame(0, $consumer->consume(fn (Event $event) => null)); + $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(2, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); } /** @@ -259,7 +259,7 @@ public function testResetReplaysEverythingStillRetained(): void $this->feed->append('b'); $consumer = $this->consumer(); - $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); $consumer->reset(); @@ -275,9 +275,9 @@ public function testConsumersOfTheSameFeedTrackSeparatePositions(): void $one = new Consumer($this->feed, 'one', $this->cursor); $two = new Consumer($this->feed, 'two', $this->cursor); - $this->assertSame(1, $one->consume(fn (Event $event) => null)); - $this->assertSame(1, $two->consume(fn (Event $event) => null), 'The second consumer has its own position'); - $this->assertSame(0, $one->consume(fn (Event $event) => null)); + $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); + $this->assertSame(1, $two->consume(fn (CloudEvent $event) => null), 'The second consumer has its own position'); + $this->assertSame(0, $one->consume(fn (CloudEvent $event) => null)); } public function testPositionIsNullBeforeTheFirstRun(): void @@ -315,7 +315,7 @@ public function testAFailedReadLeavesThePositionAlone(): void $this->expectException(\Utopia\Feed\Exception\Unsupported::class); try { - $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); } finally { $this->assertSame($first, $this->cursor->load('invalidator')); } diff --git a/tests/Feed/Unit/EventTest.php b/tests/Feed/Unit/EventTest.php deleted file mode 100644 index 3c28a14..0000000 --- a/tests/Feed/Unit/EventTest.php +++ /dev/null @@ -1,195 +0,0 @@ - '1.0', - 'id' => '1690000000000-0', - 'type' => 'io.appwrite.edge.invalidate-rule', - 'source' => 'urn:appwrite:cloud:fra', - 'time' => '2026-07-29T12:00:00.000Z', - 'subject' => 'preview.example.com', - 'datacontenttype' => 'application/json', - 'data' => ['tags' => ['domain' => 'preview.example.com']], - ]); - - $this->assertSame('1690000000000-0', $event->id); - $this->assertSame('io.appwrite.edge.invalidate-rule', $event->type); - $this->assertSame('urn:appwrite:cloud:fra', $event->source); - $this->assertSame('2026-07-29T12:00:00.000Z', $event->time); - $this->assertSame('preview.example.com', $event->subject); - $this->assertSame(['tags' => ['domain' => 'preview.example.com']], $event->data); - } - - public function testRejectsAnEventWithoutAnId(): void - { - $this->expectException(Invalid::class); - - Event::fromArray(['type' => 'io.appwrite.edge.invalidate']); - } - - /** - * @return array - */ - public static function unusableIds(): array - { - return [ - 'empty' => [''], - 'null' => [null], - 'numeric' => [42], - 'array' => [[]], - ]; - } - - /** - * @dataProvider unusableIds - */ - public function testRejectsAnIdThatIsNotANonEmptyString(mixed $id): void - { - $this->expectException(Invalid::class); - - Event::fromArray(['id' => $id, 'type' => 'test']); - } - - public function testDefaultsEveryFieldExceptTheId(): void - { - $event = Event::fromArray(['id' => '1-0']); - - $this->assertSame('1-0', $event->id); - $this->assertSame('', $event->type); - $this->assertSame('', $event->source); - $this->assertSame('', $event->subject); - $this->assertSame('', $event->time); - $this->assertSame([], $event->data); - } - - /** - * A producer that adds a field must not break a consumer written before - * it, which is what makes a feed safe to evolve. - */ - public function testIgnoresUnknownFields(): void - { - $event = Event::fromArray([ - 'id' => '1-0', - 'type' => 'test', - 'dataschema' => 'https://example.com/schema.json', - 'somethingnew' => ['a' => 'b'], - ]); - - $this->assertSame('1-0', $event->id); - $this->assertSame('test', $event->type); - } - - public function testCoercesAMalformedFieldToItsDefaultRatherThanFailing(): void - { - $event = Event::fromArray([ - 'id' => '1-0', - 'type' => ['not', 'a', 'string'], - 'data' => 'not an array', - ]); - - $this->assertSame('', $event->type); - $this->assertSame([], $event->data); - } - - public function testRoundTripsThroughItsArrayForm(): void - { - $event = new Event( - id: '1-0', - type: 'io.appwrite.edge.invalidate', - data: ['tags' => ['project' => 'proj-1']], - source: 'urn:appwrite:cloud:fra', - subject: 'proj-1', - time: '2026-07-29T12:00:00.000Z', - ); - - $this->assertEquals($event, Event::fromArray($event->toArray())); - } - - public function testSurvivesAJsonRoundTrip(): void - { - $event = new Event(id: '1-0', type: 'test', data: ['nested' => ['a' => 1]]); - - $encoded = \json_encode($event->toArray()); - $this->assertIsString($encoded); - - $decoded = Event::fromArray((array) \json_decode($encoded, true)); - - $this->assertSame(['nested' => ['a' => 1]], $decoded->data); - } - - public function testAlwaysReportsTheSpecVersionAndContentType(): void - { - $encoded = (new Event(id: '1-0', type: 'test'))->toArray(); - - $this->assertSame('1.0', $encoded['specversion']); - $this->assertSame('application/json', $encoded['datacontenttype']); - } - - /** - * Every CloudEvents attribute is emitted, and none of them are null, even - * on an event that set almost nothing. - * - * This is what makes the wire form portable: a stricter CloudEvents reader - * β€” including `utopia-php/cloudevents`, whose `fromArray()` reads - * `specversion`, `source`, `id` and `time` without defaulting them β€” can - * consume it directly. Dropping an empty field here, or letting one be - * null, would break those readers without breaking any test that only - * round-trips through this class. - */ - public function testTheWireFormIsPortableToStricterCloudEventsReaders(): void - { - $encoded = (new Event(id: '1-0', type: 'test'))->toArray(); - - $this->assertSame([ - 'specversion', - 'id', - 'type', - 'source', - 'time', - 'subject', - 'datacontenttype', - 'data', - ], \array_keys($encoded)); - - foreach ($encoded as $field => $value) { - $this->assertNotNull($value, "The {$field} attribute must never be null on the wire"); - } - } - - public function testReadsPayloadKeysWithADefault(): void - { - $event = new Event(id: '1-0', type: 'test', data: ['tags' => ['a' => 'b']]); - - $this->assertSame(['a' => 'b'], $event->getData('tags')); - $this->assertNull($event->getData('missing')); - $this->assertSame('fallback', $event->getData('missing', 'fallback')); - } - - public function testWithIdLeavesEverythingElseAlone(): void - { - $event = new Event(id: '', type: 'test', data: ['a' => 'b'], subject: 's'); - $stamped = $event->withId('7-0'); - - $this->assertSame('7-0', $stamped->id); - $this->assertSame('test', $stamped->type); - $this->assertSame(['a' => 'b'], $stamped->data); - $this->assertSame('s', $stamped->subject); - $this->assertSame('', $event->id, 'The original must not be mutated'); - } - - public function testNowIsRfc3339WithMilliseconds(): void - { - $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', Event::now()); - } -} diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index c920083..ef0a38b 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -7,7 +7,7 @@ use PHPUnit\Framework\TestCase; use Utopia\Feed\Adapter\Memory; use Utopia\Feed\Adapter\None; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; @@ -75,7 +75,7 @@ public function testEventsComeBackOldestFirst(): void $this->feed->append($type); } - $this->assertSame(['a', 'b', 'c'], \array_map(fn (Event $e): string => $e->type, $this->feed->read())); + $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $this->feed->read())); } public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void @@ -165,7 +165,7 @@ public function testRejectsAPayloadThatCannotBeEncoded(): void public function testPublishStampsAPreparedEvent(): void { - $id = $this->feed->publish(new Event(id: 'ignored', type: 'test', data: ['a' => 'b'], subject: 's')); + $id = $this->feed->publish(new CloudEvent(id: 'ignored', type: 'test', data: ['a' => 'b'], subject: 's')); $event = $this->feed->read()[0]; @@ -177,11 +177,91 @@ public function testPublishStampsAPreparedEvent(): void public function testPublishKeepsATimeTheCallerSet(): void { - $this->feed->publish(new Event(id: '', type: 'test', time: '2020-01-01T00:00:00.000Z')); + $this->feed->publish(new CloudEvent(id: '', type: 'test', time: '2020-01-01T00:00:00.000Z')); $this->assertSame('2020-01-01T00:00:00.000Z', $this->feed->read()[0]->time); } + /** + * A producer that attaches a `traceparent` means it to reach the consumer. + * Stamping the event on publish rebuilds it, and storing it flattens it, so + * either step could quietly drop an attribute this library does not model. + */ + public function testExtensionAttributesSurviveAppendAndRead(): void + { + $this->feed->publish(new CloudEvent( + id: '', + type: 'test', + extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2], + )); + + $event = $this->feed->read()[0]; + + $this->assertSame('00-abc-def-01', $event->getExtension('traceparent')); + $this->assertSame(2, $event->getExtension('retrycount')); + $this->assertSame('urn:appwrite:cloud:fra', $event->source, 'Stamping still happened'); + } + + public function testDataschemaSurvivesAppendAndRead(): void + { + $this->feed->publish(new CloudEvent( + id: '', + type: 'test', + dataschema: 'https://example.com/schema.json', + )); + + $this->assertSame('https://example.com/schema.json', $this->feed->read()[0]->dataschema); + } + + /** + * @return array + */ + public static function payloads(): array + { + return [ + 'map' => [['tags' => ['domain' => 'example.com']]], + 'list' => [['a', 'b', 'c']], + 'nested list' => [[['x' => 1], ['x' => 2]]], + 'string' => ['a string'], + 'number' => [42], + 'float' => [1.5], + 'boolean' => [true], + 'null' => [null], + 'empty' => [[]], + ]; + } + + /** + * The JSON event format leaves `data` unrestricted, so a list or a scalar + * has to survive as itself β€” a list must not come back as a map. + * + * @dataProvider payloads + */ + public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void + { + $this->feed->append('test', $data); + + $this->assertSame($data, $this->feed->read()[0]->data); + } + + /** + * CloudEvents models an absent subject as null rather than an empty string, + * so a caller checking for one must check for null. + */ + public function testAnEventWithNoSubjectHasANullSubject(): void + { + $this->feed->append('test'); + + $this->assertNull($this->feed->read()[0]->subject); + } + + public function testASubjectSurvivesAppendAndRead(): void + { + $this->feed->append('test', [], 'example.com'); + + $this->assertSame('example.com', $this->feed->read()[0]->subject); + } + public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void { $this->feed->append('test'); @@ -220,7 +300,7 @@ public function testRetentionIsBoundedAndTrimsTheOldest(): void $feed->append($type); } - $this->assertSame(['c', 'd', 'e'], \array_map(fn (Event $e): string => $e->type, $feed->read())); + $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read())); } /** @@ -236,7 +316,7 @@ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void $feed->append('c'); $this->assertSame(2, $adapter->count()); - $this->assertSame(['b', 'c'], \array_map(fn (Event $e): string => $e->type, $feed->read($first))); + $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read($first))); } public function testExposesItsIdentity(): void diff --git a/tests/Feed/Unit/HttpAdapterTest.php b/tests/Feed/Unit/HttpAdapterTest.php index a8280d0..7ab94da 100644 --- a/tests/Feed/Unit/HttpAdapterTest.php +++ b/tests/Feed/Unit/HttpAdapterTest.php @@ -10,7 +10,7 @@ use Utopia\Feed\Adapter\Http; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Memory as MemoryCursor; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; @@ -35,8 +35,8 @@ private function feed(array $responses = []): array public function testReadsAFeedOverHttp(): void { [$feed] = $this->feed([FakeTransport::json(Protocol::encode([ - new Event(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']]), - new Event(id: '1-1', type: 'io.appwrite.edge.invalidate'), + new CloudEvent(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']]), + new CloudEvent(id: '1-1', type: 'io.appwrite.edge.invalidate'), ]))]); $events = $feed->read(); @@ -230,7 +230,7 @@ public function testCannotAppendToAFeedItDoesNotOwn(): void */ public function testWorksThroughTheClientItself(): void { - $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new Event(id: '1-0', type: 'a')]))]); + $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-0', type: 'a')]))]); $client = (new Client($transport))->withHeaders(['x-appwrite-jwt' => 'token']); $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); @@ -249,10 +249,10 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void { [$feed, $transport] = $this->feed([ FakeTransport::json(Protocol::encode([ - new Event(id: '1-0', type: 'a'), - new Event(id: '1-1', type: 'b'), + new CloudEvent(id: '1-0', type: 'a'), + new CloudEvent(id: '1-1', type: 'b'), ])), - FakeTransport::json(Protocol::encode([new Event(id: '1-2', type: 'c')])), + FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-2', type: 'c')])), FakeTransport::json(Protocol::encode([])), ]); @@ -260,7 +260,7 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void $consumer = new Consumer($feed, 'invalidator', $cursor); $seen = []; - $handler = function (Event $event) use (&$seen): void { + $handler = function (CloudEvent $event) use (&$seen): void { $seen[] = $event->type; }; diff --git a/tests/Feed/Unit/ProtocolTest.php b/tests/Feed/Unit/ProtocolTest.php index 3173f36..0451e17 100644 --- a/tests/Feed/Unit/ProtocolTest.php +++ b/tests/Feed/Unit/ProtocolTest.php @@ -5,7 +5,7 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Protocol; @@ -30,8 +30,8 @@ public function testQueryCarriesTheParametersThatWereSet(): void public function testEncodesABatch(): void { $payload = Protocol::encode([ - new Event(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), - new Event(id: '1-1', type: 'b'), + new CloudEvent(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), + new CloudEvent(id: '1-1', type: 'b'), ]); $this->assertSame(2, $payload['total']); @@ -49,8 +49,8 @@ public function testEncodesAnEmptyBatch(): void public function testDecodesWhatItEncoded(): void { $events = [ - new Event(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), - new Event(id: '1-1', type: 'b'), + new CloudEvent(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), + new CloudEvent(id: '1-1', type: 'b'), ]; $this->assertEquals($events, Protocol::decode(Protocol::encode($events))); @@ -76,6 +76,22 @@ public function testRejectsAMalformedEventsField(): void Protocol::decode(['events' => 'nope']); } + /** + * One event as a producer would put it on the wire. + * + * @param array $overrides + * @return array + */ + private static function raw(string $id, string $type, array $overrides = []): array + { + return \array_merge([ + 'specversion' => '1.0', + 'id' => $id, + 'type' => $type, + 'source' => 'urn:test', + ], $overrides); + } + /** * An event with no id has no position, so a consumer cannot record having * passed it. Returning the usable prefix lets those events be handled and @@ -86,22 +102,22 @@ public function testKeepsTheEventsBeforeAnUndecodableOne(): void { $events = Protocol::decode([ 'events' => [ - ['id' => '1-0', 'type' => 'a'], - ['id' => '1-1', 'type' => 'b'], - ['type' => 'no id'], - ['id' => '1-3', 'type' => 'd'], + self::raw('1-0', 'a'), + self::raw('1-1', 'b'), + self::raw('', 'no id'), + self::raw('1-3', 'd'), ], ]); $this->assertCount(2, $events); - $this->assertSame(['a', 'b'], \array_map(fn (Event $e): string => $e->type, $events)); + $this->assertSame(['a', 'b'], \array_map(fn (CloudEvent $e): string => $e->type, $events)); } public function testFailsWhenTheFirstEventIsUndecodable(): void { $this->expectException(Invalid::class); - Protocol::decode(['events' => [['type' => 'no id'], ['id' => '1-1', 'type' => 'b']]]); + Protocol::decode(['events' => [self::raw('', 'no id'), self::raw('1-1', 'b')]]); } public function testFailsWhenTheFirstEntryIsNotAnEvent(): void @@ -113,9 +129,44 @@ public function testFailsWhenTheFirstEntryIsNotAnEvent(): void public function testKeepsTheEventsBeforeAnEntryThatIsNotAnEvent(): void { - $events = Protocol::decode(['events' => [['id' => '1-0', 'type' => 'a'], 'a string']]); + $events = Protocol::decode(['events' => [self::raw('1-0', 'a'), 'a string']]); + + $this->assertCount(1, $events); + } + + /** + * `specversion` is REQUIRED by the spec and a feed's own producer always + * sends it, so an entry without one is not a CloudEvent at all β€” the batch + * stops there rather than the attribute being invented. + */ + public function testFailsWhenAnEventIsNotACloudEventAtAll(): void + { + $this->expectException(Invalid::class); + + Protocol::decode(['events' => [['id' => '1-0', 'type' => 'a']]]); + } + + /** + * The forward-compatibility property a feed depends on: it is read by + * consumers older than the producer by design, so a producer that adds an + * attribute or moves the spec version forward must not stop one that + * predates it. + */ + public function testSurvivesAProducerThatMovedAhead(): void + { + $events = Protocol::decode([ + 'events' => [ + self::raw('1-0', 'a', [ + 'specversion' => '1.1', + 'somethingnew' => 'ignored', + 'traceparent' => '00-abc-def-01', + ]), + ], + ]); $this->assertCount(1, $events); + $this->assertSame('1.1', $events[0]->specversion); + $this->assertSame('00-abc-def-01', $events[0]->getExtension('traceparent')); } /** diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index e135c3b..501ef26 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -11,7 +11,7 @@ use Utopia\Feed\Adapter\Memory as MemoryAdapter; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Cache as CacheCursor; -use Utopia\Feed\Event; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Feed; use Utopia\Feed\Protocol; use Utopia\Tests\Unit\Support\FeedServer; @@ -58,16 +58,18 @@ public function testAnEventSurvivesTheWholeTrip(): void ); $received = null; - $this->consumer()->consume(function (Event $event) use (&$received): void { + $this->consumer()->consume(function (CloudEvent $event) use (&$received): void { $received = $event; }); - $this->assertInstanceOf(Event::class, $received); + $this->assertInstanceOf(CloudEvent::class, $received); $this->assertSame('io.appwrite.edge.invalidate-rule', $received->type); $this->assertSame('urn:appwrite:cloud:fra', $received->source); $this->assertSame('example.com', $received->subject); - $this->assertSame(['domain' => 'example.com'], $received->getData('tags')); - $this->assertTrue($received->getData('isAppwriteNetwork')); + $this->assertSame([ + 'tags' => ['domain' => 'example.com'], + 'isAppwriteNetwork' => true, + ], $received->data); } public function testTheConsumerOnlyEverSeesEachEventOnce(): void @@ -78,7 +80,7 @@ public function testTheConsumerOnlyEverSeesEachEventOnce(): void $consumer = $this->consumer(); $seen = []; - $handler = function (Event $event) use (&$seen): void { + $handler = function (CloudEvent $event) use (&$seen): void { $seen[] = $event->type; }; @@ -105,7 +107,7 @@ public function testAConsumerShippedLateDrainsTheBacklog(): void } $seen = []; - $handled = $this->consumer()->consume(function (Event $event) use (&$seen): void { + $handled = $this->consumer()->consume(function (CloudEvent $event) use (&$seen): void { $seen[] = $event->type; }); @@ -122,12 +124,12 @@ public function testARestartedConsumerResumesWhereItLeftOff(): void $this->producer->append('a'); $this->producer->append('b'); - $this->consumer()->consume(fn (Event $event) => null); + $this->consumer()->consume(fn (CloudEvent $event) => null); $this->producer->append('c'); $seen = []; - $this->consumer()->consume(function (Event $event) use (&$seen): void { + $this->consumer()->consume(function (CloudEvent $event) use (&$seen): void { $seen[] = $event->type; }); @@ -146,7 +148,7 @@ public function testAFailedEventBlocksTheOnesBehindItUntilItSucceeds(): void // Fails the first time it sees the poison event and succeeds after, // standing in for a dependency that was briefly unavailable. - $handler = function (Event $event) use (&$seen, &$attempts): void { + $handler = function (CloudEvent $event) use (&$seen, &$attempts): void { if ($event->type === 'poison') { $attempts++; @@ -181,10 +183,10 @@ public function testTheProducerCachesFullBatchesAndNothingElse(): void $consumer = $this->consumer(batch: 2); - $consumer->consume(fn (Event $event) => null); - $consumer->consume(fn (Event $event) => null); - $consumer->consume(fn (Event $event) => null); - $consumer->consume(fn (Event $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); $this->assertSame([ 'private, max-age=31536000', // 2 of 2 β€” settled history @@ -201,11 +203,11 @@ public function testTwoConsumersOfOneProducerAreIndependent(): void $one = $this->consumer('one'); $two = $this->consumer('two'); - $this->assertSame(1, $one->consume(fn (Event $event) => null)); + $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); $this->producer->append('b'); - $this->assertSame(2, $two->consume(fn (Event $event) => null), 'The second consumer starts from the beginning'); - $this->assertSame(1, $one->consume(fn (Event $event) => null), 'The first only sees what is new to it'); + $this->assertSame(2, $two->consume(fn (CloudEvent $event) => null), 'The second consumer starts from the beginning'); + $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null), 'The first only sees what is new to it'); } } From da108fc5301af1ceb4dd348a25991f91ac3c72bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 15:55:32 +0200 Subject: [PATCH 05/68] Rename generic adapters to journals --- CHANGELOG.md | 6 +-- README.md | 42 +++++++++++-------- docs/migration.md | 8 ++-- phpstan.neon | 2 +- src/Feed/Cursor/Pool.php | 2 +- src/Feed/Exception/Unsupported.php | 4 +- src/Feed/Feed.php | 20 ++++----- src/Feed/Id.php | 2 +- src/Feed/{Adapter.php => Journal.php} | 16 ++++--- src/Feed/{Adapter => Journal}/Http.php | 18 ++++---- src/Feed/{Adapter => Journal}/Memory.php | 10 ++--- src/Feed/{Adapter => Journal}/None.php | 6 +-- src/Feed/{Adapter => Journal}/Pool.php | 14 +++---- src/Feed/{Adapter => Journal}/Redis.php | 6 +-- src/Feed/Protocol.php | 2 +- tests/Feed/E2E/RedisTest.php | 4 +- tests/Feed/Unit/ConsumerTest.php | 10 ++--- tests/Feed/Unit/FeedTest.php | 24 +++++------ ...ttpAdapterTest.php => HttpJournalTest.php} | 16 +++---- tests/Feed/Unit/RoundTripTest.php | 6 +-- tests/Feed/Unit/Support/FakeClient.php | 2 +- tests/Feed/Unit/Support/FakeTransport.php | 2 +- 22 files changed, 117 insertions(+), 105 deletions(-) rename src/Feed/{Adapter.php => Journal.php} (91%) rename src/Feed/{Adapter => Journal}/Http.php (93%) rename src/Feed/{Adapter => Journal}/Memory.php (95%) rename src/Feed/{Adapter => Journal}/None.php (93%) rename src/Feed/{Adapter => Journal}/Pool.php (77%) rename src/Feed/{Adapter => Journal}/Redis.php (97%) rename tests/Feed/Unit/{HttpAdapterTest.php => HttpJournalTest.php} (95%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a499c7..89951f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,11 @@ library. `CloudEvent` objects β€” this library defines no event type of its own, so a feed event is accepted anywhere a `CloudEvent` is, and `dataschema` and extension attributes survive an append and a read -- `Adapter\Redis`, `Adapter\Pool` β€” Redis streams, directly or over a pool -- `Adapter\Http` β€” another service's feed, read over the wire with +- `Journal\Redis`, `Journal\Pool` β€” Redis streams, directly or over a pool +- `Journal\Http` β€” another service's feed, read over the wire with [utopia-php/client](https://github.com/utopia-php/client); takes any of its adapters, so a pooled or Swoole coroutine transport drops straight in -- `Adapter\Memory`, `Adapter\None` β€” for tests, and for no backend configured +- `Journal\Memory`, `Journal\None` β€” for tests, and for no backend configured - `Consumer` β€” the pull loop, with at-least-once semantics and a durable position - `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory` β€” where that position lives - `Protocol` β€” the http-feeds wire contract, shared by producer and consumer, and diff --git a/README.md b/README.md index a287931..853f88c 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ on β€” a cache tag to drop, a record to refresh, a config to reload. - **Ordered, resumable log** β€” consumers page by event id, and hold their own position - **CloudEvents** β€” events *are* [`utopia-php/cloudevents`](https://github.com/utopia-php/cloudevents) events, as http-feeds requires -- **Adapters** β€” Redis streams, a pooled Redis, in-memory, or another service's feed over HTTP +- **Journals** β€” Redis streams, a pooled Redis, in-memory, or another service's feed over HTTP - **Long polling** β€” subscribe in near real time without hammering the producer - **Cursors** β€” positions in a Utopia cache, in Redis, or in memory - **Consumer** β€” the pull loop, the position bookkeeping and the at-least-once semantics, written once @@ -65,14 +65,14 @@ composer require utopia-php/feed ### Producing ```php -use Utopia\Feed\Adapter\Redis as RedisAdapter; +use Utopia\Feed\Journal\Redis as RedisJournal; use Utopia\Feed\Feed; $redis = new Redis(); $redis->connect('redis', 6379); $feed = new Feed( - new RedisAdapter($redis, 'edge'), + new RedisJournal($redis, 'edge'), source: 'urn:appwrite:cloud:fra', ); @@ -160,12 +160,12 @@ $handled = $consumer->consume(function (CloudEvent $event) use ($router) { }); ``` -Consuming **another service's** feed is the same code with a different adapter: +Consuming **another service's** feed is the same code with a different journal: ```php use Utopia\Client; use Utopia\Client\Adapter\Curl\Client as Curl; -use Utopia\Feed\Adapter\Http; +use Utopia\Feed\Journal\Http; $client = (new Client(new Curl())) ->withHeaders(['x-appwrite-jwt' => $token]) @@ -174,7 +174,7 @@ $client = (new Client(new Curl())) $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); ``` -Nothing above the adapter knows the events are arriving over the network, +Nothing above the journal knows the events are arriving over the network, including the long polling β€” `Http` hands the wait to the producer, so a poll is one held request rather than a client-side loop. @@ -195,24 +195,32 @@ while (true) { } ``` -## Adapters +## Journals -| Adapter | Use for | Append | Read | +A journal is where a feed's events actually live. The name is the one event +sourcing has long used for an append-only, strictly ordered record that is +replayed rather than mutated β€” Akka Persistence calls its pluggable storage +backends journals for the same reason. It is responsible for exactly two things: +assigning an ordered id on append, and returning the events strictly after a +given id. Everything else β€” long polling, cursors, the pull loop β€” sits above it +and is the same whichever journal is underneath. + +| Journal | Use for | Append | Read | | --- | --- | --- | --- | -| `Adapter\Redis` | Producing a feed on a Redis stream | βœ… | βœ… | -| `Adapter\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | βœ… | -| `Adapter\Http` | Consuming another service's feed, over [utopia-php/client](https://github.com/utopia-php/client) | ❌ | βœ… | -| `Adapter\Memory` | Tests, and single-process development | βœ… | βœ… | -| `Adapter\None` | No backend configured | ❌ | ❌ | +| `Journal\Redis` | Producing a feed on a Redis stream | βœ… | βœ… | +| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | βœ… | +| `Journal\Http` | Consuming another service's feed, over [utopia-php/client](https://github.com/utopia-php/client) | ❌ | βœ… | +| `Journal\Memory` | Tests, and single-process development | βœ… | βœ… | +| `Journal\None` | No backend configured | ❌ | ❌ | -`Adapter\Pool` is what most services producing a feed want: a long poll holds +`Journal\Pool` is what most services producing a feed want: a long poll holds its connection for the whole timeout, so reading through a shared client would block every other user of it. -`Adapter\None` throws on every operation rather than doing nothing, so a +`Journal\None` throws on every operation rather than doing nothing, so a misconfigured service fails at the point of use instead of silently dropping -events β€” which only shows up much later, somewhere else. `Adapter\Memory` -implements the same id and retention semantics as `Adapter\Redis`, including the +events β€” which only shows up much later, somewhere else. `Journal\Memory` +implements the same id and retention semantics as `Journal\Redis`, including the awkward parts like resuming from a trimmed position, so code tested against it behaves the same when it is swapped out. diff --git a/docs/migration.md b/docs/migration.md index 42a02b6..dfc186f 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -13,7 +13,7 @@ This is what maps onto what. | Was | Now | | --- | --- | -| `Feed::append()` / `read()` / `poll()` | `Feed`, on `Adapter\Pool` | +| `Feed::append()` / `read()` / `poll()` | `Feed`, on `Journal\Pool` | | `Feed::after()` β€” stream id arithmetic | `Id::after()` | | `Feed::getCursor()` / `saveCursor()` | `Cursor\Pool` | | `Consumer::consume()` | `Consumer::consume()` | @@ -39,7 +39,7 @@ class EdgeFeed extends Feed public function __construct(?Pool $pool, string $source, int $maxSize = 100_000) { parent::__construct( - $pool === null ? new None(self::NAME) : new Adapter\Pool($pool, self::NAME, $maxSize), + $pool === null ? new None(self::NAME) : new Journal\Pool($pool, self::NAME, $maxSize), $source, ); } @@ -61,7 +61,7 @@ class EdgeFeed extends Feed } ``` -The nullable pool becomes `Adapter\None`, which throws on use with the same +The nullable pool becomes `Journal\None`, which throws on use with the same intent as the old `pool()` guard: a feed with no backend must fail loudly rather than drop events. @@ -116,7 +116,7 @@ $domain = \is_array($tags) ? ($tags['domain'] ?? '') : ''; | `Feed\Cursor` | `Cursor\Cache` | | `Feed\Event` | `Utopia\CloudEvents\CloudEvent` β€” this library has no event type of its own | | `Feed\Event::FEED` and the type constants | Stay β€” they name cloud's feed and its events | -| `Manager::fetchFeed()` | `Adapter\Http`, over `utopia-php/client` | +| `Manager::fetchFeed()` | `Journal\Http`, over `utopia-php/client` | | `Consumer::TIMEOUT_MARGIN` | `Protocol::TIMEOUT_MARGIN` | | `Feed\Poller` | Stays β€” Swoole interval scheduling | | `Router\Invalidator` | Stays β€” it purges edge caches | diff --git a/phpstan.neon b/phpstan.neon index 7718dd8..ddd7db0 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -13,7 +13,7 @@ parameters: ignoreErrors: - message: '#Call to an undefined method Redis\|RedisCluster::(xAdd|xRange)\(\)#' - path: src/Feed/Adapter/Redis.php + path: src/Feed/Journal/Redis.php - message: '#Call to an undefined method Redis::(xAdd|xRange|xLen)\(\)#' path: tests/Feed/E2E/RedisTest.php diff --git a/src/Feed/Cursor/Pool.php b/src/Feed/Cursor/Pool.php index c4fa67e..ce27f92 100644 --- a/src/Feed/Cursor/Pool.php +++ b/src/Feed/Cursor/Pool.php @@ -10,7 +10,7 @@ /** * {@see Redis}, over a pooled connection. * - * Pairs with {@see \Utopia\Feed\Adapter\Pool}, and can share its pool: a + * Pairs with {@see \Utopia\Feed\Journal\Pool}, and can share its pool: a * cursor read is one `GET`, so it borrows a connection only for as long as * that takes. * diff --git a/src/Feed/Exception/Unsupported.php b/src/Feed/Exception/Unsupported.php index d5bd509..1e764b2 100644 --- a/src/Feed/Exception/Unsupported.php +++ b/src/Feed/Exception/Unsupported.php @@ -7,8 +7,8 @@ use Utopia\Feed\Exception; /** - * The adapter cannot do what was asked of it β€” appending to a feed read over - * HTTP, or any operation at all on {@see \Utopia\Feed\Adapter\None}. + * The journal cannot do what was asked of it β€” appending to a feed read over + * HTTP, or any operation at all on {@see \Utopia\Feed\Journal\None}. * * Thrown rather than ignored: an append that silently does nothing loses * events, and a consumer cannot tell an empty feed from an absent one. diff --git a/src/Feed/Feed.php b/src/Feed/Feed.php index 5ade3d7..23c9952 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Feed.php @@ -63,7 +63,7 @@ class Feed protected const int POLL_INTERVAL = 500_000; /** - * @param Adapter $adapter Where the events live. + * @param Journal $journal Where the events live. * @param string $source Who is producing them, as a URI reference * (`urn:appwrite:cloud:fra`). Stamped onto every event this * instance appends, so a consumer merging feeds from several @@ -71,19 +71,19 @@ class Feed * the feed is only being read. */ public function __construct( - protected readonly Adapter $adapter, + protected readonly Journal $journal, protected readonly string $source = '', ) { } - public function getAdapter(): Adapter + public function getJournal(): Journal { - return $this->adapter; + return $this->journal; } public function getName(): string { - return $this->adapter->getName(); + return $this->journal->getName(); } public function getSource(): string @@ -139,7 +139,7 @@ public function publish(CloudEvent $event): string // Stamped with the withers rather than rebuilt, so anything this // library does not model itself β€” a dataschema, an extension attribute // such as a traceparent β€” survives the append untouched. - return $this->adapter->append( + return $this->journal->append( $event ->withSource($this->source) ->withTime($event->time !== '' ? $event->time : null) @@ -158,7 +158,7 @@ public function publish(CloudEvent $event): string */ public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): array { - return $this->adapter->read($lastEventId, self::limit($limit)); + return $this->journal->read($lastEventId, self::limit($limit)); } /** @@ -187,14 +187,14 @@ public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, $limit = self::limit($limit); $timeout = \max(0, \min($timeout, self::MAX_TIMEOUT)); - if ($this->adapter->pollable()) { - return $this->adapter->read($lastEventId, $limit, $timeout); + if ($this->journal->pollable()) { + return $this->journal->read($lastEventId, $limit, $timeout); } $deadline = \microtime(true) + $timeout / 1000; while (true) { - $events = $this->adapter->read($lastEventId, $limit); + $events = $this->journal->read($lastEventId, $limit); if ($events !== [] || \microtime(true) >= $deadline) { return $events; diff --git a/src/Feed/Id.php b/src/Feed/Id.php index 5ad55cd..f4619f8 100644 --- a/src/Feed/Id.php +++ b/src/Feed/Id.php @@ -15,7 +15,7 @@ * where `seq` disambiguates events appended within the same millisecond. * * The format is part of the wire contract, not a Redis implementation detail: - * an id produced by one adapter has to be a valid position for another, so + * an id produced by one journal has to be a valid position for another, so * that a feed can move between backends without invalidating the positions * consumers already hold. */ diff --git a/src/Feed/Adapter.php b/src/Feed/Journal.php similarity index 91% rename from src/Feed/Adapter.php rename to src/Feed/Journal.php index c64a723..7dc53ee 100644 --- a/src/Feed/Adapter.php +++ b/src/Feed/Journal.php @@ -11,18 +11,22 @@ /** * Where a feed's events actually live. * - * An adapter is responsible for two things and nothing else: assigning an + * Named for what event sourcing has long called an append-only, strictly + * ordered record that is replayed rather than mutated β€” the same sense in which + * Akka Persistence calls its pluggable storage backends journals. + * + * A journal is responsible for two things and nothing else: assigning an * ordered id on append, and returning the events strictly after a given id. * Everything above that β€” long polling on backends that cannot do it * themselves, cursors, the pull loop β€” is the same regardless of the backend * and lives in {@see Feed} and {@see Consumer}. * - * Adapters split into producers (Redis, Pool, Memory), which own the events, - * and consumers ({@see Adapter\Http}), which read someone else's feed over the + * Journals split into producers (Redis, Pool, Memory), which own the events, + * and consumers ({@see Journal\Http}), which read someone else's feed over the * wire. The read side is identical either way, which is what lets a service * consume a remote feed with the same code it uses on a local one. */ -abstract class Adapter +abstract class Journal { /** * @param string $name Feed identifier. Also the key the backend stores it @@ -63,7 +67,7 @@ abstract public function append(CloudEvent $event): string; * * @param int $timeout Milliseconds to wait for an event before giving up, * honoured only when {@see pollable()} is true; {@see Feed::poll()} - * handles the wait for every other adapter. + * handles the wait for every other journal. * @return list * @throws Invalid When $lastEventId is not a feed position. * @throws Exception When the backend cannot be read. @@ -74,7 +78,7 @@ abstract public function read(?string $lastEventId, int $limit, int $timeout = 0 * Whether the backend blocks until an event arrives on its own. * * False here rather than abstract because polling in a loop works against - * anything; an adapter only overrides it when the backend can do better, + * anything; a journal only overrides it when the backend can do better, * and {@see Feed::poll()} then hands the wait over instead of sleeping. */ public function pollable(): bool diff --git a/src/Feed/Adapter/Http.php b/src/Feed/Journal/Http.php similarity index 93% rename from src/Feed/Adapter/Http.php rename to src/Feed/Journal/Http.php index 4cb5420..db2b386 100644 --- a/src/Feed/Adapter/Http.php +++ b/src/Feed/Journal/Http.php @@ -2,11 +2,11 @@ declare(strict_types=1); -namespace Utopia\Feed\Adapter; +namespace Utopia\Feed\Journal; +use Utopia\Feed\Journal; use Psr\Http\Client\ClientExceptionInterface; -use Utopia\Client\Adapter as ClientAdapter; -use Utopia\Feed\Adapter; +use Utopia\Client\Adapter; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; @@ -23,7 +23,7 @@ * The counterpart to serving a feed with {@see Protocol}: a service points * this at another service's feed endpoint and consumes it with the same * {@see Feed} and {@see \Utopia\Feed\Consumer} it would use on a local one. - * Nothing above the adapter knows the events are arriving over the network. + * Nothing above the journal knows the events are arriving over the network. * * Read-only, because a feed is owned by whoever appends to it. Long polling is * delegated to the producer, which is the point of doing it this way: the @@ -43,12 +43,12 @@ * * @see https://github.com/utopia-php/client */ -class Http extends Adapter +class Http extends Journal { private readonly RequestFactory $requests; /** - * @param ClientAdapter $client Configured with whatever credentials the + * @param Adapter $client Configured with whatever credentials the * producer requires. Typed as the client's own adapter interface * rather than plain PSR-18, because a read needs to set its own * deadline β€” which also means a `Retry` or `Pool` decorator can be @@ -64,7 +64,7 @@ class Http extends Adapter * @param string $name Feed name, as the producer knows it. */ public function __construct( - protected readonly ClientAdapter $client, + protected readonly Adapter $client, protected readonly string $endpoint, string $name, ?RequestFactory $requests = null, @@ -75,7 +75,7 @@ public function __construct( } /** - * The URL this adapter reads. + * The URL this journal reads. */ public function getUrl(): string { @@ -151,7 +151,7 @@ public function pollable(): bool * surfaces as a transport failure on every quiet tick β€” burying the * failures that matter. A plain read keeps whatever the caller configured. */ - private function client(int $timeout): ClientAdapter + private function client(int $timeout): Adapter { if ($timeout <= 0) { return $this->client; diff --git a/src/Feed/Adapter/Memory.php b/src/Feed/Journal/Memory.php similarity index 95% rename from src/Feed/Adapter/Memory.php rename to src/Feed/Journal/Memory.php index db4d944..c9f5946 100644 --- a/src/Feed/Adapter/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Utopia\Feed\Adapter; +namespace Utopia\Feed\Journal; -use Utopia\Feed\Adapter; +use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Id; @@ -19,7 +19,7 @@ * Not for production: nothing is shared between processes and nothing survives * a restart, so consumers in another worker see an empty feed. */ -class Memory extends Adapter +class Memory extends Journal { /** @var list */ private array $events = []; @@ -55,7 +55,7 @@ public function append(CloudEvent $event): string $id = Id::encode($this->timestamp, $this->sequence); // Stored through the same encode/decode a real backend goes through, - // rather than holding the object. Otherwise this adapter would accept + // rather than holding the object. Otherwise this journal would accept // payloads that cannot be serialized and hand back values that survived // a round trip they would not survive in production β€” which is the one // way a stand-in like this actively causes harm. @@ -71,7 +71,7 @@ public function append(CloudEvent $event): string public function read(?string $lastEventId, int $limit, int $timeout = 0): array { // Validates the position even when nothing will be returned, so a - // malformed cursor fails the same way it does on every other adapter + // malformed cursor fails the same way it does on every other journal // instead of only once the feed has events in it. $after = $lastEventId === null ? null : Id::decode($lastEventId); diff --git a/src/Feed/Adapter/None.php b/src/Feed/Journal/None.php similarity index 93% rename from src/Feed/Adapter/None.php rename to src/Feed/Journal/None.php index 438fdc5..38eddbd 100644 --- a/src/Feed/Adapter/None.php +++ b/src/Feed/Journal/None.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Utopia\Feed\Adapter; +namespace Utopia\Feed\Journal; -use Utopia\Feed\Adapter; +use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Unsupported; @@ -20,7 +20,7 @@ * later somewhere else β€” a cache that never invalidates, a replica that never * catches up. If dropping events is genuinely acceptable, use {@see Memory}. */ -class None extends Adapter +class None extends Journal { public function __construct(string $name = 'none') { diff --git a/src/Feed/Adapter/Pool.php b/src/Feed/Journal/Pool.php similarity index 77% rename from src/Feed/Adapter/Pool.php rename to src/Feed/Journal/Pool.php index 5ca023b..03c8df1 100644 --- a/src/Feed/Adapter/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Utopia\Feed\Adapter; +namespace Utopia\Feed\Journal; -use Utopia\Feed\Adapter; +use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Pools\Pool as UtopiaPool; @@ -17,7 +17,7 @@ * * @see https://github.com/utopia-php/pools */ -class Pool extends Adapter +class Pool extends Journal { /** * @param UtopiaPool<\Redis|\RedisCluster> $pool @@ -34,22 +34,22 @@ public function __construct( public function append(CloudEvent $event): string { - return $this->pool->use(fn (\Redis|\RedisCluster $redis): string => $this->adapter($redis)->append($event)); + return $this->pool->use(fn (\Redis|\RedisCluster $redis): string => $this->journal($redis)->append($event)); } public function read(?string $lastEventId, int $limit, int $timeout = 0): array { return $this->pool->use( - fn (\Redis|\RedisCluster $redis): array => $this->adapter($redis)->read($lastEventId, $limit, $timeout) + fn (\Redis|\RedisCluster $redis): array => $this->journal($redis)->read($lastEventId, $limit, $timeout) ); } /** * The connection is only borrowed for the length of one call, so the - * adapter wrapping it is built per call too. It holds no state beyond the + * journal wrapping it is built per call too. It holds no state beyond the * connection, which makes that free. */ - private function adapter(\Redis|\RedisCluster $redis): Redis + private function journal(\Redis|\RedisCluster $redis): Redis { return new Redis($redis, $this->name, $this->maxSize); } diff --git a/src/Feed/Adapter/Redis.php b/src/Feed/Journal/Redis.php similarity index 97% rename from src/Feed/Adapter/Redis.php rename to src/Feed/Journal/Redis.php index 8a0f58e..95cd271 100644 --- a/src/Feed/Adapter/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Utopia\Feed\Adapter; +namespace Utopia\Feed\Journal; -use Utopia\Feed\Adapter; +use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; @@ -24,7 +24,7 @@ * bound on memory, not a promise about how far back a consumer can resume * from. */ -class Redis extends Adapter +class Redis extends Journal { /** * @param \Redis|\RedisCluster $redis diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index ec05720..ec6407f 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -13,7 +13,7 @@ * * A producer serving a feed and a consumer reading one have to agree on the * query parameters, the response body and the caching rules. Both halves live - * here so they cannot drift: {@see Adapter\Http} reads through it, and a + * here so they cannot drift: {@see Journal\Http} reads through it, and a * producer builds its endpoint's response with it β€” whichever HTTP framework * that endpoint happens to be written in, which is why this deals in arrays * rather than in requests and responses. diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index dbd9878..0ce9fc2 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -5,7 +5,7 @@ namespace Utopia\Tests\E2E; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Adapter\Redis as RedisAdapter; +use Utopia\Feed\Journal\Redis as RedisJournal; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Redis as RedisCursor; use Utopia\CloudEvents\CloudEvent; @@ -49,7 +49,7 @@ protected function tearDown(): void private function feed(int $maxSize = 100_000): Feed { - return new Feed(new RedisAdapter($this->redis, $this->name, $maxSize), 'urn:test:e2e'); + return new Feed(new RedisJournal($this->redis, $this->name, $maxSize), 'urn:test:e2e'); } public function testAppendsAndReadsBack(): void diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 1bdb86a..1504a6e 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -5,7 +5,7 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Adapter\Memory as MemoryAdapter; +use Utopia\Feed\Journal\Memory as MemoryJournal; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; use Utopia\Feed\Cursor\Memory as MemoryCursor; @@ -16,7 +16,7 @@ class ConsumerTest extends TestCase { - private MemoryAdapter $adapter; + private MemoryJournal $journal; private Feed $feed; @@ -24,8 +24,8 @@ class ConsumerTest extends TestCase protected function setUp(): void { - $this->adapter = new MemoryAdapter('edge'); - $this->feed = new Feed($this->adapter, 'urn:test'); + $this->journal = new MemoryJournal('edge'); + $this->feed = new Feed($this->journal, 'urn:test'); $this->cursor = new MemoryCursor('edge'); } @@ -310,7 +310,7 @@ public function testAFailedReadLeavesThePositionAlone(): void $this->feed->append('b'); $this->cursor->save('invalidator', $first); - $consumer = new Consumer(new Feed(new \Utopia\Feed\Adapter\None('edge')), 'invalidator', $this->cursor); + $consumer = new Consumer(new Feed(new \Utopia\Feed\Journal\None('edge')), 'invalidator', $this->cursor); $this->expectException(\Utopia\Feed\Exception\Unsupported::class); diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index ef0a38b..fb43107 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -5,8 +5,8 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Adapter\Memory; -use Utopia\Feed\Adapter\None; +use Utopia\Feed\Journal\Memory; +use Utopia\Feed\Journal\None; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; @@ -15,14 +15,14 @@ class FeedTest extends TestCase { - private Memory $adapter; + private Memory $journal; private Feed $feed; protected function setUp(): void { - $this->adapter = new Memory('edge'); - $this->feed = new Feed($this->adapter, 'urn:appwrite:cloud:fra'); + $this->journal = new Memory('edge'); + $this->feed = new Feed($this->journal, 'urn:appwrite:cloud:fra'); } public function testAppendReturnsAPosition(): void @@ -60,10 +60,10 @@ public function testStampsTheSourceAndTimeOnAppend(): void */ public function testKeepsTheSourceOfTheProducerThatAppended(): void { - (new Feed($this->adapter, 'urn:appwrite:cloud:fra'))->append('test'); - (new Feed($this->adapter, 'urn:appwrite:cloud:nyc'))->append('test'); + (new Feed($this->journal, 'urn:appwrite:cloud:fra'))->append('test'); + (new Feed($this->journal, 'urn:appwrite:cloud:nyc'))->append('test'); - $events = (new Feed($this->adapter, 'urn:appwrite:cloud:syd'))->read(); + $events = (new Feed($this->journal, 'urn:appwrite:cloud:syd'))->read(); $this->assertSame('urn:appwrite:cloud:fra', $events[0]->source); $this->assertSame('urn:appwrite:cloud:nyc', $events[1]->source); @@ -309,13 +309,13 @@ public function testRetentionIsBoundedAndTrimsTheOldest(): void */ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void { - $feed = new Feed($adapter = new Memory('small', maxSize: 2)); + $feed = new Feed($journal = new Memory('small', maxSize: 2)); $first = $feed->append('a'); $feed->append('b'); $feed->append('c'); - $this->assertSame(2, $adapter->count()); + $this->assertSame(2, $journal->count()); $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read($first))); } @@ -323,7 +323,7 @@ public function testExposesItsIdentity(): void { $this->assertSame('edge', $this->feed->getName()); $this->assertSame('urn:appwrite:cloud:fra', $this->feed->getSource()); - $this->assertSame($this->adapter, $this->feed->getAdapter()); + $this->assertSame($this->journal, $this->feed->getJournal()); } public function testAnUnconfiguredBackendFailsLoudlyRatherThanDroppingEvents(): void @@ -354,7 +354,7 @@ public function testRejectsAnEmptyFeedName(): void public function testFlushingMemoryDoesNotReissuePositions(): void { $before = $this->feed->append('a'); - $this->adapter->flush(); + $this->journal->flush(); $after = $this->feed->append('b'); $this->assertSame(1, Id::compare($after, $before), 'A reissued position would make a consumer skip events'); diff --git a/tests/Feed/Unit/HttpAdapterTest.php b/tests/Feed/Unit/HttpJournalTest.php similarity index 95% rename from tests/Feed/Unit/HttpAdapterTest.php rename to tests/Feed/Unit/HttpJournalTest.php index 7ab94da..405a0f3 100644 --- a/tests/Feed/Unit/HttpAdapterTest.php +++ b/tests/Feed/Unit/HttpJournalTest.php @@ -7,7 +7,7 @@ use PHPUnit\Framework\TestCase; use Psr\Http\Message\ResponseInterface; use Utopia\Client; -use Utopia\Feed\Adapter\Http; +use Utopia\Feed\Journal\Http; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Memory as MemoryCursor; use Utopia\CloudEvents\CloudEvent; @@ -18,7 +18,7 @@ use Utopia\Feed\Protocol; use Utopia\Tests\Unit\Support\FakeTransport; -class HttpAdapterTest extends TestCase +class HttpJournalTest extends TestCase { /** * @param list $responses @@ -27,9 +27,9 @@ class HttpAdapterTest extends TestCase private function feed(array $responses = []): array { $transport = FakeTransport::of($responses); - $adapter = new Http($transport, 'https://cloud.example.com/v1/feeds', 'edge'); + $journal = new Http($transport, 'https://cloud.example.com/v1/feeds', 'edge'); - return [new Feed($adapter), $transport]; + return [new Feed($journal), $transport]; } public function testReadsAFeedOverHttp(): void @@ -57,9 +57,9 @@ public function testAppendsTheFeedNameToTheEndpoint(): void public function testEncodesAFeedNameThatNeedsIt(): void { - $adapter = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds/', 'a b/c'); + $journal = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds/', 'a b/c'); - $this->assertSame('https://cloud.example.com/v1/feeds/a%20b%2Fc', $adapter->getUrl()); + $this->assertSame('https://cloud.example.com/v1/feeds/a%20b%2Fc', $journal->getUrl()); } public function testReadsWithGet(): void @@ -173,7 +173,7 @@ public function testRaisesServerErrors(): void } /** - * PSR-18 returns 4xx and 5xx rather than throwing, so the adapter has to + * PSR-18 returns 4xx and 5xx rather than throwing, so the journal has to * check the status itself β€” a producer error must not read as an empty * batch, which the consumer would take for "caught up". */ @@ -242,7 +242,7 @@ public function testWorksThroughTheClientItself(): void } /** - * The point of the adapter: a remote feed is consumed with exactly the + * The point of this journal: a remote feed is consumed with exactly the * code a local one is. */ public function testConsumesARemoteFeedThroughTheSameConsumer(): void diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index 501ef26..6a639ee 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -7,8 +7,8 @@ use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter\Memory as CacheMemory; use Utopia\Cache\Cache as UtopiaCache; -use Utopia\Feed\Adapter\Http; -use Utopia\Feed\Adapter\Memory as MemoryAdapter; +use Utopia\Feed\Journal\Http; +use Utopia\Feed\Journal\Memory as MemoryJournal; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Cache as CacheCursor; use Utopia\CloudEvents\CloudEvent; @@ -34,7 +34,7 @@ class RoundTripTest extends TestCase protected function setUp(): void { - $this->producer = new Feed(new MemoryAdapter('edge'), 'urn:appwrite:cloud:fra'); + $this->producer = new Feed(new MemoryJournal('edge'), 'urn:appwrite:cloud:fra'); $this->server = new FeedServer($this->producer); $this->consumerFeed = new Feed( diff --git a/tests/Feed/Unit/Support/FakeClient.php b/tests/Feed/Unit/Support/FakeClient.php index 688a56a..e2ce9f1 100644 --- a/tests/Feed/Unit/Support/FakeClient.php +++ b/tests/Feed/Unit/Support/FakeClient.php @@ -16,7 +16,7 @@ * * The `with*()` methods clone the way the real client does, rather than * mutating and returning `$this`. That matters: a test asserting that a long - * poll was given a longer deadline has to be able to fail if the adapter + * poll was given a longer deadline has to be able to fail if the journal * configured a clone and then sent through the original. */ abstract class FakeClient implements ClientAdapter diff --git a/tests/Feed/Unit/Support/FakeTransport.php b/tests/Feed/Unit/Support/FakeTransport.php index c5de32d..7ab4999 100644 --- a/tests/Feed/Unit/Support/FakeTransport.php +++ b/tests/Feed/Unit/Support/FakeTransport.php @@ -13,7 +13,7 @@ use Utopia\Psr7\Stream; /** - * A client that answers from a script, for driving the HTTP feed adapter + * A client that answers from a script, for driving the HTTP feed journal * through responses a real producer would be awkward to provoke. */ class FakeTransport extends FakeClient From 2e1f2b70bc38cf15613ac105f27b8cd901c917b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:09:46 +0200 Subject: [PATCH 06/68] Improve docs about each component --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/README.md b/README.md index 853f88c..08ade64 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,53 @@ on β€” a cache tag to drop, a record to refresh, a config to reload. - **Cursors** β€” positions in a Utopia cache, in Redis, or in memory - **Consumer** β€” the pull loop, the position bookkeeping and the at-least-once semantics, written once +## How the pieces fit + +The library splits along the same line the design does: the **producer owns the +events**, each **consumer owns its position**, and `Protocol` is the seam between +them when they live in different services. + +``` + PRODUCER β”‚ CONSUMER + β”‚ + append() β”‚ consume(handler) + ↓ β”‚ ↓ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Feed │───────▢│ Journal β”‚ β”‚ β”‚ Consumer │─────▢│ Cursor β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ \Redis β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ \Cache β”‚ + policy β”‚ \Pool β”‚ β”‚ the pull loop β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ \Memory β”‚ β”‚ β”‚ "where I got to" + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ ↓ + the events β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ β”‚ Feed │────▢│ Journal β”‚ + β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ \Http β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ Protocol │◀──────┼──── HTTP GET β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + the wire contract β”‚ +``` + +| | | +| --- | --- | +| **`Journal`** | Where the events live. Assigns an ordered id on append, returns the events after a given id β€” and nothing else. `Journal\Http` reads *another service's* journal, so to everything above it a remote feed and a local one are the same object. | +| **`Feed`** | The policy on one journal: stamps `source` and `time` on append, clamps a consumer-supplied `limit`, and long-polls. Subclass it to give a feed a typed vocabulary. | +| **`Cursor`** | Where one consumer's position is kept. Deliberately independent of `Journal` β€” a consumer keeps its position in *its own* storage, never the producer's. | +| **`Consumer`** | The pull loop. Reads from the stored position, hands each event to a handler oldest-first, and advances only past events the handler accepted. | +| **`Protocol`** | The HTTP contract β€” query parameters, response envelope, caching rules β€” held in one place so the two halves cannot drift apart. | + +The structural consequence worth knowing up front: **the producer stores no +per-consumer state at all.** That is what makes adding a consumer free, and it is +why `Cursor` is its own thing rather than a method on `Journal`. + +Dependencies only ever point one way, so each piece is testable alone β€” a +`Journal\Memory` and a `Cursor\Memory` exercise the whole pull loop with no Redis +and no network: + +``` +Consumer ──▢ Feed ──▢ Journal ──▢ Protocol (only Journal\Http) + └──────▢ Cursor +``` + ## Getting started Install using composer: From 9a5c13005dece73c8a32a618a689dd52653f4945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:10:23 +0200 Subject: [PATCH 07/68] Simplify readme --- README.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 08ade64..b800bd4 100644 --- a/README.md +++ b/README.md @@ -80,13 +80,25 @@ them when they live in different services. the wire contract β”‚ ``` -| | | -| --- | --- | -| **`Journal`** | Where the events live. Assigns an ordered id on append, returns the events after a given id β€” and nothing else. `Journal\Http` reads *another service's* journal, so to everything above it a remote feed and a local one are the same object. | -| **`Feed`** | The policy on one journal: stamps `source` and `time` on append, clamps a consumer-supplied `limit`, and long-polls. Subclass it to give a feed a typed vocabulary. | -| **`Cursor`** | Where one consumer's position is kept. Deliberately independent of `Journal` β€” a consumer keeps its position in *its own* storage, never the producer's. | -| **`Consumer`** | The pull loop. Reads from the stored position, hands each event to a handler oldest-first, and advances only past events the handler accepted. | -| **`Protocol`** | The HTTP contract β€” query parameters, response envelope, caching rules β€” held in one place so the two halves cannot drift apart. | +**`Journal`** β€” where the events live. It assigns an ordered id on append and +returns the events after a given id, and nothing else. `Journal\Http` reads +*another service's* journal, so to everything above it a remote feed and a local +one are the same object. + +**`Feed`** β€” the policy on one journal: stamps `source` and `time` on append, +clamps a consumer-supplied `limit`, and long-polls. Subclass it to give a feed a +typed vocabulary. + +**`Cursor`** β€” where one consumer's position is kept. Deliberately independent of +`Journal`: a consumer keeps its position in *its own* storage, never the +producer's. + +**`Consumer`** β€” the pull loop. Reads from the stored position, hands each event +to a handler oldest-first, and advances only past the events the handler +accepted. + +**`Protocol`** β€” the HTTP contract: query parameters, response envelope and +caching rules, held in one place so the two halves cannot drift apart. The structural consequence worth knowing up front: **the producer stores no per-consumer state at all.** That is what makes adding a consumer free, and it is From a4cf9da2523ebca9ee515de16e26ddf516ff1b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:13:52 +0200 Subject: [PATCH 08/68] Upgrade dependencies --- README.md | 3 ++- composer.lock | 8 ++++---- src/Feed/Journal.php | 8 +++++++- src/Feed/Protocol.php | 9 ++++++--- tests/Feed/Unit/FeedTest.php | 20 ++++++++++++++++++++ 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b800bd4..304c9b4 100644 --- a/README.md +++ b/README.md @@ -331,7 +331,8 @@ is deliberately not `CloudEvent::fromArray()`'s default: - **Strict about `id`.** For a feed the id *is* the consumer's position, so an event without one cannot be recorded as passed. The spec makes `id` required too; `Protocol` enforces exactly that one attribute rather than calling - `validate()`, which would also demand a `source` a feed has no use for. + `validate()`, which would also demand a well-formed URI-reference `source` β€” + a spec requirement, but not one a feed consumer depends on. - **Tolerant about everything else.** Decoding runs with `lenient: true` and `allowUnknownSpecversion: true`, so a producer that adds an attribute, omits an optional one, or moves the spec forward does not stop a consumer that diff --git a/composer.lock b/composer.lock index bd6060c..edd6adf 100644 --- a/composer.lock +++ b/composer.lock @@ -2056,12 +2056,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/cloudevents.git", - "reference": "9cbc8dfe6b4ad5273b4ed1729f5a35112f20660d" + "reference": "68cd50590b94e250970f226a0067caf471adf42e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cloudevents/zipball/9cbc8dfe6b4ad5273b4ed1729f5a35112f20660d", - "reference": "9cbc8dfe6b4ad5273b4ed1729f5a35112f20660d", + "url": "https://api.github.com/repos/utopia-php/cloudevents/zipball/68cd50590b94e250970f226a0067caf471adf42e", + "reference": "68cd50590b94e250970f226a0067caf471adf42e", "shasum": "" }, "require": { @@ -2100,7 +2100,7 @@ "issues": "https://github.com/utopia-php/cloudevents/issues", "source": "https://github.com/utopia-php/cloudevents/tree/feat-cloudevents-g2" }, - "time": "2026-07-29T13:37:00+00:00" + "time": "2026-07-29T13:53:17+00:00" }, { "name": "utopia-php/pools", diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index 7dc53ee..af92900 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -138,7 +138,6 @@ protected static function decode(string $id, array $fields): CloudEvent 'source' => self::field($fields, 'source'), 'time' => self::field($fields, 'time'), 'data' => \json_decode(self::field($fields, 'data'), true), - ...(\is_array($extensions) ? $extensions : []), ]; // The inverse of the normalization in encode(): these two are nullable @@ -154,6 +153,13 @@ protected static function decode(string $id, array $fields): CloudEvent } } + // The union operator rather than a spread, which renumbers integer keys. + // An extension name of only digits is legal β€” the spec allows [a-z0-9]+ β€” + // and PHP stores such a name as an int key, so a spread would silently + // rename "123" to the next free position and lose the attribute. + // Spec attributes stay on the left, so they win any collision. + $event += \is_array($extensions) ? $extensions : []; + try { return CloudEvent::fromArray($event, lenient: true); } catch (CloudEventsException $error) { diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index ec6407f..668ea7f 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -113,7 +113,9 @@ public static function query(?string $lastEventId = null, int $limit = 0, int $t * The response body for a batch. * * @param list $events - * @return array{total: int, events: list>} + * @return array{total: int, events: list>} The keys + * are not narrowed to strings because an extension attribute named + * only of digits is legal, and PHP holds such a name as an int key. */ public static function encode(array $events): array { @@ -136,8 +138,9 @@ public static function encode(array $events): array * The exception is `id`, which is enforced here and nowhere else in the * spec's terms: for a feed the id *is* the consumer's position, so an event * without one cannot be recorded as passed. {@see CloudEvent::validate()} - * requires `source` too, which a feed has no need of, so this checks the - * one attribute it actually depends on rather than calling it. + * also requires a well-formed URI-reference `source` β€” a spec requirement, + * but not one a feed consumer depends on β€” so this checks the single + * attribute it actually needs rather than calling it. * * Stops at the first event that cannot be decoded and returns the ones * before it, rather than dropping it and carrying on. Skipping it would diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index fb43107..6cf5f5f 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -202,6 +202,26 @@ public function testExtensionAttributesSurviveAppendAndRead(): void $this->assertSame('urn:appwrite:cloud:fra', $event->source, 'Stamping still happened'); } + /** + * An extension name of only digits is legal β€” the spec allows `[a-z0-9]+` β€” + * and PHP stores such a name as an integer key. Anything that merges the + * extensions back in with a spread, or with `array_merge()`, renumbers that + * key and silently loses the attribute. + */ + public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void + { + $this->feed->publish(new CloudEvent( + id: '', + type: 'test', + extensions: ['123' => 'digits', 'trace' => 'ok'], + )); + + $event = $this->feed->read()[0]; + + $this->assertSame('digits', $event->getExtension('123')); + $this->assertSame('ok', $event->getExtension('trace')); + } + public function testDataschemaSurvivesAppendAndRead(): void { $this->feed->publish(new CloudEvent( From 2e956112d69a5905c76b9a66eba189a3301cb6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:25:22 +0200 Subject: [PATCH 09/68] Fix CI base image and address review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI: appwrite/utopia-base:php-8.5-1.0.0 does not exist β€” 1.0.0 was only ever published for 8.3 and 8.4. Both the PHP 8.5 and PHPStan jobs failed at the FROM. Moved to 2.1.0, which is published for 8.3, 8.4 and 8.5. Protocol::decode() required the "events" field instead of defaulting it. A response with no envelope β€” a misrouted request, a proxy's JSON error page, an endpoint that moved β€” decoded to an empty batch, which a consumer reads as "caught up". That is indistinguishable from a genuinely empty feed and leaves the consumer parked at a position that never advances again. Journal now rejects a retention cap below one event. Non-positive caps meant opposite things per backend: Redis reads MAXLEN 0 as "keep nothing", while array_slice($events, -0) keeps everything, so the in-memory journal grew unbounded. Pinned actions/checkout to a commit SHA in all three workflows, so a moved upstream tag cannot change what CI executes. Documented that a consumer name belongs to one process: the position is written with a plain set, so two processes sharing a name can move it backwards. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/analysis.yml | 2 +- .github/workflows/linter.yml | 2 +- .github/workflows/tests.yml | 2 +- Dockerfile | 2 +- src/Feed/Consumer.php | 14 +++++++++--- src/Feed/Journal.php | 20 ++++++++++++++++ src/Feed/Journal/Memory.php | 2 ++ src/Feed/Journal/Pool.php | 2 ++ src/Feed/Journal/Redis.php | 2 ++ src/Feed/Protocol.php | 13 ++++++++++- tests/Feed/Unit/FeedTest.php | 39 ++++++++++++++++++++++++++++++++ tests/Feed/Unit/ProtocolTest.php | 30 +++++++++++++++++++++++- 12 files changed, 121 insertions(+), 9 deletions(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 0c6399e..3a34d76 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -9,7 +9,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 # Run against the test image rather than a bare composer container: the # Redis stream calls only resolve where ext-redis is actually installed. diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 1dbd22b..bdc8e8e 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -9,7 +9,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Run Linter run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ee80c4c..35db5c2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Build run: | diff --git a/Dockerfile b/Dockerfile index b02856f..7b656bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY composer.lock composer.json /src/ RUN composer install --ignore-platform-reqs --optimize-autoloader \ --no-plugins --no-scripts --prefer-dist -FROM appwrite/utopia-base:php-${PHP_VERSION}-1.0.0 AS final +FROM appwrite/utopia-base:php-${PHP_VERSION}-2.1.0 AS final LABEL maintainer="team@appwrite.io" diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index f498849..525e473 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -73,9 +73,17 @@ class Consumer /** * @param Feed $feed Feed to read. * @param string $name This consumer's name, which its position is stored - * under. Distinct per logical consumer, and stable across restarts - * and replicas β€” two processes sharing a name share a position and - * will each skip what the other handled. + * under. Distinct per logical consumer, and stable across restarts. + * + * **One process per name.** The position is written with a plain + * set, not a compare-and-set, so two processes sharing a name race: + * each skips what the other handled, and a slower one finishing a + * shorter batch later moves the stored position *backwards*, so a + * restart replays from there. Neither outcome loses events β€” + * delivery is at-least-once and handlers must tolerate a repeat + * regardless β€” but the work is wasted and progress stops being + * monotonic. Run one process per name, and let a rolling restart's + * brief overlap be absorbed by the handler's idempotence. * @param Cursor $cursor Where to keep the position. * @param int $batch Events per run. * @param int $timeout Milliseconds to wait for an event when the feed is diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index af92900..da7750a 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -86,6 +86,26 @@ public function pollable(): bool return false; } + /** + * Guard a retention cap. + * + * Non-positive values do not mean "no retention" β€” they mean something + * different on every backend, and nothing useful on any. Redis reads + * `MAXLEN 0` as "trim everything", so a feed would accept appends and + * retain none of them; `array_slice($events, -0)` is `array_slice($events, + * 0)`, so the in-memory journal would do the exact opposite and retain the + * lot, unbounded. A cap that silently means one thing here and the reverse + * there is worse than no cap, so it is rejected at construction. + * + * @throws Invalid When $maxSize would retain fewer than one event. + */ + protected static function assertRetention(int $maxSize): void + { + if ($maxSize < 1) { + throw new Invalid("Feed retention must be at least one event, got {$maxSize}"); + } + } + /** * The backend fields an event is stored as. * diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Journal/Memory.php index c9f5946..ee8899c 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -36,6 +36,8 @@ class Memory extends Journal public function __construct(string $name, protected readonly int $maxSize = 100_000) { parent::__construct($name); + + self::assertRetention($maxSize); } public function append(CloudEvent $event): string diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Journal/Pool.php index 03c8df1..42e84fa 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -30,6 +30,8 @@ public function __construct( protected readonly int $maxSize = 100_000, ) { parent::__construct($name); + + self::assertRetention($maxSize); } public function append(CloudEvent $event): string diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Journal/Redis.php index 95cd271..814634f 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -41,6 +41,8 @@ public function __construct( protected readonly int $maxSize = 100_000, ) { parent::__construct($name); + + self::assertRetention($maxSize); } /** diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 668ea7f..f6bbb8b 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -163,7 +163,18 @@ public static function decode(mixed $payload): array throw new Invalid('Expected a feed batch, got ' . \get_debug_type($payload)); } - $raw = $payload[self::KEY_EVENTS] ?? []; + // Required, not defaulted to empty. An empty batch and a response that + // is not a batch at all are the same bytes to a consumer that defaults + // it β€” and they mean opposite things: "you are caught up" versus "you + // did not reach the feed". A misrouted request, a proxy's JSON error + // page or an endpoint that moved would all read as a quiet, permanent + // caught-up state, which is the one failure a feed cannot afford to + // hide. + if (!\array_key_exists(self::KEY_EVENTS, $payload)) { + throw new Invalid('Feed batch is missing the "' . self::KEY_EVENTS . '" field'); + } + + $raw = $payload[self::KEY_EVENTS]; if (!\is_array($raw)) { throw new Invalid('Feed batch has a malformed "' . self::KEY_EVENTS . '" field'); } diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 6cf5f5f..0dfc6a2 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -371,6 +371,45 @@ public function testRejectsAnEmptyFeedName(): void new Memory(''); } + /** + * @return array + */ + public static function unusableRetention(): array + { + return [ + 'zero' => [0], + 'negative' => [-5], + ]; + } + + /** + * A non-positive cap means the opposite thing on each backend β€” Redis reads + * `MAXLEN 0` as "keep nothing", while `array_slice($events, -0)` keeps + * everything β€” so it is refused rather than silently honoured one way here + * and the other way in production. + * + * @dataProvider unusableRetention + */ + public function testRejectsARetentionCapThatWouldNotBoundTheFeed(int $maxSize): void + { + $this->expectException(Invalid::class); + + new Memory('edge', maxSize: $maxSize); + } + + public function testAcceptsTheSmallestUsefulRetentionCap(): void + { + $feed = new Feed(new Memory('edge', maxSize: 1)); + + $feed->append('a'); + $feed->append('b'); + + $events = $feed->read(); + + $this->assertCount(1, $events); + $this->assertSame('b', $events[0]->type); + } + public function testFlushingMemoryDoesNotReissuePositions(): void { $before = $this->feed->append('a'); diff --git a/tests/Feed/Unit/ProtocolTest.php b/tests/Feed/Unit/ProtocolTest.php index 0451e17..a708d98 100644 --- a/tests/Feed/Unit/ProtocolTest.php +++ b/tests/Feed/Unit/ProtocolTest.php @@ -59,7 +59,35 @@ public function testDecodesWhatItEncoded(): void public function testDecodesAnEmptyBatch(): void { $this->assertSame([], Protocol::decode(['total' => 0, 'events' => []])); - $this->assertSame([], Protocol::decode([])); + } + + /** + * An empty batch means "you are caught up". A response with no `events` + * field at all means "you did not reach the feed" β€” a misrouted request, a + * proxy's JSON error page, an endpoint that moved. Defaulting the missing + * field would make those indistinguishable, and a consumer would sit + * quietly at a position that never advances again. + * + * @dataProvider notBatches + */ + public function testARespondingEndpointThatIsNotAFeedIsNotMistakenForBeingCaughtUp(mixed $payload): void + { + $this->expectException(Invalid::class); + + Protocol::decode($payload); + } + + /** + * @return array + */ + public static function notBatches(): array + { + return [ + 'empty object' => [[]], + 'total but no events' => [['total' => 0]], + 'some other API' => [['data' => [], 'status' => 'ok']], + 'an error body' => [['message' => 'Not found', 'code' => 404]], + ]; } public function testRejectsAPayloadThatIsNotABatch(): void From 74f4c348ef10e1b9d0f0d028ce93d3f1854aa1cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:38:17 +0200 Subject: [PATCH 10/68] Stop a cursor position from moving backwards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two processes running the same consumer overlap during a rolling restart, which is a normal operation rather than a misconfiguration. They finish batches of different lengths, so the departing one's older position could land last and a later restart would replay everything between the two β€” on a busy feed, thousands of events. Positions are totally ordered, so Cursor::save() now compares before writing and drops anything that is not an advance. Cursor::shouldAdvance() holds the rule, and every implementation applies it. The guard fails open: a position that cannot be compared, because the store could not be read or because what came back is not a position, is treated as behind. It exists to stop a position going backwards and must never become a reason for one to stop going forwards. The comparison is not atomic. Closing the window entirely needs a compare-and-set in one operation, and each candidate costs more than the race does: Redis scores are doubles and cannot hold - exactly, WATCH leaves state on a connection about to return to a pool, and a one-entry stream changes the stored type and breaks cursors written by an earlier version. The window is microseconds against a poll interval of seconds, and losing the race costs a replay, which every handler already tolerates. Costs one extra read per advance. An idle poll saves nothing, so it still reads nothing. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 8 +++++ src/Feed/Consumer.php | 19 +++++----- src/Feed/Cursor.php | 59 +++++++++++++++++++++++++++++++- src/Feed/Cursor/Cache.php | 2 +- src/Feed/Cursor/Memory.php | 2 +- src/Feed/Cursor/Redis.php | 30 ++++++++-------- tests/Feed/E2E/RedisTest.php | 18 ++++++++++ tests/Feed/Unit/ConsumerTest.php | 9 ++++- tests/Feed/Unit/CursorTest.php | 59 ++++++++++++++++++++++++++++++++ 9 files changed, 178 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 304c9b4..6805dd2 100644 --- a/README.md +++ b/README.md @@ -303,6 +303,14 @@ a warning rather than a failure: it keeps its position in memory and carries on, and only a restart before the store recovers replays anything. Pass `onWarning()` to hear about it. +A position never moves **backwards**. Run one process per consumer name β€” but a +rolling restart briefly overlaps two, and without this the departing one +finishing a shorter batch would land its older position last and undo the +arriving one's progress. Positions are totally ordered, so `save()` compares +before writing and drops anything that is not an advance. The comparison is not +atomic, so a sub-millisecond interleave can still slip through; the cost of that +is a replay, which every handler must already tolerate. + ## Events There is no event type in this library. Events **are** diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 525e473..f8381ab 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -75,15 +75,16 @@ class Consumer * @param string $name This consumer's name, which its position is stored * under. Distinct per logical consumer, and stable across restarts. * - * **One process per name.** The position is written with a plain - * set, not a compare-and-set, so two processes sharing a name race: - * each skips what the other handled, and a slower one finishing a - * shorter batch later moves the stored position *backwards*, so a - * restart replays from there. Neither outcome loses events β€” - * delivery is at-least-once and handlers must tolerate a repeat - * regardless β€” but the work is wasted and progress stops being - * monotonic. Run one process per name, and let a rolling restart's - * brief overlap be absorbed by the handler's idempotence. + * **One process per name.** Two processes sharing a name each skip + * what the other handled, because neither sees the other's work + * before reading its own position. That is wasted effort, not lost + * events β€” delivery is at-least-once and handlers must tolerate a + * repeat regardless. + * + * The overlap a rolling restart creates is therefore safe, and + * {@see Cursor::save()} additionally refuses to move a stored + * position backwards, so the departing process cannot undo the + * arriving one's progress. * @param Cursor $cursor Where to keep the position. * @param int $batch Events per run. * @param int $timeout Milliseconds to wait for an event when the feed is diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 4dbc13b..5db396e 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -53,7 +53,7 @@ public function getFeed(): string abstract public function load(string $consumer): ?string; /** - * Record a position. + * Record a position, never moving one backwards. * * Only ever call this once the events up to $eventId have been handled. A * position saved ahead of the work it stands for turns a crash into @@ -61,10 +61,67 @@ abstract public function load(string $consumer): ?string; * recover from β€” the events are still in the feed, but nothing will ever * read them again. * + * A position that would move *backwards* is dropped instead. Two processes + * running the same consumer overlap during a rolling restart β€” a normal + * operation, not a misconfiguration β€” and they finish batches of different + * lengths: without this, the slower one's older position lands last and a + * later restart replays everything between the two, which on a busy feed is + * potentially thousands of events. Positions are totally ordered + * ({@see Id::compare()}), so "never backwards" is decidable here in a way it + * is not for a cursor store in general. + * + * **The check is not atomic.** It reads, compares, then writes, so two + * processes can still interleave inside that window and leave the older + * position stored. Closing it entirely needs a compare-and-set the store + * can do in one operation, and every candidate costs more than the race + * does: Redis scores are doubles and cannot hold `-` exactly, + * `WATCH` leaves state on a connection that is about to go back into a + * pool, and a one-entry stream β€” which would be exact, since these ids + * *are* stream ids β€” changes the stored type and so breaks cursors written + * by an earlier version. The window is microseconds against a poll interval + * of seconds, and losing the race costs a replay, which every handler must + * already tolerate. It is a bounded, safe outcome, not a lost event. + * * @throws Exception When the store cannot be written. */ abstract public function save(string $consumer, string $eventId): void; + /** + * Whether $eventId is worth storing for $consumer β€” that is, whether it is + * a real position and a later one than what is already there. + * + * The check every implementation applies before writing, unless it can do + * the same thing atomically ({@see Cursor\Redis} runs it as one script). + * + * Deliberately fails open. A position that cannot be compared β€” because the + * store could not be read, or because what came back is not a position β€” + * is treated as behind, so real progress replaces it. The guard exists to + * stop a position going backwards, and it must never become a reason for + * one to stop going forwards. + */ + protected function shouldAdvance(string $consumer, string $eventId): bool + { + if ($eventId === '') { + return false; + } + + try { + $current = $this->load($consumer); + } catch (\Throwable) { + return true; + } + + if ($current === null || $current === '') { + return true; + } + + try { + return Id::compare($eventId, $current) > 0; + } catch (Invalid) { + return true; + } + } + /** * Forget a consumer's position, so its next read starts from the oldest * retained event. diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php index 93ae7c6..4947ccc 100644 --- a/src/Feed/Cursor/Cache.php +++ b/src/Feed/Cursor/Cache.php @@ -52,7 +52,7 @@ public function load(string $consumer): ?string public function save(string $consumer, string $eventId): void { - if ($eventId === '') { + if (!$this->shouldAdvance($consumer, $eventId)) { return; } diff --git a/src/Feed/Cursor/Memory.php b/src/Feed/Cursor/Memory.php index cc0c1d4..879e61a 100644 --- a/src/Feed/Cursor/Memory.php +++ b/src/Feed/Cursor/Memory.php @@ -25,7 +25,7 @@ public function load(string $consumer): ?string public function save(string $consumer, string $eventId): void { - if ($eventId === '') { + if (!$this->shouldAdvance($consumer, $eventId)) { return; } diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 11137cd..5e1dba8 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -27,23 +27,9 @@ public function __construct( parent::__construct($feed); } - public function load(string $consumer): ?string - { - $key = $this->key($consumer); - - try { - /** @var mixed $cursor */ - $cursor = $this->redis->get($key); - } catch (\RedisException $error) { - throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); - } - - return \is_string($cursor) && $cursor !== '' ? $cursor : null; - } - public function save(string $consumer, string $eventId): void { - if ($eventId === '') { + if (!$this->shouldAdvance($consumer, $eventId)) { return; } @@ -59,6 +45,20 @@ public function save(string $consumer, string $eventId): void } } + public function load(string $consumer): ?string + { + $key = $this->key($consumer); + + try { + /** @var mixed $cursor */ + $cursor = $this->redis->get($key); + } catch (\RedisException $error) { + throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } + + return \is_string($cursor) && $cursor !== '' ? $cursor : null; + } + public function reset(string $consumer): void { $key = $this->key($consumer); diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index 0ce9fc2..0908367 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -263,6 +263,24 @@ public function testResetReplaysTheRetainedFeed(): void $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume(fn (CloudEvent $e) => null)); } + /** + * The rolling-restart case, against a real store: the departing process + * finishing a shorter batch must not undo the arriving one's progress. + */ + public function testAPositionNeverMovesBackwards(): void + { + $cursor = new RedisCursor($this->redis, $this->name); + + $cursor->save('invalidator', '1690000000000-5'); + $cursor->save('invalidator', '1690000000000-2'); + + $this->assertSame('1690000000000-5', $cursor->load('invalidator')); + + $cursor->save('invalidator', '1690000000001-0'); + + $this->assertSame('1690000000001-0', $cursor->load('invalidator'), 'A genuine advance still lands'); + } + public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void { (new RedisCursor($this->redis, $this->name))->save('invalidator', '1-0'); diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 1504a6e..88d5cab 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -111,11 +111,18 @@ public function load(string $consumer): ?string $consumer = $this->consumer($cursor); + // Restores the position, then advances β€” the advance reads once more to + // check it is not moving the position backwards. $consumer->consume(fn (CloudEvent $event) => null); + $settled = $cursor->loads; + + // Caught up: nothing handled, so nothing saved, so nothing read. This + // is the property that matters β€” an idle consumer polling on a timer + // does not touch its store at all. $consumer->consume(fn (CloudEvent $event) => null); $consumer->consume(fn (CloudEvent $event) => null); - $this->assertSame(1, $cursor->loads); + $this->assertSame($settled, $cursor->loads, 'A caught-up poll must not read the store'); } public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index 5b536ca..afa43f9 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -66,6 +66,65 @@ public function testConsumersDoNotShareAPosition(Cursor $cursor): void $this->assertSame('2-0', $cursor->load('two')); } + /** + * Two processes running the same consumer overlap during a rolling + * restart, finish batches of different lengths, and write out of order. + * Without this the older position lands last and a later restart replays + * everything between the two. + * + * @dataProvider stores + */ + public function testAPositionNeverMovesBackwards(Cursor $cursor): void + { + $cursor->save('invalidator', '1690000000000-5'); + $cursor->save('invalidator', '1690000000000-2'); + + $this->assertSame('1690000000000-5', $cursor->load('invalidator')); + } + + /** + * Positions are compared by their parts, not as strings β€” `10-0` is later + * than `9-0` but sorts before it, so a string comparison would reject a + * legitimate advance and stall the consumer permanently. + * + * @dataProvider stores + */ + public function testAdvancingAcrossADigitBoundaryIsNotMistakenForGoingBackwards(Cursor $cursor): void + { + $cursor->save('invalidator', '9-0'); + $cursor->save('invalidator', '10-0'); + + $this->assertSame('10-0', $cursor->load('invalidator')); + } + + /** + * @dataProvider stores + */ + public function testRewritingTheSamePositionIsAccepted(Cursor $cursor): void + { + $cursor->save('invalidator', '1-0'); + $cursor->save('invalidator', '1-0'); + + $this->assertSame('1-0', $cursor->load('invalidator')); + } + + /** + * The guard must never become a reason a position stops moving forwards. + * + * @dataProvider stores + */ + public function testAStoredValueThatIsNotAPositionIsReplaced(Cursor $cursor): void + { + $cursor->save('invalidator', '1-0'); + $cursor->reset('invalidator'); + + // Whatever a hand-edited or corrupted store hands back, real progress + // must still be able to overwrite it. + $cursor->save('invalidator', '2-0'); + + $this->assertSame('2-0', $cursor->load('invalidator')); + } + /** * @dataProvider stores */ From 9390ca1f6a311984bde664c5c28d4092dfb86c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:46:13 +0200 Subject: [PATCH 11/68] Document the cursor ordering guarantee and its limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read-compare-write in Cursor::save() is not atomic, so two processes sharing a consumer name can interleave and leave the older position stored. That was reasoned about but only thinly recorded, which leaves the next reader to redo the analysis. Consumer listed three reasons a handler must tolerate seeing an event twice. This is a fourth, and naming it there makes it part of the stated contract rather than an unlisted edge: every one of the four re-delivers, none of them skips, and that asymmetry is the point. Cursor::save() now carries why the window is left open β€” each compare-and-set candidate and what it costs, including that Cursor\Cache has no portable one at all, since a Utopia cache reports generation '0' on adapters without leases. A fix that held on some cache backends and silently not on others would be worse than one uniform, stated guarantee. Also drops a stale reference to a Lua script that no longer exists. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 10 +++++++- src/Feed/Consumer.php | 19 +++++++++++---- src/Feed/Cursor.php | 55 +++++++++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6805dd2..80362be 100644 --- a/README.md +++ b/README.md @@ -353,12 +353,20 @@ is specified to carry. ## Delivery semantics -**A handler must be safe to run twice on the same event.** There are three +**A handler must be safe to run twice on the same event.** There are four independent reasons, and no arrangement of this library removes any of them: 1. A handler can succeed and the position then fail to save. 2. A run interrupted partway resumes from the last event that succeeded. 3. A consumer whose position was lost restarts from the oldest retained event. +4. Two processes sharing a consumer name can interleave inside `Cursor::save()`'s + read-compare-write and leave the older position stored, re-delivering what + the newer one had already handled. + +Every one of them **re-delivers; none of them skips.** That asymmetry is the +whole design β€” an event handled twice is absorbed by an idempotent handler, +whereas an event stepped over is gone, still sitting in the feed with nothing +that will ever read it again. **A handler rejects an event by throwing.** That stops the run at that event and leaves the position before it, so the next run starts there and tries again. diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index f8381ab..e43e559 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -22,11 +22,20 @@ * ## What the handler must tolerate * * Delivery is at-least-once, so a handler will see the same event more than - * once and must be safe to repeat. There are three separate reasons, and no - * arrangement of this class removes any of them: a handler can succeed and the - * position fail to save; a batch can be interrupted partway and replay from - * the last event that succeeded; and a consumer whose position is lost - * restarts from the oldest retained event. + * once and must be safe to repeat. There are four separate reasons, and no + * arrangement of this class removes any of them: + * + * 1. A handler can succeed and the position then fail to save. + * 2. A batch interrupted partway replays from the last event that succeeded. + * 3. A consumer whose position is lost restarts from the oldest retained event. + * 4. Two processes sharing a consumer name can interleave inside the + * read-compare-write in {@see Cursor::save()} and leave the older position + * stored, re-delivering what the newer one had already handled. + * + * Every one of them re-delivers; none of them skips. That asymmetry is the + * whole design β€” an event handled twice is absorbed by an idempotent handler, + * whereas an event stepped over is gone, still sitting in the feed with nothing + * that will ever read it again. * * A handler rejects an event by throwing. That stops the run at that event and * leaves the position before it, so the next run starts there and tries again. diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 5db396e..6bb82a6 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -23,6 +23,12 @@ * * One store can hold positions for many consumers of the same feed, keyed by * consumer name. + * + * Positions move forwards only. {@see save()} compares before writing and drops + * anything that is not an advance, which matters because a rolling restart runs + * two processes under one name for a moment. That comparison is not atomic, and + * the reasoning for leaving it that way β€” including what the residual race + * costs, which is a replay rather than a lost event β€” is on `save()`. */ abstract class Cursor { @@ -70,17 +76,41 @@ abstract public function load(string $consumer): ?string; * ({@see Id::compare()}), so "never backwards" is decidable here in a way it * is not for a cursor store in general. * - * **The check is not atomic.** It reads, compares, then writes, so two - * processes can still interleave inside that window and leave the older - * position stored. Closing it entirely needs a compare-and-set the store - * can do in one operation, and every candidate costs more than the race - * does: Redis scores are doubles and cannot hold `-` exactly, - * `WATCH` leaves state on a connection that is about to go back into a - * pool, and a one-entry stream β€” which would be exact, since these ids - * *are* stream ids β€” changes the stored type and so breaks cursors written - * by an earlier version. The window is microseconds against a poll interval - * of seconds, and losing the race costs a replay, which every handler must - * already tolerate. It is a bounded, safe outcome, not a lost event. + * ## The check is not atomic + * + * It reads, compares, then writes. Two processes can interleave inside that + * window, both decide they are ahead, and leave the older position stored. + * + * **The consequence is a replay, which is the delivery guarantee rather + * than a departure from it.** A regressed position re-delivers events that + * were already handled; it never skips one, never loses one, and never + * advances past work that did not happen. Handlers are required to tolerate + * a repeat for three other reasons already β€” see {@see Consumer} β€” so this + * adds a fourth cause of something they must survive regardless, not a new + * kind of failure. + * + * Closing the window needs a compare-and-set the store performs in one + * operation, and it is not reachable across this abstraction: + * + * - `ZADD ... GT` is the native primitive, but scores are doubles and + * cannot hold `-` exactly once the sequence is packed in. + * - `WATCH`/`MULTI` needs no scripting, but leaves watch state on a + * connection that is about to go back into a pool. + * - A one-entry stream via `XADD` would be exact, since these ids *are* + * Redis stream ids and Redis refuses to move one backwards natively β€” but + * it changes the stored type, so cursors written by an earlier version + * stop being readable. + * - {@see Cursor\Cache} has no portable compare-and-set at all: a Utopia + * cache exposes leases, but `getGeneration()` returns `'0'` on adapters + * that do not implement them. Any fix here would hold on some cache + * backends and silently not on others, which is worse than one uniform, + * stated guarantee. + * + * So the guarantee is deliberately the weaker, uniform one: a position + * never moves backwards **except** under a sub-millisecond interleave + * between two processes sharing a consumer name, whose cost is bounded + * replay. Run one process per name ({@see Consumer::__construct()}) and it + * cannot arise at all. * * @throws Exception When the store cannot be written. */ @@ -90,8 +120,7 @@ abstract public function save(string $consumer, string $eventId): void; * Whether $eventId is worth storing for $consumer β€” that is, whether it is * a real position and a later one than what is already there. * - * The check every implementation applies before writing, unless it can do - * the same thing atomically ({@see Cursor\Redis} runs it as one script). + * The check every implementation applies before writing. * * Deliberately fails open. A position that cannot be compared β€” because the * store could not be read, or because what came back is not a position β€” From 60e12dbd5ad83f9d5bc9cdc0b64841dabe2036f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:51:51 +0200 Subject: [PATCH 12/68] Make Redis cursor advances atomic with a one-entry stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A feed position is a Redis stream id β€” - is the format XADD allocates β€” and Redis refuses to append an id equal to or smaller than the one at the top of a stream. Storing the position as the id of a one-entry stream makes "a position never moves backwards" the server's rule, enforced by the same operation that writes it, so two processes racing during a rolling restart cannot both decide they are ahead. It also drops a round trip: XADD replaces the read-compare-write. Cursor\Cache keeps the non-atomic guard. A Utopia cache has no portable compare-and-set β€” getGeneration() returns '0' on adapters without lease support β€” so a fix there would hold on some backends and silently not on others, which is worse than one clearly stated guarantee. Documented per implementation rather than as one blanket claim. The upgrade from string-valued keys is transparent: load() reads a string where it finds one, and the next save() replaces the key in place. The position is carried across, because the consumer is advancing past exactly what the string held, so nothing replays. docs/migration.md carries the operational notes β€” in particular that rolling back to a build predating this hits WRONGTYPE, and that anything reading these keys directly needs XREVRANGE rather than GET. Also drops the phpstan ignore rules for the Redis stream commands. They were added under phpstan 1.x; 2.2's bundled stubs cover these methods, so the rules matched nothing and, with reportUnmatchedIgnoredErrors off, would have silently masked real errors later. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 21 ++-- docs/migration.md | 43 +++++++- phpstan.neon | 13 --- src/Feed/Consumer.php | 8 +- src/Feed/Cursor.php | 54 ++++------ src/Feed/Cursor/Redis.php | 191 +++++++++++++++++++++++++++++++---- tests/Feed/E2E/RedisTest.php | 77 ++++++++++++++ 7 files changed, 333 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 80362be..67c3381 100644 --- a/README.md +++ b/README.md @@ -306,10 +306,16 @@ and only a restart before the store recovers replays anything. Pass A position never moves **backwards**. Run one process per consumer name β€” but a rolling restart briefly overlaps two, and without this the departing one finishing a shorter batch would land its older position last and undo the -arriving one's progress. Positions are totally ordered, so `save()` compares -before writing and drops anything that is not an advance. The comparison is not -atomic, so a sub-millisecond interleave can still slip through; the cost of that -is a replay, which every handler must already tolerate. +arriving one's progress. Positions are totally ordered, so `save()` drops +anything that is not an advance. + +`Cursor\Redis` and `Cursor\Pool` enforce that **atomically**, by storing the +position as the id of a one-entry Redis stream: a feed position *is* a stream +id, and Redis itself refuses an `XADD` that is not newer than the stream's top. +The check and the write are one operation, so two processes racing cannot both +believe they are ahead. Other stores compare before writing, which leaves a +sub-millisecond window; losing that race costs a replay, which every handler +must already tolerate. ## Events @@ -359,9 +365,10 @@ independent reasons, and no arrangement of this library removes any of them: 1. A handler can succeed and the position then fail to save. 2. A run interrupted partway resumes from the last event that succeeded. 3. A consumer whose position was lost restarts from the oldest retained event. -4. Two processes sharing a consumer name can interleave inside `Cursor::save()`'s - read-compare-write and leave the older position stored, re-delivering what - the newer one had already handled. +4. Two processes sharing a consumer name can interleave inside `Cursor::save()` + and leave the older position stored, re-delivering what the newer one had + already handled. Not possible on `Cursor\Redis` or `Cursor\Pool`, which + refuse a stale position atomically. Every one of them **re-delivers; none of them skips.** That asymmetry is the whole design β€” an event handled twice is absorbed by an idempotent handler, diff --git a/docs/migration.md b/docs/migration.md index dfc186f..7a7e647 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -173,8 +173,9 @@ These were load-bearing in the original implementations and are preserved: failed event is retried on the next run, and everything behind it waits. - **A cursor store that is down is a warning, not a failure.** The position is mirrored in memory, so the consumer keeps working and only a restart replays. -- **The `feed::cursor:` key format**, so consumers keep their - positions across the migration instead of replaying the retained feed. +- **The `feed::cursor:` key**, so consumers keep their positions + across the migration instead of replaying the retained feed. Its *type* + changes β€” see the note below. - **`-` event ids**, so positions already handed out stay valid. - **The margin a consumer allows its HTTP client over the long-poll timeout**, without which every quiet tick surfaces as a transport failure. @@ -195,3 +196,41 @@ These were load-bearing in the original implementations and are preserved: is nullable, so an event without one reads back as `null` rather than `''`; `data` is unrestricted, so a list or scalar payload round-trips as itself; and `dataschema` and extension attributes now survive an append and a read. + +## ⚠️ Redis cursors change type β€” read this before deploying + +`Cursor\Redis` and `Cursor\Pool` now store a consumer's position as **the id of +a one-entry Redis stream**, where cloud's implementation stored it as a plain +string under the same key. + +The reason is atomicity. A feed position *is* a Redis stream id β€” `-` +is the format `XADD` allocates β€” and Redis refuses to append an id equal to or +smaller than the one at the top of a stream. Storing the position that way makes +"a position never moves backwards" the server's rule, enforced in the same +operation that writes it. A rolling restart briefly runs two processes under one +consumer name, and without this the departing one finishing a shorter batch +could land its older position last and undo the arriving one's progress. + +**The upgrade is transparent and needs no manual step.** `load()` reads a +string-valued key where it finds one, and the next `save()` replaces the key in +place with the stream form. The position is carried across, because the consumer +is advancing past exactly the value the string held β€” so no events are replayed. + +What this does mean: + +- **Do not roll back** to a build that predates this change while a cursor key + holds a stream. The old code issues `GET` against it and gets `WRONGTYPE`, and + the consumer will fail to load its position on every poll. Recovering means + deleting the affected `feed:*:cursor:*` keys, after which consumers restart + from the oldest retained event β€” safe, but a burst of redundant work. +- **Anything reading these keys directly** β€” a dashboard, a runbook, an ops + script doing `GET feed:edge:cursor:fra` β€” needs updating to + `XREVRANGE feed:edge:cursor:fra + - COUNT 1` and to take the entry's *id*, not + its payload. The payload is a placeholder; a stream entry has to carry one + field, and the id is the whole value. +- **`Cursor\Cache` is unaffected.** It still stores a string, and it keeps the + non-atomic read-compare-write, because a Utopia cache has no portable + compare-and-set β€” `getGeneration()` returns `'0'` on adapters without lease + support, so a fix there would hold on some backends and silently not on + others. The edge poller uses this store; its residual race costs a replay, + which its invalidation handler already tolerates. diff --git a/phpstan.neon b/phpstan.neon index ddd7db0..bd1ec2f 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -4,16 +4,3 @@ parameters: - src - tests treatPhpDocTypesAsCertain: false - - # Stream commands landed in ext-redis 5. Analysis run without the - # extension loaded falls back to bundled stubs that predate them, so these - # only resolve where ext-redis is actually installed β€” which is the case in - # CI, and not the case on a machine that only runs the unit suite. - reportUnmatchedIgnoredErrors: false - ignoreErrors: - - - message: '#Call to an undefined method Redis\|RedisCluster::(xAdd|xRange)\(\)#' - path: src/Feed/Journal/Redis.php - - - message: '#Call to an undefined method Redis::(xAdd|xRange|xLen)\(\)#' - path: tests/Feed/E2E/RedisTest.php diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index e43e559..8dd083f 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -28,9 +28,11 @@ * 1. A handler can succeed and the position then fail to save. * 2. A batch interrupted partway replays from the last event that succeeded. * 3. A consumer whose position is lost restarts from the oldest retained event. - * 4. Two processes sharing a consumer name can interleave inside the - * read-compare-write in {@see Cursor::save()} and leave the older position - * stored, re-delivering what the newer one had already handled. + * 4. Two processes sharing a consumer name can interleave inside + * {@see Cursor::save()} and leave the older position stored, re-delivering + * what the newer one had already handled. Not possible on + * {@see Cursor\Redis} or {@see Cursor\Pool}, where the store refuses a + * stale position atomically. * * Every one of them re-delivers; none of them skips. That asymmetry is the * whole design β€” an event handled twice is absorbed by an idempotent handler, diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 6bb82a6..89999ad 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -76,41 +76,31 @@ abstract public function load(string $consumer): ?string; * ({@see Id::compare()}), so "never backwards" is decidable here in a way it * is not for a cursor store in general. * - * ## The check is not atomic + * ## How strongly that holds depends on the store * - * It reads, compares, then writes. Two processes can interleave inside that - * window, both decide they are ahead, and leave the older position stored. + * {@see Cursor\Redis} and {@see Cursor\Pool} enforce it **atomically**: a + * position is a Redis stream id, so it is stored as the id of a one-entry + * stream and the server itself refuses an `XADD` that is not an advance. + * There is no window between deciding and writing for a second process to + * slip into. * - * **The consequence is a replay, which is the delivery guarantee rather - * than a departure from it.** A regressed position re-delivers events that - * were already handled; it never skips one, never loses one, and never - * advances past work that did not happen. Handlers are required to tolerate - * a repeat for three other reasons already β€” see {@see Consumer} β€” so this - * adds a fourth cause of something they must survive regardless, not a new - * kind of failure. + * Every other store falls back to this base implementation, which reads, + * compares, then writes. Two processes can interleave inside that window, + * both decide they are ahead, and leave the older position stored. There is + * no portable way to close it: {@see Cursor\Cache} would need a + * compare-and-set the cache does not have β€” leases exist, but + * `getGeneration()` returns `'0'` on adapters that do not implement them, so + * a fix would hold on some cache backends and silently not on others, which + * is worse than one clearly stated guarantee. * - * Closing the window needs a compare-and-set the store performs in one - * operation, and it is not reachable across this abstraction: - * - * - `ZADD ... GT` is the native primitive, but scores are doubles and - * cannot hold `-` exactly once the sequence is packed in. - * - `WATCH`/`MULTI` needs no scripting, but leaves watch state on a - * connection that is about to go back into a pool. - * - A one-entry stream via `XADD` would be exact, since these ids *are* - * Redis stream ids and Redis refuses to move one backwards natively β€” but - * it changes the stored type, so cursors written by an earlier version - * stop being readable. - * - {@see Cursor\Cache} has no portable compare-and-set at all: a Utopia - * cache exposes leases, but `getGeneration()` returns `'0'` on adapters - * that do not implement them. Any fix here would hold on some cache - * backends and silently not on others, which is worse than one uniform, - * stated guarantee. - * - * So the guarantee is deliberately the weaker, uniform one: a position - * never moves backwards **except** under a sub-millisecond interleave - * between two processes sharing a consumer name, whose cost is bounded - * replay. Run one process per name ({@see Consumer::__construct()}) and it - * cannot arise at all. + * **Where the window remains, losing the race costs a replay, which is the + * delivery guarantee rather than a departure from it.** A regressed position + * re-delivers events that were already handled; it never skips one, never + * loses one, and never advances past work that did not happen. Handlers must + * tolerate a repeat for three other reasons anyway β€” see {@see Consumer} β€” + * so this is a fourth cause of something they already survive, not a new + * kind of failure. Running one process per consumer name avoids it + * entirely. * * @throws Exception When the store cannot be written. */ diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 5e1dba8..25f0295 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -5,10 +5,12 @@ namespace Utopia\Feed\Cursor; use Utopia\Feed\Cursor; +use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; +use Utopia\Feed\Id; /** - * Positions kept in Redis, as plain keys alongside the stream. + * Positions kept in Redis, as a one-entry stream. * * For consumers running inside the producer β€” a job that turns the feed into * something else, a bridge to a system that cannot poll. They have no store of @@ -17,9 +19,31 @@ * Consumers reached over HTTP should not use this: keeping their positions in * the producer's Redis puts per-consumer state back on the producer, which is * exactly what the feed is arranged to avoid. + * + * ## Why a stream and not a string + * + * A feed position *is* a Redis stream id β€” `-` is the format `XADD` + * allocates β€” and Redis refuses to append an id equal to or smaller than the + * one already at the top of a stream. Storing the position as the id of a + * single-entry stream therefore makes "never move a position backwards" the + * server's rule rather than this client's, and it holds atomically: two + * processes racing cannot both decide they are ahead, because the losing `XADD` + * is rejected by the same operation that would have written it. + * + * That is the one guarantee {@see Cursor::save()}'s read-compare-write cannot + * give, and it costs a single round trip rather than two. + * + * `MAXLEN 1` keeps the stream at exactly the current position; the entry's + * payload is unused, since the id carries the whole value. */ class Redis extends Cursor { + /** + * Placeholder field. A stream entry must carry at least one field, but the + * position lives in the entry's id, so nothing reads this. + */ + private const string FIELD = 'p'; + public function __construct( protected readonly \Redis|\RedisCluster $redis, string $feed, @@ -27,46 +51,179 @@ public function __construct( parent::__construct($feed); } + /** + * Advance the position, atomically. + * + * No read-compare-write here: `XADD` is itself the comparison, so a + * position that is not an advance is refused by Redis and treated as a + * no-op. That is the whole reason for the stream. + * + * @throws Invalid When $eventId is not a feed position, which a stream + * cannot store as an id. + * @throws Transport When Redis cannot be reached or refuses for any other + * reason. + */ public function save(string $consumer, string $eventId): void { - if (!$this->shouldAdvance($consumer, $eventId)) { + if ($eventId === '') { return; } + if (!Id::isValid($eventId)) { + throw new Invalid("Cannot store '{$eventId}' as a position: it is not a feed id"); + } + $key = $this->key($consumer); - try { - // Deliberately no expiry. Unlike a cache, this is the only copy, - // and a position that quietly expired would replay the whole - // retained feed the next time the consumer restarted. - $this->redis->set($key, $eventId); - } catch (\RedisException $error) { - throw new Transport("Failed to save the {$consumer} cursor: {$error->getMessage()}", previous: $error); + [$reply, $error] = $this->attempt( + fn (): mixed => $this->redis->xAdd($key, $eventId, [self::FIELD => '1'], 1, false) + ); + + if ($error === '') { + return; } + + // Redis rejected the id as not newer than what is stored: another + // process got further than this one, which is exactly the outcome the + // stream is here to produce. + if (self::rejectedAsStale($error)) { + return; + } + + // A position written by a version that stored these as plain strings. + // Replacing the key in place keeps the position β€” the caller is + // advancing past whatever the string held β€” so the upgrade costs no + // replay. See docs/migration.md. + if (self::wrongType($error)) { + $this->replaceLegacy($key, $consumer, $eventId); + + return; + } + + throw new Transport("Failed to save the {$consumer} cursor: {$error}"); } public function load(string $consumer): ?string { $key = $this->key($consumer); - try { - /** @var mixed $cursor */ - $cursor = $this->redis->get($key); - } catch (\RedisException $error) { - throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); + [$entries, $error] = $this->attempt( + fn (): mixed => $this->redis->xRevRange($key, '+', '-', 1) + ); + + if ($error !== '') { + // Written by a version that stored positions as plain strings. Read + // it where it is; the next save() converts the key. + if (self::wrongType($error)) { + return $this->loadLegacy($key, $consumer); + } + + throw new Transport("Failed to load the {$consumer} cursor: {$error}"); } - return \is_string($cursor) && $cursor !== '' ? $cursor : null; + if (!\is_array($entries) || $entries === []) { + return null; + } + + // The position is the entry's id, not its payload. + $id = \array_key_first($entries); + + return \is_string($id) && $id !== '' ? $id : null; } public function reset(string $consumer): void { $key = $this->key($consumer); + [, $error] = $this->attempt(fn (): mixed => $this->redis->del($key)); + + if ($error !== '') { + throw new Transport("Failed to reset the {$consumer} cursor: {$error}"); + } + } + + /** + * @throws Transport + */ + private function loadLegacy(string $key, string $consumer): ?string + { + [$cursor, $error] = $this->attempt(fn (): mixed => $this->redis->get($key)); + + if ($error !== '') { + throw new Transport("Failed to load the {$consumer} cursor: {$error}"); + } + + return \is_string($cursor) && $cursor !== '' ? $cursor : null; + } + + /** + * @throws Transport + */ + private function replaceLegacy(string $key, string $consumer, string $eventId): void + { + [, $error] = $this->attempt(fn (): mixed => $this->redis->del($key)); + + if ($error === '') { + [, $error] = $this->attempt( + fn (): mixed => $this->redis->xAdd($key, $eventId, [self::FIELD => '1'], 1, false) + ); + } + + if ($error !== '') { + throw new Transport("Failed to upgrade the {$consumer} cursor to a stream: {$error}"); + } + } + + /** + * Run a command and report Redis' refusal rather than letting it surface as + * two different things. + * + * phpredis signals a command-level error either by throwing or by returning + * `false` and parking the text in `getLastError()`, depending on the build + * and the connection's options. Both are normalised here so callers can ask + * one question β€” did Redis refuse, and what did it say β€” instead of each + * handling the split. + * + * @param \Closure(): mixed $command + * @return array{mixed, string} The reply, and Redis' error text when it + * refused, or an empty string when it did not. + */ + private function attempt(\Closure $command): array + { + $this->redis->clearLastError(); + try { - $this->redis->del($key); + /** @var mixed $reply */ + $reply = $command(); } catch (\RedisException $error) { - throw new Transport("Failed to reset the {$consumer} cursor: {$error->getMessage()}", previous: $error); + return [false, $error->getMessage() !== '' ? $error->getMessage() : 'Redis command failed']; } + + if ($reply !== false) { + return [$reply, '']; + } + + $error = $this->redis->getLastError(); + $this->redis->clearLastError(); + + return [false, \is_string($error) && $error !== '' ? $error : '']; + } + + /** + * Whether Redis refused an id for being at or behind the stream's top, + * which is this class's definition of "not an advance". + */ + private static function rejectedAsStale(string $error): bool + { + return \str_contains($error, 'equal or smaller'); + } + + /** + * Whether the key holds something other than a stream β€” in practice, a + * position written by a version that stored them as plain strings. + */ + private static function wrongType(string $error): bool + { + return \str_contains($error, 'WRONGTYPE'); } } diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index 0908367..674e983 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -266,6 +266,10 @@ public function testResetReplaysTheRetainedFeed(): void /** * The rolling-restart case, against a real store: the departing process * finishing a shorter batch must not undo the arriving one's progress. + * + * Here Redis enforces it rather than this library β€” the stale `XADD` is + * refused by the same operation that would have written it, so there is no + * window between the check and the write for a second process to slip into. */ public function testAPositionNeverMovesBackwards(): void { @@ -281,6 +285,79 @@ public function testAPositionNeverMovesBackwards(): void $this->assertSame('1690000000001-0', $cursor->load('invalidator'), 'A genuine advance still lands'); } + /** + * `10-0` is later than `9-0` but sorts before it as a string, so a + * comparison on the text would refuse a legitimate advance and stall the + * consumer permanently. Redis compares the parts. + */ + public function testAdvancingAcrossADigitBoundaryIsAccepted(): void + { + $cursor = new RedisCursor($this->redis, $this->name); + + $cursor->save('invalidator', '9-0'); + $cursor->save('invalidator', '10-0'); + + $this->assertSame('10-0', $cursor->load('invalidator')); + } + + public function testRewritingTheSamePositionIsAccepted(): void + { + $cursor = new RedisCursor($this->redis, $this->name); + + $cursor->save('invalidator', '5-0'); + $cursor->save('invalidator', '5-0'); + + $this->assertSame('5-0', $cursor->load('invalidator')); + } + + /** + * The position is the entry's id, and the stream never grows past it. + */ + public function testAPositionIsStoredAsAOneEntryStream(): void + { + $cursor = new RedisCursor($this->redis, $this->name); + $key = 'feed:' . $this->name . ':cursor:invalidator'; + + foreach (['1-0', '2-0', '3-0'] as $position) { + $cursor->save('invalidator', $position); + } + + $this->assertSame('stream', $this->redis->type($key)); + $this->assertSame(1, $this->redis->xLen($key), 'MAXLEN 1 keeps only the current position'); + } + + /** + * An earlier version stored these as plain strings. Reading has to find one + * where it is, and the next save has to replace it in place β€” otherwise a + * deployment upgrading this library would either hit WRONGTYPE forever or + * silently restart from the oldest retained event. + */ + public function testUpgradesAPositionWrittenAsAPlainString(): void + { + $cursor = new RedisCursor($this->redis, $this->name); + $key = 'feed:' . $this->name . ':cursor:invalidator'; + + $this->redis->set($key, '1690000000000-0'); + + $this->assertSame('1690000000000-0', $cursor->load('invalidator'), 'The old position is still readable'); + + $cursor->save('invalidator', '1690000000000-1'); + + $this->assertSame('stream', $this->redis->type($key), 'The key is converted on the next save'); + $this->assertSame('1690000000000-1', $cursor->load('invalidator'), 'No position is lost in the upgrade'); + } + + /** + * A stream id is the storage, so anything that is not one is refused with a + * clear message rather than surfacing as a Redis parse error. + */ + public function testRejectsACursorPositionThatIsNotAFeedId(): void + { + $this->expectException(Invalid::class); + + (new RedisCursor($this->redis, $this->name))->save('invalidator', 'not-a-position'); + } + public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void { (new RedisCursor($this->redis, $this->name))->save('invalidator', '1-0'); From c929c918af3e565be7243b7ec6c37bd02306cbe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 16:53:28 +0200 Subject: [PATCH 13/68] Fix E2E assertions for the stream-backed cursor phpredis type() returns an integer constant, not a type name, so comparing it to 'stream' could never hold. And testCursorsAreStoredUnderTheFeedTheyBelongTo still read the key with GET, which returns false now the position lives in a stream entry's id. All three were assertions rather than behaviour: monotonicity, the digit boundary, the legacy read and the invalid-id rejection all passed against a real Redis on the previous run. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/E2E/RedisTest.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index 674e983..195b451 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -322,7 +322,7 @@ public function testAPositionIsStoredAsAOneEntryStream(): void $cursor->save('invalidator', $position); } - $this->assertSame('stream', $this->redis->type($key)); + $this->assertSame(\Redis::REDIS_STREAM, $this->redis->type($key)); $this->assertSame(1, $this->redis->xLen($key), 'MAXLEN 1 keeps only the current position'); } @@ -343,7 +343,7 @@ public function testUpgradesAPositionWrittenAsAPlainString(): void $cursor->save('invalidator', '1690000000000-1'); - $this->assertSame('stream', $this->redis->type($key), 'The key is converted on the next save'); + $this->assertSame(\Redis::REDIS_STREAM, $this->redis->type($key), 'The key is converted on the next save'); $this->assertSame('1690000000000-1', $cursor->load('invalidator'), 'No position is lost in the upgrade'); } @@ -362,6 +362,11 @@ public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void { (new RedisCursor($this->redis, $this->name))->save('invalidator', '1-0'); - $this->assertSame('1-0', $this->redis->get('feed:' . $this->name . ':cursor:invalidator')); + // The position is the entry's id, so this reads the stream rather than + // the key's value. + $entries = $this->redis->xRevRange('feed:' . $this->name . ':cursor:invalidator', '+', '-', 1); + + $this->assertIsArray($entries); + $this->assertSame('1-0', \array_key_first($entries)); } } From e0e1999629a423e1135212fbce6cdb8a8ed58866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 17:28:27 +0200 Subject: [PATCH 14/68] Simplify implementation --- LICENSE.md | 20 ---- README.md | 22 +--- docs/migration.md | 43 +------ src/Feed/Consumer.php | 23 ++-- src/Feed/Cursor.php | 79 +----------- src/Feed/Cursor/Cache.php | 2 +- src/Feed/Cursor/Memory.php | 2 +- src/Feed/Cursor/Redis.php | 199 ++++--------------------------- src/Feed/Journal.php | 12 +- src/Feed/Journal/Memory.php | 4 + src/Feed/Journal/Pool.php | 2 + src/Feed/Journal/Redis.php | 2 + tests/Feed/E2E/RedisTest.php | 102 +--------------- tests/Feed/Unit/ConsumerTest.php | 12 +- tests/Feed/Unit/CursorTest.php | 69 ++--------- tests/Feed/Unit/FeedTest.php | 7 +- 16 files changed, 69 insertions(+), 531 deletions(-) delete mode 100755 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100755 index 27feb08..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013 Eldad Fux - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 67c3381..d2ea5d6 100644 --- a/README.md +++ b/README.md @@ -303,19 +303,9 @@ a warning rather than a failure: it keeps its position in memory and carries on, and only a restart before the store recovers replays anything. Pass `onWarning()` to hear about it. -A position never moves **backwards**. Run one process per consumer name β€” but a -rolling restart briefly overlaps two, and without this the departing one -finishing a shorter batch would land its older position last and undo the -arriving one's progress. Positions are totally ordered, so `save()` drops -anything that is not an advance. - -`Cursor\Redis` and `Cursor\Pool` enforce that **atomically**, by storing the -position as the id of a one-entry Redis stream: a feed position *is* a stream -id, and Redis itself refuses an `XADD` that is not newer than the stream's top. -The check and the write are one operation, so two processes racing cannot both -believe they are ahead. Other stores compare before writing, which leaves a -sub-millisecond window; losing that race costs a replay, which every handler -must already tolerate. +Run **one process per consumer name.** Two sharing a name share one position, so +each sees only the events the other has not already advanced past β€” the feed is +split between them rather than delivered to both. ## Events @@ -359,16 +349,12 @@ is specified to carry. ## Delivery semantics -**A handler must be safe to run twice on the same event.** There are four +**A handler must be safe to run twice on the same event.** There are three independent reasons, and no arrangement of this library removes any of them: 1. A handler can succeed and the position then fail to save. 2. A run interrupted partway resumes from the last event that succeeded. 3. A consumer whose position was lost restarts from the oldest retained event. -4. Two processes sharing a consumer name can interleave inside `Cursor::save()` - and leave the older position stored, re-delivering what the newer one had - already handled. Not possible on `Cursor\Redis` or `Cursor\Pool`, which - refuse a stale position atomically. Every one of them **re-delivers; none of them skips.** That asymmetry is the whole design β€” an event handled twice is absorbed by an idempotent handler, diff --git a/docs/migration.md b/docs/migration.md index 7a7e647..dfc186f 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -173,9 +173,8 @@ These were load-bearing in the original implementations and are preserved: failed event is retried on the next run, and everything behind it waits. - **A cursor store that is down is a warning, not a failure.** The position is mirrored in memory, so the consumer keeps working and only a restart replays. -- **The `feed::cursor:` key**, so consumers keep their positions - across the migration instead of replaying the retained feed. Its *type* - changes β€” see the note below. +- **The `feed::cursor:` key format**, so consumers keep their + positions across the migration instead of replaying the retained feed. - **`-` event ids**, so positions already handed out stay valid. - **The margin a consumer allows its HTTP client over the long-poll timeout**, without which every quiet tick surfaces as a transport failure. @@ -196,41 +195,3 @@ These were load-bearing in the original implementations and are preserved: is nullable, so an event without one reads back as `null` rather than `''`; `data` is unrestricted, so a list or scalar payload round-trips as itself; and `dataschema` and extension attributes now survive an append and a read. - -## ⚠️ Redis cursors change type β€” read this before deploying - -`Cursor\Redis` and `Cursor\Pool` now store a consumer's position as **the id of -a one-entry Redis stream**, where cloud's implementation stored it as a plain -string under the same key. - -The reason is atomicity. A feed position *is* a Redis stream id β€” `-` -is the format `XADD` allocates β€” and Redis refuses to append an id equal to or -smaller than the one at the top of a stream. Storing the position that way makes -"a position never moves backwards" the server's rule, enforced in the same -operation that writes it. A rolling restart briefly runs two processes under one -consumer name, and without this the departing one finishing a shorter batch -could land its older position last and undo the arriving one's progress. - -**The upgrade is transparent and needs no manual step.** `load()` reads a -string-valued key where it finds one, and the next `save()` replaces the key in -place with the stream form. The position is carried across, because the consumer -is advancing past exactly the value the string held β€” so no events are replayed. - -What this does mean: - -- **Do not roll back** to a build that predates this change while a cursor key - holds a stream. The old code issues `GET` against it and gets `WRONGTYPE`, and - the consumer will fail to load its position on every poll. Recovering means - deleting the affected `feed:*:cursor:*` keys, after which consumers restart - from the oldest retained event β€” safe, but a burst of redundant work. -- **Anything reading these keys directly** β€” a dashboard, a runbook, an ops - script doing `GET feed:edge:cursor:fra` β€” needs updating to - `XREVRANGE feed:edge:cursor:fra + - COUNT 1` and to take the entry's *id*, not - its payload. The payload is a placeholder; a stream entry has to carry one - field, and the id is the whole value. -- **`Cursor\Cache` is unaffected.** It still stores a string, and it keeps the - non-atomic read-compare-write, because a Utopia cache has no portable - compare-and-set β€” `getGeneration()` returns `'0'` on adapters without lease - support, so a fix there would hold on some backends and silently not on - others. The edge poller uses this store; its residual race costs a replay, - which its invalidation handler already tolerates. diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 8dd083f..c961b7f 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -22,17 +22,12 @@ * ## What the handler must tolerate * * Delivery is at-least-once, so a handler will see the same event more than - * once and must be safe to repeat. There are four separate reasons, and no + * once and must be safe to repeat. There are three separate reasons, and no * arrangement of this class removes any of them: * * 1. A handler can succeed and the position then fail to save. * 2. A batch interrupted partway replays from the last event that succeeded. * 3. A consumer whose position is lost restarts from the oldest retained event. - * 4. Two processes sharing a consumer name can interleave inside - * {@see Cursor::save()} and leave the older position stored, re-delivering - * what the newer one had already handled. Not possible on - * {@see Cursor\Redis} or {@see Cursor\Pool}, where the store refuses a - * stale position atomically. * * Every one of them re-delivers; none of them skips. That asymmetry is the * whole design β€” an event handled twice is absorbed by an idempotent handler, @@ -86,16 +81,12 @@ class Consumer * @param string $name This consumer's name, which its position is stored * under. Distinct per logical consumer, and stable across restarts. * - * **One process per name.** Two processes sharing a name each skip - * what the other handled, because neither sees the other's work - * before reading its own position. That is wasted effort, not lost - * events β€” delivery is at-least-once and handlers must tolerate a - * repeat regardless. - * - * The overlap a rolling restart creates is therefore safe, and - * {@see Cursor::save()} additionally refuses to move a stored - * position backwards, so the departing process cannot undo the - * arriving one's progress. + * **One process per name.** Two sharing a name share one position, + * so each sees only the events the other has not already advanced + * past β€” the feed is split between them rather than delivered to + * both, which is not what a handler written against this class + * expects. Give each replica the same name only if you mean them to + * divide the work. * @param Cursor $cursor Where to keep the position. * @param int $batch Events per run. * @param int $timeout Milliseconds to wait for an event when the feed is diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 89999ad..c9bf310 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -23,12 +23,6 @@ * * One store can hold positions for many consumers of the same feed, keyed by * consumer name. - * - * Positions move forwards only. {@see save()} compares before writing and drops - * anything that is not an advance, which matters because a rolling restart runs - * two processes under one name for a moment. That comparison is not atomic, and - * the reasoning for leaving it that way β€” including what the residual race - * costs, which is a replay rather than a lost event β€” is on `save()`. */ abstract class Cursor { @@ -59,7 +53,7 @@ public function getFeed(): string abstract public function load(string $consumer): ?string; /** - * Record a position, never moving one backwards. + * Record a position. * * Only ever call this once the events up to $eventId have been handled. A * position saved ahead of the work it stands for turns a crash into @@ -67,80 +61,13 @@ abstract public function load(string $consumer): ?string; * recover from β€” the events are still in the feed, but nothing will ever * read them again. * - * A position that would move *backwards* is dropped instead. Two processes - * running the same consumer overlap during a rolling restart β€” a normal - * operation, not a misconfiguration β€” and they finish batches of different - * lengths: without this, the slower one's older position lands last and a - * later restart replays everything between the two, which on a busy feed is - * potentially thousands of events. Positions are totally ordered - * ({@see Id::compare()}), so "never backwards" is decidable here in a way it - * is not for a cursor store in general. - * - * ## How strongly that holds depends on the store - * - * {@see Cursor\Redis} and {@see Cursor\Pool} enforce it **atomically**: a - * position is a Redis stream id, so it is stored as the id of a one-entry - * stream and the server itself refuses an `XADD` that is not an advance. - * There is no window between deciding and writing for a second process to - * slip into. - * - * Every other store falls back to this base implementation, which reads, - * compares, then writes. Two processes can interleave inside that window, - * both decide they are ahead, and leave the older position stored. There is - * no portable way to close it: {@see Cursor\Cache} would need a - * compare-and-set the cache does not have β€” leases exist, but - * `getGeneration()` returns `'0'` on adapters that do not implement them, so - * a fix would hold on some cache backends and silently not on others, which - * is worse than one clearly stated guarantee. - * - * **Where the window remains, losing the race costs a replay, which is the - * delivery guarantee rather than a departure from it.** A regressed position - * re-delivers events that were already handled; it never skips one, never - * loses one, and never advances past work that did not happen. Handlers must - * tolerate a repeat for three other reasons anyway β€” see {@see Consumer} β€” - * so this is a fourth cause of something they already survive, not a new - * kind of failure. Running one process per consumer name avoids it - * entirely. + * An empty $eventId is ignored rather than rejected: it means "nothing + * handled yet", and must not erase a real position. * * @throws Exception When the store cannot be written. */ abstract public function save(string $consumer, string $eventId): void; - /** - * Whether $eventId is worth storing for $consumer β€” that is, whether it is - * a real position and a later one than what is already there. - * - * The check every implementation applies before writing. - * - * Deliberately fails open. A position that cannot be compared β€” because the - * store could not be read, or because what came back is not a position β€” - * is treated as behind, so real progress replaces it. The guard exists to - * stop a position going backwards, and it must never become a reason for - * one to stop going forwards. - */ - protected function shouldAdvance(string $consumer, string $eventId): bool - { - if ($eventId === '') { - return false; - } - - try { - $current = $this->load($consumer); - } catch (\Throwable) { - return true; - } - - if ($current === null || $current === '') { - return true; - } - - try { - return Id::compare($eventId, $current) > 0; - } catch (Invalid) { - return true; - } - } - /** * Forget a consumer's position, so its next read starts from the oldest * retained event. diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php index 4947ccc..93ae7c6 100644 --- a/src/Feed/Cursor/Cache.php +++ b/src/Feed/Cursor/Cache.php @@ -52,7 +52,7 @@ public function load(string $consumer): ?string public function save(string $consumer, string $eventId): void { - if (!$this->shouldAdvance($consumer, $eventId)) { + if ($eventId === '') { return; } diff --git a/src/Feed/Cursor/Memory.php b/src/Feed/Cursor/Memory.php index 879e61a..cc0c1d4 100644 --- a/src/Feed/Cursor/Memory.php +++ b/src/Feed/Cursor/Memory.php @@ -25,7 +25,7 @@ public function load(string $consumer): ?string public function save(string $consumer, string $eventId): void { - if (!$this->shouldAdvance($consumer, $eventId)) { + if ($eventId === '') { return; } diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 25f0295..11137cd 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -5,12 +5,10 @@ namespace Utopia\Feed\Cursor; use Utopia\Feed\Cursor; -use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; -use Utopia\Feed\Id; /** - * Positions kept in Redis, as a one-entry stream. + * Positions kept in Redis, as plain keys alongside the stream. * * For consumers running inside the producer β€” a job that turns the feed into * something else, a bridge to a system that cannot poll. They have no store of @@ -19,31 +17,9 @@ * Consumers reached over HTTP should not use this: keeping their positions in * the producer's Redis puts per-consumer state back on the producer, which is * exactly what the feed is arranged to avoid. - * - * ## Why a stream and not a string - * - * A feed position *is* a Redis stream id β€” `-` is the format `XADD` - * allocates β€” and Redis refuses to append an id equal to or smaller than the - * one already at the top of a stream. Storing the position as the id of a - * single-entry stream therefore makes "never move a position backwards" the - * server's rule rather than this client's, and it holds atomically: two - * processes racing cannot both decide they are ahead, because the losing `XADD` - * is rejected by the same operation that would have written it. - * - * That is the one guarantee {@see Cursor::save()}'s read-compare-write cannot - * give, and it costs a single round trip rather than two. - * - * `MAXLEN 1` keeps the stream at exactly the current position; the entry's - * payload is unused, since the id carries the whole value. */ class Redis extends Cursor { - /** - * Placeholder field. A stream entry must carry at least one field, but the - * position lives in the entry's id, so nothing reads this. - */ - private const string FIELD = 'p'; - public function __construct( protected readonly \Redis|\RedisCluster $redis, string $feed, @@ -51,179 +27,46 @@ public function __construct( parent::__construct($feed); } - /** - * Advance the position, atomically. - * - * No read-compare-write here: `XADD` is itself the comparison, so a - * position that is not an advance is refused by Redis and treated as a - * no-op. That is the whole reason for the stream. - * - * @throws Invalid When $eventId is not a feed position, which a stream - * cannot store as an id. - * @throws Transport When Redis cannot be reached or refuses for any other - * reason. - */ - public function save(string $consumer, string $eventId): void + public function load(string $consumer): ?string { - if ($eventId === '') { - return; - } - - if (!Id::isValid($eventId)) { - throw new Invalid("Cannot store '{$eventId}' as a position: it is not a feed id"); - } - $key = $this->key($consumer); - [$reply, $error] = $this->attempt( - fn (): mixed => $this->redis->xAdd($key, $eventId, [self::FIELD => '1'], 1, false) - ); - - if ($error === '') { - return; - } - - // Redis rejected the id as not newer than what is stored: another - // process got further than this one, which is exactly the outcome the - // stream is here to produce. - if (self::rejectedAsStale($error)) { - return; + try { + /** @var mixed $cursor */ + $cursor = $this->redis->get($key); + } catch (\RedisException $error) { + throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); } - // A position written by a version that stored these as plain strings. - // Replacing the key in place keeps the position β€” the caller is - // advancing past whatever the string held β€” so the upgrade costs no - // replay. See docs/migration.md. - if (self::wrongType($error)) { - $this->replaceLegacy($key, $consumer, $eventId); + return \is_string($cursor) && $cursor !== '' ? $cursor : null; + } + public function save(string $consumer, string $eventId): void + { + if ($eventId === '') { return; } - throw new Transport("Failed to save the {$consumer} cursor: {$error}"); - } - - public function load(string $consumer): ?string - { $key = $this->key($consumer); - [$entries, $error] = $this->attempt( - fn (): mixed => $this->redis->xRevRange($key, '+', '-', 1) - ); - - if ($error !== '') { - // Written by a version that stored positions as plain strings. Read - // it where it is; the next save() converts the key. - if (self::wrongType($error)) { - return $this->loadLegacy($key, $consumer); - } - - throw new Transport("Failed to load the {$consumer} cursor: {$error}"); - } - - if (!\is_array($entries) || $entries === []) { - return null; + try { + // Deliberately no expiry. Unlike a cache, this is the only copy, + // and a position that quietly expired would replay the whole + // retained feed the next time the consumer restarted. + $this->redis->set($key, $eventId); + } catch (\RedisException $error) { + throw new Transport("Failed to save the {$consumer} cursor: {$error->getMessage()}", previous: $error); } - - // The position is the entry's id, not its payload. - $id = \array_key_first($entries); - - return \is_string($id) && $id !== '' ? $id : null; } public function reset(string $consumer): void { $key = $this->key($consumer); - [, $error] = $this->attempt(fn (): mixed => $this->redis->del($key)); - - if ($error !== '') { - throw new Transport("Failed to reset the {$consumer} cursor: {$error}"); - } - } - - /** - * @throws Transport - */ - private function loadLegacy(string $key, string $consumer): ?string - { - [$cursor, $error] = $this->attempt(fn (): mixed => $this->redis->get($key)); - - if ($error !== '') { - throw new Transport("Failed to load the {$consumer} cursor: {$error}"); - } - - return \is_string($cursor) && $cursor !== '' ? $cursor : null; - } - - /** - * @throws Transport - */ - private function replaceLegacy(string $key, string $consumer, string $eventId): void - { - [, $error] = $this->attempt(fn (): mixed => $this->redis->del($key)); - - if ($error === '') { - [, $error] = $this->attempt( - fn (): mixed => $this->redis->xAdd($key, $eventId, [self::FIELD => '1'], 1, false) - ); - } - - if ($error !== '') { - throw new Transport("Failed to upgrade the {$consumer} cursor to a stream: {$error}"); - } - } - - /** - * Run a command and report Redis' refusal rather than letting it surface as - * two different things. - * - * phpredis signals a command-level error either by throwing or by returning - * `false` and parking the text in `getLastError()`, depending on the build - * and the connection's options. Both are normalised here so callers can ask - * one question β€” did Redis refuse, and what did it say β€” instead of each - * handling the split. - * - * @param \Closure(): mixed $command - * @return array{mixed, string} The reply, and Redis' error text when it - * refused, or an empty string when it did not. - */ - private function attempt(\Closure $command): array - { - $this->redis->clearLastError(); - try { - /** @var mixed $reply */ - $reply = $command(); + $this->redis->del($key); } catch (\RedisException $error) { - return [false, $error->getMessage() !== '' ? $error->getMessage() : 'Redis command failed']; - } - - if ($reply !== false) { - return [$reply, '']; + throw new Transport("Failed to reset the {$consumer} cursor: {$error->getMessage()}", previous: $error); } - - $error = $this->redis->getLastError(); - $this->redis->clearLastError(); - - return [false, \is_string($error) && $error !== '' ? $error : '']; - } - - /** - * Whether Redis refused an id for being at or behind the stream's top, - * which is this class's definition of "not an advance". - */ - private static function rejectedAsStale(string $error): bool - { - return \str_contains($error, 'equal or smaller'); - } - - /** - * Whether the key holds something other than a stream β€” in practice, a - * position written by a version that stored them as plain strings. - */ - private static function wrongType(string $error): bool - { - return \str_contains($error, 'WRONGTYPE'); } } diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index da7750a..c64d5e7 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -87,15 +87,11 @@ public function pollable(): bool } /** - * Guard a retention cap. + * Guard a retention cap, at construction. * - * Non-positive values do not mean "no retention" β€” they mean something - * different on every backend, and nothing useful on any. Redis reads - * `MAXLEN 0` as "trim everything", so a feed would accept appends and - * retain none of them; `array_slice($events, -0)` is `array_slice($events, - * 0)`, so the in-memory journal would do the exact opposite and retain the - * lot, unbounded. A cap that silently means one thing here and the reverse - * there is worse than no cap, so it is rejected at construction. + * A feed must retain at least one event. Backends disagree about what a + * non-positive cap means β€” some keep nothing, some keep everything β€” so it + * is refused here rather than resolved differently on each one. * * @throws Invalid When $maxSize would retain fewer than one event. */ diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Journal/Memory.php index ee8899c..dfa66c8 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -6,6 +6,7 @@ use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Id; /** @@ -33,6 +34,9 @@ class Memory extends Journal private int $sequence = -1; + /** + * @throws Invalid When $name is empty, or $maxSize is below one event. + */ public function __construct(string $name, protected readonly int $maxSize = 100_000) { parent::__construct($name); diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Journal/Pool.php index 42e84fa..85302ec 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -6,6 +6,7 @@ use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Exception\Invalid; use Utopia\Pools\Pool as UtopiaPool; /** @@ -23,6 +24,7 @@ class Pool extends Journal * @param UtopiaPool<\Redis|\RedisCluster> $pool * @param string $name Feed name; the stream is stored at `feed:`. * @param int $maxSize Approximate cap on retained events. + * @throws Invalid When $name is empty, or $maxSize is below one event. */ public function __construct( protected readonly UtopiaPool $pool, diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Journal/Redis.php index 814634f..20ad1eb 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -6,6 +6,7 @@ use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; @@ -34,6 +35,7 @@ class Redis extends Journal * matters: a consumer down for longer than its feed's retention * resumes from the oldest event it can, rather than from where it * left off. + * @throws Invalid When $name is empty, or $maxSize is below one event. */ public function __construct( protected readonly \Redis|\RedisCluster $redis, diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index 195b451..0ce9fc2 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -263,110 +263,10 @@ public function testResetReplaysTheRetainedFeed(): void $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume(fn (CloudEvent $e) => null)); } - /** - * The rolling-restart case, against a real store: the departing process - * finishing a shorter batch must not undo the arriving one's progress. - * - * Here Redis enforces it rather than this library β€” the stale `XADD` is - * refused by the same operation that would have written it, so there is no - * window between the check and the write for a second process to slip into. - */ - public function testAPositionNeverMovesBackwards(): void - { - $cursor = new RedisCursor($this->redis, $this->name); - - $cursor->save('invalidator', '1690000000000-5'); - $cursor->save('invalidator', '1690000000000-2'); - - $this->assertSame('1690000000000-5', $cursor->load('invalidator')); - - $cursor->save('invalidator', '1690000000001-0'); - - $this->assertSame('1690000000001-0', $cursor->load('invalidator'), 'A genuine advance still lands'); - } - - /** - * `10-0` is later than `9-0` but sorts before it as a string, so a - * comparison on the text would refuse a legitimate advance and stall the - * consumer permanently. Redis compares the parts. - */ - public function testAdvancingAcrossADigitBoundaryIsAccepted(): void - { - $cursor = new RedisCursor($this->redis, $this->name); - - $cursor->save('invalidator', '9-0'); - $cursor->save('invalidator', '10-0'); - - $this->assertSame('10-0', $cursor->load('invalidator')); - } - - public function testRewritingTheSamePositionIsAccepted(): void - { - $cursor = new RedisCursor($this->redis, $this->name); - - $cursor->save('invalidator', '5-0'); - $cursor->save('invalidator', '5-0'); - - $this->assertSame('5-0', $cursor->load('invalidator')); - } - - /** - * The position is the entry's id, and the stream never grows past it. - */ - public function testAPositionIsStoredAsAOneEntryStream(): void - { - $cursor = new RedisCursor($this->redis, $this->name); - $key = 'feed:' . $this->name . ':cursor:invalidator'; - - foreach (['1-0', '2-0', '3-0'] as $position) { - $cursor->save('invalidator', $position); - } - - $this->assertSame(\Redis::REDIS_STREAM, $this->redis->type($key)); - $this->assertSame(1, $this->redis->xLen($key), 'MAXLEN 1 keeps only the current position'); - } - - /** - * An earlier version stored these as plain strings. Reading has to find one - * where it is, and the next save has to replace it in place β€” otherwise a - * deployment upgrading this library would either hit WRONGTYPE forever or - * silently restart from the oldest retained event. - */ - public function testUpgradesAPositionWrittenAsAPlainString(): void - { - $cursor = new RedisCursor($this->redis, $this->name); - $key = 'feed:' . $this->name . ':cursor:invalidator'; - - $this->redis->set($key, '1690000000000-0'); - - $this->assertSame('1690000000000-0', $cursor->load('invalidator'), 'The old position is still readable'); - - $cursor->save('invalidator', '1690000000000-1'); - - $this->assertSame(\Redis::REDIS_STREAM, $this->redis->type($key), 'The key is converted on the next save'); - $this->assertSame('1690000000000-1', $cursor->load('invalidator'), 'No position is lost in the upgrade'); - } - - /** - * A stream id is the storage, so anything that is not one is refused with a - * clear message rather than surfacing as a Redis parse error. - */ - public function testRejectsACursorPositionThatIsNotAFeedId(): void - { - $this->expectException(Invalid::class); - - (new RedisCursor($this->redis, $this->name))->save('invalidator', 'not-a-position'); - } - public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void { (new RedisCursor($this->redis, $this->name))->save('invalidator', '1-0'); - // The position is the entry's id, so this reads the stream rather than - // the key's value. - $entries = $this->redis->xRevRange('feed:' . $this->name . ':cursor:invalidator', '+', '-', 1); - - $this->assertIsArray($entries); - $this->assertSame('1-0', \array_key_first($entries)); + $this->assertSame('1-0', $this->redis->get('feed:' . $this->name . ':cursor:invalidator')); } } diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 88d5cab..771bc30 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -111,18 +111,14 @@ public function load(string $consumer): ?string $consumer = $this->consumer($cursor); - // Restores the position, then advances β€” the advance reads once more to - // check it is not moving the position backwards. + // One read on the first pass to restore the position; the two + // caught-up polls after it must not touch the store at all, which is + // what keeps an idle consumer on a timer free. $consumer->consume(fn (CloudEvent $event) => null); - $settled = $cursor->loads; - - // Caught up: nothing handled, so nothing saved, so nothing read. This - // is the property that matters β€” an idle consumer polling on a timer - // does not touch its store at all. $consumer->consume(fn (CloudEvent $event) => null); $consumer->consume(fn (CloudEvent $event) => null); - $this->assertSame($settled, $cursor->loads, 'A caught-up poll must not read the store'); + $this->assertSame(1, $cursor->loads); } public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index afa43f9..723b294 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -66,65 +66,6 @@ public function testConsumersDoNotShareAPosition(Cursor $cursor): void $this->assertSame('2-0', $cursor->load('two')); } - /** - * Two processes running the same consumer overlap during a rolling - * restart, finish batches of different lengths, and write out of order. - * Without this the older position lands last and a later restart replays - * everything between the two. - * - * @dataProvider stores - */ - public function testAPositionNeverMovesBackwards(Cursor $cursor): void - { - $cursor->save('invalidator', '1690000000000-5'); - $cursor->save('invalidator', '1690000000000-2'); - - $this->assertSame('1690000000000-5', $cursor->load('invalidator')); - } - - /** - * Positions are compared by their parts, not as strings β€” `10-0` is later - * than `9-0` but sorts before it, so a string comparison would reject a - * legitimate advance and stall the consumer permanently. - * - * @dataProvider stores - */ - public function testAdvancingAcrossADigitBoundaryIsNotMistakenForGoingBackwards(Cursor $cursor): void - { - $cursor->save('invalidator', '9-0'); - $cursor->save('invalidator', '10-0'); - - $this->assertSame('10-0', $cursor->load('invalidator')); - } - - /** - * @dataProvider stores - */ - public function testRewritingTheSamePositionIsAccepted(Cursor $cursor): void - { - $cursor->save('invalidator', '1-0'); - $cursor->save('invalidator', '1-0'); - - $this->assertSame('1-0', $cursor->load('invalidator')); - } - - /** - * The guard must never become a reason a position stops moving forwards. - * - * @dataProvider stores - */ - public function testAStoredValueThatIsNotAPositionIsReplaced(Cursor $cursor): void - { - $cursor->save('invalidator', '1-0'); - $cursor->reset('invalidator'); - - // Whatever a hand-edited or corrupted store hands back, real progress - // must still be able to overwrite it. - $cursor->save('invalidator', '2-0'); - - $this->assertSame('2-0', $cursor->load('invalidator')); - } - /** * @dataProvider stores */ @@ -157,6 +98,16 @@ public function testRejectsAnEmptyConsumerName(Cursor $cursor): void $cursor->load(''); } + /** + * @dataProvider stores + */ + public function testRejectsAnEmptyConsumerNameOnSave(Cursor $cursor): void + { + $this->expectException(Invalid::class); + + $cursor->save('', '1-0'); + } + public function testFeedsDoNotShareAPosition(): void { $cache = new UtopiaCache(new CacheMemory()); diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 0dfc6a2..2c9b901 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -383,10 +383,9 @@ public static function unusableRetention(): array } /** - * A non-positive cap means the opposite thing on each backend β€” Redis reads - * `MAXLEN 0` as "keep nothing", while `array_slice($events, -0)` keeps - * everything β€” so it is refused rather than silently honoured one way here - * and the other way in production. + * Backends disagree about what a non-positive cap means β€” some keep + * nothing, some keep everything β€” so it is refused at construction rather + * than resolved one way in a test and the other way in production. * * @dataProvider unusableRetention */ From 976d3966d69c2bab98cb88e53c4a32aac32a8e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 17:46:32 +0200 Subject: [PATCH 15/68] Further library simplification --- CHANGELOG.md | 12 +- README.md | 436 ++++++------------ docs/migration.md | 197 -------- src/Feed/Consumer.php | 102 ++-- src/Feed/Cursor.php | 62 +-- src/Feed/Cursor/Cache.php | 35 +- src/Feed/Cursor/Memory.php | 15 +- src/Feed/Cursor/Pool.php | 33 +- src/Feed/Cursor/Redis.php | 33 +- src/Feed/Exception/Unsupported.php | 2 +- src/Feed/Feed.php | 149 ++---- src/Feed/Id.php | 37 +- src/Feed/Journal.php | 107 ++--- src/Feed/Journal/Http.php | 98 ++-- src/Feed/Journal/Memory.php | 52 +-- src/Feed/Journal/Pool.php | 12 +- src/Feed/Journal/Redis.php | 50 +- .../Journal/{None.php => Unconfigured.php} | 17 +- src/Feed/Protocol.php | 115 ++--- tests/Feed/E2E/RedisTest.php | 14 +- tests/Feed/Unit/ConsumerTest.php | 37 +- tests/Feed/Unit/CursorTest.php | 74 ++- tests/Feed/Unit/FeedTest.php | 55 ++- tests/Feed/Unit/HttpJournalTest.php | 16 +- tests/Feed/Unit/IdTest.php | 19 +- tests/Feed/Unit/RoundTripTest.php | 2 +- tests/Feed/Unit/Support/FailingCursor.php | 10 +- tests/Feed/Unit/Support/FeedServer.php | 2 +- 28 files changed, 595 insertions(+), 1198 deletions(-) delete mode 100644 docs/migration.md rename src/Feed/Journal/{None.php => Unconfigured.php} (55%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89951f8..2288da4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,6 @@ Initial release. -Extracted from the pull-based feed built across `appwrite-labs/cloud` (producer) -and `appwrite-labs/edge` (consumer), where the event model, the pull loop, the -cursor bookkeeping and the HTTP contract joining the two had each been written -twice. See [docs/migration.md](docs/migration.md) for how those map onto this -library. - - `Feed` β€” append, read and long-poll an ordered event log - Events are [utopia-php/cloudevents](https://github.com/utopia-php/cloudevents) `CloudEvent` objects β€” this library defines no event type of its own, so a feed @@ -19,9 +13,11 @@ library. - `Journal\Http` β€” another service's feed, read over the wire with [utopia-php/client](https://github.com/utopia-php/client); takes any of its adapters, so a pooled or Swoole coroutine transport drops straight in -- `Journal\Memory`, `Journal\None` β€” for tests, and for no backend configured +- `Journal\Memory`, `Journal\Unconfigured` β€” for tests, and for no backend + configured - `Consumer` β€” the pull loop, with at-least-once semantics and a durable position -- `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory` β€” where that position lives +- `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory` β€” where that + position lives, keyed by feed and consumer name - `Protocol` β€” the http-feeds wire contract, shared by producer and consumer, and the one place the feed's decode policy lives: strict about `id` because it is the consumer's position, tolerant of everything else so a consumer older than diff --git a/README.md b/README.md index d2ea5d6..0b0553f 100644 --- a/README.md +++ b/README.md @@ -4,138 +4,44 @@ ![Total Downloads](https://img.shields.io/packagist/dt/utopia-php/feed.svg) [![Discord](https://img.shields.io/discord/564160730845151244)](https://appwrite.io/discord) -Utopia Feed is a simple and lite library for moving events between services with -**pull-based HTTP event feeds** ([http-feeds.org](https://www.http-feeds.org/)), -instead of pushing them to every service that needs them. This library is aiming -to be as simple and easy to learn and use. This library is maintained by the -[Appwrite team](https://appwrite.io). - -Although this library is part of the [Utopia -Framework](https://github.com/utopia-php/framework) project, it is dependency -light and can be used as standalone with any other PHP project or framework. - -## Why pull instead of push - -A service that pushes an event to its consumers has to reach all of them at the -moment it happens. Any consumer that is down, redeploying, rate limited or -simply new misses the event, and there is nothing in the system that will ever -tell it. The producer also grows an outbound call per consumer, has to hold a -retry queue per consumer, and has to be told when a consumer is added. - -A feed inverts that. The producer appends to an ordered log and forgets about -it. Each consumer asks *"what has happened since the last thing I saw?"*, -quoting the id of that event. A consumer that was down catches up on its next -poll. A consumer that is added later starts from whatever is still retained. The -producer keeps no per-consumer state at all, so nothing about it changes when -consumers come and go. - -``` - append GET /feeds/edge?lastEventId=... - producer ───────────────▢ feed ◀───────────────────────────── consumer A - (log) ◀───────────────────────────── consumer B - consumer C ← added later, - catches up on its own -``` - -The trade is **at-least-once delivery**: consumers retry, replay, and restart -from positions they have already passed, so every event has to be safe to -process twice. Retention is bounded, so a consumer that falls a long way behind -resumes from the oldest retained event rather than failing. That makes a feed a -poor fit for events whose effect depends on seeing every one of them (a balance -built out of deltas) and a good fit for events that describe a state to converge -on β€” a cache tag to drop, a record to refresh, a config to reload. - -## Features - -- **Ordered, resumable log** β€” consumers page by event id, and hold their own position -- **CloudEvents** β€” events *are* [`utopia-php/cloudevents`](https://github.com/utopia-php/cloudevents) events, as http-feeds requires -- **Journals** β€” Redis streams, a pooled Redis, in-memory, or another service's feed over HTTP -- **Long polling** β€” subscribe in near real time without hammering the producer -- **Cursors** β€” positions in a Utopia cache, in Redis, or in memory -- **Consumer** β€” the pull loop, the position bookkeeping and the at-least-once semantics, written once - -## How the pieces fit - -The library splits along the same line the design does: the **producer owns the -events**, each **consumer owns its position**, and `Protocol` is the seam between -them when they live in different services. - -``` - PRODUCER β”‚ CONSUMER - β”‚ - append() β”‚ consume(handler) - ↓ β”‚ ↓ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ Feed │───────▢│ Journal β”‚ β”‚ β”‚ Consumer │─────▢│ Cursor β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ \Redis β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ \Cache β”‚ - policy β”‚ \Pool β”‚ β”‚ the pull loop β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ \Memory β”‚ β”‚ β”‚ "where I got to" - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ ↓ - the events β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ β”‚ β”‚ Feed │────▢│ Journal β”‚ - β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ \Http β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ Protocol │◀──────┼──── HTTP GET β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - the wire contract β”‚ -``` - -**`Journal`** β€” where the events live. It assigns an ordered id on append and -returns the events after a given id, and nothing else. `Journal\Http` reads -*another service's* journal, so to everything above it a remote feed and a local -one are the same object. - -**`Feed`** β€” the policy on one journal: stamps `source` and `time` on append, -clamps a consumer-supplied `limit`, and long-polls. Subclass it to give a feed a -typed vocabulary. - -**`Cursor`** β€” where one consumer's position is kept. Deliberately independent of -`Journal`: a consumer keeps its position in *its own* storage, never the -producer's. - -**`Consumer`** β€” the pull loop. Reads from the stored position, hands each event -to a handler oldest-first, and advances only past the events the handler -accepted. - -**`Protocol`** β€” the HTTP contract: query parameters, response envelope and -caching rules, held in one place so the two halves cannot drift apart. - -The structural consequence worth knowing up front: **the producer stores no -per-consumer state at all.** That is what makes adding a consumer free, and it is -why `Cursor` is its own thing rather than a method on `Journal`. - -Dependencies only ever point one way, so each piece is testable alone β€” a -`Journal\Memory` and a `Cursor\Memory` exercise the whole pull loop with no Redis -and no network: - -``` -Consumer ──▢ Feed ──▢ Journal ──▢ Protocol (only Journal\Http) - └──────▢ Cursor -``` +Utopia Feed moves events between services with **pull-based HTTP event feeds** +([http-feeds.org](https://www.http-feeds.org/)) instead of pushing them to every +service that needs them. + +A producer appends events to an ordered log. Each consumer asks *"what has +happened since the last event I saw?"*, quoting that event's id, and keeps track +of its own position. A consumer that was down catches up on its next poll; a +consumer added later starts from whatever is still retained. The producer stores +nothing per consumer, so nothing about it changes when consumers come and go. + +The trade is **at-least-once delivery**: every event must be safe to handle +twice. Retention is bounded, so a feed suits events that describe a state to +converge on β€” a cache tag to drop, a record to refresh β€” rather than ones whose +effect depends on seeing every single one. + +This library is maintained by the [Appwrite team](https://appwrite.io). Although +it is part of the [Utopia +Framework](https://github.com/utopia-php/framework), it is dependency light and +works standalone with any PHP project. ## Getting started -Install using composer: - ```bash composer require utopia-php/feed ``` -### Producing +### Produce ```php -use Utopia\Feed\Journal\Redis as RedisJournal; use Utopia\Feed\Feed; - -$redis = new Redis(); -$redis->connect('redis', 6379); +use Utopia\Feed\Journal; $feed = new Feed( - new RedisJournal($redis, 'edge'), + new Journal\Redis($redis, 'edge'), source: 'urn:appwrite:cloud:fra', ); -$feed->append( +$id = $feed->append( type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']], subject: 'example.com', @@ -144,20 +50,14 @@ $feed->append( `append()` returns the event's id, which is its position in the feed. -Give a feed a typed vocabulary by subclassing it, so callers cannot invent an -event type or misspell a payload key: +Subclass `Feed` to give it a typed vocabulary, so callers cannot invent an event +type or misspell a payload key: ```php class EdgeFeed extends Feed { - public const string NAME = 'edge'; - public function invalidateRule(string $domain): string { - if ($domain === '') { - throw new \InvalidArgumentException('Rule invalidation requires a domain'); - } - return $this->append( 'io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => $domain]], @@ -167,229 +67,179 @@ class EdgeFeed extends Feed } ``` -### Serving a feed over HTTP - -`Protocol` holds the wire contract β€” the query parameters, the response body and -the caching rules β€” so the endpoint and its consumers cannot drift apart. It -deals in arrays rather than requests and responses, so it fits whichever HTTP -layer the producer is written in: - -```php -use Utopia\Feed\Feed; -use Utopia\Feed\Protocol; - -// GET /v1/feeds/:feedId -$limit = (int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH); - -$events = $feed->poll( - $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, - $limit, - (int) $request->getParam(Protocol::PARAM_TIMEOUT, 0), -); - -$response - ->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)) - ->json(Protocol::encode($events)); -``` - -`cacheControl()` marks a full batch immutable β€” the same query returns the same -events forever β€” and a short batch `no-store`, because it is the live end of the -feed and will grow. It defaults to `private`, since a feed is usually served -behind authorization and `public` would let a shared cache hand one consumer's -events to a requester that never presented a credential. +### Consume -### Consuming - -A `Consumer` reads from where it last got to, hands each new event to a handler, -and records how far it got: +A `Consumer` reads from where it last got to, hands each new event to your +handler, and records how far it got: ```php use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Consumer; -use Utopia\Feed\Cursor\Cache as CacheCursor; +use Utopia\Feed\Cursor; -$consumer = new Consumer( - feed: $feed, - name: 'cache-invalidator', - cursor: new CacheCursor($cache, 'edge'), -); +$consumer = new Consumer($feed, 'cache-invalidator', new Cursor\Cache($cache)); $handled = $consumer->consume(function (CloudEvent $event) use ($router) { $router->invalidate($event->data['tags'] ?? []); }); ``` -Consuming **another service's** feed is the same code with a different journal: +Call `consume()` on a timer, or give the consumer a `timeout` and loop β€” each +call then returns the moment an event arrives, or empty after the timeout: + +```php +$consumer = new Consumer($feed, 'cache-invalidator', $cursor, timeout: 20_000); + +while (true) { + $consumer->consume($handler); +} +``` + +### Consume another service's feed + +Same code, different journal β€” nothing above it knows the events arrive over the +network: ```php use Utopia\Client; use Utopia\Client\Adapter\Curl\Client as Curl; -use Utopia\Feed\Journal\Http; $client = (new Client(new Curl())) ->withHeaders(['x-appwrite-jwt' => $token]) ->withConnectionReuse(); -$feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); +$feed = new Feed(new Journal\Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); ``` -Nothing above the journal knows the events are arriving over the network, -including the long polling β€” `Http` hands the wait to the producer, so a poll is -one held request rather than a client-side loop. +Long polling is handled by the producer, so a poll is one held request rather +than a client-side loop. `Journal\Http` takes any +[utopia-php/client](https://github.com/utopia-php/client) adapter, so a pooled or +Swoole coroutine transport drops straight in. Leave the `Retry` decorator off: a +failed read leaves the position where it was, so the next poll is already the +retry. -`Http` takes any [`utopia-php/client`](https://github.com/utopia-php/client) -adapter, so a `Pool` or a Swoole coroutine transport drops straight in. Leave the -`Retry` decorator off, though: a failed read leaves the cursor where it was, so -the next poll is already the retry, and retrying inside a long poll only -multiplies how long a single tick can take. +### Serve a feed over HTTP -Call `consume()` on a timer, or give the consumer a `timeout` and loop: +`Protocol` holds the wire contract β€” query parameters, response body, caching +rules β€” and deals in arrays, so it fits whichever HTTP layer you use: ```php -// Returns as soon as an event arrives, or after 20s with nothing. -$consumer = new Consumer($feed, 'cache-invalidator', $cursor, timeout: 20_000); +use Utopia\Feed\Feed; +use Utopia\Feed\Protocol; -while (true) { - $consumer->consume($handler); -} +// GET /v1/feeds/:feedId +$limit = Feed::limit((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH)); + +$events = $feed->poll( + $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, + $limit, + (int) $request->getParam(Protocol::PARAM_TIMEOUT, 0), +); + +$response + ->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)) + ->json(Protocol::encode($events)); ``` +`Feed::limit()` clamps what the consumer asked for to what a read will actually +return, so the same number reaches `cacheControl()`. A full batch is settled +history and is marked cacheable; a short one is the live end of the feed and is +marked `no-store`. Caching is `private` unless you pass `public: true`. + +## Events + +Events **are** +[`Utopia\CloudEvents\CloudEvent`](https://github.com/utopia-php/cloudevents) +objects β€” this library defines no event type of its own: + +```php +$consumer->consume(function (CloudEvent $event) { + $tags = $event->data['tags'] ?? []; + $trace = $event->getExtension('traceparent'); +}); +``` + +`data` is unrestricted β€” a map, list, string, number or null all round-trip as +themselves. `subject` is nullable, so an event with no subject reads back as +`null`. `dataschema` and extension attributes survive an append and a read. + +A batch is decoded strictly about `id`, because for a feed the id *is* the +consumer's position, and leniently about everything else β€” a producer that adds +an attribute or moves the spec forward must not stop a consumer that predates it. + ## Journals -A journal is where a feed's events actually live. The name is the one event -sourcing has long used for an append-only, strictly ordered record that is -replayed rather than mutated β€” Akka Persistence calls its pluggable storage -backends journals for the same reason. It is responsible for exactly two things: -assigning an ordered id on append, and returning the events strictly after a -given id. Everything else β€” long polling, cursors, the pull loop β€” sits above it -and is the same whichever journal is underneath. +A journal is where a feed's events live. It assigns an ordered id on append and +returns the events after a given id; everything else sits above it. | Journal | Use for | Append | Read | | --- | --- | --- | --- | | `Journal\Redis` | Producing a feed on a Redis stream | βœ… | βœ… | | `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | βœ… | -| `Journal\Http` | Consuming another service's feed, over [utopia-php/client](https://github.com/utopia-php/client) | ❌ | βœ… | -| `Journal\Memory` | Tests, and single-process development | βœ… | βœ… | -| `Journal\None` | No backend configured | ❌ | ❌ | - -`Journal\Pool` is what most services producing a feed want: a long poll holds -its connection for the whole timeout, so reading through a shared client would -block every other user of it. +| `Journal\Http` | Consuming another service's feed | ❌ | βœ… | +| `Journal\Memory` | Tests and single-process development | βœ… | βœ… | +| `Journal\Unconfigured` | No backend configured β€” throws on use | ❌ | ❌ | -`Journal\None` throws on every operation rather than doing nothing, so a -misconfigured service fails at the point of use instead of silently dropping -events β€” which only shows up much later, somewhere else. `Journal\Memory` -implements the same id and retention semantics as `Journal\Redis`, including the -awkward parts like resuming from a trimmed position, so code tested against it -behaves the same when it is swapped out. +`Journal\Pool` is what most services producing a feed want: a long poll holds its +connection for the whole timeout, so reading through a shared client would block +every other user of it. ## Cursors -http-feeds puts the position on the consumer rather than the producer, which is -what makes adding a consumer free. A cursor is just somewhere to write a string: +A cursor is where one consumer keeps its position. It is keyed by feed and +consumer name, so a single store serves every feed a service consumes: | Cursor | Use for | | --- | --- | -| `Cursor\Cache` | A consumer with a [Utopia cache](https://github.com/utopia-php/cache) β€” the usual choice for one reading a remote feed | +| `Cursor\Cache` | A consumer with a [Utopia cache](https://github.com/utopia-php/cache) β€” the usual choice when reading a remote feed | | `Cursor\Redis` | A consumer running inside the producer, with no store of its own | | `Cursor\Pool` | The same, over a pooled connection | | `Cursor\Memory` | Tests, or a consumer that should replay from the beginning on every restart | -The store is allowed to be lossy. A lost position is not a lost event β€” a -consumer with no position resumes from the oldest retained event β€” so the -consequence is redundant work, not a gap. That is why a cache is a reasonable -place to put one, and it is also why a `Consumer` treats a store that is down as -a warning rather than a failure: it keeps its position in memory and carries on, -and only a restart before the store recovers replays anything. Pass -`onWarning()` to hear about it. - -Run **one process per consumer name.** Two sharing a name share one position, so -each sees only the events the other has not already advanced past β€” the feed is -split between them rather than delivered to both. - -## Events - -There is no event type in this library. Events **are** -[`Utopia\CloudEvents\CloudEvent`](https://github.com/utopia-php/cloudevents) -objects, so anything already typed against one takes a feed event directly, and -everything a CloudEvent carries β€” `dataschema`, extension attributes such as a -`traceparent` β€” survives an append and a read untouched: +The store is allowed to be lossy: a lost position costs a replay, not a gap. A +store that is down is a warning rather than a failure β€” the consumer keeps its +position in memory and carries on. Pass `onWarning()` to hear about it: ```php -use Utopia\CloudEvents\CloudEvent; - -$consumer->consume(function (CloudEvent $event) { - $tags = $event->data['tags'] ?? []; - $trace = $event->getExtension('traceparent'); -}); +$consumer->onWarning(fn (\Throwable $error, string $context) => + Console::warning("[feed] Could not {$context} the cursor: {$error->getMessage()}")); ``` -`data` is unrestricted, as the JSON event format requires β€” a map, a list, a -string, a number or null are all valid payloads and all round-trip as -themselves. `subject` is nullable, so an event with no subject reads back as -`null` rather than `''`. - -The one thing this library decides for itself is how a batch is decoded, and it -is deliberately not `CloudEvent::fromArray()`'s default: - -- **Strict about `id`.** For a feed the id *is* the consumer's position, so an - event without one cannot be recorded as passed. The spec makes `id` required - too; `Protocol` enforces exactly that one attribute rather than calling - `validate()`, which would also demand a well-formed URI-reference `source` β€” - a spec requirement, but not one a feed consumer depends on. -- **Tolerant about everything else.** Decoding runs with `lenient: true` and - `allowUnknownSpecversion: true`, so a producer that adds an attribute, omits - an optional one, or moves the spec forward does not stop a consumer that - predates it. A feed is read by consumers older than the producer *by design*, - and that is what makes a staged rollout safe. - -An entry that is not a CloudEvent at all β€” no `specversion`, no `type` β€” is not -tolerated, because that is a producer sending something other than what the feed -is specified to carry. +Run **one process per consumer name.** Two sharing a name share one position, so +the feed is split between them rather than delivered to both. -## Delivery semantics +## What a handler must tolerate **A handler must be safe to run twice on the same event.** There are three -independent reasons, and no arrangement of this library removes any of them: +reasons, and none of them can be arranged away: 1. A handler can succeed and the position then fail to save. 2. A run interrupted partway resumes from the last event that succeeded. 3. A consumer whose position was lost restarts from the oldest retained event. -Every one of them **re-delivers; none of them skips.** That asymmetry is the -whole design β€” an event handled twice is absorbed by an idempotent handler, -whereas an event stepped over is gone, still sitting in the feed with nothing -that will ever read it again. - -**A handler rejects an event by throwing.** That stops the run at that event and -leaves the position before it, so the next run starts there and tries again. -Everything already handled in that run stays handled β€” progress is committed -before the failure is re-raised. A handler that keeps failing on one event -therefore blocks everything behind it, which is the intended behaviour: a feed -is ordered, and stepping over a failure would deliver later events on top of -state that was never updated. - -**A consumer with no recorded position starts at the oldest retained event, -never at the tip.** Starting at the tip would drop whatever is already in the -feed, and for a consumer being deployed for the first time that is not a -hypothetical backlog β€” it is exactly the events it was meant to catch up on. -This is what makes a staged rollout safe: ship the producer first, let events -accumulate, then ship the consumer, and nothing in between is lost. +Every one of them re-delivers; none of them skips. An event handled twice is +absorbed by an idempotent handler, whereas an event stepped over is gone. + +**A handler rejects an event by throwing.** The run stops there, the position +stays before it, and the next run tries again. Everything handled earlier in that +run stays handled. A handler that keeps failing blocks everything behind it β€” +intentionally, because a feed is ordered and stepping over a failure would apply +later events on top of state that was never updated. + +**A consumer with no position starts at the oldest retained event, never at the +tip**, so a consumer deployed after the producer catches up rather than dropping +the backlog. ## Rolling out a feed -Replacing push delivery with a feed is a two-release change, and the order -matters: +Replacing push delivery with a feed is a two-release change: 1. **Release the producer.** It appends events; nothing reads them yet. -2. **Release the consumers.** Each drains the backlog from its first poll, - because a consumer with no position starts at the oldest retained event. -3. **Only then remove the push path.** Until every consumer is polling, removing - it means nothing is delivered. +2. **Release the consumers.** Each drains the backlog from its first poll. +3. **Only then remove the push path.** -While step 2 is in progress, consumers that have not shipped yet will get a 404 -from a producer that does not serve the feed β€” normal, not a fault. That status -is on the exception, so it can be told apart from a real failure: +While step 2 is in progress, consumers get a 404 from a producer that does not +serve the feed yet β€” normal, not a fault. The status is on the exception: ```php use Utopia\Feed\Exception\Transport; @@ -414,31 +264,17 @@ composer install composer test ``` -The E2E suite runs against a real Redis, which is where the assumptions about -stream ids and `MAXLEN` trimming are actually confirmed: +The E2E suite runs against a real Redis, and static analysis needs `ext-redis`, +so both run in the container: ```bash docker compose up -d docker compose exec tests composer test:e2e -``` - -Static analysis runs at PHPStan level max. Run it inside the container, where -`ext-redis` is installed: - -```bash docker compose exec tests composer check ``` -The image is built from one parameterized `Dockerfile`, so testing against -another PHP version needs no new file: - -```bash -PHP_VERSION=8.6 docker compose build -PHP_VERSION=8.6 docker compose up -d -``` - -To add that version to CI, add it to the `php-versions` matrix in -`.github/workflows/tests.yml` β€” that is the only place versions are listed. +To test another PHP version, build with `PHP_VERSION=8.6 docker compose build`, +and add it to the `php-versions` matrix in `.github/workflows/tests.yml`. ## System requirements diff --git a/docs/migration.md b/docs/migration.md deleted file mode 100644 index dfc186f..0000000 --- a/docs/migration.md +++ /dev/null @@ -1,197 +0,0 @@ -# Migrating an existing feed onto this library - -This library was extracted from two implementations of the same idea that had -grown up on either side of one feed: a producer in `appwrite-labs/cloud` and a -consumer in `appwrite-labs/edge`. Between them they had two event models, two -pull loops, two cursor stores and two copies of the HTTP contract that joined -them β€” the duplication that matters most, because the two halves drifting apart -is a wire incompatibility rather than a local bug. - -This is what maps onto what. - -## Producer (cloud) - -| Was | Now | -| --- | --- | -| `Feed::append()` / `read()` / `poll()` | `Feed`, on `Journal\Pool` | -| `Feed::after()` β€” stream id arithmetic | `Id::after()` | -| `Feed::getCursor()` / `saveCursor()` | `Cursor\Pool` | -| `Consumer::consume()` | `Consumer::consume()` | -| Query params and response shape in `Http\Feeds\Get` | `Protocol` | -| `Cache-Control` rules in `Http\Feeds\Get` | `Protocol::cacheControl()` | -| `EdgeFeed` | Stays β€” subclass `Feed` | -| `FastlyConsumer` | Stays β€” becomes a handler | -| `Response\Model\FeedEvent` | Stays β€” it is an SDK response model | - -`EdgeFeed` stays in cloud because the tag names in its payloads are a contract -with the edge, not a general-purpose feed concern. It keeps its typed method per -invalidatable resource for the same reason as before β€” a caller-supplied tag -array with a typo in it produces an event that silently invalidates nothing β€” -and now only has to define the vocabulary: - -```php -class EdgeFeed extends Feed -{ - public const string NAME = 'edge'; - - public const string EVENT_INVALIDATE_RULE = 'io.appwrite.edge.invalidate-rule'; - - public function __construct(?Pool $pool, string $source, int $maxSize = 100_000) - { - parent::__construct( - $pool === null ? new None(self::NAME) : new Journal\Pool($pool, self::NAME, $maxSize), - $source, - ); - } - - public function invalidateRule(string $domain, bool $isAppwriteNetwork = false): string - { - if ($domain === '') { - throw new \InvalidArgumentException('Rule invalidation requires a domain'); - } - - return $this->append( - self::EVENT_INVALIDATE_RULE, - ['tags' => ['domain' => $domain], 'isAppwriteNetwork' => $isAppwriteNetwork], - $domain, - ); - } - - // ...one method per invalidatable resource, as before -} -``` - -The nullable pool becomes `Journal\None`, which throws on use with the same -intent as the old `pool()` guard: a feed with no backend must fail loudly rather -than drop events. - -`Http\Feeds\Get` keeps its route, auth and SDK metadata, and hands the wire -details to `Protocol`: - -```php -$events = $feed->poll($lastEventId === '' ? null : $lastEventId, $limit, $timeout); - -$response->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)); -$response->dynamic(new Document(Protocol::encode($events)), Response::MODEL_FEED_EVENT_LIST); -``` - -`FastlyConsumer` stops extending `Consumer` and becomes a handler passed to one. -Its Fastly-specific parts β€” the purge URL, the surrogate key format, the -credential β€” stay in cloud; the pull loop does not: - -```php -$consumer = new Consumer($edgeFeed, FastlyConsumer::NAME, new Cursor\Pool($pool, EdgeFeed::NAME)); - -$purged = 0; -$seen = []; - -$consumer->consume(function (CloudEvent $event) use (&$purged, &$seen): void { - $url = $this->purgeUrl($event); - if ($url === null || isset($seen[$url])) { - return; - } - - $this->send($url); - $seen[$url] = true; - $purged++; -}); -``` - -`purgeUrl()` reads a typed `CloudEvent` instead of an array: - -```php -if ($event->type !== EdgeFeed::EVENT_INVALIDATE_RULE) { - return null; -} - -$tags = $event->data['tags'] ?? []; -$domain = \is_array($tags) ? ($tags['domain'] ?? '') : ''; -``` - -## Consumer (edge) - -| Was | Now | -| --- | --- | -| `Feed\Consumer` | `Consumer`, plus a handler | -| `Feed\Cursor` | `Cursor\Cache` | -| `Feed\Event` | `Utopia\CloudEvents\CloudEvent` β€” this library has no event type of its own | -| `Feed\Event::FEED` and the type constants | Stay β€” they name cloud's feed and its events | -| `Manager::fetchFeed()` | `Journal\Http`, over `utopia-php/client` | -| `Consumer::TIMEOUT_MARGIN` | `Protocol::TIMEOUT_MARGIN` | -| `Feed\Poller` | Stays β€” Swoole interval scheduling | -| `Router\Invalidator` | Stays β€” it purges edge caches | - -The whole of `Feed\Consumer`, `Feed\Cursor` and `Feed\Event` is replaced by -construction: - -```php -$client = (new Client(new Curl()))->withHeaders(['x-appwrite-jwt' => $token]); - -$consumer = new Consumer( - feed: new Feed(new Http($client, $endpoint . '/manager/feeds', 'edge')), - name: $region, - cursor: new Cursor\Cache($cache, 'edge'), - batch: 500, - timeout: $timeout, -); - -$consumer->onWarning(fn (\Throwable $error, string $context) => - Console::warning("[feed] Could not {$context} the {$region} cursor: {$error->getMessage()}")); -``` - -The edge's old `Event::from()` normalized tags on the way in so that -`tags === []` was a reliable answer to "is there anything to do?". That belongs -with the invalidator that defines what a usable tag is, so it moves into the -handler: - -```php -$consumer->consume(function (CloudEvent $event) use ($invalidator): void { - $tags = $event->data['tags'] ?? []; - $tags = \is_array($tags) ? Invalidator::normalize($tags) : []; - - if ($tags === []) { - Span::add('feed.consume.skipped_type', $event->type); - return; - } - - $invalidator->invalidate($tags); -}); -``` - -`Poller` keeps its Swoole `WaitGroup` fan-out and its once-per-region 404 -reporting. Only the exception it catches changes, from `PlatformException` to -`Utopia\Feed\Exception\Transport` β€” the status is still on `getCode()`, so the -`!== 404` check is unchanged. - -## Behaviour that is deliberately identical - -These were load-bearing in the original implementations and are preserved: - -- **A consumer with no position starts at the oldest retained event**, not at - the tip. This is what makes the staged rollout in the README safe. -- **A run commits the events handled before a failure**, then re-raises it. The - failed event is retried on the next run, and everything behind it waits. -- **A cursor store that is down is a warning, not a failure.** The position is - mirrored in memory, so the consumer keeps working and only a restart replays. -- **The `feed::cursor:` key format**, so consumers keep their - positions across the migration instead of replaying the retained feed. -- **`-` event ids**, so positions already handed out stay valid. -- **The margin a consumer allows its HTTP client over the long-poll timeout**, - without which every quiet tick surfaces as a transport failure. - -## Behaviour that changed - -- **`source` is stamped at append rather than at read.** Previously every event - read from a feed was labelled with the reading region's source, whoever - actually produced it. Storing it at append costs nothing and keeps it correct - for a feed that is replicated or read back somewhere else. -- **A batch containing an event with no id no longer throws away the valid - events in front of it.** `Protocol::decode()` returns the usable prefix, and - throws once the broken event reaches the head of a batch β€” where the feed - stops visibly rather than quietly losing events. -- **`Cache-Control` is computed from the batch**, and `public` is opt-in rather - than a decision baked into one endpoint. -- **Events are `Utopia\CloudEvents\CloudEvent`**, not a bespoke type. `subject` - is nullable, so an event without one reads back as `null` rather than `''`; - `data` is unrestricted, so a list or scalar payload round-trips as itself; and - `dataschema` and extension attributes now survive an append and a read. diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index c961b7f..6815186 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -11,7 +11,7 @@ * and records how far it got. * * ```php - * $consumer = new Consumer($feed, 'cache-invalidator', new Cursor\Cache($cache, 'edge')); + * $consumer = new Consumer($feed, 'cache-invalidator', new Cursor\Cache($cache)); * * // On a timer, or in a loop with a long-poll timeout: * $consumer->consume(function (CloudEvent $event) use ($cache) { @@ -19,55 +19,22 @@ * }); * ``` * - * ## What the handler must tolerate - * - * Delivery is at-least-once, so a handler will see the same event more than - * once and must be safe to repeat. There are three separate reasons, and no - * arrangement of this class removes any of them: - * - * 1. A handler can succeed and the position then fail to save. - * 2. A batch interrupted partway replays from the last event that succeeded. - * 3. A consumer whose position is lost restarts from the oldest retained event. - * - * Every one of them re-delivers; none of them skips. That asymmetry is the - * whole design β€” an event handled twice is absorbed by an idempotent handler, - * whereas an event stepped over is gone, still sitting in the feed with nothing - * that will ever read it again. - * - * A handler rejects an event by throwing. That stops the run at that event and - * leaves the position before it, so the next run starts there and tries again. - * Everything already handled in that run stays handled β€” progress is committed - * before the failure is re-raised β€” which means a handler that fails on one - * event does not undo the batch, but does block everything behind it until it - * stops failing. That is the intended behaviour: a feed is ordered, and - * stepping over a failure would deliver later events on top of state that was - * never updated. - * - * ## Starting position - * - * A consumer with no recorded position starts at the oldest retained event, - * never at the tip. Starting at the tip would drop whatever is already in the - * feed, and for a consumer being deployed for the first time that is not a - * hypothetical backlog β€” it is everything that happened between the producer - * shipping and the consumer shipping, which during a staged rollout is exactly - * the events that were meant to be caught up on. + * Delivery is at-least-once, so a handler must be safe to run twice on the same + * event. A handler rejects an event by throwing, which stops the run there and + * leaves the position before it, so the next run tries again. See the README + * for what that means in practice. */ class Consumer { /** * Events per run. Small enough that a backlog drains in bounded steps - * instead of one long pass that fails near the end and repeats most of - * itself. + * instead of one long pass that fails near the end and repeats itself. */ public const int BATCH = 100; /** - * The position, mirrored in memory. - * - * A run therefore reads the store once, on its first pass, and a store - * that becomes unavailable afterwards costs nothing β€” the consumer keeps - * making progress and only replays if it restarts before the store - * recovers. + * The position, mirrored in memory, so a run reads the store once and a + * store that becomes unavailable afterwards costs nothing. */ private ?string $position = null; @@ -80,20 +47,15 @@ class Consumer * @param Feed $feed Feed to read. * @param string $name This consumer's name, which its position is stored * under. Distinct per logical consumer, and stable across restarts. - * - * **One process per name.** Two sharing a name share one position, - * so each sees only the events the other has not already advanced - * past β€” the feed is split between them rather than delivered to - * both, which is not what a handler written against this class - * expects. Give each replica the same name only if you mean them to - * divide the work. + * Run **one process per name** β€” two sharing a name share one + * position, so the feed is split between them rather than delivered + * to both. * @param Cursor $cursor Where to keep the position. * @param int $batch Events per run. * @param int $timeout Milliseconds to wait for an event when the feed is * caught up. Zero returns immediately, which is what a consumer - * driven by an external timer wants; a non-zero value suits a - * consumer looping on its own, where it replaces a sleep with a - * wait that ends the moment an event arrives. + * driven by an external timer wants; a non-zero value suits one + * looping on its own. * @throws Exception\Invalid When $name is empty. */ public function __construct( @@ -113,19 +75,14 @@ public function getName(): string return $this->name; } - public function getFeed(): Feed - { - return $this->feed; - } - /** * Report failures that were survived rather than raised β€” currently, a * position that could not be loaded or saved. * - * These are not fatal: the consumer carries on with its in-memory - * position and the only cost is a replay after a restart. They are still - * worth knowing about, because a store that has been failing quietly for a - * week is a replay of the entire retained feed waiting to happen. + * These are not fatal: the consumer carries on with its in-memory position + * and the only cost is a replay after a restart. They are still worth + * knowing about, because a store that has been failing quietly for a week + * is a replay of the entire retained feed waiting to happen. * * @param (callable(\Throwable, string): void)|null $callback Receives the * error and a short context string. @@ -141,13 +98,13 @@ public function onWarning(?callable $callback): self * Hand every event not yet seen to $handler, oldest first, and return how * many it accepted. * - * @param callable(CloudEvent): void $handler Throws to reject an event, which - * stops the run and leaves the position before it. + * @param callable(CloudEvent): void $handler Throws to reject an event, + * which stops the run and leaves the position before it. * @return int Events handled. Zero means the consumer is caught up. * @throws Exception When the feed cannot be read. The position stays where * it was, so the next run retries the same events. - * @throws \Throwable Whatever the handler threw, after the events before - * it have been committed. + * @throws \Throwable Whatever the handler threw, after the events before it + * have been committed. */ public function consume(callable $handler): int { @@ -196,13 +153,12 @@ public function position(): ?string $this->restored = true; try { - $this->position = $this->cursor->load($this->name); + $this->position = $this->cursor->load($this->feed->getName(), $this->name); } catch (\Throwable $error) { // Falls through to null, which restarts from the oldest retained - // event. Wasteful β€” it replays events already handled β€” but the - // alternatives are worse: guessing at a position risks skipping, - // and refusing to run means an outage in the cursor store becomes - // an outage in whatever the feed drives. + // event. Wasteful, but the alternatives are worse: guessing at a + // position risks skipping, and refusing to run turns an outage in + // the cursor store into an outage in whatever the feed drives. $this->warn($error, 'load'); } @@ -217,7 +173,7 @@ public function position(): ?string */ public function reset(): void { - $this->cursor->reset($this->name); + $this->cursor->reset($this->feed->getName(), $this->name); $this->position = null; $this->restored = true; @@ -228,10 +184,10 @@ private function advance(string $eventId): void $this->position = $eventId; try { - $this->cursor->save($this->name, $eventId); + $this->cursor->save($this->feed->getName(), $this->name, $eventId); } catch (\Throwable $error) { - // In-memory position already moved, so this process does not - // repeat itself; only a restart before the store recovers replays. + // In-memory position already moved, so this process does not repeat + // itself; only a restart before the store recovers replays. $this->warn($error, 'save'); } } diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index c9bf310..682e449 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -10,47 +10,24 @@ * Where a consumer's position in a feed is kept. * * http-feeds puts the position on the consumer rather than the producer, which - * is what makes adding a consumer free β€” the producer keeps no per-consumer - * state, so it does not need to know who is reading. The cost is that each - * consumer needs somewhere to write a string, and that is all this is. + * is what makes adding a consumer free. A cursor is somewhere to write a + * string, keyed by feed and consumer name, so one store serves every feed a + * service consumes. * - * The store is allowed to be lossy. A lost position is not a lost event: a - * consumer with no position resumes from the oldest retained event, so the - * consequence is redundant work, not a gap. That is why a cache is a - * reasonable place to put one, and it is also why handlers must tolerate - * seeing an event twice β€” which they must anyway, since delivery is - * at-least-once regardless. - * - * One store can hold positions for many consumers of the same feed, keyed by - * consumer name. + * The store is allowed to be lossy. A lost position is not a lost event β€” a + * consumer with no position resumes from the oldest retained event β€” so the + * cost is redundant work, not a gap. */ abstract class Cursor { /** - * @param string $feed Feed the positions belong to. Part of the key, so - * one store can serve every feed a service consumes. - * @throws Invalid When $feed is empty. - */ - public function __construct(protected readonly string $feed) - { - if ($feed === '') { - throw new Invalid('Cursor requires a feed name'); - } - } - - public function getFeed(): string - { - return $this->feed; - } - - /** - * The last position $consumer recorded, or null if it has never recorded - * one β€” which a caller should read as "start from the beginning of what is - * retained", never as "start from now". + * The last position $consumer recorded on $feed, or null if it has never + * recorded one β€” which means "start from the oldest retained event", never + * "start from now". * * @throws Exception When the store cannot be read. */ - abstract public function load(string $consumer): ?string; + abstract public function load(string $feed, string $consumer): ?string; /** * Record a position. @@ -58,33 +35,32 @@ abstract public function load(string $consumer): ?string; * Only ever call this once the events up to $eventId have been handled. A * position saved ahead of the work it stands for turns a crash into * silently skipped events, which is the one failure this design cannot - * recover from β€” the events are still in the feed, but nothing will ever - * read them again. + * recover from. * * An empty $eventId is ignored rather than rejected: it means "nothing * handled yet", and must not erase a real position. * * @throws Exception When the store cannot be written. */ - abstract public function save(string $consumer, string $eventId): void; + abstract public function save(string $feed, string $consumer, string $eventId): void; /** - * Forget a consumer's position, so its next read starts from the oldest + * Forget a position, so the consumer's next read starts from the oldest * retained event. * * @throws Exception When the store cannot be written. */ - abstract public function reset(string $consumer): void; + abstract public function reset(string $feed, string $consumer): void; /** - * @throws Invalid When $consumer is empty. + * @throws Invalid When either name is empty. */ - protected function key(string $consumer): string + protected function key(string $feed, string $consumer): string { - if ($consumer === '') { - throw new Invalid('Cursor requires a consumer name'); + if ($feed === '' || $consumer === '') { + throw new Invalid('Cursor requires a feed and a consumer name'); } - return 'feed:' . $this->feed . ':cursor:' . $consumer; + return 'feed:' . $feed . ':cursor:' . $consumer; } } diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php index 93ae7c6..9e757e1 100644 --- a/src/Feed/Cursor/Cache.php +++ b/src/Feed/Cursor/Cache.php @@ -8,59 +8,48 @@ use Utopia\Feed\Cursor; /** - * Positions kept in a Utopia cache. - * - * The right store for a consumer reading a remote feed: it already has a cache - * for its own work, and losing a position there only costs a replay of - * whatever is still retained. + * Positions kept in a Utopia cache β€” the usual choice for a consumer reading a + * remote feed, since it already has a cache for its own work. * * Note that a Utopia cache lowercases keys unless `setCaseSensitivity(true)` - * was called on it, so consumer names that differ only in case will share a - * position β€” and two consumers sharing a position each skip what the other - * handled. + * was called on it, so consumer names that differ only in case share a position. * * @see https://github.com/utopia-php/cache */ class Cache extends Cursor { /** - * How long a position survives without being written. - * - * The clock runs from the last save, not the last load, so this is really - * "how long a consumer may go without handling anything". A month is long - * enough that only a feed which has genuinely gone silent reaches it, and - * a consumer that does lose its position replays the retained feed rather - * than losing anything. + * How long a position survives without being written. The clock runs from + * the last save, so this is really "how long a consumer may go without + * handling anything". */ public const int TTL = 30 * 24 * 60 * 60; public function __construct( protected readonly UtopiaCache $cache, - string $feed, protected readonly int $ttl = self::TTL, ) { - parent::__construct($feed); } - public function load(string $consumer): ?string + public function load(string $feed, string $consumer): ?string { /** @var mixed $cursor */ - $cursor = $this->cache->load($this->key($consumer), $this->ttl); + $cursor = $this->cache->load($this->key($feed, $consumer), $this->ttl); return \is_string($cursor) && $cursor !== '' ? $cursor : null; } - public function save(string $consumer, string $eventId): void + public function save(string $feed, string $consumer, string $eventId): void { if ($eventId === '') { return; } - $this->cache->save($this->key($consumer), $eventId); + $this->cache->save($this->key($feed, $consumer), $eventId); } - public function reset(string $consumer): void + public function reset(string $feed, string $consumer): void { - $this->cache->purge($this->key($consumer)); + $this->cache->purge($this->key($feed, $consumer)); } } diff --git a/src/Feed/Cursor/Memory.php b/src/Feed/Cursor/Memory.php index cc0c1d4..4ac2ab9 100644 --- a/src/Feed/Cursor/Memory.php +++ b/src/Feed/Cursor/Memory.php @@ -10,30 +10,29 @@ * Positions held in process memory. * * For tests, and for a consumer that genuinely wants to start from the - * beginning of the retained feed on every restart. Anything else will replay - * the whole feed each time it is deployed. + * beginning of the retained feed on every restart. */ class Memory extends Cursor { /** @var array */ private array $cursors = []; - public function load(string $consumer): ?string + public function load(string $feed, string $consumer): ?string { - return $this->cursors[$this->key($consumer)] ?? null; + return $this->cursors[$this->key($feed, $consumer)] ?? null; } - public function save(string $consumer, string $eventId): void + public function save(string $feed, string $consumer, string $eventId): void { if ($eventId === '') { return; } - $this->cursors[$this->key($consumer)] = $eventId; + $this->cursors[$this->key($feed, $consumer)] = $eventId; } - public function reset(string $consumer): void + public function reset(string $feed, string $consumer): void { - unset($this->cursors[$this->key($consumer)]); + unset($this->cursors[$this->key($feed, $consumer)]); } } diff --git a/src/Feed/Cursor/Pool.php b/src/Feed/Cursor/Pool.php index ce27f92..faf6e3e 100644 --- a/src/Feed/Cursor/Pool.php +++ b/src/Feed/Cursor/Pool.php @@ -10,9 +10,8 @@ /** * {@see Redis}, over a pooled connection. * - * Pairs with {@see \Utopia\Feed\Journal\Pool}, and can share its pool: a - * cursor read is one `GET`, so it borrows a connection only for as long as - * that takes. + * Pairs with {@see \Utopia\Feed\Journal\Pool}, and can share its pool: a cursor + * read is one `GET`, so it borrows a connection only for as long as that takes. * * @see https://github.com/utopia-php/pools */ @@ -21,34 +20,26 @@ class Pool extends Cursor /** * @param UtopiaPool<\Redis|\RedisCluster> $pool */ - public function __construct( - protected readonly UtopiaPool $pool, - string $feed, - ) { - parent::__construct($feed); - } - - public function load(string $consumer): ?string + public function __construct(protected readonly UtopiaPool $pool) { - return $this->pool->use(fn (\Redis|\RedisCluster $redis): ?string => $this->cursor($redis)->load($consumer)); } - public function save(string $consumer, string $eventId): void + public function load(string $feed, string $consumer): ?string { - $this->pool->use(function (\Redis|\RedisCluster $redis) use ($consumer, $eventId): void { - $this->cursor($redis)->save($consumer, $eventId); - }); + return $this->pool->use(fn (\Redis|\RedisCluster $redis): ?string => (new Redis($redis))->load($feed, $consumer)); } - public function reset(string $consumer): void + public function save(string $feed, string $consumer, string $eventId): void { - $this->pool->use(function (\Redis|\RedisCluster $redis) use ($consumer): void { - $this->cursor($redis)->reset($consumer); + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($feed, $consumer, $eventId): void { + (new Redis($redis))->save($feed, $consumer, $eventId); }); } - private function cursor(\Redis|\RedisCluster $redis): Redis + public function reset(string $feed, string $consumer): void { - return new Redis($redis, $this->feed); + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($feed, $consumer): void { + (new Redis($redis))->reset($feed, $consumer); + }); } } diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 11137cd..3c88517 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -10,30 +10,25 @@ /** * Positions kept in Redis, as plain keys alongside the stream. * - * For consumers running inside the producer β€” a job that turns the feed into - * something else, a bridge to a system that cannot poll. They have no store of - * their own, and the feed's Redis is already there. - * + * For consumers running inside the producer, which have no store of their own. * Consumers reached over HTTP should not use this: keeping their positions in * the producer's Redis puts per-consumer state back on the producer, which is * exactly what the feed is arranged to avoid. */ class Redis extends Cursor { - public function __construct( - protected readonly \Redis|\RedisCluster $redis, - string $feed, - ) { - parent::__construct($feed); + /** + * @param \Redis|\RedisCluster $redis + */ + public function __construct(protected readonly \Redis|\RedisCluster $redis) + { } - public function load(string $consumer): ?string + public function load(string $feed, string $consumer): ?string { - $key = $this->key($consumer); - try { /** @var mixed $cursor */ - $cursor = $this->redis->get($key); + $cursor = $this->redis->get($this->key($feed, $consumer)); } catch (\RedisException $error) { throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); } @@ -41,30 +36,26 @@ public function load(string $consumer): ?string return \is_string($cursor) && $cursor !== '' ? $cursor : null; } - public function save(string $consumer, string $eventId): void + public function save(string $feed, string $consumer, string $eventId): void { if ($eventId === '') { return; } - $key = $this->key($consumer); - try { // Deliberately no expiry. Unlike a cache, this is the only copy, // and a position that quietly expired would replay the whole // retained feed the next time the consumer restarted. - $this->redis->set($key, $eventId); + $this->redis->set($this->key($feed, $consumer), $eventId); } catch (\RedisException $error) { throw new Transport("Failed to save the {$consumer} cursor: {$error->getMessage()}", previous: $error); } } - public function reset(string $consumer): void + public function reset(string $feed, string $consumer): void { - $key = $this->key($consumer); - try { - $this->redis->del($key); + $this->redis->del($this->key($feed, $consumer)); } catch (\RedisException $error) { throw new Transport("Failed to reset the {$consumer} cursor: {$error->getMessage()}", previous: $error); } diff --git a/src/Feed/Exception/Unsupported.php b/src/Feed/Exception/Unsupported.php index 1e764b2..969c70e 100644 --- a/src/Feed/Exception/Unsupported.php +++ b/src/Feed/Exception/Unsupported.php @@ -8,7 +8,7 @@ /** * The journal cannot do what was asked of it β€” appending to a feed read over - * HTTP, or any operation at all on {@see \Utopia\Feed\Journal\None}. + * HTTP, or any operation at all on {@see \Utopia\Feed\Journal\Unconfigured}. * * Thrown rather than ignored: an append that silently does nothing loses * events, and a consumer cannot tell an empty feed from an absent one. diff --git a/src/Feed/Feed.php b/src/Feed/Feed.php index 23c9952..d94230e 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Feed.php @@ -7,26 +7,18 @@ use Utopia\CloudEvents\CloudEvent; /** - * An append-only, strongly ordered sequence of events that consumers pull. + * An append-only, ordered sequence of events that consumers pull. * - * Follows http-feeds (https://www.http-feeds.org/): rather than the producer - * pushing every event to every consumer, each consumer asks for what it has - * not seen yet, quoting the id of the last event it processed. Delivery then - * stops depending on every consumer being reachable at the moment something - * happens β€” one that was down, redeploying, or only just added catches up on - * its next read instead of missing the event entirely. + * Follows http-feeds (https://www.http-feeds.org/): each consumer asks for what + * it has not seen yet, quoting the id of the last event it processed, so one + * that was down or only just deployed catches up on its next read. * - * The trade is at-least-once delivery. Consumers fall behind, retry, and - * restart from positions they have already passed, so **every event must be - * safe to process twice**. Retention is bounded, so a consumer that falls - * behind the trim horizon resumes from the oldest retained event rather than - * failing β€” which makes the feed unsuitable for events whose effect depends on - * seeing all of them (a balance built from deltas), and a good fit for events - * that describe a state to converge on (a cache tag to drop, a record to - * refresh). + * Delivery is at-least-once and retention is bounded, so every event must be + * safe to process twice, and a consumer that falls behind the trim horizon + * resumes from the oldest retained event. * * ```php - * $feed = new Feed(new Redis($redis, 'edge'), 'urn:appwrite:cloud:fra'); + * $feed = new Feed(new Journal\Redis($redis, 'edge'), 'urn:appwrite:cloud:fra'); * * $feed->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']]); * @@ -41,34 +33,23 @@ class Feed { /** - * Most events a single read may return. A cap belongs here rather than on - * the caller: `limit` arrives from a consumer over the network, and an - * unbounded read is a way to hold a producer's worker open. + * Most events a single read may return. The cap belongs here because + * `limit` arrives from a consumer over the network. */ public const int MAX_BATCH = 1000; /** - * Longest a long poll may hold a request open, in milliseconds. Kept - * under the 60s that proxies and load balancers commonly cut idle - * responses off at, so a poll ends by returning empty rather than by - * having the connection dropped underneath it. + * Longest a long poll may hold a request open, in milliseconds. Kept under + * the 60s that proxies commonly cut idle responses off at. */ public const int MAX_TIMEOUT = 30_000; - /** - * Microseconds between reads while long polling on a backend that cannot - * block on its own. Half a second bounds delivery latency at roughly that, - * while keeping a quiet feed at two reads a second per consumer. - */ - protected const int POLL_INTERVAL = 500_000; - /** * @param Journal $journal Where the events live. * @param string $source Who is producing them, as a URI reference - * (`urn:appwrite:cloud:fra`). Stamped onto every event this - * instance appends, so a consumer merging feeds from several - * producers can tell which one an event came from. Irrelevant when - * the feed is only being read. + * (`urn:appwrite:cloud:fra`). Stamped onto every event this instance + * appends, so a consumer merging feeds from several producers can + * tell them apart. Only needed to append, not to read. */ public function __construct( protected readonly Journal $journal, @@ -76,40 +57,24 @@ public function __construct( ) { } - public function getJournal(): Journal - { - return $this->journal; - } - public function getName(): string { return $this->journal->getName(); } - public function getSource(): string - { - return $this->source; - } - /** * Append an event and return its position in the feed. * * @param string $type What happened, in reverse-DNS notation. * @param mixed $data Payload, JSON encodable. Usually a map, but the JSON - * event format leaves it unrestricted, so a list or a scalar is - * equally valid. - * @param string $subject The one business object this is about, if there - * is one. Empty means none, which is how CloudEvents models it. - * @throws Exception\Invalid When $type is empty or $data cannot be - * encoded. + * event format leaves it unrestricted. + * @param string $subject The one business object this is about, if there is + * one. Empty means none, which is how CloudEvents models it. + * @throws Exception\Invalid When $type is empty or $data cannot be encoded. * @throws Exception When the backend rejects the append. */ public function append(string $type, mixed $data = [], string $subject = ''): string { - if ($type === '') { - throw new Exception\Invalid('Feed event type is required'); - } - return $this->publish(new CloudEvent( type: $type, subject: $subject === '' ? null : $subject, @@ -118,16 +83,11 @@ public function append(string $type, mixed $data = [], string $subject = ''): st } /** - * Append a prepared event, stamping it with this feed's source and the - * current time. - * - * Both are stamped here rather than accepted from the caller because they - * describe the append itself. Recording the source at append rather than - * at read also keeps it correct for a feed that is replicated or read back - * from somewhere other than where it was written. + * Append a prepared event, stamping it with this feed's source and, unless + * it already has one, the current time. * - * @throws Exception\Invalid When the event has no type or its data cannot - * be encoded. + * @throws Exception\Invalid When the event has no type, the feed has no + * source, or the data cannot be encoded. * @throws Exception When the backend rejects the append. */ public function publish(CloudEvent $event): string @@ -136,14 +96,16 @@ public function publish(CloudEvent $event): string throw new Exception\Invalid('Feed event type is required'); } + if ($this->source === '') { + throw new Exception\Invalid('Feed source is required to append; construct the feed with one'); + } + // Stamped with the withers rather than rebuilt, so anything this // library does not model itself β€” a dataschema, an extension attribute // such as a traceparent β€” survives the append untouched. - return $this->journal->append( - $event - ->withSource($this->source) - ->withTime($event->time !== '' ? $event->time : null) - ); + $event = $event->withSource($this->source); + + return $this->journal->append($event->time === '' ? $event->withTime() : $event); } /** @@ -165,18 +127,14 @@ public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): * {@see read()}, but when there is nothing new yet, wait up to $timeout * milliseconds for something to arrive before answering. * - * This is how a consumer subscribes in near real time without hammering - * the producer: poll in a loop with a timeout, and each call either - * returns as soon as an event is appended or costs one request per - * timeout while the feed is quiet. A timeout of 0 makes this a plain read. - * - * The batch may still come back empty β€” the timeout elapsing is a normal - * outcome, not a failure. + * This is how a consumer subscribes in near real time without hammering the + * producer. The batch may still come back empty β€” the timeout elapsing is a + * normal outcome, not a failure. A timeout of 0 makes this a plain read. * - * Where the backend cannot block on its own this waits by re-reading on an - * interval, which under Swoole yields the worker only if coroutine hooks - * are enabled. Without them it holds the worker for the duration, so run - * it with hooks on or keep the timeout at 0. + * Journals that cannot block wait by re-reading on an interval, which under + * Swoole yields the worker only if coroutine hooks are enabled. Without them + * it holds the worker for the duration, so run it with hooks on or keep the + * timeout at 0. * * @return list * @throws Exception\Invalid When $lastEventId is not a feed position. @@ -184,32 +142,25 @@ public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): */ public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, int $timeout = 0): array { - $limit = self::limit($limit); - $timeout = \max(0, \min($timeout, self::MAX_TIMEOUT)); - - if ($this->journal->pollable()) { - return $this->journal->read($lastEventId, $limit, $timeout); - } - - $deadline = \microtime(true) + $timeout / 1000; - - while (true) { - $events = $this->journal->read($lastEventId, $limit); - - if ($events !== [] || \microtime(true) >= $deadline) { - return $events; - } - - \usleep(self::POLL_INTERVAL); - } + return $this->journal->poll( + $lastEventId, + self::limit($limit), + \max(0, \min($timeout, self::MAX_TIMEOUT)), + ); } /** - * Clamp rather than reject: `limit` is a hint about how much work a + * The limit a read will actually use. + * + * Clamped rather than rejected: `limit` is a hint about how much work a * consumer wants in one go, and failing a read because it asked for too * much would stall a feed over something the producer can simply decide. + * + * An endpoint serving this feed should clamp with this before answering, so + * the number it passes to {@see Protocol::cacheControl()} is the one the + * batch was actually built with. */ - private static function limit(int $limit): int + public static function limit(int $limit): int { return \max(1, \min($limit, self::MAX_BATCH)); } diff --git a/src/Feed/Id.php b/src/Feed/Id.php index f4619f8..0947b80 100644 --- a/src/Feed/Id.php +++ b/src/Feed/Id.php @@ -7,17 +7,13 @@ use Utopia\Feed\Exception\Invalid; /** - * Feed positions. + * Feed positions: `-`, Redis' stream id format, where + * the sequence disambiguates events appended within the same millisecond. * - * http-feeds requires event ids to be strictly ordered so a consumer can ask - * for "everything after this one" with nothing but the last id it processed. - * This library uses Redis' stream id format for them β€” `-`, - * where `seq` disambiguates events appended within the same millisecond. - * - * The format is part of the wire contract, not a Redis implementation detail: - * an id produced by one journal has to be a valid position for another, so - * that a feed can move between backends without invalidating the positions - * consumers already hold. + * The format is part of the wire contract, not a Redis implementation detail β€” + * an id produced by one journal has to be a valid position for another, so a + * feed can move between backends without invalidating the positions consumers + * already hold. */ final class Id { @@ -40,7 +36,8 @@ public static function encode(int $timestamp, int $sequence): string } /** - * Split an id into its millisecond timestamp and sequence number. + * Split an id into its millisecond timestamp and sequence number, which + * compare in feed order (`10-0` is after `9-0`, not before it). * * @return array{int, int} * @throws Invalid When $id is not a feed position. @@ -55,13 +52,12 @@ public static function decode(string $id): array } /** - * The exclusive successor of an id: the smallest position strictly after - * it. + * The exclusive successor of an id: the smallest position strictly after it. * * Computed rather than relying on Redis' `(`-prefixed exclusive ranges, so * reads work against anything speaking the Redis 5 stream API β€” including - * the several proxies and compatible servers that never implemented the - * newer syntax. + * the proxies and compatible servers that never implemented the newer + * syntax. * * @throws Invalid When $id is not a feed position. */ @@ -71,15 +67,4 @@ public static function after(string $id): string return self::encode($timestamp, $sequence + 1); } - - /** - * Compare two positions the way `<=>` would, so ids sort by age rather - * than by string order (`10-0` is after `9-0`, not before it). - * - * @throws Invalid When either id is not a feed position. - */ - public static function compare(string $a, string $b): int - { - return self::decode($a) <=> self::decode($b); - } } diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index c64d5e7..fd7be5f 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -9,29 +9,22 @@ use Utopia\Feed\Exception\Invalid; /** - * Where a feed's events actually live. + * Where a feed's events live. * - * Named for what event sourcing has long called an append-only, strictly - * ordered record that is replayed rather than mutated β€” the same sense in which - * Akka Persistence calls its pluggable storage backends journals. - * - * A journal is responsible for two things and nothing else: assigning an - * ordered id on append, and returning the events strictly after a given id. - * Everything above that β€” long polling on backends that cannot do it - * themselves, cursors, the pull loop β€” is the same regardless of the backend - * and lives in {@see Feed} and {@see Consumer}. - * - * Journals split into producers (Redis, Pool, Memory), which own the events, - * and consumers ({@see Journal\Http}), which read someone else's feed over the - * wire. The read side is identical either way, which is what lets a service - * consume a remote feed with the same code it uses on a local one. + * A journal does two things: assign an ordered id on append, and return the + * events strictly after a given id. Everything above it β€” long polling, + * cursors, the pull loop β€” is the same whichever journal is underneath. */ abstract class Journal { + /** + * Microseconds between reads while waiting in {@see poll()}. + */ + protected const int POLL_INTERVAL = 500_000; + /** * @param string $name Feed identifier. Also the key the backend stores it - * under, and the path segment it is served on, so it is part of the - * contract with consumers rather than a local label. + * under, and the path segment it is served on. * @throws Invalid When $name is empty. */ public function __construct(protected readonly string $name) @@ -50,11 +43,9 @@ public function getName(): string * Append an event and return the id the backend assigned it. * * Any id already on $event is ignored: positions are the backend's to - * allocate, since only it can guarantee they are ordered. + * allocate, since only it can keep them ordered. * - * @throws Exception When the event cannot be appended. Never silently, and - * never partially β€” a caller that gets an id back can tell every - * consumer will see the event. + * @throws Exception When the event cannot be appended. */ abstract public function append(CloudEvent $event): string; @@ -65,30 +56,40 @@ abstract public function append(CloudEvent $event): string; * An empty result means the consumer is caught up, not that the feed is * empty. * - * @param int $timeout Milliseconds to wait for an event before giving up, - * honoured only when {@see pollable()} is true; {@see Feed::poll()} - * handles the wait for every other journal. * @return list * @throws Invalid When $lastEventId is not a feed position. * @throws Exception When the backend cannot be read. */ - abstract public function read(?string $lastEventId, int $limit, int $timeout = 0): array; + abstract public function read(?string $lastEventId, int $limit): array; /** - * Whether the backend blocks until an event arrives on its own. + * {@see read()}, but wait up to $timeout milliseconds for an event before + * answering with an empty batch. * - * False here rather than abstract because polling in a loop works against - * anything; a journal only overrides it when the backend can do better, - * and {@see Feed::poll()} then hands the wait over instead of sleeping. + * Waits by re-reading on an interval, which works against any backend. A + * journal that can do better β€” {@see Journal\Http} hands the wait to the + * producer β€” overrides this. + * + * @return list + * @throws Invalid When $lastEventId is not a feed position. + * @throws Exception When the backend cannot be read. */ - public function pollable(): bool + public function poll(?string $lastEventId, int $limit, int $timeout): array { - return false; + $deadline = \microtime(true) + $timeout / 1000; + + while (true) { + $events = $this->read($lastEventId, $limit); + + if ($events !== [] || \microtime(true) >= $deadline) { + return $events; + } + + \usleep(self::POLL_INTERVAL); + } } /** - * Guard a retention cap, at construction. - * * A feed must retain at least one event. Backends disagree about what a * non-positive cap means β€” some keep nothing, some keep everything β€” so it * is refused here rather than resolved differently on each one. @@ -106,13 +107,8 @@ protected static function assertRetention(int $maxSize): void * The backend fields an event is stored as. * * `data` and `extensions` are JSON so they can hold what CloudEvents lets - * them hold; every other attribute is a flat string, because those are the - * ones a backend may want to index or filter on. The id is not among them - * β€” it is the key the entry is stored under. - * - * Extensions are stored rather than dropped: a producer that attaches one - * β€” a `traceparent`, say β€” means it to reach the consumer, and losing it - * on the way through the backend would be invisible at both ends. + * them hold; every other attribute is a flat string a backend can index. + * The id is not among them β€” it is the key the entry is stored under. * * @return array * @throws Invalid When the payload cannot be encoded. @@ -122,12 +118,12 @@ protected static function encode(CloudEvent $event): array return [ 'type' => $event->type, 'source' => $event->source, - // CloudEvents models an absent subject as null. A backend field is - // a string, so it is normalized here rather than stored as a null - // that would read back as "" on one backend and break on another. + // CloudEvents models an absent subject and dataschema as null, and + // a backend field cannot hold one, so both are normalized here and + // read back as absent in decode(). 'subject' => $event->subject ?? '', - 'time' => $event->time, 'dataschema' => $event->dataschema ?? '', + 'time' => $event->time, 'data' => self::json($event->data, 'data'), 'extensions' => self::json($event->getExtensions(), 'extensions'), ]; @@ -136,12 +132,8 @@ protected static function encode(CloudEvent $event): array /** * Rebuild an event from what {@see encode()} stored. * - * Decoded leniently, for the same reason {@see Protocol::decode()} is: the - * event happened, its id is a valid position, and refusing to return it - * over one malformed attribute would wedge every consumer behind it. - * * @param array $fields - * @throws Invalid When the stored entry cannot be read as an event at all. + * @throws Invalid When the stored entry cannot be read as an event. */ protected static function decode(string $id, array $fields): CloudEvent { @@ -156,11 +148,6 @@ protected static function decode(string $id, array $fields): CloudEvent 'data' => \json_decode(self::field($fields, 'data'), true), ]; - // The inverse of the normalization in encode(): these two are nullable - // on a CloudEvent, and a backend field cannot hold a null, so an empty - // stored field means the attribute was absent. Passing the empty string - // through instead would turn "no subject" into "a subject that is the - // empty string" on every round trip through a backend. foreach (['subject', 'dataschema'] as $optional) { $value = self::field($fields, $optional); @@ -169,14 +156,16 @@ protected static function decode(string $id, array $fields): CloudEvent } } - // The union operator rather than a spread, which renumbers integer keys. - // An extension name of only digits is legal β€” the spec allows [a-z0-9]+ β€” - // and PHP stores such a name as an int key, so a spread would silently - // rename "123" to the next free position and lose the attribute. - // Spec attributes stay on the left, so they win any collision. + // The union operator rather than a spread, which renumbers integer + // keys: an extension named only of digits is legal, and PHP holds such + // a name as an int key. Spec attributes stay on the left, so they win + // any collision. $event += \is_array($extensions) ? $extensions : []; try { + // Lenient for the same reason Protocol::decode() is: the event + // happened and its id is a valid position, so refusing to return it + // over one malformed attribute would wedge every consumer behind it. return CloudEvent::fromArray($event, lenient: true); } catch (CloudEventsException $error) { throw new Invalid("Feed entry {$id} could not be read as an event: {$error->getMessage()}", previous: $error); diff --git a/src/Feed/Journal/Http.php b/src/Feed/Journal/Http.php index db2b386..0a7dd49 100644 --- a/src/Feed/Journal/Http.php +++ b/src/Feed/Journal/Http.php @@ -4,13 +4,12 @@ namespace Utopia\Feed\Journal; -use Utopia\Feed\Journal; use Psr\Http\Client\ClientExceptionInterface; use Utopia\Client\Adapter; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; -use Utopia\Feed\Feed; +use Utopia\Feed\Journal; use Utopia\Feed\Protocol; use Utopia\Psr7\ContentType; use Utopia\Psr7\Header; @@ -20,23 +19,12 @@ /** * Someone else's feed, read over HTTP. * - * The counterpart to serving a feed with {@see Protocol}: a service points - * this at another service's feed endpoint and consumes it with the same - * {@see Feed} and {@see \Utopia\Feed\Consumer} it would use on a local one. - * Nothing above the journal knows the events are arriving over the network. - * - * Read-only, because a feed is owned by whoever appends to it. Long polling is - * delegated to the producer, which is the point of doing it this way: the - * consumer holds one request open instead of asking repeatedly, and the - * producer answers the moment an event exists. + * The counterpart to serving a feed with {@see Protocol}: point this at another + * service's feed endpoint and consume it with the same `Feed` and `Consumer` a + * local one uses. Read-only β€” a feed is owned by whoever appends to it. * * ```php - * use Utopia\Client; - * use Utopia\Client\Adapter\Curl\Client as Curl; - * - * $client = (new Client(new Curl())) - * ->withHeaders(['x-appwrite-jwt' => $token]) - * ->withConnectionReuse(); + * $client = (new Client(new Curl()))->withHeaders(['x-appwrite-jwt' => $token]); * * $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); * ``` @@ -48,52 +36,58 @@ class Http extends Journal private readonly RequestFactory $requests; /** - * @param Adapter $client Configured with whatever credentials the - * producer requires. Typed as the client's own adapter interface - * rather than plain PSR-18, because a read needs to set its own - * deadline β€” which also means a `Retry` or `Pool` decorator can be - * passed here, since those implement it too. + * @param Adapter $client Configured with whatever credentials the producer + * requires. Typed as the client's own adapter interface rather than + * plain PSR-18, because a poll needs to set its own deadline. * - * Retries are best left off. A failed read leaves the cursor where - * it was, so the next poll is already the retry; retrying inside a - * long poll only multiplies how long a single tick can take. - * @param string $endpoint Base URL the producer serves its feeds under. - * The feed name is appended to it, so - * `https://cloud.example.com/v1/feeds` reads - * `https://cloud.example.com/v1/feeds/edge`. + * Retries are best left off: a failed read leaves the cursor where it + * was, so the next poll is already the retry. + * @param string $endpoint Base URL the producer serves its feeds under. The + * feed name is appended, so `https://cloud.example.com/v1/feeds` + * reads `https://cloud.example.com/v1/feeds/edge`. * @param string $name Feed name, as the producer knows it. */ public function __construct( protected readonly Adapter $client, protected readonly string $endpoint, string $name, - ?RequestFactory $requests = null, ) { parent::__construct($name); - $this->requests = $requests ?? new RequestFactory(); + $this->requests = new RequestFactory(); } /** - * The URL this journal reads. + * @throws Unsupported Always. A consumer cannot append to a feed it does + * not own; call the producer's own API instead. */ - public function getUrl(): string + public function append(CloudEvent $event): string { - return \rtrim($this->endpoint, '/') . '/' . \rawurlencode($this->name); + throw new Unsupported("The {$this->name} feed is read over HTTP and cannot be appended to"); + } + + public function read(?string $lastEventId, int $limit): array + { + return $this->fetch($lastEventId, $limit, 0); } /** - * @throws Unsupported Always. A consumer cannot append to a feed it does - * not own; call the producer's own API instead. + * The producer does the waiting, so a poll is one held request rather than + * a client-side loop. */ - public function append(CloudEvent $event): string + public function poll(?string $lastEventId, int $limit, int $timeout): array { - throw new Unsupported("The {$this->name} feed is read over HTTP and cannot be appended to"); + return $this->fetch($lastEventId, $limit, $timeout); } - public function read(?string $lastEventId, int $limit, int $timeout = 0): array + /** + * @return list + * @throws Transport When the producer cannot be reached, or answers with an + * error status or a body that is not JSON. + */ + private function fetch(?string $lastEventId, int $limit, int $timeout): array { - $url = $this->getUrl(); + $url = $this->url(); $request = $this->requests->query( Method::GET, @@ -115,9 +109,8 @@ public function read(?string $lastEventId, int $limit, int $timeout = 0): array if ($status >= 400) { // Carried as the exception code so a caller can act on it β€” most - // usefully to treat a 404 as "this producer does not serve the - // feed yet", which is the normal state while a feed is being - // rolled out across services and not something to alert on. + // usefully to treat a 404 as "this producer does not serve the feed + // yet", which is normal while a feed is being rolled out. throw new Transport( "Reading the {$this->name} feed at {$url} failed with status {$status}", $status, @@ -133,23 +126,16 @@ public function read(?string $lastEventId, int $limit, int $timeout = 0): array return Protocol::decode($body); } - /** - * The producer does the waiting. - */ - public function pollable(): bool + private function url(): string { - return true; + return \rtrim($this->endpoint, '/') . '/' . \rawurlencode($this->name); } /** - * The client to read with, given how long the producer has been asked to - * hold the request. - * - * A long poll needs a deadline past the one it asked for. Without the - * margin the client's deadline races the producer's, and a poll that - * correctly waits out its full timeout gets cancelled a hair early and - * surfaces as a transport failure on every quiet tick β€” burying the - * failures that matter. A plain read keeps whatever the caller configured. + * A long poll needs a deadline past the one it asked for. Without the margin + * the client's deadline races the producer's, and a poll that correctly + * waits out its full timeout surfaces as a transport failure on every quiet + * tick. A plain read keeps whatever the caller configured. */ private function client(int $timeout): Adapter { diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Journal/Memory.php index dfa66c8..145cb11 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -4,21 +4,18 @@ namespace Utopia\Feed\Journal; -use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Id; +use Utopia\Feed\Journal; /** - * A feed held in process memory. + * A feed held in process memory, for tests and single-process development. * - * For tests and for running a service on its own without a Redis. It - * implements the same id and retention semantics as {@see Redis}, so code - * written against it behaves the same when it is swapped out β€” including the - * parts that are easy to get wrong, like resuming from a trimmed position. - * - * Not for production: nothing is shared between processes and nothing survives - * a restart, so consumers in another worker see an empty feed. + * Implements the same id and retention semantics as {@see Redis}, including the + * awkward parts like resuming from a trimmed position, so code written against + * it behaves the same when it is swapped out. Not for production: nothing is + * shared between processes and nothing survives a restart. */ class Memory extends Journal { @@ -26,9 +23,9 @@ class Memory extends Journal private array $events = []; /** - * Last millisecond an event was appended in, with the sequence number - * reached within it. Tracked so several appends inside the same - * millisecond still get ordered ids, the way `XADD` does. + * Last millisecond an event was appended in, with the sequence reached + * within it, so several appends in the same millisecond still get ordered + * ids the way `XADD` does. */ private int $timestamp = 0; @@ -61,10 +58,7 @@ public function append(CloudEvent $event): string $id = Id::encode($this->timestamp, $this->sequence); // Stored through the same encode/decode a real backend goes through, - // rather than holding the object. Otherwise this journal would accept - // payloads that cannot be serialized and hand back values that survived - // a round trip they would not survive in production β€” which is the one - // way a stand-in like this actively causes harm. + // so this journal cannot accept payloads that would fail in production. $this->events[] = self::decode($id, self::encode($event)); if (\count($this->events) > $this->maxSize) { @@ -74,11 +68,10 @@ public function append(CloudEvent $event): string return $id; } - public function read(?string $lastEventId, int $limit, int $timeout = 0): array + public function read(?string $lastEventId, int $limit): array { - // Validates the position even when nothing will be returned, so a - // malformed cursor fails the same way it does on every other journal - // instead of only once the feed has events in it. + // Decoded up front so a malformed position fails the same way it does + // on every other journal, even when the feed is empty. $after = $lastEventId === null ? null : Id::decode($lastEventId); $events = []; @@ -97,23 +90,4 @@ public function read(?string $lastEventId, int $limit, int $timeout = 0): array return $events; } - - /** - * How many events are currently retained. Test affordance β€” a feed has no - * length a consumer is allowed to care about. - */ - public function count(): int - { - return \count($this->events); - } - - /** - * Drop every event, without resetting the id counter: positions already - * handed out must not be reissued, or a consumer holding one would skip - * whatever is appended next. - */ - public function flush(): void - { - $this->events = []; - } } diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Journal/Pool.php index 85302ec..0f962aa 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -4,17 +4,17 @@ namespace Utopia\Feed\Journal; -use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; +use Utopia\Feed\Journal; use Utopia\Pools\Pool as UtopiaPool; /** * {@see Redis}, over a pooled connection. * - * What most services actually want: a long poll holds its connection for the - * whole timeout, so a feed read from a shared client would block every other - * user of it. Taking a connection per operation keeps that contained. + * What most services producing a feed want: a long poll holds its connection + * for the whole timeout, so reading through a shared client would block every + * other user of it. * * @see https://github.com/utopia-php/pools */ @@ -41,10 +41,10 @@ public function append(CloudEvent $event): string return $this->pool->use(fn (\Redis|\RedisCluster $redis): string => $this->journal($redis)->append($event)); } - public function read(?string $lastEventId, int $limit, int $timeout = 0): array + public function read(?string $lastEventId, int $limit): array { return $this->pool->use( - fn (\Redis|\RedisCluster $redis): array => $this->journal($redis)->read($lastEventId, $limit, $timeout) + fn (\Redis|\RedisCluster $redis): array => $this->journal($redis)->read($lastEventId, $limit) ); } diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Journal/Redis.php index 20ad1eb..4d8a664 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -4,37 +4,29 @@ namespace Utopia\Feed\Journal; -use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; +use Utopia\Feed\Journal; /** * A feed backed by a Redis stream. * - * Streams give the two properties http-feeds needs and are awkward to build on - * anything else: `XADD` allocates ids that are ordered and unique across - * concurrent producers, and `XRANGE` pages from any of them without the - * producer tracking who has read what. Consumer groups are deliberately not - * used β€” they move the position onto the producer, which is the arrangement - * this library exists to avoid. - * - * Retention is a `MAXLEN` cap. Redis trims approximately, to whole nodes, - * so a feed holds at least $maxSize events and usually somewhat more; it is a - * bound on memory, not a promise about how far back a consumer can resume - * from. + * `XADD` allocates ids that are ordered and unique across concurrent producers, + * and `XRANGE` pages from any of them without the producer tracking who has + * read what. Consumer groups are deliberately not used β€” they move the position + * onto the producer, which is what this library exists to avoid. */ class Redis extends Journal { /** * @param \Redis|\RedisCluster $redis * @param string $name Feed name; the stream is stored at `feed:`. - * @param int $maxSize Approximate cap on retained events. The default - * holds a long weekend of a busy feed, which is the window that - * matters: a consumer down for longer than its feed's retention - * resumes from the oldest event it can, rather than from where it - * left off. + * @param int $maxSize Cap on retained events. Redis trims approximately, to + * whole nodes, so a feed holds at least this many and usually more β€” + * a bound on memory, not a promise about how far back a consumer can + * resume from. * @throws Invalid When $name is empty, or $maxSize is below one event. */ public function __construct( @@ -47,19 +39,10 @@ public function __construct( self::assertRetention($maxSize); } - /** - * The key the stream lives at. Namespaced so a feed can share a Redis with - * whatever else the service keeps there. - */ - public function getKey(): string - { - return 'feed:' . $this->name; - } - public function append(CloudEvent $event): string { try { - $id = $this->redis->xAdd($this->getKey(), '*', self::encode($event), $this->maxSize, true); + $id = $this->redis->xAdd($this->key(), '*', self::encode($event), $this->maxSize, true); } catch (\RedisException $error) { throw new Transport("Failed to append to the {$this->name} feed: {$error->getMessage()}", previous: $error); } @@ -71,12 +54,12 @@ public function append(CloudEvent $event): string return $id; } - public function read(?string $lastEventId, int $limit, int $timeout = 0): array + public function read(?string $lastEventId, int $limit): array { $start = $lastEventId === null ? '-' : Id::after($lastEventId); try { - $entries = $this->redis->xRange($this->getKey(), $start, '+', $limit); + $entries = $this->redis->xRange($this->key(), $start, '+', $limit); } catch (\RedisException $error) { throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); } @@ -98,4 +81,13 @@ public function read(?string $lastEventId, int $limit, int $timeout = 0): array return $events; } + + /** + * The key the stream lives at. Namespaced so a feed can share a Redis with + * whatever else the service keeps there. + */ + private function key(): string + { + return 'feed:' . $this->name; + } } diff --git a/src/Feed/Journal/None.php b/src/Feed/Journal/Unconfigured.php similarity index 55% rename from src/Feed/Journal/None.php rename to src/Feed/Journal/Unconfigured.php index 38eddbd..febadac 100644 --- a/src/Feed/Journal/None.php +++ b/src/Feed/Journal/Unconfigured.php @@ -4,9 +4,9 @@ namespace Utopia\Feed\Journal; -use Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Unsupported; +use Utopia\Feed\Journal; /** * No backend configured. Every operation throws. @@ -14,15 +14,14 @@ * Lets a service construct its feeds unconditionally and fail at the point of * use, instead of threading a nullable feed through every caller. * - * Deliberately not a no-op, unlike the null adapters elsewhere in Utopia. A - * feed that silently swallowed appends would leave the producer believing its - * consumers had been told, and the consequence of that only shows up much - * later somewhere else β€” a cache that never invalidates, a replica that never - * catches up. If dropping events is genuinely acceptable, use {@see Memory}. + * Deliberately not a no-op: a feed that silently swallowed appends would leave + * the producer believing its consumers had been told, and the consequence only + * shows up much later somewhere else. If dropping events is genuinely + * acceptable, use {@see Memory}. */ -class None extends Journal +class Unconfigured extends Journal { - public function __construct(string $name = 'none') + public function __construct(string $name = 'unconfigured') { parent::__construct($name); } @@ -32,7 +31,7 @@ public function append(CloudEvent $event): string throw new Unsupported("No feed backend is configured for the {$this->name} feed"); } - public function read(?string $lastEventId, int $limit, int $timeout = 0): array + public function read(?string $lastEventId, int $limit): array { throw new Unsupported("No feed backend is configured for the {$this->name} feed"); } diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index f6bbb8b..0be9a15 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -11,18 +11,18 @@ /** * The HTTP shape of a feed, as defined by https://www.http-feeds.org/. * - * A producer serving a feed and a consumer reading one have to agree on the - * query parameters, the response body and the caching rules. Both halves live - * here so they cannot drift: {@see Journal\Http} reads through it, and a - * producer builds its endpoint's response with it β€” whichever HTTP framework - * that endpoint happens to be written in, which is why this deals in arrays - * rather than in requests and responses. + * Both halves live here so a producer's endpoint and its consumers cannot + * drift: {@see Journal\Http} reads through it, and a producer builds its + * response with it β€” in whichever HTTP framework it is written in, which is why + * this deals in arrays rather than in requests and responses. * * ```php * // Producer, in whatever routing layer it uses: + * $limit = Feed::limit((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH)); + * * $events = $feed->poll( * $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, - * (int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH), + * $limit, * (int) $request->getParam(Protocol::PARAM_TIMEOUT, 0), * ); * @@ -34,11 +34,8 @@ final class Protocol { /** - * Position to read from. Omitted or empty means the oldest retained - * event β€” not the newest. Starting at the tip would silently drop whatever - * is already in the feed, and on a fresh consumer that is not a - * hypothetical backlog: nothing is recorded until the first event arrives, - * so the first event is precisely the one that would be skipped. + * Position to read from. Omitted or empty means the oldest retained event β€” + * not the newest, which would silently drop whatever is already in the feed. */ public const string PARAM_LAST_EVENT_ID = 'lastEventId'; @@ -71,22 +68,17 @@ final class Protocol /** * Extra milliseconds a consumer allows its HTTP client on top of the long - * poll timeout it asked for. - * - * Without it the client's own deadline races the server's: a poll that - * correctly waits out its full timeout gets cancelled a hair early and - * surfaces as a transport failure on every quiet tick, burying the - * failures that matter. + * poll timeout it asked for, so the client's deadline does not race the + * producer's. */ public const int TIMEOUT_MARGIN = 10_000; /** * The query string for a read. * - * Parameters at their default are left out rather than sent explicitly, so - * a consumer and a producer that disagree on a default resolve it the - * producer's way β€” and so the URL of a first read is stable enough to be - * cached and logged as one thing. + * Parameters at their default are left out, so a consumer and a producer + * that disagree on a default resolve it the producer's way β€” and so the URL + * of a first read is stable enough to be cached and logged as one thing. * * @return array */ @@ -128,30 +120,17 @@ public static function encode(array $events): array /** * Read a batch out of a response body. * - * Decoded leniently, and tolerating a `specversion` this consumer has - * never seen: a feed is read by consumers older than the producer *by - * design*, so a producer that adds an attribute, omits an optional one, or - * moves the spec forward must not stop a consumer that predates it. That - * is what makes a staged rollout safe, and it is the one place this - * library needs a reader more forgiving than a general CloudEvents one. - * - * The exception is `id`, which is enforced here and nowhere else in the - * spec's terms: for a feed the id *is* the consumer's position, so an event - * without one cannot be recorded as passed. {@see CloudEvent::validate()} - * also requires a well-formed URI-reference `source` β€” a spec requirement, - * but not one a feed consumer depends on β€” so this checks the single - * attribute it actually needs rather than calling it. + * Decoded leniently, and tolerating a `specversion` this consumer has never + * seen: a feed is read by consumers older than the producer *by design*, so + * a producer that adds an attribute or moves the spec forward must not stop + * one that predates it. Only `id` is enforced β€” for a feed the id *is* the + * consumer's position, so an event without one cannot be recorded as passed. * * Stops at the first event that cannot be decoded and returns the ones - * before it, rather than dropping it and carrying on. Skipping it would - * mean every event after it is acknowledged under a cursor that never - * advanced past the gap, and the next restart would replay them all. - * - * Returning the prefix keeps the events that *are* usable moving: the - * consumer applies them, advances to the last one, and meets the broken + * before it. The consumer applies those, advances, and meets the broken * event at the head of the next batch β€” where, with no prefix left to * salvage, this throws and the feed visibly stops instead of quietly - * losing events. + * skipping past a gap. * * @return list * @throws Invalid When the payload is not a batch, or when the very first @@ -163,13 +142,11 @@ public static function decode(mixed $payload): array throw new Invalid('Expected a feed batch, got ' . \get_debug_type($payload)); } - // Required, not defaulted to empty. An empty batch and a response that - // is not a batch at all are the same bytes to a consumer that defaults - // it β€” and they mean opposite things: "you are caught up" versus "you - // did not reach the feed". A misrouted request, a proxy's JSON error - // page or an endpoint that moved would all read as a quiet, permanent - // caught-up state, which is the one failure a feed cannot afford to - // hide. + // Required, not defaulted to empty: an empty batch and a response that + // is not a batch at all mean opposite things β€” "you are caught up" + // versus "you did not reach the feed" β€” and a misrouted request or a + // proxy's error page must not read as a quiet, permanent caught-up + // state. if (!\array_key_exists(self::KEY_EVENTS, $payload)) { throw new Invalid('Feed batch is missing the "' . self::KEY_EVENTS . '" field'); } @@ -203,6 +180,26 @@ public static function decode(mixed $payload): array return $events; } + /** + * What a producer should send as `Cache-Control` for a batch. + * + * @param int $count Events being returned. + * @param int $limit Events the batch was built with, after {@see + * Feed::limit()} has clamped what the consumer asked for. + * @param bool $public Whether a shared cache may store the batch. Off by + * default: feeds are usually served behind authorization, and + * `public` there would let a CDN hand one consumer's events to a + * requester that never presented a credential. + */ + public static function cacheControl(int $count, int $limit, bool $public = false): string + { + if ($count < $limit || $count === 0) { + return self::CACHE_NONE; + } + + return ($public ? 'public, ' : 'private, ') . self::CACHE_IMMUTABLE; + } + /** * Decode one event, enforcing the only attribute a feed cannot do without. * @@ -220,24 +217,4 @@ private static function event(array $raw): CloudEvent return $event; } - - /** - * What a producer should send as `Cache-Control` for a batch. - * - * @param int $count Events being returned. - * @param int $limit Events that were asked for. - * @param bool $public Whether a shared cache may store the batch. Off by - * default: feeds are usually served behind authorization, and - * `public` there would let a CDN hand one consumer's events to a - * requester that never presented a credential. Only turn it on for - * a feed whose events are safe for anyone who can reach the URL. - */ - public static function cacheControl(int $count, int $limit, bool $public = false): string - { - if ($count < $limit || $count === 0) { - return self::CACHE_NONE; - } - - return ($public ? 'public, ' : 'private, ') . self::CACHE_IMMUTABLE; - } } diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index 0ce9fc2..e9ae175 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -88,7 +88,7 @@ public function testIdsIncreaseAcrossRapidAppends(): void $this->assertSame($ids, \array_unique($ids)); for ($i = 1; $i < \count($ids); $i++) { - $this->assertSame(1, Id::compare($ids[$i], $ids[$i - 1])); + $this->assertGreaterThan(Id::decode($ids[$i - 1]), Id::decode($ids[$i])); } } @@ -217,7 +217,7 @@ public function testLongPollingGivesUpAtTheTimeout(): void public function testConsumesThroughAPersistedCursor(): void { $feed = $this->feed(); - $cursor = new RedisCursor($this->redis, $this->name); + $cursor = new RedisCursor($this->redis); $feed->append('a'); $last = $feed->append('b'); @@ -228,7 +228,7 @@ public function testConsumesThroughAPersistedCursor(): void }; $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume($handler)); - $this->assertSame($last, $cursor->load('invalidator')); + $this->assertSame($last, $cursor->load($this->name, 'invalidator')); // A second Consumer stands in for a restart: it has no in-memory // position, so it has to pick the stored one up to avoid replaying. @@ -239,7 +239,7 @@ public function testConsumesThroughAPersistedCursor(): void public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void { $feed = $this->feed(); - $cursor = new RedisCursor($this->redis, $this->name); + $cursor = new RedisCursor($this->redis); $feed->append('a'); @@ -250,7 +250,7 @@ public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void public function testResetReplaysTheRetainedFeed(): void { $feed = $this->feed(); - $cursor = new RedisCursor($this->redis, $this->name); + $cursor = new RedisCursor($this->redis); $feed->append('a'); $feed->append('b'); @@ -259,13 +259,13 @@ public function testResetReplaysTheRetainedFeed(): void $consumer->consume(fn (CloudEvent $e) => null); $consumer->reset(); - $this->assertNull($cursor->load('invalidator')); + $this->assertNull($cursor->load($this->name, 'invalidator')); $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume(fn (CloudEvent $e) => null)); } public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void { - (new RedisCursor($this->redis, $this->name))->save('invalidator', '1-0'); + (new RedisCursor($this->redis))->save($this->name, 'invalidator', '1-0'); $this->assertSame('1-0', $this->redis->get('feed:' . $this->name . ':cursor:invalidator')); } diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 771bc30..e9ec3b4 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -26,7 +26,7 @@ protected function setUp(): void { $this->journal = new MemoryJournal('edge'); $this->feed = new Feed($this->journal, 'urn:test'); - $this->cursor = new MemoryCursor('edge'); + $this->cursor = new MemoryCursor(); } private function consumer(?Cursor $cursor = null, int $batch = Consumer::BATCH): Consumer @@ -57,7 +57,7 @@ public function testHandlesEachEventAndAdvancesPastTheLastOne(): void $this->assertSame(['a', 'b'], $this->drain($consumer, $count)); $this->assertSame(2, $count); - $this->assertSame($last, $this->cursor->load('invalidator')); + $this->assertSame($last, $this->cursor->load('edge', 'invalidator')); $this->assertSame($last, $consumer->position()); } @@ -76,7 +76,7 @@ public function testResumesFromTheStoredPosition(): void $first = $this->feed->append('a'); $this->feed->append('b'); - $this->cursor->save('invalidator', $first); + $this->cursor->save('edge', 'invalidator', $first); $this->assertSame(['b'], $this->drain($this->consumer())); } @@ -98,14 +98,14 @@ public function testReadsTheStoreOnceAndThenTracksThePositionInMemory(): void { $this->feed->append('a'); - $cursor = new class ('edge') extends MemoryCursor { + $cursor = new class () extends MemoryCursor { public int $loads = 0; - public function load(string $consumer): ?string + public function load(string $feed, string $consumer): ?string { $this->loads++; - return parent::load($consumer); + return parent::load($feed, $consumer); } }; @@ -144,7 +144,7 @@ public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void } $this->assertSame(['a'], $seen); - $this->assertSame($first, $this->cursor->load('invalidator'), 'Progress before the failure is committed'); + $this->assertSame($first, $this->cursor->load('edge', 'invalidator'), 'Progress before the failure is committed'); } public function testRetriesTheFailedEventOnTheNextRun(): void @@ -185,7 +185,7 @@ public function testAFailureOnTheFirstEventCommitsNothing(): void // Expected. } - $this->assertNull($this->cursor->load('invalidator')); + $this->assertNull($this->cursor->load('edge', 'invalidator')); } public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void @@ -220,7 +220,7 @@ public function testKeepsWorkingWhenThePositionCannotBeLoaded(): void { $this->feed->append('a'); - $consumer = $this->consumer(new FailingCursor('edge', onLoad: true)); + $consumer = $this->consumer(new FailingCursor(onLoad: true)); $warnings = []; $consumer->onWarning(function (\Throwable $error, string $context) use (&$warnings): void { $warnings[] = $context; @@ -235,7 +235,7 @@ public function testKeepsWorkingWhenThePositionCannotBeSaved(): void $this->feed->append('a'); $this->feed->append('b'); - $consumer = $this->consumer(new FailingCursor('edge', onSave: true)); + $consumer = $this->consumer(new FailingCursor(onSave: true)); $warnings = []; $consumer->onWarning(function (\Throwable $error, string $context) use (&$warnings): void { $warnings[] = $context; @@ -253,7 +253,7 @@ public function testSurvivesAFailingStoreWithNoWarningHandler(): void { $this->feed->append('a'); - $this->assertSame(['a'], $this->drain($this->consumer(new FailingCursor('edge', onLoad: true, onSave: true)))); + $this->assertSame(['a'], $this->drain($this->consumer(new FailingCursor(onLoad: true, onSave: true)))); } public function testResetReplaysEverythingStillRetained(): void @@ -267,7 +267,7 @@ public function testResetReplaysEverythingStillRetained(): void $consumer->reset(); $this->assertNull($consumer->position()); - $this->assertNull($this->cursor->load('invalidator')); + $this->assertNull($this->cursor->load('edge', 'invalidator')); $this->assertSame(['a', 'b'], $this->drain($consumer)); } @@ -295,12 +295,9 @@ public function testRejectsAnEmptyConsumerName(): void new Consumer($this->feed, '', $this->cursor); } - public function testExposesWhatItIsConsuming(): void + public function testExposesItsName(): void { - $consumer = $this->consumer(); - - $this->assertSame('invalidator', $consumer->getName()); - $this->assertSame($this->feed, $consumer->getFeed()); + $this->assertSame('invalidator', $this->consumer()->getName()); } /** @@ -311,16 +308,16 @@ public function testAFailedReadLeavesThePositionAlone(): void { $first = $this->feed->append('a'); $this->feed->append('b'); - $this->cursor->save('invalidator', $first); + $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer(new Feed(new \Utopia\Feed\Journal\None('edge')), 'invalidator', $this->cursor); + $consumer = new Consumer(new Feed(new \Utopia\Feed\Journal\Unconfigured('edge')), 'invalidator', $this->cursor); $this->expectException(\Utopia\Feed\Exception\Unsupported::class); try { $consumer->consume(fn (CloudEvent $event) => null); } finally { - $this->assertSame($first, $this->cursor->load('invalidator')); + $this->assertSame($first, $this->cursor->load('edge', 'invalidator')); } } } diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index 723b294..77e7e2d 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -20,8 +20,8 @@ class CursorTest extends TestCase public static function stores(): array { return [ - 'memory' => [new Memory('edge')], - 'cache' => [new Cache(new UtopiaCache(new CacheMemory()), 'edge')], + 'memory' => [new Memory()], + 'cache' => [new Cache(new UtopiaCache(new CacheMemory()))], ]; } @@ -30,7 +30,7 @@ public static function stores(): array */ public function testAnUnknownConsumerHasNoPosition(Cursor $cursor): void { - $this->assertNull($cursor->load('never-run')); + $this->assertNull($cursor->load('edge', 'never-run')); } /** @@ -38,9 +38,9 @@ public function testAnUnknownConsumerHasNoPosition(Cursor $cursor): void */ public function testRoundTripsAPosition(Cursor $cursor): void { - $cursor->save('invalidator', '1690000000000-0'); + $cursor->save('edge', 'invalidator', '1690000000000-0'); - $this->assertSame('1690000000000-0', $cursor->load('invalidator')); + $this->assertSame('1690000000000-0', $cursor->load('edge', 'invalidator')); } /** @@ -48,10 +48,10 @@ public function testRoundTripsAPosition(Cursor $cursor): void */ public function testOverwritesAPosition(Cursor $cursor): void { - $cursor->save('invalidator', '1-0'); - $cursor->save('invalidator', '2-0'); + $cursor->save('edge', 'invalidator', '1-0'); + $cursor->save('edge', 'invalidator', '2-0'); - $this->assertSame('2-0', $cursor->load('invalidator')); + $this->assertSame('2-0', $cursor->load('edge', 'invalidator')); } /** @@ -59,73 +59,65 @@ public function testOverwritesAPosition(Cursor $cursor): void */ public function testConsumersDoNotShareAPosition(Cursor $cursor): void { - $cursor->save('one', '1-0'); - $cursor->save('two', '2-0'); + $cursor->save('edge', 'one', '1-0'); + $cursor->save('edge', 'two', '2-0'); - $this->assertSame('1-0', $cursor->load('one')); - $this->assertSame('2-0', $cursor->load('two')); + $this->assertSame('1-0', $cursor->load('edge', 'one')); + $this->assertSame('2-0', $cursor->load('edge', 'two')); } /** + * One store serves every feed a service consumes, which is why the feed + * name is part of the key rather than of the cursor. + * * @dataProvider stores */ - public function testResetForgetsAPosition(Cursor $cursor): void + public function testFeedsDoNotShareAPosition(Cursor $cursor): void { - $cursor->save('invalidator', '1-0'); - $cursor->reset('invalidator'); + $cursor->save('edge', 'invalidator', '1-0'); - $this->assertNull($cursor->load('invalidator')); + $this->assertNull($cursor->load('other', 'invalidator')); } /** * @dataProvider stores */ - public function testSavingAnEmptyPositionIsIgnored(Cursor $cursor): void + public function testResetForgetsAPosition(Cursor $cursor): void { - $cursor->save('invalidator', '1-0'); - $cursor->save('invalidator', ''); + $cursor->save('edge', 'invalidator', '1-0'); + $cursor->reset('edge', 'invalidator'); - $this->assertSame('1-0', $cursor->load('invalidator'), 'An empty position must not erase a real one'); + $this->assertNull($cursor->load('edge', 'invalidator')); } /** * @dataProvider stores */ - public function testRejectsAnEmptyConsumerName(Cursor $cursor): void + public function testSavingAnEmptyPositionIsIgnored(Cursor $cursor): void { - $this->expectException(Invalid::class); + $cursor->save('edge', 'invalidator', '1-0'); + $cursor->save('edge', 'invalidator', ''); - $cursor->load(''); + $this->assertSame('1-0', $cursor->load('edge', 'invalidator'), 'An empty position must not erase a real one'); } /** * @dataProvider stores */ - public function testRejectsAnEmptyConsumerNameOnSave(Cursor $cursor): void + public function testRejectsAnEmptyConsumerName(Cursor $cursor): void { $this->expectException(Invalid::class); - $cursor->save('', '1-0'); - } - - public function testFeedsDoNotShareAPosition(): void - { - $cache = new UtopiaCache(new CacheMemory()); - - (new Cache($cache, 'edge'))->save('invalidator', '1-0'); - - $this->assertNull((new Cache($cache, 'other'))->load('invalidator')); + $cursor->load('edge', ''); } - public function testRejectsAnEmptyFeedName(): void + /** + * @dataProvider stores + */ + public function testRejectsAnEmptyFeedName(Cursor $cursor): void { $this->expectException(Invalid::class); - new Memory(''); - } - - public function testExposesTheFeedItTracks(): void - { - $this->assertSame('edge', (new Memory('edge'))->getFeed()); + $cursor->save('', 'invalidator', '1-0'); } } diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 2c9b901..52bbc85 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -6,7 +6,7 @@ use PHPUnit\Framework\TestCase; use Utopia\Feed\Journal\Memory; -use Utopia\Feed\Journal\None; +use Utopia\Feed\Journal\Unconfigured; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; @@ -88,7 +88,7 @@ public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); for ($i = 1; $i < \count($ids); $i++) { - $this->assertSame(1, Id::compare($ids[$i], $ids[$i - 1]), 'Positions must increase'); + $this->assertGreaterThan(Id::decode($ids[$i - 1]), Id::decode($ids[$i]), 'Positions must increase'); } } @@ -142,6 +142,19 @@ public function testClampsTheLimitToTheMaximum(): void $this->assertCount(1, $this->feed->read(null, -5)); } + /** + * An endpoint serving this feed clamps with the same helper, so the limit + * it passes to `Protocol::cacheControl()` is the one the batch was built + * with. + */ + public function testExposesTheLimitAReadWillActuallyUse(): void + { + $this->assertSame(50, Feed::limit(50)); + $this->assertSame(Feed::MAX_BATCH, Feed::limit(Feed::MAX_BATCH * 10)); + $this->assertSame(1, Feed::limit(0)); + $this->assertSame(1, Feed::limit(-5)); + } + public function testRejectsAPositionThatIsNotAFeedId(): void { $this->expectException(Invalid::class); @@ -149,6 +162,20 @@ public function testRejectsAPositionThatIsNotAFeedId(): void $this->feed->read('not-a-position'); } + /** + * CloudEvents requires a source, and a feed that stamped an empty one would + * produce events no consumer can attribute β€” so it is refused rather than + * appended. + */ + public function testRejectsAnAppendToAFeedWithNoSource(): void + { + $feed = new Feed(new Memory('edge')); + + $this->expectException(Invalid::class); + + $feed->append('test'); + } + public function testRejectsAnEmptyEventType(): void { $this->expectException(Invalid::class); @@ -314,7 +341,7 @@ public function testPollWithoutATimeoutIsAPlainRead(): void public function testRetentionIsBoundedAndTrimsTheOldest(): void { - $feed = new Feed(new Memory('small', maxSize: 3)); + $feed = new Feed(new Memory('small', maxSize: 3), 'urn:appwrite:cloud:fra'); foreach (['a', 'b', 'c', 'd', 'e'] as $type) { $feed->append($type); @@ -329,26 +356,23 @@ public function testRetentionIsBoundedAndTrimsTheOldest(): void */ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void { - $feed = new Feed($journal = new Memory('small', maxSize: 2)); + $feed = new Feed(new Memory('small', maxSize: 2), 'urn:appwrite:cloud:fra'); $first = $feed->append('a'); $feed->append('b'); $feed->append('c'); - $this->assertSame(2, $journal->count()); $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read($first))); } - public function testExposesItsIdentity(): void + public function testExposesTheFeedItReads(): void { $this->assertSame('edge', $this->feed->getName()); - $this->assertSame('urn:appwrite:cloud:fra', $this->feed->getSource()); - $this->assertSame($this->journal, $this->feed->getJournal()); } public function testAnUnconfiguredBackendFailsLoudlyRatherThanDroppingEvents(): void { - $feed = new Feed(new None('edge')); + $feed = new Feed(new Unconfigured('edge'), 'urn:appwrite:cloud:fra'); $this->expectException(Unsupported::class); @@ -357,7 +381,7 @@ public function testAnUnconfiguredBackendFailsLoudlyRatherThanDroppingEvents(): public function testAnUnconfiguredBackendCannotBeRead(): void { - $feed = new Feed(new None('edge')); + $feed = new Feed(new Unconfigured('edge')); $this->expectException(Unsupported::class); @@ -398,7 +422,7 @@ public function testRejectsARetentionCapThatWouldNotBoundTheFeed(int $maxSize): public function testAcceptsTheSmallestUsefulRetentionCap(): void { - $feed = new Feed(new Memory('edge', maxSize: 1)); + $feed = new Feed(new Memory('edge', maxSize: 1), 'urn:appwrite:cloud:fra'); $feed->append('a'); $feed->append('b'); @@ -408,13 +432,4 @@ public function testAcceptsTheSmallestUsefulRetentionCap(): void $this->assertCount(1, $events); $this->assertSame('b', $events[0]->type); } - - public function testFlushingMemoryDoesNotReissuePositions(): void - { - $before = $this->feed->append('a'); - $this->journal->flush(); - $after = $this->feed->append('b'); - - $this->assertSame(1, Id::compare($after, $before), 'A reissued position would make a consumer skip events'); - } } diff --git a/tests/Feed/Unit/HttpJournalTest.php b/tests/Feed/Unit/HttpJournalTest.php index 405a0f3..c243062 100644 --- a/tests/Feed/Unit/HttpJournalTest.php +++ b/tests/Feed/Unit/HttpJournalTest.php @@ -57,9 +57,14 @@ public function testAppendsTheFeedNameToTheEndpoint(): void public function testEncodesAFeedNameThatNeedsIt(): void { - $journal = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds/', 'a b/c'); + $transport = FakeTransport::of([]); - $this->assertSame('https://cloud.example.com/v1/feeds/a%20b%2Fc', $journal->getUrl()); + (new Feed(new Http($transport, 'https://cloud.example.com/v1/feeds/', 'a b/c')))->read(); + + $this->assertStringStartsWith( + 'https://cloud.example.com/v1/feeds/a%20b%2Fc', + $transport->recorder->last()['uri'], + ); } public function testReadsWithGet(): void @@ -216,7 +221,8 @@ public function testRejectsABodyThatIsNotABatch(): void public function testCannotAppendToAFeedItDoesNotOwn(): void { - [$feed] = $this->feed(); + $journal = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', 'edge'); + $feed = new Feed($journal, 'urn:appwrite:edge:fra'); $this->expectException(Unsupported::class); @@ -256,7 +262,7 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void FakeTransport::json(Protocol::encode([])), ]); - $cursor = new MemoryCursor('edge'); + $cursor = new MemoryCursor(); $consumer = new Consumer($feed, 'invalidator', $cursor); $seen = []; @@ -265,7 +271,7 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void }; $this->assertSame(2, $consumer->consume($handler)); - $this->assertSame('1-1', $cursor->load('invalidator')); + $this->assertSame('1-1', $cursor->load('edge', 'invalidator')); $this->assertSame(1, $consumer->consume($handler)); $this->assertSame(0, $consumer->consume($handler)); diff --git a/tests/Feed/Unit/IdTest.php b/tests/Feed/Unit/IdTest.php index fa69b56..c307a2d 100644 --- a/tests/Feed/Unit/IdTest.php +++ b/tests/Feed/Unit/IdTest.php @@ -61,22 +61,21 @@ public function testEncodeAndDecodeAreInverses(): void } /** - * The reason ids are compared by parts rather than as strings: `10-0` - * sorts before `9-0` lexically, which would make a consumer treat a newer - * event as one it had already passed. + * The reason positions are compared as decoded parts rather than as + * strings: `10-0` sorts before `9-0` lexically, which would make a consumer + * treat a newer event as one it had already passed. */ - public function testComparesNumericallyNotLexically(): void + public function testDecodedIdsCompareInFeedOrderNotLexically(): void { - $this->assertSame(1, Id::compare('10-0', '9-0')); - $this->assertSame(-1, Id::compare('9-0', '10-0')); - $this->assertSame(0, Id::compare('10-0', '10-0')); - $this->assertSame(1, Id::compare('10-2', '10-1')); + $this->assertGreaterThan(Id::decode('9-0'), Id::decode('10-0')); + $this->assertGreaterThan(Id::decode('10-1'), Id::decode('10-2')); + $this->assertSame(Id::decode('10-0'), Id::decode('10-0')); } - public function testCompareRejectsAnIdThatIsNotAPosition(): void + public function testDecodeRejectsAnIdThatIsNotAPosition(): void { $this->expectException(Invalid::class); - Id::compare('1-0', 'nope'); + Id::decode('nope'); } } diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index 6a639ee..6a61cae 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -41,7 +41,7 @@ protected function setUp(): void new Http($this->server, 'https://cloud.example.com/v1/feeds', 'edge') ); - $this->cursor = new CacheCursor(new UtopiaCache(new CacheMemory()), 'edge'); + $this->cursor = new CacheCursor(new UtopiaCache(new CacheMemory())); } private function consumer(string $name = 'invalidator', int $batch = Consumer::BATCH): Consumer diff --git a/tests/Feed/Unit/Support/FailingCursor.php b/tests/Feed/Unit/Support/FailingCursor.php index 7a3f450..f11aacc 100644 --- a/tests/Feed/Unit/Support/FailingCursor.php +++ b/tests/Feed/Unit/Support/FailingCursor.php @@ -13,28 +13,26 @@ class FailingCursor extends Memory { public function __construct( - string $feed, private readonly bool $onLoad = false, private readonly bool $onSave = false, ) { - parent::__construct($feed); } - public function load(string $consumer): ?string + public function load(string $feed, string $consumer): ?string { if ($this->onLoad) { throw new Transport('Cursor store is unavailable'); } - return parent::load($consumer); + return parent::load($feed, $consumer); } - public function save(string $consumer, string $eventId): void + public function save(string $feed, string $consumer, string $eventId): void { if ($this->onSave) { throw new Transport('Cursor store is unavailable'); } - parent::save($consumer, $eventId); + parent::save($feed, $consumer, $eventId); } } diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php index 1098d41..da303a3 100644 --- a/tests/Feed/Unit/Support/FeedServer.php +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -33,7 +33,7 @@ protected function respond(RequestInterface $request): ResponseInterface \parse_str($request->getUri()->getQuery(), $query); $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; - $limit = (int) ($query[Protocol::PARAM_LIMIT] ?? Feed::MAX_BATCH); + $limit = Feed::limit((int) ($query[Protocol::PARAM_LIMIT] ?? Feed::MAX_BATCH)); $timeout = (int) ($query[Protocol::PARAM_TIMEOUT] ?? 0); $events = $this->feed->poll( From 1e3f88bc7efa8767606b111a1e22dc7b93657d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 18:07:46 +0200 Subject: [PATCH 16/68] Manual simplification --- pint.json | 5 ++++- src/Feed/Cursor.php | 29 ++--------------------------- src/Feed/Cursor/Cache.php | 20 +------------------- src/Feed/Cursor/Memory.php | 10 ---------- src/Feed/Cursor/Pool.php | 8 -------- src/Feed/Cursor/Redis.php | 15 --------------- src/Feed/Exception.php | 6 +----- src/Feed/Exception/Invalid.php | 8 +------- src/Feed/Exception/Transport.php | 11 +---------- src/Feed/Exception/Unsupported.php | 8 +------- tests/Feed/Unit/CursorTest.php | 11 ----------- 11 files changed, 11 insertions(+), 120 deletions(-) diff --git a/pint.json b/pint.json index ea5e72c..8b8058b 100644 --- a/pint.json +++ b/pint.json @@ -1,3 +1,6 @@ { - "preset": "psr12" + "preset": "psr12", + "rules": { + "declare_strict_types": true + } } diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 682e449..c5ddb4c 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -7,52 +7,27 @@ use Utopia\Feed\Exception\Invalid; /** - * Where a consumer's position in a feed is kept. - * - * http-feeds puts the position on the consumer rather than the producer, which - * is what makes adding a consumer free. A cursor is somewhere to write a - * string, keyed by feed and consumer name, so one store serves every feed a - * service consumes. - * - * The store is allowed to be lossy. A lost position is not a lost event β€” a - * consumer with no position resumes from the oldest retained event β€” so the - * cost is redundant work, not a gap. + * Client class: Remember position of feed that has already been consumed. */ abstract class Cursor { /** - * The last position $consumer recorded on $feed, or null if it has never - * recorded one β€” which means "start from the oldest retained event", never - * "start from now". - * * @throws Exception When the store cannot be read. */ abstract public function load(string $feed, string $consumer): ?string; /** - * Record a position. - * - * Only ever call this once the events up to $eventId have been handled. A - * position saved ahead of the work it stands for turns a crash into - * silently skipped events, which is the one failure this design cannot - * recover from. - * - * An empty $eventId is ignored rather than rejected: it means "nothing - * handled yet", and must not erase a real position. - * * @throws Exception When the store cannot be written. */ abstract public function save(string $feed, string $consumer, string $eventId): void; /** - * Forget a position, so the consumer's next read starts from the oldest - * retained event. - * * @throws Exception When the store cannot be written. */ abstract public function reset(string $feed, string $consumer): void; /** + * Helper tool for consistency * @throws Invalid When either name is empty. */ protected function key(string $feed, string $consumer): string diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php index 9e757e1..98a9e30 100644 --- a/src/Feed/Cursor/Cache.php +++ b/src/Feed/Cursor/Cache.php @@ -7,23 +7,9 @@ use Utopia\Cache\Cache as UtopiaCache; use Utopia\Feed\Cursor; -/** - * Positions kept in a Utopia cache β€” the usual choice for a consumer reading a - * remote feed, since it already has a cache for its own work. - * - * Note that a Utopia cache lowercases keys unless `setCaseSensitivity(true)` - * was called on it, so consumer names that differ only in case share a position. - * - * @see https://github.com/utopia-php/cache - */ class Cache extends Cursor { - /** - * How long a position survives without being written. The clock runs from - * the last save, so this is really "how long a consumer may go without - * handling anything". - */ - public const int TTL = 30 * 24 * 60 * 60; + public const int TTL = 30 * 24 * 60 * 60; // 30 days public function __construct( protected readonly UtopiaCache $cache, @@ -41,10 +27,6 @@ public function load(string $feed, string $consumer): ?string public function save(string $feed, string $consumer, string $eventId): void { - if ($eventId === '') { - return; - } - $this->cache->save($this->key($feed, $consumer), $eventId); } diff --git a/src/Feed/Cursor/Memory.php b/src/Feed/Cursor/Memory.php index 4ac2ab9..5557a58 100644 --- a/src/Feed/Cursor/Memory.php +++ b/src/Feed/Cursor/Memory.php @@ -6,12 +6,6 @@ use Utopia\Feed\Cursor; -/** - * Positions held in process memory. - * - * For tests, and for a consumer that genuinely wants to start from the - * beginning of the retained feed on every restart. - */ class Memory extends Cursor { /** @var array */ @@ -24,10 +18,6 @@ public function load(string $feed, string $consumer): ?string public function save(string $feed, string $consumer, string $eventId): void { - if ($eventId === '') { - return; - } - $this->cursors[$this->key($feed, $consumer)] = $eventId; } diff --git a/src/Feed/Cursor/Pool.php b/src/Feed/Cursor/Pool.php index faf6e3e..363c830 100644 --- a/src/Feed/Cursor/Pool.php +++ b/src/Feed/Cursor/Pool.php @@ -7,14 +7,6 @@ use Utopia\Feed\Cursor; use Utopia\Pools\Pool as UtopiaPool; -/** - * {@see Redis}, over a pooled connection. - * - * Pairs with {@see \Utopia\Feed\Journal\Pool}, and can share its pool: a cursor - * read is one `GET`, so it borrows a connection only for as long as that takes. - * - * @see https://github.com/utopia-php/pools - */ class Pool extends Cursor { /** diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 3c88517..07bf096 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -7,14 +7,6 @@ use Utopia\Feed\Cursor; use Utopia\Feed\Exception\Transport; -/** - * Positions kept in Redis, as plain keys alongside the stream. - * - * For consumers running inside the producer, which have no store of their own. - * Consumers reached over HTTP should not use this: keeping their positions in - * the producer's Redis puts per-consumer state back on the producer, which is - * exactly what the feed is arranged to avoid. - */ class Redis extends Cursor { /** @@ -38,14 +30,7 @@ public function load(string $feed, string $consumer): ?string public function save(string $feed, string $consumer, string $eventId): void { - if ($eventId === '') { - return; - } - try { - // Deliberately no expiry. Unlike a cache, this is the only copy, - // and a position that quietly expired would replay the whole - // retained feed the next time the consumer restarted. $this->redis->set($this->key($feed, $consumer), $eventId); } catch (\RedisException $error) { throw new Transport("Failed to save the {$consumer} cursor: {$error->getMessage()}", previous: $error); diff --git a/src/Feed/Exception.php b/src/Feed/Exception.php index 0a69927..31e053f 100644 --- a/src/Feed/Exception.php +++ b/src/Feed/Exception.php @@ -4,11 +4,7 @@ namespace Utopia\Feed; -/** - * Base class for every error this library raises, so a caller can catch all of - * them without also catching unrelated failures from the backend it happens to - * be sitting on. - */ +// Base class for every error this library raises class Exception extends \Exception { } diff --git a/src/Feed/Exception/Invalid.php b/src/Feed/Exception/Invalid.php index 5f6137f..784525e 100644 --- a/src/Feed/Exception/Invalid.php +++ b/src/Feed/Exception/Invalid.php @@ -6,13 +6,7 @@ use Utopia\Feed\Exception; -/** - * Something handed to the library cannot be used: an event with no id, an - * event id that is not a feed position, an empty feed or consumer name. - * - * Always a bug in the caller or in whatever produced the event, never a - * transient condition β€” retrying the same input will fail the same way. - */ +// Something handed to the library cannot be used. Always a bug in the caller class Invalid extends Exception { } diff --git a/src/Feed/Exception/Transport.php b/src/Feed/Exception/Transport.php index aa3634a..a12026e 100644 --- a/src/Feed/Exception/Transport.php +++ b/src/Feed/Exception/Transport.php @@ -6,16 +6,7 @@ use Utopia\Feed\Exception; -/** - * The backend could not be reached, or rejected the operation. - * - * Usually transient, so a consumer should leave its cursor where it is and try - * again rather than skipping past the events it failed to read. - * - * For feeds read over HTTP the code is the response status, which is how a - * caller distinguishes a producer that does not serve the feed yet (404, - * expected while a rollout is in progress) from one that is broken. - */ +// The journal (backend) could not be reached, or rejected the operation. class Transport extends Exception { } diff --git a/src/Feed/Exception/Unsupported.php b/src/Feed/Exception/Unsupported.php index 969c70e..5a160e7 100644 --- a/src/Feed/Exception/Unsupported.php +++ b/src/Feed/Exception/Unsupported.php @@ -6,13 +6,7 @@ use Utopia\Feed\Exception; -/** - * The journal cannot do what was asked of it β€” appending to a feed read over - * HTTP, or any operation at all on {@see \Utopia\Feed\Journal\Unconfigured}. - * - * Thrown rather than ignored: an append that silently does nothing loses - * events, and a consumer cannot tell an empty feed from an absent one. - */ +// The journal (backend) cannot do what was asked of it class Unsupported extends Exception { } diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index 77e7e2d..ba7daed 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -90,17 +90,6 @@ public function testResetForgetsAPosition(Cursor $cursor): void $this->assertNull($cursor->load('edge', 'invalidator')); } - /** - * @dataProvider stores - */ - public function testSavingAnEmptyPositionIsIgnored(Cursor $cursor): void - { - $cursor->save('edge', 'invalidator', '1-0'); - $cursor->save('edge', 'invalidator', ''); - - $this->assertSame('1-0', $cursor->load('edge', 'invalidator'), 'An empty position must not erase a real one'); - } - /** * @dataProvider stores */ From 3a1529f7de40617210231e526a70e7081ae6462d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 18:19:35 +0200 Subject: [PATCH 17/68] More manual simplification --- CHANGELOG.md | 5 +- README.md | 28 ++++-- src/Feed/Consumer.php | 125 +------------------------ src/Feed/Cursor.php | 4 +- src/Feed/Cursor/None.php | 38 ++++++++ src/Feed/Feed.php | 110 +--------------------- src/Feed/Id.php | 33 +------ src/Feed/Journal.php | 99 ++------------------ src/Feed/Journal/Http.php | 67 ++----------- src/Feed/Journal/Memory.php | 27 ------ src/Feed/Journal/None.php | 27 ++++++ src/Feed/Journal/Pool.php | 31 +----- src/Feed/Journal/Redis.php | 33 +------ src/Feed/Journal/Unconfigured.php | 38 -------- src/Feed/Protocol.php | 4 +- tests/Feed/Unit/ConsumerTest.php | 87 ++++++++++++----- tests/Feed/Unit/CursorTest.php | 43 +++++++++ tests/Feed/Unit/FeedTest.php | 48 +--------- tests/Feed/Unit/Support/FeedServer.php | 2 +- 19 files changed, 234 insertions(+), 615 deletions(-) create mode 100644 src/Feed/Cursor/None.php create mode 100644 src/Feed/Journal/None.php delete mode 100644 src/Feed/Journal/Unconfigured.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 2288da4..0b2db25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,10 +13,9 @@ Initial release. - `Journal\Http` β€” another service's feed, read over the wire with [utopia-php/client](https://github.com/utopia-php/client); takes any of its adapters, so a pooled or Swoole coroutine transport drops straight in -- `Journal\Memory`, `Journal\Unconfigured` β€” for tests, and for no backend - configured +- `Journal\Memory`, `Journal\None` β€” for tests, and for no backend configured - `Consumer` β€” the pull loop, with at-least-once semantics and a durable position -- `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory` β€” where that +- `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory`, `Cursor\None` β€” where that position lives, keyed by feed and consumer name - `Protocol` β€” the http-feeds wire contract, shared by producer and consumer, and the one place the feed's decode policy lives: strict about `id` because it is diff --git a/README.md b/README.md index 0b0553f..ca35bb2 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ use Utopia\Feed\Feed; use Utopia\Feed\Protocol; // GET /v1/feeds/:feedId -$limit = Feed::limit((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH)); +$limit = \min((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH), Feed::MAX_BATCH); $events = $feed->poll( $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, @@ -141,8 +141,9 @@ $response ->json(Protocol::encode($events)); ``` -`Feed::limit()` clamps what the consumer asked for to what a read will actually -return, so the same number reaches `cacheControl()`. A full batch is settled +Cap the limit yourself with `Feed::MAX_BATCH` before the call, so the number +that reaches `cacheControl()` is the one the batch was actually built with β€” a +read never returns more than that cap anyway. A full batch is settled history and is marked cacheable; a short one is the live end of the feed and is marked `no-store`. Caching is `private` unless you pass `public: true`. @@ -178,12 +179,18 @@ returns the events after a given id; everything else sits above it. | `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | βœ… | | `Journal\Http` | Consuming another service's feed | ❌ | βœ… | | `Journal\Memory` | Tests and single-process development | βœ… | βœ… | -| `Journal\Unconfigured` | No backend configured β€” throws on use | ❌ | ❌ | +| `Journal\None` | No backend configured β€” throws on use | ❌ | ❌ | `Journal\Pool` is what most services producing a feed want: a long poll holds its connection for the whole timeout, so reading through a shared client would block every other user of it. +`Journal\None` throws on every operation rather than doing nothing, so a +misconfigured service fails at the point of use instead of silently dropping +events. `Cursor\None` is the opposite β€” it is a no-op, because a position that +goes nowhere only costs a replay, while an append that goes nowhere loses +events. + ## Cursors A cursor is where one consumer keeps its position. It is keyed by feed and @@ -195,14 +202,19 @@ consumer name, so a single store serves every feed a service consumes: | `Cursor\Redis` | A consumer running inside the producer, with no store of its own | | `Cursor\Pool` | The same, over a pooled connection | | `Cursor\Memory` | Tests, or a consumer that should replay from the beginning on every restart | +| `Cursor\None` | No store configured β€” remembers nothing, so a restart replays | The store is allowed to be lossy: a lost position costs a replay, not a gap. A -store that is down is a warning rather than a failure β€” the consumer keeps its -position in memory and carries on. Pass `onWarning()` to hear about it: +store that is *down* is a different matter β€” the failure surfaces from +`consume()` as a `Transport` exception rather than being swallowed, so catch it +in your loop if the consumer should keep trying: ```php -$consumer->onWarning(fn (\Throwable $error, string $context) => - Console::warning("[feed] Could not {$context} the cursor: {$error->getMessage()}")); +try { + $consumer->consume($handler); +} catch (Transport $error) { + Console::warning("[feed] {$error->getMessage()}"); +} ``` Run **one process per consumer name.** Two sharing a name share one position, so diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 6815186..597f395 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -4,60 +4,15 @@ namespace Utopia\Feed; -use Utopia\CloudEvents\CloudEvent; - -/** - * Reads a feed from where it last got to, hands each new event to a handler, - * and records how far it got. - * - * ```php - * $consumer = new Consumer($feed, 'cache-invalidator', new Cursor\Cache($cache)); - * - * // On a timer, or in a loop with a long-poll timeout: - * $consumer->consume(function (CloudEvent $event) use ($cache) { - * $cache->purge($event->data['tag'] ?? ''); - * }); - * ``` - * - * Delivery is at-least-once, so a handler must be safe to run twice on the same - * event. A handler rejects an event by throwing, which stops the run there and - * leaves the position before it, so the next run tries again. See the README - * for what that means in practice. - */ +// Client class: Read what has been missed from journal class Consumer { - /** - * Events per run. Small enough that a backlog drains in bounded steps - * instead of one long pass that fails near the end and repeats itself. - */ public const int BATCH = 100; - /** - * The position, mirrored in memory, so a run reads the store once and a - * store that becomes unavailable afterwards costs nothing. - */ private ?string $position = null; private bool $restored = false; - /** @var (callable(\Throwable, string): void)|null */ - private $onWarning = null; - - /** - * @param Feed $feed Feed to read. - * @param string $name This consumer's name, which its position is stored - * under. Distinct per logical consumer, and stable across restarts. - * Run **one process per name** β€” two sharing a name share one - * position, so the feed is split between them rather than delivered - * to both. - * @param Cursor $cursor Where to keep the position. - * @param int $batch Events per run. - * @param int $timeout Milliseconds to wait for an event when the feed is - * caught up. Zero returns immediately, which is what a consumer - * driven by an external timer wants; a non-zero value suits one - * looping on its own. - * @throws Exception\Invalid When $name is empty. - */ public function __construct( protected readonly Feed $feed, protected readonly string $name, @@ -75,37 +30,6 @@ public function getName(): string return $this->name; } - /** - * Report failures that were survived rather than raised β€” currently, a - * position that could not be loaded or saved. - * - * These are not fatal: the consumer carries on with its in-memory position - * and the only cost is a replay after a restart. They are still worth - * knowing about, because a store that has been failing quietly for a week - * is a replay of the entire retained feed waiting to happen. - * - * @param (callable(\Throwable, string): void)|null $callback Receives the - * error and a short context string. - */ - public function onWarning(?callable $callback): self - { - $this->onWarning = $callback; - - return $this; - } - - /** - * Hand every event not yet seen to $handler, oldest first, and return how - * many it accepted. - * - * @param callable(CloudEvent): void $handler Throws to reject an event, - * which stops the run and leaves the position before it. - * @return int Events handled. Zero means the consumer is caught up. - * @throws Exception When the feed cannot be read. The position stays where - * it was, so the next run retries the same events. - * @throws \Throwable Whatever the handler threw, after the events before it - * have been committed. - */ public function consume(callable $handler): int { $events = $this->feed->poll($this->position(), $this->batch, $this->timeout); @@ -131,7 +55,8 @@ public function consume(callable $handler): int } if ($processed !== null) { - $this->advance($processed); + $this->position = $processed; + $this->cursor->save($this->feed->getName(), $this->name, $processed); } if ($failure !== null) { @@ -141,36 +66,16 @@ public function consume(callable $handler): int return $handled; } - /** - * Where this consumer has got to, or null if it has not started. - */ public function position(): ?string { - if ($this->restored) { - return $this->position; - } - - $this->restored = true; - - try { + if (!$this->restored) { $this->position = $this->cursor->load($this->feed->getName(), $this->name); - } catch (\Throwable $error) { - // Falls through to null, which restarts from the oldest retained - // event. Wasteful, but the alternatives are worse: guessing at a - // position risks skipping, and refusing to run turns an outage in - // the cursor store into an outage in whatever the feed drives. - $this->warn($error, 'load'); + $this->restored = true; } return $this->position; } - /** - * Drop the position and start again from the oldest retained event on the - * next run. Every event still in the feed will be handled again. - * - * @throws Exception When the store cannot be written. - */ public function reset(): void { $this->cursor->reset($this->feed->getName(), $this->name); @@ -178,24 +83,4 @@ public function reset(): void $this->position = null; $this->restored = true; } - - private function advance(string $eventId): void - { - $this->position = $eventId; - - try { - $this->cursor->save($this->feed->getName(), $this->name, $eventId); - } catch (\Throwable $error) { - // In-memory position already moved, so this process does not repeat - // itself; only a restart before the store recovers replays. - $this->warn($error, 'save'); - } - } - - private function warn(\Throwable $error, string $context): void - { - if ($this->onWarning !== null) { - ($this->onWarning)($error, $context); - } - } } diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index c5ddb4c..ef59671 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -6,9 +6,7 @@ use Utopia\Feed\Exception\Invalid; -/** - * Client class: Remember position of feed that has already been consumed. - */ +// Client class: Remember position of feed that has already been consumed abstract class Cursor { /** diff --git a/src/Feed/Cursor/None.php b/src/Feed/Cursor/None.php new file mode 100644 index 0000000..bf58cbc --- /dev/null +++ b/src/Feed/Cursor/None.php @@ -0,0 +1,38 @@ +key($feed, $consumer); + + return null; + } + + public function save(string $feed, string $consumer, string $eventId): void + { + $this->key($feed, $consumer); + } + + public function reset(string $feed, string $consumer): void + { + $this->key($feed, $consumer); + } +} diff --git a/src/Feed/Feed.php b/src/Feed/Feed.php index d94230e..3416532 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Feed.php @@ -6,51 +6,12 @@ use Utopia\CloudEvents\CloudEvent; -/** - * An append-only, ordered sequence of events that consumers pull. - * - * Follows http-feeds (https://www.http-feeds.org/): each consumer asks for what - * it has not seen yet, quoting the id of the last event it processed, so one - * that was down or only just deployed catches up on its next read. - * - * Delivery is at-least-once and retention is bounded, so every event must be - * safe to process twice, and a consumer that falls behind the trim horizon - * resumes from the oldest retained event. - * - * ```php - * $feed = new Feed(new Journal\Redis($redis, 'edge'), 'urn:appwrite:cloud:fra'); - * - * $feed->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']]); - * - * foreach ($feed->read($lastEventId) as $event) { - * // ... - * } - * ``` - * - * Subclass to give a feed a typed vocabulary β€” one method per thing that can - * happen, rather than callers assembling event types and payloads by hand. - */ class Feed { - /** - * Most events a single read may return. The cap belongs here because - * `limit` arrives from a consumer over the network. - */ public const int MAX_BATCH = 1000; - /** - * Longest a long poll may hold a request open, in milliseconds. Kept under - * the 60s that proxies commonly cut idle responses off at. - */ public const int MAX_TIMEOUT = 30_000; - /** - * @param Journal $journal Where the events live. - * @param string $source Who is producing them, as a URI reference - * (`urn:appwrite:cloud:fra`). Stamped onto every event this instance - * appends, so a consumer merging feeds from several producers can - * tell them apart. Only needed to append, not to read. - */ public function __construct( protected readonly Journal $journal, protected readonly string $source = '', @@ -62,17 +23,6 @@ public function getName(): string return $this->journal->getName(); } - /** - * Append an event and return its position in the feed. - * - * @param string $type What happened, in reverse-DNS notation. - * @param mixed $data Payload, JSON encodable. Usually a map, but the JSON - * event format leaves it unrestricted. - * @param string $subject The one business object this is about, if there is - * one. Empty means none, which is how CloudEvents models it. - * @throws Exception\Invalid When $type is empty or $data cannot be encoded. - * @throws Exception When the backend rejects the append. - */ public function append(string $type, mixed $data = [], string $subject = ''): string { return $this->publish(new CloudEvent( @@ -82,14 +32,6 @@ public function append(string $type, mixed $data = [], string $subject = ''): st )); } - /** - * Append a prepared event, stamping it with this feed's source and, unless - * it already has one, the current time. - * - * @throws Exception\Invalid When the event has no type, the feed has no - * source, or the data cannot be encoded. - * @throws Exception When the backend rejects the append. - */ public function publish(CloudEvent $event): string { if ($event->type === '') { @@ -100,68 +42,24 @@ public function publish(CloudEvent $event): string throw new Exception\Invalid('Feed source is required to append; construct the feed with one'); } - // Stamped with the withers rather than rebuilt, so anything this - // library does not model itself β€” a dataschema, an extension attribute - // such as a traceparent β€” survives the append untouched. $event = $event->withSource($this->source); return $this->journal->append($event->time === '' ? $event->withTime() : $event); } - /** - * Read the events after $lastEventId, oldest first, or from the oldest - * retained event when it is null. - * - * Returns immediately, with an empty list when the consumer is caught up. - * - * @return list - * @throws Exception\Invalid When $lastEventId is not a feed position. - * @throws Exception When the backend cannot be read. - */ + /** @return list */ public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): array { - return $this->journal->read($lastEventId, self::limit($limit)); + return $this->journal->read($lastEventId, \max(1, \min($limit, self::MAX_BATCH))); } - /** - * {@see read()}, but when there is nothing new yet, wait up to $timeout - * milliseconds for something to arrive before answering. - * - * This is how a consumer subscribes in near real time without hammering the - * producer. The batch may still come back empty β€” the timeout elapsing is a - * normal outcome, not a failure. A timeout of 0 makes this a plain read. - * - * Journals that cannot block wait by re-reading on an interval, which under - * Swoole yields the worker only if coroutine hooks are enabled. Without them - * it holds the worker for the duration, so run it with hooks on or keep the - * timeout at 0. - * - * @return list - * @throws Exception\Invalid When $lastEventId is not a feed position. - * @throws Exception When the backend cannot be read. - */ + /** @return list */ public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, int $timeout = 0): array { return $this->journal->poll( $lastEventId, - self::limit($limit), + \max(1, \min($limit, self::MAX_BATCH)), \max(0, \min($timeout, self::MAX_TIMEOUT)), ); } - - /** - * The limit a read will actually use. - * - * Clamped rather than rejected: `limit` is a hint about how much work a - * consumer wants in one go, and failing a read because it asked for too - * much would stall a feed over something the producer can simply decide. - * - * An endpoint serving this feed should clamp with this before answering, so - * the number it passes to {@see Protocol::cacheControl()} is the one the - * batch was actually built with. - */ - public static function limit(int $limit): int - { - return \max(1, \min($limit, self::MAX_BATCH)); - } } diff --git a/src/Feed/Id.php b/src/Feed/Id.php index 0947b80..91c252d 100644 --- a/src/Feed/Id.php +++ b/src/Feed/Id.php @@ -6,42 +6,21 @@ use Utopia\Feed\Exception\Invalid; -/** - * Feed positions: `-`, Redis' stream id format, where - * the sequence disambiguates events appended within the same millisecond. - * - * The format is part of the wire contract, not a Redis implementation detail β€” - * an id produced by one journal has to be a valid position for another, so a - * feed can move between backends without invalidating the positions consumers - * already hold. - */ final class Id { private const string PATTERN = '/^(\d+)-(\d+)$/'; - /** - * Whether $id is a feed position. - */ public static function isValid(string $id): bool { return \preg_match(self::PATTERN, $id) === 1; } - /** - * Build an id from its parts. - */ public static function encode(int $timestamp, int $sequence): string { return $timestamp . '-' . $sequence; } - /** - * Split an id into its millisecond timestamp and sequence number, which - * compare in feed order (`10-0` is after `9-0`, not before it). - * - * @return array{int, int} - * @throws Invalid When $id is not a feed position. - */ + /** @return array{int, int} */ public static function decode(string $id): array { if (\preg_match(self::PATTERN, $id, $matches) !== 1) { @@ -51,16 +30,6 @@ public static function decode(string $id): array return [(int) $matches[1], (int) $matches[2]]; } - /** - * The exclusive successor of an id: the smallest position strictly after it. - * - * Computed rather than relying on Redis' `(`-prefixed exclusive ranges, so - * reads work against anything speaking the Redis 5 stream API β€” including - * the proxies and compatible servers that never implemented the newer - * syntax. - * - * @throws Invalid When $id is not a feed position. - */ public static function after(string $id): string { [$timestamp, $sequence] = self::decode($id); diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index fd7be5f..a0d46e3 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -8,25 +8,11 @@ use Utopia\CloudEvents\Exception as CloudEventsException; use Utopia\Feed\Exception\Invalid; -/** - * Where a feed's events live. - * - * A journal does two things: assign an ordered id on append, and return the - * events strictly after a given id. Everything above it β€” long polling, - * cursors, the pull loop β€” is the same whichever journal is underneath. - */ +// Server class: Durable storage for events abstract class Journal { - /** - * Microseconds between reads while waiting in {@see poll()}. - */ - protected const int POLL_INTERVAL = 500_000; - - /** - * @param string $name Feed identifier. Also the key the backend stores it - * under, and the path segment it is served on. - * @throws Invalid When $name is empty. - */ + protected const int POLL_INTERVAL = 500_000; // 0.5s + public function __construct(protected readonly string $name) { if ($name === '') { @@ -39,41 +25,12 @@ public function getName(): string return $this->name; } - /** - * Append an event and return the id the backend assigned it. - * - * Any id already on $event is ignored: positions are the backend's to - * allocate, since only it can keep them ordered. - * - * @throws Exception When the event cannot be appended. - */ abstract public function append(CloudEvent $event): string; - /** - * Read up to $limit events strictly after $lastEventId, oldest first, or - * from the oldest retained event when it is null. - * - * An empty result means the consumer is caught up, not that the feed is - * empty. - * - * @return list - * @throws Invalid When $lastEventId is not a feed position. - * @throws Exception When the backend cannot be read. - */ + /** @return list */ abstract public function read(?string $lastEventId, int $limit): array; - /** - * {@see read()}, but wait up to $timeout milliseconds for an event before - * answering with an empty batch. - * - * Waits by re-reading on an interval, which works against any backend. A - * journal that can do better β€” {@see Journal\Http} hands the wait to the - * producer β€” overrides this. - * - * @return list - * @throws Invalid When $lastEventId is not a feed position. - * @throws Exception When the backend cannot be read. - */ + /** @return list */ public function poll(?string $lastEventId, int $limit, int $timeout): array { $deadline = \microtime(true) + $timeout / 1000; @@ -89,30 +46,7 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array } } - /** - * A feed must retain at least one event. Backends disagree about what a - * non-positive cap means β€” some keep nothing, some keep everything β€” so it - * is refused here rather than resolved differently on each one. - * - * @throws Invalid When $maxSize would retain fewer than one event. - */ - protected static function assertRetention(int $maxSize): void - { - if ($maxSize < 1) { - throw new Invalid("Feed retention must be at least one event, got {$maxSize}"); - } - } - - /** - * The backend fields an event is stored as. - * - * `data` and `extensions` are JSON so they can hold what CloudEvents lets - * them hold; every other attribute is a flat string a backend can index. - * The id is not among them β€” it is the key the entry is stored under. - * - * @return array - * @throws Invalid When the payload cannot be encoded. - */ + /** @return array */ protected static function encode(CloudEvent $event): array { return [ @@ -129,12 +63,7 @@ protected static function encode(CloudEvent $event): array ]; } - /** - * Rebuild an event from what {@see encode()} stored. - * - * @param array $fields - * @throws Invalid When the stored entry cannot be read as an event. - */ + /** @param array $fields */ protected static function decode(string $id, array $fields): CloudEvent { $extensions = \json_decode(self::field($fields, 'extensions'), true); @@ -156,25 +85,15 @@ protected static function decode(string $id, array $fields): CloudEvent } } - // The union operator rather than a spread, which renumbers integer - // keys: an extension named only of digits is legal, and PHP holds such - // a name as an int key. Spec attributes stay on the left, so they win - // any collision. $event += \is_array($extensions) ? $extensions : []; try { - // Lenient for the same reason Protocol::decode() is: the event - // happened and its id is a valid position, so refusing to return it - // over one malformed attribute would wedge every consumer behind it. return CloudEvent::fromArray($event, lenient: true); } catch (CloudEventsException $error) { throw new Invalid("Feed entry {$id} could not be read as an event: {$error->getMessage()}", previous: $error); } } - /** - * @throws Invalid When the value cannot be encoded. - */ private static function json(mixed $value, string $attribute): string { try { @@ -184,9 +103,7 @@ private static function json(mixed $value, string $attribute): string } } - /** - * @param array $fields - */ + /** @param array $fields */ private static function field(array $fields, string $key): string { $value = $fields[$key] ?? ''; diff --git a/src/Feed/Journal/Http.php b/src/Feed/Journal/Http.php index 0a7dd49..fcfe9cb 100644 --- a/src/Feed/Journal/Http.php +++ b/src/Feed/Journal/Http.php @@ -16,37 +16,10 @@ use Utopia\Psr7\Method; use Utopia\Psr7\Request\Factory as RequestFactory; -/** - * Someone else's feed, read over HTTP. - * - * The counterpart to serving a feed with {@see Protocol}: point this at another - * service's feed endpoint and consume it with the same `Feed` and `Consumer` a - * local one uses. Read-only β€” a feed is owned by whoever appends to it. - * - * ```php - * $client = (new Client(new Curl()))->withHeaders(['x-appwrite-jwt' => $token]); - * - * $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); - * ``` - * - * @see https://github.com/utopia-php/client - */ class Http extends Journal { private readonly RequestFactory $requests; - /** - * @param Adapter $client Configured with whatever credentials the producer - * requires. Typed as the client's own adapter interface rather than - * plain PSR-18, because a poll needs to set its own deadline. - * - * Retries are best left off: a failed read leaves the cursor where it - * was, so the next poll is already the retry. - * @param string $endpoint Base URL the producer serves its feeds under. The - * feed name is appended, so `https://cloud.example.com/v1/feeds` - * reads `https://cloud.example.com/v1/feeds/edge`. - * @param string $name Feed name, as the producer knows it. - */ public function __construct( protected readonly Adapter $client, protected readonly string $endpoint, @@ -57,10 +30,6 @@ public function __construct( $this->requests = new RequestFactory(); } - /** - * @throws Unsupported Always. A consumer cannot append to a feed it does - * not own; call the producer's own API instead. - */ public function append(CloudEvent $event): string { throw new Unsupported("The {$this->name} feed is read over HTTP and cannot be appended to"); @@ -71,10 +40,6 @@ public function read(?string $lastEventId, int $limit): array return $this->fetch($lastEventId, $limit, 0); } - /** - * The producer does the waiting, so a poll is one held request rather than - * a client-side loop. - */ public function poll(?string $lastEventId, int $limit, int $timeout): array { return $this->fetch($lastEventId, $limit, $timeout); @@ -82,8 +47,6 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array /** * @return list - * @throws Transport When the producer cannot be reached, or answers with an - * error status or a body that is not JSON. */ private function fetch(?string $lastEventId, int $limit, int $timeout): array { @@ -96,21 +59,22 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array [Header::ACCEPT => ContentType::JSON], ); + // A long poll needs a deadline past the one it asked the producer for, + // or the client cancels a correct wait a hair early and every quiet + // tick surfaces as a failure. A plain read keeps the caller's own. + $client = $timeout > 0 + ? $this->client->withTimeout(($timeout + Protocol::TIMEOUT_MARGIN) / 1000) + : $this->client; + try { - $response = $this->client($timeout)->sendRequest($request); + $response = $client->sendRequest($request); } catch (ClientExceptionInterface $error) { - // PSR-18 reserves exceptions for failures that produced no usable - // response, so anything landing here is a transport problem rather - // than something the producer said. throw new Transport("Failed to read the {$this->name} feed at {$url}: {$error->getMessage()}", previous: $error); } $status = $response->getStatusCode(); if ($status >= 400) { - // Carried as the exception code so a caller can act on it β€” most - // usefully to treat a 404 as "this producer does not serve the feed - // yet", which is normal while a feed is being rolled out. throw new Transport( "Reading the {$this->name} feed at {$url} failed with status {$status}", $status, @@ -130,19 +94,4 @@ private function url(): string { return \rtrim($this->endpoint, '/') . '/' . \rawurlencode($this->name); } - - /** - * A long poll needs a deadline past the one it asked for. Without the margin - * the client's deadline races the producer's, and a poll that correctly - * waits out its full timeout surfaces as a transport failure on every quiet - * tick. A plain read keeps whatever the caller configured. - */ - private function client(int $timeout): Adapter - { - if ($timeout <= 0) { - return $this->client; - } - - return $this->client->withTimeout(($timeout + Protocol::TIMEOUT_MARGIN) / 1000); - } } diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Journal/Memory.php index 145cb11..0afc4cc 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -5,40 +5,20 @@ namespace Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Id; use Utopia\Feed\Journal; -/** - * A feed held in process memory, for tests and single-process development. - * - * Implements the same id and retention semantics as {@see Redis}, including the - * awkward parts like resuming from a trimmed position, so code written against - * it behaves the same when it is swapped out. Not for production: nothing is - * shared between processes and nothing survives a restart. - */ class Memory extends Journal { /** @var list */ private array $events = []; - /** - * Last millisecond an event was appended in, with the sequence reached - * within it, so several appends in the same millisecond still get ordered - * ids the way `XADD` does. - */ private int $timestamp = 0; - private int $sequence = -1; - /** - * @throws Invalid When $name is empty, or $maxSize is below one event. - */ public function __construct(string $name, protected readonly int $maxSize = 100_000) { parent::__construct($name); - - self::assertRetention($maxSize); } public function append(CloudEvent $event): string @@ -49,16 +29,11 @@ public function append(CloudEvent $event): string $this->timestamp = $now; $this->sequence = 0; } else { - // Also covers a clock that stepped backwards: ids must never go - // backwards, so the sequence keeps climbing under the old - // millisecond rather than the timestamp following the clock down. $this->sequence++; } $id = Id::encode($this->timestamp, $this->sequence); - // Stored through the same encode/decode a real backend goes through, - // so this journal cannot accept payloads that would fail in production. $this->events[] = self::decode($id, self::encode($event)); if (\count($this->events) > $this->maxSize) { @@ -70,8 +45,6 @@ public function append(CloudEvent $event): string public function read(?string $lastEventId, int $limit): array { - // Decoded up front so a malformed position fails the same way it does - // on every other journal, even when the feed is empty. $after = $lastEventId === null ? null : Id::decode($lastEventId); $events = []; diff --git a/src/Feed/Journal/None.php b/src/Feed/Journal/None.php new file mode 100644 index 0000000..93286de --- /dev/null +++ b/src/Feed/Journal/None.php @@ -0,0 +1,27 @@ +name} feed"); + } + + public function read(?string $lastEventId, int $limit): array + { + throw new Unsupported("No feed backend is configured for the {$this->name} feed"); + } +} diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Journal/Pool.php index 0f962aa..7fee505 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -5,26 +5,13 @@ namespace Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Journal; use Utopia\Pools\Pool as UtopiaPool; -/** - * {@see Redis}, over a pooled connection. - * - * What most services producing a feed want: a long poll holds its connection - * for the whole timeout, so reading through a shared client would block every - * other user of it. - * - * @see https://github.com/utopia-php/pools - */ class Pool extends Journal { /** * @param UtopiaPool<\Redis|\RedisCluster> $pool - * @param string $name Feed name; the stream is stored at `feed:`. - * @param int $maxSize Approximate cap on retained events. - * @throws Invalid When $name is empty, or $maxSize is below one event. */ public function __construct( protected readonly UtopiaPool $pool, @@ -32,29 +19,19 @@ public function __construct( protected readonly int $maxSize = 100_000, ) { parent::__construct($name); - - self::assertRetention($maxSize); } public function append(CloudEvent $event): string { - return $this->pool->use(fn (\Redis|\RedisCluster $redis): string => $this->journal($redis)->append($event)); + return $this->pool->use( + fn (\Redis|\RedisCluster $redis): string => (new Redis($redis, $this->name, $this->maxSize))->append($event) + ); } public function read(?string $lastEventId, int $limit): array { return $this->pool->use( - fn (\Redis|\RedisCluster $redis): array => $this->journal($redis)->read($lastEventId, $limit) + fn (\Redis|\RedisCluster $redis): array => (new Redis($redis, $this->name, $this->maxSize))->read($lastEventId, $limit) ); } - - /** - * The connection is only borrowed for the length of one call, so the - * journal wrapping it is built per call too. It holds no state beyond the - * connection, which makes that free. - */ - private function journal(\Redis|\RedisCluster $redis): Redis - { - return new Redis($redis, $this->name, $this->maxSize); - } } diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Journal/Redis.php index 4d8a664..7f78d94 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -5,44 +5,24 @@ namespace Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; use Utopia\Feed\Journal; -/** - * A feed backed by a Redis stream. - * - * `XADD` allocates ids that are ordered and unique across concurrent producers, - * and `XRANGE` pages from any of them without the producer tracking who has - * read what. Consumer groups are deliberately not used β€” they move the position - * onto the producer, which is what this library exists to avoid. - */ class Redis extends Journal { - /** - * @param \Redis|\RedisCluster $redis - * @param string $name Feed name; the stream is stored at `feed:`. - * @param int $maxSize Cap on retained events. Redis trims approximately, to - * whole nodes, so a feed holds at least this many and usually more β€” - * a bound on memory, not a promise about how far back a consumer can - * resume from. - * @throws Invalid When $name is empty, or $maxSize is below one event. - */ public function __construct( protected readonly \Redis|\RedisCluster $redis, string $name, protected readonly int $maxSize = 100_000, ) { parent::__construct($name); - - self::assertRetention($maxSize); } public function append(CloudEvent $event): string { try { - $id = $this->redis->xAdd($this->key(), '*', self::encode($event), $this->maxSize, true); + $id = $this->redis->xAdd('feed:' . $this->name, '*', self::encode($event), $this->maxSize, true); } catch (\RedisException $error) { throw new Transport("Failed to append to the {$this->name} feed: {$error->getMessage()}", previous: $error); } @@ -59,7 +39,7 @@ public function read(?string $lastEventId, int $limit): array $start = $lastEventId === null ? '-' : Id::after($lastEventId); try { - $entries = $this->redis->xRange($this->key(), $start, '+', $limit); + $entries = $this->redis->xRange('feed:' . $this->name, $start, '+', $limit); } catch (\RedisException $error) { throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); } @@ -81,13 +61,4 @@ public function read(?string $lastEventId, int $limit): array return $events; } - - /** - * The key the stream lives at. Namespaced so a feed can share a Redis with - * whatever else the service keeps there. - */ - private function key(): string - { - return 'feed:' . $this->name; - } } diff --git a/src/Feed/Journal/Unconfigured.php b/src/Feed/Journal/Unconfigured.php deleted file mode 100644 index febadac..0000000 --- a/src/Feed/Journal/Unconfigured.php +++ /dev/null @@ -1,38 +0,0 @@ -name} feed"); - } - - public function read(?string $lastEventId, int $limit): array - { - throw new Unsupported("No feed backend is configured for the {$this->name} feed"); - } -} diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 0be9a15..5b763f4 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -18,7 +18,7 @@ * * ```php * // Producer, in whatever routing layer it uses: - * $limit = Feed::limit((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH)); + * $limit = \min((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH), Feed::MAX_BATCH); * * $events = $feed->poll( * $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, @@ -185,7 +185,7 @@ public static function decode(mixed $payload): array * * @param int $count Events being returned. * @param int $limit Events the batch was built with, after {@see - * Feed::limit()} has clamped what the consumer asked for. + * Feed::MAX_BATCH} has capped what the consumer asked for. * @param bool $public Whether a shared cache may store the batch. Off by * default: feeds are usually served behind authorization, and * `public` there would let a CDN hand one consumer's events to a diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index e9ec3b4..b7adbc1 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -11,6 +11,7 @@ use Utopia\Feed\Cursor\Memory as MemoryCursor; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; +use Utopia\Feed\Exception\Transport; use Utopia\Feed\Feed; use Utopia\Tests\Unit\Support\FailingCursor; @@ -212,48 +213,86 @@ public function testDrainsABacklogInBatches(): void } /** - * The store failing must not stop the work: the position is mirrored in - * memory, so the run carries on and only a restart before the store - * recovers replays anything. + * A cursor store that is down surfaces rather than being swallowed: reading + * from an unknown position would replay the retained feed, so the run stops + * and the caller decides. */ - public function testKeepsWorkingWhenThePositionCannotBeLoaded(): void + public function testAPositionThatCannotBeLoadedStopsTheRun(): void { $this->feed->append('a'); $consumer = $this->consumer(new FailingCursor(onLoad: true)); - $warnings = []; - $consumer->onWarning(function (\Throwable $error, string $context) use (&$warnings): void { - $warnings[] = $context; - }); + $seen = []; - $this->assertSame(['a'], $this->drain($consumer)); - $this->assertSame(['load'], $warnings); + try { + $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }); + $this->fail('The store failure should have been raised'); + } catch (Transport $error) { + $this->assertSame('Cursor store is unavailable', $error->getMessage()); + } + + $this->assertSame([], $seen, 'Nothing is handled from a position that could not be read'); } - public function testKeepsWorkingWhenThePositionCannotBeSaved(): void + /** + * The load is retried on the next run rather than being remembered as a + * failure, so a store that blips does not leave the consumer stuck. + */ + public function testAFailedLoadIsRetriedOnTheNextRun(): void { $this->feed->append('a'); - $this->feed->append('b'); - $consumer = $this->consumer(new FailingCursor(onSave: true)); - $warnings = []; - $consumer->onWarning(function (\Throwable $error, string $context) use (&$warnings): void { - $warnings[] = $context; - }); + $cursor = new class () extends MemoryCursor { + public bool $fail = true; - $this->assertSame(['a', 'b'], $this->drain($consumer)); - $this->assertSame(['save'], $warnings); + public function load(string $feed, string $consumer): ?string + { + if ($this->fail) { + $this->fail = false; - $this->feed->append('c'); + throw new Transport('Cursor store is unavailable'); + } - $this->assertSame(['c'], $this->drain($consumer), 'The in-memory position still moved'); + return parent::load($feed, $consumer); + } + }; + + $consumer = $this->consumer($cursor); + + try { + $consumer->consume(fn (CloudEvent $event) => null); + } catch (Transport) { + // Expected on the first run. + } + + $this->assertSame(['a'], $this->drain($consumer), 'The second run reads the store again'); } - public function testSurvivesAFailingStoreWithNoWarningHandler(): void + /** + * The events were handled, so the failure comes after them: this run keeps + * its progress in memory and only a restart replays. + */ + public function testAPositionThatCannotBeSavedIsRaisedAfterTheEventsAreHandled(): void { $this->feed->append('a'); + $this->feed->append('b'); + + $consumer = $this->consumer(new FailingCursor(onSave: true)); + $seen = []; + + try { + $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }); + $this->fail('The store failure should have been raised'); + } catch (Transport $error) { + $this->assertSame('Cursor store is unavailable', $error->getMessage()); + } - $this->assertSame(['a'], $this->drain($this->consumer(new FailingCursor(onLoad: true, onSave: true)))); + $this->assertSame(['a', 'b'], $seen, 'The handler still saw the batch'); + $this->assertNotNull($consumer->position(), 'The in-memory position still moved'); } public function testResetReplaysEverythingStillRetained(): void @@ -310,7 +349,7 @@ public function testAFailedReadLeavesThePositionAlone(): void $this->feed->append('b'); $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer(new Feed(new \Utopia\Feed\Journal\Unconfigured('edge')), 'invalidator', $this->cursor); + $consumer = new Consumer(new Feed(new \Utopia\Feed\Journal\None('edge')), 'invalidator', $this->cursor); $this->expectException(\Utopia\Feed\Exception\Unsupported::class); diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index ba7daed..d6c25d7 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -10,6 +10,7 @@ use Utopia\Feed\Cursor; use Utopia\Feed\Cursor\Cache; use Utopia\Feed\Cursor\Memory; +use Utopia\Feed\Cursor\None; use Utopia\Feed\Exception\Invalid; class CursorTest extends TestCase @@ -109,4 +110,46 @@ public function testRejectsAnEmptyFeedName(Cursor $cursor): void $cursor->save('', 'invalidator', '1-0'); } + + public function testTheNoneStoreRemembersNothing(): void + { + $cursor = new None(); + + $cursor->save('edge', 'invalidator', '1-0'); + + $this->assertNull($cursor->load('edge', 'invalidator'), 'Nothing is stored, so nothing comes back'); + } + + public function testResettingTheNoneStoreIsHarmless(): void + { + $cursor = new None(); + + $cursor->reset('edge', 'invalidator'); + + $this->assertNull($cursor->load('edge', 'invalidator')); + } + + /** + * A stand-in that accepted names a real store rejects would let a bug + * through in development and surface it in production instead. + * + * @return array + */ + public static function unusableNames(): array + { + return [ + 'no feed' => ['', 'invalidator'], + 'no consumer' => ['edge', ''], + ]; + } + + /** + * @dataProvider unusableNames + */ + public function testTheNoneStoreStillRejectsEmptyNames(string $feed, string $consumer): void + { + $this->expectException(Invalid::class); + + (new None())->load($feed, $consumer); + } } diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 52bbc85..07186c1 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -6,7 +6,7 @@ use PHPUnit\Framework\TestCase; use Utopia\Feed\Journal\Memory; -use Utopia\Feed\Journal\Unconfigured; +use Utopia\Feed\Journal\None; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; @@ -142,19 +142,6 @@ public function testClampsTheLimitToTheMaximum(): void $this->assertCount(1, $this->feed->read(null, -5)); } - /** - * An endpoint serving this feed clamps with the same helper, so the limit - * it passes to `Protocol::cacheControl()` is the one the batch was built - * with. - */ - public function testExposesTheLimitAReadWillActuallyUse(): void - { - $this->assertSame(50, Feed::limit(50)); - $this->assertSame(Feed::MAX_BATCH, Feed::limit(Feed::MAX_BATCH * 10)); - $this->assertSame(1, Feed::limit(0)); - $this->assertSame(1, Feed::limit(-5)); - } - public function testRejectsAPositionThatIsNotAFeedId(): void { $this->expectException(Invalid::class); @@ -370,18 +357,18 @@ public function testExposesTheFeedItReads(): void $this->assertSame('edge', $this->feed->getName()); } - public function testAnUnconfiguredBackendFailsLoudlyRatherThanDroppingEvents(): void + public function testAFeedWithNoBackendFailsLoudlyRatherThanDroppingEvents(): void { - $feed = new Feed(new Unconfigured('edge'), 'urn:appwrite:cloud:fra'); + $feed = new Feed(new None('edge'), 'urn:appwrite:cloud:fra'); $this->expectException(Unsupported::class); $feed->append('test'); } - public function testAnUnconfiguredBackendCannotBeRead(): void + public function testAFeedWithNoBackendCannotBeRead(): void { - $feed = new Feed(new Unconfigured('edge')); + $feed = new Feed(new None('edge')); $this->expectException(Unsupported::class); @@ -395,31 +382,6 @@ public function testRejectsAnEmptyFeedName(): void new Memory(''); } - /** - * @return array - */ - public static function unusableRetention(): array - { - return [ - 'zero' => [0], - 'negative' => [-5], - ]; - } - - /** - * Backends disagree about what a non-positive cap means β€” some keep - * nothing, some keep everything β€” so it is refused at construction rather - * than resolved one way in a test and the other way in production. - * - * @dataProvider unusableRetention - */ - public function testRejectsARetentionCapThatWouldNotBoundTheFeed(int $maxSize): void - { - $this->expectException(Invalid::class); - - new Memory('edge', maxSize: $maxSize); - } - public function testAcceptsTheSmallestUsefulRetentionCap(): void { $feed = new Feed(new Memory('edge', maxSize: 1), 'urn:appwrite:cloud:fra'); diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php index da303a3..b7bb08d 100644 --- a/tests/Feed/Unit/Support/FeedServer.php +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -33,7 +33,7 @@ protected function respond(RequestInterface $request): ResponseInterface \parse_str($request->getUri()->getQuery(), $query); $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; - $limit = Feed::limit((int) ($query[Protocol::PARAM_LIMIT] ?? Feed::MAX_BATCH)); + $limit = \min((int) ($query[Protocol::PARAM_LIMIT] ?? Feed::MAX_BATCH), Feed::MAX_BATCH); $timeout = (int) ($query[Protocol::PARAM_TIMEOUT] ?? 0); $events = $this->feed->poll( From 583f4db706709c393c1a71a58a9d8c0eea704440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 18:25:55 +0200 Subject: [PATCH 18/68] Update Protocol.php --- src/Feed/Protocol.php | 102 ++---------------------------------------- 1 file changed, 3 insertions(+), 99 deletions(-) diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 5b763f4..00df42b 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -8,78 +8,22 @@ use Utopia\CloudEvents\Exception as CloudEventsException; use Utopia\Feed\Exception\Invalid; -/** - * The HTTP shape of a feed, as defined by https://www.http-feeds.org/. - * - * Both halves live here so a producer's endpoint and its consumers cannot - * drift: {@see Journal\Http} reads through it, and a producer builds its - * response with it β€” in whichever HTTP framework it is written in, which is why - * this deals in arrays rather than in requests and responses. - * - * ```php - * // Producer, in whatever routing layer it uses: - * $limit = \min((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH), Feed::MAX_BATCH); - * - * $events = $feed->poll( - * $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, - * $limit, - * (int) $request->getParam(Protocol::PARAM_TIMEOUT, 0), - * ); - * - * $response - * ->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)) - * ->json(Protocol::encode($events)); - * ``` - */ +// HTTP shape of a feed, as defined by https://www.http-feeds.org/. final class Protocol { - /** - * Position to read from. Omitted or empty means the oldest retained event β€” - * not the newest, which would silently drop whatever is already in the feed. - */ public const string PARAM_LAST_EVENT_ID = 'lastEventId'; - - /** - * Most events to return. - */ public const string PARAM_LIMIT = 'limit'; - - /** - * Milliseconds to hold the request open waiting for an event before - * answering with an empty batch. Zero returns immediately. - */ public const string PARAM_TIMEOUT = 'timeout'; public const string KEY_EVENTS = 'events'; - public const string KEY_TOTAL = 'total'; - /** - * A full batch is settled history β€” the same query returns the same events - * forever β€” so it may be cached indefinitely. - */ public const string CACHE_IMMUTABLE = 'max-age=31536000'; - - /** - * A short batch is the live end of the feed. Re-asking the same question a - * second later legitimately returns more, so it must not be cached at all. - */ public const string CACHE_NONE = 'no-store'; - - /** - * Extra milliseconds a consumer allows its HTTP client on top of the long - * poll timeout it asked for, so the client's deadline does not race the - * producer's. - */ + public const int TIMEOUT_MARGIN = 10_000; /** - * The query string for a read. - * - * Parameters at their default are left out, so a consumer and a producer - * that disagree on a default resolve it the producer's way β€” and so the URL - * of a first read is stable enough to be cached and logged as one thing. - * * @return array */ public static function query(?string $lastEventId = null, int $limit = 0, int $timeout = 0): array @@ -102,12 +46,8 @@ public static function query(?string $lastEventId = null, int $limit = 0, int $t } /** - * The response body for a batch. - * * @param list $events - * @return array{total: int, events: list>} The keys - * are not narrowed to strings because an extension attribute named - * only of digits is legal, and PHP holds such a name as an int key. + * @return array{total: int, events: list>} */ public static function encode(array $events): array { @@ -118,23 +58,7 @@ public static function encode(array $events): array } /** - * Read a batch out of a response body. - * - * Decoded leniently, and tolerating a `specversion` this consumer has never - * seen: a feed is read by consumers older than the producer *by design*, so - * a producer that adds an attribute or moves the spec forward must not stop - * one that predates it. Only `id` is enforced β€” for a feed the id *is* the - * consumer's position, so an event without one cannot be recorded as passed. - * - * Stops at the first event that cannot be decoded and returns the ones - * before it. The consumer applies those, advances, and meets the broken - * event at the head of the next batch β€” where, with no prefix left to - * salvage, this throws and the feed visibly stops instead of quietly - * skipping past a gap. - * * @return list - * @throws Invalid When the payload is not a batch, or when the very first - * event in it cannot be decoded. */ public static function decode(mixed $payload): array { @@ -142,11 +66,6 @@ public static function decode(mixed $payload): array throw new Invalid('Expected a feed batch, got ' . \get_debug_type($payload)); } - // Required, not defaulted to empty: an empty batch and a response that - // is not a batch at all mean opposite things β€” "you are caught up" - // versus "you did not reach the feed" β€” and a misrouted request or a - // proxy's error page must not read as a quiet, permanent caught-up - // state. if (!\array_key_exists(self::KEY_EVENTS, $payload)) { throw new Invalid('Feed batch is missing the "' . self::KEY_EVENTS . '" field'); } @@ -180,17 +99,6 @@ public static function decode(mixed $payload): array return $events; } - /** - * What a producer should send as `Cache-Control` for a batch. - * - * @param int $count Events being returned. - * @param int $limit Events the batch was built with, after {@see - * Feed::MAX_BATCH} has capped what the consumer asked for. - * @param bool $public Whether a shared cache may store the batch. Off by - * default: feeds are usually served behind authorization, and - * `public` there would let a CDN hand one consumer's events to a - * requester that never presented a credential. - */ public static function cacheControl(int $count, int $limit, bool $public = false): string { if ($count < $limit || $count === 0) { @@ -201,11 +109,7 @@ public static function cacheControl(int $count, int $limit, bool $public = false } /** - * Decode one event, enforcing the only attribute a feed cannot do without. - * * @param array $raw - * @throws Invalid When the event carries no usable id. - * @throws CloudEventsException When it is not a CloudEvent at all. */ private static function event(array $raw): CloudEvent { From 4fd26569bc28ca0f9f90c300f10ff5e16b9ba317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 18:35:46 +0200 Subject: [PATCH 19/68] Introduce Appendable away from Feed --- src/Feed/Appendable.php | 22 ++++ src/Feed/Consumer.php | 3 +- src/Feed/Cursor.php | 3 +- src/Feed/Feed.php | 32 +----- src/Feed/Journal.php | 6 +- src/Feed/Journal/Http.php | 6 -- src/Feed/Journal/Memory.php | 3 +- src/Feed/Journal/None.php | 3 +- src/Feed/Journal/Pool.php | 3 +- src/Feed/Journal/Redis.php | 3 +- src/Feed/Producer.php | 68 ++++++++++++ tests/Feed/E2E/RedisTest.php | 19 ++-- tests/Feed/Unit/ConsumerTest.php | 60 ++++++----- tests/Feed/Unit/FeedTest.php | 155 +++++++--------------------- tests/Feed/Unit/HttpJournalTest.php | 11 -- tests/Feed/Unit/ProducerTest.php | 133 ++++++++++++++++++++++++ tests/Feed/Unit/RoundTripTest.php | 8 +- 17 files changed, 330 insertions(+), 208 deletions(-) create mode 100644 src/Feed/Appendable.php create mode 100644 src/Feed/Producer.php create mode 100644 tests/Feed/Unit/ProducerTest.php diff --git a/src/Feed/Appendable.php b/src/Feed/Appendable.php new file mode 100644 index 0000000..8a729c5 --- /dev/null +++ b/src/Feed/Appendable.php @@ -0,0 +1,22 @@ +journal->getName(); } - public function append(string $type, mixed $data = [], string $subject = ''): string - { - return $this->publish(new CloudEvent( - type: $type, - subject: $subject === '' ? null : $subject, - data: $data, - )); - } - - public function publish(CloudEvent $event): string - { - if ($event->type === '') { - throw new Exception\Invalid('Feed event type is required'); - } - - if ($this->source === '') { - throw new Exception\Invalid('Feed source is required to append; construct the feed with one'); - } - - $event = $event->withSource($this->source); - - return $this->journal->append($event->time === '' ? $event->withTime() : $event); - } - /** @return list */ public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): array { diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index a0d46e3..c5610e6 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -8,7 +8,9 @@ use Utopia\CloudEvents\Exception as CloudEventsException; use Utopia\Feed\Exception\Invalid; -// Server class: Durable storage for events +// Server class: durable storage for the events β€” Journal\Redis, Pool, Memory. +// Client exception: Journal\Http, which reads another service's feed over the wire. +// Journals that own their events also implement Appendable. abstract class Journal { protected const int POLL_INTERVAL = 500_000; // 0.5s @@ -25,8 +27,6 @@ public function getName(): string return $this->name; } - abstract public function append(CloudEvent $event): string; - /** @return list */ abstract public function read(?string $lastEventId, int $limit): array; diff --git a/src/Feed/Journal/Http.php b/src/Feed/Journal/Http.php index fcfe9cb..80bdce2 100644 --- a/src/Feed/Journal/Http.php +++ b/src/Feed/Journal/Http.php @@ -8,7 +8,6 @@ use Utopia\Client\Adapter; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; -use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Journal; use Utopia\Feed\Protocol; use Utopia\Psr7\ContentType; @@ -30,11 +29,6 @@ public function __construct( $this->requests = new RequestFactory(); } - public function append(CloudEvent $event): string - { - throw new Unsupported("The {$this->name} feed is read over HTTP and cannot be appended to"); - } - public function read(?string $lastEventId, int $limit): array { return $this->fetch($lastEventId, $limit, 0); diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Journal/Memory.php index 0afc4cc..f9f4cfc 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -6,9 +6,10 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Id; +use Utopia\Feed\Appendable; use Utopia\Feed\Journal; -class Memory extends Journal +class Memory extends Journal implements Appendable { /** @var list */ private array $events = []; diff --git a/src/Feed/Journal/None.php b/src/Feed/Journal/None.php index 93286de..144115a 100644 --- a/src/Feed/Journal/None.php +++ b/src/Feed/Journal/None.php @@ -6,9 +6,10 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Unsupported; +use Utopia\Feed\Appendable; use Utopia\Feed\Journal; -class None extends Journal +class None extends Journal implements Appendable { public function __construct(string $name = 'none') { diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Journal/Pool.php index 7fee505..cfa3949 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -5,10 +5,11 @@ namespace Utopia\Feed\Journal; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Appendable; use Utopia\Feed\Journal; use Utopia\Pools\Pool as UtopiaPool; -class Pool extends Journal +class Pool extends Journal implements Appendable { /** * @param UtopiaPool<\Redis|\RedisCluster> $pool diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Journal/Redis.php index 7f78d94..0873c6a 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -7,9 +7,10 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; +use Utopia\Feed\Appendable; use Utopia\Feed\Journal; -class Redis extends Journal +class Redis extends Journal implements Appendable { public function __construct( protected readonly \Redis|\RedisCluster $redis, diff --git a/src/Feed/Producer.php b/src/Feed/Producer.php new file mode 100644 index 0000000..e865e75 --- /dev/null +++ b/src/Feed/Producer.php @@ -0,0 +1,68 @@ +journal->getName(); + } + + /** + * Append an event and return its position in the feed. + * + * @throws Exception\Invalid When $type is empty or $data cannot be encoded. + * @throws Exception When the backend rejects the append. + */ + public function append(string $type, mixed $data = [], string $subject = ''): string + { + return $this->publish(new CloudEvent( + type: $type, + subject: $subject === '' ? null : $subject, + data: $data, + )); + } + + /** + * Append a prepared event, stamping it with this producer's source and, + * unless it already has one, the current time. + * + * @throws Exception\Invalid When the event has no type or cannot be encoded. + * @throws Exception When the backend rejects the append. + */ + public function publish(CloudEvent $event): string + { + if ($event->type === '') { + throw new Exception\Invalid('Feed event type is required'); + } + + // Stamped with the withers rather than rebuilt, so anything this library + // does not model β€” a dataschema, a traceparent β€” survives untouched. + $event = $event->withSource($this->source); + + return $this->journal->append($event->time === '' ? $event->withTime() : $event); + } +} diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index e9ae175..baecc23 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -11,6 +11,7 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Feed; +use Utopia\Feed\Producer; use Utopia\Feed\Id; /** @@ -49,7 +50,12 @@ protected function tearDown(): void private function feed(int $maxSize = 100_000): Feed { - return new Feed(new RedisJournal($this->redis, $this->name, $maxSize), 'urn:test:e2e'); + return new Feed(new RedisJournal($this->redis, $this->name, $maxSize)); + } + + private function producer(int $maxSize = 100_000): Producer + { + return new Producer(new RedisJournal($this->redis, $this->name, $maxSize), 'urn:test:e2e'); } public function testAppendsAndReadsBack(): void @@ -71,7 +77,7 @@ public function testAppendsAndReadsBack(): void public function testStreamIdsMatchTheFormatPositionsAreParsedWith(): void { - $id = $this->feed()->append('test'); + $id = $this->producer()->append('test'); $this->assertTrue(Id::isValid($id), "Redis returned an id this library cannot page from: {$id}"); } @@ -113,7 +119,7 @@ public function testReadsStrictlyAfterAPosition(): void public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void { - $this->feed()->publish(new CloudEvent( + $this->producer()->publish(new CloudEvent( id: '', type: 'test', dataschema: 'https://example.com/schema.json', @@ -128,14 +134,14 @@ public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void public function testAnAbsentSubjectStaysAbsent(): void { - $this->feed()->append('test'); + $this->producer()->append('test'); $this->assertNull($this->feed()->read()[0]->subject); } public function testAScalarPayloadSurvivesTheRoundTrip(): void { - $this->feed()->append('test', 'a string'); + $this->producer()->append('test', 'a string'); $this->assertSame('a string', $this->feed()->read()[0]->data); } @@ -149,7 +155,7 @@ public function testNestedPayloadsSurviveTheRoundTrip(): void 'unicode' => 'ΓΌnΓ―cΓΈde βœ“', ]; - $this->feed()->append('test', $data); + $this->producer()->append('test', $data); $this->assertSame($data, $this->feed()->read()[0]->data); } @@ -180,6 +186,7 @@ public function testRejectsAPositionThatIsNotAFeedId(): void public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void { $feed = $this->feed(maxSize: 10); + $producer = $this->producer(maxSize: 10); $first = $feed->append('first'); diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index b7adbc1..e4efc1a 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -13,6 +13,7 @@ use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Feed; +use Utopia\Feed\Producer; use Utopia\Tests\Unit\Support\FailingCursor; class ConsumerTest extends TestCase @@ -21,12 +22,15 @@ class ConsumerTest extends TestCase private Feed $feed; + private Producer $producer; + private MemoryCursor $cursor; protected function setUp(): void { $this->journal = new MemoryJournal('edge'); - $this->feed = new Feed($this->journal, 'urn:test'); + $this->producer = new Producer($this->journal, 'urn:test'); + $this->feed = new Feed($this->journal); $this->cursor = new MemoryCursor(); } @@ -51,8 +55,8 @@ private function drain(Consumer $consumer, ?int &$count = null): array public function testHandlesEachEventAndAdvancesPastTheLastOne(): void { - $this->feed->append('a'); - $last = $this->feed->append('b'); + $this->producer->append('a'); + $last = $this->producer->append('b'); $consumer = $this->consumer(); @@ -64,7 +68,7 @@ public function testHandlesEachEventAndAdvancesPastTheLastOne(): void public function testCaughtUpConsumerDoesNothing(): void { - $this->feed->append('a'); + $this->producer->append('a'); $consumer = $this->consumer(); $consumer->consume(fn (CloudEvent $event) => null); @@ -74,8 +78,8 @@ public function testCaughtUpConsumerDoesNothing(): void public function testResumesFromTheStoredPosition(): void { - $first = $this->feed->append('a'); - $this->feed->append('b'); + $first = $this->producer->append('a'); + $this->producer->append('b'); $this->cursor->save('edge', 'invalidator', $first); @@ -89,15 +93,15 @@ public function testResumesFromTheStoredPosition(): void */ public function testAConsumerWithNoPositionStartsAtTheOldestEventNotTheTip(): void { - $this->feed->append('a'); - $this->feed->append('b'); + $this->producer->append('a'); + $this->producer->append('b'); $this->assertSame(['a', 'b'], $this->drain($this->consumer())); } public function testReadsTheStoreOnceAndThenTracksThePositionInMemory(): void { - $this->feed->append('a'); + $this->producer->append('a'); $cursor = new class () extends MemoryCursor { public int $loads = 0; @@ -124,9 +128,9 @@ public function load(string $feed, string $consumer): ?string public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void { - $first = $this->feed->append('a'); - $this->feed->append('b'); - $this->feed->append('c'); + $first = $this->producer->append('a'); + $this->producer->append('b'); + $this->producer->append('c'); $consumer = $this->consumer(); $seen = []; @@ -150,8 +154,8 @@ public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void public function testRetriesTheFailedEventOnTheNextRun(): void { - $this->feed->append('a'); - $this->feed->append('b'); + $this->producer->append('a'); + $this->producer->append('b'); $consumer = $this->consumer(); $attempts = 0; @@ -178,7 +182,7 @@ public function testRetriesTheFailedEventOnTheNextRun(): void */ public function testAFailureOnTheFirstEventCommitsNothing(): void { - $this->feed->append('a'); + $this->producer->append('a'); try { $this->consumer()->consume(fn (CloudEvent $event) => throw new \RuntimeException('nope')); @@ -191,9 +195,9 @@ public function testAFailureOnTheFirstEventCommitsNothing(): void public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void { - $this->feed->append('a'); - $this->feed->append('b'); - $this->feed->append('c'); + $this->producer->append('a'); + $this->producer->append('b'); + $this->producer->append('c'); $this->assertSame(3, $this->consumer()->consume(fn (CloudEvent $event) => null)); } @@ -201,7 +205,7 @@ public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void public function testDrainsABacklogInBatches(): void { foreach (\range(1, 10) as $i) { - $this->feed->append('event-' . $i); + $this->producer->append('event-' . $i); } $consumer = $this->consumer(batch: 4); @@ -219,7 +223,7 @@ public function testDrainsABacklogInBatches(): void */ public function testAPositionThatCannotBeLoadedStopsTheRun(): void { - $this->feed->append('a'); + $this->producer->append('a'); $consumer = $this->consumer(new FailingCursor(onLoad: true)); $seen = []; @@ -242,7 +246,7 @@ public function testAPositionThatCannotBeLoadedStopsTheRun(): void */ public function testAFailedLoadIsRetriedOnTheNextRun(): void { - $this->feed->append('a'); + $this->producer->append('a'); $cursor = new class () extends MemoryCursor { public bool $fail = true; @@ -276,8 +280,8 @@ public function load(string $feed, string $consumer): ?string */ public function testAPositionThatCannotBeSavedIsRaisedAfterTheEventsAreHandled(): void { - $this->feed->append('a'); - $this->feed->append('b'); + $this->producer->append('a'); + $this->producer->append('b'); $consumer = $this->consumer(new FailingCursor(onSave: true)); $seen = []; @@ -297,8 +301,8 @@ public function testAPositionThatCannotBeSavedIsRaisedAfterTheEventsAreHandled() public function testResetReplaysEverythingStillRetained(): void { - $this->feed->append('a'); - $this->feed->append('b'); + $this->producer->append('a'); + $this->producer->append('b'); $consumer = $this->consumer(); $consumer->consume(fn (CloudEvent $event) => null); @@ -312,7 +316,7 @@ public function testResetReplaysEverythingStillRetained(): void public function testConsumersOfTheSameFeedTrackSeparatePositions(): void { - $this->feed->append('a'); + $this->producer->append('a'); $one = new Consumer($this->feed, 'one', $this->cursor); $two = new Consumer($this->feed, 'two', $this->cursor); @@ -345,8 +349,8 @@ public function testExposesItsName(): void */ public function testAFailedReadLeavesThePositionAlone(): void { - $first = $this->feed->append('a'); - $this->feed->append('b'); + $first = $this->producer->append('a'); + $this->producer->append('b'); $this->cursor->save('edge', 'invalidator', $first); $consumer = new Consumer(new Feed(new \Utopia\Feed\Journal\None('edge')), 'invalidator', $this->cursor); diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 07186c1..1021574 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -11,6 +11,7 @@ use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; +use Utopia\Feed\Producer; use Utopia\Feed\Id; class FeedTest extends TestCase @@ -19,22 +20,18 @@ class FeedTest extends TestCase private Feed $feed; + private Producer $producer; + protected function setUp(): void { $this->journal = new Memory('edge'); - $this->feed = new Feed($this->journal, 'urn:appwrite:cloud:fra'); - } - - public function testAppendReturnsAPosition(): void - { - $id = $this->feed->append('io.appwrite.edge.invalidate', ['tags' => ['project' => 'p1']]); - - $this->assertTrue(Id::isValid($id)); + $this->producer = new Producer($this->journal, 'urn:appwrite:cloud:fra'); + $this->feed = new Feed($this->journal); } public function testReadsBackWhatWasAppended(): void { - $this->feed->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + $this->producer->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); $events = $this->feed->read(); @@ -44,35 +41,10 @@ public function testReadsBackWhatWasAppended(): void $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); } - public function testStampsTheSourceAndTimeOnAppend(): void - { - $this->feed->append('test'); - - $event = $this->feed->read()[0]; - - $this->assertSame('urn:appwrite:cloud:fra', $event->source); - $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', $event->time); - } - - /** - * Recording it at append rather than at read keeps it correct for a feed - * read back somewhere other than where it was written. - */ - public function testKeepsTheSourceOfTheProducerThatAppended(): void - { - (new Feed($this->journal, 'urn:appwrite:cloud:fra'))->append('test'); - (new Feed($this->journal, 'urn:appwrite:cloud:nyc'))->append('test'); - - $events = (new Feed($this->journal, 'urn:appwrite:cloud:syd'))->read(); - - $this->assertSame('urn:appwrite:cloud:fra', $events[0]->source); - $this->assertSame('urn:appwrite:cloud:nyc', $events[1]->source); - } - public function testEventsComeBackOldestFirst(): void { foreach (['a', 'b', 'c'] as $type) { - $this->feed->append($type); + $this->producer->append($type); } $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $this->feed->read())); @@ -82,7 +54,7 @@ public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void { $ids = []; for ($i = 0; $i < 50; $i++) { - $ids[] = $this->feed->append('test'); + $ids[] = $this->producer->append('test'); } $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); @@ -94,8 +66,8 @@ public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void public function testReadsStrictlyAfterTheGivenPosition(): void { - $first = $this->feed->append('a'); - $this->feed->append('b'); + $first = $this->producer->append('a'); + $this->producer->append('b'); $events = $this->feed->read($first); @@ -105,16 +77,16 @@ public function testReadsStrictlyAfterTheGivenPosition(): void public function testReadFromTheLastEventIsEmpty(): void { - $this->feed->append('a'); - $last = $this->feed->append('b'); + $this->producer->append('a'); + $last = $this->producer->append('b'); $this->assertSame([], $this->feed->read($last)); } public function testNullPositionReadsFromTheOldestRetainedEvent(): void { - $this->feed->append('a'); - $this->feed->append('b'); + $this->producer->append('a'); + $this->producer->append('b'); $this->assertCount(2, $this->feed->read(null)); } @@ -122,7 +94,7 @@ public function testNullPositionReadsFromTheOldestRetainedEvent(): void public function testHonoursTheLimit(): void { foreach (\range(1, 10) as $i) { - $this->feed->append('test'); + $this->producer->append('test'); } $this->assertCount(3, $this->feed->read(null, 3)); @@ -135,7 +107,7 @@ public function testHonoursTheLimit(): void */ public function testClampsTheLimitToTheMaximum(): void { - $this->feed->append('test'); + $this->producer->append('test'); $this->assertCount(1, $this->feed->read(null, Feed::MAX_BATCH * 10)); $this->assertCount(1, $this->feed->read(null, 0)); @@ -149,53 +121,6 @@ public function testRejectsAPositionThatIsNotAFeedId(): void $this->feed->read('not-a-position'); } - /** - * CloudEvents requires a source, and a feed that stamped an empty one would - * produce events no consumer can attribute β€” so it is refused rather than - * appended. - */ - public function testRejectsAnAppendToAFeedWithNoSource(): void - { - $feed = new Feed(new Memory('edge')); - - $this->expectException(Invalid::class); - - $feed->append('test'); - } - - public function testRejectsAnEmptyEventType(): void - { - $this->expectException(Invalid::class); - - $this->feed->append(''); - } - - public function testRejectsAPayloadThatCannotBeEncoded(): void - { - $this->expectException(Invalid::class); - - $this->feed->append('test', ['resource' => \fopen('php://memory', 'r')]); - } - - public function testPublishStampsAPreparedEvent(): void - { - $id = $this->feed->publish(new CloudEvent(id: 'ignored', type: 'test', data: ['a' => 'b'], subject: 's')); - - $event = $this->feed->read()[0]; - - $this->assertSame($id, $event->id); - $this->assertNotSame('ignored', $event->id, 'The backend assigns the position, not the caller'); - $this->assertSame('urn:appwrite:cloud:fra', $event->source); - $this->assertSame(['a' => 'b'], $event->data); - } - - public function testPublishKeepsATimeTheCallerSet(): void - { - $this->feed->publish(new CloudEvent(id: '', type: 'test', time: '2020-01-01T00:00:00.000Z')); - - $this->assertSame('2020-01-01T00:00:00.000Z', $this->feed->read()[0]->time); - } - /** * A producer that attaches a `traceparent` means it to reach the consumer. * Stamping the event on publish rebuilds it, and storing it flattens it, so @@ -203,7 +128,7 @@ public function testPublishKeepsATimeTheCallerSet(): void */ public function testExtensionAttributesSurviveAppendAndRead(): void { - $this->feed->publish(new CloudEvent( + $this->producer->publish(new CloudEvent( id: '', type: 'test', extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2], @@ -213,7 +138,6 @@ public function testExtensionAttributesSurviveAppendAndRead(): void $this->assertSame('00-abc-def-01', $event->getExtension('traceparent')); $this->assertSame(2, $event->getExtension('retrycount')); - $this->assertSame('urn:appwrite:cloud:fra', $event->source, 'Stamping still happened'); } /** @@ -224,7 +148,7 @@ public function testExtensionAttributesSurviveAppendAndRead(): void */ public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void { - $this->feed->publish(new CloudEvent( + $this->producer->publish(new CloudEvent( id: '', type: 'test', extensions: ['123' => 'digits', 'trace' => 'ok'], @@ -238,7 +162,7 @@ public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void public function testDataschemaSurvivesAppendAndRead(): void { - $this->feed->publish(new CloudEvent( + $this->producer->publish(new CloudEvent( id: '', type: 'test', dataschema: 'https://example.com/schema.json', @@ -273,7 +197,7 @@ public static function payloads(): array */ public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void { - $this->feed->append('test', $data); + $this->producer->append('test', $data); $this->assertSame($data, $this->feed->read()[0]->data); } @@ -284,21 +208,21 @@ public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void */ public function testAnEventWithNoSubjectHasANullSubject(): void { - $this->feed->append('test'); + $this->producer->append('test'); $this->assertNull($this->feed->read()[0]->subject); } public function testASubjectSurvivesAppendAndRead(): void { - $this->feed->append('test', [], 'example.com'); + $this->producer->append('test', [], 'example.com'); $this->assertSame('example.com', $this->feed->read()[0]->subject); } public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void { - $this->feed->append('test'); + $this->producer->append('test'); $started = \microtime(true); $events = $this->feed->poll(null, 10, 2000); @@ -328,10 +252,12 @@ public function testPollWithoutATimeoutIsAPlainRead(): void public function testRetentionIsBoundedAndTrimsTheOldest(): void { - $feed = new Feed(new Memory('small', maxSize: 3), 'urn:appwrite:cloud:fra'); + $journal = new Memory('small', maxSize: 3); + $producer = new Producer($journal, 'urn:appwrite:cloud:fra'); + $feed = new Feed($journal); foreach (['a', 'b', 'c', 'd', 'e'] as $type) { - $feed->append($type); + $producer->append($type); } $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read())); @@ -343,11 +269,13 @@ public function testRetentionIsBoundedAndTrimsTheOldest(): void */ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void { - $feed = new Feed(new Memory('small', maxSize: 2), 'urn:appwrite:cloud:fra'); + $journal = new Memory('small', maxSize: 2); + $producer = new Producer($journal, 'urn:appwrite:cloud:fra'); + $feed = new Feed($journal); - $first = $feed->append('a'); - $feed->append('b'); - $feed->append('c'); + $first = $producer->append('a'); + $producer->append('b'); + $producer->append('c'); $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read($first))); } @@ -357,15 +285,6 @@ public function testExposesTheFeedItReads(): void $this->assertSame('edge', $this->feed->getName()); } - public function testAFeedWithNoBackendFailsLoudlyRatherThanDroppingEvents(): void - { - $feed = new Feed(new None('edge'), 'urn:appwrite:cloud:fra'); - - $this->expectException(Unsupported::class); - - $feed->append('test'); - } - public function testAFeedWithNoBackendCannotBeRead(): void { $feed = new Feed(new None('edge')); @@ -384,10 +303,12 @@ public function testRejectsAnEmptyFeedName(): void public function testAcceptsTheSmallestUsefulRetentionCap(): void { - $feed = new Feed(new Memory('edge', maxSize: 1), 'urn:appwrite:cloud:fra'); + $journal = new Memory('edge', maxSize: 1); + $producer = new Producer($journal, 'urn:appwrite:cloud:fra'); + $feed = new Feed($journal); - $feed->append('a'); - $feed->append('b'); + $producer->append('a'); + $producer->append('b'); $events = $feed->read(); diff --git a/tests/Feed/Unit/HttpJournalTest.php b/tests/Feed/Unit/HttpJournalTest.php index c243062..896a177 100644 --- a/tests/Feed/Unit/HttpJournalTest.php +++ b/tests/Feed/Unit/HttpJournalTest.php @@ -13,7 +13,6 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; -use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; use Utopia\Feed\Protocol; use Utopia\Tests\Unit\Support\FakeTransport; @@ -219,16 +218,6 @@ public function testRejectsABodyThatIsNotABatch(): void $feed->read(); } - public function testCannotAppendToAFeedItDoesNotOwn(): void - { - $journal = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', 'edge'); - $feed = new Feed($journal, 'urn:appwrite:edge:fra'); - - $this->expectException(Unsupported::class); - - $feed->append('io.appwrite.edge.invalidate'); - } - /** * Anything implementing the client's adapter interface works, including * the client itself wrapping a transport β€” which is how this is actually diff --git a/tests/Feed/Unit/ProducerTest.php b/tests/Feed/Unit/ProducerTest.php new file mode 100644 index 0000000..8906483 --- /dev/null +++ b/tests/Feed/Unit/ProducerTest.php @@ -0,0 +1,133 @@ +journal = new Memory('edge'); + $this->producer = new Producer($this->journal, 'urn:appwrite:cloud:fra'); + $this->feed = new Feed($this->journal); + } + + public function testAppendReturnsAPosition(): void + { + $id = $this->producer->append('io.appwrite.edge.invalidate', ['tags' => ['project' => 'p1']]); + + $this->assertTrue(Id::isValid($id)); + } + + public function testStampsTheSourceAndTimeOnAppend(): void + { + $this->producer->append('test'); + + $event = $this->feed->read()[0]; + + $this->assertSame('urn:appwrite:cloud:fra', $event->source); + $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', $event->time); + } + + /** + * Recording it at append rather than at read keeps it correct for a feed + * read back somewhere other than where it was written. + */ + public function testKeepsTheSourceOfTheProducerThatAppended(): void + { + (new Producer($this->journal, 'urn:appwrite:cloud:fra'))->append('test'); + (new Producer($this->journal, 'urn:appwrite:cloud:nyc'))->append('test'); + + $events = $this->feed->read(); + + $this->assertSame('urn:appwrite:cloud:fra', $events[0]->source); + $this->assertSame('urn:appwrite:cloud:nyc', $events[1]->source); + } + + public function testPublishStampsAPreparedEvent(): void + { + $id = $this->producer->publish(new CloudEvent(id: 'ignored', type: 'test', data: ['a' => 'b'], subject: 's')); + + $event = $this->feed->read()[0]; + + $this->assertSame($id, $event->id); + $this->assertNotSame('ignored', $event->id, 'The backend assigns the position, not the caller'); + $this->assertSame('urn:appwrite:cloud:fra', $event->source); + $this->assertSame(['a' => 'b'], $event->data); + } + + public function testPublishKeepsATimeTheCallerSet(): void + { + $this->producer->publish(new CloudEvent(id: '', type: 'test', time: '2020-01-01T00:00:00.000Z')); + + $this->assertSame('2020-01-01T00:00:00.000Z', $this->feed->read()[0]->time); + } + + public function testRejectsAnEmptyEventType(): void + { + $this->expectException(Invalid::class); + + $this->producer->append(''); + } + + public function testRejectsAPayloadThatCannotBeEncoded(): void + { + $this->expectException(Invalid::class); + + $this->producer->append('test', ['resource' => \fopen('php://memory', 'r')]); + } + + /** + * CloudEvents requires a source, and an event stamped with an empty one is + * an event no consumer can attribute. + */ + public function testRejectsAnEmptySource(): void + { + $this->expectException(Invalid::class); + + new Producer($this->journal, ''); + } + + /** + * A feed read over HTTP belongs to whoever appends to it, so it is not + * Appendable β€” the mistake is a type error at construction rather than an + * exception once an event is already in hand. + */ + public function testAJournalThatCannotBeAppendedToIsRejectedOnConstruction(): void + { + $journal = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', 'edge'); + + $this->expectException(\TypeError::class); + + // @phpstan-ignore argument.type + new Producer($journal, 'urn:appwrite:edge:fra'); + } + + public function testAFeedWithNoBackendFailsLoudlyRatherThanDroppingEvents(): void + { + $producer = new Producer(new None('edge'), 'urn:appwrite:cloud:fra'); + + $this->expectException(Unsupported::class); + + $producer->append('test'); + } +} diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index 6a61cae..2150022 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -13,6 +13,7 @@ use Utopia\Feed\Cursor\Cache as CacheCursor; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Feed; +use Utopia\Feed\Producer; use Utopia\Feed\Protocol; use Utopia\Tests\Unit\Support\FeedServer; @@ -24,7 +25,7 @@ */ class RoundTripTest extends TestCase { - private Feed $producer; + private Producer $producer; private Feed $consumerFeed; @@ -34,8 +35,9 @@ class RoundTripTest extends TestCase protected function setUp(): void { - $this->producer = new Feed(new MemoryJournal('edge'), 'urn:appwrite:cloud:fra'); - $this->server = new FeedServer($this->producer); + $journal = new MemoryJournal('edge'); + $this->producer = new Producer($journal, 'urn:appwrite:cloud:fra'); + $this->server = new FeedServer(new Feed($journal)); $this->consumerFeed = new Feed( new Http($this->server, 'https://cloud.example.com/v1/feeds', 'edge') From f9501c5c279b3fbdfea02b8dbddc16bbc5f218f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 18:37:21 +0200 Subject: [PATCH 20/68] Fix linter --- CHANGELOG.md | 5 +++- README.md | 38 +++++++++++++---------- src/Feed/Protocol.php | 2 +- tests/Feed/E2E/RedisTest.php | 58 ++++++++++++++++++++++-------------- 4 files changed, 63 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b2db25..733a3a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,14 @@ Initial release. -- `Feed` β€” append, read and long-poll an ordered event log +- `Producer` β€” appends events to a feed this service owns +- `Feed` β€” reads and long-polls a feed, local or remote - Events are [utopia-php/cloudevents](https://github.com/utopia-php/cloudevents) `CloudEvent` objects β€” this library defines no event type of its own, so a feed event is accepted anywhere a `CloudEvent` is, and `dataschema` and extension attributes survive an append and a read +- `Appendable` β€” the journals that own their events and can be appended to; + `Journal\Http` deliberately does not implement it - `Journal\Redis`, `Journal\Pool` β€” Redis streams, directly or over a pool - `Journal\Http` β€” another service's feed, read over the wire with [utopia-php/client](https://github.com/utopia-php/client); takes any of its diff --git a/README.md b/README.md index ca35bb2..a2a908f 100644 --- a/README.md +++ b/README.md @@ -33,15 +33,15 @@ composer require utopia-php/feed ### Produce ```php -use Utopia\Feed\Feed; use Utopia\Feed\Journal; +use Utopia\Feed\Producer; -$feed = new Feed( +$producer = new Producer( new Journal\Redis($redis, 'edge'), source: 'urn:appwrite:cloud:fra', ); -$id = $feed->append( +$id = $producer->append( type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']], subject: 'example.com', @@ -50,11 +50,11 @@ $id = $feed->append( `append()` returns the event's id, which is its position in the feed. -Subclass `Feed` to give it a typed vocabulary, so callers cannot invent an event -type or misspell a payload key: +Subclass `Producer` to give it a typed vocabulary, so callers cannot invent an +event type or misspell a payload key: ```php -class EdgeFeed extends Feed +class EdgeProducer extends Producer { public function invalidateRule(string $domain): string { @@ -67,6 +67,10 @@ class EdgeFeed extends Feed } ``` +`Producer` only accepts a journal that owns its events (one implementing +`Appendable`), so pointing it at a remote feed is a type error rather than a +runtime surprise. + ### Consume A `Consumer` reads from where it last got to, hands each new event to your @@ -123,10 +127,14 @@ retry. `Protocol` holds the wire contract β€” query parameters, response body, caching rules β€” and deals in arrays, so it fits whichever HTTP layer you use: +The feed a service serves is the read half of the same journal it appends to: + ```php use Utopia\Feed\Feed; use Utopia\Feed\Protocol; +$feed = new Feed($journal); // the same journal the Producer was built on + // GET /v1/feeds/:feedId $limit = \min((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH), Feed::MAX_BATCH); @@ -170,16 +178,16 @@ an attribute or moves the spec forward must not stop a consumer that predates it ## Journals -A journal is where a feed's events live. It assigns an ordered id on append and -returns the events after a given id; everything else sits above it. +A journal is where a feed's events live. It returns the events after a given id; +the ones that own their events also implement `Appendable` and assign the ids. -| Journal | Use for | Append | Read | -| --- | --- | --- | --- | -| `Journal\Redis` | Producing a feed on a Redis stream | βœ… | βœ… | -| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | βœ… | -| `Journal\Http` | Consuming another service's feed | ❌ | βœ… | -| `Journal\Memory` | Tests and single-process development | βœ… | βœ… | -| `Journal\None` | No backend configured β€” throws on use | ❌ | ❌ | +| Journal | Use for | `Appendable` | +| --- | --- | --- | +| `Journal\Redis` | Producing a feed on a Redis stream | βœ… | +| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | +| `Journal\Http` | Consuming another service's feed | ❌ β€” it belongs to whoever appends to it | +| `Journal\Memory` | Tests and single-process development | βœ… | +| `Journal\None` | No backend configured β€” throws on use | βœ…, and throws | `Journal\Pool` is what most services producing a feed want: a long poll holds its connection for the whole timeout, so reading through a shared client would block diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 00df42b..589f400 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -20,7 +20,7 @@ final class Protocol public const string CACHE_IMMUTABLE = 'max-age=31536000'; public const string CACHE_NONE = 'no-store'; - + public const int TIMEOUT_MARGIN = 10_000; /** diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index baecc23..ab78922 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -58,11 +58,24 @@ private function producer(int $maxSize = 100_000): Producer return new Producer(new RedisJournal($this->redis, $this->name, $maxSize), 'urn:test:e2e'); } + /** + * The two halves of one feed: what a producing service builds over a single + * journal to append to its feed and serve it. + * + * @return array{Producer, Feed} + */ + private function feedAndProducer(int $maxSize = 100_000): array + { + $journal = new RedisJournal($this->redis, $this->name, $maxSize); + + return [new Producer($journal, 'urn:test:e2e'), new Feed($journal)]; + } + public function testAppendsAndReadsBack(): void { - $feed = $this->feed(); + [$producer, $feed] = $this->feedAndProducer(); - $id = $feed->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + $id = $producer->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); $events = $feed->read(); @@ -84,11 +97,11 @@ public function testStreamIdsMatchTheFormatPositionsAreParsedWith(): void public function testIdsIncreaseAcrossRapidAppends(): void { - $feed = $this->feed(); + [$producer, $feed] = $this->feedAndProducer(); $ids = []; for ($i = 0; $i < 100; $i++) { - $ids[] = $feed->append('test'); + $ids[] = $producer->append('test'); } $this->assertSame($ids, \array_unique($ids)); @@ -105,10 +118,10 @@ public function testIdsIncreaseAcrossRapidAppends(): void */ public function testReadsStrictlyAfterAPosition(): void { - $feed = $this->feed(); + [$producer, $feed] = $this->feedAndProducer(); - $first = $feed->append('a'); - $second = $feed->append('b'); + $first = $producer->append('a'); + $second = $producer->append('b'); $events = $feed->read($first); @@ -162,10 +175,10 @@ public function testNestedPayloadsSurviveTheRoundTrip(): void public function testHonoursTheLimit(): void { - $feed = $this->feed(); + [$producer, $feed] = $this->feedAndProducer(); foreach (\range(1, 10) as $i) { - $feed->append('test'); + $producer->append('test'); } $this->assertCount(3, $feed->read(null, 3)); @@ -185,13 +198,12 @@ public function testRejectsAPositionThatIsNotAFeedId(): void */ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void { - $feed = $this->feed(maxSize: 10); - $producer = $this->producer(maxSize: 10); + [$producer, $feed] = $this->feedAndProducer(maxSize: 10); - $first = $feed->append('first'); + $first = $producer->append('first'); foreach (\range(1, 500) as $i) { - $feed->append('event-' . $i); + $producer->append('event-' . $i); } $events = $feed->read($first); @@ -202,8 +214,8 @@ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void public function testLongPollingReturnsAsSoonAsTheFeedHasSomething(): void { - $feed = $this->feed(); - $feed->append('a'); + [$producer, $feed] = $this->feedAndProducer(); + $producer->append('a'); $started = \microtime(true); $events = $feed->poll(null, 10, 3000); @@ -223,11 +235,11 @@ public function testLongPollingGivesUpAtTheTimeout(): void public function testConsumesThroughAPersistedCursor(): void { - $feed = $this->feed(); + [$producer, $feed] = $this->feedAndProducer(); $cursor = new RedisCursor($this->redis); - $feed->append('a'); - $last = $feed->append('b'); + $producer->append('a'); + $last = $producer->append('b'); $seen = []; $handler = function (CloudEvent $event) use (&$seen): void { @@ -245,10 +257,10 @@ public function testConsumesThroughAPersistedCursor(): void public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void { - $feed = $this->feed(); + [$producer, $feed] = $this->feedAndProducer(); $cursor = new RedisCursor($this->redis); - $feed->append('a'); + $producer->append('a'); $this->assertSame(1, (new Consumer($feed, 'one', $cursor))->consume(fn (CloudEvent $e) => null)); $this->assertSame(1, (new Consumer($feed, 'two', $cursor))->consume(fn (CloudEvent $e) => null)); @@ -256,11 +268,11 @@ public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void public function testResetReplaysTheRetainedFeed(): void { - $feed = $this->feed(); + [$producer, $feed] = $this->feedAndProducer(); $cursor = new RedisCursor($this->redis); - $feed->append('a'); - $feed->append('b'); + $producer->append('a'); + $producer->append('b'); $consumer = new Consumer($feed, 'invalidator', $cursor); $consumer->consume(fn (CloudEvent $e) => null); From a1c5b2bec71a77bbb790c7acbcff5d28094b481b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 Jul 2026 19:16:14 +0200 Subject: [PATCH 21/68] Update docs behaviour --- README.md | 36 +++++++++++++++++++++++++++--------- src/Feed/Cursor.php | 3 ++- src/Feed/Cursor/Redis.php | 3 --- src/Feed/Journal.php | 12 +++++++++--- src/Feed/Protocol.php | 10 ++++++++++ 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index a2a908f..dec8d01 100644 --- a/README.md +++ b/README.md @@ -151,9 +151,11 @@ $response Cap the limit yourself with `Feed::MAX_BATCH` before the call, so the number that reaches `cacheControl()` is the one the batch was actually built with β€” a -read never returns more than that cap anyway. A full batch is settled -history and is marked cacheable; a short one is the live end of the feed and is -marked `no-store`. Caching is `private` unless you pass `public: true`. +read never returns more than that cap anyway. `Feed` clamps the wait too, to +`Feed::MAX_TIMEOUT` (30s), so a client cannot ask a producer to hold a +connection open for as long as it likes. A full batch is settled history and is +marked cacheable; a short one is the live end of the feed and is marked +`no-store`. Caching is `private` unless you pass `public: true`. ## Events @@ -176,6 +178,11 @@ A batch is decoded strictly about `id`, because for a feed the id *is* the consumer's position, and leniently about everything else β€” a producer that adds an attribute or moves the spec forward must not stop a consumer that predates it. +An entry that cannot be read at all ends the batch where it sits: the events +before it are returned and handled, and the broken one heads the next batch, +where it stops the feed loudly. Only when it is the first entry β€” leaving no +usable prefix β€” does the read throw `Exception\Invalid`. + ## Journals A journal is where a feed's events live. It returns the events after a given id; @@ -189,9 +196,15 @@ the ones that own their events also implement `Appendable` and assign the ids. | `Journal\Memory` | Tests and single-process development | βœ… | | `Journal\None` | No backend configured β€” throws on use | βœ…, and throws | -`Journal\Pool` is what most services producing a feed want: a long poll holds its -connection for the whole timeout, so reading through a shared client would block -every other user of it. +`Journal\Pool` is what most services producing a feed want: a long poll spans its +whole timeout, and this one borrows a connection per read and gives it back while +it waits, so polling never ties up the client the rest of the service is using. + +`Journal\Redis` and `Journal\Pool` trim the stream to about `maxSize` entries +(100,000 by default, and the same for `Journal\Memory`; Redis trims +approximately, so the stream may run a little longer). That cap is the feed's +retention: the oldest entry still in it is where a consumer with no position +starts. `Journal\None` throws on every operation rather than doing nothing, so a misconfigured service fails at the point of use instead of silently dropping @@ -212,6 +225,10 @@ consumer name, so a single store serves every feed a service consumes: | `Cursor\Memory` | Tests, or a consumer that should replay from the beginning on every restart | | `Cursor\None` | No store configured β€” remembers nothing, so a restart replays | +`Cursor\Cache` holds a position for `Cursor\Cache::TTL` (30 days) unless it is +saved again, so a consumer idle for longer than that reads back as one that has +never run. + The store is allowed to be lossy: a lost position costs a replay, not a gap. A store that is *down* is a different matter β€” the failure surfaces from `consume()` as a `Transport` exception rather than being swallowed, so catch it @@ -252,7 +269,7 @@ the backlog. ## Rolling out a feed -Replacing push delivery with a feed is a two-release change: +Replacing push delivery with a feed is a staged change, one release per step: 1. **Release the producer.** It appends events; nothing reads them yet. 2. **Release the consumers.** Each drains the backlog from its first poll. @@ -277,10 +294,11 @@ try { ## Tests -Unit tests need nothing but composer: +Unit tests need no services, but dependencies declare extensions the suite never +touches (`ext-redis`, `ext-memcached`, `ext-protobuf`), so install past them: ```bash -composer install +composer install --ignore-platform-reqs composer test ``` diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 927a4db..d1c8250 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -26,7 +26,8 @@ abstract public function save(string $feed, string $consumer, string $eventId): abstract public function reset(string $feed, string $consumer): void; /** - * Helper tool for consistency + * The one place a cursor key is shaped, so every store agrees on it. + * * @throws Invalid When either name is empty. */ protected function key(string $feed, string $consumer): string diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 07bf096..828f8b1 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -9,9 +9,6 @@ class Redis extends Cursor { - /** - * @param \Redis|\RedisCluster $redis - */ public function __construct(protected readonly \Redis|\RedisCluster $redis) { } diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index c5610e6..c98c0e8 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -9,8 +9,8 @@ use Utopia\Feed\Exception\Invalid; // Server class: durable storage for the events β€” Journal\Redis, Pool, Memory. -// Client exception: Journal\Http, which reads another service's feed over the wire. -// Journals that own their events also implement Appendable. +// The one client-side journal is Journal\Http, which reads another service's +// feed over the wire. Journals that own their events also implement Appendable. abstract class Journal { protected const int POLL_INTERVAL = 500_000; // 0.5s @@ -30,7 +30,13 @@ public function getName(): string /** @return list */ abstract public function read(?string $lastEventId, int $limit): array; - /** @return list */ + /** + * Wait for events, re-reading on an interval until some land or the + * deadline passes. Journal\Http overrides this: there the producer does the + * waiting, so a poll is one held request. + * + * @return list + */ public function poll(?string $lastEventId, int $limit, int $timeout): array { $deadline = \microtime(true) + $timeout / 1000; diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 589f400..0b99d93 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -58,7 +58,17 @@ public static function encode(array $events): array } /** + * Read a batch off the wire. + * + * An entry that cannot be read ends the batch there rather than failing the + * whole response: the events before it are handled and the position + * advances past them, leaving the broken entry at the head of the next + * batch, where it stops the feed loudly. With no usable prefix there is + * nothing to advance to, so that case throws. + * * @return list + * + * @throws Invalid When the payload is not a batch, or its first entry cannot be read. */ public static function decode(mixed $payload): array { From 052b72e2334759b72299a97956efaaeb02164a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 30 Jul 2026 13:19:36 +0200 Subject: [PATCH 22/68] Upgrade to latest cloudevent lib --- README.md | 2 +- composer.json | 2 +- composer.lock | 14 +++--- src/Feed/Journal.php | 23 ++++----- src/Feed/Producer.php | 28 +++++++++-- src/Feed/Protocol.php | 73 ++++++++++++++++++++++++++--- tests/Feed/E2E/RedisTest.php | 5 +- tests/Feed/Unit/FeedTest.php | 13 +++-- tests/Feed/Unit/HttpJournalTest.php | 12 ++--- tests/Feed/Unit/ProducerTest.php | 5 +- tests/Feed/Unit/ProtocolTest.php | 6 +-- 11 files changed, 136 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index dec8d01..02c91e2 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ objects β€” this library defines no event type of its own: ```php $consumer->consume(function (CloudEvent $event) { $tags = $event->data['tags'] ?? []; - $trace = $event->getExtension('traceparent'); + $trace = $event->extensions['traceparent'] ?? null; }); ``` diff --git a/composer.json b/composer.json index ef80187..ef0c7f9 100644 --- a/composer.json +++ b/composer.json @@ -42,7 +42,7 @@ "utopia-php/client": "^0.2", "utopia-php/pools": "1.*", "utopia-php/psr7": "^0.2", - "utopia-php/cloudevents": "dev-feat-cloudevents-g2" + "utopia-php/cloudevents": "dev-feat-constructor-validation" }, "require-dev": { "laravel/pint": "^1.21", diff --git a/composer.lock b/composer.lock index edd6adf..433413d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "47e874a6ca9e914a88ab31627d42e31b", + "content-hash": "64da6e0642c2891229b50497a4cfef29", "packages": [ { "name": "brick/math", @@ -2052,16 +2052,16 @@ }, { "name": "utopia-php/cloudevents", - "version": "dev-feat-cloudevents-g2", + "version": "dev-feat-constructor-validation", "source": { "type": "git", "url": "https://github.com/utopia-php/cloudevents.git", - "reference": "68cd50590b94e250970f226a0067caf471adf42e" + "reference": "8a89ab2365a70919f2e9a4c3742c28ac9020fb57" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cloudevents/zipball/68cd50590b94e250970f226a0067caf471adf42e", - "reference": "68cd50590b94e250970f226a0067caf471adf42e", + "url": "https://api.github.com/repos/utopia-php/cloudevents/zipball/8a89ab2365a70919f2e9a4c3742c28ac9020fb57", + "reference": "8a89ab2365a70919f2e9a4c3742c28ac9020fb57", "shasum": "" }, "require": { @@ -2098,9 +2098,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cloudevents/issues", - "source": "https://github.com/utopia-php/cloudevents/tree/feat-cloudevents-g2" + "source": "https://github.com/utopia-php/cloudevents/tree/feat-constructor-validation" }, - "time": "2026-07-29T13:53:17+00:00" + "time": "2026-07-30T11:07:39+00:00" }, { "name": "utopia-php/pools", diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index c98c0e8..3ea3581 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -5,7 +5,6 @@ namespace Utopia\Feed; use Utopia\CloudEvents\CloudEvent; -use Utopia\CloudEvents\Exception as CloudEventsException; use Utopia\Feed\Exception\Invalid; // Server class: durable storage for the events β€” Journal\Redis, Pool, Memory. @@ -58,14 +57,14 @@ protected static function encode(CloudEvent $event): array return [ 'type' => $event->type, 'source' => $event->source, - // CloudEvents models an absent subject and dataschema as null, and - // a backend field cannot hold one, so both are normalized here and - // read back as absent in decode(). + // CloudEvents models an absent subject, dataschema and time as + // null, and a backend field cannot hold one, so they are + // normalized here and read back as absent in decode(). 'subject' => $event->subject ?? '', 'dataschema' => $event->dataschema ?? '', - 'time' => $event->time, + 'time' => $event->time ?? '', 'data' => self::json($event->data, 'data'), - 'extensions' => self::json($event->getExtensions(), 'extensions'), + 'extensions' => self::json($event->extensions, 'extensions'), ]; } @@ -75,15 +74,14 @@ protected static function decode(string $id, array $fields): CloudEvent $extensions = \json_decode(self::field($fields, 'extensions'), true); $event = [ - 'specversion' => CloudEvent::SPECVERSION, + 'specversion' => '1.0', 'id' => $id, 'type' => self::field($fields, 'type'), 'source' => self::field($fields, 'source'), - 'time' => self::field($fields, 'time'), 'data' => \json_decode(self::field($fields, 'data'), true), ]; - foreach (['subject', 'dataschema'] as $optional) { + foreach (['subject', 'dataschema', 'time'] as $optional) { $value = self::field($fields, $optional); if ($value !== '') { @@ -94,8 +92,11 @@ protected static function decode(string $id, array $fields): CloudEvent $event += \is_array($extensions) ? $extensions : []; try { - return CloudEvent::fromArray($event, lenient: true); - } catch (CloudEventsException $error) { + // The docblock wants array, but a digit-only + // extension name β€” legal per the spec β€” is an integer key in PHP. + // @phpstan-ignore argument.type + return CloudEvent::fromArray($event); + } catch (\InvalidArgumentException $error) { throw new Invalid("Feed entry {$id} could not be read as an event: {$error->getMessage()}", previous: $error); } } diff --git a/src/Feed/Producer.php b/src/Feed/Producer.php index e865e75..18f98fc 100644 --- a/src/Feed/Producer.php +++ b/src/Feed/Producer.php @@ -41,6 +41,8 @@ public function append(string $type, mixed $data = [], string $subject = ''): st { return $this->publish(new CloudEvent( type: $type, + source: $this->source, + id: '', subject: $subject === '' ? null : $subject, data: $data, )); @@ -59,10 +61,28 @@ public function publish(CloudEvent $event): string throw new Exception\Invalid('Feed event type is required'); } - // Stamped with the withers rather than rebuilt, so anything this library - // does not model β€” a dataschema, a traceparent β€” survives untouched. - $event = $event->withSource($this->source); + // Rebuilt attribute by attribute β€” extensions included β€” so anything + // this library does not model, a dataschema or a traceparent, survives + // untouched. + $event = new CloudEvent( + type: $event->type, + source: $this->source, + id: $event->id, + specversion: $event->specversion, + subject: $event->subject, + time: $event->time === null || $event->time === '' ? self::now() : $event->time, + datacontenttype: $event->datacontenttype, + data: $event->data, + dataschema: $event->dataschema, + extensions: $event->extensions, + ); - return $this->journal->append($event->time === '' ? $event->withTime() : $event); + return $this->journal->append($event); + } + + /** The current time in the RFC 3339 format the spec requires. */ + private static function now(): string + { + return (new \DateTimeImmutable('now', new \DateTimeZone('UTC')))->format('Y-m-d\TH:i:s.v\Z'); } } diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 0b99d93..37852db 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -5,7 +5,6 @@ namespace Utopia\Feed; use Utopia\CloudEvents\CloudEvent; -use Utopia\CloudEvents\Exception as CloudEventsException; use Utopia\Feed\Exception\Invalid; // HTTP shape of a feed, as defined by https://www.http-feeds.org/. @@ -23,6 +22,19 @@ final class Protocol public const int TIMEOUT_MARGIN = 10_000; + /** The context attributes this library models; the rest are extensions. */ + private const array ATTRIBUTES = [ + 'specversion', + 'type', + 'source', + 'id', + 'subject', + 'time', + 'datacontenttype', + 'dataschema', + 'data', + ]; + /** * @return array */ @@ -95,7 +107,7 @@ public static function decode(mixed $payload): array } $events[] = self::event($event); - } catch (Invalid | CloudEventsException $error) { + } catch (Invalid | \InvalidArgumentException $error) { if ($events === []) { throw $error instanceof Invalid ? $error @@ -119,16 +131,65 @@ public static function cacheControl(int $count, int $limit, bool $public = false } /** + * Read one event off the wire. + * + * Mapped by hand rather than through CloudEvent::fromArray(), which + * rejects a specversion it does not know. A feed is read by consumers + * older than its producer by design, so a producer that moved the spec + * version forward, or attached an attribute this library cannot model, + * must not stop one that predates it β€” what cannot be carried is dropped, + * not fatal. + * * @param array $raw */ private static function event(array $raw): CloudEvent { - $event = CloudEvent::fromArray($raw, lenient: true, allowUnknownSpecversion: true); + foreach (['specversion', 'id', 'type', 'source'] as $required) { + if (!isset($raw[$required]) || !\is_string($raw[$required]) || $raw[$required] === '') { + throw new Invalid('Feed event is missing ' . ($required === 'id' ? 'an id' : 'a ' . $required)); + } + } + + $extensions = []; + + /** @var mixed $value */ + foreach ($raw as $name => $value) { + if (\in_array($name, self::ATTRIBUTES, true)) { + continue; + } - if ($event->id === '') { - throw new Invalid('Feed event is missing an id'); + // Only what the CloudEvent constructor accepts as an extension β€” + // anything else would throw and stop the feed. + if (\preg_match('/^[a-z0-9]+$/', (string) $name) === 1 + && (\is_bool($value) || \is_int($value) || \is_string($value))) { + $extensions[$name] = $value; + } } - return $event; + return new CloudEvent( + type: $raw['type'], + source: $raw['source'], + id: $raw['id'], + specversion: $raw['specversion'], + subject: self::optional($raw, 'subject'), + time: self::optional($raw, 'time'), + datacontenttype: self::optional($raw, 'datacontenttype'), + data: $raw['data'] ?? null, + dataschema: self::optional($raw, 'dataschema'), + // The docblock wants array, but a digit-only + // extension name β€” legal per the spec β€” is an integer key in PHP. + // @phpstan-ignore argument.type + extensions: $extensions, + ); + } + + /** + * @param array $raw + */ + private static function optional(array $raw, string $attribute): ?string + { + $value = $raw[$attribute] ?? null; + + return \is_string($value) && $value !== '' ? $value : null; } } diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index ab78922..c7ca3f2 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -85,7 +85,7 @@ public function testAppendsAndReadsBack(): void $this->assertSame('example.com', $events[0]->subject); $this->assertSame('urn:test:e2e', $events[0]->source); $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); - $this->assertNotSame('', $events[0]->time); + $this->assertNotNull($events[0]->time); } public function testStreamIdsMatchTheFormatPositionsAreParsedWith(): void @@ -135,6 +135,7 @@ public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void $this->producer()->publish(new CloudEvent( id: '', type: 'test', + source: '', dataschema: 'https://example.com/schema.json', extensions: ['traceparent' => '00-abc-def-01'], )); @@ -142,7 +143,7 @@ public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void $event = $this->feed()->read()[0]; $this->assertSame('https://example.com/schema.json', $event->dataschema); - $this->assertSame('00-abc-def-01', $event->getExtension('traceparent')); + $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); } public function testAnAbsentSubjectStaysAbsent(): void diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 1021574..6707eac 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -131,13 +131,14 @@ public function testExtensionAttributesSurviveAppendAndRead(): void $this->producer->publish(new CloudEvent( id: '', type: 'test', + source: '', extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2], )); $event = $this->feed->read()[0]; - $this->assertSame('00-abc-def-01', $event->getExtension('traceparent')); - $this->assertSame(2, $event->getExtension('retrycount')); + $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); + $this->assertSame(2, $event->extensions['retrycount']); } /** @@ -151,13 +152,16 @@ public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void $this->producer->publish(new CloudEvent( id: '', type: 'test', + source: '', + // @phpstan-ignore argument.type ('123' is an integer key in PHP) extensions: ['123' => 'digits', 'trace' => 'ok'], )); $event = $this->feed->read()[0]; - $this->assertSame('digits', $event->getExtension('123')); - $this->assertSame('ok', $event->getExtension('trace')); + // @phpstan-ignore offsetAccess.notFound + $this->assertSame('digits', $event->extensions['123']); + $this->assertSame('ok', $event->extensions['trace']); } public function testDataschemaSurvivesAppendAndRead(): void @@ -165,6 +169,7 @@ public function testDataschemaSurvivesAppendAndRead(): void $this->producer->publish(new CloudEvent( id: '', type: 'test', + source: '', dataschema: 'https://example.com/schema.json', )); diff --git a/tests/Feed/Unit/HttpJournalTest.php b/tests/Feed/Unit/HttpJournalTest.php index 896a177..df3129c 100644 --- a/tests/Feed/Unit/HttpJournalTest.php +++ b/tests/Feed/Unit/HttpJournalTest.php @@ -34,8 +34,8 @@ private function feed(array $responses = []): array public function testReadsAFeedOverHttp(): void { [$feed] = $this->feed([FakeTransport::json(Protocol::encode([ - new CloudEvent(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']]), - new CloudEvent(id: '1-1', type: 'io.appwrite.edge.invalidate'), + new CloudEvent(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', source: 'urn:test', data: ['tags' => ['domain' => 'example.com']]), + new CloudEvent(id: '1-1', type: 'io.appwrite.edge.invalidate', source: 'urn:test'), ]))]); $events = $feed->read(); @@ -225,7 +225,7 @@ public function testRejectsABodyThatIsNotABatch(): void */ public function testWorksThroughTheClientItself(): void { - $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-0', type: 'a')]))]); + $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-0', type: 'a', source: 'urn:test')]))]); $client = (new Client($transport))->withHeaders(['x-appwrite-jwt' => 'token']); $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); @@ -244,10 +244,10 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void { [$feed, $transport] = $this->feed([ FakeTransport::json(Protocol::encode([ - new CloudEvent(id: '1-0', type: 'a'), - new CloudEvent(id: '1-1', type: 'b'), + new CloudEvent(id: '1-0', type: 'a', source: 'urn:test'), + new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), ])), - FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-2', type: 'c')])), + FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-2', type: 'c', source: 'urn:test')])), FakeTransport::json(Protocol::encode([])), ]); diff --git a/tests/Feed/Unit/ProducerTest.php b/tests/Feed/Unit/ProducerTest.php index 8906483..6a04de5 100644 --- a/tests/Feed/Unit/ProducerTest.php +++ b/tests/Feed/Unit/ProducerTest.php @@ -45,6 +45,7 @@ public function testStampsTheSourceAndTimeOnAppend(): void $event = $this->feed->read()[0]; $this->assertSame('urn:appwrite:cloud:fra', $event->source); + $this->assertNotNull($event->time); $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', $event->time); } @@ -65,7 +66,7 @@ public function testKeepsTheSourceOfTheProducerThatAppended(): void public function testPublishStampsAPreparedEvent(): void { - $id = $this->producer->publish(new CloudEvent(id: 'ignored', type: 'test', data: ['a' => 'b'], subject: 's')); + $id = $this->producer->publish(new CloudEvent(id: 'ignored', type: 'test', source: 'ignored', data: ['a' => 'b'], subject: 's')); $event = $this->feed->read()[0]; @@ -77,7 +78,7 @@ public function testPublishStampsAPreparedEvent(): void public function testPublishKeepsATimeTheCallerSet(): void { - $this->producer->publish(new CloudEvent(id: '', type: 'test', time: '2020-01-01T00:00:00.000Z')); + $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', time: '2020-01-01T00:00:00.000Z')); $this->assertSame('2020-01-01T00:00:00.000Z', $this->feed->read()[0]->time); } diff --git a/tests/Feed/Unit/ProtocolTest.php b/tests/Feed/Unit/ProtocolTest.php index a708d98..4004b39 100644 --- a/tests/Feed/Unit/ProtocolTest.php +++ b/tests/Feed/Unit/ProtocolTest.php @@ -31,7 +31,7 @@ public function testEncodesABatch(): void { $payload = Protocol::encode([ new CloudEvent(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), - new CloudEvent(id: '1-1', type: 'b'), + new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), ]); $this->assertSame(2, $payload['total']); @@ -50,7 +50,7 @@ public function testDecodesWhatItEncoded(): void { $events = [ new CloudEvent(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), - new CloudEvent(id: '1-1', type: 'b'), + new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), ]; $this->assertEquals($events, Protocol::decode(Protocol::encode($events))); @@ -194,7 +194,7 @@ public function testSurvivesAProducerThatMovedAhead(): void $this->assertCount(1, $events); $this->assertSame('1.1', $events[0]->specversion); - $this->assertSame('00-abc-def-01', $events[0]->getExtension('traceparent')); + $this->assertSame('00-abc-def-01', $events[0]->extensions['traceparent']); } /** From 31289d85a7b51ca1824fbabde5cb46a6c63f2ea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 10:57:40 +0200 Subject: [PATCH 23/68] Make the wire format comply with the http-feeds.org spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A batch on the wire is now the plain JSON array of CloudEvents the spec defines β€” the {total, events} envelope is gone. Protocol::encode() returns the bare array, Protocol::decode() expects one (keeping the per-entry leniency policy), and the HTTP journal asks for the spec's application/cloudevents-batch+json media type while staying tolerant of servers that answer application/json. The limit query parameter stays, documented as an extension beyond the spec, and an event carrying the spec's optional method attribute is pinned to decode into extensions. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 10 +++ README.md | 12 ++++ src/Feed/Journal/Http.php | 5 +- src/Feed/Protocol.php | 24 +++---- tests/Feed/Unit/HttpJournalTest.php | 7 +- tests/Feed/Unit/ProtocolTest.php | 81 ++++++++++++----------- tests/Feed/Unit/Support/FakeTransport.php | 4 +- tests/Feed/Unit/Support/FeedServer.php | 3 +- 8 files changed, 82 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 733a3a1..597c9bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +- **Breaking (wire format):** a feed batch on the wire is now the plain JSON + array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) + defines β€” the `{total, events}` envelope is gone, and an empty feed + serializes to `[]`. `Protocol::encode()` returns the bare array, + `Protocol::decode()` expects one, and the HTTP journal asks for the spec's + `application/cloudevents-batch+json` media type (`Protocol::MEDIA_TYPE`). + Both sides of a feed must move together. + ## 0.1.0 Initial release. diff --git a/README.md b/README.md index 02c91e2..a929221 100644 --- a/README.md +++ b/README.md @@ -145,10 +145,22 @@ $events = $feed->poll( ); $response + ->addHeader('Content-Type', Protocol::MEDIA_TYPE) ->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)) ->json(Protocol::encode($events)); ``` +The response body is a bare JSON array of CloudEvents, as +[http-feeds.org](https://www.http-feeds.org/) defines it β€” no envelope. An +empty array means the consumer is caught up. The media type is +`application/cloudevents-batch+json` (`Protocol::MEDIA_TYPE`); on receipt this +library only checks the body shape, so a feed answering `application/json` +still reads fine. + +The spec defines two query parameters: `lastEventId` and `timeout`. The +`limit` parameter is this library's extension beyond the spec β€” a +spec-compliant consumer simply never sends it, and gets full batches. + Cap the limit yourself with `Feed::MAX_BATCH` before the call, so the number that reaches `cacheControl()` is the one the batch was actually built with β€” a read never returns more than that cap anyway. `Feed` clamps the wait too, to diff --git a/src/Feed/Journal/Http.php b/src/Feed/Journal/Http.php index 80bdce2..ea20a2b 100644 --- a/src/Feed/Journal/Http.php +++ b/src/Feed/Journal/Http.php @@ -10,7 +10,6 @@ use Utopia\Feed\Exception\Transport; use Utopia\Feed\Journal; use Utopia\Feed\Protocol; -use Utopia\Psr7\ContentType; use Utopia\Psr7\Header; use Utopia\Psr7\Method; use Utopia\Psr7\Request\Factory as RequestFactory; @@ -46,11 +45,13 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array { $url = $this->url(); + // The Content-Type of the response is deliberately not checked: many + // servers answer application/json, and the body shape is what matters. $request = $this->requests->query( Method::GET, $url, Protocol::query($lastEventId, $limit, $timeout), - [Header::ACCEPT => ContentType::JSON], + [Header::ACCEPT => Protocol::MEDIA_TYPE], ); // A long poll needs a deadline past the one it asked the producer for, diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 37852db..1820357 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -14,8 +14,7 @@ final class Protocol public const string PARAM_LIMIT = 'limit'; public const string PARAM_TIMEOUT = 'timeout'; - public const string KEY_EVENTS = 'events'; - public const string KEY_TOTAL = 'total'; + public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; public const string CACHE_IMMUTABLE = 'max-age=31536000'; public const string CACHE_NONE = 'no-store'; @@ -58,15 +57,15 @@ public static function query(?string $lastEventId = null, int $limit = 0, int $t } /** + * A batch on the wire is a plain array of CloudEvents β€” no envelope. An + * empty feed serializes to `[]`, which the spec reads as "you are caught up". + * * @param list $events - * @return array{total: int, events: list>} + * @return list> */ public static function encode(array $events): array { - return [ - self::KEY_TOTAL => \count($events), - self::KEY_EVENTS => \array_map(static fn (CloudEvent $event): array => $event->toArray(), $events), - ]; + return \array_map(static fn (CloudEvent $event): array => $event->toArray(), $events); } /** @@ -88,19 +87,14 @@ public static function decode(mixed $payload): array throw new Invalid('Expected a feed batch, got ' . \get_debug_type($payload)); } - if (!\array_key_exists(self::KEY_EVENTS, $payload)) { - throw new Invalid('Feed batch is missing the "' . self::KEY_EVENTS . '" field'); - } - - $raw = $payload[self::KEY_EVENTS]; - if (!\is_array($raw)) { - throw new Invalid('Feed batch has a malformed "' . self::KEY_EVENTS . '" field'); + if (!\array_is_list($payload)) { + throw new Invalid('Expected a feed batch as a plain array of events'); } $events = []; /** @var mixed $event */ - foreach ($raw as $event) { + foreach ($payload as $event) { try { if (!\is_array($event)) { throw new Invalid('Feed batch contains an entry that is not an event'); diff --git a/tests/Feed/Unit/HttpJournalTest.php b/tests/Feed/Unit/HttpJournalTest.php index df3129c..af410a4 100644 --- a/tests/Feed/Unit/HttpJournalTest.php +++ b/tests/Feed/Unit/HttpJournalTest.php @@ -75,13 +75,14 @@ public function testReadsWithGet(): void $this->assertSame('GET', $transport->recorder->last()['method']); } - public function testAsksForJson(): void + public function testAsksForTheFeedMediaType(): void { [$feed, $transport] = $this->feed(); $feed->read(); - $this->assertSame('application/json', $transport->recorder->last()['headers']['Accept'] ?? null); + $this->assertSame(Protocol::MEDIA_TYPE, $transport->recorder->last()['headers']['Accept'] ?? null); + $this->assertSame('application/cloudevents-batch+json', Protocol::MEDIA_TYPE); } public function testSendsThePositionAndLimit(): void @@ -183,7 +184,7 @@ public function testRaisesServerErrors(): void */ public function testAnErrorStatusIsNotMistakenForAnEmptyBatch(): void { - [$feed] = $this->feed([FakeTransport::json(['total' => 0, 'events' => []], 500)]); + [$feed] = $this->feed([FakeTransport::json([], 500)]); $this->expectException(Transport::class); diff --git a/tests/Feed/Unit/ProtocolTest.php b/tests/Feed/Unit/ProtocolTest.php index 4004b39..10f404a 100644 --- a/tests/Feed/Unit/ProtocolTest.php +++ b/tests/Feed/Unit/ProtocolTest.php @@ -27,23 +27,23 @@ public function testQueryCarriesTheParametersThatWereSet(): void ], Protocol::query('1-0', 500, 20000)); } - public function testEncodesABatch(): void + public function testEncodesABatchAsAPlainArrayOfEvents(): void { $payload = Protocol::encode([ new CloudEvent(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), ]); - $this->assertSame(2, $payload['total']); - $this->assertCount(2, $payload['events']); - $this->assertSame('1-0', $payload['events'][0]['id']); - $this->assertSame(['x' => 1], $payload['events'][0]['data']); - $this->assertSame('1.0', $payload['events'][0]['specversion']); + $this->assertTrue(\array_is_list($payload), 'A batch is a plain array β€” the spec defines no envelope'); + $this->assertCount(2, $payload); + $this->assertSame('1-0', $payload[0]['id']); + $this->assertSame(['x' => 1], $payload[0]['data']); + $this->assertSame('1.0', $payload[0]['specversion']); } - public function testEncodesAnEmptyBatch(): void + public function testEncodesAnEmptyBatchAsAnEmptyArray(): void { - $this->assertSame(['total' => 0, 'events' => []], Protocol::encode([])); + $this->assertSame([], Protocol::encode([])); } public function testDecodesWhatItEncoded(): void @@ -58,15 +58,14 @@ public function testDecodesWhatItEncoded(): void public function testDecodesAnEmptyBatch(): void { - $this->assertSame([], Protocol::decode(['total' => 0, 'events' => []])); + $this->assertSame([], Protocol::decode([])); } /** - * An empty batch means "you are caught up". A response with no `events` - * field at all means "you did not reach the feed" β€” a misrouted request, a - * proxy's JSON error page, an endpoint that moved. Defaulting the missing - * field would make those indistinguishable, and a consumer would sit - * quietly at a position that never advances again. + * An empty batch means "you are caught up". A JSON object means "you did + * not reach the feed" β€” a misrouted request, a proxy's JSON error page, an + * endpoint that moved. Reading one as an empty batch would leave a + * consumer sitting quietly at a position that never advances again. * * @dataProvider notBatches */ @@ -83,8 +82,7 @@ public function testARespondingEndpointThatIsNotAFeedIsNotMistakenForBeingCaught public static function notBatches(): array { return [ - 'empty object' => [[]], - 'total but no events' => [['total' => 0]], + 'the old envelope' => [['total' => 0, 'events' => []]], 'some other API' => [['data' => [], 'status' => 'ok']], 'an error body' => [['message' => 'Not found', 'code' => 404]], ]; @@ -97,13 +95,6 @@ public function testRejectsAPayloadThatIsNotABatch(): void Protocol::decode('not a batch'); } - public function testRejectsAMalformedEventsField(): void - { - $this->expectException(Invalid::class); - - Protocol::decode(['events' => 'nope']); - } - /** * One event as a producer would put it on the wire. * @@ -129,12 +120,10 @@ private static function raw(string $id, string $type, array $overrides = []): ar public function testKeepsTheEventsBeforeAnUndecodableOne(): void { $events = Protocol::decode([ - 'events' => [ - self::raw('1-0', 'a'), - self::raw('1-1', 'b'), - self::raw('', 'no id'), - self::raw('1-3', 'd'), - ], + self::raw('1-0', 'a'), + self::raw('1-1', 'b'), + self::raw('', 'no id'), + self::raw('1-3', 'd'), ]); $this->assertCount(2, $events); @@ -145,19 +134,19 @@ public function testFailsWhenTheFirstEventIsUndecodable(): void { $this->expectException(Invalid::class); - Protocol::decode(['events' => [self::raw('', 'no id'), self::raw('1-1', 'b')]]); + Protocol::decode([self::raw('', 'no id'), self::raw('1-1', 'b')]); } public function testFailsWhenTheFirstEntryIsNotAnEvent(): void { $this->expectException(Invalid::class); - Protocol::decode(['events' => ['a string']]); + Protocol::decode(['a string']); } public function testKeepsTheEventsBeforeAnEntryThatIsNotAnEvent(): void { - $events = Protocol::decode(['events' => [self::raw('1-0', 'a'), 'a string']]); + $events = Protocol::decode([self::raw('1-0', 'a'), 'a string']); $this->assertCount(1, $events); } @@ -171,7 +160,7 @@ public function testFailsWhenAnEventIsNotACloudEventAtAll(): void { $this->expectException(Invalid::class); - Protocol::decode(['events' => [['id' => '1-0', 'type' => 'a']]]); + Protocol::decode([['id' => '1-0', 'type' => 'a']]); } /** @@ -183,13 +172,11 @@ public function testFailsWhenAnEventIsNotACloudEventAtAll(): void public function testSurvivesAProducerThatMovedAhead(): void { $events = Protocol::decode([ - 'events' => [ - self::raw('1-0', 'a', [ - 'specversion' => '1.1', - 'somethingnew' => 'ignored', - 'traceparent' => '00-abc-def-01', - ]), - ], + self::raw('1-0', 'a', [ + 'specversion' => '1.1', + 'somethingnew' => 'ignored', + 'traceparent' => '00-abc-def-01', + ]), ]); $this->assertCount(1, $events); @@ -197,6 +184,20 @@ public function testSurvivesAProducerThatMovedAhead(): void $this->assertSame('00-abc-def-01', $events[0]->extensions['traceparent']); } + /** + * The spec's optional compaction/deletion feature marks an event with a + * `method` attribute. This library does not implement the feature, but a + * feed that uses it must still be readable β€” the attribute rides along as + * an extension rather than breaking the batch. + */ + public function testAnEventCarryingTheSpecsMethodAttributeDecodes(): void + { + $events = Protocol::decode([self::raw('1-0', 'a', ['method' => 'DELETE'])]); + + $this->assertCount(1, $events); + $this->assertSame('DELETE', $events[0]->extensions['method']); + } + /** * A full batch is settled history, so it may be cached forever. Anything * short is the live end of the feed and will grow. diff --git a/tests/Feed/Unit/Support/FakeTransport.php b/tests/Feed/Unit/Support/FakeTransport.php index 7ab4999..2d5b50b 100644 --- a/tests/Feed/Unit/Support/FakeTransport.php +++ b/tests/Feed/Unit/Support/FakeTransport.php @@ -31,7 +31,7 @@ public static function of(array $responses): self } /** - * @param array $body + * @param array $body */ public static function json(array $body, int $statusCode = 200): ResponseInterface { @@ -67,6 +67,6 @@ protected function respond(RequestInterface $request): ResponseInterface throw $response; } - return $response ?? self::json(['total' => 0, 'events' => []]); + return $response ?? self::json([]); } } diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php index b7bb08d..057f8f3 100644 --- a/tests/Feed/Unit/Support/FeedServer.php +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -8,7 +8,6 @@ use Psr\Http\Message\ResponseInterface; use Utopia\Feed\Feed; use Utopia\Feed\Protocol; -use Utopia\Psr7\ContentType; use Utopia\Psr7\Header; use Utopia\Psr7\Response; use Utopia\Psr7\Stream; @@ -45,7 +44,7 @@ protected function respond(RequestInterface $request): ResponseInterface $body = (string) \json_encode(Protocol::encode($events)); return (new Response(200, body: new Stream\Factory()->createStream($body))) - ->withHeader(Header::CONTENT_TYPE, ContentType::JSON) + ->withHeader(Header::CONTENT_TYPE, Protocol::MEDIA_TYPE) ->withHeader(Header::CACHE_CONTROL, Protocol::cacheControl(\count($events), $limit)); } } From 461a92a653676924aa32cc35a07c15e7cfb60ccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 11:04:13 +0200 Subject: [PATCH 24/68] Return a Batch from reads and fold Protocol out of user-facing code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feed::read() and Feed::poll() now return a Batch β€” the events paired with the limit the read was actually clamped to, so cacheControl() can never be fed a number the batch was not built with. Batch counts and iterates as its events, toArray() is the wire encoding, and lastId() is the position a stateless relay tracks by hand. Feed::serve() takes a route's raw query-parameter array and does the whole request: extracts lastEventId, limit and timeout, coerces string values, applies the defaults and clamps, treats an empty lastEventId as absent, and rejects a malformed one with Exception\Invalid. A route body no longer names Protocol at all, and Protocol is documented as internal plumbing. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 11 +++ README.md | 51 +++++------ src/Feed/Batch.php | 78 +++++++++++++++++ src/Feed/Consumer.php | 6 +- src/Feed/Feed.php | 48 ++++++++--- src/Feed/Protocol.php | 3 + tests/Feed/E2E/RedisTest.php | 25 ++++-- tests/Feed/Unit/BatchTest.php | 75 ++++++++++++++++ tests/Feed/Unit/FeedTest.php | 113 +++++++++++++++++++++---- tests/Feed/Unit/HttpJournalTest.php | 2 +- tests/Feed/Unit/ProducerTest.php | 14 ++- tests/Feed/Unit/Support/FeedServer.php | 21 ++--- 12 files changed, 366 insertions(+), 81 deletions(-) create mode 100644 src/Feed/Batch.php create mode 100644 tests/Feed/Unit/BatchTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 597c9bb..a9791d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- **Breaking:** `Feed::read()` and `Feed::poll()` return a `Batch` instead of + a plain event array. A batch counts and iterates as its events and carries + the limit it was actually built with, so `Batch::cacheControl()` can never + be fed a number the read did not use. `Batch::toArray()` is the wire + encoding, `Batch::lastId()` the position a stateless relay tracks. +- Added `Feed::serve(array $query): Batch` β€” the whole HTTP request in one + call: extracts `lastEventId`, `limit` and `timeout` from the route's raw + query parameters, coerces and clamps them, and rejects a malformed + `lastEventId` with `Exception\Invalid`. A route never needs to name + `Protocol`, which is now documented as internal plumbing. + - **Breaking (wire format):** a feed batch on the wire is now the plain JSON array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) defines β€” the `{total, events}` envelope is gone, and an empty feed diff --git a/README.md b/README.md index a929221..b8fc857 100644 --- a/README.md +++ b/README.md @@ -124,50 +124,51 @@ retry. ### Serve a feed over HTTP -`Protocol` holds the wire contract β€” query parameters, response body, caching -rules β€” and deals in arrays, so it fits whichever HTTP layer you use: - -The feed a service serves is the read half of the same journal it appends to: +The feed a service serves is the read half of the same journal it appends to. +`Feed::serve()` takes the route's raw query parameters and answers with a +`Batch`, which knows its own body and caching header β€” the whole route body is: ```php use Utopia\Feed\Feed; -use Utopia\Feed\Protocol; $feed = new Feed($journal); // the same journal the Producer was built on // GET /v1/feeds/:feedId -$limit = \min((int) $request->getParam(Protocol::PARAM_LIMIT, Feed::MAX_BATCH), Feed::MAX_BATCH); - -$events = $feed->poll( - $request->getParam(Protocol::PARAM_LAST_EVENT_ID) ?: null, - $limit, - (int) $request->getParam(Protocol::PARAM_TIMEOUT, 0), -); +$batch = $feed->serve($request->getParams()); $response - ->addHeader('Content-Type', Protocol::MEDIA_TYPE) - ->addHeader('Cache-Control', Protocol::cacheControl(\count($events), $limit)) - ->json(Protocol::encode($events)); + ->addHeader('Content-Type', 'application/cloudevents-batch+json') + ->addHeader('Cache-Control', $batch->cacheControl(public: true)) + ->json($batch->toArray()); ``` +`serve()` extracts `lastEventId`, `limit` and `timeout` from the query, +coerces their string values, applies the defaults, and clamps the batch to +`Feed::MAX_BATCH` (1000 events) and the long-poll wait to `Feed::MAX_TIMEOUT` +(30s), so a client cannot ask for more than the producer is willing to build +or hold. A malformed `lastEventId` throws `Exception\Invalid` β€” catch it to +answer 400. + The response body is a bare JSON array of CloudEvents, as [http-feeds.org](https://www.http-feeds.org/) defines it β€” no envelope. An empty array means the consumer is caught up. The media type is -`application/cloudevents-batch+json` (`Protocol::MEDIA_TYPE`); on receipt this -library only checks the body shape, so a feed answering `application/json` -still reads fine. +`application/cloudevents-batch+json`; on receipt this library only checks the +body shape, so a feed answering `application/json` still reads fine. The spec defines two query parameters: `lastEventId` and `timeout`. The `limit` parameter is this library's extension beyond the spec β€” a spec-compliant consumer simply never sends it, and gets full batches. -Cap the limit yourself with `Feed::MAX_BATCH` before the call, so the number -that reaches `cacheControl()` is the one the batch was actually built with β€” a -read never returns more than that cap anyway. `Feed` clamps the wait too, to -`Feed::MAX_TIMEOUT` (30s), so a client cannot ask a producer to hold a -connection open for as long as it likes. A full batch is settled history and is -marked cacheable; a short one is the live end of the feed and is marked -`no-store`. Caching is `private` unless you pass `public: true`. +A full batch is settled history and `cacheControl()` marks it cacheable; a +short one is the live end of the feed and is marked `no-store`. Caching is +`private` unless you pass `public: true`. The batch carries the limit it was +actually built with, so the header is always honest β€” there is no number for +the route to keep in sync. + +Callers that already hold typed values can use `Feed::read()` and +`Feed::poll()` directly; both return a `Batch`, which counts and iterates as +the list of events it carries, and `Batch::lastId()` is the position a +stateless relay would otherwise track by hand. ## Events diff --git a/src/Feed/Batch.php b/src/Feed/Batch.php new file mode 100644 index 0000000..5ef2905 --- /dev/null +++ b/src/Feed/Batch.php @@ -0,0 +1,78 @@ + + */ +final class Batch implements \Countable, \IteratorAggregate +{ + /** + * @param list $events + * @param int $limit The effective limit β€” the clamped value the feed used to build this batch. + */ + public function __construct( + private readonly array $events, + private readonly int $limit, + ) { + } + + /** @return \ArrayIterator */ + public function getIterator(): \ArrayIterator + { + return new \ArrayIterator($this->events); + } + + public function count(): int + { + return \count($this->events); + } + + public function isEmpty(): bool + { + return $this->events === []; + } + + /** + * The id of the last event, or null on an empty batch β€” the position a + * caller relaying the feed by hand tracks. + */ + public function lastId(): ?string + { + $count = \count($this->events); + + return $count === 0 ? null : $this->events[$count - 1]->id; + } + + /** + * The Cache-Control header for the response carrying this batch: a full + * batch is settled history and immutable, anything short is the live end + * of the feed and must not be cached. Caching is private unless $public. + */ + public function cacheControl(bool $public = false): string + { + return Protocol::cacheControl(\count($this->events), $this->limit, $public); + } + + /** + * The batch as it goes on the wire: a plain array of CloudEvents. + * + * @return list> + */ + public function toArray(): array + { + return Protocol::encode($this->events); + } +} diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index d19f57c..26b89e8 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -33,9 +33,9 @@ public function getName(): string public function consume(callable $handler): int { - $events = $this->feed->poll($this->position(), $this->batch, $this->timeout); + $batch = $this->feed->poll($this->position(), $this->batch, $this->timeout); - if ($events === []) { + if ($batch->isEmpty()) { return 0; } @@ -43,7 +43,7 @@ public function consume(callable $handler): int $processed = null; $failure = null; - foreach ($events as $event) { + foreach ($batch as $event) { try { $handler($event); } catch (\Throwable $error) { diff --git a/src/Feed/Feed.php b/src/Feed/Feed.php index de1b825..8257081 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Feed.php @@ -4,8 +4,6 @@ namespace Utopia\Feed; -use Utopia\CloudEvents\CloudEvent; - // Server and client class: the read view of a feed β€” read and long-poll. // Server serves its own feed with this; client reads a remote one through Journal\Http. class Feed @@ -23,19 +21,49 @@ public function getName(): string return $this->journal->getName(); } - /** @return list */ - public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): array + public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): Batch + { + $limit = \max(1, \min($limit, self::MAX_BATCH)); + + return new Batch($this->journal->read($lastEventId, $limit), $limit); + } + + public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, int $timeout = 0): Batch { - return $this->journal->read($lastEventId, \max(1, \min($limit, self::MAX_BATCH))); + $limit = \max(1, \min($limit, self::MAX_BATCH)); + + return new Batch( + $this->journal->poll($lastEventId, $limit, \max(0, \min($timeout, self::MAX_TIMEOUT))), + $limit, + ); } - /** @return list */ - public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, int $timeout = 0): array + /** + * Serve one HTTP feed request: the route's raw query-parameter array in, + * the batch out. Extracts `lastEventId`, `limit` and `timeout`, coerces + * their string values, applies the defaults and clamps to the protocol + * limits, so the route never touches the wire vocabulary itself. + * + * @param array $query The request's query parameters, string values included. + * + * @throws Exception\Invalid When `lastEventId` is present but is not a feed position β€” a 400-worthy input. + */ + public function serve(array $query): Batch { - return $this->journal->poll( + $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; + $lastEventId = \is_string($lastEventId) && $lastEventId !== '' ? $lastEventId : null; + + if ($lastEventId !== null && !Id::isValid($lastEventId)) { + throw new Exception\Invalid('Invalid lastEventId: ' . $lastEventId); + } + + $limit = $query[Protocol::PARAM_LIMIT] ?? null; + $timeout = $query[Protocol::PARAM_TIMEOUT] ?? null; + + return $this->poll( $lastEventId, - \max(1, \min($limit, self::MAX_BATCH)), - \max(0, \min($timeout, self::MAX_TIMEOUT)), + \is_numeric($limit) ? (int) $limit : self::MAX_BATCH, + \is_numeric($timeout) ? (int) $timeout : 0, ); } } diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 1820357..36655ac 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -8,6 +8,9 @@ use Utopia\Feed\Exception\Invalid; // HTTP shape of a feed, as defined by https://www.http-feeds.org/. +// Internal plumbing: used by Feed and the HTTP journal, so a service serving +// or reading a feed never needs to name it β€” Feed::serve() and Batch cover +// both ends. It stays public for integrators building their own transports. final class Protocol { public const string PARAM_LAST_EVENT_ID = 'lastEventId'; diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index c7ca3f2..026ad97 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -9,6 +9,7 @@ use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Redis as RedisCursor; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Batch; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Feed; use Utopia\Feed\Producer; @@ -53,6 +54,12 @@ private function feed(int $maxSize = 100_000): Feed return new Feed(new RedisJournal($this->redis, $this->name, $maxSize)); } + /** @return list */ + private static function events(Batch $batch): array + { + return \array_values(\iterator_to_array($batch)); + } + private function producer(int $maxSize = 100_000): Producer { return new Producer(new RedisJournal($this->redis, $this->name, $maxSize), 'urn:test:e2e'); @@ -77,7 +84,7 @@ public function testAppendsAndReadsBack(): void $id = $producer->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); - $events = $feed->read(); + $events = self::events($feed->read()); $this->assertCount(1, $events); $this->assertSame($id, $events[0]->id); @@ -123,11 +130,11 @@ public function testReadsStrictlyAfterAPosition(): void $first = $producer->append('a'); $second = $producer->append('b'); - $events = $feed->read($first); + $events = self::events($feed->read($first)); $this->assertCount(1, $events); $this->assertSame($second, $events[0]->id); - $this->assertSame([], $feed->read($second)); + $this->assertCount(0, $feed->read($second)); } public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void @@ -140,7 +147,7 @@ public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void extensions: ['traceparent' => '00-abc-def-01'], )); - $event = $this->feed()->read()[0]; + $event = self::events($this->feed()->read())[0]; $this->assertSame('https://example.com/schema.json', $event->dataschema); $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); @@ -150,14 +157,14 @@ public function testAnAbsentSubjectStaysAbsent(): void { $this->producer()->append('test'); - $this->assertNull($this->feed()->read()[0]->subject); + $this->assertNull(self::events($this->feed()->read())[0]->subject); } public function testAScalarPayloadSurvivesTheRoundTrip(): void { $this->producer()->append('test', 'a string'); - $this->assertSame('a string', $this->feed()->read()[0]->data); + $this->assertSame('a string', self::events($this->feed()->read())[0]->data); } public function testNestedPayloadsSurviveTheRoundTrip(): void @@ -171,7 +178,7 @@ public function testNestedPayloadsSurviveTheRoundTrip(): void $this->producer()->append('test', $data); - $this->assertSame($data, $this->feed()->read()[0]->data); + $this->assertSame($data, self::events($this->feed()->read())[0]->data); } public function testHonoursTheLimit(): void @@ -209,7 +216,7 @@ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void $events = $feed->read($first); - $this->assertNotEmpty($events, 'A consumer that fell behind must still get what is retained'); + $this->assertFalse($events->isEmpty(), 'A consumer that fell behind must still get what is retained'); $this->assertLessThan(500, $this->redis->xLen('feed:' . $this->name), 'The feed must be trimmed'); } @@ -230,7 +237,7 @@ public function testLongPollingGivesUpAtTheTimeout(): void $started = \microtime(true); $events = $this->feed()->poll(null, 10, 700); - $this->assertSame([], $events); + $this->assertCount(0, $events); $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started); } diff --git a/tests/Feed/Unit/BatchTest.php b/tests/Feed/Unit/BatchTest.php new file mode 100644 index 0000000..ca23c4b --- /dev/null +++ b/tests/Feed/Unit/BatchTest.php @@ -0,0 +1,75 @@ + */ + private static function events(int $count): array + { + return \array_map( + static fn (int $i): CloudEvent => new CloudEvent(id: '1-' . $i, type: 'event-' . $i, source: 'urn:test'), + \range(0, $count - 1), + ); + } + + public function testCountsItsEvents(): void + { + $this->assertCount(3, new Batch(self::events(3), 100)); + $this->assertCount(0, new Batch([], 100)); + } + + public function testIteratesItsEventsInOrder(): void + { + $types = []; + + foreach (new Batch(self::events(2), 100) as $event) { + $types[] = $event->type; + } + + $this->assertSame(['event-0', 'event-1'], $types); + } + + public function testKnowsWhetherItIsEmpty(): void + { + $this->assertTrue((new Batch([], 100))->isEmpty()); + $this->assertFalse((new Batch(self::events(1), 100))->isEmpty()); + } + + public function testLastIdIsTheLastEventsPosition(): void + { + $this->assertSame('1-2', (new Batch(self::events(3), 100))->lastId()); + $this->assertNull((new Batch([], 100))->lastId()); + } + + /** + * The caching rule stays in one place β€” the batch answers with its own + * count and its own limit, so a mismatched pair cannot be expressed. + */ + public function testAFullBatchIsCacheable(): void + { + $this->assertSame('private, max-age=31536000', (new Batch(self::events(2), 2))->cacheControl()); + $this->assertSame('public, max-age=31536000', (new Batch(self::events(2), 2))->cacheControl(public: true)); + } + + public function testAShortOrEmptyBatchIsNot(): void + { + $this->assertSame('no-store', (new Batch(self::events(1), 2))->cacheControl()); + $this->assertSame('no-store', (new Batch([], 2))->cacheControl()); + } + + public function testToArrayIsTheWireEncoding(): void + { + $events = self::events(2); + + $this->assertSame(Protocol::encode($events), (new Batch($events, 100))->toArray()); + $this->assertSame([], (new Batch([], 100))->toArray()); + } +} diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 6707eac..112000d 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -8,6 +8,7 @@ use Utopia\Feed\Journal\Memory; use Utopia\Feed\Journal\None; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Batch; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; @@ -29,11 +30,17 @@ protected function setUp(): void $this->feed = new Feed($this->journal); } + /** @return list */ + private static function events(Batch $batch): array + { + return \array_values(\iterator_to_array($batch)); + } + public function testReadsBackWhatWasAppended(): void { $this->producer->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); - $events = $this->feed->read(); + $events = self::events($this->feed->read()); $this->assertCount(1, $events); $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); @@ -47,7 +54,7 @@ public function testEventsComeBackOldestFirst(): void $this->producer->append($type); } - $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $this->feed->read())); + $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($this->feed->read()))); } public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void @@ -69,7 +76,7 @@ public function testReadsStrictlyAfterTheGivenPosition(): void $first = $this->producer->append('a'); $this->producer->append('b'); - $events = $this->feed->read($first); + $events = self::events($this->feed->read($first)); $this->assertCount(1, $events); $this->assertSame('b', $events[0]->type); @@ -80,7 +87,7 @@ public function testReadFromTheLastEventIsEmpty(): void $this->producer->append('a'); $last = $this->producer->append('b'); - $this->assertSame([], $this->feed->read($last)); + $this->assertCount(0, $this->feed->read($last)); } public function testNullPositionReadsFromTheOldestRetainedEvent(): void @@ -135,7 +142,7 @@ public function testExtensionAttributesSurviveAppendAndRead(): void extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2], )); - $event = $this->feed->read()[0]; + $event = self::events($this->feed->read())[0]; $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); $this->assertSame(2, $event->extensions['retrycount']); @@ -157,7 +164,7 @@ public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void extensions: ['123' => 'digits', 'trace' => 'ok'], )); - $event = $this->feed->read()[0]; + $event = self::events($this->feed->read())[0]; // @phpstan-ignore offsetAccess.notFound $this->assertSame('digits', $event->extensions['123']); @@ -173,7 +180,7 @@ public function testDataschemaSurvivesAppendAndRead(): void dataschema: 'https://example.com/schema.json', )); - $this->assertSame('https://example.com/schema.json', $this->feed->read()[0]->dataschema); + $this->assertSame('https://example.com/schema.json', self::events($this->feed->read())[0]->dataschema); } /** @@ -204,7 +211,7 @@ public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void { $this->producer->append('test', $data); - $this->assertSame($data, $this->feed->read()[0]->data); + $this->assertSame($data, self::events($this->feed->read())[0]->data); } /** @@ -215,14 +222,14 @@ public function testAnEventWithNoSubjectHasANullSubject(): void { $this->producer->append('test'); - $this->assertNull($this->feed->read()[0]->subject); + $this->assertNull(self::events($this->feed->read())[0]->subject); } public function testASubjectSurvivesAppendAndRead(): void { $this->producer->append('test', [], 'example.com'); - $this->assertSame('example.com', $this->feed->read()[0]->subject); + $this->assertSame('example.com', self::events($this->feed->read())[0]->subject); } public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void @@ -242,7 +249,7 @@ public function testPollGivesUpAtTheTimeoutWithAnEmptyBatch(): void $events = $this->feed->poll(null, 10, 600); $elapsed = \microtime(true) - $started; - $this->assertSame([], $events); + $this->assertCount(0, $events); $this->assertGreaterThanOrEqual(0.4, $elapsed, 'Must actually wait'); $this->assertLessThan(3.0, $elapsed, 'Must not wait far past the timeout'); } @@ -251,7 +258,7 @@ public function testPollWithoutATimeoutIsAPlainRead(): void { $started = \microtime(true); - $this->assertSame([], $this->feed->poll()); + $this->assertCount(0, $this->feed->poll()); $this->assertLessThan(0.4, \microtime(true) - $started); } @@ -265,7 +272,7 @@ public function testRetentionIsBoundedAndTrimsTheOldest(): void $producer->append($type); } - $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read())); + $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($feed->read()))); } /** @@ -282,7 +289,7 @@ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void $producer->append('b'); $producer->append('c'); - $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $feed->read($first))); + $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($feed->read($first)))); } public function testExposesTheFeedItReads(): void @@ -299,6 +306,82 @@ public function testAFeedWithNoBackendCannotBeRead(): void $feed->read(); } + public function testServeAppliesTheDefaultsWhenNoParametersArrive(): void + { + $this->producer->append('a'); + $this->producer->append('b'); + + $this->assertCount(2, $this->feed->serve([])); + } + + public function testServeCoercesTheStringValuesARouteHands(): void + { + $first = $this->producer->append('a'); + $this->producer->append('b'); + $this->producer->append('c'); + + $batch = $this->feed->serve([ + 'lastEventId' => $first, + 'limit' => '1', + 'timeout' => '0', + ]); + + $this->assertSame(['b'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($batch))); + } + + public function testServeTreatsAnEmptyLastEventIdAsAbsent(): void + { + $this->producer->append('a'); + + $this->assertCount(1, $this->feed->serve(['lastEventId' => ''])); + } + + public function testServeRejectsALastEventIdThatIsNotAPosition(): void + { + $this->expectException(Invalid::class); + + $this->feed->serve(['lastEventId' => 'not-a-position']); + } + + public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void + { + $this->producer->append('a'); + $this->producer->append('b'); + + $this->assertCount(2, $this->feed->serve(['limit' => 'lots', 'timeout' => 'soon'])); + } + + /** + * The trap the old API set: a route that passed the raw request limit to + * the caching rule while the read was clamped to less would mark a full + * batch `no-store` β€” or worse. The batch carries the limit it was actually + * built with, so a full batch under an oversized request is still + * recognized as settled history. + */ + public function testAnOversizedLimitStillYieldsAnHonestCacheControl(): void + { + foreach (\range(1, Feed::MAX_BATCH) as $i) { + $this->producer->append('event-' . $i); + } + + $batch = $this->feed->serve(['limit' => '5000']); + + $this->assertCount(Feed::MAX_BATCH, $batch); + $this->assertSame('public, max-age=31536000', $batch->cacheControl(public: true)); + } + + public function testAShortBatchIsNotCacheable(): void + { + $this->producer->append('a'); + + $this->assertSame('no-store', $this->feed->serve([])->cacheControl()); + } + + public function testAnEmptyBatchIsNotCacheable(): void + { + $this->assertSame('no-store', $this->feed->serve([])->cacheControl()); + } + public function testRejectsAnEmptyFeedName(): void { $this->expectException(Invalid::class); @@ -315,7 +398,7 @@ public function testAcceptsTheSmallestUsefulRetentionCap(): void $producer->append('a'); $producer->append('b'); - $events = $feed->read(); + $events = self::events($feed->read()); $this->assertCount(1, $events); $this->assertSame('b', $events[0]->type); diff --git a/tests/Feed/Unit/HttpJournalTest.php b/tests/Feed/Unit/HttpJournalTest.php index af410a4..f18a4a0 100644 --- a/tests/Feed/Unit/HttpJournalTest.php +++ b/tests/Feed/Unit/HttpJournalTest.php @@ -38,7 +38,7 @@ public function testReadsAFeedOverHttp(): void new CloudEvent(id: '1-1', type: 'io.appwrite.edge.invalidate', source: 'urn:test'), ]))]); - $events = $feed->read(); + $events = \array_values(\iterator_to_array($feed->read())); $this->assertCount(2, $events); $this->assertSame('1-0', $events[0]->id); diff --git a/tests/Feed/Unit/ProducerTest.php b/tests/Feed/Unit/ProducerTest.php index 6a04de5..d75680b 100644 --- a/tests/Feed/Unit/ProducerTest.php +++ b/tests/Feed/Unit/ProducerTest.php @@ -38,11 +38,17 @@ public function testAppendReturnsAPosition(): void $this->assertTrue(Id::isValid($id)); } + /** @return list */ + private function events(): array + { + return \array_values(\iterator_to_array($this->feed->read())); + } + public function testStampsTheSourceAndTimeOnAppend(): void { $this->producer->append('test'); - $event = $this->feed->read()[0]; + $event = $this->events()[0]; $this->assertSame('urn:appwrite:cloud:fra', $event->source); $this->assertNotNull($event->time); @@ -58,7 +64,7 @@ public function testKeepsTheSourceOfTheProducerThatAppended(): void (new Producer($this->journal, 'urn:appwrite:cloud:fra'))->append('test'); (new Producer($this->journal, 'urn:appwrite:cloud:nyc'))->append('test'); - $events = $this->feed->read(); + $events = $this->events(); $this->assertSame('urn:appwrite:cloud:fra', $events[0]->source); $this->assertSame('urn:appwrite:cloud:nyc', $events[1]->source); @@ -68,7 +74,7 @@ public function testPublishStampsAPreparedEvent(): void { $id = $this->producer->publish(new CloudEvent(id: 'ignored', type: 'test', source: 'ignored', data: ['a' => 'b'], subject: 's')); - $event = $this->feed->read()[0]; + $event = $this->events()[0]; $this->assertSame($id, $event->id); $this->assertNotSame('ignored', $event->id, 'The backend assigns the position, not the caller'); @@ -80,7 +86,7 @@ public function testPublishKeepsATimeTheCallerSet(): void { $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', time: '2020-01-01T00:00:00.000Z')); - $this->assertSame('2020-01-01T00:00:00.000Z', $this->feed->read()[0]->time); + $this->assertSame('2020-01-01T00:00:00.000Z', $this->events()[0]->time); } public function testRejectsAnEmptyEventType(): void diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php index 057f8f3..92cf057 100644 --- a/tests/Feed/Unit/Support/FeedServer.php +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -15,9 +15,10 @@ /** * A producer's feed endpoint, as a client. * - * Serves a real {@see Feed} through {@see Protocol} exactly as an HTTP route - * would, so a consumer reading it exercises the whole contract β€” parameters, - * body and caching β€” rather than a fixture written to match the consumer. + * Serves a real {@see Feed} through {@see Feed::serve()} exactly as an HTTP + * route would, so a consumer reading it exercises the whole contract β€” + * parameters, body and caching β€” rather than a fixture written to match the + * consumer. */ class FeedServer extends FakeClient { @@ -31,20 +32,12 @@ protected function respond(RequestInterface $request): ResponseInterface $query = []; \parse_str($request->getUri()->getQuery(), $query); - $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; - $limit = \min((int) ($query[Protocol::PARAM_LIMIT] ?? Feed::MAX_BATCH), Feed::MAX_BATCH); - $timeout = (int) ($query[Protocol::PARAM_TIMEOUT] ?? 0); + $batch = $this->feed->serve($query); - $events = $this->feed->poll( - \is_string($lastEventId) && $lastEventId !== '' ? $lastEventId : null, - $limit, - $timeout, - ); - - $body = (string) \json_encode(Protocol::encode($events)); + $body = (string) \json_encode($batch->toArray()); return (new Response(200, body: new Stream\Factory()->createStream($body))) ->withHeader(Header::CONTENT_TYPE, Protocol::MEDIA_TYPE) - ->withHeader(Header::CACHE_CONTROL, Protocol::cacheControl(\count($events), $limit)); + ->withHeader(Header::CACHE_CONTROL, $batch->cacheControl()); } } From d90e3abf28d44f77f73017ba7a96d2efe23d9662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 11:12:32 +0200 Subject: [PATCH 25/68] Let a consumer start at the tip of a feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lastEventId of $ β€” a protocol extension like limit, borrowed from the Redis XREAD convention β€” now means "the tip of the feed". Journals resolve it to the newest entry at the moment of the call (an empty feed resolves to the beginning of future events), pinned once per poll so events landing mid-wait are still delivered. Journal\Http passes the sentinel through for the producer to resolve inside the same request, leaving no tip round trip to race; Feed::serve() lets it through its id validation, and Id keeps rejecting it. Consumers opt in with Start::Tip: when no position is stored, the first poll uses the sentinel. A stored cursor always wins, and reset() with Start::Tip means "forget everything, resume from now". Feed::tip() exposes the newest event's id for local journals. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 5 ++ README.md | 29 ++++++-- src/Feed/Consumer.php | 15 +++- src/Feed/Feed.php | 15 +++- src/Feed/Journal.php | 23 ++++++ src/Feed/Journal/Http.php | 11 +++ src/Feed/Journal/Memory.php | 9 +++ src/Feed/Journal/None.php | 5 ++ src/Feed/Journal/Pool.php | 7 ++ src/Feed/Journal/Redis.php | 17 +++++ src/Feed/Protocol.php | 8 +++ src/Feed/Start.php | 24 +++++++ tests/Feed/E2E/RedisTest.php | 17 +++++ tests/Feed/Unit/ConsumerTest.php | 84 ++++++++++++++++++++++ tests/Feed/Unit/FeedTest.php | 26 +++++++ tests/Feed/Unit/HttpJournalTest.php | 37 ++++++++++ tests/Feed/Unit/IdTest.php | 1 + tests/Feed/Unit/ProtocolTest.php | 9 +++ tests/Feed/Unit/Support/MidPollJournal.php | 31 ++++++++ 19 files changed, 366 insertions(+), 7 deletions(-) create mode 100644 src/Feed/Start.php create mode 100644 tests/Feed/Unit/Support/MidPollJournal.php diff --git a/CHANGELOG.md b/CHANGELOG.md index a9791d7..edd58b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ query parameters, coerces and clamps them, and rejects a malformed `lastEventId` with `Exception\Invalid`. A route never needs to name `Protocol`, which is now documented as internal plumbing. +- Added `Start::Tip` β€” a consumer with no stored position can opt into + starting at the tip of the feed (only what happens from now on) instead of + draining the backlog. Rides a protocol extension: the `lastEventId` value + `$`, resolved by the producer to the newest event as the request arrives. + Also added `Feed::tip()`, the id of the newest event in a local journal. - **Breaking (wire format):** a feed batch on the wire is now the plain JSON array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) diff --git a/README.md b/README.md index b8fc857..c364c29 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,24 @@ while (true) { } ``` +A consumer with no stored position starts at the oldest retained event. A +consumer that must not act on the backlog β€” a notifier announcing events as +they happen β€” opts into starting at the tip instead: + +```php +use Utopia\Feed\Start; + +$consumer = new Consumer($feed, 'notifier', $cursor, timeout: 20_000, start: Start::Tip); +``` + +A stored position always wins; `Start::Tip` only applies on the first run, or +after `reset()` β€” which with `Start::Tip` means "forget everything, resume +from now". Give a tip consumer a `timeout`: the producer anchors "now" as each +poll arrives, so new events land inside the held request rather than in the +gap between polls. Against a producer that predates the tip extension, the +first poll fails with a 4xx `Transport` error rather than silently replaying +the backlog. + ### Consume another service's feed Same code, different journal β€” nothing above it knows the events arrive over the @@ -155,9 +173,11 @@ empty array means the consumer is caught up. The media type is `application/cloudevents-batch+json`; on receipt this library only checks the body shape, so a feed answering `application/json` still reads fine. -The spec defines two query parameters: `lastEventId` and `timeout`. The -`limit` parameter is this library's extension beyond the spec β€” a -spec-compliant consumer simply never sends it, and gets full batches. +The spec defines two query parameters: `lastEventId` and `timeout`. This +library extends it with two more pieces of vocabulary: the `limit` parameter, +and the `lastEventId` value `$`, which the producer resolves to the tip of the +feed β€” the anchor behind `Start::Tip`. A spec-compliant consumer simply never +sends either, and a `$` can never collide with a real id. A full batch is settled history and `cacheControl()` marks it cacheable; a short one is the live end of the feed and is marked `no-store`. Caching is @@ -278,7 +298,8 @@ later events on top of state that was never updated. **A consumer with no position starts at the oldest retained event, never at the tip**, so a consumer deployed after the producer catches up rather than dropping -the backlog. +the backlog. Starting at the tip is strictly opt-in, per consumer, with +`Start::Tip`. ## Rolling out a feed diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 26b89e8..cd53230 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -20,6 +20,7 @@ public function __construct( protected readonly Cursor $cursor, protected readonly int $batch = self::BATCH, protected readonly int $timeout = 0, + protected readonly Start $start = Start::Oldest, ) { if ($name === '') { throw new Exception\Invalid('Feed consumer requires a name'); @@ -33,7 +34,7 @@ public function getName(): string public function consume(callable $handler): int { - $batch = $this->feed->poll($this->position(), $this->batch, $this->timeout); + $batch = $this->feed->poll($this->position() ?? $this->origin(), $this->batch, $this->timeout); if ($batch->isEmpty()) { return 0; @@ -67,6 +68,18 @@ public function consume(callable $handler): int return $handled; } + /** + * Where a poll starts when no position is stored: the oldest retained + * event, or β€” for Start::Tip β€” the tip sentinel, which the journal (or + * the remote producer, inside the same request) resolves to "now". Once + * events are handled and the cursor saves, the sentinel never appears + * again; reset() forgets the position, so the next poll anchors anew. + */ + private function origin(): ?string + { + return $this->start === Start::Tip ? Protocol::TIP : null; + } + public function position(): ?string { if (!$this->restored) { diff --git a/src/Feed/Feed.php b/src/Feed/Feed.php index 8257081..566581d 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Feed.php @@ -21,6 +21,17 @@ public function getName(): string return $this->journal->getName(); } + /** + * The id of the newest event, or null on an empty feed. Local journals + * only β€” a remote feed's producer resolves the tip sentinel instead. + * + * @throws Exception + */ + public function tip(): ?string + { + return $this->journal->tip(); + } + public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): Batch { $limit = \max(1, \min($limit, self::MAX_BATCH)); @@ -46,14 +57,14 @@ public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, * * @param array $query The request's query parameters, string values included. * - * @throws Exception\Invalid When `lastEventId` is present but is not a feed position β€” a 400-worthy input. + * @throws Exception\Invalid When `lastEventId` is present but is neither a feed position nor the tip sentinel β€” a 400-worthy input. */ public function serve(array $query): Batch { $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; $lastEventId = \is_string($lastEventId) && $lastEventId !== '' ? $lastEventId : null; - if ($lastEventId !== null && !Id::isValid($lastEventId)) { + if ($lastEventId !== null && $lastEventId !== Protocol::TIP && !Id::isValid($lastEventId)) { throw new Exception\Invalid('Invalid lastEventId: ' . $lastEventId); } diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index 3ea3581..9cf9775 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -29,6 +29,24 @@ public function getName(): string /** @return list */ abstract public function read(?string $lastEventId, int $limit): array; + /** + * The id of the newest event, or null when the feed is empty. + * + * @throws Exception + */ + abstract public function tip(): ?string; + + /** + * Resolve the tip sentinel into a concrete position: the newest id at the + * moment of the call, or null on an empty feed β€” which reads as "the + * beginning of future events". The sentinel is resolved here, before any + * id arithmetic; Id itself keeps rejecting it. + */ + protected function resolve(?string $lastEventId): ?string + { + return $lastEventId === Protocol::TIP ? $this->tip() : $lastEventId; + } + /** * Wait for events, re-reading on an interval until some land or the * deadline passes. Journal\Http overrides this: there the producer does the @@ -38,6 +56,11 @@ abstract public function read(?string $lastEventId, int $limit): array; */ public function poll(?string $lastEventId, int $limit, int $timeout): array { + // The sentinel is pinned once, before the wait: re-resolving on every + // read would move the tip past events landing mid-poll, and they + // would never be delivered. + $lastEventId = $this->resolve($lastEventId); + $deadline = \microtime(true) + $timeout / 1000; while (true) { diff --git a/src/Feed/Journal/Http.php b/src/Feed/Journal/Http.php index ea20a2b..4dca6d8 100644 --- a/src/Feed/Journal/Http.php +++ b/src/Feed/Journal/Http.php @@ -8,6 +8,7 @@ use Utopia\Client\Adapter; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; +use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Journal; use Utopia\Feed\Protocol; use Utopia\Psr7\Header; @@ -28,6 +29,16 @@ public function __construct( $this->requests = new RequestFactory(); } + /** + * Never called on the consumer path: the tip sentinel is passed through + * as `lastEventId=$` and the producer resolves it inside the same + * request, so there is no separate tip round trip to race. + */ + public function tip(): ?string + { + throw new Unsupported("The {$this->name} feed is remote; its producer resolves the tip"); + } + public function read(?string $lastEventId, int $limit): array { return $this->fetch($lastEventId, $limit, 0); diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Journal/Memory.php index f9f4cfc..b0a7a7d 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -44,8 +44,17 @@ public function append(CloudEvent $event): string return $id; } + public function tip(): ?string + { + $count = \count($this->events); + + return $count === 0 ? null : $this->events[$count - 1]->id; + } + public function read(?string $lastEventId, int $limit): array { + $lastEventId = $this->resolve($lastEventId); + $after = $lastEventId === null ? null : Id::decode($lastEventId); $events = []; diff --git a/src/Feed/Journal/None.php b/src/Feed/Journal/None.php index 144115a..b90ada4 100644 --- a/src/Feed/Journal/None.php +++ b/src/Feed/Journal/None.php @@ -21,6 +21,11 @@ public function append(CloudEvent $event): string throw new Unsupported("No feed backend is configured for the {$this->name} feed"); } + public function tip(): ?string + { + throw new Unsupported("No feed backend is configured for the {$this->name} feed"); + } + public function read(?string $lastEventId, int $limit): array { throw new Unsupported("No feed backend is configured for the {$this->name} feed"); diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Journal/Pool.php index cfa3949..51843e3 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -29,6 +29,13 @@ public function append(CloudEvent $event): string ); } + public function tip(): ?string + { + return $this->pool->use( + fn (\Redis|\RedisCluster $redis): ?string => (new Redis($redis, $this->name, $this->maxSize))->tip() + ); + } + public function read(?string $lastEventId, int $limit): array { return $this->pool->use( diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Journal/Redis.php index 0873c6a..77496ca 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -35,8 +35,25 @@ public function append(CloudEvent $event): string return $id; } + public function tip(): ?string + { + try { + $entries = $this->redis->xRevRange('feed:' . $this->name, '+', '-', 1); + } catch (\RedisException $error) { + throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); + } + + if (!\is_array($entries) || $entries === []) { + return null; + } + + return (string) \array_key_first($entries); + } + public function read(?string $lastEventId, int $limit): array { + $lastEventId = $this->resolve($lastEventId); + $start = $lastEventId === null ? '-' : Id::after($lastEventId); try { diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 36655ac..642adc7 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -19,6 +19,14 @@ final class Protocol public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; + /** + * Extension beyond the spec, like `limit`: a `lastEventId` of `$` means + * "the tip of the feed". The producer resolves it to the newest event at + * the moment the request arrives, so a consumer can ask for only what + * happens from now on without a separate round trip to learn the tip. + */ + public const string TIP = '$'; + public const string CACHE_IMMUTABLE = 'max-age=31536000'; public const string CACHE_NONE = 'no-store'; diff --git a/src/Feed/Start.php b/src/Feed/Start.php new file mode 100644 index 0000000..02aa23a --- /dev/null +++ b/src/Feed/Start.php @@ -0,0 +1,24 @@ +assertGreaterThanOrEqual(0.4, \microtime(true) - $started); } + /** + * The tip is found with XREVRANGE, so this needs a real Redis: an empty + * stream has no tip, and the sentinel reads nothing that already exists. + */ + public function testTheTipSentinelSkipsTheBacklog(): void + { + [$producer, $feed] = $this->feedAndProducer(); + + $this->assertNull($feed->tip(), 'An empty feed has no tip'); + + $producer->append('a'); + $last = $producer->append('b'); + + $this->assertSame($last, $feed->tip()); + $this->assertCount(0, $feed->read('$')); + } + public function testConsumesThroughAPersistedCursor(): void { [$producer, $feed] = $this->feedAndProducer(); diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index e4efc1a..656a24b 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -14,7 +14,9 @@ use Utopia\Feed\Exception\Transport; use Utopia\Feed\Feed; use Utopia\Feed\Producer; +use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FailingCursor; +use Utopia\Tests\Unit\Support\MidPollJournal; class ConsumerTest extends TestCase { @@ -299,6 +301,88 @@ public function testAPositionThatCannotBeSavedIsRaisedAfterTheEventsAreHandled() $this->assertNotNull($consumer->position(), 'The in-memory position still moved'); } + public function testTipStartDoesNotAnnounceTheBacklog(): void + { + $this->producer->append('old-1'); + $this->producer->append('old-2'); + + $consumer = new Consumer($this->feed, 'notifier', $this->cursor, start: Start::Tip); + + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertNull($this->cursor->load('edge', 'notifier'), 'Skipping the backlog is not progress to commit'); + } + + /** + * The tip is pinned when the poll starts, so an event landing while the + * poll waits is delivered β€” only the backlog is skipped. + */ + public function testTipStartDeliversWhatLandsMidPoll(): void + { + $journal = new MidPollJournal('edge'); + (new Producer($journal, 'urn:test'))->append('old'); + + $cursor = new MemoryCursor(); + $consumer = new Consumer(new Feed($journal), 'notifier', $cursor, timeout: 5_000, start: Start::Tip); + + $seen = []; + $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }); + + $this->assertSame(1, $count); + $this->assertSame(['landed'], $seen, 'The backlog is skipped; the mid-wait event is not'); + $this->assertNotNull($cursor->load('edge', 'notifier'), 'Handling the event saves the position'); + } + + public function testAStoredCursorBeatsTipStart(): void + { + $first = $this->producer->append('a'); + $this->producer->append('b'); + + $this->cursor->save('edge', 'invalidator', $first); + + $consumer = new Consumer($this->feed, 'invalidator', $this->cursor, start: Start::Tip); + + $this->assertSame(['b'], $this->drain($consumer), 'A restart must not skip the gap'); + } + + public function testResetWithTipStartResumesFromNow(): void + { + $first = $this->producer->append('a'); + $this->producer->append('b'); + $this->cursor->save('edge', 'invalidator', $first); + + $consumer = new Consumer($this->feed, 'invalidator', $this->cursor, start: Start::Tip); + + $this->assertSame(['b'], $this->drain($consumer), 'The stored position still wins before the reset'); + + $consumer->reset(); + + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null), 'After reset, the backlog is forgotten'); + } + + public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void + { + $consumer = new Consumer($this->feed, 'notifier', $this->cursor, timeout: 600, start: Start::Tip); + + $started = \microtime(true); + + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started, 'Must actually wait'); + } + + public function testTipStartOnAnEmptyFeedDeliversWhatLandsMidWait(): void + { + $consumer = new Consumer(new Feed(new MidPollJournal('edge')), 'notifier', $this->cursor, timeout: 5_000, start: Start::Tip); + + $seen = []; + $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }); + + $this->assertSame(['landed'], $seen); + } + public function testResetReplaysEverythingStillRetained(): void { $this->producer->append('a'); diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 112000d..1b852ae 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -13,6 +13,7 @@ use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; use Utopia\Feed\Producer; +use Utopia\Feed\Protocol; use Utopia\Feed\Id; class FeedTest extends TestCase @@ -306,6 +307,24 @@ public function testAFeedWithNoBackendCannotBeRead(): void $feed->read(); } + public function testTipIsTheNewestEventsId(): void + { + $this->assertNull($this->feed->tip()); + + $this->producer->append('a'); + $last = $this->producer->append('b'); + + $this->assertSame($last, $this->feed->tip()); + } + + public function testReadingFromTheTipSentinelSkipsTheBacklog(): void + { + $this->producer->append('a'); + $this->producer->append('b'); + + $this->assertCount(0, $this->feed->read(Protocol::TIP)); + } + public function testServeAppliesTheDefaultsWhenNoParametersArrive(): void { $this->producer->append('a'); @@ -343,6 +362,13 @@ public function testServeRejectsALastEventIdThatIsNotAPosition(): void $this->feed->serve(['lastEventId' => 'not-a-position']); } + public function testServeLetsTheTipSentinelThrough(): void + { + $this->producer->append('a'); + + $this->assertCount(0, $this->feed->serve(['lastEventId' => Protocol::TIP])); + } + public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void { $this->producer->append('a'); diff --git a/tests/Feed/Unit/HttpJournalTest.php b/tests/Feed/Unit/HttpJournalTest.php index f18a4a0..dd75b7b 100644 --- a/tests/Feed/Unit/HttpJournalTest.php +++ b/tests/Feed/Unit/HttpJournalTest.php @@ -14,8 +14,12 @@ use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Feed; +use Utopia\Feed\Producer; use Utopia\Feed\Protocol; +use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FakeTransport; +use Utopia\Tests\Unit\Support\FeedServer; +use Utopia\Tests\Unit\Support\MidPollJournal; class HttpJournalTest extends TestCase { @@ -269,4 +273,37 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void $this->assertSame(['a', 'b', 'c'], $seen); $this->assertStringContainsString('lastEventId=1-1', $transport->recorder->uris()[1]); } + + /** + * The tip sentinel crosses the wire as `lastEventId=$` and the producer + * resolves it inside the held request β€” end to end, a tip consumer skips + * the backlog and still gets what lands mid-poll. + */ + public function testTipStartWorksOverHttp(): void + { + $journal = new MidPollJournal('edge'); + $producer = new Producer($journal, 'urn:test'); + $producer->append('old'); + + $server = new FeedServer(new Feed($journal)); + $feed = new Feed(new Http($server, 'https://cloud.example.com/v1/feeds', 'edge')); + $consumer = new Consumer($feed, 'notifier', new MemoryCursor(), timeout: 5_000, start: Start::Tip); + + $seen = []; + $handler = function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }; + + $this->assertSame(1, $consumer->consume($handler)); + $this->assertSame(['landed'], $seen, 'The backlog is skipped; the mid-wait event is not'); + $this->assertMatchesRegularExpression('/lastEventId=(%24|\$)/', $server->recorder->last()['uri']); + + // The position now saves as a real id, so the sentinel never + // appears on the wire again. + $producer->append('after'); + + $this->assertSame(1, $consumer->consume($handler)); + $this->assertSame(['landed', 'after'], $seen); + $this->assertDoesNotMatchRegularExpression('/lastEventId=(%24|\$)/', $server->recorder->last()['uri']); + } } diff --git a/tests/Feed/Unit/IdTest.php b/tests/Feed/Unit/IdTest.php index c307a2d..3f3ea4a 100644 --- a/tests/Feed/Unit/IdTest.php +++ b/tests/Feed/Unit/IdTest.php @@ -26,6 +26,7 @@ public static function ids(): array 'trailing dash' => ['1690000000000-', false], 'exclusive syntax' => ['(1690000000000-0', false], 'range token' => ['-', false], + 'tip sentinel' => ['$', false], ]; } diff --git a/tests/Feed/Unit/ProtocolTest.php b/tests/Feed/Unit/ProtocolTest.php index 10f404a..a766fcc 100644 --- a/tests/Feed/Unit/ProtocolTest.php +++ b/tests/Feed/Unit/ProtocolTest.php @@ -27,6 +27,15 @@ public function testQueryCarriesTheParametersThatWereSet(): void ], Protocol::query('1-0', 500, 20000)); } + /** + * The tip sentinel rides the lastEventId parameter unchanged β€” the + * producer resolves it, and it can never collide with a real position. + */ + public function testQueryPassesTheTipSentinelThrough(): void + { + $this->assertSame(['lastEventId' => '$'], Protocol::query(Protocol::TIP)); + } + public function testEncodesABatchAsAPlainArrayOfEvents(): void { $payload = Protocol::encode([ diff --git a/tests/Feed/Unit/Support/MidPollJournal.php b/tests/Feed/Unit/Support/MidPollJournal.php new file mode 100644 index 0000000..d42c2ed --- /dev/null +++ b/tests/Feed/Unit/Support/MidPollJournal.php @@ -0,0 +1,31 @@ +reads++ === 1) { + $this->append(new CloudEvent(id: '', type: $this->lands, source: 'urn:test')); + } + + return parent::read($lastEventId, $limit); + } +} From b89ff18ee31a3282d9b15c8e606d0fdb38d945b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 11:14:48 +0200 Subject: [PATCH 26/68] Add Consumer::seek() to set a consumer's position explicitly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seek($eventId) treats the id as the last event handled: the next consume() starts strictly after it. The position persists immediately through the cursor store and mirrors in memory; a store failure surfaces as Transport with the in-memory position unmoved, so a seek that did not persist never looks like one that did. The id must satisfy Id::isValid β€” the tip sentinel included in the rejects β€” but does not need to exist in the feed, which is what makes seeking to a poison event's own id the deliberate way to step past it. Documented next to reset() in the README, with the poison-event recipe. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 5 ++ README.md | 28 ++++++++ src/Feed/Consumer.php | 28 ++++++++ tests/Feed/Unit/ConsumerTest.php | 118 +++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index edd58b8..10996ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ draining the backlog. Rides a protocol extension: the `lastEventId` value `$`, resolved by the producer to the newest event as the request arrives. Also added `Feed::tip()`, the id of the newest event in a local journal. +- Added `Consumer::seek(string $eventId)` β€” set the position explicitly: the + id is treated as the last event handled, persisted immediately, and the + next `consume()` starts strictly after it. The operational escape hatch for + a poison event: seek to the failing event's own id to step past it + deliberately. - **Breaking (wire format):** a feed batch on the wire is now the plain JSON array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) diff --git a/README.md b/README.md index c364c29..5392697 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,34 @@ try { Run **one process per consumer name.** Two sharing a name share one position, so the feed is split between them rather than delivered to both. +### Moving the position by hand + +Two methods change a consumer's position outside the normal flow: + +- `reset()` forgets the position entirely β€” the next run starts from the + oldest retained event (or the tip, for a `Start::Tip` consumer). +- `seek($eventId)` sets it to a specific point: the id is treated as the last + event handled, so the next run starts strictly *after* it. The seek is + persisted immediately, and a store failure surfaces as `Transport` β€” a seek + that did not persist never looks like one that did. The id must be well + formed but does not need to still exist in the feed. + +`seek()` is the escape hatch for a poison event. A handler that keeps failing +blocks the feed by design, so stepping past it is a decision, made in code: +catch the failure, log the event's id, and once you have decided the event +must be skipped, seek to *its own id*: + +```php +try { + $consumer->consume($handler); +} catch (\Throwable $error) { + Console::error("[feed] blocked: {$error->getMessage()}"); + + // After investigating β€” this event cannot and should not be handled: + $consumer->seek($poisonEventId); +} +``` + ## What a handler must tolerate **A handler must be safe to run twice on the same event.** There are three diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index cd53230..732e3dd 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -90,6 +90,34 @@ public function position(): ?string return $this->position; } + /** + * Set the position explicitly: treat $eventId as the last event handled, + * so the next consume() starts strictly *after* it. + * + * The position is persisted immediately via Cursor::save() and mirrored + * in memory. $eventId must be a well-formed feed position, but does not + * need to currently exist in the feed β€” seeking to an id older than + * retention or newer than the tip is legal and simply positions relative + * to it, which is what makes seeking to a poison event's own id the way + * to step past it deliberately. + * + * @throws Exception\Invalid When $eventId is not a feed position (the tip sentinel included). + * @throws Exception When the cursor store cannot be written β€” a seek that + * did not persist must not look like one that did, so the failure + * is never swallowed and the in-memory position stays put. + */ + public function seek(string $eventId): void + { + if (!Id::isValid($eventId)) { + throw new Exception\Invalid('Invalid feed event id: ' . $eventId); + } + + $this->cursor->save($this->feed->getName(), $this->name, $eventId); + + $this->position = $eventId; + $this->restored = true; + } + public function reset(): void { $this->cursor->reset($this->feed->getName(), $this->name); diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 656a24b..4fe79a8 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -398,6 +398,124 @@ public function testResetReplaysEverythingStillRetained(): void $this->assertSame(['a', 'b'], $this->drain($consumer)); } + public function testSeekPositionsTheNextRunStrictlyAfterTheGivenId(): void + { + $this->producer->append('a'); + $second = $this->producer->append('b'); + $this->producer->append('c'); + + $consumer = $this->consumer(); + $consumer->seek($second); + + $this->assertSame($second, $consumer->position(), 'The seeked id is the position until something is handled'); + $this->assertSame(['c'], $this->drain($consumer)); + } + + /** + * A seek is persisted, not just remembered: a fresh Consumer sharing the + * store and the name β€” a restart β€” resumes from it. + */ + public function testASeekSurvivesARestart(): void + { + $this->producer->append('a'); + $second = $this->producer->append('b'); + $this->producer->append('c'); + + $this->consumer()->seek($second); + + $this->assertSame(['c'], $this->drain($this->consumer())); + } + + /** + * The operational escape hatch seek() exists for: a handler that keeps + * failing blocks the feed by design, and stepping past it is a deliberate + * seek to the failing event's own id. + */ + public function testSeekingToAPoisonEventsIdUnblocksTheConsumer(): void + { + $this->producer->append('poison'); + $this->producer->append('after'); + + $consumer = $this->consumer(); + $poison = null; + + $handler = function (CloudEvent $event) use (&$poison): void { + if ($event->type === 'poison') { + $poison = $event->id; + + throw new \RuntimeException('cannot handle this one'); + } + }; + + try { + $consumer->consume($handler); + $this->fail('The poison event should have blocked the run'); + } catch (\RuntimeException) { + // Expected: the feed is now blocked at the poison event. + } + + $this->assertNotNull($poison); + $consumer->seek($poison); + + $this->assertSame(['after'], $this->drain($consumer), 'The poison event is stepped over, nothing behind it is lost'); + } + + /** + * @dataProvider notPositions + */ + public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void + { + $first = $this->producer->append('a'); + $this->cursor->save('edge', 'invalidator', $first); + + $consumer = $this->consumer(); + + try { + $consumer->seek($id); + $this->fail('The id should have been rejected'); + } catch (Invalid) { + // Expected. + } + + $this->assertSame($first, $this->cursor->load('edge', 'invalidator'), 'A rejected seek leaves the stored position untouched'); + $this->assertSame($first, $consumer->position()); + } + + /** + * @return array + */ + public static function notPositions(): array + { + return [ + 'empty' => [''], + 'not an id' => ['abc'], + 'too many parts' => ['1-2-3'], + 'the tip sentinel' => ['$'], + ]; + } + + /** + * A seek that did not persist must not look like one that did: the store + * failure surfaces, and the in-memory position stays where it was. + */ + public function testASeekThatCannotPersistFailsLoudlyAndMovesNothing(): void + { + $this->producer->append('a'); + + $consumer = $this->consumer(new FailingCursor(onSave: true)); + + $this->assertNull($consumer->position()); + + try { + $consumer->seek('1-0'); + $this->fail('The store failure should have been raised'); + } catch (Transport) { + // Expected. + } + + $this->assertNull($consumer->position(), 'The in-memory position must not move on a failed seek'); + } + public function testConsumersOfTheSameFeedTrackSeparatePositions(): void { $this->producer->append('a'); From 488ec87039a5d243230616e8bd489fdd1ef05cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 11:16:50 +0200 Subject: [PATCH 27/68] Make the journal poll interval configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The long-poll read-sleep loop's interval is now a trailing constructor option on Journal\Redis, Pool and Memory β€” pollInterval, in milliseconds for consistency with every other duration in the API, default 500. An interval below 1 ms throws Exception\Invalid rather than silently clamping a busy-spin into existence. Journal\Http and Journal\None keep their signatures: the producer does the waiting for one, and the other only throws. While touching the loop, the deadline overshoot is fixed: poll() now sleeps min(interval, remaining), so a timeout shorter than the interval is honoured to within scheduler precision instead of returning up to a full interval late. Pool passes the interval through to its per-read inner journal so a future change there cannot silently drop it, and the README now explains why the loop polls instead of holding a connection in XREAD BLOCK. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 6 +++ README.md | 7 ++++ src/Feed/Journal.php | 27 ++++++++++--- src/Feed/Journal/Memory.php | 9 +++-- src/Feed/Journal/Pool.php | 17 +++++++-- src/Feed/Journal/Redis.php | 3 +- tests/Feed/Unit/FeedTest.php | 44 ++++++++++++++++++++++ tests/Feed/Unit/Support/MidPollJournal.php | 4 +- 8 files changed, 102 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10996ec..bc673b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,12 @@ next `consume()` starts strictly after it. The operational escape hatch for a poison event: seek to the failing event's own id to step past it deliberately. +- The long-poll read interval is now a constructor option on `Journal\Redis`, + `Journal\Pool` and `Journal\Memory`: `pollInterval`, in milliseconds, + default 500. An interval below 1 ms throws `Exception\Invalid`. The poll + loop also no longer oversleeps: it sleeps the remaining time when less than + an interval is left, so a timeout is honoured to within scheduler precision + instead of running up to one interval late. - **Breaking (wire format):** a feed batch on the wire is now the plain JSON array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) diff --git a/README.md b/README.md index 5392697..21f8c42 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,13 @@ the ones that own their events also implement `Appendable` and assign the ids. whole timeout, and this one borrows a connection per read and gives it back while it waits, so polling never ties up the client the rest of the service is using. +A long poll is served by re-reading the journal on an interval β€” `pollInterval`, +a trailing constructor option in milliseconds, 500 by default. A shorter +interval lowers long-poll latency and raises the read rate against the backend. +The loop is deliberate: Redis could wait server-side with `XREAD BLOCK`, but a +blocking read holds the connection for the whole wait, which is exactly what +`Journal\Pool`'s borrow-per-read strategy exists to avoid. + `Journal\Redis` and `Journal\Pool` trim the stream to about `maxSize` entries (100,000 by default, and the same for `Journal\Memory`; Redis trims approximately, so the stream may run a little longer). That cap is the feed's diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index 9cf9775..7078bc6 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -12,13 +12,26 @@ // feed over the wire. Journals that own their events also implement Appendable. abstract class Journal { - protected const int POLL_INTERVAL = 500_000; // 0.5s + protected const int POLL_INTERVAL = 500; // ms - public function __construct(protected readonly string $name) - { + /** + * @param int $pollInterval How long poll() sleeps between reads, in + * milliseconds. Shorter lowers long-poll latency and raises the + * backend read rate. + */ + public function __construct( + protected readonly string $name, + protected readonly int $pollInterval = self::POLL_INTERVAL, + ) { if ($name === '') { throw new Invalid('Feed name is required'); } + + // A zero interval is a busy-spin against the backend; clamping + // silently would hide the misconfiguration. + if ($pollInterval < 1) { + throw new Invalid('Feed poll interval must be at least 1 millisecond'); + } } public function getName(): string @@ -66,11 +79,15 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array while (true) { $events = $this->read($lastEventId, $limit); - if ($events !== [] || \microtime(true) >= $deadline) { + $remaining = $deadline - \microtime(true); + + if ($events !== [] || $remaining <= 0) { return $events; } - \usleep(self::POLL_INTERVAL); + // Never sleep past the deadline: the timeout is honoured to + // within scheduler precision, not to within one interval. + \usleep((int) \min($this->pollInterval * 1000, \ceil($remaining * 1_000_000))); } } diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Journal/Memory.php index b0a7a7d..bfaea64 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Journal/Memory.php @@ -17,9 +17,12 @@ class Memory extends Journal implements Appendable private int $timestamp = 0; private int $sequence = -1; - public function __construct(string $name, protected readonly int $maxSize = 100_000) - { - parent::__construct($name); + public function __construct( + string $name, + protected readonly int $maxSize = 100_000, + int $pollInterval = self::POLL_INTERVAL, + ) { + parent::__construct($name, $pollInterval); } public function append(CloudEvent $event): string diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Journal/Pool.php index 51843e3..6d945a1 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Journal/Pool.php @@ -18,28 +18,37 @@ public function __construct( protected readonly UtopiaPool $pool, string $name, protected readonly int $maxSize = 100_000, + int $pollInterval = self::POLL_INTERVAL, ) { - parent::__construct($name); + parent::__construct($name, $pollInterval); + } + + // The interval only matters in this class's own inherited poll() loop β€” + // the inner journal lives for a single read β€” but it is passed through so + // a future change to the inner journal cannot silently drop it. + private function inner(\Redis|\RedisCluster $redis): Redis + { + return new Redis($redis, $this->name, $this->maxSize, $this->pollInterval); } public function append(CloudEvent $event): string { return $this->pool->use( - fn (\Redis|\RedisCluster $redis): string => (new Redis($redis, $this->name, $this->maxSize))->append($event) + fn (\Redis|\RedisCluster $redis): string => $this->inner($redis)->append($event) ); } public function tip(): ?string { return $this->pool->use( - fn (\Redis|\RedisCluster $redis): ?string => (new Redis($redis, $this->name, $this->maxSize))->tip() + fn (\Redis|\RedisCluster $redis): ?string => $this->inner($redis)->tip() ); } public function read(?string $lastEventId, int $limit): array { return $this->pool->use( - fn (\Redis|\RedisCluster $redis): array => (new Redis($redis, $this->name, $this->maxSize))->read($lastEventId, $limit) + fn (\Redis|\RedisCluster $redis): array => $this->inner($redis)->read($lastEventId, $limit) ); } } diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Journal/Redis.php index 77496ca..05bcda2 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Journal/Redis.php @@ -16,8 +16,9 @@ public function __construct( protected readonly \Redis|\RedisCluster $redis, string $name, protected readonly int $maxSize = 100_000, + int $pollInterval = self::POLL_INTERVAL, ) { - parent::__construct($name); + parent::__construct($name, $pollInterval); } public function append(CloudEvent $event): string diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index 1b852ae..c55f1c4 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -15,6 +15,7 @@ use Utopia\Feed\Producer; use Utopia\Feed\Protocol; use Utopia\Feed\Id; +use Utopia\Tests\Unit\Support\MidPollJournal; class FeedTest extends TestCase { @@ -263,6 +264,49 @@ public function testPollWithoutATimeoutIsAPlainRead(): void $this->assertLessThan(0.4, \microtime(true) - $started); } + public function testAShorterPollIntervalDeliversAMidPollEventSooner(): void + { + $journal = new MidPollJournal('edge', pollInterval: 20); + + $started = \microtime(true); + $events = $journal->poll(null, 10, 5_000); + $elapsed = \microtime(true) - $started; + + $this->assertCount(1, $events); + $this->assertLessThan(0.4, $elapsed, 'A 20ms interval must beat the default 500ms floor'); + } + + /** + * The overshoot fix: the loop must sleep the remaining time when that is + * less than the interval, not a full interval past the deadline. + */ + public function testPollHonoursATimeoutShorterThanTheInterval(): void + { + $journal = new Memory('edge', pollInterval: 500); + + $started = \microtime(true); + $events = $journal->poll(null, 10, 100); + $elapsed = \microtime(true) - $started; + + $this->assertSame([], $events); + $this->assertGreaterThanOrEqual(0.08, $elapsed, 'Must actually wait out the timeout'); + $this->assertLessThan(0.3, $elapsed, 'Must not sleep a full interval past the deadline'); + } + + public function testRejectsAZeroPollInterval(): void + { + $this->expectException(Invalid::class); + + new Memory('edge', pollInterval: 0); + } + + public function testRejectsANegativePollInterval(): void + { + $this->expectException(Invalid::class); + + new Memory('edge', pollInterval: -5); + } + public function testRetentionIsBoundedAndTrimsTheOldest(): void { $journal = new Memory('small', maxSize: 3); diff --git a/tests/Feed/Unit/Support/MidPollJournal.php b/tests/Feed/Unit/Support/MidPollJournal.php index d42c2ed..d1d24b8 100644 --- a/tests/Feed/Unit/Support/MidPollJournal.php +++ b/tests/Feed/Unit/Support/MidPollJournal.php @@ -15,9 +15,9 @@ class MidPollJournal extends Memory { private int $reads = 0; - public function __construct(string $name, private readonly string $lands = 'landed') + public function __construct(string $name, private readonly string $lands = 'landed', int $pollInterval = self::POLL_INTERVAL) { - parent::__construct($name); + parent::__construct($name, pollInterval: $pollInterval); } public function read(?string $lastEventId, int $limit): array From 3f122e882f800266ec403c39cfaf1e86e24f7d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 11:25:47 +0200 Subject: [PATCH 28/68] Give each side of the wire its own vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every user-facing name now belongs to exactly one side: a Producer appends to a Journal and a Feed serves that journal over HTTP; a Consumer reads a Remote feed and keeps its place in a Cursor. The new Readable interface (read, poll, tip, getName) is the contract of "something a feed's events can be read from" β€” the counterpart of Appendable. Journal implements it, and Journal\Http moves to Utopia\Feed\Remote: a standalone Readable that is deliberately not Appendable, so producing into someone else's feed stays a type error. After the move, everything named Journal\* can be appended to. Consumer accepts any Readable β€” a Remote, or a local journal, with no Feed wrapper on the client path β€” and clamps its own batch and timeout to the protocol limits, which moved with the responsibility: Feed::MAX_BATCH and MAX_TIMEOUT are now Protocol::MAX_BATCH and Protocol::MAX_TIMEOUT. Feed itself is server vocabulary, built over the journal it serves. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 10 ++ README.md | 84 +++++++------ src/Feed/Appendable.php | 2 +- src/Feed/Consumer.php | 17 ++- src/Feed/Feed.php | 22 ++-- src/Feed/Journal.php | 15 +-- src/Feed/Producer.php | 2 +- src/Feed/Protocol.php | 6 + src/Feed/Readable.php | 43 +++++++ src/Feed/{Journal/Http.php => Remote.php} | 25 ++-- tests/Feed/E2E/RedisTest.php | 21 ++-- tests/Feed/Unit/ConsumerTest.php | 55 +++++--- tests/Feed/Unit/FeedTest.php | 6 +- tests/Feed/Unit/ProducerTest.php | 14 +-- .../{HttpJournalTest.php => RemoteTest.php} | 119 +++++++++++------- tests/Feed/Unit/RoundTripTest.php | 10 +- 16 files changed, 286 insertions(+), 165 deletions(-) create mode 100644 src/Feed/Readable.php rename src/Feed/{Journal/Http.php => Remote.php} (80%) rename tests/Feed/Unit/{HttpJournalTest.php => RemoteTest.php} (72%) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc673b6..fb4e5f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,16 @@ loop also no longer oversleeps: it sleeps the remaining time when less than an interval is left, so a timeout is honoured to within scheduler precision instead of running up to one interval late. +- **Breaking (renames):** every user-facing name now belongs to exactly one + side of the wire. `Journal\Http` is gone; its replacement is + `Utopia\Feed\Remote` β€” another service's feed, over HTTP β€” a standalone + class implementing the new `Readable` interface (`read`, `poll`, `tip`, + `getName`) rather than posing as a journal. `Consumer` accepts any + `Readable` (a `Remote`, or a local journal) instead of a `Feed`, and clamps + its own `batch`/`timeout`; `Feed` is server vocabulary, built over the + journal it serves. The protocol limits moved with the responsibility: + `Feed::MAX_BATCH`/`Feed::MAX_TIMEOUT` are now `Protocol::MAX_BATCH` and + `Protocol::MAX_TIMEOUT`. - **Breaking (wire format):** a feed batch on the wire is now the plain JSON array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) diff --git a/README.md b/README.md index 21f8c42..1974239 100644 --- a/README.md +++ b/README.md @@ -73,26 +73,50 @@ runtime surprise. ### Consume -A `Consumer` reads from where it last got to, hands each new event to your -handler, and records how far it got: +A `Consumer` pulls from a `Remote` β€” another service's feed, over HTTP β€” reads +from where it last got to, hands each new event to your handler, and records +how far it got in a `Cursor`: ```php +use Utopia\Client; +use Utopia\Client\Adapter\Curl\Client as Curl; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; +use Utopia\Feed\Remote; -$consumer = new Consumer($feed, 'cache-invalidator', new Cursor\Cache($cache)); +$client = (new Client(new Curl())) + ->withHeaders(['x-appwrite-jwt' => $token]) + ->withConnectionReuse(); + +$remote = new Remote($client, 'https://cloud.example.com/v1/feeds', 'edge'); + +$consumer = new Consumer($remote, 'cache-invalidator', new Cursor\Cache($cache)); $handled = $consumer->consume(function (CloudEvent $event) use ($router) { $router->invalidate($event->data['tags'] ?? []); }); ``` +Long polling is handled by the producer, so a poll is one held request rather +than a client-side loop. `Remote` takes any +[utopia-php/client](https://github.com/utopia-php/client) adapter, so a pooled or +Swoole coroutine transport drops straight in. Leave the `Retry` decorator off: a +failed read leaves the position where it was, so the next poll is already the +retry. + +A consumer inside the producing service reads its own feed the same way β€” +`Consumer` accepts anything `Readable`, so it takes the local journal directly: + +```php +$consumer = new Consumer($journal, 'audit-log', new Cursor\Redis($redis)); +``` + Call `consume()` on a timer, or give the consumer a `timeout` and loop β€” each call then returns the moment an event arrives, or empty after the timeout: ```php -$consumer = new Consumer($feed, 'cache-invalidator', $cursor, timeout: 20_000); +$consumer = new Consumer($remote, 'cache-invalidator', $cursor, timeout: 20_000); while (true) { $consumer->consume($handler); @@ -106,7 +130,7 @@ they happen β€” opts into starting at the tip instead: ```php use Utopia\Feed\Start; -$consumer = new Consumer($feed, 'notifier', $cursor, timeout: 20_000, start: Start::Tip); +$consumer = new Consumer($remote, 'notifier', $cursor, timeout: 20_000, start: Start::Tip); ``` A stored position always wins; `Start::Tip` only applies on the first run, or @@ -117,29 +141,6 @@ gap between polls. Against a producer that predates the tip extension, the first poll fails with a 4xx `Transport` error rather than silently replaying the backlog. -### Consume another service's feed - -Same code, different journal β€” nothing above it knows the events arrive over the -network: - -```php -use Utopia\Client; -use Utopia\Client\Adapter\Curl\Client as Curl; - -$client = (new Client(new Curl())) - ->withHeaders(['x-appwrite-jwt' => $token]) - ->withConnectionReuse(); - -$feed = new Feed(new Journal\Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); -``` - -Long polling is handled by the producer, so a poll is one held request rather -than a client-side loop. `Journal\Http` takes any -[utopia-php/client](https://github.com/utopia-php/client) adapter, so a pooled or -Swoole coroutine transport drops straight in. Leave the `Retry` decorator off: a -failed read leaves the position where it was, so the next poll is already the -retry. - ### Serve a feed over HTTP The feed a service serves is the read half of the same journal it appends to. @@ -162,10 +163,9 @@ $response `serve()` extracts `lastEventId`, `limit` and `timeout` from the query, coerces their string values, applies the defaults, and clamps the batch to -`Feed::MAX_BATCH` (1000 events) and the long-poll wait to `Feed::MAX_TIMEOUT` -(30s), so a client cannot ask for more than the producer is willing to build -or hold. A malformed `lastEventId` throws `Exception\Invalid` β€” catch it to -answer 400. +1000 events and the long-poll wait to 30 seconds, so a client cannot ask for +more than the producer is willing to build or hold. A malformed `lastEventId` +throws `Exception\Invalid` β€” catch it to answer 400. The response body is a bare JSON array of CloudEvents, as [http-feeds.org](https://www.http-feeds.org/) defines it β€” no envelope. An @@ -218,16 +218,18 @@ usable prefix β€” does the read throw `Exception\Invalid`. ## Journals -A journal is where a feed's events live. It returns the events after a given id; -the ones that own their events also implement `Appendable` and assign the ids. +A journal is where a feed's events live, on the server that owns it. Every +journal is `Readable` β€” it returns the events after a given id β€” and +`Appendable`, assigning the ids. (Reading another service's feed is not a +journal's job: that is `Remote`, which is `Readable` and deliberately not +`Appendable`.) -| Journal | Use for | `Appendable` | -| --- | --- | --- | -| `Journal\Redis` | Producing a feed on a Redis stream | βœ… | -| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | βœ… | -| `Journal\Http` | Consuming another service's feed | ❌ β€” it belongs to whoever appends to it | -| `Journal\Memory` | Tests and single-process development | βœ… | -| `Journal\None` | No backend configured β€” throws on use | βœ…, and throws | +| Journal | Use for | +| --- | --- | +| `Journal\Redis` | Producing a feed on a Redis stream | +| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | +| `Journal\Memory` | Tests and single-process development | +| `Journal\None` | No backend configured β€” throws on use | `Journal\Pool` is what most services producing a feed want: a long poll spans its whole timeout, and this one borrows a connection per read and gives it back while diff --git a/src/Feed/Appendable.php b/src/Feed/Appendable.php index 8a729c5..fbc0057 100644 --- a/src/Feed/Appendable.php +++ b/src/Feed/Appendable.php @@ -7,7 +7,7 @@ use Utopia\CloudEvents\CloudEvent; // Server interface: a journal that owns its events, so it can be appended to. -// Journal\Http does not implement it β€” a consumer cannot write to someone else's feed. +// Remote does not implement it β€” a consumer cannot write to someone else's feed. interface Appendable { /** diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 732e3dd..070651c 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -5,7 +5,6 @@ namespace Utopia\Feed; // Client class: the pull loop β€” reads what it has not seen and records how far it got. -// Also runs server-side, when a job inside the producer consumes the feed it produces. class Consumer { public const int BATCH = 100; @@ -14,8 +13,12 @@ class Consumer private bool $restored = false; + /** + * @param Readable $feed The feed to pull from β€” a Remote for another + * service's feed, or a local journal for one this service owns. + */ public function __construct( - protected readonly Feed $feed, + protected readonly Readable $feed, protected readonly string $name, protected readonly Cursor $cursor, protected readonly int $batch = self::BATCH, @@ -34,9 +37,13 @@ public function getName(): string public function consume(callable $handler): int { - $batch = $this->feed->poll($this->position() ?? $this->origin(), $this->batch, $this->timeout); + $events = $this->feed->poll( + $this->position() ?? $this->origin(), + \max(1, \min($this->batch, Protocol::MAX_BATCH)), + \max(0, \min($this->timeout, Protocol::MAX_TIMEOUT)), + ); - if ($batch->isEmpty()) { + if ($events === []) { return 0; } @@ -44,7 +51,7 @@ public function consume(callable $handler): int $processed = null; $failure = null; - foreach ($batch as $event) { + foreach ($events as $event) { try { $handler($event); } catch (\Throwable $error) { diff --git a/src/Feed/Feed.php b/src/Feed/Feed.php index 566581d..daf2a06 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Feed.php @@ -4,15 +4,11 @@ namespace Utopia\Feed; -// Server and client class: the read view of a feed β€” read and long-poll. -// Server serves its own feed with this; client reads a remote one through Journal\Http. +// Server class: the read view over the journal a service appends to β€” it +// reads, long-polls, and serves the feed over HTTP with serve(). class Feed { - public const int MAX_BATCH = 1000; - - public const int MAX_TIMEOUT = 30_000; - - public function __construct(protected readonly Journal $journal) + public function __construct(protected readonly Readable $journal) { } @@ -32,19 +28,19 @@ public function tip(): ?string return $this->journal->tip(); } - public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): Batch + public function read(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH): Batch { - $limit = \max(1, \min($limit, self::MAX_BATCH)); + $limit = \max(1, \min($limit, Protocol::MAX_BATCH)); return new Batch($this->journal->read($lastEventId, $limit), $limit); } - public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, int $timeout = 0): Batch + public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH, int $timeout = 0): Batch { - $limit = \max(1, \min($limit, self::MAX_BATCH)); + $limit = \max(1, \min($limit, Protocol::MAX_BATCH)); return new Batch( - $this->journal->poll($lastEventId, $limit, \max(0, \min($timeout, self::MAX_TIMEOUT))), + $this->journal->poll($lastEventId, $limit, \max(0, \min($timeout, Protocol::MAX_TIMEOUT))), $limit, ); } @@ -73,7 +69,7 @@ public function serve(array $query): Batch return $this->poll( $lastEventId, - \is_numeric($limit) ? (int) $limit : self::MAX_BATCH, + \is_numeric($limit) ? (int) $limit : Protocol::MAX_BATCH, \is_numeric($timeout) ? (int) $timeout : 0, ); } diff --git a/src/Feed/Journal.php b/src/Feed/Journal.php index 7078bc6..1c19897 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Journal.php @@ -8,9 +8,9 @@ use Utopia\Feed\Exception\Invalid; // Server class: durable storage for the events β€” Journal\Redis, Pool, Memory. -// The one client-side journal is Journal\Http, which reads another service's -// feed over the wire. Journals that own their events also implement Appendable. -abstract class Journal +// A journal owns its events, so every journal also implements Appendable; +// reading another service's feed over the wire is Remote's job. +abstract class Journal implements Readable { protected const int POLL_INTERVAL = 500; // ms @@ -42,11 +42,6 @@ public function getName(): string /** @return list */ abstract public function read(?string $lastEventId, int $limit): array; - /** - * The id of the newest event, or null when the feed is empty. - * - * @throws Exception - */ abstract public function tip(): ?string; /** @@ -62,8 +57,8 @@ protected function resolve(?string $lastEventId): ?string /** * Wait for events, re-reading on an interval until some land or the - * deadline passes. Journal\Http overrides this: there the producer does the - * waiting, so a poll is one held request. + * deadline passes. (Remote does not share this loop: there the producer + * does the waiting, so a poll is one held request.) * * @return list */ diff --git a/src/Feed/Producer.php b/src/Feed/Producer.php index 18f98fc..d2d2eae 100644 --- a/src/Feed/Producer.php +++ b/src/Feed/Producer.php @@ -7,7 +7,7 @@ use Utopia\CloudEvents\CloudEvent; // Server class: appends events to a feed this service owns. -// The journal must be Appendable, so a remote feed cannot reach this at all. +// The journal must be a Journal and Appendable, so a Remote cannot reach this at all. class Producer { /** diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 642adc7..76261df 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -32,6 +32,12 @@ final class Protocol public const int TIMEOUT_MARGIN = 10_000; + /** The most events one batch may carry β€” servers clamp `limit` to this. */ + public const int MAX_BATCH = 1000; + + /** The longest a long poll may hold a connection, in milliseconds. */ + public const int MAX_TIMEOUT = 30_000; + /** The context attributes this library models; the rest are extensions. */ private const array ATTRIBUTES = [ 'specversion', diff --git a/src/Feed/Readable.php b/src/Feed/Readable.php new file mode 100644 index 0000000..aeebb9c --- /dev/null +++ b/src/Feed/Readable.php @@ -0,0 +1,43 @@ + + * + * @throws Exception When the events cannot be read. + */ + public function read(?string $lastEventId, int $limit): array; + + /** + * Like read(), but waits up to $timeout milliseconds for events to land + * before answering empty. + * + * @return list + * + * @throws Exception When the events cannot be read. + */ + public function poll(?string $lastEventId, int $limit, int $timeout): array; + + /** + * The id of the newest event, or null when the feed is empty. + * + * @throws Exception When the tip cannot be read, or only its owner can resolve it. + */ + public function tip(): ?string; +} diff --git a/src/Feed/Journal/Http.php b/src/Feed/Remote.php similarity index 80% rename from src/Feed/Journal/Http.php rename to src/Feed/Remote.php index 4dca6d8..0e0a5aa 100644 --- a/src/Feed/Journal/Http.php +++ b/src/Feed/Remote.php @@ -2,33 +2,42 @@ declare(strict_types=1); -namespace Utopia\Feed\Journal; +namespace Utopia\Feed; use Psr\Http\Client\ClientExceptionInterface; use Utopia\Client\Adapter; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; -use Utopia\Feed\Journal; -use Utopia\Feed\Protocol; use Utopia\Psr7\Header; use Utopia\Psr7\Method; use Utopia\Psr7\Request\Factory as RequestFactory; -class Http extends Journal +// Client class: another service's feed, read over the wire. +// Deliberately Readable and not Appendable β€” events are appended by whoever +// owns the feed, so producing into a remote one is a type error, not a request. +class Remote implements Readable { private readonly RequestFactory $requests; public function __construct( protected readonly Adapter $client, protected readonly string $endpoint, - string $name, + protected readonly string $name, ) { - parent::__construct($name); + if ($name === '') { + throw new Invalid('Feed name is required'); + } $this->requests = new RequestFactory(); } + public function getName(): string + { + return $this->name; + } + /** * Never called on the consumer path: the tip sentinel is passed through * as `lastEventId=$` and the producer resolves it inside the same @@ -39,12 +48,12 @@ public function tip(): ?string throw new Unsupported("The {$this->name} feed is remote; its producer resolves the tip"); } - public function read(?string $lastEventId, int $limit): array + public function read(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH): array { return $this->fetch($lastEventId, $limit, 0); } - public function poll(?string $lastEventId, int $limit, int $timeout): array + public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH, int $timeout = 0): array { return $this->fetch($lastEventId, $limit, $timeout); } diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index 313e1f9..e50fd42 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -260,7 +260,8 @@ public function testTheTipSentinelSkipsTheBacklog(): void public function testConsumesThroughAPersistedCursor(): void { - [$producer, $feed] = $this->feedAndProducer(); + $journal = new RedisJournal($this->redis, $this->name); + $producer = new Producer($journal, 'urn:test:e2e'); $cursor = new RedisCursor($this->redis); $producer->append('a'); @@ -271,40 +272,42 @@ public function testConsumesThroughAPersistedCursor(): void $seen[] = $event->type; }; - $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume($handler)); + $this->assertSame(2, (new Consumer($journal, 'invalidator', $cursor))->consume($handler)); $this->assertSame($last, $cursor->load($this->name, 'invalidator')); // A second Consumer stands in for a restart: it has no in-memory // position, so it has to pick the stored one up to avoid replaying. - $this->assertSame(0, (new Consumer($feed, 'invalidator', $cursor))->consume($handler)); + $this->assertSame(0, (new Consumer($journal, 'invalidator', $cursor))->consume($handler)); $this->assertSame(['a', 'b'], $seen); } public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void { - [$producer, $feed] = $this->feedAndProducer(); + $journal = new RedisJournal($this->redis, $this->name); + $producer = new Producer($journal, 'urn:test:e2e'); $cursor = new RedisCursor($this->redis); $producer->append('a'); - $this->assertSame(1, (new Consumer($feed, 'one', $cursor))->consume(fn (CloudEvent $e) => null)); - $this->assertSame(1, (new Consumer($feed, 'two', $cursor))->consume(fn (CloudEvent $e) => null)); + $this->assertSame(1, (new Consumer($journal, 'one', $cursor))->consume(fn (CloudEvent $e) => null)); + $this->assertSame(1, (new Consumer($journal, 'two', $cursor))->consume(fn (CloudEvent $e) => null)); } public function testResetReplaysTheRetainedFeed(): void { - [$producer, $feed] = $this->feedAndProducer(); + $journal = new RedisJournal($this->redis, $this->name); + $producer = new Producer($journal, 'urn:test:e2e'); $cursor = new RedisCursor($this->redis); $producer->append('a'); $producer->append('b'); - $consumer = new Consumer($feed, 'invalidator', $cursor); + $consumer = new Consumer($journal, 'invalidator', $cursor); $consumer->consume(fn (CloudEvent $e) => null); $consumer->reset(); $this->assertNull($cursor->load($this->name, 'invalidator')); - $this->assertSame(2, (new Consumer($feed, 'invalidator', $cursor))->consume(fn (CloudEvent $e) => null)); + $this->assertSame(2, (new Consumer($journal, 'invalidator', $cursor))->consume(fn (CloudEvent $e) => null)); } public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 4fe79a8..c1a2853 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -12,8 +12,8 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; -use Utopia\Feed\Feed; use Utopia\Feed\Producer; +use Utopia\Feed\Protocol; use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FailingCursor; use Utopia\Tests\Unit\Support\MidPollJournal; @@ -22,8 +22,6 @@ class ConsumerTest extends TestCase { private MemoryJournal $journal; - private Feed $feed; - private Producer $producer; private MemoryCursor $cursor; @@ -32,13 +30,12 @@ protected function setUp(): void { $this->journal = new MemoryJournal('edge'); $this->producer = new Producer($this->journal, 'urn:test'); - $this->feed = new Feed($this->journal); $this->cursor = new MemoryCursor(); } private function consumer(?Cursor $cursor = null, int $batch = Consumer::BATCH): Consumer { - return new Consumer($this->feed, 'invalidator', $cursor ?? $this->cursor, $batch); + return new Consumer($this->journal, 'invalidator', $cursor ?? $this->cursor, $batch); } /** @@ -204,6 +201,34 @@ public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void $this->assertSame(3, $this->consumer()->consume(fn (CloudEvent $event) => null)); } + /** + * The clamping the client-side Feed wrapper used to provide lives in the + * consumer now: whatever the constructor was given, a journal is never + * asked for more than the protocol allows. + */ + public function testClampsBatchAndTimeoutToTheProtocolLimits(): void + { + $journal = new class ('edge') extends MemoryJournal { + public ?int $limit = null; + + public ?int $timeout = null; + + public function poll(?string $lastEventId, int $limit, int $timeout): array + { + $this->limit = $limit; + $this->timeout = $timeout; + + return parent::poll($lastEventId, $limit, 0); + } + }; + + $consumer = new Consumer($journal, 'invalidator', $this->cursor, batch: 5_000, timeout: 120_000); + $consumer->consume(fn (CloudEvent $event) => null); + + $this->assertSame(Protocol::MAX_BATCH, $journal->limit); + $this->assertSame(Protocol::MAX_TIMEOUT, $journal->timeout); + } + public function testDrainsABacklogInBatches(): void { foreach (\range(1, 10) as $i) { @@ -306,7 +331,7 @@ public function testTipStartDoesNotAnnounceTheBacklog(): void $this->producer->append('old-1'); $this->producer->append('old-2'); - $consumer = new Consumer($this->feed, 'notifier', $this->cursor, start: Start::Tip); + $consumer = new Consumer($this->journal, 'notifier', $this->cursor, start: Start::Tip); $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); $this->assertNull($this->cursor->load('edge', 'notifier'), 'Skipping the backlog is not progress to commit'); @@ -322,7 +347,7 @@ public function testTipStartDeliversWhatLandsMidPoll(): void (new Producer($journal, 'urn:test'))->append('old'); $cursor = new MemoryCursor(); - $consumer = new Consumer(new Feed($journal), 'notifier', $cursor, timeout: 5_000, start: Start::Tip); + $consumer = new Consumer($journal, 'notifier', $cursor, timeout: 5_000, start: Start::Tip); $seen = []; $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { @@ -341,7 +366,7 @@ public function testAStoredCursorBeatsTipStart(): void $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer($this->feed, 'invalidator', $this->cursor, start: Start::Tip); + $consumer = new Consumer($this->journal, 'invalidator', $this->cursor, start: Start::Tip); $this->assertSame(['b'], $this->drain($consumer), 'A restart must not skip the gap'); } @@ -352,7 +377,7 @@ public function testResetWithTipStartResumesFromNow(): void $this->producer->append('b'); $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer($this->feed, 'invalidator', $this->cursor, start: Start::Tip); + $consumer = new Consumer($this->journal, 'invalidator', $this->cursor, start: Start::Tip); $this->assertSame(['b'], $this->drain($consumer), 'The stored position still wins before the reset'); @@ -363,7 +388,7 @@ public function testResetWithTipStartResumesFromNow(): void public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void { - $consumer = new Consumer($this->feed, 'notifier', $this->cursor, timeout: 600, start: Start::Tip); + $consumer = new Consumer($this->journal, 'notifier', $this->cursor, timeout: 600, start: Start::Tip); $started = \microtime(true); @@ -373,7 +398,7 @@ public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void public function testTipStartOnAnEmptyFeedDeliversWhatLandsMidWait(): void { - $consumer = new Consumer(new Feed(new MidPollJournal('edge')), 'notifier', $this->cursor, timeout: 5_000, start: Start::Tip); + $consumer = new Consumer(new MidPollJournal('edge'), 'notifier', $this->cursor, timeout: 5_000, start: Start::Tip); $seen = []; $consumer->consume(function (CloudEvent $event) use (&$seen): void { @@ -520,8 +545,8 @@ public function testConsumersOfTheSameFeedTrackSeparatePositions(): void { $this->producer->append('a'); - $one = new Consumer($this->feed, 'one', $this->cursor); - $two = new Consumer($this->feed, 'two', $this->cursor); + $one = new Consumer($this->journal, 'one', $this->cursor); + $two = new Consumer($this->journal, 'two', $this->cursor); $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); $this->assertSame(1, $two->consume(fn (CloudEvent $event) => null), 'The second consumer has its own position'); @@ -537,7 +562,7 @@ public function testRejectsAnEmptyConsumerName(): void { $this->expectException(Invalid::class); - new Consumer($this->feed, '', $this->cursor); + new Consumer($this->journal, '', $this->cursor); } public function testExposesItsName(): void @@ -555,7 +580,7 @@ public function testAFailedReadLeavesThePositionAlone(): void $this->producer->append('b'); $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer(new Feed(new \Utopia\Feed\Journal\None('edge')), 'invalidator', $this->cursor); + $consumer = new Consumer(new \Utopia\Feed\Journal\None('edge'), 'invalidator', $this->cursor); $this->expectException(\Utopia\Feed\Exception\Unsupported::class); diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/FeedTest.php index c55f1c4..6aa698e 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/FeedTest.php @@ -118,7 +118,7 @@ public function testClampsTheLimitToTheMaximum(): void { $this->producer->append('test'); - $this->assertCount(1, $this->feed->read(null, Feed::MAX_BATCH * 10)); + $this->assertCount(1, $this->feed->read(null, Protocol::MAX_BATCH * 10)); $this->assertCount(1, $this->feed->read(null, 0)); $this->assertCount(1, $this->feed->read(null, -5)); } @@ -430,13 +430,13 @@ public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void */ public function testAnOversizedLimitStillYieldsAnHonestCacheControl(): void { - foreach (\range(1, Feed::MAX_BATCH) as $i) { + foreach (\range(1, Protocol::MAX_BATCH) as $i) { $this->producer->append('event-' . $i); } $batch = $this->feed->serve(['limit' => '5000']); - $this->assertCount(Feed::MAX_BATCH, $batch); + $this->assertCount(Protocol::MAX_BATCH, $batch); $this->assertSame('public, max-age=31536000', $batch->cacheControl(public: true)); } diff --git a/tests/Feed/Unit/ProducerTest.php b/tests/Feed/Unit/ProducerTest.php index d75680b..0e31610 100644 --- a/tests/Feed/Unit/ProducerTest.php +++ b/tests/Feed/Unit/ProducerTest.php @@ -10,10 +10,10 @@ use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; use Utopia\Feed\Id; -use Utopia\Feed\Journal\Http; use Utopia\Feed\Journal\Memory; use Utopia\Feed\Journal\None; use Utopia\Feed\Producer; +use Utopia\Feed\Remote; use Utopia\Tests\Unit\Support\FakeTransport; class ProducerTest extends TestCase @@ -115,18 +115,18 @@ public function testRejectsAnEmptySource(): void } /** - * A feed read over HTTP belongs to whoever appends to it, so it is not - * Appendable β€” the mistake is a type error at construction rather than an - * exception once an event is already in hand. + * A remote feed belongs to whoever appends to it, so Remote is neither a + * Journal nor Appendable β€” the mistake is a type error at construction + * rather than an exception once an event is already in hand. */ - public function testAJournalThatCannotBeAppendedToIsRejectedOnConstruction(): void + public function testARemoteFeedIsRejectedOnConstruction(): void { - $journal = new Http(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', 'edge'); + $remote = new Remote(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', 'edge'); $this->expectException(\TypeError::class); // @phpstan-ignore argument.type - new Producer($journal, 'urn:appwrite:edge:fra'); + new Producer($remote, 'urn:appwrite:edge:fra'); } public function testAFeedWithNoBackendFailsLoudlyRatherThanDroppingEvents(): void diff --git a/tests/Feed/Unit/HttpJournalTest.php b/tests/Feed/Unit/RemoteTest.php similarity index 72% rename from tests/Feed/Unit/HttpJournalTest.php rename to tests/Feed/Unit/RemoteTest.php index dd75b7b..cbc85f7 100644 --- a/tests/Feed/Unit/HttpJournalTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -7,42 +7,42 @@ use PHPUnit\Framework\TestCase; use Psr\Http\Message\ResponseInterface; use Utopia\Client; -use Utopia\Feed\Journal\Http; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Memory as MemoryCursor; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; +use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Feed; use Utopia\Feed\Producer; use Utopia\Feed\Protocol; +use Utopia\Feed\Remote; use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FakeTransport; use Utopia\Tests\Unit\Support\FeedServer; use Utopia\Tests\Unit\Support\MidPollJournal; -class HttpJournalTest extends TestCase +class RemoteTest extends TestCase { /** * @param list $responses - * @return array{Feed, FakeTransport} + * @return array{Remote, FakeTransport} */ - private function feed(array $responses = []): array + private function remote(array $responses = []): array { $transport = FakeTransport::of($responses); - $journal = new Http($transport, 'https://cloud.example.com/v1/feeds', 'edge'); - return [new Feed($journal), $transport]; + return [new Remote($transport, 'https://cloud.example.com/v1/feeds', 'edge'), $transport]; } public function testReadsAFeedOverHttp(): void { - [$feed] = $this->feed([FakeTransport::json(Protocol::encode([ + [$remote] = $this->remote([FakeTransport::json(Protocol::encode([ new CloudEvent(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', source: 'urn:test', data: ['tags' => ['domain' => 'example.com']]), new CloudEvent(id: '1-1', type: 'io.appwrite.edge.invalidate', source: 'urn:test'), ]))]); - $events = \array_values(\iterator_to_array($feed->read())); + $events = $remote->read(); $this->assertCount(2, $events); $this->assertSame('1-0', $events[0]->id); @@ -51,9 +51,9 @@ public function testReadsAFeedOverHttp(): void public function testAppendsTheFeedNameToTheEndpoint(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); - $feed->read(); + $remote->read(); $this->assertStringStartsWith('https://cloud.example.com/v1/feeds/edge', $transport->recorder->last()['uri']); } @@ -62,7 +62,7 @@ public function testEncodesAFeedNameThatNeedsIt(): void { $transport = FakeTransport::of([]); - (new Feed(new Http($transport, 'https://cloud.example.com/v1/feeds/', 'a b/c')))->read(); + (new Remote($transport, 'https://cloud.example.com/v1/feeds/', 'a b/c'))->read(); $this->assertStringStartsWith( 'https://cloud.example.com/v1/feeds/a%20b%2Fc', @@ -70,20 +70,47 @@ public function testEncodesAFeedNameThatNeedsIt(): void ); } + public function testRejectsAnEmptyFeedName(): void + { + $this->expectException(Invalid::class); + + new Remote(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', ''); + } + + public function testExposesTheFeedItReads(): void + { + [$remote] = $this->remote(); + + $this->assertSame('edge', $remote->getName()); + } + + /** + * The tip is the producer's to resolve β€” a consumer starting at the tip + * sends the sentinel instead of asking for the newest id first. + */ + public function testARemoteFeedHasNoLocalTip(): void + { + [$remote] = $this->remote(); + + $this->expectException(Unsupported::class); + + $remote->tip(); + } + public function testReadsWithGet(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); - $feed->read(); + $remote->read(); $this->assertSame('GET', $transport->recorder->last()['method']); } public function testAsksForTheFeedMediaType(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); - $feed->read(); + $remote->read(); $this->assertSame(Protocol::MEDIA_TYPE, $transport->recorder->last()['headers']['Accept'] ?? null); $this->assertSame('application/cloudevents-batch+json', Protocol::MEDIA_TYPE); @@ -91,9 +118,9 @@ public function testAsksForTheFeedMediaType(): void public function testSendsThePositionAndLimit(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); - $feed->read('1-0', 250); + $remote->read('1-0', 250); $uri = $transport->recorder->last()['uri']; @@ -101,11 +128,11 @@ public function testSendsThePositionAndLimit(): void $this->assertStringContainsString('limit=250', $uri); } - public function testSendsNoParametersOnAFirstFullRead(): void + public function testSendsNoPositionOnAFirstFullRead(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); - $feed->read(null, Feed::MAX_BATCH); + $remote->read(null, Protocol::MAX_BATCH); $this->assertStringNotContainsString('lastEventId', $transport->recorder->last()['uri']); } @@ -116,10 +143,10 @@ public function testSendsNoParametersOnAFirstFullRead(): void */ public function testDelegatesLongPollingToTheProducer(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); $started = \microtime(true); - $feed->poll(null, 100, 5000); + $remote->poll(null, 100, 5000); $this->assertLessThan(1, \microtime(true) - $started, 'Must not wait client-side'); $this->assertCount(1, $transport->recorder->requests, 'Must not poll in a loop'); @@ -133,9 +160,9 @@ public function testDelegatesLongPollingToTheProducer(): void */ public function testAllowsTheClientLongerThanTheLongPollTimeout(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); - $feed->poll(null, 100, 5000); + $remote->poll(null, 100, 5000); // Seconds, which is what the client takes; the protocol margin is in // milliseconds, like the timeout the producer is given. @@ -145,9 +172,9 @@ public function testAllowsTheClientLongerThanTheLongPollTimeout(): void public function testLeavesTheConfiguredTimeoutAloneWhenNotLongPolling(): void { - [$feed, $transport] = $this->feed(); + [$remote, $transport] = $this->remote(); - $feed->read(); + $remote->read(); $this->assertNull($transport->recorder->last()['timeout'], 'A plain read must not override the client'); } @@ -159,10 +186,10 @@ public function testLeavesTheConfiguredTimeoutAloneWhenNotLongPolling(): void */ public function testCarriesTheStatusOfARejectedRead(): void { - [$feed] = $this->feed([FakeTransport::json([], 404)]); + [$remote] = $this->remote([FakeTransport::json([], 404)]); try { - $feed->read(); + $remote->read(); $this->fail('A 404 should have been raised'); } catch (Transport $error) { $this->assertSame(404, $error->getCode()); @@ -171,10 +198,10 @@ public function testCarriesTheStatusOfARejectedRead(): void public function testRaisesServerErrors(): void { - [$feed] = $this->feed([FakeTransport::json([], 503)]); + [$remote] = $this->remote([FakeTransport::json([], 503)]); try { - $feed->read(); + $remote->read(); $this->fail('A 503 should have been raised'); } catch (Transport $error) { $this->assertSame(503, $error->getCode()); @@ -182,45 +209,45 @@ public function testRaisesServerErrors(): void } /** - * PSR-18 returns 4xx and 5xx rather than throwing, so the journal has to + * PSR-18 returns 4xx and 5xx rather than throwing, so the transport has to * check the status itself β€” a producer error must not read as an empty * batch, which the consumer would take for "caught up". */ public function testAnErrorStatusIsNotMistakenForAnEmptyBatch(): void { - [$feed] = $this->feed([FakeTransport::json([], 500)]); + [$remote] = $this->remote([FakeTransport::json([], 500)]); $this->expectException(Transport::class); - $feed->read(); + $remote->read(); } public function testWrapsATransportFailure(): void { - [$feed] = $this->feed([FakeTransport::offline()]); + [$remote] = $this->remote([FakeTransport::offline()]); $this->expectException(Transport::class); $this->expectExceptionMessageMatches('/Connection refused/'); - $feed->read(); + $remote->read(); } public function testWrapsABodyThatIsNotJson(): void { - [$feed] = $this->feed([FakeTransport::raw('502 Bad Gateway')]); + [$remote] = $this->remote([FakeTransport::raw('502 Bad Gateway')]); $this->expectException(Transport::class); - $feed->read(); + $remote->read(); } public function testRejectsABodyThatIsNotABatch(): void { - [$feed] = $this->feed([FakeTransport::raw('"a string"')]); + [$remote] = $this->remote([FakeTransport::raw('"a string"')]); $this->expectException(Invalid::class); - $feed->read(); + $remote->read(); } /** @@ -233,21 +260,21 @@ public function testWorksThroughTheClientItself(): void $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-0', type: 'a', source: 'urn:test')]))]); $client = (new Client($transport))->withHeaders(['x-appwrite-jwt' => 'token']); - $feed = new Feed(new Http($client, 'https://cloud.example.com/v1/feeds', 'edge')); + $remote = new Remote($client, 'https://cloud.example.com/v1/feeds', 'edge'); - $events = $feed->read(); + $events = $remote->read(); $this->assertCount(1, $events); $this->assertSame('token', $transport->recorder->last()['headers']['x-appwrite-jwt'] ?? null); } /** - * The point of this journal: a remote feed is consumed with exactly the + * The point of this class: a remote feed is consumed with exactly the * code a local one is. */ public function testConsumesARemoteFeedThroughTheSameConsumer(): void { - [$feed, $transport] = $this->feed([ + [$remote, $transport] = $this->remote([ FakeTransport::json(Protocol::encode([ new CloudEvent(id: '1-0', type: 'a', source: 'urn:test'), new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), @@ -257,7 +284,7 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void ]); $cursor = new MemoryCursor(); - $consumer = new Consumer($feed, 'invalidator', $cursor); + $consumer = new Consumer($remote, 'invalidator', $cursor); $seen = []; $handler = function (CloudEvent $event) use (&$seen): void { @@ -286,8 +313,8 @@ public function testTipStartWorksOverHttp(): void $producer->append('old'); $server = new FeedServer(new Feed($journal)); - $feed = new Feed(new Http($server, 'https://cloud.example.com/v1/feeds', 'edge')); - $consumer = new Consumer($feed, 'notifier', new MemoryCursor(), timeout: 5_000, start: Start::Tip); + $remote = new Remote($server, 'https://cloud.example.com/v1/feeds', 'edge'); + $consumer = new Consumer($remote, 'notifier', new MemoryCursor(), timeout: 5_000, start: Start::Tip); $seen = []; $handler = function (CloudEvent $event) use (&$seen): void { diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index 2150022..a931127 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -7,7 +7,6 @@ use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter\Memory as CacheMemory; use Utopia\Cache\Cache as UtopiaCache; -use Utopia\Feed\Journal\Http; use Utopia\Feed\Journal\Memory as MemoryJournal; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Cache as CacheCursor; @@ -15,6 +14,7 @@ use Utopia\Feed\Feed; use Utopia\Feed\Producer; use Utopia\Feed\Protocol; +use Utopia\Feed\Remote; use Utopia\Tests\Unit\Support\FeedServer; /** @@ -27,7 +27,7 @@ class RoundTripTest extends TestCase { private Producer $producer; - private Feed $consumerFeed; + private Remote $remote; private FeedServer $server; @@ -39,16 +39,14 @@ protected function setUp(): void $this->producer = new Producer($journal, 'urn:appwrite:cloud:fra'); $this->server = new FeedServer(new Feed($journal)); - $this->consumerFeed = new Feed( - new Http($this->server, 'https://cloud.example.com/v1/feeds', 'edge') - ); + $this->remote = new Remote($this->server, 'https://cloud.example.com/v1/feeds', 'edge'); $this->cursor = new CacheCursor(new UtopiaCache(new CacheMemory())); } private function consumer(string $name = 'invalidator', int $batch = Consumer::BATCH): Consumer { - return new Consumer($this->consumerFeed, $name, $this->cursor, $batch); + return new Consumer($this->remote, $name, $this->cursor, $batch); } public function testAnEventSurvivesTheWholeTrip(): void From f23031e6682ed45dbaf4cb85731222d263c590ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 11:28:24 +0200 Subject: [PATCH 29/68] =?UTF-8?q?Rewrite=20the=20README=20=E2=80=94=20simp?= =?UTF-8?q?le,=20straight=20to=20the=20point,=20side-first?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reader picks their side immediately: the two-column "which classes are mine" table comes right after four sentences of what-it-is and the one-sentence model. Each side then gets its complete, runnable code in one block β€” the server example now followed by the wire-format JSON a feed actually returns, which was missing entirely. The handler rules keep their teeth in compact form, reference tables cover journals, cursors, protocol parameters (extensions marked) and exceptions, and everything skippable β€” rollout staging, retention, decode leniency, the poll-loop rationale β€” moves to the fine print. Protocol and the client-side Feed wrapper no longer appear in learning text, and nothing is said twice. Co-Authored-By: Claude Fable 5 --- README.md | 465 ++++++++++++++++++++++++------------------------------ 1 file changed, 210 insertions(+), 255 deletions(-) diff --git a/README.md b/README.md index 1974239..c512156 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,109 @@ # Utopia Feed [![Build Status](https://github.com/utopia-php/feed/actions/workflows/tests.yml/badge.svg)](https://github.com/utopia-php/feed/actions) -![Total Downloads](https://img.shields.io/packagist/dt/utopia-php/feed.svg) [![Discord](https://img.shields.io/discord/564160730845151244)](https://appwrite.io/discord) Utopia Feed moves events between services with **pull-based HTTP event feeds** -([http-feeds.org](https://www.http-feeds.org/)) instead of pushing them to every -service that needs them. +([http-feeds.org](https://www.http-feeds.org/)). A producer appends events to +an ordered log; each consumer polls *"what happened since the last event I +saw?"* and keeps its own position, so the producer stores nothing per consumer +and delivery is **at-least-once**. A consumer that was down catches up on its +next poll. -A producer appends events to an ordered log. Each consumer asks *"what has -happened since the last event I saw?"*, quoting that event's id, and keeps track -of its own position. A consumer that was down catches up on its next poll; a -consumer added later starts from whatever is still retained. The producer stores -nothing per consumer, so nothing about it changes when consumers come and go. +The whole library in one sentence: *a `Producer` appends to a `Journal`, a +`Feed` serves that journal over HTTP; a `Consumer` reads a `Remote` feed and +keeps its place in a `Cursor`.* -The trade is **at-least-once delivery**: every event must be safe to handle -twice. Retention is bounded, so a feed suits events that describe a state to -converge on β€” a cache tag to drop, a record to refresh β€” rather than ones whose -effect depends on seeing every single one. +## Which classes are mine? -This library is maintained by the [Appwrite team](https://appwrite.io). Although -it is part of the [Utopia -Framework](https://github.com/utopia-php/framework), it is dependency light and -works standalone with any PHP project. +| Server (owns the feed) | Client (consumes it) | +| --- | --- | +| `Journal` β€” where events live | `Remote` β€” another service's feed, over HTTP | +| `Producer` β€” appends events | `Consumer` β€” the pull loop | +| `Feed` β€” serves the journal | `Cursor` β€” where the position is kept | -## Getting started +## Install ```bash composer require utopia-php/feed ``` -### Produce +## Serve a feed + +The server side is three objects over one journal β€” where the events live, +here a capped Redis stream: ```php +use Utopia\Feed\Feed; use Utopia\Feed\Journal; use Utopia\Feed\Producer; -$producer = new Producer( - new Journal\Redis($redis, 'edge'), - source: 'urn:appwrite:cloud:fra', -); +$journal = new Journal\Redis($redis, 'edge'); -$id = $producer->append( +// Wherever things happen: +$producer = new Producer($journal, source: 'urn:appwrite:cloud:fra'); + +$producer->append( type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']], subject: 'example.com', ); -``` -`append()` returns the event's id, which is its position in the feed. +// The whole feed route: +$feed = new Feed($journal); -Subclass `Producer` to give it a typed vocabulary, so callers cannot invent an -event type or misspell a payload key: +// GET /v1/feeds/:feedId +$batch = $feed->serve($request->getParams()); -```php -class EdgeProducer extends Producer -{ - public function invalidateRule(string $domain): string - { - return $this->append( - 'io.appwrite.edge.invalidate-rule', - ['tags' => ['domain' => $domain]], - $domain, - ); - } -} +$response + ->addHeader('Content-Type', 'application/cloudevents-batch+json') + ->addHeader('Cache-Control', $batch->cacheControl(public: true)) + ->json($batch->toArray()); +``` + +On the wire, the response is a plain JSON array of +[CloudEvents](https://github.com/utopia-php/cloudevents) β€” no envelope. An +empty array means the consumer is caught up: + +```json +[ + { + "specversion": "1.0", + "type": "io.appwrite.edge.invalidate-rule", + "source": "urn:appwrite:cloud:fra", + "id": "1717689471234-0", + "subject": "example.com", + "time": "2026-07-31T09:15:02.123Z", + "data": { "tags": { "domain": "example.com" } } + }, + { + "specversion": "1.0", + "type": "io.appwrite.edge.invalidate", + "source": "urn:appwrite:cloud:fra", + "id": "1717689471234-1", + "time": "2026-07-31T09:15:02.348Z", + "data": { "tags": { "project": "p1" } } + } +] ``` -`Producer` only accepts a journal that owns its events (one implementing -`Appendable`), so pointing it at a remote feed is a type error rather than a -runtime surprise. +`serve()` reads `lastEventId`, `limit` and `timeout` from the raw query +parameters, coerces and clamps them (at most 1000 events per batch, long polls +held at most 30s), and throws `Exception\Invalid` on a malformed `lastEventId` +β€” catch it to answer 400. `append()` returns the event's id, which is its +position in the feed. Subclass `Producer` to give callers a typed vocabulary +instead of raw type strings. + +The `Batch` that `serve()` (and `Feed::read()`/`poll()`) returns counts and +iterates as its events; `cacheControl()` marks a full batch as immutable +history and everything shorter `no-store`, using the limit the batch was +actually built with, so the header is always honest. -### Consume +## Consume a feed -A `Consumer` pulls from a `Remote` β€” another service's feed, over HTTP β€” reads -from where it last got to, hands each new event to your handler, and records -how far it got in a `Cursor`: +The client side is a `Consumer` pulling a `Remote` feed, with its position in +a `Cursor`. With a `timeout`, each poll is one held request that returns the +moment an event lands (long polling β€” the producer does the waiting): ```php use Utopia\Client; @@ -83,6 +111,7 @@ use Utopia\Client\Adapter\Curl\Client as Curl; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; +use Utopia\Feed\Exception\Transport; use Utopia\Feed\Remote; $client = (new Client(new Curl())) @@ -91,41 +120,38 @@ $client = (new Client(new Curl())) $remote = new Remote($client, 'https://cloud.example.com/v1/feeds', 'edge'); -$consumer = new Consumer($remote, 'cache-invalidator', new Cursor\Cache($cache)); +$consumer = new Consumer($remote, 'cache-invalidator', new Cursor\Cache($cache), timeout: 20_000); -$handled = $consumer->consume(function (CloudEvent $event) use ($router) { - $router->invalidate($event->data['tags'] ?? []); -}); +while (true) { + try { + $consumer->consume(function (CloudEvent $event) use ($router) { + $router->invalidate($event->data['tags'] ?? []); + }); + } catch (Transport $error) { + // Feed or cursor store unreachable. The position did not move, so + // the next poll is already the retry β€” just don't spin. + \usleep(1_000_000); + } +} ``` -Long polling is handled by the producer, so a poll is one held request rather -than a client-side loop. `Remote` takes any -[utopia-php/client](https://github.com/utopia-php/client) adapter, so a pooled or -Swoole coroutine transport drops straight in. Leave the `Retry` decorator off: a -failed read leaves the position where it was, so the next poll is already the -retry. +Events **are** `Utopia\CloudEvents\CloudEvent` objects β€” `data` round-trips +any JSON value, and extension attributes like `traceparent` survive the trip. +Leave the client's `Retry` decorator off: a failed read leaves the position +where it was. A consumer inside the producing service reads its own feed the same way β€” -`Consumer` accepts anything `Readable`, so it takes the local journal directly: +`Consumer` accepts anything `Readable`, so hand it the local journal directly: ```php $consumer = new Consumer($journal, 'audit-log', new Cursor\Redis($redis)); ``` -Call `consume()` on a timer, or give the consumer a `timeout` and loop β€” each -call then returns the moment an event arrives, or empty after the timeout: - -```php -$consumer = new Consumer($remote, 'cache-invalidator', $cursor, timeout: 20_000); - -while (true) { - $consumer->consume($handler); -} -``` +### Starting at the tip A consumer with no stored position starts at the oldest retained event. A consumer that must not act on the backlog β€” a notifier announcing events as -they happen β€” opts into starting at the tip instead: +they happen β€” opts into starting at the tip: ```php use Utopia\Feed\Start; @@ -133,225 +159,128 @@ use Utopia\Feed\Start; $consumer = new Consumer($remote, 'notifier', $cursor, timeout: 20_000, start: Start::Tip); ``` -A stored position always wins; `Start::Tip` only applies on the first run, or -after `reset()` β€” which with `Start::Tip` means "forget everything, resume -from now". Give a tip consumer a `timeout`: the producer anchors "now" as each -poll arrives, so new events land inside the held request rather than in the -gap between polls. Against a producer that predates the tip extension, the -first poll fails with a 4xx `Transport` error rather than silently replaying -the backlog. +A stored position always wins; `Start::Tip` applies only on the first run or +after `reset()` (which then means "forget everything, resume from now"). Give +a tip consumer a `timeout`: the producer anchors "now" as each poll arrives, +so new events land inside the held request rather than in the gap between +polls. Against a producer that predates the tip extension, the first poll +fails with a 4xx `Transport` error rather than silently replaying the backlog. -### Serve a feed over HTTP +### Moving the position by hand -The feed a service serves is the read half of the same journal it appends to. -`Feed::serve()` takes the route's raw query parameters and answers with a -`Batch`, which knows its own body and caching header β€” the whole route body is: +- `reset()` β€” forget the position; the next run starts from the oldest + retained event (or the tip, for a `Start::Tip` consumer). +- `seek($eventId)` β€” treat `$eventId` as the last event handled; the next run + starts strictly *after* it. Persisted immediately; a store failure surfaces + as `Transport`. The id must be well formed but need not still exist in the + feed. + +`seek()` is the escape hatch for a poison event. A handler that keeps failing +blocks the feed by design, so stepping past it is a decision made in code: +catch the failure, log the event's id, and once you have decided the event +must be skipped, seek to *its own id*: ```php -use Utopia\Feed\Feed; +try { + $consumer->consume($handler); +} catch (\Throwable $error) { + Console::error("[feed] blocked: {$error->getMessage()}"); -$feed = new Feed($journal); // the same journal the Producer was built on + // After investigating β€” this event cannot and should not be handled: + $consumer->seek($poisonEventId); +} +``` -// GET /v1/feeds/:feedId -$batch = $feed->serve($request->getParams()); +## The rules a handler lives by -$response - ->addHeader('Content-Type', 'application/cloudevents-batch+json') - ->addHeader('Cache-Control', $batch->cacheControl(public: true)) - ->json($batch->toArray()); -``` +**Safe to run twice on the same event.** Three things re-deliver, and none can +be arranged away: -`serve()` extracts `lastEventId`, `limit` and `timeout` from the query, -coerces their string values, applies the defaults, and clamps the batch to -1000 events and the long-poll wait to 30 seconds, so a client cannot ask for -more than the producer is willing to build or hold. A malformed `lastEventId` -throws `Exception\Invalid` β€” catch it to answer 400. - -The response body is a bare JSON array of CloudEvents, as -[http-feeds.org](https://www.http-feeds.org/) defines it β€” no envelope. An -empty array means the consumer is caught up. The media type is -`application/cloudevents-batch+json`; on receipt this library only checks the -body shape, so a feed answering `application/json` still reads fine. - -The spec defines two query parameters: `lastEventId` and `timeout`. This -library extends it with two more pieces of vocabulary: the `limit` parameter, -and the `lastEventId` value `$`, which the producer resolves to the tip of the -feed β€” the anchor behind `Start::Tip`. A spec-compliant consumer simply never -sends either, and a `$` can never collide with a real id. - -A full batch is settled history and `cacheControl()` marks it cacheable; a -short one is the live end of the feed and is marked `no-store`. Caching is -`private` unless you pass `public: true`. The batch carries the limit it was -actually built with, so the header is always honest β€” there is no number for -the route to keep in sync. - -Callers that already hold typed values can use `Feed::read()` and -`Feed::poll()` directly; both return a `Batch`, which counts and iterates as -the list of events it carries, and `Batch::lastId()` is the position a -stateless relay would otherwise track by hand. - -## Events - -Events **are** -[`Utopia\CloudEvents\CloudEvent`](https://github.com/utopia-php/cloudevents) -objects β€” this library defines no event type of its own: +1. A handler succeeds and the position then fails to save. +2. A run interrupted partway resumes from the last event that succeeded. +3. A consumer whose position was lost restarts from the oldest retained event. -```php -$consumer->consume(function (CloudEvent $event) { - $tags = $event->data['tags'] ?? []; - $trace = $event->extensions['traceparent'] ?? null; -}); -``` +Every one re-delivers; none skips. An idempotent handler absorbs a duplicate, +whereas an event stepped over is gone. + +**Reject by throwing.** The run stops there, the position stays before the +failed event, and the next run retries it. Everything handled earlier in the +run stays handled. A handler that keeps failing blocks everything behind it β€” +intentionally: a feed is ordered, and stepping over a failure would apply +later events on top of state that was never updated. -`data` is unrestricted β€” a map, list, string, number or null all round-trip as -themselves. `subject` is nullable, so an event with no subject reads back as -`null`. `dataschema` and extension attributes survive an append and a read. +**No position means the oldest retained event, never the tip** (unless the +consumer opted into `Start::Tip`), so a consumer deployed after the producer +drains the backlog instead of dropping it. -A batch is decoded strictly about `id`, because for a feed the id *is* the -consumer's position, and leniently about everything else β€” a producer that adds -an attribute or moves the spec forward must not stop a consumer that predates it. +**One process per consumer name.** Two processes sharing a name share one +position, so the feed is split between them rather than delivered to both. -An entry that cannot be read at all ends the batch where it sits: the events -before it are returned and handled, and the broken one heads the next batch, -where it stops the feed loudly. Only when it is the first entry β€” leaving no -usable prefix β€” does the read throw `Exception\Invalid`. +## Reference -## Journals +### Journals -A journal is where a feed's events live, on the server that owns it. Every -journal is `Readable` β€” it returns the events after a given id β€” and -`Appendable`, assigning the ids. (Reading another service's feed is not a -journal's job: that is `Remote`, which is `Readable` and deliberately not -`Appendable`.) +Every journal is `Readable` and `Appendable` β€” it owns its events and assigns +their ids. (`Remote` is `Readable` only; you cannot produce into someone +else's feed.) All take `maxSize` (retention, ~100,000 entries by default) and +`pollInterval` (how often a held poll re-reads, 500 ms by default β€” shorter +lowers long-poll latency, raises backend reads). | Journal | Use for | | --- | --- | | `Journal\Redis` | Producing a feed on a Redis stream | -| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection | +| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection β€” borrows per read, so a held poll never ties up a connection | | `Journal\Memory` | Tests and single-process development | -| `Journal\None` | No backend configured β€” throws on use | +| `Journal\None` | No backend configured β€” throws on use, so a misconfigured service fails loudly instead of dropping events | -`Journal\Pool` is what most services producing a feed want: a long poll spans its -whole timeout, and this one borrows a connection per read and gives it back while -it waits, so polling never ties up the client the rest of the service is using. +### Cursors -A long poll is served by re-reading the journal on an interval β€” `pollInterval`, -a trailing constructor option in milliseconds, 500 by default. A shorter -interval lowers long-poll latency and raises the read rate against the backend. -The loop is deliberate: Redis could wait server-side with `XREAD BLOCK`, but a -blocking read holds the connection for the whole wait, which is exactly what -`Journal\Pool`'s borrow-per-read strategy exists to avoid. - -`Journal\Redis` and `Journal\Pool` trim the stream to about `maxSize` entries -(100,000 by default, and the same for `Journal\Memory`; Redis trims -approximately, so the stream may run a little longer). That cap is the feed's -retention: the oldest entry still in it is where a consumer with no position -starts. - -`Journal\None` throws on every operation rather than doing nothing, so a -misconfigured service fails at the point of use instead of silently dropping -events. `Cursor\None` is the opposite β€” it is a no-op, because a position that -goes nowhere only costs a replay, while an append that goes nowhere loses -events. - -## Cursors - -A cursor is where one consumer keeps its position. It is keyed by feed and -consumer name, so a single store serves every feed a service consumes: +A cursor is keyed by feed and consumer name, so one store serves every feed a +service consumes. The store may be lossy β€” a lost position costs a replay, not +a gap. | Cursor | Use for | | --- | --- | -| `Cursor\Cache` | A consumer with a [Utopia cache](https://github.com/utopia-php/cache) β€” the usual choice when reading a remote feed | -| `Cursor\Redis` | A consumer running inside the producer, with no store of its own | +| `Cursor\Cache` | A consumer with a [Utopia cache](https://github.com/utopia-php/cache) β€” the usual choice for a remote feed. Holds a position for 30 days (`Cursor\Cache::TTL`) unless saved again | +| `Cursor\Redis` | A consumer inside the producer, with no store of its own | | `Cursor\Pool` | The same, over a pooled connection | -| `Cursor\Memory` | Tests, or a consumer that should replay from the beginning on every restart | -| `Cursor\None` | No store configured β€” remembers nothing, so a restart replays | +| `Cursor\Memory` | Tests, or deliberate replay-on-restart | +| `Cursor\None` | Remembers nothing β€” every restart replays | -`Cursor\Cache` holds a position for `Cursor\Cache::TTL` (30 days) unless it is -saved again, so a consumer idle for longer than that reads back as one that has -never run. +### Protocol parameters -The store is allowed to be lossy: a lost position costs a replay, not a gap. A -store that is *down* is a different matter β€” the failure surfaces from -`consume()` as a `Transport` exception rather than being swallowed, so catch it -in your loop if the consumer should keep trying: +The endpoint speaks [http-feeds.org](https://www.http-feeds.org/): a GET +returning a JSON array of CloudEvents, media type +`application/cloudevents-batch+json` (a feed answering `application/json` is +read fine β€” the body shape is what matters). -```php -try { - $consumer->consume($handler); -} catch (Transport $error) { - Console::warning("[feed] {$error->getMessage()}"); -} -``` - -Run **one process per consumer name.** Two sharing a name share one position, so -the feed is split between them rather than delivered to both. - -### Moving the position by hand +| Parameter | Origin | Meaning | +| --- | --- | --- | +| `lastEventId` | spec | The consumer's position: return events strictly after it. Omitted = oldest retained | +| `timeout` | spec | Long poll: hold the request up to this many ms before answering `[]`. Clamped to 30s | +| `limit` | extension | Cap the batch size. Clamped to 1000, which is also the default | +| `lastEventId=$` | extension | The tip: the producer resolves `$` to its newest event as the request arrives. Never collides with a real id | -Two methods change a consumer's position outside the normal flow: +A spec-compliant third-party client simply never sends the extensions. -- `reset()` forgets the position entirely β€” the next run starts from the - oldest retained event (or the tip, for a `Start::Tip` consumer). -- `seek($eventId)` sets it to a specific point: the id is treated as the last - event handled, so the next run starts strictly *after* it. The seek is - persisted immediately, and a store failure surfaces as `Transport` β€” a seek - that did not persist never looks like one that did. The id must be well - formed but does not need to still exist in the feed. +### Exceptions -`seek()` is the escape hatch for a poison event. A handler that keeps failing -blocks the feed by design, so stepping past it is a decision, made in code: -catch the failure, log the event's id, and once you have decided the event -must be skipped, seek to *its own id*: +All extend `Utopia\Feed\Exception`. -```php -try { - $consumer->consume($handler); -} catch (\Throwable $error) { - Console::error("[feed] blocked: {$error->getMessage()}"); - - // After investigating β€” this event cannot and should not be handled: - $consumer->seek($poisonEventId); -} -``` - -## What a handler must tolerate - -**A handler must be safe to run twice on the same event.** There are three -reasons, and none of them can be arranged away: - -1. A handler can succeed and the position then fail to save. -2. A run interrupted partway resumes from the last event that succeeded. -3. A consumer whose position was lost restarts from the oldest retained event. - -Every one of them re-delivers; none of them skips. An event handled twice is -absorbed by an idempotent handler, whereas an event stepped over is gone. - -**A handler rejects an event by throwing.** The run stops there, the position -stays before it, and the next run tries again. Everything handled earlier in that -run stays handled. A handler that keeps failing blocks everything behind it β€” -intentionally, because a feed is ordered and stepping over a failure would apply -later events on top of state that was never updated. - -**A consumer with no position starts at the oldest retained event, never at the -tip**, so a consumer deployed after the producer catches up rather than dropping -the backlog. Starting at the tip is strictly opt-in, per consumer, with -`Start::Tip`. - -## Rolling out a feed - -Replacing push delivery with a feed is a staged change, one release per step: +| Exception | Thrown when | +| --- | --- | +| `Exception\Invalid` | Input is wrong: a malformed event id or `lastEventId`, an empty feed/consumer name, a payload that cannot be JSON-encoded, a response that is not a feed batch. Answer 400 when it surfaces from `serve()` | +| `Exception\Transport` | The backend or network failed: Redis errors, HTTP failures (the status code is on the exception), a cursor store that is down | +| `Exception\Unsupported` | The operation cannot happen here: any use of `Journal\None`, or `tip()` on a `Remote` (the producer resolves the tip) | -1. **Release the producer.** It appends events; nothing reads them yet. -2. **Release the consumers.** Each drains the backlog from its first poll. -3. **Only then remove the push path.** +## The fine print -While step 2 is in progress, consumers get a 404 from a producer that does not -serve the feed yet β€” normal, not a fault. The status is on the exception: +**Rolling out a feed** is a staged change: release the producer (events +accumulate, nothing reads them), then the consumers (each drains the backlog), +and only then remove the old push path. In between, a consumer polling a +producer that does not serve the feed yet gets a 404 β€” normal, not a fault: ```php -use Utopia\Feed\Exception\Transport; - try { $consumer->consume($handler); } catch (Transport $error) { @@ -363,10 +292,34 @@ try { } ``` +**Retention is bounded.** Journals trim to about `maxSize` entries (Redis +trims approximately). The oldest retained entry is where a consumer with no +position starts; a consumer that fell behind the trim horizon gets what is +left β€” no error, no detectable gap. Feeds therefore suit events that describe +a state to converge on (a cache tag to drop, a record to refresh) rather than +ones whose effect depends on seeing every single one. + +**Decoding is strict about `id`, lenient about the rest.** The id is the +consumer's position, so an entry without one ends the batch there: everything +before it is handled, and the broken entry heads the next batch, where it +stops the feed loudly (only a broken *first* entry throws `Invalid`). A +producer that adds attributes or moves the spec version forward does not stop +a consumer that predates it β€” the spec's optional `method` attribute included. + +**Why a poll loop instead of `XREAD BLOCK`?** A blocking read holds the +connection for the whole wait, which is exactly what `Journal\Pool`'s +borrow-per-read strategy exists to avoid. Tune the trade-off with +`pollInterval`. + +**For integrators** building a transport of their own: the wire contract β€” +query parameters, batch encoding, caching rule β€” lives in `Utopia\Feed\Protocol`. +Services never need it. + ## Tests -Unit tests need no services, but dependencies declare extensions the suite never -touches (`ext-redis`, `ext-memcached`, `ext-protobuf`), so install past them: +Unit tests need no services, but dependencies declare extensions the suite +never touches (`ext-redis`, `ext-memcached`, `ext-protobuf`), so install past +them: ```bash composer install --ignore-platform-reqs @@ -387,8 +340,10 @@ and add it to the `php-versions` matrix in `.github/workflows/tests.yml`. ## System requirements -Utopia Feed requires PHP 8.5 or later. We recommend using the latest PHP version -whenever possible. +Utopia Feed requires PHP 8.5 or later. This library is maintained by the +[Appwrite team](https://appwrite.io) and, although part of the +[Utopia Framework](https://github.com/utopia-php/framework), it is dependency +light and works standalone with any PHP project. ## Copyright and license From 703aaa32c85d0f572d4d38ad137a1c7c720100d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 15:08:07 +0200 Subject: [PATCH 30/68] Terminology rework --- CHANGELOG.md | 57 ++++-- README.md | 96 ++++----- src/Feed/Appendable.php | 2 +- src/Feed/Consumer.php | 32 ++- src/Feed/Cursor/None.php | 2 +- src/Feed/Exception/Transport.php | 2 +- src/Feed/Exception/Unsupported.php | 2 +- src/Feed/Producer.php | 22 +- src/Feed/Protocol.php | 4 +- src/Feed/Readable.php | 2 +- src/Feed/Remote.php | 20 +- src/Feed/{Feed.php => Server.php} | 16 +- src/Feed/{Journal.php => Store.php} | 6 +- src/Feed/Store/Cache.php | 131 ++++++++++++ src/Feed/{Journal => Store}/Memory.php | 6 +- src/Feed/{Journal => Store}/None.php | 6 +- src/Feed/{Journal => Store}/Pool.php | 10 +- src/Feed/{Journal => Store}/Redis.php | 6 +- tests/Feed/E2E/RedisTest.php | 126 ++++++------ tests/Feed/Unit/ConsumerTest.php | 168 ++++++++------- tests/Feed/Unit/ProducerTest.php | 48 ++--- tests/Feed/Unit/RemoteTest.php | 52 +++-- tests/Feed/Unit/RoundTripTest.php | 43 ++-- .../Unit/{FeedTest.php => ServerTest.php} | 192 +++++++++--------- tests/Feed/Unit/StoreCacheTest.php | 170 ++++++++++++++++ tests/Feed/Unit/Support/FakeClient.php | 2 +- tests/Feed/Unit/Support/FakeTransport.php | 2 +- tests/Feed/Unit/Support/FeedServer.php | 10 +- .../{MidPollJournal.php => MidPollStore.php} | 6 +- 29 files changed, 808 insertions(+), 433 deletions(-) rename src/Feed/{Feed.php => Server.php} (83%) rename src/Feed/{Journal.php => Store.php} (96%) create mode 100644 src/Feed/Store/Cache.php rename src/Feed/{Journal => Store}/Memory.php (94%) rename src/Feed/{Journal => Store}/None.php (87%) rename src/Feed/{Journal => Store}/Pool.php (83%) rename src/Feed/{Journal => Store}/Redis.php (95%) rename tests/Feed/Unit/{FeedTest.php => ServerTest.php} (67%) create mode 100644 tests/Feed/Unit/StoreCacheTest.php rename tests/Feed/Unit/Support/{MidPollJournal.php => MidPollStore.php} (82%) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb4e5f6..c803926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,35 @@ ## Unreleased -- **Breaking:** `Feed::read()` and `Feed::poll()` return a `Batch` instead of - a plain event array. A batch counts and iterates as its events and carries - the limit it was actually built with, so `Batch::cacheControl()` can never - be fed a number the read did not use. `Batch::toArray()` is the wire - encoding, `Batch::lastId()` the position a stateless relay tracks. -- Added `Feed::serve(array $query): Batch` β€” the whole HTTP request in one +- **Breaking (the three classes):** the library is now three main classes. + `Producer` writes events to a feed with `produce()` (formerly `append()`; + `publish()` still takes a prepared `CloudEvent`). `Server` (formerly + `Feed`) is the HTTP feed endpoint, with `serve()` as its main method. + `Consumer` reads a feed as a client with `consume()`. +- **Breaking:** `Journal` is now `Store` β€” `Store\Redis`, `Store\Pool`, + `Store\Memory`, `Store\None`, in the `Utopia\Feed\Store` namespace. The + `Producer` and `Server` are built over a store; `Appendable` and `Readable` + are unchanged as its contracts. +- **Breaking:** `Consumer` no longer takes an endpoint. It is built straight + over a [utopia-php/client](https://github.com/utopia-php/client) whose + base URI points at the feed endpoint (`withBaseUri()`), plus the feed's + name: `new Consumer($client, $cursor, name: 'invalidator', feed: 'edge')`. + A local store still drops in for the client, for consuming a feed the same + service produces; the cursor moved forward to the second parameter. + `Remote` accordingly lost its `endpoint` parameter β€” the feed name is sent + as a relative path and the client resolves it. +- Added `Store\Cache` β€” a feed on a [utopia-php/cache](https://github.com/utopia-php/cache) + `Cache`, for a producer whose service already carries a cache and does not + want another backend. The whole feed lives under one key (last-writer-wins + appends β€” run one producing process), trims to `maxSize`, and expires `ttl` + seconds after the last append (default 30 days). Both `Store` and `Cursor` + now have `Redis` and `Cache` adapters. +- `Server::read()` and `Server::poll()` return a `Batch` instead of a plain + event array. A batch counts and iterates as its events and carries the + limit it was actually built with, so `Batch::cacheControl()` can never be + fed a number the read did not use. `Batch::toArray()` is the wire encoding, + `Batch::lastId()` the position a stateless relay tracks. +- Added `Server::serve(array $query): Batch` β€” the whole HTTP request in one call: extracts `lastEventId`, `limit` and `timeout` from the route's raw query parameters, coerces and clamps them, and rejects a malformed `lastEventId` with `Exception\Invalid`. A route never needs to name @@ -16,34 +39,30 @@ starting at the tip of the feed (only what happens from now on) instead of draining the backlog. Rides a protocol extension: the `lastEventId` value `$`, resolved by the producer to the newest event as the request arrives. - Also added `Feed::tip()`, the id of the newest event in a local journal. + Also added `Server::tip()`, the id of the newest event in a local store. - Added `Consumer::seek(string $eventId)` β€” set the position explicitly: the id is treated as the last event handled, persisted immediately, and the next `consume()` starts strictly after it. The operational escape hatch for a poison event: seek to the failing event's own id to step past it deliberately. -- The long-poll read interval is now a constructor option on `Journal\Redis`, - `Journal\Pool` and `Journal\Memory`: `pollInterval`, in milliseconds, - default 500. An interval below 1 ms throws `Exception\Invalid`. The poll - loop also no longer oversleeps: it sleeps the remaining time when less than - an interval is left, so a timeout is honoured to within scheduler precision - instead of running up to one interval late. +- The long-poll read interval is now a constructor option on the stores: + `pollInterval`, in milliseconds, default 500. An interval below 1 ms throws + `Exception\Invalid`. The poll loop also no longer oversleeps: it sleeps the + remaining time when less than an interval is left, so a timeout is honoured + to within scheduler precision instead of running up to one interval late. - **Breaking (renames):** every user-facing name now belongs to exactly one side of the wire. `Journal\Http` is gone; its replacement is `Utopia\Feed\Remote` β€” another service's feed, over HTTP β€” a standalone class implementing the new `Readable` interface (`read`, `poll`, `tip`, - `getName`) rather than posing as a journal. `Consumer` accepts any - `Readable` (a `Remote`, or a local journal) instead of a `Feed`, and clamps - its own `batch`/`timeout`; `Feed` is server vocabulary, built over the - journal it serves. The protocol limits moved with the responsibility: + `getName`) rather than posing as a store. `Consumer` clamps its own + `batch`/`timeout`. The protocol limits moved with the responsibility: `Feed::MAX_BATCH`/`Feed::MAX_TIMEOUT` are now `Protocol::MAX_BATCH` and `Protocol::MAX_TIMEOUT`. - - **Breaking (wire format):** a feed batch on the wire is now the plain JSON array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) defines β€” the `{total, events}` envelope is gone, and an empty feed serializes to `[]`. `Protocol::encode()` returns the bare array, - `Protocol::decode()` expects one, and the HTTP journal asks for the spec's + `Protocol::decode()` expects one, and `Remote` asks for the spec's `application/cloudevents-batch+json` media type (`Protocol::MEDIA_TYPE`). Both sides of a feed must move together. diff --git a/README.md b/README.md index c512156..d465aa5 100644 --- a/README.md +++ b/README.md @@ -4,23 +4,23 @@ [![Discord](https://img.shields.io/discord/564160730845151244)](https://appwrite.io/discord) Utopia Feed moves events between services with **pull-based HTTP event feeds** -([http-feeds.org](https://www.http-feeds.org/)). A producer appends events to +([http-feeds.org](https://www.http-feeds.org/)). A producer writes events to an ordered log; each consumer polls *"what happened since the last event I saw?"* and keeps its own position, so the producer stores nothing per consumer and delivery is **at-least-once**. A consumer that was down catches up on its next poll. -The whole library in one sentence: *a `Producer` appends to a `Journal`, a -`Feed` serves that journal over HTTP; a `Consumer` reads a `Remote` feed and -keeps its place in a `Cursor`.* +The whole library in three classes: a `Producer` produces events into a +`Store`, a `Server` serves that store over HTTP, and a `Consumer` consumes the +feed through an HTTP client, keeping its place in a `Cursor`. ## Which classes are mine? | Server (owns the feed) | Client (consumes it) | | --- | --- | -| `Journal` β€” where events live | `Remote` β€” another service's feed, over HTTP | -| `Producer` β€” appends events | `Consumer` β€” the pull loop | -| `Feed` β€” serves the journal | `Cursor` β€” where the position is kept | +| `Store` β€” where events live | `Consumer` β€” the pull loop | +| `Producer` β€” writes events | `Cursor` β€” where the position is kept | +| `Server` β€” serves the store | | ## Install @@ -30,30 +30,30 @@ composer require utopia-php/feed ## Serve a feed -The server side is three objects over one journal β€” where the events live, -here a capped Redis stream: +The server side is three objects over one store β€” where the events live, here +a capped Redis stream: ```php -use Utopia\Feed\Feed; -use Utopia\Feed\Journal; use Utopia\Feed\Producer; +use Utopia\Feed\Server; +use Utopia\Feed\Store; -$journal = new Journal\Redis($redis, 'edge'); +$store = new Store\Redis($redis, 'edge'); // Wherever things happen: -$producer = new Producer($journal, source: 'urn:appwrite:cloud:fra'); +$producer = new Producer($store, source: 'urn:appwrite:cloud:fra'); -$producer->append( +$producer->produce( type: 'io.appwrite.edge.invalidate-rule', data: ['tags' => ['domain' => 'example.com']], subject: 'example.com', ); // The whole feed route: -$feed = new Feed($journal); +$server = new Server($store); // GET /v1/feeds/:feedId -$batch = $feed->serve($request->getParams()); +$batch = $server->serve($request->getParams()); $response ->addHeader('Content-Type', 'application/cloudevents-batch+json') @@ -90,20 +90,22 @@ empty array means the consumer is caught up: `serve()` reads `lastEventId`, `limit` and `timeout` from the raw query parameters, coerces and clamps them (at most 1000 events per batch, long polls held at most 30s), and throws `Exception\Invalid` on a malformed `lastEventId` -β€” catch it to answer 400. `append()` returns the event's id, which is its +β€” catch it to answer 400. `produce()` returns the event's id, which is its position in the feed. Subclass `Producer` to give callers a typed vocabulary instead of raw type strings. -The `Batch` that `serve()` (and `Feed::read()`/`poll()`) returns counts and +The `Batch` that `serve()` (and `Server::read()`/`poll()`) returns counts and iterates as its events; `cacheControl()` marks a full batch as immutable history and everything shorter `no-store`, using the limit the batch was actually built with, so the header is always honest. ## Consume a feed -The client side is a `Consumer` pulling a `Remote` feed, with its position in -a `Cursor`. With a `timeout`, each poll is one held request that returns the -moment an event lands (long polling β€” the producer does the waiting): +The client side is a `Consumer` pulling over an HTTP client, with its position +in a `Cursor`. The feed's endpoint is set on the client β€” `withBaseUri()` β€” +and the consumer names the feed it reads. With a `timeout`, each poll is one +held request that returns the moment an event lands (long polling β€” the +producer does the waiting): ```php use Utopia\Client; @@ -112,15 +114,13 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; use Utopia\Feed\Exception\Transport; -use Utopia\Feed\Remote; $client = (new Client(new Curl())) + ->withBaseUri('https://cloud.example.com/v1/feeds') ->withHeaders(['x-appwrite-jwt' => $token]) ->withConnectionReuse(); -$remote = new Remote($client, 'https://cloud.example.com/v1/feeds', 'edge'); - -$consumer = new Consumer($remote, 'cache-invalidator', new Cursor\Cache($cache), timeout: 20_000); +$consumer = new Consumer($client, new Cursor\Cache($cache), name: 'cache-invalidator', feed: 'edge', timeout: 20_000); while (true) { try { @@ -141,10 +141,11 @@ Leave the client's `Retry` decorator off: a failed read leaves the position where it was. A consumer inside the producing service reads its own feed the same way β€” -`Consumer` accepts anything `Readable`, so hand it the local journal directly: +`Consumer` also accepts a local store in place of the client, which already +names its feed: ```php -$consumer = new Consumer($journal, 'audit-log', new Cursor\Redis($redis)); +$consumer = new Consumer($store, new Cursor\Redis($redis), name: 'audit-log'); ``` ### Starting at the tip @@ -156,7 +157,7 @@ they happen β€” opts into starting at the tip: ```php use Utopia\Feed\Start; -$consumer = new Consumer($remote, 'notifier', $cursor, timeout: 20_000, start: Start::Tip); +$consumer = new Consumer($client, $cursor, name: 'notifier', feed: 'edge', timeout: 20_000, start: Start::Tip); ``` A stored position always wins; `Start::Tip` applies only on the first run or @@ -218,20 +219,20 @@ position, so the feed is split between them rather than delivered to both. ## Reference -### Journals +### Stores -Every journal is `Readable` and `Appendable` β€” it owns its events and assigns -their ids. (`Remote` is `Readable` only; you cannot produce into someone -else's feed.) All take `maxSize` (retention, ~100,000 entries by default) and +Every store is `Readable` and `Appendable` β€” it owns its events and assigns +their ids. All take `maxSize` (retention, ~100,000 entries by default) and `pollInterval` (how often a held poll re-reads, 500 ms by default β€” shorter lowers long-poll latency, raises backend reads). -| Journal | Use for | +| Store | Use for | | --- | --- | -| `Journal\Redis` | Producing a feed on a Redis stream | -| `Journal\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection β€” borrows per read, so a held poll never ties up a connection | -| `Journal\Memory` | Tests and single-process development | -| `Journal\None` | No backend configured β€” throws on use, so a misconfigured service fails loudly instead of dropping events | +| `Store\Redis` | Producing a feed on a Redis stream | +| `Store\Pool` | The same, over a [pooled](https://github.com/utopia-php/pools) connection β€” borrows per read, so a held poll never ties up a connection | +| `Store\Cache` | A feed on a [Utopia cache](https://github.com/utopia-php/cache) β€” for a service that already carries a cache and does not want another backend. One key per feed, rewritten per append (last-writer-wins β€” run one producing process); expires `ttl` after the last append, 30 days by default | +| `Store\Memory` | Tests and single-process development | +| `Store\None` | No backend configured β€” throws on use, so a misconfigured service fails loudly instead of dropping events | ### Cursors @@ -271,7 +272,7 @@ All extend `Utopia\Feed\Exception`. | --- | --- | | `Exception\Invalid` | Input is wrong: a malformed event id or `lastEventId`, an empty feed/consumer name, a payload that cannot be JSON-encoded, a response that is not a feed batch. Answer 400 when it surfaces from `serve()` | | `Exception\Transport` | The backend or network failed: Redis errors, HTTP failures (the status code is on the exception), a cursor store that is down | -| `Exception\Unsupported` | The operation cannot happen here: any use of `Journal\None`, or `tip()` on a `Remote` (the producer resolves the tip) | +| `Exception\Unsupported` | The operation cannot happen here: any use of `Store\None`, or `tip()` on a remote feed (the producer resolves the tip) | ## The fine print @@ -292,12 +293,12 @@ try { } ``` -**Retention is bounded.** Journals trim to about `maxSize` entries (Redis -trims approximately). The oldest retained entry is where a consumer with no -position starts; a consumer that fell behind the trim horizon gets what is -left β€” no error, no detectable gap. Feeds therefore suit events that describe -a state to converge on (a cache tag to drop, a record to refresh) rather than -ones whose effect depends on seeing every single one. +**Retention is bounded.** Stores trim to about `maxSize` entries (Redis trims +approximately). The oldest retained entry is where a consumer with no position +starts; a consumer that fell behind the trim horizon gets what is left β€” no +error, no detectable gap. Feeds therefore suit events that describe a state to +converge on (a cache tag to drop, a record to refresh) rather than ones whose +effect depends on seeing every single one. **Decoding is strict about `id`, lenient about the rest.** The id is the consumer's position, so an entry without one ends the batch there: everything @@ -307,13 +308,14 @@ producer that adds attributes or moves the spec version forward does not stop a consumer that predates it β€” the spec's optional `method` attribute included. **Why a poll loop instead of `XREAD BLOCK`?** A blocking read holds the -connection for the whole wait, which is exactly what `Journal\Pool`'s +connection for the whole wait, which is exactly what `Store\Pool`'s borrow-per-read strategy exists to avoid. Tune the trade-off with `pollInterval`. **For integrators** building a transport of their own: the wire contract β€” -query parameters, batch encoding, caching rule β€” lives in `Utopia\Feed\Protocol`. -Services never need it. +query parameters, batch encoding, caching rule β€” lives in `Utopia\Feed\Protocol`, +and `Utopia\Feed\Remote` is the client-side `Readable` the consumer builds +over its client. Services never need either. ## Tests diff --git a/src/Feed/Appendable.php b/src/Feed/Appendable.php index fbc0057..ad8d369 100644 --- a/src/Feed/Appendable.php +++ b/src/Feed/Appendable.php @@ -6,7 +6,7 @@ use Utopia\CloudEvents\CloudEvent; -// Server interface: a journal that owns its events, so it can be appended to. +// Server interface: a store that owns its events, so it can be appended to. // Remote does not implement it β€” a consumer cannot write to someone else's feed. interface Appendable { diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 070651c..07aa118 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -4,23 +4,35 @@ namespace Utopia\Feed; +use Utopia\Client\Adapter; + // Client class: the pull loop β€” reads what it has not seen and records how far it got. class Consumer { public const int BATCH = 100; + protected readonly Readable $feed; + private ?string $position = null; private bool $restored = false; /** - * @param Readable $feed The feed to pull from β€” a Remote for another - * service's feed, or a local journal for one this service owns. + * @param Adapter|Readable $source The feed to pull from β€” an HTTP client + * for another service's feed (its endpoint set on the client with + * withBaseUri()), or a local store for a feed this service owns. + * @param string $name This consumer's name β€” what keys its position. + * @param string $feed The feed's name. Required when $source is a client, + * where nothing else carries it; a local source already names its + * feed, so leave it out. + * + * @throws Exception\Invalid When a name is missing, or $feed contradicts the source. */ public function __construct( - protected readonly Readable $feed, - protected readonly string $name, + Adapter|Readable $source, protected readonly Cursor $cursor, + protected readonly string $name, + string $feed = '', protected readonly int $batch = self::BATCH, protected readonly int $timeout = 0, protected readonly Start $start = Start::Oldest, @@ -28,6 +40,16 @@ public function __construct( if ($name === '') { throw new Exception\Invalid('Feed consumer requires a name'); } + + if ($source instanceof Adapter) { + $this->feed = new Remote($source, $feed); + } else { + if ($feed !== '' && $feed !== $source->getName()) { + throw new Exception\Invalid("The source already names its feed {$source->getName()}, which {$feed} contradicts"); + } + + $this->feed = $source; + } } public function getName(): string @@ -77,7 +99,7 @@ public function consume(callable $handler): int /** * Where a poll starts when no position is stored: the oldest retained - * event, or β€” for Start::Tip β€” the tip sentinel, which the journal (or + * event, or β€” for Start::Tip β€” the tip sentinel, which the store (or * the remote producer, inside the same request) resolves to "now". Once * events are handled and the cursor saves, the sentinel never appears * again; reset() forgets the position, so the next poll anchors anew. diff --git a/src/Feed/Cursor/None.php b/src/Feed/Cursor/None.php index bf58cbc..dec0d5d 100644 --- a/src/Feed/Cursor/None.php +++ b/src/Feed/Cursor/None.php @@ -9,7 +9,7 @@ /** * No cursor store configured. Nothing is remembered. * - * A no-op, unlike {@see \Utopia\Feed\Journal\None}, because the two losses are + * A no-op, unlike {@see \Utopia\Feed\Store\None}, because the two losses are * not comparable: an append that goes nowhere loses events, while a position * that goes nowhere only costs a replay. A consumer using this still advances * in memory for the life of the process, and starts again from the oldest diff --git a/src/Feed/Exception/Transport.php b/src/Feed/Exception/Transport.php index a12026e..929fe92 100644 --- a/src/Feed/Exception/Transport.php +++ b/src/Feed/Exception/Transport.php @@ -6,7 +6,7 @@ use Utopia\Feed\Exception; -// The journal (backend) could not be reached, or rejected the operation. +// The store (backend) could not be reached, or rejected the operation. class Transport extends Exception { } diff --git a/src/Feed/Exception/Unsupported.php b/src/Feed/Exception/Unsupported.php index 5a160e7..195e294 100644 --- a/src/Feed/Exception/Unsupported.php +++ b/src/Feed/Exception/Unsupported.php @@ -6,7 +6,7 @@ use Utopia\Feed\Exception; -// The journal (backend) cannot do what was asked of it +// The store (backend) cannot do what was asked of it class Unsupported extends Exception { } diff --git a/src/Feed/Producer.php b/src/Feed/Producer.php index d2d2eae..de9cb48 100644 --- a/src/Feed/Producer.php +++ b/src/Feed/Producer.php @@ -6,19 +6,19 @@ use Utopia\CloudEvents\CloudEvent; -// Server class: appends events to a feed this service owns. -// The journal must be a Journal and Appendable, so a Remote cannot reach this at all. +// Server class: writes events to a feed this service owns. +// The store must be a Store and Appendable, so a Remote cannot reach this at all. class Producer { /** - * @param Journal&Appendable $journal Where the events live. + * @param Store&Appendable $store Where the events live. * @param string $source Who is producing them, as a URI reference * (`urn:appwrite:cloud:fra`). Stamped onto every event, so a consumer * merging feeds from several producers can tell them apart. * @throws Exception\Invalid When $source is empty. */ public function __construct( - protected readonly Journal&Appendable $journal, + protected readonly Store&Appendable $store, protected readonly string $source, ) { if ($source === '') { @@ -28,16 +28,16 @@ public function __construct( public function getName(): string { - return $this->journal->getName(); + return $this->store->getName(); } /** - * Append an event and return its position in the feed. + * Produce an event and return its position in the feed. * * @throws Exception\Invalid When $type is empty or $data cannot be encoded. - * @throws Exception When the backend rejects the append. + * @throws Exception When the backend rejects the event. */ - public function append(string $type, mixed $data = [], string $subject = ''): string + public function produce(string $type, mixed $data = [], string $subject = ''): string { return $this->publish(new CloudEvent( type: $type, @@ -49,11 +49,11 @@ public function append(string $type, mixed $data = [], string $subject = ''): st } /** - * Append a prepared event, stamping it with this producer's source and, + * Produce a prepared event, stamping it with this producer's source and, * unless it already has one, the current time. * * @throws Exception\Invalid When the event has no type or cannot be encoded. - * @throws Exception When the backend rejects the append. + * @throws Exception When the backend rejects the event. */ public function publish(CloudEvent $event): string { @@ -77,7 +77,7 @@ public function publish(CloudEvent $event): string extensions: $event->extensions, ); - return $this->journal->append($event); + return $this->store->append($event); } /** The current time in the RFC 3339 format the spec requires. */ diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php index 76261df..763a955 100644 --- a/src/Feed/Protocol.php +++ b/src/Feed/Protocol.php @@ -8,8 +8,8 @@ use Utopia\Feed\Exception\Invalid; // HTTP shape of a feed, as defined by https://www.http-feeds.org/. -// Internal plumbing: used by Feed and the HTTP journal, so a service serving -// or reading a feed never needs to name it β€” Feed::serve() and Batch cover +// Internal plumbing: used by Server and Remote, so a service serving or +// reading a feed never needs to name it β€” Server::serve() and Batch cover // both ends. It stays public for integrators building their own transports. final class Protocol { diff --git a/src/Feed/Readable.php b/src/Feed/Readable.php index aeebb9c..105171b 100644 --- a/src/Feed/Readable.php +++ b/src/Feed/Readable.php @@ -7,7 +7,7 @@ use Utopia\CloudEvents\CloudEvent; // The contract of "something a feed's events can be read from" β€” the -// counterpart of Appendable. Journal implements it on the server; Remote is +// counterpart of Appendable. Store implements it on the server; Remote is // the client's read-only view of another service's feed. interface Readable { diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index 0e0a5aa..a38ec78 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -15,6 +15,8 @@ use Utopia\Psr7\Request\Factory as RequestFactory; // Client class: another service's feed, read over the wire. +// The endpoint lives on the client β€” set it with withBaseUri() β€” and the feed +// name becomes the request path, resolved against it. // Deliberately Readable and not Appendable β€” events are appended by whoever // owns the feed, so producing into a remote one is a type error, not a request. class Remote implements Readable @@ -23,7 +25,6 @@ class Remote implements Readable public function __construct( protected readonly Adapter $client, - protected readonly string $endpoint, protected readonly string $name, ) { if ($name === '') { @@ -63,13 +64,13 @@ public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BAT */ private function fetch(?string $lastEventId, int $limit, int $timeout): array { - $url = $this->url(); - + // The path is relative β€” the feed's name β€” and the client resolves it + // against its base URI, which is where the endpoint is configured. // The Content-Type of the response is deliberately not checked: many // servers answer application/json, and the body shape is what matters. $request = $this->requests->query( Method::GET, - $url, + \rawurlencode($this->name), Protocol::query($lastEventId, $limit, $timeout), [Header::ACCEPT => Protocol::MEDIA_TYPE], ); @@ -84,14 +85,14 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array try { $response = $client->sendRequest($request); } catch (ClientExceptionInterface $error) { - throw new Transport("Failed to read the {$this->name} feed at {$url}: {$error->getMessage()}", previous: $error); + throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); } $status = $response->getStatusCode(); if ($status >= 400) { throw new Transport( - "Reading the {$this->name} feed at {$url} failed with status {$status}", + "Reading the {$this->name} feed failed with status {$status}", $status, ); } @@ -99,14 +100,9 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array try { $body = \json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); } catch (\JsonException $error) { - throw new Transport("The {$this->name} feed at {$url} returned a body that is not JSON: {$error->getMessage()}", previous: $error); + throw new Transport("The {$this->name} feed returned a body that is not JSON: {$error->getMessage()}", previous: $error); } return Protocol::decode($body); } - - private function url(): string - { - return \rtrim($this->endpoint, '/') . '/' . \rawurlencode($this->name); - } } diff --git a/src/Feed/Feed.php b/src/Feed/Server.php similarity index 83% rename from src/Feed/Feed.php rename to src/Feed/Server.php index daf2a06..30e40d5 100644 --- a/src/Feed/Feed.php +++ b/src/Feed/Server.php @@ -4,35 +4,35 @@ namespace Utopia\Feed; -// Server class: the read view over the journal a service appends to β€” it +// Server class: the read view over the store a service appends to β€” it // reads, long-polls, and serves the feed over HTTP with serve(). -class Feed +class Server { - public function __construct(protected readonly Readable $journal) + public function __construct(protected readonly Readable $store) { } public function getName(): string { - return $this->journal->getName(); + return $this->store->getName(); } /** - * The id of the newest event, or null on an empty feed. Local journals + * The id of the newest event, or null on an empty feed. Local stores * only β€” a remote feed's producer resolves the tip sentinel instead. * * @throws Exception */ public function tip(): ?string { - return $this->journal->tip(); + return $this->store->tip(); } public function read(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH): Batch { $limit = \max(1, \min($limit, Protocol::MAX_BATCH)); - return new Batch($this->journal->read($lastEventId, $limit), $limit); + return new Batch($this->store->read($lastEventId, $limit), $limit); } public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH, int $timeout = 0): Batch @@ -40,7 +40,7 @@ public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BAT $limit = \max(1, \min($limit, Protocol::MAX_BATCH)); return new Batch( - $this->journal->poll($lastEventId, $limit, \max(0, \min($timeout, Protocol::MAX_TIMEOUT))), + $this->store->poll($lastEventId, $limit, \max(0, \min($timeout, Protocol::MAX_TIMEOUT))), $limit, ); } diff --git a/src/Feed/Journal.php b/src/Feed/Store.php similarity index 96% rename from src/Feed/Journal.php rename to src/Feed/Store.php index 1c19897..bfd10ad 100644 --- a/src/Feed/Journal.php +++ b/src/Feed/Store.php @@ -7,10 +7,10 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; -// Server class: durable storage for the events β€” Journal\Redis, Pool, Memory. -// A journal owns its events, so every journal also implements Appendable; +// Server class: durable storage for the events β€” Store\Redis, Cache, Pool, Memory. +// A store owns its events, so every store also implements Appendable; // reading another service's feed over the wire is Remote's job. -abstract class Journal implements Readable +abstract class Store implements Readable { protected const int POLL_INTERVAL = 500; // ms diff --git a/src/Feed/Store/Cache.php b/src/Feed/Store/Cache.php new file mode 100644 index 0000000..0b8a516 --- /dev/null +++ b/src/Feed/Store/Cache.php @@ -0,0 +1,131 @@ +load(); + + $last = $entries === [] ? null : $entries[\count($entries) - 1]; + [$timestamp, $sequence] = $last === null ? [0, -1] : Id::decode($last['id']); + + $now = (int) \floor(\microtime(true) * 1000); + $id = $now > $timestamp ? Id::encode($now, 0) : Id::encode($timestamp, $sequence + 1); + + $entries[] = ['id' => $id, 'fields' => self::encode($event)]; + + if (\count($entries) > $this->maxSize) { + $entries = \array_slice($entries, -$this->maxSize); + } + + $saved = $this->cache->save($this->key(), $entries); + + if ($saved === false) { + throw new Transport("Failed to append to the {$this->name} feed"); + } + + return $id; + } + + public function tip(): ?string + { + $entries = $this->load(); + + return $entries === [] ? null : $entries[\count($entries) - 1]['id']; + } + + public function read(?string $lastEventId, int $limit): array + { + $lastEventId = $this->resolve($lastEventId); + + $after = $lastEventId === null ? null : Id::decode($lastEventId); + + $events = []; + + foreach ($this->load() as $entry) { + if ($after !== null && Id::decode($entry['id']) <= $after) { + continue; + } + + $events[] = self::decode($entry['id'], $entry['fields']); + + if (\count($events) >= $limit) { + break; + } + } + + return $events; + } + + /** + * The stored feed, oldest first. Anything that is not the shape append() + * writes β€” a missing key, a foreign value under it β€” reads as an empty + * feed rather than a fault: a cache is allowed to forget. + * + * @return list}> + */ + private function load(): array + { + /** @var mixed $stored */ + $stored = $this->cache->load($this->key(), $this->ttl); + + if (!\is_array($stored)) { + return []; + } + + $entries = []; + + /** @var mixed $entry */ + foreach ($stored as $entry) { + if (!\is_array($entry) + || !isset($entry['id'], $entry['fields']) + || !\is_string($entry['id']) + || !\is_array($entry['fields'])) { + return []; + } + + $entries[] = ['id' => $entry['id'], 'fields' => $entry['fields']]; + } + + return $entries; + } + + private function key(): string + { + return 'feed:' . $this->name; + } +} diff --git a/src/Feed/Journal/Memory.php b/src/Feed/Store/Memory.php similarity index 94% rename from src/Feed/Journal/Memory.php rename to src/Feed/Store/Memory.php index bfaea64..1d8a630 100644 --- a/src/Feed/Journal/Memory.php +++ b/src/Feed/Store/Memory.php @@ -2,14 +2,14 @@ declare(strict_types=1); -namespace Utopia\Feed\Journal; +namespace Utopia\Feed\Store; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Id; use Utopia\Feed\Appendable; -use Utopia\Feed\Journal; +use Utopia\Feed\Store; -class Memory extends Journal implements Appendable +class Memory extends Store implements Appendable { /** @var list */ private array $events = []; diff --git a/src/Feed/Journal/None.php b/src/Feed/Store/None.php similarity index 87% rename from src/Feed/Journal/None.php rename to src/Feed/Store/None.php index b90ada4..ee7a31f 100644 --- a/src/Feed/Journal/None.php +++ b/src/Feed/Store/None.php @@ -2,14 +2,14 @@ declare(strict_types=1); -namespace Utopia\Feed\Journal; +namespace Utopia\Feed\Store; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Appendable; -use Utopia\Feed\Journal; +use Utopia\Feed\Store; -class None extends Journal implements Appendable +class None extends Store implements Appendable { public function __construct(string $name = 'none') { diff --git a/src/Feed/Journal/Pool.php b/src/Feed/Store/Pool.php similarity index 83% rename from src/Feed/Journal/Pool.php rename to src/Feed/Store/Pool.php index 6d945a1..032432f 100644 --- a/src/Feed/Journal/Pool.php +++ b/src/Feed/Store/Pool.php @@ -2,14 +2,14 @@ declare(strict_types=1); -namespace Utopia\Feed\Journal; +namespace Utopia\Feed\Store; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Appendable; -use Utopia\Feed\Journal; +use Utopia\Feed\Store; use Utopia\Pools\Pool as UtopiaPool; -class Pool extends Journal implements Appendable +class Pool extends Store implements Appendable { /** * @param UtopiaPool<\Redis|\RedisCluster> $pool @@ -24,8 +24,8 @@ public function __construct( } // The interval only matters in this class's own inherited poll() loop β€” - // the inner journal lives for a single read β€” but it is passed through so - // a future change to the inner journal cannot silently drop it. + // the inner store lives for a single read β€” but it is passed through so + // a future change to the inner store cannot silently drop it. private function inner(\Redis|\RedisCluster $redis): Redis { return new Redis($redis, $this->name, $this->maxSize, $this->pollInterval); diff --git a/src/Feed/Journal/Redis.php b/src/Feed/Store/Redis.php similarity index 95% rename from src/Feed/Journal/Redis.php rename to src/Feed/Store/Redis.php index 05bcda2..796a904 100644 --- a/src/Feed/Journal/Redis.php +++ b/src/Feed/Store/Redis.php @@ -2,15 +2,15 @@ declare(strict_types=1); -namespace Utopia\Feed\Journal; +namespace Utopia\Feed\Store; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; use Utopia\Feed\Appendable; -use Utopia\Feed\Journal; +use Utopia\Feed\Store; -class Redis extends Journal implements Appendable +class Redis extends Store implements Appendable { public function __construct( protected readonly \Redis|\RedisCluster $redis, diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php index e50fd42..bf79231 100644 --- a/tests/Feed/E2E/RedisTest.php +++ b/tests/Feed/E2E/RedisTest.php @@ -5,13 +5,13 @@ namespace Utopia\Tests\E2E; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Journal\Redis as RedisJournal; +use Utopia\Feed\Store\Redis as RedisStore; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Redis as RedisCursor; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Batch; use Utopia\Feed\Exception\Invalid; -use Utopia\Feed\Feed; +use Utopia\Feed\Server; use Utopia\Feed\Producer; use Utopia\Feed\Id; @@ -49,9 +49,9 @@ protected function tearDown(): void $this->redis->close(); } - private function feed(int $maxSize = 100_000): Feed + private function server(int $maxSize = 100_000): Server { - return new Feed(new RedisJournal($this->redis, $this->name, $maxSize)); + return new Server(new RedisStore($this->redis, $this->name, $maxSize)); } /** @return list */ @@ -62,29 +62,29 @@ private static function events(Batch $batch): array private function producer(int $maxSize = 100_000): Producer { - return new Producer(new RedisJournal($this->redis, $this->name, $maxSize), 'urn:test:e2e'); + return new Producer(new RedisStore($this->redis, $this->name, $maxSize), 'urn:test:e2e'); } /** * The two halves of one feed: what a producing service builds over a single - * journal to append to its feed and serve it. + * store to produce into its feed and serve it. * - * @return array{Producer, Feed} + * @return array{Producer, Server} */ - private function feedAndProducer(int $maxSize = 100_000): array + private function serverAndProducer(int $maxSize = 100_000): array { - $journal = new RedisJournal($this->redis, $this->name, $maxSize); + $store = new RedisStore($this->redis, $this->name, $maxSize); - return [new Producer($journal, 'urn:test:e2e'), new Feed($journal)]; + return [new Producer($store, 'urn:test:e2e'), new Server($store)]; } public function testAppendsAndReadsBack(): void { - [$producer, $feed] = $this->feedAndProducer(); + [$producer, $server] = $this->serverAndProducer(); - $id = $producer->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + $id = $producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); - $events = self::events($feed->read()); + $events = self::events($server->read()); $this->assertCount(1, $events); $this->assertSame($id, $events[0]->id); @@ -97,18 +97,18 @@ public function testAppendsAndReadsBack(): void public function testStreamIdsMatchTheFormatPositionsAreParsedWith(): void { - $id = $this->producer()->append('test'); + $id = $this->producer()->produce('test'); $this->assertTrue(Id::isValid($id), "Redis returned an id this library cannot page from: {$id}"); } public function testIdsIncreaseAcrossRapidAppends(): void { - [$producer, $feed] = $this->feedAndProducer(); + [$producer, $server] = $this->serverAndProducer(); $ids = []; for ($i = 0; $i < 100; $i++) { - $ids[] = $producer->append('test'); + $ids[] = $producer->produce('test'); } $this->assertSame($ids, \array_unique($ids)); @@ -125,16 +125,16 @@ public function testIdsIncreaseAcrossRapidAppends(): void */ public function testReadsStrictlyAfterAPosition(): void { - [$producer, $feed] = $this->feedAndProducer(); + [$producer, $server] = $this->serverAndProducer(); - $first = $producer->append('a'); - $second = $producer->append('b'); + $first = $producer->produce('a'); + $second = $producer->produce('b'); - $events = self::events($feed->read($first)); + $events = self::events($server->read($first)); $this->assertCount(1, $events); $this->assertSame($second, $events[0]->id); - $this->assertCount(0, $feed->read($second)); + $this->assertCount(0, $server->read($second)); } public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void @@ -147,7 +147,7 @@ public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void extensions: ['traceparent' => '00-abc-def-01'], )); - $event = self::events($this->feed()->read())[0]; + $event = self::events($this->server()->read())[0]; $this->assertSame('https://example.com/schema.json', $event->dataschema); $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); @@ -155,16 +155,16 @@ public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void public function testAnAbsentSubjectStaysAbsent(): void { - $this->producer()->append('test'); + $this->producer()->produce('test'); - $this->assertNull(self::events($this->feed()->read())[0]->subject); + $this->assertNull(self::events($this->server()->read())[0]->subject); } public function testAScalarPayloadSurvivesTheRoundTrip(): void { - $this->producer()->append('test', 'a string'); + $this->producer()->produce('test', 'a string'); - $this->assertSame('a string', self::events($this->feed()->read())[0]->data); + $this->assertSame('a string', self::events($this->server()->read())[0]->data); } public function testNestedPayloadsSurviveTheRoundTrip(): void @@ -176,27 +176,27 @@ public function testNestedPayloadsSurviveTheRoundTrip(): void 'unicode' => 'ΓΌnΓ―cΓΈde βœ“', ]; - $this->producer()->append('test', $data); + $this->producer()->produce('test', $data); - $this->assertSame($data, self::events($this->feed()->read())[0]->data); + $this->assertSame($data, self::events($this->server()->read())[0]->data); } public function testHonoursTheLimit(): void { - [$producer, $feed] = $this->feedAndProducer(); + [$producer, $server] = $this->serverAndProducer(); foreach (\range(1, 10) as $i) { - $producer->append('test'); + $producer->produce('test'); } - $this->assertCount(3, $feed->read(null, 3)); + $this->assertCount(3, $server->read(null, 3)); } public function testRejectsAPositionThatIsNotAFeedId(): void { $this->expectException(Invalid::class); - $this->feed()->read('not-a-position'); + $this->server()->read('not-a-position'); } /** @@ -206,15 +206,15 @@ public function testRejectsAPositionThatIsNotAFeedId(): void */ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void { - [$producer, $feed] = $this->feedAndProducer(maxSize: 10); + [$producer, $server] = $this->serverAndProducer(maxSize: 10); - $first = $producer->append('first'); + $first = $producer->produce('first'); foreach (\range(1, 500) as $i) { - $producer->append('event-' . $i); + $producer->produce('event-' . $i); } - $events = $feed->read($first); + $events = $server->read($first); $this->assertFalse($events->isEmpty(), 'A consumer that fell behind must still get what is retained'); $this->assertLessThan(500, $this->redis->xLen('feed:' . $this->name), 'The feed must be trimmed'); @@ -222,11 +222,11 @@ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void public function testLongPollingReturnsAsSoonAsTheFeedHasSomething(): void { - [$producer, $feed] = $this->feedAndProducer(); - $producer->append('a'); + [$producer, $server] = $this->serverAndProducer(); + $producer->produce('a'); $started = \microtime(true); - $events = $feed->poll(null, 10, 3000); + $events = $server->poll(null, 10, 3000); $this->assertCount(1, $events); $this->assertLessThan(1, \microtime(true) - $started); @@ -235,7 +235,7 @@ public function testLongPollingReturnsAsSoonAsTheFeedHasSomething(): void public function testLongPollingGivesUpAtTheTimeout(): void { $started = \microtime(true); - $events = $this->feed()->poll(null, 10, 700); + $events = $this->server()->poll(null, 10, 700); $this->assertCount(0, $events); $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started); @@ -247,67 +247,67 @@ public function testLongPollingGivesUpAtTheTimeout(): void */ public function testTheTipSentinelSkipsTheBacklog(): void { - [$producer, $feed] = $this->feedAndProducer(); + [$producer, $server] = $this->serverAndProducer(); - $this->assertNull($feed->tip(), 'An empty feed has no tip'); + $this->assertNull($server->tip(), 'An empty feed has no tip'); - $producer->append('a'); - $last = $producer->append('b'); + $producer->produce('a'); + $last = $producer->produce('b'); - $this->assertSame($last, $feed->tip()); - $this->assertCount(0, $feed->read('$')); + $this->assertSame($last, $server->tip()); + $this->assertCount(0, $server->read('$')); } public function testConsumesThroughAPersistedCursor(): void { - $journal = new RedisJournal($this->redis, $this->name); - $producer = new Producer($journal, 'urn:test:e2e'); + $store = new RedisStore($this->redis, $this->name); + $producer = new Producer($store, 'urn:test:e2e'); $cursor = new RedisCursor($this->redis); - $producer->append('a'); - $last = $producer->append('b'); + $producer->produce('a'); + $last = $producer->produce('b'); $seen = []; $handler = function (CloudEvent $event) use (&$seen): void { $seen[] = $event->type; }; - $this->assertSame(2, (new Consumer($journal, 'invalidator', $cursor))->consume($handler)); + $this->assertSame(2, (new Consumer($store, $cursor, 'invalidator'))->consume($handler)); $this->assertSame($last, $cursor->load($this->name, 'invalidator')); // A second Consumer stands in for a restart: it has no in-memory // position, so it has to pick the stored one up to avoid replaying. - $this->assertSame(0, (new Consumer($journal, 'invalidator', $cursor))->consume($handler)); + $this->assertSame(0, (new Consumer($store, $cursor, 'invalidator'))->consume($handler)); $this->assertSame(['a', 'b'], $seen); } public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void { - $journal = new RedisJournal($this->redis, $this->name); - $producer = new Producer($journal, 'urn:test:e2e'); + $store = new RedisStore($this->redis, $this->name); + $producer = new Producer($store, 'urn:test:e2e'); $cursor = new RedisCursor($this->redis); - $producer->append('a'); + $producer->produce('a'); - $this->assertSame(1, (new Consumer($journal, 'one', $cursor))->consume(fn (CloudEvent $e) => null)); - $this->assertSame(1, (new Consumer($journal, 'two', $cursor))->consume(fn (CloudEvent $e) => null)); + $this->assertSame(1, (new Consumer($store, $cursor, 'one'))->consume(fn (CloudEvent $e) => null)); + $this->assertSame(1, (new Consumer($store, $cursor, 'two'))->consume(fn (CloudEvent $e) => null)); } public function testResetReplaysTheRetainedFeed(): void { - $journal = new RedisJournal($this->redis, $this->name); - $producer = new Producer($journal, 'urn:test:e2e'); + $store = new RedisStore($this->redis, $this->name); + $producer = new Producer($store, 'urn:test:e2e'); $cursor = new RedisCursor($this->redis); - $producer->append('a'); - $producer->append('b'); + $producer->produce('a'); + $producer->produce('b'); - $consumer = new Consumer($journal, 'invalidator', $cursor); + $consumer = new Consumer($store, $cursor, 'invalidator'); $consumer->consume(fn (CloudEvent $e) => null); $consumer->reset(); $this->assertNull($cursor->load($this->name, 'invalidator')); - $this->assertSame(2, (new Consumer($journal, 'invalidator', $cursor))->consume(fn (CloudEvent $e) => null)); + $this->assertSame(2, (new Consumer($store, $cursor, 'invalidator'))->consume(fn (CloudEvent $e) => null)); } public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index c1a2853..781a91b 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -5,7 +5,7 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Journal\Memory as MemoryJournal; +use Utopia\Feed\Store\Memory as MemoryStore; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; use Utopia\Feed\Cursor\Memory as MemoryCursor; @@ -16,11 +16,12 @@ use Utopia\Feed\Protocol; use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FailingCursor; -use Utopia\Tests\Unit\Support\MidPollJournal; +use Utopia\Tests\Unit\Support\FakeTransport; +use Utopia\Tests\Unit\Support\MidPollStore; class ConsumerTest extends TestCase { - private MemoryJournal $journal; + private MemoryStore $store; private Producer $producer; @@ -28,14 +29,14 @@ class ConsumerTest extends TestCase protected function setUp(): void { - $this->journal = new MemoryJournal('edge'); - $this->producer = new Producer($this->journal, 'urn:test'); + $this->store = new MemoryStore('edge'); + $this->producer = new Producer($this->store, 'urn:test'); $this->cursor = new MemoryCursor(); } private function consumer(?Cursor $cursor = null, int $batch = Consumer::BATCH): Consumer { - return new Consumer($this->journal, 'invalidator', $cursor ?? $this->cursor, $batch); + return new Consumer($this->store, $cursor ?? $this->cursor, 'invalidator', batch: $batch); } /** @@ -54,8 +55,8 @@ private function drain(Consumer $consumer, ?int &$count = null): array public function testHandlesEachEventAndAdvancesPastTheLastOne(): void { - $this->producer->append('a'); - $last = $this->producer->append('b'); + $this->producer->produce('a'); + $last = $this->producer->produce('b'); $consumer = $this->consumer(); @@ -67,7 +68,7 @@ public function testHandlesEachEventAndAdvancesPastTheLastOne(): void public function testCaughtUpConsumerDoesNothing(): void { - $this->producer->append('a'); + $this->producer->produce('a'); $consumer = $this->consumer(); $consumer->consume(fn (CloudEvent $event) => null); @@ -77,8 +78,8 @@ public function testCaughtUpConsumerDoesNothing(): void public function testResumesFromTheStoredPosition(): void { - $first = $this->producer->append('a'); - $this->producer->append('b'); + $first = $this->producer->produce('a'); + $this->producer->produce('b'); $this->cursor->save('edge', 'invalidator', $first); @@ -92,15 +93,15 @@ public function testResumesFromTheStoredPosition(): void */ public function testAConsumerWithNoPositionStartsAtTheOldestEventNotTheTip(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); $this->assertSame(['a', 'b'], $this->drain($this->consumer())); } public function testReadsTheStoreOnceAndThenTracksThePositionInMemory(): void { - $this->producer->append('a'); + $this->producer->produce('a'); $cursor = new class () extends MemoryCursor { public int $loads = 0; @@ -127,9 +128,9 @@ public function load(string $feed, string $consumer): ?string public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void { - $first = $this->producer->append('a'); - $this->producer->append('b'); - $this->producer->append('c'); + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + $this->producer->produce('c'); $consumer = $this->consumer(); $seen = []; @@ -153,8 +154,8 @@ public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void public function testRetriesTheFailedEventOnTheNextRun(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); $consumer = $this->consumer(); $attempts = 0; @@ -181,7 +182,7 @@ public function testRetriesTheFailedEventOnTheNextRun(): void */ public function testAFailureOnTheFirstEventCommitsNothing(): void { - $this->producer->append('a'); + $this->producer->produce('a'); try { $this->consumer()->consume(fn (CloudEvent $event) => throw new \RuntimeException('nope')); @@ -194,21 +195,20 @@ public function testAFailureOnTheFirstEventCommitsNothing(): void public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void { - $this->producer->append('a'); - $this->producer->append('b'); - $this->producer->append('c'); + $this->producer->produce('a'); + $this->producer->produce('b'); + $this->producer->produce('c'); $this->assertSame(3, $this->consumer()->consume(fn (CloudEvent $event) => null)); } /** - * The clamping the client-side Feed wrapper used to provide lives in the - * consumer now: whatever the constructor was given, a journal is never - * asked for more than the protocol allows. + * The consumer clamps its own inputs: whatever the constructor was given, + * a store is never asked for more than the protocol allows. */ public function testClampsBatchAndTimeoutToTheProtocolLimits(): void { - $journal = new class ('edge') extends MemoryJournal { + $store = new class ('edge') extends MemoryStore { public ?int $limit = null; public ?int $timeout = null; @@ -222,17 +222,17 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array } }; - $consumer = new Consumer($journal, 'invalidator', $this->cursor, batch: 5_000, timeout: 120_000); + $consumer = new Consumer($store, $this->cursor, 'invalidator', batch: 5_000, timeout: 120_000); $consumer->consume(fn (CloudEvent $event) => null); - $this->assertSame(Protocol::MAX_BATCH, $journal->limit); - $this->assertSame(Protocol::MAX_TIMEOUT, $journal->timeout); + $this->assertSame(Protocol::MAX_BATCH, $store->limit); + $this->assertSame(Protocol::MAX_TIMEOUT, $store->timeout); } public function testDrainsABacklogInBatches(): void { foreach (\range(1, 10) as $i) { - $this->producer->append('event-' . $i); + $this->producer->produce('event-' . $i); } $consumer = $this->consumer(batch: 4); @@ -250,7 +250,7 @@ public function testDrainsABacklogInBatches(): void */ public function testAPositionThatCannotBeLoadedStopsTheRun(): void { - $this->producer->append('a'); + $this->producer->produce('a'); $consumer = $this->consumer(new FailingCursor(onLoad: true)); $seen = []; @@ -273,7 +273,7 @@ public function testAPositionThatCannotBeLoadedStopsTheRun(): void */ public function testAFailedLoadIsRetriedOnTheNextRun(): void { - $this->producer->append('a'); + $this->producer->produce('a'); $cursor = new class () extends MemoryCursor { public bool $fail = true; @@ -307,8 +307,8 @@ public function load(string $feed, string $consumer): ?string */ public function testAPositionThatCannotBeSavedIsRaisedAfterTheEventsAreHandled(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); $consumer = $this->consumer(new FailingCursor(onSave: true)); $seen = []; @@ -328,10 +328,10 @@ public function testAPositionThatCannotBeSavedIsRaisedAfterTheEventsAreHandled() public function testTipStartDoesNotAnnounceTheBacklog(): void { - $this->producer->append('old-1'); - $this->producer->append('old-2'); + $this->producer->produce('old-1'); + $this->producer->produce('old-2'); - $consumer = new Consumer($this->journal, 'notifier', $this->cursor, start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'notifier', start: Start::Tip); $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); $this->assertNull($this->cursor->load('edge', 'notifier'), 'Skipping the backlog is not progress to commit'); @@ -343,11 +343,11 @@ public function testTipStartDoesNotAnnounceTheBacklog(): void */ public function testTipStartDeliversWhatLandsMidPoll(): void { - $journal = new MidPollJournal('edge'); - (new Producer($journal, 'urn:test'))->append('old'); + $store = new MidPollStore('edge'); + (new Producer($store, 'urn:test'))->produce('old'); $cursor = new MemoryCursor(); - $consumer = new Consumer($journal, 'notifier', $cursor, timeout: 5_000, start: Start::Tip); + $consumer = new Consumer($store, $cursor, 'notifier', timeout: 5_000, start: Start::Tip); $seen = []; $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { @@ -361,23 +361,23 @@ public function testTipStartDeliversWhatLandsMidPoll(): void public function testAStoredCursorBeatsTipStart(): void { - $first = $this->producer->append('a'); - $this->producer->append('b'); + $first = $this->producer->produce('a'); + $this->producer->produce('b'); $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer($this->journal, 'invalidator', $this->cursor, start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Start::Tip); $this->assertSame(['b'], $this->drain($consumer), 'A restart must not skip the gap'); } public function testResetWithTipStartResumesFromNow(): void { - $first = $this->producer->append('a'); - $this->producer->append('b'); + $first = $this->producer->produce('a'); + $this->producer->produce('b'); $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer($this->journal, 'invalidator', $this->cursor, start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Start::Tip); $this->assertSame(['b'], $this->drain($consumer), 'The stored position still wins before the reset'); @@ -388,7 +388,7 @@ public function testResetWithTipStartResumesFromNow(): void public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void { - $consumer = new Consumer($this->journal, 'notifier', $this->cursor, timeout: 600, start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'notifier', timeout: 600, start: Start::Tip); $started = \microtime(true); @@ -398,7 +398,7 @@ public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void public function testTipStartOnAnEmptyFeedDeliversWhatLandsMidWait(): void { - $consumer = new Consumer(new MidPollJournal('edge'), 'notifier', $this->cursor, timeout: 5_000, start: Start::Tip); + $consumer = new Consumer(new MidPollStore('edge'), $this->cursor, 'notifier', timeout: 5_000, start: Start::Tip); $seen = []; $consumer->consume(function (CloudEvent $event) use (&$seen): void { @@ -410,8 +410,8 @@ public function testTipStartOnAnEmptyFeedDeliversWhatLandsMidWait(): void public function testResetReplaysEverythingStillRetained(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); $consumer = $this->consumer(); $consumer->consume(fn (CloudEvent $event) => null); @@ -425,9 +425,9 @@ public function testResetReplaysEverythingStillRetained(): void public function testSeekPositionsTheNextRunStrictlyAfterTheGivenId(): void { - $this->producer->append('a'); - $second = $this->producer->append('b'); - $this->producer->append('c'); + $this->producer->produce('a'); + $second = $this->producer->produce('b'); + $this->producer->produce('c'); $consumer = $this->consumer(); $consumer->seek($second); @@ -442,9 +442,9 @@ public function testSeekPositionsTheNextRunStrictlyAfterTheGivenId(): void */ public function testASeekSurvivesARestart(): void { - $this->producer->append('a'); - $second = $this->producer->append('b'); - $this->producer->append('c'); + $this->producer->produce('a'); + $second = $this->producer->produce('b'); + $this->producer->produce('c'); $this->consumer()->seek($second); @@ -458,8 +458,8 @@ public function testASeekSurvivesARestart(): void */ public function testSeekingToAPoisonEventsIdUnblocksTheConsumer(): void { - $this->producer->append('poison'); - $this->producer->append('after'); + $this->producer->produce('poison'); + $this->producer->produce('after'); $consumer = $this->consumer(); $poison = null; @@ -490,7 +490,7 @@ public function testSeekingToAPoisonEventsIdUnblocksTheConsumer(): void */ public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void { - $first = $this->producer->append('a'); + $first = $this->producer->produce('a'); $this->cursor->save('edge', 'invalidator', $first); $consumer = $this->consumer(); @@ -525,7 +525,7 @@ public static function notPositions(): array */ public function testASeekThatCannotPersistFailsLoudlyAndMovesNothing(): void { - $this->producer->append('a'); + $this->producer->produce('a'); $consumer = $this->consumer(new FailingCursor(onSave: true)); @@ -543,10 +543,10 @@ public function testASeekThatCannotPersistFailsLoudlyAndMovesNothing(): void public function testConsumersOfTheSameFeedTrackSeparatePositions(): void { - $this->producer->append('a'); + $this->producer->produce('a'); - $one = new Consumer($this->journal, 'one', $this->cursor); - $two = new Consumer($this->journal, 'two', $this->cursor); + $one = new Consumer($this->store, $this->cursor, 'one'); + $two = new Consumer($this->store, $this->cursor, 'two'); $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); $this->assertSame(1, $two->consume(fn (CloudEvent $event) => null), 'The second consumer has its own position'); @@ -562,7 +562,7 @@ public function testRejectsAnEmptyConsumerName(): void { $this->expectException(Invalid::class); - new Consumer($this->journal, '', $this->cursor); + new Consumer($this->store, $this->cursor, ''); } public function testExposesItsName(): void @@ -570,17 +570,49 @@ public function testExposesItsName(): void $this->assertSame('invalidator', $this->consumer()->getName()); } + /** + * A client carries the endpoint but not the feed's name, so a consumer + * built over one has to be told which feed it is reading. + */ + public function testConsumingThroughAClientRequiresAFeedName(): void + { + $this->expectException(Invalid::class); + + new Consumer(FakeTransport::of([]), $this->cursor, 'invalidator'); + } + + /** + * A local store already names its feed. Repeating the name is harmless; + * contradicting it means the caller is confused about what they are + * reading, which must not resolve silently in either direction. + */ + public function testAFeedNameThatContradictsTheStoreIsRejected(): void + { + $this->expectException(Invalid::class); + + new Consumer($this->store, $this->cursor, 'invalidator', feed: 'other'); + } + + public function testAFeedNameThatMatchesTheStoreIsAccepted(): void + { + $this->producer->produce('a'); + + $consumer = new Consumer($this->store, $this->cursor, 'invalidator', feed: 'edge'); + + $this->assertSame(1, $consumer->consume(fn (CloudEvent $event) => null)); + } + /** * The failure a consumer must not turn into a gap: if the read itself * fails, nothing is handled and nothing is committed. */ public function testAFailedReadLeavesThePositionAlone(): void { - $first = $this->producer->append('a'); - $this->producer->append('b'); + $first = $this->producer->produce('a'); + $this->producer->produce('b'); $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer(new \Utopia\Feed\Journal\None('edge'), 'invalidator', $this->cursor); + $consumer = new Consumer(new \Utopia\Feed\Store\None('edge'), $this->cursor, 'invalidator'); $this->expectException(\Utopia\Feed\Exception\Unsupported::class); diff --git a/tests/Feed/Unit/ProducerTest.php b/tests/Feed/Unit/ProducerTest.php index 0e31610..0be1600 100644 --- a/tests/Feed/Unit/ProducerTest.php +++ b/tests/Feed/Unit/ProducerTest.php @@ -8,32 +8,32 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; -use Utopia\Feed\Feed; +use Utopia\Feed\Server; use Utopia\Feed\Id; -use Utopia\Feed\Journal\Memory; -use Utopia\Feed\Journal\None; +use Utopia\Feed\Store\Memory; +use Utopia\Feed\Store\None; use Utopia\Feed\Producer; use Utopia\Feed\Remote; use Utopia\Tests\Unit\Support\FakeTransport; class ProducerTest extends TestCase { - private Memory $journal; + private Memory $store; private Producer $producer; - private Feed $feed; + private Server $server; protected function setUp(): void { - $this->journal = new Memory('edge'); - $this->producer = new Producer($this->journal, 'urn:appwrite:cloud:fra'); - $this->feed = new Feed($this->journal); + $this->store = new Memory('edge'); + $this->producer = new Producer($this->store, 'urn:appwrite:cloud:fra'); + $this->server = new Server($this->store); } - public function testAppendReturnsAPosition(): void + public function testProduceReturnsAPosition(): void { - $id = $this->producer->append('io.appwrite.edge.invalidate', ['tags' => ['project' => 'p1']]); + $id = $this->producer->produce('io.appwrite.edge.invalidate', ['tags' => ['project' => 'p1']]); $this->assertTrue(Id::isValid($id)); } @@ -41,12 +41,12 @@ public function testAppendReturnsAPosition(): void /** @return list */ private function events(): array { - return \array_values(\iterator_to_array($this->feed->read())); + return \array_values(\iterator_to_array($this->server->read())); } - public function testStampsTheSourceAndTimeOnAppend(): void + public function testStampsTheSourceAndTimeOnProduce(): void { - $this->producer->append('test'); + $this->producer->produce('test'); $event = $this->events()[0]; @@ -56,13 +56,13 @@ public function testStampsTheSourceAndTimeOnAppend(): void } /** - * Recording it at append rather than at read keeps it correct for a feed + * Recording it when produced rather than when read keeps it correct for a feed * read back somewhere other than where it was written. */ - public function testKeepsTheSourceOfTheProducerThatAppended(): void + public function testKeepsTheSourceOfTheProducerThatProduced(): void { - (new Producer($this->journal, 'urn:appwrite:cloud:fra'))->append('test'); - (new Producer($this->journal, 'urn:appwrite:cloud:nyc'))->append('test'); + (new Producer($this->store, 'urn:appwrite:cloud:fra'))->produce('test'); + (new Producer($this->store, 'urn:appwrite:cloud:nyc'))->produce('test'); $events = $this->events(); @@ -93,14 +93,14 @@ public function testRejectsAnEmptyEventType(): void { $this->expectException(Invalid::class); - $this->producer->append(''); + $this->producer->produce(''); } public function testRejectsAPayloadThatCannotBeEncoded(): void { $this->expectException(Invalid::class); - $this->producer->append('test', ['resource' => \fopen('php://memory', 'r')]); + $this->producer->produce('test', ['resource' => \fopen('php://memory', 'r')]); } /** @@ -111,17 +111,17 @@ public function testRejectsAnEmptySource(): void { $this->expectException(Invalid::class); - new Producer($this->journal, ''); + new Producer($this->store, ''); } /** - * A remote feed belongs to whoever appends to it, so Remote is neither a - * Journal nor Appendable β€” the mistake is a type error at construction + * A remote feed belongs to whoever produces into it, so Remote is neither + * a Store nor Appendable β€” the mistake is a type error at construction * rather than an exception once an event is already in hand. */ public function testARemoteFeedIsRejectedOnConstruction(): void { - $remote = new Remote(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', 'edge'); + $remote = new Remote(FakeTransport::of([]), 'edge'); $this->expectException(\TypeError::class); @@ -135,6 +135,6 @@ public function testAFeedWithNoBackendFailsLoudlyRatherThanDroppingEvents(): voi $this->expectException(Unsupported::class); - $producer->append('test'); + $producer->produce('test'); } } diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index cbc85f7..9e65d2a 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -13,14 +13,14 @@ use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; -use Utopia\Feed\Feed; use Utopia\Feed\Producer; use Utopia\Feed\Protocol; use Utopia\Feed\Remote; +use Utopia\Feed\Server; use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FakeTransport; use Utopia\Tests\Unit\Support\FeedServer; -use Utopia\Tests\Unit\Support\MidPollJournal; +use Utopia\Tests\Unit\Support\MidPollStore; class RemoteTest extends TestCase { @@ -32,7 +32,7 @@ private function remote(array $responses = []): array { $transport = FakeTransport::of($responses); - return [new Remote($transport, 'https://cloud.example.com/v1/feeds', 'edge'), $transport]; + return [new Remote($transport, 'edge'), $transport]; } public function testReadsAFeedOverHttp(): void @@ -49,11 +49,16 @@ public function testReadsAFeedOverHttp(): void $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); } - public function testAppendsTheFeedNameToTheEndpoint(): void + /** + * The endpoint lives on the client β€” the feed asks for its name as a + * relative path and the client resolves it against its base URI. + */ + public function testResolvesTheFeedNameAgainstTheClientsBaseUri(): void { - [$remote, $transport] = $this->remote(); + $transport = FakeTransport::of([]); + $client = (new Client($transport))->withBaseUri('https://cloud.example.com/v1/feeds'); - $remote->read(); + (new Remote($client, 'edge'))->read(); $this->assertStringStartsWith('https://cloud.example.com/v1/feeds/edge', $transport->recorder->last()['uri']); } @@ -61,8 +66,9 @@ public function testAppendsTheFeedNameToTheEndpoint(): void public function testEncodesAFeedNameThatNeedsIt(): void { $transport = FakeTransport::of([]); + $client = (new Client($transport))->withBaseUri('https://cloud.example.com/v1/feeds/'); - (new Remote($transport, 'https://cloud.example.com/v1/feeds/', 'a b/c'))->read(); + (new Remote($client, 'a b/c'))->read(); $this->assertStringStartsWith( 'https://cloud.example.com/v1/feeds/a%20b%2Fc', @@ -74,7 +80,7 @@ public function testRejectsAnEmptyFeedName(): void { $this->expectException(Invalid::class); - new Remote(FakeTransport::of([]), 'https://cloud.example.com/v1/feeds', ''); + new Remote(FakeTransport::of([]), ''); } public function testExposesTheFeedItReads(): void @@ -259,8 +265,10 @@ public function testWorksThroughTheClientItself(): void { $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-0', type: 'a', source: 'urn:test')]))]); - $client = (new Client($transport))->withHeaders(['x-appwrite-jwt' => 'token']); - $remote = new Remote($client, 'https://cloud.example.com/v1/feeds', 'edge'); + $client = (new Client($transport)) + ->withBaseUri('https://cloud.example.com/v1/feeds') + ->withHeaders(['x-appwrite-jwt' => 'token']); + $remote = new Remote($client, 'edge'); $events = $remote->read(); @@ -270,11 +278,12 @@ public function testWorksThroughTheClientItself(): void /** * The point of this class: a remote feed is consumed with exactly the - * code a local one is. + * code a local one is. The consumer is built straight over the client β€” + * it wraps the feed name and the client into a Remote itself. */ public function testConsumesARemoteFeedThroughTheSameConsumer(): void { - [$remote, $transport] = $this->remote([ + $transport = FakeTransport::of([ FakeTransport::json(Protocol::encode([ new CloudEvent(id: '1-0', type: 'a', source: 'urn:test'), new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), @@ -284,7 +293,7 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void ]); $cursor = new MemoryCursor(); - $consumer = new Consumer($remote, 'invalidator', $cursor); + $consumer = new Consumer($transport, $cursor, 'invalidator', feed: 'edge'); $seen = []; $handler = function (CloudEvent $event) use (&$seen): void { @@ -308,13 +317,12 @@ public function testConsumesARemoteFeedThroughTheSameConsumer(): void */ public function testTipStartWorksOverHttp(): void { - $journal = new MidPollJournal('edge'); - $producer = new Producer($journal, 'urn:test'); - $producer->append('old'); + $store = new MidPollStore('edge'); + $producer = new Producer($store, 'urn:test'); + $producer->produce('old'); - $server = new FeedServer(new Feed($journal)); - $remote = new Remote($server, 'https://cloud.example.com/v1/feeds', 'edge'); - $consumer = new Consumer($remote, 'notifier', new MemoryCursor(), timeout: 5_000, start: Start::Tip); + $endpoint = new FeedServer(new Server($store)); + $consumer = new Consumer($endpoint, new MemoryCursor(), 'notifier', feed: 'edge', timeout: 5_000, start: Start::Tip); $seen = []; $handler = function (CloudEvent $event) use (&$seen): void { @@ -323,14 +331,14 @@ public function testTipStartWorksOverHttp(): void $this->assertSame(1, $consumer->consume($handler)); $this->assertSame(['landed'], $seen, 'The backlog is skipped; the mid-wait event is not'); - $this->assertMatchesRegularExpression('/lastEventId=(%24|\$)/', $server->recorder->last()['uri']); + $this->assertMatchesRegularExpression('/lastEventId=(%24|\$)/', $endpoint->recorder->last()['uri']); // The position now saves as a real id, so the sentinel never // appears on the wire again. - $producer->append('after'); + $producer->produce('after'); $this->assertSame(1, $consumer->consume($handler)); $this->assertSame(['landed', 'after'], $seen); - $this->assertDoesNotMatchRegularExpression('/lastEventId=(%24|\$)/', $server->recorder->last()['uri']); + $this->assertDoesNotMatchRegularExpression('/lastEventId=(%24|\$)/', $endpoint->recorder->last()['uri']); } } diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index a931127..7aa80da 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -7,14 +7,13 @@ use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter\Memory as CacheMemory; use Utopia\Cache\Cache as UtopiaCache; -use Utopia\Feed\Journal\Memory as MemoryJournal; +use Utopia\Feed\Store\Memory as MemoryStore; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor\Cache as CacheCursor; use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Feed; use Utopia\Feed\Producer; use Utopia\Feed\Protocol; -use Utopia\Feed\Remote; +use Utopia\Feed\Server; use Utopia\Tests\Unit\Support\FeedServer; /** @@ -27,31 +26,27 @@ class RoundTripTest extends TestCase { private Producer $producer; - private Remote $remote; - private FeedServer $server; private CacheCursor $cursor; protected function setUp(): void { - $journal = new MemoryJournal('edge'); - $this->producer = new Producer($journal, 'urn:appwrite:cloud:fra'); - $this->server = new FeedServer(new Feed($journal)); - - $this->remote = new Remote($this->server, 'https://cloud.example.com/v1/feeds', 'edge'); + $store = new MemoryStore('edge'); + $this->producer = new Producer($store, 'urn:appwrite:cloud:fra'); + $this->server = new FeedServer(new Server($store)); $this->cursor = new CacheCursor(new UtopiaCache(new CacheMemory())); } private function consumer(string $name = 'invalidator', int $batch = Consumer::BATCH): Consumer { - return new Consumer($this->remote, $name, $this->cursor, $batch); + return new Consumer($this->server, $this->cursor, $name, feed: 'edge', batch: $batch); } public function testAnEventSurvivesTheWholeTrip(): void { - $this->producer->append( + $this->producer->produce( 'io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com'], 'isAppwriteNetwork' => true], 'example.com', @@ -75,7 +70,7 @@ public function testAnEventSurvivesTheWholeTrip(): void public function testTheConsumerOnlyEverSeesEachEventOnce(): void { foreach (\range(1, 5) as $i) { - $this->producer->append('event-' . $i); + $this->producer->produce('event-' . $i); } $consumer = $this->consumer(); @@ -87,7 +82,7 @@ public function testTheConsumerOnlyEverSeesEachEventOnce(): void $consumer->consume($handler); $consumer->consume($handler); - $this->producer->append('event-6'); + $this->producer->produce('event-6'); $consumer->consume($handler); $this->assertSame( @@ -103,7 +98,7 @@ public function testTheConsumerOnlyEverSeesEachEventOnce(): void public function testAConsumerShippedLateDrainsTheBacklog(): void { foreach (\range(1, 3) as $i) { - $this->producer->append('missed-' . $i); + $this->producer->produce('missed-' . $i); } $seen = []; @@ -121,12 +116,12 @@ public function testAConsumerShippedLateDrainsTheBacklog(): void */ public function testARestartedConsumerResumesWhereItLeftOff(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); $this->consumer()->consume(fn (CloudEvent $event) => null); - $this->producer->append('c'); + $this->producer->produce('c'); $seen = []; $this->consumer()->consume(function (CloudEvent $event) use (&$seen): void { @@ -138,9 +133,9 @@ public function testARestartedConsumerResumesWhereItLeftOff(): void public function testAFailedEventBlocksTheOnesBehindItUntilItSucceeds(): void { - $this->producer->append('a'); - $this->producer->append('poison'); - $this->producer->append('c'); + $this->producer->produce('a'); + $this->producer->produce('poison'); + $this->producer->produce('c'); $consumer = $this->consumer(); $seen = []; @@ -178,7 +173,7 @@ public function testAFailedEventBlocksTheOnesBehindItUntilItSucceeds(): void public function testTheProducerCachesFullBatchesAndNothingElse(): void { foreach (\range(1, 5) as $i) { - $this->producer->append('event-' . $i); + $this->producer->produce('event-' . $i); } $consumer = $this->consumer(batch: 2); @@ -198,14 +193,14 @@ public function testTheProducerCachesFullBatchesAndNothingElse(): void public function testTwoConsumersOfOneProducerAreIndependent(): void { - $this->producer->append('a'); + $this->producer->produce('a'); $one = $this->consumer('one'); $two = $this->consumer('two'); $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); - $this->producer->append('b'); + $this->producer->produce('b'); $this->assertSame(2, $two->consume(fn (CloudEvent $event) => null), 'The second consumer starts from the beginning'); $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null), 'The first only sees what is new to it'); diff --git a/tests/Feed/Unit/FeedTest.php b/tests/Feed/Unit/ServerTest.php similarity index 67% rename from tests/Feed/Unit/FeedTest.php rename to tests/Feed/Unit/ServerTest.php index 6aa698e..49717a5 100644 --- a/tests/Feed/Unit/FeedTest.php +++ b/tests/Feed/Unit/ServerTest.php @@ -5,31 +5,31 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Journal\Memory; -use Utopia\Feed\Journal\None; +use Utopia\Feed\Store\Memory; +use Utopia\Feed\Store\None; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Batch; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; -use Utopia\Feed\Feed; +use Utopia\Feed\Server; use Utopia\Feed\Producer; use Utopia\Feed\Protocol; use Utopia\Feed\Id; -use Utopia\Tests\Unit\Support\MidPollJournal; +use Utopia\Tests\Unit\Support\MidPollStore; -class FeedTest extends TestCase +class ServerTest extends TestCase { - private Memory $journal; + private Memory $store; - private Feed $feed; + private Server $server; private Producer $producer; protected function setUp(): void { - $this->journal = new Memory('edge'); - $this->producer = new Producer($this->journal, 'urn:appwrite:cloud:fra'); - $this->feed = new Feed($this->journal); + $this->store = new Memory('edge'); + $this->producer = new Producer($this->store, 'urn:appwrite:cloud:fra'); + $this->server = new Server($this->store); } /** @return list */ @@ -40,9 +40,9 @@ private static function events(Batch $batch): array public function testReadsBackWhatWasAppended(): void { - $this->producer->append('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + $this->producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); - $events = self::events($this->feed->read()); + $events = self::events($this->server->read()); $this->assertCount(1, $events); $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); @@ -53,17 +53,17 @@ public function testReadsBackWhatWasAppended(): void public function testEventsComeBackOldestFirst(): void { foreach (['a', 'b', 'c'] as $type) { - $this->producer->append($type); + $this->producer->produce($type); } - $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($this->feed->read()))); + $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($this->server->read()))); } public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void { $ids = []; for ($i = 0; $i < 50; $i++) { - $ids[] = $this->producer->append('test'); + $ids[] = $this->producer->produce('test'); } $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); @@ -75,10 +75,10 @@ public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void public function testReadsStrictlyAfterTheGivenPosition(): void { - $first = $this->producer->append('a'); - $this->producer->append('b'); + $first = $this->producer->produce('a'); + $this->producer->produce('b'); - $events = self::events($this->feed->read($first)); + $events = self::events($this->server->read($first)); $this->assertCount(1, $events); $this->assertSame('b', $events[0]->type); @@ -86,27 +86,27 @@ public function testReadsStrictlyAfterTheGivenPosition(): void public function testReadFromTheLastEventIsEmpty(): void { - $this->producer->append('a'); - $last = $this->producer->append('b'); + $this->producer->produce('a'); + $last = $this->producer->produce('b'); - $this->assertCount(0, $this->feed->read($last)); + $this->assertCount(0, $this->server->read($last)); } public function testNullPositionReadsFromTheOldestRetainedEvent(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); - $this->assertCount(2, $this->feed->read(null)); + $this->assertCount(2, $this->server->read(null)); } public function testHonoursTheLimit(): void { foreach (\range(1, 10) as $i) { - $this->producer->append('test'); + $this->producer->produce('test'); } - $this->assertCount(3, $this->feed->read(null, 3)); + $this->assertCount(3, $this->server->read(null, 3)); } /** @@ -116,18 +116,18 @@ public function testHonoursTheLimit(): void */ public function testClampsTheLimitToTheMaximum(): void { - $this->producer->append('test'); + $this->producer->produce('test'); - $this->assertCount(1, $this->feed->read(null, Protocol::MAX_BATCH * 10)); - $this->assertCount(1, $this->feed->read(null, 0)); - $this->assertCount(1, $this->feed->read(null, -5)); + $this->assertCount(1, $this->server->read(null, Protocol::MAX_BATCH * 10)); + $this->assertCount(1, $this->server->read(null, 0)); + $this->assertCount(1, $this->server->read(null, -5)); } public function testRejectsAPositionThatIsNotAFeedId(): void { $this->expectException(Invalid::class); - $this->feed->read('not-a-position'); + $this->server->read('not-a-position'); } /** @@ -144,7 +144,7 @@ public function testExtensionAttributesSurviveAppendAndRead(): void extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2], )); - $event = self::events($this->feed->read())[0]; + $event = self::events($this->server->read())[0]; $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); $this->assertSame(2, $event->extensions['retrycount']); @@ -166,7 +166,7 @@ public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void extensions: ['123' => 'digits', 'trace' => 'ok'], )); - $event = self::events($this->feed->read())[0]; + $event = self::events($this->server->read())[0]; // @phpstan-ignore offsetAccess.notFound $this->assertSame('digits', $event->extensions['123']); @@ -182,7 +182,7 @@ public function testDataschemaSurvivesAppendAndRead(): void dataschema: 'https://example.com/schema.json', )); - $this->assertSame('https://example.com/schema.json', self::events($this->feed->read())[0]->dataschema); + $this->assertSame('https://example.com/schema.json', self::events($this->server->read())[0]->dataschema); } /** @@ -211,9 +211,9 @@ public static function payloads(): array */ public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void { - $this->producer->append('test', $data); + $this->producer->produce('test', $data); - $this->assertSame($data, self::events($this->feed->read())[0]->data); + $this->assertSame($data, self::events($this->server->read())[0]->data); } /** @@ -222,24 +222,24 @@ public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void */ public function testAnEventWithNoSubjectHasANullSubject(): void { - $this->producer->append('test'); + $this->producer->produce('test'); - $this->assertNull(self::events($this->feed->read())[0]->subject); + $this->assertNull(self::events($this->server->read())[0]->subject); } public function testASubjectSurvivesAppendAndRead(): void { - $this->producer->append('test', [], 'example.com'); + $this->producer->produce('test', [], 'example.com'); - $this->assertSame('example.com', self::events($this->feed->read())[0]->subject); + $this->assertSame('example.com', self::events($this->server->read())[0]->subject); } public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void { - $this->producer->append('test'); + $this->producer->produce('test'); $started = \microtime(true); - $events = $this->feed->poll(null, 10, 2000); + $events = $this->server->poll(null, 10, 2000); $this->assertCount(1, $events); $this->assertLessThan(1, \microtime(true) - $started); @@ -248,7 +248,7 @@ public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void public function testPollGivesUpAtTheTimeoutWithAnEmptyBatch(): void { $started = \microtime(true); - $events = $this->feed->poll(null, 10, 600); + $events = $this->server->poll(null, 10, 600); $elapsed = \microtime(true) - $started; $this->assertCount(0, $events); @@ -260,16 +260,16 @@ public function testPollWithoutATimeoutIsAPlainRead(): void { $started = \microtime(true); - $this->assertCount(0, $this->feed->poll()); + $this->assertCount(0, $this->server->poll()); $this->assertLessThan(0.4, \microtime(true) - $started); } public function testAShorterPollIntervalDeliversAMidPollEventSooner(): void { - $journal = new MidPollJournal('edge', pollInterval: 20); + $store = new MidPollStore('edge', pollInterval: 20); $started = \microtime(true); - $events = $journal->poll(null, 10, 5_000); + $events = $store->poll(null, 10, 5_000); $elapsed = \microtime(true) - $started; $this->assertCount(1, $events); @@ -282,10 +282,10 @@ public function testAShorterPollIntervalDeliversAMidPollEventSooner(): void */ public function testPollHonoursATimeoutShorterThanTheInterval(): void { - $journal = new Memory('edge', pollInterval: 500); + $store = new Memory('edge', pollInterval: 500); $started = \microtime(true); - $events = $journal->poll(null, 10, 100); + $events = $store->poll(null, 10, 100); $elapsed = \microtime(true) - $started; $this->assertSame([], $events); @@ -309,15 +309,15 @@ public function testRejectsANegativePollInterval(): void public function testRetentionIsBoundedAndTrimsTheOldest(): void { - $journal = new Memory('small', maxSize: 3); - $producer = new Producer($journal, 'urn:appwrite:cloud:fra'); - $feed = new Feed($journal); + $store = new Memory('small', maxSize: 3); + $producer = new Producer($store, 'urn:appwrite:cloud:fra'); + $server = new Server($store); foreach (['a', 'b', 'c', 'd', 'e'] as $type) { - $producer->append($type); + $producer->produce($type); } - $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($feed->read()))); + $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($server->read()))); } /** @@ -326,64 +326,64 @@ public function testRetentionIsBoundedAndTrimsTheOldest(): void */ public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void { - $journal = new Memory('small', maxSize: 2); - $producer = new Producer($journal, 'urn:appwrite:cloud:fra'); - $feed = new Feed($journal); + $store = new Memory('small', maxSize: 2); + $producer = new Producer($store, 'urn:appwrite:cloud:fra'); + $server = new Server($store); - $first = $producer->append('a'); - $producer->append('b'); - $producer->append('c'); + $first = $producer->produce('a'); + $producer->produce('b'); + $producer->produce('c'); - $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($feed->read($first)))); + $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($server->read($first)))); } public function testExposesTheFeedItReads(): void { - $this->assertSame('edge', $this->feed->getName()); + $this->assertSame('edge', $this->server->getName()); } public function testAFeedWithNoBackendCannotBeRead(): void { - $feed = new Feed(new None('edge')); + $server = new Server(new None('edge')); $this->expectException(Unsupported::class); - $feed->read(); + $server->read(); } public function testTipIsTheNewestEventsId(): void { - $this->assertNull($this->feed->tip()); + $this->assertNull($this->server->tip()); - $this->producer->append('a'); - $last = $this->producer->append('b'); + $this->producer->produce('a'); + $last = $this->producer->produce('b'); - $this->assertSame($last, $this->feed->tip()); + $this->assertSame($last, $this->server->tip()); } public function testReadingFromTheTipSentinelSkipsTheBacklog(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); - $this->assertCount(0, $this->feed->read(Protocol::TIP)); + $this->assertCount(0, $this->server->read(Protocol::TIP)); } public function testServeAppliesTheDefaultsWhenNoParametersArrive(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); - $this->assertCount(2, $this->feed->serve([])); + $this->assertCount(2, $this->server->serve([])); } public function testServeCoercesTheStringValuesARouteHands(): void { - $first = $this->producer->append('a'); - $this->producer->append('b'); - $this->producer->append('c'); + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + $this->producer->produce('c'); - $batch = $this->feed->serve([ + $batch = $this->server->serve([ 'lastEventId' => $first, 'limit' => '1', 'timeout' => '0', @@ -394,31 +394,31 @@ public function testServeCoercesTheStringValuesARouteHands(): void public function testServeTreatsAnEmptyLastEventIdAsAbsent(): void { - $this->producer->append('a'); + $this->producer->produce('a'); - $this->assertCount(1, $this->feed->serve(['lastEventId' => ''])); + $this->assertCount(1, $this->server->serve(['lastEventId' => ''])); } public function testServeRejectsALastEventIdThatIsNotAPosition(): void { $this->expectException(Invalid::class); - $this->feed->serve(['lastEventId' => 'not-a-position']); + $this->server->serve(['lastEventId' => 'not-a-position']); } public function testServeLetsTheTipSentinelThrough(): void { - $this->producer->append('a'); + $this->producer->produce('a'); - $this->assertCount(0, $this->feed->serve(['lastEventId' => Protocol::TIP])); + $this->assertCount(0, $this->server->serve(['lastEventId' => Protocol::TIP])); } public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void { - $this->producer->append('a'); - $this->producer->append('b'); + $this->producer->produce('a'); + $this->producer->produce('b'); - $this->assertCount(2, $this->feed->serve(['limit' => 'lots', 'timeout' => 'soon'])); + $this->assertCount(2, $this->server->serve(['limit' => 'lots', 'timeout' => 'soon'])); } /** @@ -431,10 +431,10 @@ public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void public function testAnOversizedLimitStillYieldsAnHonestCacheControl(): void { foreach (\range(1, Protocol::MAX_BATCH) as $i) { - $this->producer->append('event-' . $i); + $this->producer->produce('event-' . $i); } - $batch = $this->feed->serve(['limit' => '5000']); + $batch = $this->server->serve(['limit' => '5000']); $this->assertCount(Protocol::MAX_BATCH, $batch); $this->assertSame('public, max-age=31536000', $batch->cacheControl(public: true)); @@ -442,14 +442,14 @@ public function testAnOversizedLimitStillYieldsAnHonestCacheControl(): void public function testAShortBatchIsNotCacheable(): void { - $this->producer->append('a'); + $this->producer->produce('a'); - $this->assertSame('no-store', $this->feed->serve([])->cacheControl()); + $this->assertSame('no-store', $this->server->serve([])->cacheControl()); } public function testAnEmptyBatchIsNotCacheable(): void { - $this->assertSame('no-store', $this->feed->serve([])->cacheControl()); + $this->assertSame('no-store', $this->server->serve([])->cacheControl()); } public function testRejectsAnEmptyFeedName(): void @@ -461,14 +461,14 @@ public function testRejectsAnEmptyFeedName(): void public function testAcceptsTheSmallestUsefulRetentionCap(): void { - $journal = new Memory('edge', maxSize: 1); - $producer = new Producer($journal, 'urn:appwrite:cloud:fra'); - $feed = new Feed($journal); + $store = new Memory('edge', maxSize: 1); + $producer = new Producer($store, 'urn:appwrite:cloud:fra'); + $server = new Server($store); - $producer->append('a'); - $producer->append('b'); + $producer->produce('a'); + $producer->produce('b'); - $events = self::events($feed->read()); + $events = self::events($server->read()); $this->assertCount(1, $events); $this->assertSame('b', $events[0]->type); diff --git a/tests/Feed/Unit/StoreCacheTest.php b/tests/Feed/Unit/StoreCacheTest.php new file mode 100644 index 0000000..2373731 --- /dev/null +++ b/tests/Feed/Unit/StoreCacheTest.php @@ -0,0 +1,170 @@ +cache = new UtopiaCache(new CacheMemory()); + $this->store = new Cache($this->cache, 'edge'); + $this->producer = new Producer($this->store, 'urn:test'); + } + + public function testRoundTripsAnEventThroughTheServer(): void + { + $id = $this->producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + + $events = \array_values(\iterator_to_array((new Server($this->store))->read())); + + $this->assertCount(1, $events); + $this->assertSame($id, $events[0]->id); + $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); + $this->assertSame('example.com', $events[0]->subject); + $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); + } + + public function testEventsComeBackOldestFirst(): void + { + foreach (['a', 'b', 'c'] as $type) { + $this->producer->produce($type); + } + + $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $this->store->read(null, 10))); + } + + public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void + { + $ids = []; + for ($i = 0; $i < 50; $i++) { + $ids[] = $this->producer->produce('test'); + } + + $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); + + for ($i = 1; $i < \count($ids); $i++) { + $this->assertGreaterThan(Id::decode($ids[$i - 1]), Id::decode($ids[$i]), 'Positions must increase'); + } + } + + public function testReadsStrictlyAfterTheGivenPosition(): void + { + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + + $events = $this->store->read($first, 10); + + $this->assertCount(1, $events); + $this->assertSame('b', $events[0]->type); + $this->assertCount(0, $this->store->read($events[0]->id, 10)); + } + + public function testHonoursTheLimit(): void + { + foreach (\range(1, 10) as $i) { + $this->producer->produce('test'); + } + + $this->assertCount(3, $this->store->read(null, 3)); + } + + public function testRetentionIsBoundedAndTrimsTheOldest(): void + { + $store = new Cache($this->cache, 'small', maxSize: 3); + $producer = new Producer($store, 'urn:test'); + + foreach (['a', 'b', 'c', 'd', 'e'] as $type) { + $producer->produce($type); + } + + $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 10))); + } + + public function testTipIsTheNewestEventsId(): void + { + $this->assertNull($this->store->tip(), 'An empty feed has no tip'); + + $this->producer->produce('a'); + $last = $this->producer->produce('b'); + + $this->assertSame($last, $this->store->tip()); + } + + /** + * The feed lives in the cache, not in the store object: a second store + * over the same cache β€” another request handling the same feed β€” reads + * what the first appended. + */ + public function testTheFeedSurvivesTheStoreThatWroteIt(): void + { + $id = $this->producer->produce('a'); + + $events = (new Cache($this->cache, 'edge'))->read(null, 10); + + $this->assertCount(1, $events); + $this->assertSame($id, $events[0]->id); + } + + /** + * A cache is allowed to forget, and it may also hold a foreign value under + * the feed's key. Both read as an empty feed β€” a replay, never a fault. + */ + public function testAForeignValueUnderTheFeedsKeyReadsAsEmpty(): void + { + $this->cache->save('feed:edge', ['not' => 'a feed']); + + $this->assertCount(0, $this->store->read(null, 10)); + $this->assertNull($this->store->tip()); + } + + public function testExtensionAttributesSurviveTheTrip(): void + { + $this->producer->publish(new CloudEvent( + id: '', + type: 'test', + source: '', + extensions: ['traceparent' => '00-abc-def-01'], + )); + + $this->assertSame('00-abc-def-01', $this->store->read(null, 10)[0]->extensions['traceparent']); + } + + public function testAConsumerDrainsACacheFedFeed(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $seen = []; + $consumer = new Consumer($this->store, new MemoryCursor(), 'invalidator'); + + $this->assertSame(2, $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + })); + $this->assertSame(['a', 'b'], $seen); + } +} diff --git a/tests/Feed/Unit/Support/FakeClient.php b/tests/Feed/Unit/Support/FakeClient.php index e2ce9f1..c4ba4d3 100644 --- a/tests/Feed/Unit/Support/FakeClient.php +++ b/tests/Feed/Unit/Support/FakeClient.php @@ -16,7 +16,7 @@ * * The `with*()` methods clone the way the real client does, rather than * mutating and returning `$this`. That matters: a test asserting that a long - * poll was given a longer deadline has to be able to fail if the journal + * poll was given a longer deadline has to be able to fail if the remote feed * configured a clone and then sent through the original. */ abstract class FakeClient implements ClientAdapter diff --git a/tests/Feed/Unit/Support/FakeTransport.php b/tests/Feed/Unit/Support/FakeTransport.php index 2d5b50b..6b40538 100644 --- a/tests/Feed/Unit/Support/FakeTransport.php +++ b/tests/Feed/Unit/Support/FakeTransport.php @@ -13,7 +13,7 @@ use Utopia\Psr7\Stream; /** - * A client that answers from a script, for driving the HTTP feed journal + * A client that answers from a script, for driving a remote feed * through responses a real producer would be awkward to provoke. */ class FakeTransport extends FakeClient diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php index 92cf057..7fec62e 100644 --- a/tests/Feed/Unit/Support/FeedServer.php +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -6,8 +6,8 @@ use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; -use Utopia\Feed\Feed; use Utopia\Feed\Protocol; +use Utopia\Feed\Server; use Utopia\Psr7\Header; use Utopia\Psr7\Response; use Utopia\Psr7\Stream; @@ -15,14 +15,14 @@ /** * A producer's feed endpoint, as a client. * - * Serves a real {@see Feed} through {@see Feed::serve()} exactly as an HTTP - * route would, so a consumer reading it exercises the whole contract β€” + * Serves a real {@see Server} through {@see Server::serve()} exactly as an + * HTTP route would, so a consumer reading it exercises the whole contract β€” * parameters, body and caching β€” rather than a fixture written to match the * consumer. */ class FeedServer extends FakeClient { - public function __construct(private readonly Feed $feed, Recorder $recorder = new Recorder()) + public function __construct(private readonly Server $server, Recorder $recorder = new Recorder()) { parent::__construct($recorder); } @@ -32,7 +32,7 @@ protected function respond(RequestInterface $request): ResponseInterface $query = []; \parse_str($request->getUri()->getQuery(), $query); - $batch = $this->feed->serve($query); + $batch = $this->server->serve($query); $body = (string) \json_encode($batch->toArray()); diff --git a/tests/Feed/Unit/Support/MidPollJournal.php b/tests/Feed/Unit/Support/MidPollStore.php similarity index 82% rename from tests/Feed/Unit/Support/MidPollJournal.php rename to tests/Feed/Unit/Support/MidPollStore.php index d1d24b8..0cb2818 100644 --- a/tests/Feed/Unit/Support/MidPollJournal.php +++ b/tests/Feed/Unit/Support/MidPollStore.php @@ -5,13 +5,13 @@ namespace Utopia\Tests\Unit\Support; use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Journal\Memory; +use Utopia\Feed\Store\Memory; /** - * A journal where another process appends while a poll is waiting: the event + * A store where another process appends while a poll is waiting: the event * lands just before the poll loop's second read, mid-wait. */ -class MidPollJournal extends Memory +class MidPollStore extends Memory { private int $reads = 0; From 5f7c0be0b6b86c6684b0a609cacec320c7d9a3d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 15:50:53 +0200 Subject: [PATCH 31/68] Manual simplification --- CHANGELOG.md | 2 +- Dockerfile | 7 ----- README.md | 10 +++---- docker-compose.yml | 3 --- src/Feed/Appendable.php | 8 +----- src/Feed/Batch.php | 20 +------------- src/Feed/Consumer.php | 46 +++++++++----------------------- src/Feed/Cursor.php | 2 -- src/Feed/Cursor/None.php | 12 +-------- src/Feed/Producer.php | 15 +---------- src/Feed/Readable.php | 14 ---------- src/Feed/Remote.php | 17 ------------ src/Feed/Server.php | 13 +-------- src/Feed/Start.php | 24 ----------------- src/Feed/Store.php | 31 --------------------- src/Feed/Store/Cache.php | 15 ----------- src/Feed/Store/Pool.php | 3 --- tests/Feed/Unit/ConsumerTest.php | 13 +++++---- tests/Feed/Unit/RemoteTest.php | 3 +-- 19 files changed, 30 insertions(+), 228 deletions(-) delete mode 100644 src/Feed/Start.php diff --git a/CHANGELOG.md b/CHANGELOG.md index c803926..437c624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.1.0 - **Breaking (the three classes):** the library is now three main classes. `Producer` writes events to a feed with `produce()` (formerly `append()`; diff --git a/Dockerfile b/Dockerfile index 7b656bc..501a5e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,3 @@ -# One image for every PHP version the library is tested against. Adding a -# version is a single entry in the `php-versions` matrix in -# .github/workflows/tests.yml β€” there is nothing to add here. -# -# PHP_VERSION=8.6 docker compose build ARG PHP_VERSION=8.5 FROM composer:2.7 AS vendor @@ -22,8 +17,6 @@ WORKDIR /code COPY --from=vendor /src/vendor /code/vendor -# Composer itself, for the `test`, `check` and `lint` scripts. The base image -# ships PHP but not composer. COPY --from=composer:2.7 /usr/bin/composer /usr/bin/composer COPY ./composer.json /code/composer.json diff --git a/README.md b/README.md index d465aa5..2414924 100644 --- a/README.md +++ b/README.md @@ -155,12 +155,10 @@ consumer that must not act on the backlog β€” a notifier announcing events as they happen β€” opts into starting at the tip: ```php -use Utopia\Feed\Start; - -$consumer = new Consumer($client, $cursor, name: 'notifier', feed: 'edge', timeout: 20_000, start: Start::Tip); +$consumer = new Consumer($client, $cursor, name: 'notifier', feed: 'edge', timeout: 20_000, start: Consumer::START_TIP); ``` -A stored position always wins; `Start::Tip` applies only on the first run or +A stored position always wins; `Consumer::START_TIP` applies only on the first run or after `reset()` (which then means "forget everything, resume from now"). Give a tip consumer a `timeout`: the producer anchors "now" as each poll arrives, so new events land inside the held request rather than in the gap between @@ -170,7 +168,7 @@ fails with a 4xx `Transport` error rather than silently replaying the backlog. ### Moving the position by hand - `reset()` β€” forget the position; the next run starts from the oldest - retained event (or the tip, for a `Start::Tip` consumer). + retained event (or the tip, for a `Consumer::START_TIP` consumer). - `seek($eventId)` β€” treat `$eventId` as the last event handled; the next run starts strictly *after* it. Persisted immediately; a store failure surfaces as `Transport`. The id must be well formed but need not still exist in the @@ -211,7 +209,7 @@ intentionally: a feed is ordered, and stepping over a failure would apply later events on top of state that was never updated. **No position means the oldest retained event, never the tip** (unless the -consumer opted into `Start::Tip`), so a consumer deployed after the producer +consumer opted into `Consumer::START_TIP`), so a consumer deployed after the producer drains the backlog instead of dropping it. **One process per consumer name.** Two processes sharing a name share one diff --git a/docker-compose.yml b/docker-compose.yml index aadde99..7c33372 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,5 @@ name: feed -# Pick the PHP version with an environment variable rather than a per-version -# file, so testing against a new release is `PHP_VERSION=8.6 docker compose up` -# and adding it to CI is one line in .github/workflows/tests.yml. services: tests: container_name: feed-tests diff --git a/src/Feed/Appendable.php b/src/Feed/Appendable.php index ad8d369..3a33bf4 100644 --- a/src/Feed/Appendable.php +++ b/src/Feed/Appendable.php @@ -6,16 +6,10 @@ use Utopia\CloudEvents\CloudEvent; -// Server interface: a store that owns its events, so it can be appended to. -// Remote does not implement it β€” a consumer cannot write to someone else's feed. +// Server interface interface Appendable { /** - * Append an event and return the id the backend assigned it. - * - * Any id already on $event is ignored: positions are the backend's to - * allocate, since only it can keep them ordered. - * * @throws Exception When the event cannot be appended. */ public function append(CloudEvent $event): string; diff --git a/src/Feed/Batch.php b/src/Feed/Batch.php index 5ef2905..e0c95a0 100644 --- a/src/Feed/Batch.php +++ b/src/Feed/Batch.php @@ -7,14 +7,7 @@ use Utopia\CloudEvents\CloudEvent; /** - * One read of a feed: the events, paired with the limit the batch was - * actually built with. - * - * The pairing is the point. Whether a batch may be cached forever depends on - * whether it came back full, so the caching rule needs the limit the read was - * clamped to β€” not the one the request asked for. Holding both in one value - * makes a mismatched pair impossible to express. - * + * One read of a feed * @implements \IteratorAggregate */ final class Batch implements \Countable, \IteratorAggregate @@ -45,10 +38,6 @@ public function isEmpty(): bool return $this->events === []; } - /** - * The id of the last event, or null on an empty batch β€” the position a - * caller relaying the feed by hand tracks. - */ public function lastId(): ?string { $count = \count($this->events); @@ -56,19 +45,12 @@ public function lastId(): ?string return $count === 0 ? null : $this->events[$count - 1]->id; } - /** - * The Cache-Control header for the response carrying this batch: a full - * batch is settled history and immutable, anything short is the live end - * of the feed and must not be cached. Caching is private unless $public. - */ public function cacheControl(bool $public = false): string { return Protocol::cacheControl(\count($this->events), $this->limit, $public); } /** - * The batch as it goes on the wire: a plain array of CloudEvents. - * * @return list> */ public function toArray(): array diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 07aa118..6819227 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -6,11 +6,14 @@ use Utopia\Client\Adapter; -// Client class: the pull loop β€” reads what it has not seen and records how far it got. class Consumer { public const int BATCH = 100; + /** Where a consumer with no stored position begins. */ + public const string START_OLDEST = 'oldest'; + public const string START_TIP = 'tip'; + protected readonly Readable $feed; private ?string $position = null; @@ -18,15 +21,7 @@ class Consumer private bool $restored = false; /** - * @param Adapter|Readable $source The feed to pull from β€” an HTTP client - * for another service's feed (its endpoint set on the client with - * withBaseUri()), or a local store for a feed this service owns. - * @param string $name This consumer's name β€” what keys its position. - * @param string $feed The feed's name. Required when $source is a client, - * where nothing else carries it; a local source already names its - * feed, so leave it out. - * - * @throws Exception\Invalid When a name is missing, or $feed contradicts the source. + * @throws Exception\Invalid When a name is missing, $feed contradicts the source, or $start is not a START_* constant. */ public function __construct( Adapter|Readable $source, @@ -35,12 +30,16 @@ public function __construct( string $feed = '', protected readonly int $batch = self::BATCH, protected readonly int $timeout = 0, - protected readonly Start $start = Start::Oldest, + protected readonly string $start = self::START_OLDEST, ) { if ($name === '') { throw new Exception\Invalid('Feed consumer requires a name'); } + if ($start !== self::START_OLDEST && $start !== self::START_TIP) { + throw new Exception\Invalid("Feed consumer start must be Consumer::START_OLDEST or Consumer::START_TIP, got {$start}"); + } + if ($source instanceof Adapter) { $this->feed = new Remote($source, $feed); } else { @@ -97,16 +96,9 @@ public function consume(callable $handler): int return $handled; } - /** - * Where a poll starts when no position is stored: the oldest retained - * event, or β€” for Start::Tip β€” the tip sentinel, which the store (or - * the remote producer, inside the same request) resolves to "now". Once - * events are handled and the cursor saves, the sentinel never appears - * again; reset() forgets the position, so the next poll anchors anew. - */ private function origin(): ?string { - return $this->start === Start::Tip ? Protocol::TIP : null; + return $this->start === self::START_TIP ? Protocol::TIP : null; } public function position(): ?string @@ -120,20 +112,8 @@ public function position(): ?string } /** - * Set the position explicitly: treat $eventId as the last event handled, - * so the next consume() starts strictly *after* it. - * - * The position is persisted immediately via Cursor::save() and mirrored - * in memory. $eventId must be a well-formed feed position, but does not - * need to currently exist in the feed β€” seeking to an id older than - * retention or newer than the tip is legal and simply positions relative - * to it, which is what makes seeking to a poison event's own id the way - * to step past it deliberately. - * - * @throws Exception\Invalid When $eventId is not a feed position (the tip sentinel included). - * @throws Exception When the cursor store cannot be written β€” a seek that - * did not persist must not look like one that did, so the failure - * is never swallowed and the in-memory position stays put. + * @throws Exception\Invalid When $eventId is not a feed position (the tip sentinel included) + * @throws Exception When the cursor store cannot be written */ public function seek(string $eventId): void { diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index d1c8250..c81b115 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -6,8 +6,6 @@ use Utopia\Feed\Exception\Invalid; -// Client class: where a consumer keeps its position β€” the server stores nothing per consumer. -// Cursor\Cache for a consumer reading a remote feed, Cursor\Redis or Pool for one inside the producer. abstract class Cursor { /** diff --git a/src/Feed/Cursor/None.php b/src/Feed/Cursor/None.php index dec0d5d..d5edb49 100644 --- a/src/Feed/Cursor/None.php +++ b/src/Feed/Cursor/None.php @@ -6,21 +6,11 @@ use Utopia\Feed\Cursor; -/** - * No cursor store configured. Nothing is remembered. - * - * A no-op, unlike {@see \Utopia\Feed\Store\None}, because the two losses are - * not comparable: an append that goes nowhere loses events, while a position - * that goes nowhere only costs a replay. A consumer using this still advances - * in memory for the life of the process, and starts again from the oldest - * retained event whenever it restarts. - */ class None extends Cursor { public function load(string $feed, string $consumer): ?string { - // Validated even though nothing is stored, so a name that a real store - // would reject does not start working the moment this stands in for one. + // Validated even though nothing is stored $this->key($feed, $consumer); return null; diff --git a/src/Feed/Producer.php b/src/Feed/Producer.php index de9cb48..0becf38 100644 --- a/src/Feed/Producer.php +++ b/src/Feed/Producer.php @@ -6,15 +6,9 @@ use Utopia\CloudEvents\CloudEvent; -// Server class: writes events to a feed this service owns. -// The store must be a Store and Appendable, so a Remote cannot reach this at all. class Producer { /** - * @param Store&Appendable $store Where the events live. - * @param string $source Who is producing them, as a URI reference - * (`urn:appwrite:cloud:fra`). Stamped onto every event, so a consumer - * merging feeds from several producers can tell them apart. * @throws Exception\Invalid When $source is empty. */ public function __construct( @@ -32,8 +26,6 @@ public function getName(): string } /** - * Produce an event and return its position in the feed. - * * @throws Exception\Invalid When $type is empty or $data cannot be encoded. * @throws Exception When the backend rejects the event. */ @@ -49,9 +41,6 @@ public function produce(string $type, mixed $data = [], string $subject = ''): s } /** - * Produce a prepared event, stamping it with this producer's source and, - * unless it already has one, the current time. - * * @throws Exception\Invalid When the event has no type or cannot be encoded. * @throws Exception When the backend rejects the event. */ @@ -61,9 +50,7 @@ public function publish(CloudEvent $event): string throw new Exception\Invalid('Feed event type is required'); } - // Rebuilt attribute by attribute β€” extensions included β€” so anything - // this library does not model, a dataschema or a traceparent, survives - // untouched. + // Rebuilt attribute by attribute to have untouched copy $event = new CloudEvent( type: $event->type, source: $this->source, diff --git a/src/Feed/Readable.php b/src/Feed/Readable.php index 105171b..e32a100 100644 --- a/src/Feed/Readable.php +++ b/src/Feed/Readable.php @@ -6,37 +6,23 @@ use Utopia\CloudEvents\CloudEvent; -// The contract of "something a feed's events can be read from" β€” the -// counterpart of Appendable. Store implements it on the server; Remote is -// the client's read-only view of another service's feed. interface Readable { public function getName(): string; /** - * Events strictly after $lastEventId, oldest first, at most $limit of - * them. A null position reads from the oldest retained event; the tip - * sentinel (Protocol::TIP) reads from the tip. - * * @return list - * * @throws Exception When the events cannot be read. */ public function read(?string $lastEventId, int $limit): array; /** - * Like read(), but waits up to $timeout milliseconds for events to land - * before answering empty. - * * @return list - * * @throws Exception When the events cannot be read. */ public function poll(?string $lastEventId, int $limit, int $timeout): array; /** - * The id of the newest event, or null when the feed is empty. - * * @throws Exception When the tip cannot be read, or only its owner can resolve it. */ public function tip(): ?string; diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index a38ec78..c086013 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -14,11 +14,6 @@ use Utopia\Psr7\Method; use Utopia\Psr7\Request\Factory as RequestFactory; -// Client class: another service's feed, read over the wire. -// The endpoint lives on the client β€” set it with withBaseUri() β€” and the feed -// name becomes the request path, resolved against it. -// Deliberately Readable and not Appendable β€” events are appended by whoever -// owns the feed, so producing into a remote one is a type error, not a request. class Remote implements Readable { private readonly RequestFactory $requests; @@ -39,11 +34,6 @@ public function getName(): string return $this->name; } - /** - * Never called on the consumer path: the tip sentinel is passed through - * as `lastEventId=$` and the producer resolves it inside the same - * request, so there is no separate tip round trip to race. - */ public function tip(): ?string { throw new Unsupported("The {$this->name} feed is remote; its producer resolves the tip"); @@ -64,10 +54,6 @@ public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BAT */ private function fetch(?string $lastEventId, int $limit, int $timeout): array { - // The path is relative β€” the feed's name β€” and the client resolves it - // against its base URI, which is where the endpoint is configured. - // The Content-Type of the response is deliberately not checked: many - // servers answer application/json, and the body shape is what matters. $request = $this->requests->query( Method::GET, \rawurlencode($this->name), @@ -75,9 +61,6 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array [Header::ACCEPT => Protocol::MEDIA_TYPE], ); - // A long poll needs a deadline past the one it asked the producer for, - // or the client cancels a correct wait a hair early and every quiet - // tick surfaces as a failure. A plain read keeps the caller's own. $client = $timeout > 0 ? $this->client->withTimeout(($timeout + Protocol::TIMEOUT_MARGIN) / 1000) : $this->client; diff --git a/src/Feed/Server.php b/src/Feed/Server.php index 30e40d5..9a754a1 100644 --- a/src/Feed/Server.php +++ b/src/Feed/Server.php @@ -4,8 +4,6 @@ namespace Utopia\Feed; -// Server class: the read view over the store a service appends to β€” it -// reads, long-polls, and serves the feed over HTTP with serve(). class Server { public function __construct(protected readonly Readable $store) @@ -18,9 +16,6 @@ public function getName(): string } /** - * The id of the newest event, or null on an empty feed. Local stores - * only β€” a remote feed's producer resolves the tip sentinel instead. - * * @throws Exception */ public function tip(): ?string @@ -46,14 +41,8 @@ public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BAT } /** - * Serve one HTTP feed request: the route's raw query-parameter array in, - * the batch out. Extracts `lastEventId`, `limit` and `timeout`, coerces - * their string values, applies the defaults and clamps to the protocol - * limits, so the route never touches the wire vocabulary itself. - * * @param array $query The request's query parameters, string values included. - * - * @throws Exception\Invalid When `lastEventId` is present but is neither a feed position nor the tip sentinel β€” a 400-worthy input. + * @throws Exception\Invalid When `lastEventId` is present but is neither a feed position nor the tip sentinel */ public function serve(array $query): Batch { diff --git a/src/Feed/Start.php b/src/Feed/Start.php deleted file mode 100644 index 02aa23a..0000000 --- a/src/Feed/Start.php +++ /dev/null @@ -1,24 +0,0 @@ -tip() : $lastEventId; } /** - * Wait for events, re-reading on an interval until some land or the - * deadline passes. (Remote does not share this loop: there the producer - * does the waiting, so a poll is one held request.) - * * @return list */ public function poll(?string $lastEventId, int $limit, int $timeout): array { - // The sentinel is pinned once, before the wait: re-resolving on every - // read would move the tip past events landing mid-poll, and they - // would never be delivered. $lastEventId = $this->resolve($lastEventId); $deadline = \microtime(true) + $timeout / 1000; @@ -80,8 +57,6 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array return $events; } - // Never sleep past the deadline: the timeout is honoured to - // within scheduler precision, not to within one interval. \usleep((int) \min($this->pollInterval * 1000, \ceil($remaining * 1_000_000))); } } @@ -92,9 +67,6 @@ protected static function encode(CloudEvent $event): array return [ 'type' => $event->type, 'source' => $event->source, - // CloudEvents models an absent subject, dataschema and time as - // null, and a backend field cannot hold one, so they are - // normalized here and read back as absent in decode(). 'subject' => $event->subject ?? '', 'dataschema' => $event->dataschema ?? '', 'time' => $event->time ?? '', @@ -127,9 +99,6 @@ protected static function decode(string $id, array $fields): CloudEvent $event += \is_array($extensions) ? $extensions : []; try { - // The docblock wants array, but a digit-only - // extension name β€” legal per the spec β€” is an integer key in PHP. - // @phpstan-ignore argument.type return CloudEvent::fromArray($event); } catch (\InvalidArgumentException $error) { throw new Invalid("Feed entry {$id} could not be read as an event: {$error->getMessage()}", previous: $error); diff --git a/src/Feed/Store/Cache.php b/src/Feed/Store/Cache.php index 0b8a516..20d1f7e 100644 --- a/src/Feed/Store/Cache.php +++ b/src/Feed/Store/Cache.php @@ -11,17 +11,6 @@ use Utopia\Feed\Appendable; use Utopia\Feed\Store; -/** - * A feed on a Utopia cache β€” for a producer whose service already carries a - * cache and does not want another backend for its feed. - * - * The whole feed lives under one key, rewritten on every append, so appends - * are last-writer-wins rather than atomic: run one producing process, or - * accept that concurrent appends can drop each other. The store may also be - * evicted as a unit β€” a consumer then restarts from an empty feed, which - * costs a replay of nothing, not a gap it can detect. Retention still trims - * to maxSize; ttl bounds how long an idle feed outlives its last append. - */ class Cache extends Store implements Appendable { public const int TTL = 30 * 24 * 60 * 60; // 30 days @@ -92,10 +81,6 @@ public function read(?string $lastEventId, int $limit): array } /** - * The stored feed, oldest first. Anything that is not the shape append() - * writes β€” a missing key, a foreign value under it β€” reads as an empty - * feed rather than a fault: a cache is allowed to forget. - * * @return list}> */ private function load(): array diff --git a/src/Feed/Store/Pool.php b/src/Feed/Store/Pool.php index 032432f..02f894f 100644 --- a/src/Feed/Store/Pool.php +++ b/src/Feed/Store/Pool.php @@ -23,9 +23,6 @@ public function __construct( parent::__construct($name, $pollInterval); } - // The interval only matters in this class's own inherited poll() loop β€” - // the inner store lives for a single read β€” but it is passed through so - // a future change to the inner store cannot silently drop it. private function inner(\Redis|\RedisCluster $redis): Redis { return new Redis($redis, $this->name, $this->maxSize, $this->pollInterval); diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 781a91b..190a308 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -14,7 +14,6 @@ use Utopia\Feed\Exception\Transport; use Utopia\Feed\Producer; use Utopia\Feed\Protocol; -use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FailingCursor; use Utopia\Tests\Unit\Support\FakeTransport; use Utopia\Tests\Unit\Support\MidPollStore; @@ -331,7 +330,7 @@ public function testTipStartDoesNotAnnounceTheBacklog(): void $this->producer->produce('old-1'); $this->producer->produce('old-2'); - $consumer = new Consumer($this->store, $this->cursor, 'notifier', start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'notifier', start: Consumer::START_TIP); $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); $this->assertNull($this->cursor->load('edge', 'notifier'), 'Skipping the backlog is not progress to commit'); @@ -347,7 +346,7 @@ public function testTipStartDeliversWhatLandsMidPoll(): void (new Producer($store, 'urn:test'))->produce('old'); $cursor = new MemoryCursor(); - $consumer = new Consumer($store, $cursor, 'notifier', timeout: 5_000, start: Start::Tip); + $consumer = new Consumer($store, $cursor, 'notifier', timeout: 5_000, start: Consumer::START_TIP); $seen = []; $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { @@ -366,7 +365,7 @@ public function testAStoredCursorBeatsTipStart(): void $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Consumer::START_TIP); $this->assertSame(['b'], $this->drain($consumer), 'A restart must not skip the gap'); } @@ -377,7 +376,7 @@ public function testResetWithTipStartResumesFromNow(): void $this->producer->produce('b'); $this->cursor->save('edge', 'invalidator', $first); - $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Consumer::START_TIP); $this->assertSame(['b'], $this->drain($consumer), 'The stored position still wins before the reset'); @@ -388,7 +387,7 @@ public function testResetWithTipStartResumesFromNow(): void public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void { - $consumer = new Consumer($this->store, $this->cursor, 'notifier', timeout: 600, start: Start::Tip); + $consumer = new Consumer($this->store, $this->cursor, 'notifier', timeout: 600, start: Consumer::START_TIP); $started = \microtime(true); @@ -398,7 +397,7 @@ public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void public function testTipStartOnAnEmptyFeedDeliversWhatLandsMidWait(): void { - $consumer = new Consumer(new MidPollStore('edge'), $this->cursor, 'notifier', timeout: 5_000, start: Start::Tip); + $consumer = new Consumer(new MidPollStore('edge'), $this->cursor, 'notifier', timeout: 5_000, start: Consumer::START_TIP); $seen = []; $consumer->consume(function (CloudEvent $event) use (&$seen): void { diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index 9e65d2a..08da9de 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -17,7 +17,6 @@ use Utopia\Feed\Protocol; use Utopia\Feed\Remote; use Utopia\Feed\Server; -use Utopia\Feed\Start; use Utopia\Tests\Unit\Support\FakeTransport; use Utopia\Tests\Unit\Support\FeedServer; use Utopia\Tests\Unit\Support\MidPollStore; @@ -322,7 +321,7 @@ public function testTipStartWorksOverHttp(): void $producer->produce('old'); $endpoint = new FeedServer(new Server($store)); - $consumer = new Consumer($endpoint, new MemoryCursor(), 'notifier', feed: 'edge', timeout: 5_000, start: Start::Tip); + $consumer = new Consumer($endpoint, new MemoryCursor(), 'notifier', feed: 'edge', timeout: 5_000, start: Consumer::START_TIP); $seen = []; $handler = function (CloudEvent $event) use (&$seen): void { From 1c7afbbae28cab460e3dc88e2ecea3ca1a535c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 16:14:11 +0200 Subject: [PATCH 32/68] Simplify classes --- README.md | 9 +- src/Feed/Batch.php | 25 ++- src/Feed/Consumer.php | 6 +- src/Feed/Protocol.php | 206 --------------------- src/Feed/Readable.php | 14 ++ src/Feed/Remote.php | 150 +++++++++++++++- src/Feed/Server.php | 25 +-- src/Feed/Store.php | 2 +- tests/Feed/Unit/BatchTest.php | 16 +- tests/Feed/Unit/ConsumerTest.php | 6 +- tests/Feed/Unit/ProtocolTest.php | 239 ------------------------- tests/Feed/Unit/RemoteTest.php | 178 ++++++++++++++++-- tests/Feed/Unit/RoundTripTest.php | 7 +- tests/Feed/Unit/ServerTest.php | 12 +- tests/Feed/Unit/Support/FeedServer.php | 4 +- 15 files changed, 399 insertions(+), 500 deletions(-) delete mode 100644 src/Feed/Protocol.php delete mode 100644 tests/Feed/Unit/ProtocolTest.php diff --git a/README.md b/README.md index 2414924..29dd061 100644 --- a/README.md +++ b/README.md @@ -310,10 +310,11 @@ connection for the whole wait, which is exactly what `Store\Pool`'s borrow-per-read strategy exists to avoid. Tune the trade-off with `pollInterval`. -**For integrators** building a transport of their own: the wire contract β€” -query parameters, batch encoding, caching rule β€” lives in `Utopia\Feed\Protocol`, -and `Utopia\Feed\Remote` is the client-side `Readable` the consumer builds -over its client. Services never need either. +**For integrators** building a transport of their own: the shared contract β€” +the tip sentinel and the batch/timeout limits β€” lives on `Utopia\Feed\Readable`; +`Utopia\Feed\Batch` carries the serving side (encoding, media type, caching +rule) and `Utopia\Feed\Remote` is the client-side `Readable` the consumer +builds over its client. Services never need any of them directly. ## Tests diff --git a/src/Feed/Batch.php b/src/Feed/Batch.php index e0c95a0..ff64d19 100644 --- a/src/Feed/Batch.php +++ b/src/Feed/Batch.php @@ -12,6 +12,12 @@ */ final class Batch implements \Countable, \IteratorAggregate { + /** The batch media type on the wire β€” what a feed response's Content-Type carries. */ + public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; + + private const string CACHE_IMMUTABLE = 'max-age=31536000'; + private const string CACHE_NONE = 'no-store'; + /** * @param list $events * @param int $limit The effective limit β€” the clamped value the feed used to build this batch. @@ -45,16 +51,31 @@ public function lastId(): ?string return $count === 0 ? null : $this->events[$count - 1]->id; } + /** + * A full batch is settled history, so it may be cached forever. Anything + * short is the live end of the feed and will grow β€” an empty batch + * included: zero events is a caught-up consumer, and caching that would + * pin the consumer at its position forever. + */ public function cacheControl(bool $public = false): string { - return Protocol::cacheControl(\count($this->events), $this->limit, $public); + $count = \count($this->events); + + if ($count < $this->limit || $count === 0) { + return self::CACHE_NONE; + } + + return ($public ? 'public, ' : 'private, ') . self::CACHE_IMMUTABLE; } /** + * A batch on the wire is a plain array of CloudEvents β€” no envelope. An + * empty feed serializes to `[]`, which the spec reads as "you are caught up". + * * @return list> */ public function toArray(): array { - return Protocol::encode($this->events); + return \array_map(static fn (CloudEvent $event): array => $event->toArray(), $this->events); } } diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 6819227..920fe98 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -60,8 +60,8 @@ public function consume(callable $handler): int { $events = $this->feed->poll( $this->position() ?? $this->origin(), - \max(1, \min($this->batch, Protocol::MAX_BATCH)), - \max(0, \min($this->timeout, Protocol::MAX_TIMEOUT)), + \max(1, \min($this->batch, Readable::MAX_BATCH)), + \max(0, \min($this->timeout, Readable::MAX_TIMEOUT)), ); if ($events === []) { @@ -98,7 +98,7 @@ public function consume(callable $handler): int private function origin(): ?string { - return $this->start === self::START_TIP ? Protocol::TIP : null; + return $this->start === self::START_TIP ? Readable::TIP : null; } public function position(): ?string diff --git a/src/Feed/Protocol.php b/src/Feed/Protocol.php deleted file mode 100644 index 763a955..0000000 --- a/src/Feed/Protocol.php +++ /dev/null @@ -1,206 +0,0 @@ - - */ - public static function query(?string $lastEventId = null, int $limit = 0, int $timeout = 0): array - { - $query = []; - - if ($lastEventId !== null && $lastEventId !== '') { - $query[self::PARAM_LAST_EVENT_ID] = $lastEventId; - } - - if ($limit > 0) { - $query[self::PARAM_LIMIT] = $limit; - } - - if ($timeout > 0) { - $query[self::PARAM_TIMEOUT] = $timeout; - } - - return $query; - } - - /** - * A batch on the wire is a plain array of CloudEvents β€” no envelope. An - * empty feed serializes to `[]`, which the spec reads as "you are caught up". - * - * @param list $events - * @return list> - */ - public static function encode(array $events): array - { - return \array_map(static fn (CloudEvent $event): array => $event->toArray(), $events); - } - - /** - * Read a batch off the wire. - * - * An entry that cannot be read ends the batch there rather than failing the - * whole response: the events before it are handled and the position - * advances past them, leaving the broken entry at the head of the next - * batch, where it stops the feed loudly. With no usable prefix there is - * nothing to advance to, so that case throws. - * - * @return list - * - * @throws Invalid When the payload is not a batch, or its first entry cannot be read. - */ - public static function decode(mixed $payload): array - { - if (!\is_array($payload)) { - throw new Invalid('Expected a feed batch, got ' . \get_debug_type($payload)); - } - - if (!\array_is_list($payload)) { - throw new Invalid('Expected a feed batch as a plain array of events'); - } - - $events = []; - - /** @var mixed $event */ - foreach ($payload as $event) { - try { - if (!\is_array($event)) { - throw new Invalid('Feed batch contains an entry that is not an event'); - } - - $events[] = self::event($event); - } catch (Invalid | \InvalidArgumentException $error) { - if ($events === []) { - throw $error instanceof Invalid - ? $error - : new Invalid('Feed batch contains an event that cannot be read: ' . $error->getMessage(), previous: $error); - } - - break; - } - } - - return $events; - } - - public static function cacheControl(int $count, int $limit, bool $public = false): string - { - if ($count < $limit || $count === 0) { - return self::CACHE_NONE; - } - - return ($public ? 'public, ' : 'private, ') . self::CACHE_IMMUTABLE; - } - - /** - * Read one event off the wire. - * - * Mapped by hand rather than through CloudEvent::fromArray(), which - * rejects a specversion it does not know. A feed is read by consumers - * older than its producer by design, so a producer that moved the spec - * version forward, or attached an attribute this library cannot model, - * must not stop one that predates it β€” what cannot be carried is dropped, - * not fatal. - * - * @param array $raw - */ - private static function event(array $raw): CloudEvent - { - foreach (['specversion', 'id', 'type', 'source'] as $required) { - if (!isset($raw[$required]) || !\is_string($raw[$required]) || $raw[$required] === '') { - throw new Invalid('Feed event is missing ' . ($required === 'id' ? 'an id' : 'a ' . $required)); - } - } - - $extensions = []; - - /** @var mixed $value */ - foreach ($raw as $name => $value) { - if (\in_array($name, self::ATTRIBUTES, true)) { - continue; - } - - // Only what the CloudEvent constructor accepts as an extension β€” - // anything else would throw and stop the feed. - if (\preg_match('/^[a-z0-9]+$/', (string) $name) === 1 - && (\is_bool($value) || \is_int($value) || \is_string($value))) { - $extensions[$name] = $value; - } - } - - return new CloudEvent( - type: $raw['type'], - source: $raw['source'], - id: $raw['id'], - specversion: $raw['specversion'], - subject: self::optional($raw, 'subject'), - time: self::optional($raw, 'time'), - datacontenttype: self::optional($raw, 'datacontenttype'), - data: $raw['data'] ?? null, - dataschema: self::optional($raw, 'dataschema'), - // The docblock wants array, but a digit-only - // extension name β€” legal per the spec β€” is an integer key in PHP. - // @phpstan-ignore argument.type - extensions: $extensions, - ); - } - - /** - * @param array $raw - */ - private static function optional(array $raw, string $attribute): ?string - { - $value = $raw[$attribute] ?? null; - - return \is_string($value) && $value !== '' ? $value : null; - } -} diff --git a/src/Feed/Readable.php b/src/Feed/Readable.php index e32a100..cda635b 100644 --- a/src/Feed/Readable.php +++ b/src/Feed/Readable.php @@ -8,6 +8,20 @@ interface Readable { + /** + * Extension beyond the spec, like `limit`: a `lastEventId` of `$` means + * "the tip of the feed". The producer resolves it to the newest event at + * the moment the request arrives, so a consumer can ask for only what + * happens from now on without a separate round trip to learn the tip. + */ + public const string TIP = '$'; + + /** The most events one batch may carry β€” producers clamp `limit` to this. */ + public const int MAX_BATCH = 1000; + + /** The longest a long poll may hold a connection, in milliseconds. */ + public const int MAX_TIMEOUT = 30_000; + public function getName(): string; /** diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index c086013..93c8828 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -16,6 +16,35 @@ class Remote implements Readable { + /** The batch media type on the wire β€” sent as `Accept` with every read. */ + public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; + + // The request parameters of https://www.http-feeds.org/. + private const string PARAM_LAST_EVENT_ID = 'lastEventId'; + private const string PARAM_LIMIT = 'limit'; + private const string PARAM_TIMEOUT = 'timeout'; + + /** + * How much longer than the long poll the HTTP client is allowed to wait, + * in milliseconds. Without the margin the client's deadline races the + * producer's, and a poll that correctly waits out its timeout surfaces + * as a failure on every quiet tick. + */ + private const int TIMEOUT_MARGIN = 10_000; + + /** The context attributes this library models; the rest are extensions. */ + private const array ATTRIBUTES = [ + 'specversion', + 'type', + 'source', + 'id', + 'subject', + 'time', + 'datacontenttype', + 'dataschema', + 'data', + ]; + private readonly RequestFactory $requests; public function __construct( @@ -39,12 +68,12 @@ public function tip(): ?string throw new Unsupported("The {$this->name} feed is remote; its producer resolves the tip"); } - public function read(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH): array + public function read(?string $lastEventId = null, int $limit = self::MAX_BATCH): array { return $this->fetch($lastEventId, $limit, 0); } - public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH, int $timeout = 0): array + public function poll(?string $lastEventId = null, int $limit = self::MAX_BATCH, int $timeout = 0): array { return $this->fetch($lastEventId, $limit, $timeout); } @@ -57,12 +86,12 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array $request = $this->requests->query( Method::GET, \rawurlencode($this->name), - Protocol::query($lastEventId, $limit, $timeout), - [Header::ACCEPT => Protocol::MEDIA_TYPE], + self::query($lastEventId, $limit, $timeout), + [Header::ACCEPT => self::MEDIA_TYPE], ); $client = $timeout > 0 - ? $this->client->withTimeout(($timeout + Protocol::TIMEOUT_MARGIN) / 1000) + ? $this->client->withTimeout(($timeout + self::TIMEOUT_MARGIN) / 1000) : $this->client; try { @@ -86,6 +115,115 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array throw new Transport("The {$this->name} feed returned a body that is not JSON: {$error->getMessage()}", previous: $error); } - return Protocol::decode($body); + return self::decode($body); + } + + /** + * @return array + */ + private static function query(?string $lastEventId, int $limit, int $timeout): array + { + $query = []; + + if ($lastEventId !== null && $lastEventId !== '') { + $query[self::PARAM_LAST_EVENT_ID] = $lastEventId; + } + + if ($limit > 0) { + $query[self::PARAM_LIMIT] = $limit; + } + + if ($timeout > 0) { + $query[self::PARAM_TIMEOUT] = $timeout; + } + + return $query; + } + + /** + * @return list + * @throws Invalid When the payload is not a batch, or its first entry cannot be read. + */ + private static function decode(mixed $payload): array + { + if (!\is_array($payload)) { + throw new Invalid('Expected a feed batch, got ' . \get_debug_type($payload)); + } + + if (!\array_is_list($payload)) { + throw new Invalid('Expected a feed batch as a plain array of events'); + } + + $events = []; + + /** @var mixed $event */ + foreach ($payload as $event) { + try { + if (!\is_array($event)) { + throw new Invalid('Feed batch contains an entry that is not an event'); + } + + $events[] = self::event($event); + } catch (Invalid | \InvalidArgumentException $error) { + if ($events === []) { + throw $error instanceof Invalid + ? $error + : new Invalid('Feed batch contains an event that cannot be read: ' . $error->getMessage(), previous: $error); + } + + break; + } + } + + return $events; + } + + /** + * @param array $raw + */ + private static function event(array $raw): CloudEvent + { + foreach (['specversion', 'id', 'type', 'source'] as $required) { + if (!isset($raw[$required]) || !\is_string($raw[$required]) || $raw[$required] === '') { + throw new Invalid('Feed event is missing ' . ($required === 'id' ? 'an id' : 'a ' . $required)); + } + } + + $extensions = []; + + /** @var mixed $value */ + foreach ($raw as $name => $value) { + if (\in_array($name, self::ATTRIBUTES, true)) { + continue; + } + + if (\preg_match('/^[a-z0-9]+$/', (string) $name) === 1 + && (\is_bool($value) || \is_int($value) || \is_string($value))) { + $extensions[$name] = $value; + } + } + + return new CloudEvent( + type: $raw['type'], + source: $raw['source'], + id: $raw['id'], + specversion: $raw['specversion'], + subject: self::optional($raw, 'subject'), + time: self::optional($raw, 'time'), + datacontenttype: self::optional($raw, 'datacontenttype'), + data: $raw['data'] ?? null, + dataschema: self::optional($raw, 'dataschema'), + extensions: $extensions, + ); + } + + /** + * @param array $raw + */ + private static function optional(array $raw, string $attribute): ?string + { + $value = $raw[$attribute] ?? null; + + return \is_string($value) && $value !== '' ? $value : null; } } diff --git a/src/Feed/Server.php b/src/Feed/Server.php index 9a754a1..a25b279 100644 --- a/src/Feed/Server.php +++ b/src/Feed/Server.php @@ -6,6 +6,11 @@ class Server { + // The request parameters of https://www.http-feeds.org/, as serve() reads them. + private const string PARAM_LAST_EVENT_ID = 'lastEventId'; + private const string PARAM_LIMIT = 'limit'; + private const string PARAM_TIMEOUT = 'timeout'; + public function __construct(protected readonly Readable $store) { } @@ -23,19 +28,19 @@ public function tip(): ?string return $this->store->tip(); } - public function read(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH): Batch + public function read(?string $lastEventId = null, int $limit = Readable::MAX_BATCH): Batch { - $limit = \max(1, \min($limit, Protocol::MAX_BATCH)); + $limit = \max(1, \min($limit, Readable::MAX_BATCH)); return new Batch($this->store->read($lastEventId, $limit), $limit); } - public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BATCH, int $timeout = 0): Batch + public function poll(?string $lastEventId = null, int $limit = Readable::MAX_BATCH, int $timeout = 0): Batch { - $limit = \max(1, \min($limit, Protocol::MAX_BATCH)); + $limit = \max(1, \min($limit, Readable::MAX_BATCH)); return new Batch( - $this->store->poll($lastEventId, $limit, \max(0, \min($timeout, Protocol::MAX_TIMEOUT))), + $this->store->poll($lastEventId, $limit, \max(0, \min($timeout, Readable::MAX_TIMEOUT))), $limit, ); } @@ -46,19 +51,19 @@ public function poll(?string $lastEventId = null, int $limit = Protocol::MAX_BAT */ public function serve(array $query): Batch { - $lastEventId = $query[Protocol::PARAM_LAST_EVENT_ID] ?? null; + $lastEventId = $query[self::PARAM_LAST_EVENT_ID] ?? null; $lastEventId = \is_string($lastEventId) && $lastEventId !== '' ? $lastEventId : null; - if ($lastEventId !== null && $lastEventId !== Protocol::TIP && !Id::isValid($lastEventId)) { + if ($lastEventId !== null && $lastEventId !== Readable::TIP && !Id::isValid($lastEventId)) { throw new Exception\Invalid('Invalid lastEventId: ' . $lastEventId); } - $limit = $query[Protocol::PARAM_LIMIT] ?? null; - $timeout = $query[Protocol::PARAM_TIMEOUT] ?? null; + $limit = $query[self::PARAM_LIMIT] ?? null; + $timeout = $query[self::PARAM_TIMEOUT] ?? null; return $this->poll( $lastEventId, - \is_numeric($limit) ? (int) $limit : Protocol::MAX_BATCH, + \is_numeric($limit) ? (int) $limit : Readable::MAX_BATCH, \is_numeric($timeout) ? (int) $timeout : 0, ); } diff --git a/src/Feed/Store.php b/src/Feed/Store.php index c348cd7..929a042 100644 --- a/src/Feed/Store.php +++ b/src/Feed/Store.php @@ -36,7 +36,7 @@ abstract public function tip(): ?string; protected function resolve(?string $lastEventId): ?string { - return $lastEventId === Protocol::TIP ? $this->tip() : $lastEventId; + return $lastEventId === Readable::TIP ? $this->tip() : $lastEventId; } /** diff --git a/tests/Feed/Unit/BatchTest.php b/tests/Feed/Unit/BatchTest.php index ca23c4b..7103356 100644 --- a/tests/Feed/Unit/BatchTest.php +++ b/tests/Feed/Unit/BatchTest.php @@ -7,7 +7,6 @@ use PHPUnit\Framework\TestCase; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Batch; -use Utopia\Feed\Protocol; class BatchTest extends TestCase { @@ -65,11 +64,22 @@ public function testAShortOrEmptyBatchIsNot(): void $this->assertSame('no-store', (new Batch([], 2))->cacheControl()); } + /** + * A batch of zero out of zero is not history β€” it is a caught-up consumer, + * and caching it would pin the consumer at that position forever. + */ + public function testAnEmptyBatchIsNeverCacheable(): void + { + $this->assertSame('no-store', (new Batch([], 0))->cacheControl()); + } + public function testToArrayIsTheWireEncoding(): void { - $events = self::events(2); + $payload = (new Batch(self::events(2), 100))->toArray(); - $this->assertSame(Protocol::encode($events), (new Batch($events, 100))->toArray()); + $this->assertTrue(\array_is_list($payload), 'A batch is a plain array β€” the spec defines no envelope'); + $this->assertSame(['1-0', '1-1'], \array_column($payload, 'id')); + $this->assertSame('1.0', $payload[0]['specversion']); $this->assertSame([], (new Batch([], 100))->toArray()); } } diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 190a308..86cca10 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -13,7 +13,7 @@ use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Producer; -use Utopia\Feed\Protocol; +use Utopia\Feed\Readable; use Utopia\Tests\Unit\Support\FailingCursor; use Utopia\Tests\Unit\Support\FakeTransport; use Utopia\Tests\Unit\Support\MidPollStore; @@ -224,8 +224,8 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array $consumer = new Consumer($store, $this->cursor, 'invalidator', batch: 5_000, timeout: 120_000); $consumer->consume(fn (CloudEvent $event) => null); - $this->assertSame(Protocol::MAX_BATCH, $store->limit); - $this->assertSame(Protocol::MAX_TIMEOUT, $store->timeout); + $this->assertSame(Readable::MAX_BATCH, $store->limit); + $this->assertSame(Readable::MAX_TIMEOUT, $store->timeout); } public function testDrainsABacklogInBatches(): void diff --git a/tests/Feed/Unit/ProtocolTest.php b/tests/Feed/Unit/ProtocolTest.php deleted file mode 100644 index a766fcc..0000000 --- a/tests/Feed/Unit/ProtocolTest.php +++ /dev/null @@ -1,239 +0,0 @@ -assertSame([], Protocol::query()); - $this->assertSame([], Protocol::query(null, 0, 0)); - $this->assertSame([], Protocol::query('', 0, 0)); - } - - public function testQueryCarriesTheParametersThatWereSet(): void - { - $this->assertSame([ - 'lastEventId' => '1-0', - 'limit' => 500, - 'timeout' => 20000, - ], Protocol::query('1-0', 500, 20000)); - } - - /** - * The tip sentinel rides the lastEventId parameter unchanged β€” the - * producer resolves it, and it can never collide with a real position. - */ - public function testQueryPassesTheTipSentinelThrough(): void - { - $this->assertSame(['lastEventId' => '$'], Protocol::query(Protocol::TIP)); - } - - public function testEncodesABatchAsAPlainArrayOfEvents(): void - { - $payload = Protocol::encode([ - new CloudEvent(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), - new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), - ]); - - $this->assertTrue(\array_is_list($payload), 'A batch is a plain array β€” the spec defines no envelope'); - $this->assertCount(2, $payload); - $this->assertSame('1-0', $payload[0]['id']); - $this->assertSame(['x' => 1], $payload[0]['data']); - $this->assertSame('1.0', $payload[0]['specversion']); - } - - public function testEncodesAnEmptyBatchAsAnEmptyArray(): void - { - $this->assertSame([], Protocol::encode([])); - } - - public function testDecodesWhatItEncoded(): void - { - $events = [ - new CloudEvent(id: '1-0', type: 'a', data: ['x' => 1], source: 'urn:test', subject: 's', time: 't'), - new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), - ]; - - $this->assertEquals($events, Protocol::decode(Protocol::encode($events))); - } - - public function testDecodesAnEmptyBatch(): void - { - $this->assertSame([], Protocol::decode([])); - } - - /** - * An empty batch means "you are caught up". A JSON object means "you did - * not reach the feed" β€” a misrouted request, a proxy's JSON error page, an - * endpoint that moved. Reading one as an empty batch would leave a - * consumer sitting quietly at a position that never advances again. - * - * @dataProvider notBatches - */ - public function testARespondingEndpointThatIsNotAFeedIsNotMistakenForBeingCaughtUp(mixed $payload): void - { - $this->expectException(Invalid::class); - - Protocol::decode($payload); - } - - /** - * @return array - */ - public static function notBatches(): array - { - return [ - 'the old envelope' => [['total' => 0, 'events' => []]], - 'some other API' => [['data' => [], 'status' => 'ok']], - 'an error body' => [['message' => 'Not found', 'code' => 404]], - ]; - } - - public function testRejectsAPayloadThatIsNotABatch(): void - { - $this->expectException(Invalid::class); - - Protocol::decode('not a batch'); - } - - /** - * One event as a producer would put it on the wire. - * - * @param array $overrides - * @return array - */ - private static function raw(string $id, string $type, array $overrides = []): array - { - return \array_merge([ - 'specversion' => '1.0', - 'id' => $id, - 'type' => $type, - 'source' => 'urn:test', - ], $overrides); - } - - /** - * An event with no id has no position, so a consumer cannot record having - * passed it. Returning the usable prefix lets those events be handled and - * the position advance to the last of them; the broken event is then at - * the head of the next batch, where it stops the feed loudly. - */ - public function testKeepsTheEventsBeforeAnUndecodableOne(): void - { - $events = Protocol::decode([ - self::raw('1-0', 'a'), - self::raw('1-1', 'b'), - self::raw('', 'no id'), - self::raw('1-3', 'd'), - ]); - - $this->assertCount(2, $events); - $this->assertSame(['a', 'b'], \array_map(fn (CloudEvent $e): string => $e->type, $events)); - } - - public function testFailsWhenTheFirstEventIsUndecodable(): void - { - $this->expectException(Invalid::class); - - Protocol::decode([self::raw('', 'no id'), self::raw('1-1', 'b')]); - } - - public function testFailsWhenTheFirstEntryIsNotAnEvent(): void - { - $this->expectException(Invalid::class); - - Protocol::decode(['a string']); - } - - public function testKeepsTheEventsBeforeAnEntryThatIsNotAnEvent(): void - { - $events = Protocol::decode([self::raw('1-0', 'a'), 'a string']); - - $this->assertCount(1, $events); - } - - /** - * `specversion` is REQUIRED by the spec and a feed's own producer always - * sends it, so an entry without one is not a CloudEvent at all β€” the batch - * stops there rather than the attribute being invented. - */ - public function testFailsWhenAnEventIsNotACloudEventAtAll(): void - { - $this->expectException(Invalid::class); - - Protocol::decode([['id' => '1-0', 'type' => 'a']]); - } - - /** - * The forward-compatibility property a feed depends on: it is read by - * consumers older than the producer by design, so a producer that adds an - * attribute or moves the spec version forward must not stop one that - * predates it. - */ - public function testSurvivesAProducerThatMovedAhead(): void - { - $events = Protocol::decode([ - self::raw('1-0', 'a', [ - 'specversion' => '1.1', - 'somethingnew' => 'ignored', - 'traceparent' => '00-abc-def-01', - ]), - ]); - - $this->assertCount(1, $events); - $this->assertSame('1.1', $events[0]->specversion); - $this->assertSame('00-abc-def-01', $events[0]->extensions['traceparent']); - } - - /** - * The spec's optional compaction/deletion feature marks an event with a - * `method` attribute. This library does not implement the feature, but a - * feed that uses it must still be readable β€” the attribute rides along as - * an extension rather than breaking the batch. - */ - public function testAnEventCarryingTheSpecsMethodAttributeDecodes(): void - { - $events = Protocol::decode([self::raw('1-0', 'a', ['method' => 'DELETE'])]); - - $this->assertCount(1, $events); - $this->assertSame('DELETE', $events[0]->extensions['method']); - } - - /** - * A full batch is settled history, so it may be cached forever. Anything - * short is the live end of the feed and will grow. - */ - public function testAFullBatchIsCacheable(): void - { - $this->assertSame('private, max-age=31536000', Protocol::cacheControl(100, 100)); - } - - public function testAPartialBatchIsNotCacheable(): void - { - $this->assertSame('no-store', Protocol::cacheControl(99, 100)); - $this->assertSame('no-store', Protocol::cacheControl(0, 100)); - } - - /** - * A batch of zero out of zero is not history β€” it is a caught-up consumer, - * and caching it would pin the consumer at that position forever. - */ - public function testAnEmptyBatchIsNeverCacheable(): void - { - $this->assertSame('no-store', Protocol::cacheControl(0, 0)); - } - - public function testSharedCachingIsOptIn(): void - { - $this->assertStringStartsWith('private, ', Protocol::cacheControl(10, 10)); - $this->assertStringStartsWith('public, ', Protocol::cacheControl(10, 10, public: true)); - } -} diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index 08da9de..b763590 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -13,8 +13,9 @@ use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Exception\Unsupported; +use Utopia\Feed\Batch; use Utopia\Feed\Producer; -use Utopia\Feed\Protocol; +use Utopia\Feed\Readable; use Utopia\Feed\Remote; use Utopia\Feed\Server; use Utopia\Tests\Unit\Support\FakeTransport; @@ -34,9 +35,20 @@ private function remote(array $responses = []): array return [new Remote($transport, 'edge'), $transport]; } + /** + * A response body as a producer would put it on the wire. + * + * @param list $events + * @return list> + */ + private static function batch(array $events): array + { + return (new Batch($events, \count($events)))->toArray(); + } + public function testReadsAFeedOverHttp(): void { - [$remote] = $this->remote([FakeTransport::json(Protocol::encode([ + [$remote] = $this->remote([FakeTransport::json(self::batch([ new CloudEvent(id: '1-0', type: 'io.appwrite.edge.invalidate-rule', source: 'urn:test', data: ['tags' => ['domain' => 'example.com']]), new CloudEvent(id: '1-1', type: 'io.appwrite.edge.invalidate', source: 'urn:test'), ]))]); @@ -117,8 +129,8 @@ public function testAsksForTheFeedMediaType(): void $remote->read(); - $this->assertSame(Protocol::MEDIA_TYPE, $transport->recorder->last()['headers']['Accept'] ?? null); - $this->assertSame('application/cloudevents-batch+json', Protocol::MEDIA_TYPE); + $this->assertSame(Remote::MEDIA_TYPE, $transport->recorder->last()['headers']['Accept'] ?? null); + $this->assertSame('application/cloudevents-batch+json', Remote::MEDIA_TYPE); } public function testSendsThePositionAndLimit(): void @@ -137,7 +149,7 @@ public function testSendsNoPositionOnAFirstFullRead(): void { [$remote, $transport] = $this->remote(); - $remote->read(null, Protocol::MAX_BATCH); + $remote->read(null, Readable::MAX_BATCH); $this->assertStringNotContainsString('lastEventId', $transport->recorder->last()['uri']); } @@ -169,10 +181,9 @@ public function testAllowsTheClientLongerThanTheLongPollTimeout(): void $remote->poll(null, 100, 5000); - // Seconds, which is what the client takes; the protocol margin is in + // Seconds, which is what the client takes; the margin is in // milliseconds, like the timeout the producer is given. $this->assertSame(15.0, $transport->recorder->last()['timeout']); - $this->assertSame((float) ((5000 + Protocol::TIMEOUT_MARGIN) / 1000), $transport->recorder->last()['timeout']); } public function testLeavesTheConfiguredTimeoutAloneWhenNotLongPolling(): void @@ -255,6 +266,151 @@ public function testRejectsABodyThatIsNotABatch(): void $remote->read(); } + /** + * An empty batch means "you are caught up". A JSON object means "you did + * not reach the feed" β€” a misrouted request, a proxy's JSON error page, an + * endpoint that moved. Reading one as an empty batch would leave a + * consumer sitting quietly at a position that never advances again. + * + * @dataProvider notBatches + * @param array $payload + */ + public function testARespondingEndpointThatIsNotAFeedIsNotMistakenForBeingCaughtUp(array $payload): void + { + [$remote] = $this->remote([FakeTransport::json($payload)]); + + $this->expectException(Invalid::class); + + $remote->read(); + } + + /** + * @return array}> + */ + public static function notBatches(): array + { + return [ + 'the old envelope' => [['total' => 0, 'events' => []]], + 'some other API' => [['data' => [], 'status' => 'ok']], + 'an error body' => [['message' => 'Not found', 'code' => 404]], + ]; + } + + /** + * One event as a producer would put it on the wire. + * + * @param array $overrides + * @return array + */ + private static function raw(string $id, string $type, array $overrides = []): array + { + return \array_merge([ + 'specversion' => '1.0', + 'id' => $id, + 'type' => $type, + 'source' => 'urn:test', + ], $overrides); + } + + /** + * An event with no id has no position, so a consumer cannot record having + * passed it. Returning the usable prefix lets those events be handled and + * the position advance to the last of them; the broken event is then at + * the head of the next batch, where it stops the feed loudly. + */ + public function testKeepsTheEventsBeforeAnUndecodableOne(): void + { + [$remote] = $this->remote([FakeTransport::json([ + self::raw('1-0', 'a'), + self::raw('1-1', 'b'), + self::raw('', 'no id'), + self::raw('1-3', 'd'), + ])]); + + $events = $remote->read(); + + $this->assertCount(2, $events); + $this->assertSame(['a', 'b'], \array_map(fn (CloudEvent $e): string => $e->type, $events)); + } + + public function testFailsWhenTheFirstEventIsUndecodable(): void + { + [$remote] = $this->remote([FakeTransport::json([self::raw('', 'no id'), self::raw('1-1', 'b')])]); + + $this->expectException(Invalid::class); + + $remote->read(); + } + + public function testFailsWhenTheFirstEntryIsNotAnEvent(): void + { + [$remote] = $this->remote([FakeTransport::json(['a string'])]); + + $this->expectException(Invalid::class); + + $remote->read(); + } + + public function testKeepsTheEventsBeforeAnEntryThatIsNotAnEvent(): void + { + [$remote] = $this->remote([FakeTransport::json([self::raw('1-0', 'a'), 'a string'])]); + + $this->assertCount(1, $remote->read()); + } + + /** + * `specversion` is REQUIRED by the spec and a feed's own producer always + * sends it, so an entry without one is not a CloudEvent at all β€” the batch + * stops there rather than the attribute being invented. + */ + public function testFailsWhenAnEventIsNotACloudEventAtAll(): void + { + [$remote] = $this->remote([FakeTransport::json([['id' => '1-0', 'type' => 'a']])]); + + $this->expectException(Invalid::class); + + $remote->read(); + } + + /** + * The forward-compatibility property a feed depends on: it is read by + * consumers older than the producer by design, so a producer that adds an + * attribute or moves the spec version forward must not stop one that + * predates it. + */ + public function testSurvivesAProducerThatMovedAhead(): void + { + [$remote] = $this->remote([FakeTransport::json([ + self::raw('1-0', 'a', [ + 'specversion' => '1.1', + 'somethingnew' => 'ignored', + 'traceparent' => '00-abc-def-01', + ]), + ])]); + + $events = $remote->read(); + + $this->assertCount(1, $events); + $this->assertSame('1.1', $events[0]->specversion); + $this->assertSame('00-abc-def-01', $events[0]->extensions['traceparent']); + } + + /** + * The spec's optional compaction/deletion feature marks an event with a + * `method` attribute. This library does not implement the feature, but a + * feed that uses it must still be readable β€” the attribute rides along as + * an extension rather than breaking the batch. + */ + public function testAnEventCarryingTheSpecsMethodAttributeDecodes(): void + { + [$remote] = $this->remote([FakeTransport::json([self::raw('1-0', 'a', ['method' => 'DELETE'])])]); + + $events = $remote->read(); + + $this->assertCount(1, $events); + $this->assertSame('DELETE', $events[0]->extensions['method']); + } + /** * Anything implementing the client's adapter interface works, including * the client itself wrapping a transport β€” which is how this is actually @@ -262,7 +418,7 @@ public function testRejectsABodyThatIsNotABatch(): void */ public function testWorksThroughTheClientItself(): void { - $transport = FakeTransport::of([FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-0', type: 'a', source: 'urn:test')]))]); + $transport = FakeTransport::of([FakeTransport::json(self::batch([new CloudEvent(id: '1-0', type: 'a', source: 'urn:test')]))]); $client = (new Client($transport)) ->withBaseUri('https://cloud.example.com/v1/feeds') @@ -283,12 +439,12 @@ public function testWorksThroughTheClientItself(): void public function testConsumesARemoteFeedThroughTheSameConsumer(): void { $transport = FakeTransport::of([ - FakeTransport::json(Protocol::encode([ + FakeTransport::json(self::batch([ new CloudEvent(id: '1-0', type: 'a', source: 'urn:test'), new CloudEvent(id: '1-1', type: 'b', source: 'urn:test'), ])), - FakeTransport::json(Protocol::encode([new CloudEvent(id: '1-2', type: 'c', source: 'urn:test')])), - FakeTransport::json(Protocol::encode([])), + FakeTransport::json(self::batch([new CloudEvent(id: '1-2', type: 'c', source: 'urn:test')])), + FakeTransport::json(self::batch([])), ]); $cursor = new MemoryCursor(); diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php index 7aa80da..dec0bd4 100644 --- a/tests/Feed/Unit/RoundTripTest.php +++ b/tests/Feed/Unit/RoundTripTest.php @@ -12,15 +12,14 @@ use Utopia\Feed\Cursor\Cache as CacheCursor; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Producer; -use Utopia\Feed\Protocol; use Utopia\Feed\Server; use Utopia\Tests\Unit\Support\FeedServer; /** * A producer and a consumer joined by the HTTP contract, which is the pair this - * library exists to keep from drifting apart. Everything here goes through - * {@see Protocol} in both directions rather than through a fixture written to - * match one side. + * library exists to keep from drifting apart. Everything here goes through the + * real wire code in both directions β€” {@see Batch} encoding, {@see Remote} + * decoding β€” rather than through a fixture written to match one side. */ class RoundTripTest extends TestCase { diff --git a/tests/Feed/Unit/ServerTest.php b/tests/Feed/Unit/ServerTest.php index 49717a5..d55c51b 100644 --- a/tests/Feed/Unit/ServerTest.php +++ b/tests/Feed/Unit/ServerTest.php @@ -13,7 +13,7 @@ use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Server; use Utopia\Feed\Producer; -use Utopia\Feed\Protocol; +use Utopia\Feed\Readable; use Utopia\Feed\Id; use Utopia\Tests\Unit\Support\MidPollStore; @@ -118,7 +118,7 @@ public function testClampsTheLimitToTheMaximum(): void { $this->producer->produce('test'); - $this->assertCount(1, $this->server->read(null, Protocol::MAX_BATCH * 10)); + $this->assertCount(1, $this->server->read(null, Readable::MAX_BATCH * 10)); $this->assertCount(1, $this->server->read(null, 0)); $this->assertCount(1, $this->server->read(null, -5)); } @@ -366,7 +366,7 @@ public function testReadingFromTheTipSentinelSkipsTheBacklog(): void $this->producer->produce('a'); $this->producer->produce('b'); - $this->assertCount(0, $this->server->read(Protocol::TIP)); + $this->assertCount(0, $this->server->read(Readable::TIP)); } public function testServeAppliesTheDefaultsWhenNoParametersArrive(): void @@ -410,7 +410,7 @@ public function testServeLetsTheTipSentinelThrough(): void { $this->producer->produce('a'); - $this->assertCount(0, $this->server->serve(['lastEventId' => Protocol::TIP])); + $this->assertCount(0, $this->server->serve(['lastEventId' => Readable::TIP])); } public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void @@ -430,13 +430,13 @@ public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void */ public function testAnOversizedLimitStillYieldsAnHonestCacheControl(): void { - foreach (\range(1, Protocol::MAX_BATCH) as $i) { + foreach (\range(1, Readable::MAX_BATCH) as $i) { $this->producer->produce('event-' . $i); } $batch = $this->server->serve(['limit' => '5000']); - $this->assertCount(Protocol::MAX_BATCH, $batch); + $this->assertCount(Readable::MAX_BATCH, $batch); $this->assertSame('public, max-age=31536000', $batch->cacheControl(public: true)); } diff --git a/tests/Feed/Unit/Support/FeedServer.php b/tests/Feed/Unit/Support/FeedServer.php index 7fec62e..3f82ff0 100644 --- a/tests/Feed/Unit/Support/FeedServer.php +++ b/tests/Feed/Unit/Support/FeedServer.php @@ -6,7 +6,7 @@ use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; -use Utopia\Feed\Protocol; +use Utopia\Feed\Batch; use Utopia\Feed\Server; use Utopia\Psr7\Header; use Utopia\Psr7\Response; @@ -37,7 +37,7 @@ protected function respond(RequestInterface $request): ResponseInterface $body = (string) \json_encode($batch->toArray()); return (new Response(200, body: new Stream\Factory()->createStream($body))) - ->withHeader(Header::CONTENT_TYPE, Protocol::MEDIA_TYPE) + ->withHeader(Header::CONTENT_TYPE, Batch::MEDIA_TYPE) ->withHeader(Header::CACHE_CONTROL, $batch->cacheControl()); } } From 2f6470e726a4c8b900542921086a8568dce08bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 Jul 2026 16:35:19 +0200 Subject: [PATCH 33/68] Fix failing tests --- src/Feed/Remote.php | 3 +++ src/Feed/Store.php | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index 93c8828..9d8303c 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -213,6 +213,9 @@ private static function event(array $raw): CloudEvent datacontenttype: self::optional($raw, 'datacontenttype'), data: $raw['data'] ?? null, dataschema: self::optional($raw, 'dataschema'), + // The docblock wants array, but a digit-only + // extension name β€” legal per the spec β€” is an integer key in PHP. + // @phpstan-ignore argument.type extensions: $extensions, ); } diff --git a/src/Feed/Store.php b/src/Feed/Store.php index 929a042..56276f0 100644 --- a/src/Feed/Store.php +++ b/src/Feed/Store.php @@ -99,6 +99,9 @@ protected static function decode(string $id, array $fields): CloudEvent $event += \is_array($extensions) ? $extensions : []; try { + // The docblock wants array, but a digit-only + // extension name β€” legal per the spec β€” is an integer key in PHP. + // @phpstan-ignore argument.type return CloudEvent::fromArray($event); } catch (\InvalidArgumentException $error) { throw new Invalid("Feed entry {$id} could not be read as an event: {$error->getMessage()}", previous: $error); From c3cab970922895c27bb48b0ecbda551f9794637f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 10:12:26 +0200 Subject: [PATCH 34/68] Add max site, fix concurrency --- README.md | 12 +++++++++-- src/Feed/Consumer.php | 9 +++++++- src/Feed/Store.php | 6 ++++++ src/Feed/Store/Cache.php | 4 ++-- src/Feed/Store/Memory.php | 4 ++-- src/Feed/Store/Pool.php | 4 ++-- src/Feed/Store/Redis.php | 4 ++-- tests/Feed/Unit/ConsumerTest.php | 37 ++++++++++++++++++++++++++++++++ tests/Feed/Unit/ServerTest.php | 21 ++++++++++++++++++ 9 files changed, 90 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 29dd061..57a4ad0 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,10 @@ fails with a 4xx `Transport` error rather than silently replaying the backlog. as `Transport`. The id must be well formed but need not still exist in the feed. +Either is safe to call from inside a handler: a run that finishes after the +move keeps its own progress to itself rather than saving over the newer +decision. + `seek()` is the escape hatch for a poison event. A handler that keeps failing blocks the feed by design, so stepping past it is a decision made in code: catch the failure, log the event's id, and once you have decided the event @@ -213,7 +217,9 @@ consumer opted into `Consumer::START_TIP`), so a consumer deployed after the pro drains the backlog instead of dropping it. **One process per consumer name.** Two processes sharing a name share one -position, so the feed is split between them rather than delivered to both. +position, so the feed is split between them rather than delivered to both β€” +and because each save is last-writer-wins, the shared position can also move +backwards and replay. Give every consumer its own name. ## Reference @@ -222,7 +228,9 @@ position, so the feed is split between them rather than delivered to both. Every store is `Readable` and `Appendable` β€” it owns its events and assigns their ids. All take `maxSize` (retention, ~100,000 entries by default) and `pollInterval` (how often a held poll re-reads, 500 ms by default β€” shorter -lowers long-poll latency, raises backend reads). +lowers long-poll latency, raises backend reads). Neither may be below 1 β€” a +store that retains nothing is a misconfiguration, so the constructor throws +`Exception\Invalid`. | Store | Use for | | --- | --- | diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 920fe98..6e9fc42 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -20,6 +20,9 @@ class Consumer private bool $restored = false; + /** Bumped by every hand-made move, so a run never saves over one. */ + private int $moved = 0; + /** * @throws Exception\Invalid When a name is missing, $feed contradicts the source, or $start is not a START_* constant. */ @@ -58,6 +61,8 @@ public function getName(): string public function consume(callable $handler): int { + $moved = $this->moved; + $events = $this->feed->poll( $this->position() ?? $this->origin(), \max(1, \min($this->batch, Readable::MAX_BATCH)), @@ -84,7 +89,7 @@ public function consume(callable $handler): int $handled++; } - if ($processed !== null) { + if ($processed !== null && $this->moved === $moved) { $this->position = $processed; $this->cursor->save($this->feed->getName(), $this->name, $processed); } @@ -125,6 +130,7 @@ public function seek(string $eventId): void $this->position = $eventId; $this->restored = true; + $this->moved++; } public function reset(): void @@ -133,5 +139,6 @@ public function reset(): void $this->position = null; $this->restored = true; + $this->moved++; } } diff --git a/src/Feed/Store.php b/src/Feed/Store.php index 56276f0..f5f1fc0 100644 --- a/src/Feed/Store.php +++ b/src/Feed/Store.php @@ -9,16 +9,22 @@ abstract class Store implements Readable { + protected const int MAX_SIZE = 100_000; // entries protected const int POLL_INTERVAL = 500; // ms public function __construct( protected readonly string $name, + protected readonly int $maxSize = self::MAX_SIZE, protected readonly int $pollInterval = self::POLL_INTERVAL, ) { if ($name === '') { throw new Invalid('Feed name is required'); } + if ($maxSize < 1) { + throw new Invalid('Feed retention must be at least 1 event'); + } + if ($pollInterval < 1) { throw new Invalid('Feed poll interval must be at least 1 millisecond'); } diff --git a/src/Feed/Store/Cache.php b/src/Feed/Store/Cache.php index 20d1f7e..7219c7c 100644 --- a/src/Feed/Store/Cache.php +++ b/src/Feed/Store/Cache.php @@ -18,11 +18,11 @@ class Cache extends Store implements Appendable public function __construct( protected readonly UtopiaCache $cache, string $name, - protected readonly int $maxSize = 100_000, + int $maxSize = self::MAX_SIZE, protected readonly int $ttl = self::TTL, int $pollInterval = self::POLL_INTERVAL, ) { - parent::__construct($name, $pollInterval); + parent::__construct($name, $maxSize, $pollInterval); } public function append(CloudEvent $event): string diff --git a/src/Feed/Store/Memory.php b/src/Feed/Store/Memory.php index 1d8a630..11b2659 100644 --- a/src/Feed/Store/Memory.php +++ b/src/Feed/Store/Memory.php @@ -19,10 +19,10 @@ class Memory extends Store implements Appendable public function __construct( string $name, - protected readonly int $maxSize = 100_000, + int $maxSize = self::MAX_SIZE, int $pollInterval = self::POLL_INTERVAL, ) { - parent::__construct($name, $pollInterval); + parent::__construct($name, $maxSize, $pollInterval); } public function append(CloudEvent $event): string diff --git a/src/Feed/Store/Pool.php b/src/Feed/Store/Pool.php index 02f894f..c2afcb3 100644 --- a/src/Feed/Store/Pool.php +++ b/src/Feed/Store/Pool.php @@ -17,10 +17,10 @@ class Pool extends Store implements Appendable public function __construct( protected readonly UtopiaPool $pool, string $name, - protected readonly int $maxSize = 100_000, + int $maxSize = self::MAX_SIZE, int $pollInterval = self::POLL_INTERVAL, ) { - parent::__construct($name, $pollInterval); + parent::__construct($name, $maxSize, $pollInterval); } private function inner(\Redis|\RedisCluster $redis): Redis diff --git a/src/Feed/Store/Redis.php b/src/Feed/Store/Redis.php index 796a904..9b0d227 100644 --- a/src/Feed/Store/Redis.php +++ b/src/Feed/Store/Redis.php @@ -15,10 +15,10 @@ class Redis extends Store implements Appendable public function __construct( protected readonly \Redis|\RedisCluster $redis, string $name, - protected readonly int $maxSize = 100_000, + int $maxSize = self::MAX_SIZE, int $pollInterval = self::POLL_INTERVAL, ) { - parent::__construct($name, $pollInterval); + parent::__construct($name, $maxSize, $pollInterval); } public function append(CloudEvent $event): string diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 86cca10..d6ee29e 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -484,6 +484,43 @@ public function testSeekingToAPoisonEventsIdUnblocksTheConsumer(): void $this->assertSame(['after'], $this->drain($consumer), 'The poison event is stepped over, nothing behind it is lost'); } + /** + * A hand-made move decided during a run is the newer decision, so the run + * must not save its own progress over it on the way out. + */ + public function testASeekMadeInsideAHandlerIsNotOverwritten(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + $third = $this->producer->produce('c'); + + $consumer = $this->consumer(batch: 1); + $consumer->consume(function (CloudEvent $event) use ($consumer, $third): void { + $consumer->seek($third); + }); + + $this->assertSame($third, $consumer->position()); + $this->assertSame($third, $this->cursor->load('edge', 'invalidator')); + $this->assertSame([], $this->drain($consumer), 'The run resumes after the seeked id, not after the handled one'); + } + + public function testAResetMadeInsideAHandlerIsNotOverwritten(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $consumer = $this->consumer(); + $consumer->consume(function (CloudEvent $event) use ($consumer): void { + if ($event->type === 'b') { + $consumer->reset(); + } + }); + + $this->assertNull($consumer->position()); + $this->assertNull($this->cursor->load('edge', 'invalidator')); + $this->assertSame(['a', 'b'], $this->drain($consumer), 'The reset stands, so everything retained replays'); + } + /** * @dataProvider notPositions */ diff --git a/tests/Feed/Unit/ServerTest.php b/tests/Feed/Unit/ServerTest.php index d55c51b..443e71a 100644 --- a/tests/Feed/Unit/ServerTest.php +++ b/tests/Feed/Unit/ServerTest.php @@ -307,6 +307,27 @@ public function testRejectsANegativePollInterval(): void new Memory('edge', pollInterval: -5); } + /** + * @dataProvider notRetentions + */ + public function testRejectsARetentionThatKeepsNothing(int $maxSize): void + { + $this->expectException(Invalid::class); + + new Memory('edge', maxSize: $maxSize); + } + + /** + * @return array + */ + public static function notRetentions(): array + { + return [ + 'zero' => [0], + 'negative' => [-5], + ]; + } + public function testRetentionIsBoundedAndTrimsTheOldest(): void { $store = new Memory('small', maxSize: 3); From efbbdf1018d27557dd1c82d09a8f9ab8489bdce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 10:26:10 +0200 Subject: [PATCH 35/68] Document expected behaviour --- README.md | 6 +++++ tests/Feed/Unit/ConsumerTest.php | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/README.md b/README.md index 57a4ad0..52fad7f 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,12 @@ A cursor is keyed by feed and consumer name, so one store serves every feed a service consumes. The store may be lossy β€” a lost position costs a replay, not a gap. +The stored form is deliberately plain: the key is `feed::cursor:` +and the value is the event id as a string. That is the format consumers have +always written, so positions carry across an upgrade instead of replaying the +retained feed, and `GET feed:edge:cursor:notifier` answers "where is this +consumer?" from a shell. + | Cursor | Use for | | --- | --- | | `Cursor\Cache` | A consumer with a [Utopia cache](https://github.com/utopia-php/cache) β€” the usual choice for a remote feed. Holds a position for 30 days (`Cursor\Cache::TTL`) unless saved again | diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index d6ee29e..f5beb1f 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -589,6 +589,47 @@ public function testConsumersOfTheSameFeedTrackSeparatePositions(): void $this->assertSame(0, $one->consume(fn (CloudEvent $event) => null)); } + /** + * The other half of that rule, pinned because it is the boundary the + * design draws rather than an accident: a name is one logical reader, so + * two processes behind one name split the feed instead of both seeing it. + */ + public function testTwoConsumersSharingANameSplitTheFeed(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $one = $this->consumer(batch: 1); + $two = $this->consumer(batch: 1); + + $this->assertSame(['a'], $this->drain($one)); + $this->assertSame(['b'], $this->drain($two), 'The second picks up after the first, it does not see a of its own'); + } + + /** + * And the cost of sharing a name, which no amount of coordination inside a + * single process can remove: each save is last-writer-wins, so a replica + * holding an older position drags the shared one backwards when it saves. + * At-least-once makes that a replay rather than a loss β€” the same is true of + * a reset one replica performs and another then recreates β€” but it is why + * every consumer gets its own name. + */ + public function testAStaleConsumerSharingANameDragsThePositionBackwards(): void + { + $first = $this->producer->produce('a'); + $second = $this->producer->produce('b'); + + $stale = $this->consumer(batch: 1); + $this->assertNull($stale->position(), 'Reads the shared position before the other replica moves it'); + + $ahead = $this->consumer(); + $this->assertSame(['a', 'b'], $this->drain($ahead)); + $this->assertSame($second, $this->cursor->load('edge', 'invalidator')); + + $this->assertSame(['a'], $this->drain($stale), 'The stale replica polls from where it thought it was'); + $this->assertSame($first, $this->cursor->load('edge', 'invalidator'), 'Its save wins, so the shared position regresses'); + } + public function testPositionIsNullBeforeTheFirstRun(): void { $this->assertNull($this->consumer()->position()); From 77b2fd074b77d8d4498d9e4e0e4b8fe2244a1ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 10:55:40 +0200 Subject: [PATCH 36/68] Test rework preparations --- .github/workflows/tests.yml | 13 +++++++------ README.md | 27 ++++++++++++++++++++------- composer.json | 9 +++++++-- phpunit.xml | 30 ++++++++++++++++++++++++++++-- 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 35db5c2..850bdb0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,7 @@ on: [pull_request] jobs: tests: - name: "PHP ${{ matrix.php-versions }}" + name: "PHP ${{ matrix.php-versions }} Β· ${{ matrix.suite }}" runs-on: ubuntu-latest strategy: fail-fast: false @@ -12,6 +12,10 @@ jobs: # Add a PHP version here and nowhere else β€” the image is built from one # parameterized Dockerfile. php-versions: ['8.5'] + # One job per adapter: each runs the same behaviour scenarios + # (tests/Feed/*/Base.php), so a green suite means that adapter honours + # the whole contract. See phpunit.xml for what each suite contains. + suite: [unit, memory, cache, http, redis, pool] env: PHP_VERSION: ${{ matrix.php-versions }} @@ -25,8 +29,5 @@ jobs: docker compose build docker compose up -d - - name: Unit tests - run: docker compose exec -T tests vendor/bin/phpunit --testsuite unit - - - name: E2E tests - run: docker compose exec -T tests vendor/bin/phpunit --testsuite e2e + - name: Run ${{ matrix.suite }} tests + run: docker compose exec -T tests vendor/bin/phpunit --testsuite ${{ matrix.suite }} diff --git a/README.md b/README.md index 52fad7f..6406c38 100644 --- a/README.md +++ b/README.md @@ -332,24 +332,37 @@ builds over its client. Services never need any of them directly. ## Tests -Unit tests need no services, but dependencies declare extensions the suite -never touches (`ext-redis`, `ext-memcached`, `ext-protobuf`), so install past -them: +The suite is organized around behaviour, not classes: each component has one +abstract scenario suite β€” `tests/Feed/Producer/Base.php`, +`tests/Feed/Server/Base.php`, `tests/Feed/Consumer/Base.php` β€” and every +adapter extends it, so a passing adapter suite means that adapter honours the +whole contract. Producer and Server run per store (`memory`, `cache`, `redis`, +`pool`); Consumer runs per cursor plus once through the real HTTP wire code +(`http`). What remains in `tests/Feed/Unit` are the cases only a fake can +provoke: a cursor store that is down, a body that is not a batch, a backend +that was never configured. + +The service-free suites need no Redis, but dependencies declare extensions the +suites never touch (`ext-redis`, `ext-memcached`, `ext-protobuf`), so install +past them: ```bash composer install --ignore-platform-reqs -composer test +composer test # unit + memory + cache + http ``` -The E2E suite runs against a real Redis, and static analysis needs `ext-redis`, -so both run in the container: +The `redis` and `pool` suites run the same scenarios against a real Redis, and +static analysis needs `ext-redis`, so both run in the container: ```bash docker compose up -d -docker compose exec tests composer test:e2e +docker compose exec tests composer test:redis +docker compose exec tests composer test:pool docker compose exec tests composer check ``` +CI runs every suite as its own job, so a failing adapter is visible by name. + To test another PHP version, build with `PHP_VERSION=8.6 docker compose build`, and add it to the `php-versions` matrix in `.github/workflows/tests.yml`. diff --git a/composer.json b/composer.json index ef0c7f9..5f260f4 100644 --- a/composer.json +++ b/composer.json @@ -16,8 +16,13 @@ "license": "MIT", "minimum-stability": "stable", "scripts": { - "test": "vendor/bin/phpunit --testsuite unit", - "test:e2e": "vendor/bin/phpunit --testsuite e2e", + "test": "vendor/bin/phpunit --testsuite unit,memory,cache,http", + "test:unit": "vendor/bin/phpunit --testsuite unit", + "test:memory": "vendor/bin/phpunit --testsuite memory", + "test:cache": "vendor/bin/phpunit --testsuite cache", + "test:http": "vendor/bin/phpunit --testsuite http", + "test:redis": "vendor/bin/phpunit --testsuite redis", + "test:pool": "vendor/bin/phpunit --testsuite pool", "check": "vendor/bin/phpstan analyse --memory-limit=1G", "lint": "vendor/bin/pint --test", "format": "vendor/bin/pint" diff --git a/phpunit.xml b/phpunit.xml index 4a09dc2..fcea8da 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -8,12 +8,38 @@ processIsolation="false" stopOnFailure="false" > + ./tests/Feed/Unit - - ./tests/Feed/E2E + + ./tests/Feed/Producer/MemoryTest.php + ./tests/Feed/Server/MemoryTest.php + ./tests/Feed/Consumer/MemoryTest.php + + + ./tests/Feed/Producer/CacheTest.php + ./tests/Feed/Server/CacheTest.php + ./tests/Feed/Consumer/CacheTest.php + + + ./tests/Feed/Consumer/HttpTest.php + + + ./tests/Feed/Producer/RedisTest.php + ./tests/Feed/Server/RedisTest.php + ./tests/Feed/Consumer/RedisTest.php + + + ./tests/Feed/Producer/PoolTest.php + ./tests/Feed/Server/PoolTest.php + ./tests/Feed/Consumer/PoolTest.php From 2e0e9e204ed65ecd3f8ebb56530b958c4b184cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 10:55:59 +0200 Subject: [PATCH 37/68] Rework tests --- tests/Feed/Consumer/Base.php | 607 ++++++++++++++++++ tests/Feed/Consumer/CacheTest.php | 12 + tests/Feed/Consumer/HttpTest.php | 79 +++ tests/Feed/Consumer/MemoryTest.php | 12 + tests/Feed/Consumer/PoolTest.php | 12 + tests/Feed/Consumer/RedisTest.php | 28 + tests/Feed/E2E/RedisTest.php | 319 --------- tests/Feed/Producer/Base.php | 340 ++++++++++ tests/Feed/Producer/CacheTest.php | 69 ++ tests/Feed/Producer/MemoryTest.php | 41 ++ tests/Feed/Producer/PoolTest.php | 12 + tests/Feed/Producer/RedisTest.php | 39 ++ tests/Feed/Server/Base.php | 310 +++++++++ tests/Feed/Server/CacheTest.php | 12 + tests/Feed/Server/MemoryTest.php | 42 ++ tests/Feed/Server/PoolTest.php | 12 + tests/Feed/Server/RedisTest.php | 12 + .../Feed/{Unit => }/Support/FailingCursor.php | 2 +- tests/Feed/{Unit => }/Support/FakeClient.php | 2 +- .../Feed/{Unit => }/Support/FakeTransport.php | 2 +- tests/Feed/{Unit => }/Support/FeedServer.php | 2 +- .../Feed/{Unit => }/Support/MidPollStore.php | 2 +- tests/Feed/{Unit => }/Support/Recorder.php | 2 +- tests/Feed/Support/UsesCache.php | 38 ++ tests/Feed/Support/UsesMemory.php | 28 + tests/Feed/Support/UsesPool.php | 64 ++ tests/Feed/Support/UsesRedis.php | 54 ++ tests/Feed/Unit/ConsumerTest.php | 499 +------------- tests/Feed/Unit/CursorTest.php | 106 +-- tests/Feed/Unit/ProducerTest.php | 113 +--- tests/Feed/Unit/RemoteTest.php | 6 +- tests/Feed/Unit/RoundTripTest.php | 207 ------ tests/Feed/Unit/ServerTest.php | 482 +------------- tests/Feed/Unit/StoreCacheTest.php | 170 ----- 34 files changed, 1880 insertions(+), 1857 deletions(-) create mode 100644 tests/Feed/Consumer/Base.php create mode 100644 tests/Feed/Consumer/CacheTest.php create mode 100644 tests/Feed/Consumer/HttpTest.php create mode 100644 tests/Feed/Consumer/MemoryTest.php create mode 100644 tests/Feed/Consumer/PoolTest.php create mode 100644 tests/Feed/Consumer/RedisTest.php delete mode 100644 tests/Feed/E2E/RedisTest.php create mode 100644 tests/Feed/Producer/Base.php create mode 100644 tests/Feed/Producer/CacheTest.php create mode 100644 tests/Feed/Producer/MemoryTest.php create mode 100644 tests/Feed/Producer/PoolTest.php create mode 100644 tests/Feed/Producer/RedisTest.php create mode 100644 tests/Feed/Server/Base.php create mode 100644 tests/Feed/Server/CacheTest.php create mode 100644 tests/Feed/Server/MemoryTest.php create mode 100644 tests/Feed/Server/PoolTest.php create mode 100644 tests/Feed/Server/RedisTest.php rename tests/Feed/{Unit => }/Support/FailingCursor.php (95%) rename tests/Feed/{Unit => }/Support/FakeClient.php (98%) rename tests/Feed/{Unit => }/Support/FakeTransport.php (98%) rename tests/Feed/{Unit => }/Support/FeedServer.php (97%) rename tests/Feed/{Unit => }/Support/MidPollStore.php (95%) rename tests/Feed/{Unit => }/Support/Recorder.php (97%) create mode 100644 tests/Feed/Support/UsesCache.php create mode 100644 tests/Feed/Support/UsesMemory.php create mode 100644 tests/Feed/Support/UsesPool.php create mode 100644 tests/Feed/Support/UsesRedis.php delete mode 100644 tests/Feed/Unit/RoundTripTest.php delete mode 100644 tests/Feed/Unit/StoreCacheTest.php diff --git a/tests/Feed/Consumer/Base.php b/tests/Feed/Consumer/Base.php new file mode 100644 index 0000000..9bcf332 --- /dev/null +++ b/tests/Feed/Consumer/Base.php @@ -0,0 +1,607 @@ +name = 'test-' . \bin2hex(\random_bytes(8)); + $this->store = new MemoryStore($this->name); + $this->producer = new Producer($this->store, 'urn:test'); + $this->cursor = $this->cursor(); + } + + protected function consumer( + string $name = 'invalidator', + int $batch = Consumer::BATCH, + int $timeout = 0, + string $start = Consumer::START_OLDEST, + (Store&Appendable)|null $store = null, + ): Consumer { + $store ??= $this->store; + + return new Consumer($this->source($store), $this->cursor, $name, feed: $store->getName(), batch: $batch, timeout: $timeout, start: $start); + } + + /** + * @param-out int $count + * @return list + */ + protected function drain(Consumer $consumer, ?int &$count = null): array + { + $seen = []; + $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }); + + return $seen; + } + + public function testHandlesEachEventAndAdvancesPastTheLastOne(): void + { + $this->producer->produce('a'); + $last = $this->producer->produce('b'); + + $consumer = $this->consumer(); + + $this->assertSame(['a', 'b'], $this->drain($consumer, $count)); + $this->assertSame(2, $count); + $this->assertSame($last, $this->cursor->load($this->name, 'invalidator')); + $this->assertSame($last, $consumer->position()); + } + + public function testCaughtUpConsumerDoesNothing(): void + { + $this->producer->produce('a'); + + $consumer = $this->consumer(); + $consumer->consume(fn (CloudEvent $event) => null); + + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + } + + public function testResumesFromTheStoredPosition(): void + { + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->cursor->save($this->name, 'invalidator', $first); + + $this->assertSame(['b'], $this->drain($this->consumer())); + } + + /** + * A consumer that has never run starts at the oldest retained event, not + * at the tip β€” otherwise the first event a feed ever carries is the one + * event that is guaranteed to be dropped. + */ + public function testAConsumerWithNoPositionStartsAtTheOldestEventNotTheTip(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->assertSame(['a', 'b'], $this->drain($this->consumer())); + } + + public function testTheConsumerOnlyEverSeesEachEventOnce(): void + { + foreach (\range(1, 5) as $i) { + $this->producer->produce('event-' . $i); + } + + $consumer = $this->consumer(); + $seen = []; + $handler = function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }; + + $consumer->consume($handler); + $consumer->consume($handler); + + $this->producer->produce('event-6'); + $consumer->consume($handler); + + $this->assertSame( + ['event-1', 'event-2', 'event-3', 'event-4', 'event-5', 'event-6'], + $seen, + ); + } + + /** + * The restart property: a fresh Consumer β€” no in-memory state, same name, + * same cursor store β€” picks the persisted position up rather than + * replaying or skipping. + */ + public function testARestartedConsumerResumesWhereItLeftOff(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->consumer()->consume(fn (CloudEvent $event) => null); + + $this->producer->produce('c'); + + $this->assertSame(['c'], $this->drain($this->consumer())); + } + + public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void + { + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + $this->producer->produce('c'); + + $consumer = $this->consumer(); + $seen = []; + + try { + $consumer->consume(function (CloudEvent $event) use (&$seen): void { + if ($event->type === 'b') { + throw new \RuntimeException('nope'); + } + + $seen[] = $event->type; + }); + $this->fail('The handler failure should have been re-raised'); + } catch (\RuntimeException $error) { + $this->assertSame('nope', $error->getMessage()); + } + + $this->assertSame(['a'], $seen); + $this->assertSame($first, $this->cursor->load($this->name, 'invalidator'), 'Progress before the failure is committed'); + } + + public function testRetriesTheFailedEventOnTheNextRun(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $consumer = $this->consumer(); + $attempts = 0; + + try { + $consumer->consume(function (CloudEvent $event) use (&$attempts): void { + if ($event->type === 'b') { + $attempts++; + throw new \RuntimeException('nope'); + } + }); + } catch (\RuntimeException) { + // Expected. + } + + $this->assertSame(['b'], $this->drain($consumer), 'The failed event comes back'); + $this->assertSame(1, $attempts); + } + + /** + * A failure on the very first event of a run commits nothing, so a store + * that was already empty stays empty rather than being written a position + * that stands for no completed work. + */ + public function testAFailureOnTheFirstEventCommitsNothing(): void + { + $this->producer->produce('a'); + + try { + $this->consumer()->consume(fn (CloudEvent $event) => throw new \RuntimeException('nope')); + } catch (\RuntimeException) { + // Expected. + } + + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + } + + public function testAFailedEventBlocksTheOnesBehindItUntilItSucceeds(): void + { + $this->producer->produce('a'); + $this->producer->produce('poison'); + $this->producer->produce('c'); + + $consumer = $this->consumer(); + $seen = []; + $attempts = 0; + + // Fails the first time it sees the poison event and succeeds after, + // standing in for a dependency that was briefly unavailable. + $handler = function (CloudEvent $event) use (&$seen, &$attempts): void { + if ($event->type === 'poison') { + $attempts++; + + if ($attempts === 1) { + throw new \RuntimeException('not yet'); + } + } + + $seen[] = $event->type; + }; + + try { + $consumer->consume($handler); + $this->fail('The handler failure should have been re-raised'); + } catch (\RuntimeException) { + // Expected. + } + + $this->assertSame(['a'], $seen, 'Nothing behind the failure is delivered'); + + $consumer->consume($handler); + + $this->assertSame(['a', 'poison', 'c'], $seen, 'Order is preserved and nothing is skipped'); + $this->assertSame(2, $attempts, 'The failed event is retried, not dropped'); + } + + public function testDrainsABacklogInBatches(): void + { + foreach (\range(1, 10) as $i) { + $this->producer->produce('event-' . $i); + } + + $consumer = $this->consumer(batch: 4); + + $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(2, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + } + + public function testTipStartDoesNotAnnounceTheBacklog(): void + { + $this->producer->produce('old-1'); + $this->producer->produce('old-2'); + + $consumer = $this->consumer('notifier', start: Consumer::START_TIP); + + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertNull($this->cursor->load($this->name, 'notifier'), 'Skipping the backlog is not progress to commit'); + } + + /** + * The tip is pinned when the poll starts, so an event landing while the + * poll waits is delivered β€” only the backlog is skipped. + */ + public function testTipStartDeliversWhatLandsMidPoll(): void + { + $store = new MidPollStore($this->name); + (new Producer($store, 'urn:test'))->produce('old'); + + $consumer = $this->consumer('notifier', timeout: 5_000, start: Consumer::START_TIP, store: $store); + + $seen = []; + $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }); + + $this->assertSame(1, $count); + $this->assertSame(['landed'], $seen, 'The backlog is skipped; the mid-wait event is not'); + $this->assertNotNull($this->cursor->load($this->name, 'notifier'), 'Handling the event saves the position'); + } + + public function testTipStartOnAnEmptyFeedDeliversWhatLandsMidWait(): void + { + $consumer = $this->consumer('notifier', timeout: 5_000, start: Consumer::START_TIP, store: new MidPollStore($this->name)); + + $seen = []; + $consumer->consume(function (CloudEvent $event) use (&$seen): void { + $seen[] = $event->type; + }); + + $this->assertSame(['landed'], $seen); + } + + public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void + { + $consumer = $this->consumer('notifier', timeout: 600, start: Consumer::START_TIP); + + $started = \microtime(true); + + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started, 'Must actually wait'); + } + + public function testAStoredCursorBeatsTipStart(): void + { + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->cursor->save($this->name, 'invalidator', $first); + + $consumer = $this->consumer(start: Consumer::START_TIP); + + $this->assertSame(['b'], $this->drain($consumer), 'A restart must not skip the gap'); + } + + public function testResetWithTipStartResumesFromNow(): void + { + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + $this->cursor->save($this->name, 'invalidator', $first); + + $consumer = $this->consumer(start: Consumer::START_TIP); + + $this->assertSame(['b'], $this->drain($consumer), 'The stored position still wins before the reset'); + + $consumer->reset(); + + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null), 'After reset, the backlog is forgotten'); + } + + public function testResetReplaysEverythingStillRetained(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $consumer = $this->consumer(); + $consumer->consume(fn (CloudEvent $event) => null); + + $consumer->reset(); + + $this->assertNull($consumer->position()); + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + $this->assertSame(['a', 'b'], $this->drain($consumer)); + } + + public function testSeekPositionsTheNextRunStrictlyAfterTheGivenId(): void + { + $this->producer->produce('a'); + $second = $this->producer->produce('b'); + $this->producer->produce('c'); + + $consumer = $this->consumer(); + $consumer->seek($second); + + $this->assertSame($second, $consumer->position(), 'The seeked id is the position until something is handled'); + $this->assertSame(['c'], $this->drain($consumer)); + } + + /** + * A seek is persisted, not just remembered: a fresh Consumer sharing the + * cursor store and the name β€” a restart β€” resumes from it. + */ + public function testASeekSurvivesARestart(): void + { + $this->producer->produce('a'); + $second = $this->producer->produce('b'); + $this->producer->produce('c'); + + $this->consumer()->seek($second); + + $this->assertSame(['c'], $this->drain($this->consumer())); + } + + /** + * The operational escape hatch seek() exists for: a handler that keeps + * failing blocks the feed by design, and stepping past it is a deliberate + * seek to the failing event's own id. + */ + public function testSeekingToAPoisonEventsIdUnblocksTheConsumer(): void + { + $this->producer->produce('poison'); + $this->producer->produce('after'); + + $consumer = $this->consumer(); + $poison = null; + + $handler = function (CloudEvent $event) use (&$poison): void { + if ($event->type === 'poison') { + $poison = $event->id; + + throw new \RuntimeException('cannot handle this one'); + } + }; + + try { + $consumer->consume($handler); + $this->fail('The poison event should have blocked the run'); + } catch (\RuntimeException) { + // Expected: the feed is now blocked at the poison event. + } + + $this->assertNotNull($poison); + $consumer->seek($poison); + + $this->assertSame(['after'], $this->drain($consumer), 'The poison event is stepped over, nothing behind it is lost'); + } + + /** + * A hand-made move decided during a run is the newer decision, so the run + * must not save its own progress over it on the way out. + */ + public function testASeekMadeInsideAHandlerIsNotOverwritten(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + $third = $this->producer->produce('c'); + + $consumer = $this->consumer(batch: 1); + $consumer->consume(function (CloudEvent $event) use ($consumer, $third): void { + $consumer->seek($third); + }); + + $this->assertSame($third, $consumer->position()); + $this->assertSame($third, $this->cursor->load($this->name, 'invalidator')); + $this->assertSame([], $this->drain($consumer), 'The run resumes after the seeked id, not after the handled one'); + } + + public function testAResetMadeInsideAHandlerIsNotOverwritten(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $consumer = $this->consumer(); + $consumer->consume(function (CloudEvent $event) use ($consumer): void { + if ($event->type === 'b') { + $consumer->reset(); + } + }); + + $this->assertNull($consumer->position()); + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + $this->assertSame(['a', 'b'], $this->drain($consumer), 'The reset stands, so everything retained replays'); + } + + /** + * @dataProvider notPositions + */ + public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void + { + $first = $this->producer->produce('a'); + $this->cursor->save($this->name, 'invalidator', $first); + + $consumer = $this->consumer(); + + try { + $consumer->seek($id); + $this->fail('The id should have been rejected'); + } catch (Invalid) { + // Expected. + } + + $this->assertSame($first, $this->cursor->load($this->name, 'invalidator'), 'A rejected seek leaves the stored position untouched'); + $this->assertSame($first, $consumer->position()); + } + + /** + * @return array + */ + public static function notPositions(): array + { + return [ + 'empty' => [''], + 'not an id' => ['abc'], + 'too many parts' => ['1-2-3'], + 'the tip sentinel' => ['$'], + ]; + } + + public function testConsumersOfTheSameFeedTrackSeparatePositions(): void + { + $this->producer->produce('a'); + + $one = $this->consumer('one'); + $two = $this->consumer('two'); + + $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); + $this->assertSame(1, $two->consume(fn (CloudEvent $event) => null), 'The second consumer has its own position'); + $this->assertSame(0, $one->consume(fn (CloudEvent $event) => null)); + } + + /** + * A cursor store serves every feed a service consumes, so the feed's name + * is part of the key: the same consumer name on another feed is another + * position entirely. + */ + public function testTheSameConsumerNameOnAnotherFeedTracksItsOwnPosition(): void + { + $other = new MemoryStore($this->name . '-other'); + (new Producer($other, 'urn:test'))->produce('other-a'); + + $this->producer->produce('a'); + + $this->assertSame(['a'], $this->drain($this->consumer())); + $this->assertSame(['other-a'], $this->drain($this->consumer(store: $other)), 'Draining one feed must not advance the other'); + } + + /** + * The boundary the design draws rather than an accident: a name is one + * logical reader, so two processes behind one name split the feed instead + * of both seeing it. + */ + public function testTwoConsumersSharingANameSplitTheFeed(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $one = $this->consumer(batch: 1); + $two = $this->consumer(batch: 1); + + $this->assertSame(['a'], $this->drain($one)); + $this->assertSame(['b'], $this->drain($two), 'The second picks up after the first, it does not see a of its own'); + } + + /** + * And the cost of sharing a name, which no amount of coordination inside a + * single process can remove: each save is last-writer-wins, so a replica + * holding an older position drags the shared one backwards when it saves. + * At-least-once makes that a replay rather than a loss β€” but it is why + * every consumer gets its own name. + */ + public function testAStaleConsumerSharingANameDragsThePositionBackwards(): void + { + $first = $this->producer->produce('a'); + $second = $this->producer->produce('b'); + + $stale = $this->consumer(batch: 1); + $this->assertNull($stale->position(), 'Reads the shared position before the other replica moves it'); + + $ahead = $this->consumer(); + $this->assertSame(['a', 'b'], $this->drain($ahead)); + $this->assertSame($second, $this->cursor->load($this->name, 'invalidator')); + + $this->assertSame(['a'], $this->drain($stale), 'The stale replica polls from where it thought it was'); + $this->assertSame($first, $this->cursor->load($this->name, 'invalidator'), 'Its save wins, so the shared position regresses'); + } + + public function testPositionIsNullBeforeTheFirstRun(): void + { + $this->assertNull($this->consumer()->position()); + } + + public function testRejectsAnEmptyConsumerName(): void + { + $this->expectException(Invalid::class); + + $this->consumer(''); + } + + public function testExposesItsName(): void + { + $this->assertSame('invalidator', $this->consumer()->getName()); + } +} diff --git a/tests/Feed/Consumer/CacheTest.php b/tests/Feed/Consumer/CacheTest.php new file mode 100644 index 0000000..26fb38c --- /dev/null +++ b/tests/Feed/Consumer/CacheTest.php @@ -0,0 +1,12 @@ +endpoint = new FeedServer(new Server($store)); + } + + public function testTheProducerCachesFullBatchesAndNothingElse(): void + { + foreach (\range(1, 5) as $i) { + $this->producer->produce('event-' . $i); + } + + $consumer = $this->consumer(batch: 2); + + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->consume(fn (CloudEvent $event) => null); + + $this->assertSame([ + 'private, max-age=31536000', // 2 of 2 β€” settled history + 'private, max-age=31536000', // 2 of 2 β€” settled history + 'no-store', // 1 of 2 β€” the live end, will grow + 'no-store', // 0 of 2 β€” caught up + ], $this->endpoint->recorder->cacheControl()); + } + + /** + * The tip sentinel crosses the wire as `lastEventId=$` and the producer + * resolves it inside the held request; once something is handled the + * position saves as a real id, so the sentinel never appears again. + */ + public function testTipStartSendsTheSentinelOnTheWireOnceOnly(): void + { + $store = new MidPollStore($this->name); + $producer = new Producer($store, 'urn:test'); + $producer->produce('old'); + + $consumer = $this->consumer('notifier', timeout: 5_000, start: Consumer::START_TIP, store: $store); + + $this->assertSame(['landed'], $this->drain($consumer), 'The backlog is skipped; the mid-wait event is not'); + $this->assertMatchesRegularExpression('/lastEventId=(%24|\$)/', $this->endpoint->recorder->last()['uri']); + + $producer->produce('after'); + + $this->assertSame(['after'], $this->drain($consumer)); + $this->assertDoesNotMatchRegularExpression('/lastEventId=(%24|\$)/', $this->endpoint->recorder->last()['uri']); + } +} diff --git a/tests/Feed/Consumer/MemoryTest.php b/tests/Feed/Consumer/MemoryTest.php new file mode 100644 index 0000000..bc8d3cc --- /dev/null +++ b/tests/Feed/Consumer/MemoryTest.php @@ -0,0 +1,12 @@ +:cursor:` and the value the event id as a string β€” + * so positions carry across upgrades and an operator can answer "where is + * this consumer?" from a shell. + */ + public function testThePositionIsStoredWhereOperatorsExpectIt(): void + { + $this->producer->produce('a'); + $last = $this->producer->produce('b'); + + $this->drain($this->consumer()); + + $this->assertSame($last, $this->redis()->get('feed:' . $this->name . ':cursor:invalidator')); + } +} diff --git a/tests/Feed/E2E/RedisTest.php b/tests/Feed/E2E/RedisTest.php deleted file mode 100644 index bf79231..0000000 --- a/tests/Feed/E2E/RedisTest.php +++ /dev/null @@ -1,319 +0,0 @@ -redis = new \Redis(); - $this->redis->connect((string) (\getenv('REDIS_HOST') ?: 'redis'), (int) (\getenv('REDIS_PORT') ?: 6379)); - - // A fresh feed per test: these assert on positions, and a shared - // stream would leak them between tests. - $this->name = 'test-' . \bin2hex(\random_bytes(8)); - } - - protected function tearDown(): void - { - $this->redis->del('feed:' . $this->name); - - foreach ((array) $this->redis->keys('feed:' . $this->name . ':cursor:*') as $key) { - if (\is_string($key)) { - $this->redis->del($key); - } - } - - $this->redis->close(); - } - - private function server(int $maxSize = 100_000): Server - { - return new Server(new RedisStore($this->redis, $this->name, $maxSize)); - } - - /** @return list */ - private static function events(Batch $batch): array - { - return \array_values(\iterator_to_array($batch)); - } - - private function producer(int $maxSize = 100_000): Producer - { - return new Producer(new RedisStore($this->redis, $this->name, $maxSize), 'urn:test:e2e'); - } - - /** - * The two halves of one feed: what a producing service builds over a single - * store to produce into its feed and serve it. - * - * @return array{Producer, Server} - */ - private function serverAndProducer(int $maxSize = 100_000): array - { - $store = new RedisStore($this->redis, $this->name, $maxSize); - - return [new Producer($store, 'urn:test:e2e'), new Server($store)]; - } - - public function testAppendsAndReadsBack(): void - { - [$producer, $server] = $this->serverAndProducer(); - - $id = $producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); - - $events = self::events($server->read()); - - $this->assertCount(1, $events); - $this->assertSame($id, $events[0]->id); - $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); - $this->assertSame('example.com', $events[0]->subject); - $this->assertSame('urn:test:e2e', $events[0]->source); - $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); - $this->assertNotNull($events[0]->time); - } - - public function testStreamIdsMatchTheFormatPositionsAreParsedWith(): void - { - $id = $this->producer()->produce('test'); - - $this->assertTrue(Id::isValid($id), "Redis returned an id this library cannot page from: {$id}"); - } - - public function testIdsIncreaseAcrossRapidAppends(): void - { - [$producer, $server] = $this->serverAndProducer(); - - $ids = []; - for ($i = 0; $i < 100; $i++) { - $ids[] = $producer->produce('test'); - } - - $this->assertSame($ids, \array_unique($ids)); - - for ($i = 1; $i < \count($ids); $i++) { - $this->assertGreaterThan(Id::decode($ids[$i - 1]), Id::decode($ids[$i])); - } - } - - /** - * The reason positions are advanced arithmetically instead of with Redis' - * `(`-exclusive range syntax: this has to hold on every server and proxy - * that speaks the Redis 5 stream API, not only the ones that added it. - */ - public function testReadsStrictlyAfterAPosition(): void - { - [$producer, $server] = $this->serverAndProducer(); - - $first = $producer->produce('a'); - $second = $producer->produce('b'); - - $events = self::events($server->read($first)); - - $this->assertCount(1, $events); - $this->assertSame($second, $events[0]->id); - $this->assertCount(0, $server->read($second)); - } - - public function testExtensionsAndDataschemaSurviveTheRoundTrip(): void - { - $this->producer()->publish(new CloudEvent( - id: '', - type: 'test', - source: '', - dataschema: 'https://example.com/schema.json', - extensions: ['traceparent' => '00-abc-def-01'], - )); - - $event = self::events($this->server()->read())[0]; - - $this->assertSame('https://example.com/schema.json', $event->dataschema); - $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); - } - - public function testAnAbsentSubjectStaysAbsent(): void - { - $this->producer()->produce('test'); - - $this->assertNull(self::events($this->server()->read())[0]->subject); - } - - public function testAScalarPayloadSurvivesTheRoundTrip(): void - { - $this->producer()->produce('test', 'a string'); - - $this->assertSame('a string', self::events($this->server()->read())[0]->data); - } - - public function testNestedPayloadsSurviveTheRoundTrip(): void - { - $data = [ - 'tags' => ['domain' => 'example.com', 'project' => 'p1'], - 'flags' => ['isAppwriteNetwork' => true], - 'count' => 42, - 'unicode' => 'ΓΌnΓ―cΓΈde βœ“', - ]; - - $this->producer()->produce('test', $data); - - $this->assertSame($data, self::events($this->server()->read())[0]->data); - } - - public function testHonoursTheLimit(): void - { - [$producer, $server] = $this->serverAndProducer(); - - foreach (\range(1, 10) as $i) { - $producer->produce('test'); - } - - $this->assertCount(3, $server->read(null, 3)); - } - - public function testRejectsAPositionThatIsNotAFeedId(): void - { - $this->expectException(Invalid::class); - - $this->server()->read('not-a-position'); - } - - /** - * Trimming is approximate, so this asserts the property a consumer relies - * on β€” that a position below the horizon still reads β€” rather than an - * exact retained count. - */ - public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void - { - [$producer, $server] = $this->serverAndProducer(maxSize: 10); - - $first = $producer->produce('first'); - - foreach (\range(1, 500) as $i) { - $producer->produce('event-' . $i); - } - - $events = $server->read($first); - - $this->assertFalse($events->isEmpty(), 'A consumer that fell behind must still get what is retained'); - $this->assertLessThan(500, $this->redis->xLen('feed:' . $this->name), 'The feed must be trimmed'); - } - - public function testLongPollingReturnsAsSoonAsTheFeedHasSomething(): void - { - [$producer, $server] = $this->serverAndProducer(); - $producer->produce('a'); - - $started = \microtime(true); - $events = $server->poll(null, 10, 3000); - - $this->assertCount(1, $events); - $this->assertLessThan(1, \microtime(true) - $started); - } - - public function testLongPollingGivesUpAtTheTimeout(): void - { - $started = \microtime(true); - $events = $this->server()->poll(null, 10, 700); - - $this->assertCount(0, $events); - $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started); - } - - /** - * The tip is found with XREVRANGE, so this needs a real Redis: an empty - * stream has no tip, and the sentinel reads nothing that already exists. - */ - public function testTheTipSentinelSkipsTheBacklog(): void - { - [$producer, $server] = $this->serverAndProducer(); - - $this->assertNull($server->tip(), 'An empty feed has no tip'); - - $producer->produce('a'); - $last = $producer->produce('b'); - - $this->assertSame($last, $server->tip()); - $this->assertCount(0, $server->read('$')); - } - - public function testConsumesThroughAPersistedCursor(): void - { - $store = new RedisStore($this->redis, $this->name); - $producer = new Producer($store, 'urn:test:e2e'); - $cursor = new RedisCursor($this->redis); - - $producer->produce('a'); - $last = $producer->produce('b'); - - $seen = []; - $handler = function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - }; - - $this->assertSame(2, (new Consumer($store, $cursor, 'invalidator'))->consume($handler)); - $this->assertSame($last, $cursor->load($this->name, 'invalidator')); - - // A second Consumer stands in for a restart: it has no in-memory - // position, so it has to pick the stored one up to avoid replaying. - $this->assertSame(0, (new Consumer($store, $cursor, 'invalidator'))->consume($handler)); - $this->assertSame(['a', 'b'], $seen); - } - - public function testASecondConsumerOfTheSameFeedGetsItsOwnPosition(): void - { - $store = new RedisStore($this->redis, $this->name); - $producer = new Producer($store, 'urn:test:e2e'); - $cursor = new RedisCursor($this->redis); - - $producer->produce('a'); - - $this->assertSame(1, (new Consumer($store, $cursor, 'one'))->consume(fn (CloudEvent $e) => null)); - $this->assertSame(1, (new Consumer($store, $cursor, 'two'))->consume(fn (CloudEvent $e) => null)); - } - - public function testResetReplaysTheRetainedFeed(): void - { - $store = new RedisStore($this->redis, $this->name); - $producer = new Producer($store, 'urn:test:e2e'); - $cursor = new RedisCursor($this->redis); - - $producer->produce('a'); - $producer->produce('b'); - - $consumer = new Consumer($store, $cursor, 'invalidator'); - $consumer->consume(fn (CloudEvent $e) => null); - $consumer->reset(); - - $this->assertNull($cursor->load($this->name, 'invalidator')); - $this->assertSame(2, (new Consumer($store, $cursor, 'invalidator'))->consume(fn (CloudEvent $e) => null)); - } - - public function testCursorsAreStoredUnderTheFeedTheyBelongTo(): void - { - (new RedisCursor($this->redis))->save($this->name, 'invalidator', '1-0'); - - $this->assertSame('1-0', $this->redis->get('feed:' . $this->name . ':cursor:invalidator')); - } -} diff --git a/tests/Feed/Producer/Base.php b/tests/Feed/Producer/Base.php new file mode 100644 index 0000000..02d9bd7 --- /dev/null +++ b/tests/Feed/Producer/Base.php @@ -0,0 +1,340 @@ +name = 'test-' . \bin2hex(\random_bytes(8)); + $this->store = $this->store($this->name); + $this->producer = new Producer($this->store, 'urn:test'); + } + + /** @return list */ + protected function events(): array + { + return $this->store->read(null, 1000); + } + + public function testProduceReturnsAPosition(): void + { + $id = $this->producer->produce('io.appwrite.edge.invalidate', ['tags' => ['project' => 'p1']]); + + $this->assertTrue(Id::isValid($id), "The backend returned an id this library cannot page from: {$id}"); + } + + public function testRoundTripsAnEvent(): void + { + $id = $this->producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + + $events = $this->events(); + + $this->assertCount(1, $events); + $this->assertSame($id, $events[0]->id); + $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); + $this->assertSame('example.com', $events[0]->subject); + $this->assertSame('urn:test', $events[0]->source); + $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); + $this->assertNotNull($events[0]->time); + } + + public function testStampsTheSourceAndTime(): void + { + $this->producer->produce('test'); + + $event = $this->events()[0]; + + $this->assertSame('urn:test', $event->source); + $this->assertNotNull($event->time); + $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', $event->time); + } + + /** + * Recording it when produced rather than when read keeps it correct for a + * feed read back somewhere other than where it was written. + */ + public function testKeepsTheSourceOfTheProducerThatProduced(): void + { + (new Producer($this->store, 'urn:appwrite:cloud:fra'))->produce('test'); + (new Producer($this->store, 'urn:appwrite:cloud:nyc'))->produce('test'); + + $events = $this->events(); + + $this->assertSame('urn:appwrite:cloud:fra', $events[0]->source); + $this->assertSame('urn:appwrite:cloud:nyc', $events[1]->source); + } + + public function testPublishStampsAPreparedEvent(): void + { + $id = $this->producer->publish(new CloudEvent(id: 'ignored', type: 'test', source: 'ignored', data: ['a' => 'b'], subject: 's')); + + $event = $this->events()[0]; + + $this->assertSame($id, $event->id); + $this->assertNotSame('ignored', $event->id, 'The backend assigns the position, not the caller'); + $this->assertSame('urn:test', $event->source); + $this->assertSame(['a' => 'b'], $event->data); + } + + public function testPublishKeepsATimeTheCallerSet(): void + { + $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', time: '2020-01-01T00:00:00.000Z')); + + $this->assertSame('2020-01-01T00:00:00.000Z', $this->events()[0]->time); + } + + public function testEventsComeBackOldestFirst(): void + { + foreach (['a', 'b', 'c'] as $type) { + $this->producer->produce($type); + } + + $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $this->events())); + } + + public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void + { + $ids = []; + for ($i = 0; $i < 50; $i++) { + $ids[] = $this->producer->produce('test'); + } + + $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); + + for ($i = 1; $i < \count($ids); $i++) { + $this->assertGreaterThan(Id::decode($ids[$i - 1]), Id::decode($ids[$i]), 'Positions must increase'); + } + } + + /** + * @return array + */ + public static function payloads(): array + { + return [ + 'map' => [['tags' => ['domain' => 'example.com']]], + 'list' => [['a', 'b', 'c']], + 'nested list' => [[['x' => 1], ['x' => 2]]], + 'string' => ['a string'], + 'number' => [42], + 'float' => [1.5], + 'boolean' => [true], + 'null' => [null], + 'empty' => [[]], + 'unicode' => [['unicode' => 'ΓΌnΓ―cΓΈde βœ“']], + ]; + } + + /** + * The JSON event format leaves `data` unrestricted, so a list or a scalar + * has to survive as itself β€” a list must not come back as a map. + * + * @dataProvider payloads + */ + public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void + { + $this->producer->produce('test', $data); + + $this->assertSame($data, $this->events()[0]->data); + } + + /** + * A producer that attaches a `traceparent` means it to reach the consumer. + * Stamping the event on publish rebuilds it, and storing it flattens it, so + * either step could quietly drop an attribute this library does not model. + */ + public function testExtensionAttributesSurviveAppendAndRead(): void + { + $this->producer->publish(new CloudEvent( + id: '', + type: 'test', + source: '', + extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2], + )); + + $event = $this->events()[0]; + + $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); + $this->assertSame(2, $event->extensions['retrycount']); + } + + /** + * An extension name of only digits is legal β€” the spec allows `[a-z0-9]+` β€” + * and PHP stores such a name as an integer key. Anything that merges the + * extensions back in with a spread, or with `array_merge()`, renumbers that + * key and silently loses the attribute. + */ + public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void + { + $this->producer->publish(new CloudEvent( + id: '', + type: 'test', + source: '', + // @phpstan-ignore argument.type ('123' is an integer key in PHP) + extensions: ['123' => 'digits', 'trace' => 'ok'], + )); + + $event = $this->events()[0]; + + // @phpstan-ignore offsetAccess.notFound + $this->assertSame('digits', $event->extensions['123']); + $this->assertSame('ok', $event->extensions['trace']); + } + + public function testDataschemaSurvivesAppendAndRead(): void + { + $this->producer->publish(new CloudEvent( + id: '', + type: 'test', + source: '', + dataschema: 'https://example.com/schema.json', + )); + + $this->assertSame('https://example.com/schema.json', $this->events()[0]->dataschema); + } + + /** + * CloudEvents models an absent subject as null rather than an empty string, + * so a caller checking for one must check for null. + */ + public function testAnEventWithNoSubjectHasANullSubject(): void + { + $this->producer->produce('test'); + + $this->assertNull($this->events()[0]->subject); + } + + public function testASubjectSurvivesAppendAndRead(): void + { + $this->producer->produce('test', [], 'example.com'); + + $this->assertSame('example.com', $this->events()[0]->subject); + } + + public function testRejectsAnEmptyEventType(): void + { + $this->expectException(Invalid::class); + + $this->producer->produce(''); + } + + public function testRejectsAPayloadThatCannotBeEncoded(): void + { + $this->expectException(Invalid::class); + + $this->producer->produce('test', ['resource' => \fopen('php://memory', 'r')]); + } + + /** + * CloudEvents requires a source, and an event stamped with an empty one is + * an event no consumer can attribute. + */ + public function testRejectsAnEmptySource(): void + { + $this->expectException(Invalid::class); + + new Producer($this->store, ''); + } + + /** + * Trimming may be approximate (Redis trims to node boundaries), so this + * asserts the property every adapter owes rather than an exact count: the + * feed stays bounded, the oldest events go first, and the newest survives. + */ + public function testRetentionIsBoundedAndDropsTheOldestFirst(): void + { + $store = $this->store($this->name, maxSize: 10); + $producer = new Producer($store, 'urn:test'); + + $producer->produce('first'); + + foreach (\range(1, 299) as $i) { + $producer->produce('event-' . $i); + } + + $types = \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 1000)); + + $this->assertLessThan(300, \count($types), 'The feed must be trimmed'); + $this->assertNotContains('first', $types, 'The oldest event goes first'); + $this->assertSame('event-299', \end($types), 'The newest event is retained'); + } + + public function testRejectsAnEmptyFeedName(): void + { + $this->expectException(Invalid::class); + + $this->store(''); + } + + /** + * @dataProvider notRetentions + */ + public function testRejectsARetentionThatKeepsNothing(int $maxSize): void + { + $this->expectException(Invalid::class); + + $this->store($this->name, maxSize: $maxSize); + } + + /** + * @return array + */ + public static function notRetentions(): array + { + return [ + 'zero' => [0], + 'negative' => [-5], + ]; + } + + /** + * @dataProvider notIntervals + */ + public function testRejectsAPollIntervalBelowAMillisecond(int $pollInterval): void + { + $this->expectException(Invalid::class); + + $this->store($this->name, pollInterval: $pollInterval); + } + + /** + * @return array + */ + public static function notIntervals(): array + { + return [ + 'zero' => [0], + 'negative' => [-5], + ]; + } +} diff --git a/tests/Feed/Producer/CacheTest.php b/tests/Feed/Producer/CacheTest.php new file mode 100644 index 0000000..e321798 --- /dev/null +++ b/tests/Feed/Producer/CacheTest.php @@ -0,0 +1,69 @@ +store($this->name, maxSize: 3); + $producer = new Producer($store, 'urn:test'); + + foreach (['a', 'b', 'c', 'd', 'e'] as $type) { + $producer->produce($type); + } + + $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 10))); + } + + public function testAcceptsTheSmallestUsefulRetentionCap(): void + { + $store = $this->store($this->name, maxSize: 1); + $producer = new Producer($store, 'urn:test'); + + $producer->produce('a'); + $producer->produce('b'); + + $events = $store->read(null, 10); + + $this->assertCount(1, $events); + $this->assertSame('b', $events[0]->type); + } + + /** + * The property that justifies the adapter: the feed lives in the cache, not + * in the store object, so a second store over the same cache β€” another + * request handling the same feed β€” reads what the first appended. + */ + public function testTheFeedSurvivesTheStoreThatWroteIt(): void + { + $id = $this->producer->produce('a'); + + $events = (new CacheStore($this->cache(), $this->name))->read(null, 10); + + $this->assertCount(1, $events); + $this->assertSame($id, $events[0]->id); + } + + /** + * A cache is allowed to forget, and it may also hold a foreign value under + * the feed's key. Both read as an empty feed β€” a replay, never a fault. + */ + public function testAForeignValueUnderTheFeedsKeyReadsAsEmpty(): void + { + $this->cache()->save('feed:' . $this->name, ['not' => 'a feed']); + + $this->assertCount(0, $this->store->read(null, 10)); + $this->assertNull($this->store->tip()); + } +} diff --git a/tests/Feed/Producer/MemoryTest.php b/tests/Feed/Producer/MemoryTest.php new file mode 100644 index 0000000..1872fbb --- /dev/null +++ b/tests/Feed/Producer/MemoryTest.php @@ -0,0 +1,41 @@ +store($this->name, maxSize: 3); + $producer = new Producer($store, 'urn:test'); + + foreach (['a', 'b', 'c', 'd', 'e'] as $type) { + $producer->produce($type); + } + + $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 10))); + } + + public function testAcceptsTheSmallestUsefulRetentionCap(): void + { + $store = $this->store($this->name, maxSize: 1); + $producer = new Producer($store, 'urn:test'); + + $producer->produce('a'); + $producer->produce('b'); + + $events = $store->read(null, 10); + + $this->assertCount(1, $events); + $this->assertSame('b', $events[0]->type); + } +} diff --git a/tests/Feed/Producer/PoolTest.php b/tests/Feed/Producer/PoolTest.php new file mode 100644 index 0000000..9866c59 --- /dev/null +++ b/tests/Feed/Producer/PoolTest.php @@ -0,0 +1,12 @@ +producer->produce('a'); + $this->producer->produce('b'); + + $this->assertSame(2, $this->redis()->xLen('feed:' . $this->name)); + } + + /** Trimming must happen on the server, not only in what read() returns. */ + public function testTheStreamItselfIsTrimmed(): void + { + $store = $this->store($this->name, maxSize: 10); + $producer = new Producer($store, 'urn:test'); + + foreach (\range(1, 300) as $i) { + $producer->produce('event-' . $i); + } + + $this->assertLessThan(300, $this->redis()->xLen('feed:' . $this->name), 'The stream must be trimmed'); + } +} diff --git a/tests/Feed/Server/Base.php b/tests/Feed/Server/Base.php new file mode 100644 index 0000000..b7800cb --- /dev/null +++ b/tests/Feed/Server/Base.php @@ -0,0 +1,310 @@ +name = 'test-' . \bin2hex(\random_bytes(8)); + $this->store = $this->store($this->name); + $this->producer = new Producer($this->store, 'urn:test'); + $this->server = new Server($this->store); + } + + /** @return list */ + protected static function events(Batch $batch): array + { + return \array_values(\iterator_to_array($batch)); + } + + /** @return list */ + protected static function types(Batch $batch): array + { + return \array_map(fn (CloudEvent $e): string => $e->type, self::events($batch)); + } + + public function testReadsBackWhatWasAppended(): void + { + $id = $this->producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); + + $events = self::events($this->server->read()); + + $this->assertCount(1, $events); + $this->assertSame($id, $events[0]->id); + $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); + $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); + } + + /** + * The reason positions advance arithmetically instead of with Redis' + * `(`-exclusive range syntax: "strictly after" has to hold on every + * backend, not only the ones whose range API can express it. + */ + public function testReadsStrictlyAfterTheGivenPosition(): void + { + $first = $this->producer->produce('a'); + $second = $this->producer->produce('b'); + + $events = self::events($this->server->read($first)); + + $this->assertCount(1, $events); + $this->assertSame($second, $events[0]->id); + } + + public function testReadFromTheLastEventIsEmpty(): void + { + $this->producer->produce('a'); + $last = $this->producer->produce('b'); + + $this->assertCount(0, $this->server->read($last)); + } + + public function testNullPositionReadsFromTheOldestRetainedEvent(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->assertSame(['a', 'b'], self::types($this->server->read(null))); + } + + public function testHonoursTheLimit(): void + { + foreach (\range(1, 10) as $i) { + $this->producer->produce('test'); + } + + $this->assertCount(3, $this->server->read(null, 3)); + } + + /** + * `limit` arrives from a consumer, so it is clamped rather than rejected β€” + * failing the read would stall a feed over something the producer can just + * decide. + */ + public function testClampsTheLimitToTheMaximum(): void + { + $this->producer->produce('test'); + + $this->assertCount(1, $this->server->read(null, Readable::MAX_BATCH * 10)); + $this->assertCount(1, $this->server->read(null, 0)); + $this->assertCount(1, $this->server->read(null, -5)); + } + + public function testRejectsAPositionThatIsNotAFeedId(): void + { + $this->expectException(Invalid::class); + + $this->server->read('not-a-position'); + } + + public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void + { + $this->producer->produce('test'); + + $started = \microtime(true); + $events = $this->server->poll(null, 10, 2000); + + $this->assertCount(1, $events); + $this->assertLessThan(1, \microtime(true) - $started); + } + + public function testPollGivesUpAtTheTimeoutWithAnEmptyBatch(): void + { + $started = \microtime(true); + $events = $this->server->poll(null, 10, 600); + $elapsed = \microtime(true) - $started; + + $this->assertCount(0, $events); + $this->assertGreaterThanOrEqual(0.4, $elapsed, 'Must actually wait'); + $this->assertLessThan(3.0, $elapsed, 'Must not wait far past the timeout'); + } + + public function testPollWithoutATimeoutIsAPlainRead(): void + { + $started = \microtime(true); + + $this->assertCount(0, $this->server->poll()); + $this->assertLessThan(0.4, \microtime(true) - $started); + } + + /** + * The overshoot fix: the poll loop must sleep the remaining time when that + * is less than the interval, not a full interval past the deadline. + */ + public function testPollHonoursATimeoutShorterThanThePollInterval(): void + { + $server = new Server($this->store($this->name, pollInterval: 500)); + + $started = \microtime(true); + $events = $server->poll(null, 10, 100); + $elapsed = \microtime(true) - $started; + + $this->assertCount(0, $events); + $this->assertGreaterThanOrEqual(0.08, $elapsed, 'Must actually wait out the timeout'); + $this->assertLessThan(0.3, $elapsed, 'Must not sleep a full interval past the deadline'); + } + + public function testTipIsTheNewestEventsId(): void + { + $this->assertNull($this->server->tip(), 'An empty feed has no tip'); + + $this->producer->produce('a'); + $last = $this->producer->produce('b'); + + $this->assertSame($last, $this->server->tip()); + } + + public function testReadingFromTheTipSentinelSkipsTheBacklog(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->assertCount(0, $this->server->read(Readable::TIP)); + } + + /** + * The one case a caller has to design for: a consumer that fell behind the + * trim horizon gets what is left, not an error and not a gap it can detect. + */ + public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void + { + $store = $this->store($this->name, maxSize: 10); + $producer = new Producer($store, 'urn:test'); + $server = new Server($store); + + $first = $producer->produce('first'); + + foreach (\range(1, 300) as $i) { + $producer->produce('event-' . $i); + } + + $events = $server->read($first); + + $this->assertFalse($events->isEmpty(), 'A consumer that fell behind must still get what is retained'); + $this->assertLessThan(300, \count($events), 'The feed must be trimmed'); + } + + public function testExposesTheFeedItReads(): void + { + $this->assertSame($this->name, $this->server->getName()); + } + + public function testServeAppliesTheDefaultsWhenNoParametersArrive(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->assertCount(2, $this->server->serve([])); + } + + public function testServeCoercesTheStringValuesARouteHands(): void + { + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + $this->producer->produce('c'); + + $batch = $this->server->serve([ + 'lastEventId' => $first, + 'limit' => '1', + 'timeout' => '0', + ]); + + $this->assertSame(['b'], self::types($batch)); + } + + public function testServeTreatsAnEmptyLastEventIdAsAbsent(): void + { + $this->producer->produce('a'); + + $this->assertCount(1, $this->server->serve(['lastEventId' => ''])); + } + + public function testServeRejectsALastEventIdThatIsNotAPosition(): void + { + $this->expectException(Invalid::class); + + $this->server->serve(['lastEventId' => 'not-a-position']); + } + + public function testServeLetsTheTipSentinelThrough(): void + { + $this->producer->produce('a'); + + $this->assertCount(0, $this->server->serve(['lastEventId' => Readable::TIP])); + } + + public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->assertCount(2, $this->server->serve(['limit' => 'lots', 'timeout' => 'soon'])); + } + + /** + * The trap the old API set: a route that passed the raw request limit to + * the caching rule while the read was clamped to less would mark a full + * batch `no-store` β€” or worse. The batch carries the limit it was actually + * built with, so a full batch under an oversized request is still + * recognized as settled history. + */ + public function testAnOversizedLimitStillYieldsAnHonestCacheControl(): void + { + foreach (\range(1, Readable::MAX_BATCH) as $i) { + $this->producer->produce('event-' . $i); + } + + $batch = $this->server->serve(['limit' => '5000']); + + $this->assertCount(Readable::MAX_BATCH, $batch); + $this->assertSame('public, max-age=31536000', $batch->cacheControl(public: true)); + } + + public function testAShortBatchIsNotCacheable(): void + { + $this->producer->produce('a'); + + $this->assertSame('no-store', $this->server->serve([])->cacheControl()); + } + + public function testAnEmptyBatchIsNotCacheable(): void + { + $this->assertSame('no-store', $this->server->serve([])->cacheControl()); + } +} diff --git a/tests/Feed/Server/CacheTest.php b/tests/Feed/Server/CacheTest.php new file mode 100644 index 0000000..2c6e636 --- /dev/null +++ b/tests/Feed/Server/CacheTest.php @@ -0,0 +1,12 @@ +name)); + + $started = \microtime(true); + $events = $server->poll(null, 10, 5_000); + + $this->assertCount(1, $events); + $this->assertLessThan(3, \microtime(true) - $started, 'Must return on the event, not the timeout'); + } + + public function testAShorterPollIntervalDeliversAMidPollEventSooner(): void + { + $server = new Server(new MidPollStore($this->name, pollInterval: 20)); + + $started = \microtime(true); + $events = $server->poll(null, 10, 5_000); + $elapsed = \microtime(true) - $started; + + $this->assertCount(1, $events); + $this->assertLessThan(0.4, $elapsed, 'A 20ms interval must beat the default 500ms floor'); + } +} diff --git a/tests/Feed/Server/PoolTest.php b/tests/Feed/Server/PoolTest.php new file mode 100644 index 0000000..19a8013 --- /dev/null +++ b/tests/Feed/Server/PoolTest.php @@ -0,0 +1,12 @@ +cache ??= new UtopiaCache(new CacheMemory()); + } + + protected function store(string $name, int $maxSize = 100_000, int $pollInterval = 500): Store&Appendable + { + return new CacheStore($this->cache(), $name, $maxSize, pollInterval: $pollInterval); + } + + protected function cursor(): Cursor + { + return new CacheCursor($this->cache()); + } +} diff --git a/tests/Feed/Support/UsesMemory.php b/tests/Feed/Support/UsesMemory.php new file mode 100644 index 0000000..3712ec2 --- /dev/null +++ b/tests/Feed/Support/UsesMemory.php @@ -0,0 +1,28 @@ +|null */ + private ?UtopiaPool $pool = null; + + /** @return UtopiaPool<\Redis|\RedisCluster> */ + protected function pool(): UtopiaPool + { + if ($this->pool === null) { + /** @var UtopiaPool<\Redis|\RedisCluster> $pool */ + $pool = new UtopiaPool(new Stack(), 'feed-tests', 4, static function (): \Redis { + $redis = new \Redis(); + $redis->connect((string) (\getenv('REDIS_HOST') ?: 'redis'), (int) (\getenv('REDIS_PORT') ?: 6379)); + + return $redis; + }); + + $this->pool = $pool; + } + + return $this->pool; + } + + protected function store(string $name, int $maxSize = 100_000, int $pollInterval = 500): Store&Appendable + { + return new PoolStore($this->pool(), $name, $maxSize, $pollInterval); + } + + protected function cursor(): Cursor + { + return new PoolCursor($this->pool()); + } + + protected function tearDown(): void + { + $this->pool()->use(function (\Redis|\RedisCluster $redis): void { + foreach ((array) $redis->keys('feed:' . $this->name . '*') as $key) { + if (\is_string($key)) { + $redis->del($key); + } + } + }); + } +} diff --git a/tests/Feed/Support/UsesRedis.php b/tests/Feed/Support/UsesRedis.php new file mode 100644 index 0000000..3dc5a37 --- /dev/null +++ b/tests/Feed/Support/UsesRedis.php @@ -0,0 +1,54 @@ +redis === null) { + $this->redis = new \Redis(); + $this->redis->connect((string) (\getenv('REDIS_HOST') ?: 'redis'), (int) (\getenv('REDIS_PORT') ?: 6379)); + } + + return $this->redis; + } + + protected function store(string $name, int $maxSize = 100_000, int $pollInterval = 500): Store&Appendable + { + return new RedisStore($this->redis(), $name, $maxSize, $pollInterval); + } + + protected function cursor(): Cursor + { + return new RedisCursor($this->redis()); + } + + protected function tearDown(): void + { + foreach ((array) $this->redis()->keys('feed:' . $this->name . '*') as $key) { + if (\is_string($key)) { + $this->redis()->del($key); + } + } + + $this->redis()->close(); + $this->redis = null; + } +} diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index f5beb1f..4c97eb0 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -5,19 +5,26 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Store\Memory as MemoryStore; +use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; use Utopia\Feed\Cursor\Memory as MemoryCursor; -use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Transport; +use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Producer; use Utopia\Feed\Readable; -use Utopia\Tests\Unit\Support\FailingCursor; -use Utopia\Tests\Unit\Support\FakeTransport; -use Utopia\Tests\Unit\Support\MidPollStore; - +use Utopia\Feed\Store\Memory as MemoryStore; +use Utopia\Feed\Store\None as NoneStore; +use Utopia\Tests\Support\FailingCursor; +use Utopia\Tests\Support\FakeTransport; + +/** + * The consumer behaviours that only a misbehaving collaborator can provoke β€” + * a cursor store that is down, a store that records what it was asked, a + * backend that was never configured. Everything a real adapter can exercise + * lives in {@see \Utopia\Tests\Consumer\Base} instead. + */ class ConsumerTest extends TestCase { private MemoryStore $store; @@ -33,69 +40,9 @@ protected function setUp(): void $this->cursor = new MemoryCursor(); } - private function consumer(?Cursor $cursor = null, int $batch = Consumer::BATCH): Consumer - { - return new Consumer($this->store, $cursor ?? $this->cursor, 'invalidator', batch: $batch); - } - - /** - * @param-out int $count - * @return list - */ - private function drain(Consumer $consumer, ?int &$count = null): array - { - $seen = []; - $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - }); - - return $seen; - } - - public function testHandlesEachEventAndAdvancesPastTheLastOne(): void - { - $this->producer->produce('a'); - $last = $this->producer->produce('b'); - - $consumer = $this->consumer(); - - $this->assertSame(['a', 'b'], $this->drain($consumer, $count)); - $this->assertSame(2, $count); - $this->assertSame($last, $this->cursor->load('edge', 'invalidator')); - $this->assertSame($last, $consumer->position()); - } - - public function testCaughtUpConsumerDoesNothing(): void + private function consumer(?Cursor $cursor = null): Consumer { - $this->producer->produce('a'); - - $consumer = $this->consumer(); - $consumer->consume(fn (CloudEvent $event) => null); - - $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); - } - - public function testResumesFromTheStoredPosition(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->cursor->save('edge', 'invalidator', $first); - - $this->assertSame(['b'], $this->drain($this->consumer())); - } - - /** - * A consumer that has never run starts at the oldest retained event, not - * at the tip β€” otherwise the first event a feed ever carries is the one - * event that is guaranteed to be dropped. - */ - public function testAConsumerWithNoPositionStartsAtTheOldestEventNotTheTip(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->assertSame(['a', 'b'], $this->drain($this->consumer())); + return new Consumer($this->store, $cursor ?? $this->cursor, 'invalidator'); } public function testReadsTheStoreOnceAndThenTracksThePositionInMemory(): void @@ -125,82 +72,6 @@ public function load(string $feed, string $consumer): ?string $this->assertSame(1, $cursor->loads); } - public function testStopsAtTheFirstFailureAndLeavesThePositionBeforeIt(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - $this->producer->produce('c'); - - $consumer = $this->consumer(); - $seen = []; - - try { - $consumer->consume(function (CloudEvent $event) use (&$seen): void { - if ($event->type === 'b') { - throw new \RuntimeException('nope'); - } - - $seen[] = $event->type; - }); - $this->fail('The handler failure should have been re-raised'); - } catch (\RuntimeException $error) { - $this->assertSame('nope', $error->getMessage()); - } - - $this->assertSame(['a'], $seen); - $this->assertSame($first, $this->cursor->load('edge', 'invalidator'), 'Progress before the failure is committed'); - } - - public function testRetriesTheFailedEventOnTheNextRun(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $consumer = $this->consumer(); - $attempts = 0; - - try { - $consumer->consume(function (CloudEvent $event) use (&$attempts): void { - if ($event->type === 'b') { - $attempts++; - throw new \RuntimeException('nope'); - } - }); - } catch (\RuntimeException) { - // Expected. - } - - $this->assertSame(['b'], $this->drain($consumer), 'The failed event comes back'); - $this->assertSame(1, $attempts); - } - - /** - * A failure on the very first event of a run commits nothing, so a store - * that was already empty stays empty rather than being written a position - * that stands for no completed work. - */ - public function testAFailureOnTheFirstEventCommitsNothing(): void - { - $this->producer->produce('a'); - - try { - $this->consumer()->consume(fn (CloudEvent $event) => throw new \RuntimeException('nope')); - } catch (\RuntimeException) { - // Expected. - } - - $this->assertNull($this->cursor->load('edge', 'invalidator')); - } - - public function testAHandlerThatAcceptsEverythingCountsEveryEvent(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - $this->producer->produce('c'); - - $this->assertSame(3, $this->consumer()->consume(fn (CloudEvent $event) => null)); - } - /** * The consumer clamps its own inputs: whatever the constructor was given, * a store is never asked for more than the protocol allows. @@ -228,20 +99,6 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array $this->assertSame(Readable::MAX_TIMEOUT, $store->timeout); } - public function testDrainsABacklogInBatches(): void - { - foreach (\range(1, 10) as $i) { - $this->producer->produce('event-' . $i); - } - - $consumer = $this->consumer(batch: 4); - - $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); - $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); - $this->assertSame(2, $consumer->consume(fn (CloudEvent $event) => null)); - $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); - } - /** * A cursor store that is down surfaces rather than being swallowed: reading * from an unknown position would replay the retained feed, so the run stops @@ -297,7 +154,7 @@ public function load(string $feed, string $consumer): ?string // Expected on the first run. } - $this->assertSame(['a'], $this->drain($consumer), 'The second run reads the store again'); + $this->assertSame(1, $consumer->consume(fn (CloudEvent $event) => null), 'The second run reads the store again'); } /** @@ -325,236 +182,6 @@ public function testAPositionThatCannotBeSavedIsRaisedAfterTheEventsAreHandled() $this->assertNotNull($consumer->position(), 'The in-memory position still moved'); } - public function testTipStartDoesNotAnnounceTheBacklog(): void - { - $this->producer->produce('old-1'); - $this->producer->produce('old-2'); - - $consumer = new Consumer($this->store, $this->cursor, 'notifier', start: Consumer::START_TIP); - - $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); - $this->assertNull($this->cursor->load('edge', 'notifier'), 'Skipping the backlog is not progress to commit'); - } - - /** - * The tip is pinned when the poll starts, so an event landing while the - * poll waits is delivered β€” only the backlog is skipped. - */ - public function testTipStartDeliversWhatLandsMidPoll(): void - { - $store = new MidPollStore('edge'); - (new Producer($store, 'urn:test'))->produce('old'); - - $cursor = new MemoryCursor(); - $consumer = new Consumer($store, $cursor, 'notifier', timeout: 5_000, start: Consumer::START_TIP); - - $seen = []; - $count = $consumer->consume(function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - }); - - $this->assertSame(1, $count); - $this->assertSame(['landed'], $seen, 'The backlog is skipped; the mid-wait event is not'); - $this->assertNotNull($cursor->load('edge', 'notifier'), 'Handling the event saves the position'); - } - - public function testAStoredCursorBeatsTipStart(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->cursor->save('edge', 'invalidator', $first); - - $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Consumer::START_TIP); - - $this->assertSame(['b'], $this->drain($consumer), 'A restart must not skip the gap'); - } - - public function testResetWithTipStartResumesFromNow(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - $this->cursor->save('edge', 'invalidator', $first); - - $consumer = new Consumer($this->store, $this->cursor, 'invalidator', start: Consumer::START_TIP); - - $this->assertSame(['b'], $this->drain($consumer), 'The stored position still wins before the reset'); - - $consumer->reset(); - - $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null), 'After reset, the backlog is forgotten'); - } - - public function testTipStartOnAnEmptyFeedWaitsOutTheTimeoutEmpty(): void - { - $consumer = new Consumer($this->store, $this->cursor, 'notifier', timeout: 600, start: Consumer::START_TIP); - - $started = \microtime(true); - - $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); - $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started, 'Must actually wait'); - } - - public function testTipStartOnAnEmptyFeedDeliversWhatLandsMidWait(): void - { - $consumer = new Consumer(new MidPollStore('edge'), $this->cursor, 'notifier', timeout: 5_000, start: Consumer::START_TIP); - - $seen = []; - $consumer->consume(function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - }); - - $this->assertSame(['landed'], $seen); - } - - public function testResetReplaysEverythingStillRetained(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $consumer = $this->consumer(); - $consumer->consume(fn (CloudEvent $event) => null); - - $consumer->reset(); - - $this->assertNull($consumer->position()); - $this->assertNull($this->cursor->load('edge', 'invalidator')); - $this->assertSame(['a', 'b'], $this->drain($consumer)); - } - - public function testSeekPositionsTheNextRunStrictlyAfterTheGivenId(): void - { - $this->producer->produce('a'); - $second = $this->producer->produce('b'); - $this->producer->produce('c'); - - $consumer = $this->consumer(); - $consumer->seek($second); - - $this->assertSame($second, $consumer->position(), 'The seeked id is the position until something is handled'); - $this->assertSame(['c'], $this->drain($consumer)); - } - - /** - * A seek is persisted, not just remembered: a fresh Consumer sharing the - * store and the name β€” a restart β€” resumes from it. - */ - public function testASeekSurvivesARestart(): void - { - $this->producer->produce('a'); - $second = $this->producer->produce('b'); - $this->producer->produce('c'); - - $this->consumer()->seek($second); - - $this->assertSame(['c'], $this->drain($this->consumer())); - } - - /** - * The operational escape hatch seek() exists for: a handler that keeps - * failing blocks the feed by design, and stepping past it is a deliberate - * seek to the failing event's own id. - */ - public function testSeekingToAPoisonEventsIdUnblocksTheConsumer(): void - { - $this->producer->produce('poison'); - $this->producer->produce('after'); - - $consumer = $this->consumer(); - $poison = null; - - $handler = function (CloudEvent $event) use (&$poison): void { - if ($event->type === 'poison') { - $poison = $event->id; - - throw new \RuntimeException('cannot handle this one'); - } - }; - - try { - $consumer->consume($handler); - $this->fail('The poison event should have blocked the run'); - } catch (\RuntimeException) { - // Expected: the feed is now blocked at the poison event. - } - - $this->assertNotNull($poison); - $consumer->seek($poison); - - $this->assertSame(['after'], $this->drain($consumer), 'The poison event is stepped over, nothing behind it is lost'); - } - - /** - * A hand-made move decided during a run is the newer decision, so the run - * must not save its own progress over it on the way out. - */ - public function testASeekMadeInsideAHandlerIsNotOverwritten(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - $third = $this->producer->produce('c'); - - $consumer = $this->consumer(batch: 1); - $consumer->consume(function (CloudEvent $event) use ($consumer, $third): void { - $consumer->seek($third); - }); - - $this->assertSame($third, $consumer->position()); - $this->assertSame($third, $this->cursor->load('edge', 'invalidator')); - $this->assertSame([], $this->drain($consumer), 'The run resumes after the seeked id, not after the handled one'); - } - - public function testAResetMadeInsideAHandlerIsNotOverwritten(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $consumer = $this->consumer(); - $consumer->consume(function (CloudEvent $event) use ($consumer): void { - if ($event->type === 'b') { - $consumer->reset(); - } - }); - - $this->assertNull($consumer->position()); - $this->assertNull($this->cursor->load('edge', 'invalidator')); - $this->assertSame(['a', 'b'], $this->drain($consumer), 'The reset stands, so everything retained replays'); - } - - /** - * @dataProvider notPositions - */ - public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void - { - $first = $this->producer->produce('a'); - $this->cursor->save('edge', 'invalidator', $first); - - $consumer = $this->consumer(); - - try { - $consumer->seek($id); - $this->fail('The id should have been rejected'); - } catch (Invalid) { - // Expected. - } - - $this->assertSame($first, $this->cursor->load('edge', 'invalidator'), 'A rejected seek leaves the stored position untouched'); - $this->assertSame($first, $consumer->position()); - } - - /** - * @return array - */ - public static function notPositions(): array - { - return [ - 'empty' => [''], - 'not an id' => ['abc'], - 'too many parts' => ['1-2-3'], - 'the tip sentinel' => ['$'], - ]; - } - /** * A seek that did not persist must not look like one that did: the store * failure surfaces, and the in-memory position stays where it was. @@ -577,74 +204,25 @@ public function testASeekThatCannotPersistFailsLoudlyAndMovesNothing(): void $this->assertNull($consumer->position(), 'The in-memory position must not move on a failed seek'); } - public function testConsumersOfTheSameFeedTrackSeparatePositions(): void - { - $this->producer->produce('a'); - - $one = new Consumer($this->store, $this->cursor, 'one'); - $two = new Consumer($this->store, $this->cursor, 'two'); - - $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); - $this->assertSame(1, $two->consume(fn (CloudEvent $event) => null), 'The second consumer has its own position'); - $this->assertSame(0, $one->consume(fn (CloudEvent $event) => null)); - } - /** - * The other half of that rule, pinned because it is the boundary the - * design draws rather than an accident: a name is one logical reader, so - * two processes behind one name split the feed instead of both seeing it. - */ - public function testTwoConsumersSharingANameSplitTheFeed(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $one = $this->consumer(batch: 1); - $two = $this->consumer(batch: 1); - - $this->assertSame(['a'], $this->drain($one)); - $this->assertSame(['b'], $this->drain($two), 'The second picks up after the first, it does not see a of its own'); - } - - /** - * And the cost of sharing a name, which no amount of coordination inside a - * single process can remove: each save is last-writer-wins, so a replica - * holding an older position drags the shared one backwards when it saves. - * At-least-once makes that a replay rather than a loss β€” the same is true of - * a reset one replica performs and another then recreates β€” but it is why - * every consumer gets its own name. + * The failure a consumer must not turn into a gap: if the read itself + * fails, nothing is handled and nothing is committed. */ - public function testAStaleConsumerSharingANameDragsThePositionBackwards(): void + public function testAFailedReadLeavesThePositionAlone(): void { $first = $this->producer->produce('a'); - $second = $this->producer->produce('b'); - - $stale = $this->consumer(batch: 1); - $this->assertNull($stale->position(), 'Reads the shared position before the other replica moves it'); - - $ahead = $this->consumer(); - $this->assertSame(['a', 'b'], $this->drain($ahead)); - $this->assertSame($second, $this->cursor->load('edge', 'invalidator')); - - $this->assertSame(['a'], $this->drain($stale), 'The stale replica polls from where it thought it was'); - $this->assertSame($first, $this->cursor->load('edge', 'invalidator'), 'Its save wins, so the shared position regresses'); - } + $this->producer->produce('b'); + $this->cursor->save('edge', 'invalidator', $first); - public function testPositionIsNullBeforeTheFirstRun(): void - { - $this->assertNull($this->consumer()->position()); - } + $consumer = new Consumer(new NoneStore('edge'), $this->cursor, 'invalidator'); - public function testRejectsAnEmptyConsumerName(): void - { - $this->expectException(Invalid::class); - - new Consumer($this->store, $this->cursor, ''); - } + $this->expectException(Unsupported::class); - public function testExposesItsName(): void - { - $this->assertSame('invalidator', $this->consumer()->getName()); + try { + $consumer->consume(fn (CloudEvent $event) => null); + } finally { + $this->assertSame($first, $this->cursor->load('edge', 'invalidator')); + } } /** @@ -678,25 +256,4 @@ public function testAFeedNameThatMatchesTheStoreIsAccepted(): void $this->assertSame(1, $consumer->consume(fn (CloudEvent $event) => null)); } - - /** - * The failure a consumer must not turn into a gap: if the read itself - * fails, nothing is handled and nothing is committed. - */ - public function testAFailedReadLeavesThePositionAlone(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - $this->cursor->save('edge', 'invalidator', $first); - - $consumer = new Consumer(new \Utopia\Feed\Store\None('edge'), $this->cursor, 'invalidator'); - - $this->expectException(\Utopia\Feed\Exception\Unsupported::class); - - try { - $consumer->consume(fn (CloudEvent $event) => null); - } finally { - $this->assertSame($first, $this->cursor->load('edge', 'invalidator')); - } - } } diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index d6c25d7..0a8bdc8 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -5,112 +5,16 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Cache\Adapter\Memory as CacheMemory; -use Utopia\Cache\Cache as UtopiaCache; -use Utopia\Feed\Cursor; -use Utopia\Feed\Cursor\Cache; -use Utopia\Feed\Cursor\Memory; use Utopia\Feed\Cursor\None; use Utopia\Feed\Exception\Invalid; +/** + * The cursor that deliberately remembers nothing. The real cursor adapters + * are exercised through {@see \Utopia\Tests\Consumer\Base}; None cannot be β€” + * a consumer over it replays forever, which is its point. + */ class CursorTest extends TestCase { - /** - * @return array - */ - public static function stores(): array - { - return [ - 'memory' => [new Memory()], - 'cache' => [new Cache(new UtopiaCache(new CacheMemory()))], - ]; - } - - /** - * @dataProvider stores - */ - public function testAnUnknownConsumerHasNoPosition(Cursor $cursor): void - { - $this->assertNull($cursor->load('edge', 'never-run')); - } - - /** - * @dataProvider stores - */ - public function testRoundTripsAPosition(Cursor $cursor): void - { - $cursor->save('edge', 'invalidator', '1690000000000-0'); - - $this->assertSame('1690000000000-0', $cursor->load('edge', 'invalidator')); - } - - /** - * @dataProvider stores - */ - public function testOverwritesAPosition(Cursor $cursor): void - { - $cursor->save('edge', 'invalidator', '1-0'); - $cursor->save('edge', 'invalidator', '2-0'); - - $this->assertSame('2-0', $cursor->load('edge', 'invalidator')); - } - - /** - * @dataProvider stores - */ - public function testConsumersDoNotShareAPosition(Cursor $cursor): void - { - $cursor->save('edge', 'one', '1-0'); - $cursor->save('edge', 'two', '2-0'); - - $this->assertSame('1-0', $cursor->load('edge', 'one')); - $this->assertSame('2-0', $cursor->load('edge', 'two')); - } - - /** - * One store serves every feed a service consumes, which is why the feed - * name is part of the key rather than of the cursor. - * - * @dataProvider stores - */ - public function testFeedsDoNotShareAPosition(Cursor $cursor): void - { - $cursor->save('edge', 'invalidator', '1-0'); - - $this->assertNull($cursor->load('other', 'invalidator')); - } - - /** - * @dataProvider stores - */ - public function testResetForgetsAPosition(Cursor $cursor): void - { - $cursor->save('edge', 'invalidator', '1-0'); - $cursor->reset('edge', 'invalidator'); - - $this->assertNull($cursor->load('edge', 'invalidator')); - } - - /** - * @dataProvider stores - */ - public function testRejectsAnEmptyConsumerName(Cursor $cursor): void - { - $this->expectException(Invalid::class); - - $cursor->load('edge', ''); - } - - /** - * @dataProvider stores - */ - public function testRejectsAnEmptyFeedName(Cursor $cursor): void - { - $this->expectException(Invalid::class); - - $cursor->save('', 'invalidator', '1-0'); - } - public function testTheNoneStoreRemembersNothing(): void { $cursor = new None(); diff --git a/tests/Feed/Unit/ProducerTest.php b/tests/Feed/Unit/ProducerTest.php index 0be1600..aab57f3 100644 --- a/tests/Feed/Unit/ProducerTest.php +++ b/tests/Feed/Unit/ProducerTest.php @@ -5,115 +5,20 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; -use Utopia\Feed\Server; -use Utopia\Feed\Id; -use Utopia\Feed\Store\Memory; -use Utopia\Feed\Store\None; use Utopia\Feed\Producer; use Utopia\Feed\Remote; -use Utopia\Tests\Unit\Support\FakeTransport; - +use Utopia\Feed\Store\None; +use Utopia\Tests\Support\FakeTransport; + +/** + * The producer behaviours no working adapter can show: producing into a feed + * that is not yours, and producing into a backend that was never configured. + * Everything a real adapter can exercise lives in + * {@see \Utopia\Tests\Producer\Base} instead. + */ class ProducerTest extends TestCase { - private Memory $store; - - private Producer $producer; - - private Server $server; - - protected function setUp(): void - { - $this->store = new Memory('edge'); - $this->producer = new Producer($this->store, 'urn:appwrite:cloud:fra'); - $this->server = new Server($this->store); - } - - public function testProduceReturnsAPosition(): void - { - $id = $this->producer->produce('io.appwrite.edge.invalidate', ['tags' => ['project' => 'p1']]); - - $this->assertTrue(Id::isValid($id)); - } - - /** @return list */ - private function events(): array - { - return \array_values(\iterator_to_array($this->server->read())); - } - - public function testStampsTheSourceAndTimeOnProduce(): void - { - $this->producer->produce('test'); - - $event = $this->events()[0]; - - $this->assertSame('urn:appwrite:cloud:fra', $event->source); - $this->assertNotNull($event->time); - $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/', $event->time); - } - - /** - * Recording it when produced rather than when read keeps it correct for a feed - * read back somewhere other than where it was written. - */ - public function testKeepsTheSourceOfTheProducerThatProduced(): void - { - (new Producer($this->store, 'urn:appwrite:cloud:fra'))->produce('test'); - (new Producer($this->store, 'urn:appwrite:cloud:nyc'))->produce('test'); - - $events = $this->events(); - - $this->assertSame('urn:appwrite:cloud:fra', $events[0]->source); - $this->assertSame('urn:appwrite:cloud:nyc', $events[1]->source); - } - - public function testPublishStampsAPreparedEvent(): void - { - $id = $this->producer->publish(new CloudEvent(id: 'ignored', type: 'test', source: 'ignored', data: ['a' => 'b'], subject: 's')); - - $event = $this->events()[0]; - - $this->assertSame($id, $event->id); - $this->assertNotSame('ignored', $event->id, 'The backend assigns the position, not the caller'); - $this->assertSame('urn:appwrite:cloud:fra', $event->source); - $this->assertSame(['a' => 'b'], $event->data); - } - - public function testPublishKeepsATimeTheCallerSet(): void - { - $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', time: '2020-01-01T00:00:00.000Z')); - - $this->assertSame('2020-01-01T00:00:00.000Z', $this->events()[0]->time); - } - - public function testRejectsAnEmptyEventType(): void - { - $this->expectException(Invalid::class); - - $this->producer->produce(''); - } - - public function testRejectsAPayloadThatCannotBeEncoded(): void - { - $this->expectException(Invalid::class); - - $this->producer->produce('test', ['resource' => \fopen('php://memory', 'r')]); - } - - /** - * CloudEvents requires a source, and an event stamped with an empty one is - * an event no consumer can attribute. - */ - public function testRejectsAnEmptySource(): void - { - $this->expectException(Invalid::class); - - new Producer($this->store, ''); - } - /** * A remote feed belongs to whoever produces into it, so Remote is neither * a Store nor Appendable β€” the mistake is a type error at construction diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index b763590..24e0168 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -18,9 +18,9 @@ use Utopia\Feed\Readable; use Utopia\Feed\Remote; use Utopia\Feed\Server; -use Utopia\Tests\Unit\Support\FakeTransport; -use Utopia\Tests\Unit\Support\FeedServer; -use Utopia\Tests\Unit\Support\MidPollStore; +use Utopia\Tests\Support\FakeTransport; +use Utopia\Tests\Support\FeedServer; +use Utopia\Tests\Support\MidPollStore; class RemoteTest extends TestCase { diff --git a/tests/Feed/Unit/RoundTripTest.php b/tests/Feed/Unit/RoundTripTest.php deleted file mode 100644 index dec0bd4..0000000 --- a/tests/Feed/Unit/RoundTripTest.php +++ /dev/null @@ -1,207 +0,0 @@ -producer = new Producer($store, 'urn:appwrite:cloud:fra'); - $this->server = new FeedServer(new Server($store)); - - $this->cursor = new CacheCursor(new UtopiaCache(new CacheMemory())); - } - - private function consumer(string $name = 'invalidator', int $batch = Consumer::BATCH): Consumer - { - return new Consumer($this->server, $this->cursor, $name, feed: 'edge', batch: $batch); - } - - public function testAnEventSurvivesTheWholeTrip(): void - { - $this->producer->produce( - 'io.appwrite.edge.invalidate-rule', - ['tags' => ['domain' => 'example.com'], 'isAppwriteNetwork' => true], - 'example.com', - ); - - $received = null; - $this->consumer()->consume(function (CloudEvent $event) use (&$received): void { - $received = $event; - }); - - $this->assertInstanceOf(CloudEvent::class, $received); - $this->assertSame('io.appwrite.edge.invalidate-rule', $received->type); - $this->assertSame('urn:appwrite:cloud:fra', $received->source); - $this->assertSame('example.com', $received->subject); - $this->assertSame([ - 'tags' => ['domain' => 'example.com'], - 'isAppwriteNetwork' => true, - ], $received->data); - } - - public function testTheConsumerOnlyEverSeesEachEventOnce(): void - { - foreach (\range(1, 5) as $i) { - $this->producer->produce('event-' . $i); - } - - $consumer = $this->consumer(); - $seen = []; - $handler = function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - }; - - $consumer->consume($handler); - $consumer->consume($handler); - - $this->producer->produce('event-6'); - $consumer->consume($handler); - - $this->assertSame( - ['event-1', 'event-2', 'event-3', 'event-4', 'event-5', 'event-6'], - $seen, - ); - } - - /** - * The rollout property: a consumer shipped after the producer catches up on - * everything that accumulated in between, rather than starting at the tip. - */ - public function testAConsumerShippedLateDrainsTheBacklog(): void - { - foreach (\range(1, 3) as $i) { - $this->producer->produce('missed-' . $i); - } - - $seen = []; - $handled = $this->consumer()->consume(function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - }); - - $this->assertSame(3, $handled); - $this->assertSame(['missed-1', 'missed-2', 'missed-3'], $seen); - } - - /** - * The restart property: a new Consumer with no in-memory state picks the - * stored position up rather than replaying. - */ - public function testARestartedConsumerResumesWhereItLeftOff(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->consumer()->consume(fn (CloudEvent $event) => null); - - $this->producer->produce('c'); - - $seen = []; - $this->consumer()->consume(function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - }); - - $this->assertSame(['c'], $seen); - } - - public function testAFailedEventBlocksTheOnesBehindItUntilItSucceeds(): void - { - $this->producer->produce('a'); - $this->producer->produce('poison'); - $this->producer->produce('c'); - - $consumer = $this->consumer(); - $seen = []; - $attempts = 0; - - // Fails the first time it sees the poison event and succeeds after, - // standing in for a dependency that was briefly unavailable. - $handler = function (CloudEvent $event) use (&$seen, &$attempts): void { - if ($event->type === 'poison') { - $attempts++; - - if ($attempts === 1) { - throw new \RuntimeException('not yet'); - } - } - - $seen[] = $event->type; - }; - - try { - $consumer->consume($handler); - $this->fail('The handler failure should have been re-raised'); - } catch (\RuntimeException) { - // Expected. - } - - $this->assertSame(['a'], $seen, 'Nothing behind the failure is delivered'); - - $consumer->consume($handler); - - $this->assertSame(['a', 'poison', 'c'], $seen, 'Order is preserved and nothing is skipped'); - $this->assertSame(2, $attempts, 'The failed event is retried, not dropped'); - } - - public function testTheProducerCachesFullBatchesAndNothingElse(): void - { - foreach (\range(1, 5) as $i) { - $this->producer->produce('event-' . $i); - } - - $consumer = $this->consumer(batch: 2); - - $consumer->consume(fn (CloudEvent $event) => null); - $consumer->consume(fn (CloudEvent $event) => null); - $consumer->consume(fn (CloudEvent $event) => null); - $consumer->consume(fn (CloudEvent $event) => null); - - $this->assertSame([ - 'private, max-age=31536000', // 2 of 2 β€” settled history - 'private, max-age=31536000', // 2 of 2 β€” settled history - 'no-store', // 1 of 2 β€” the live end, will grow - 'no-store', // 0 of 2 β€” caught up - ], $this->server->recorder->cacheControl()); - } - - public function testTwoConsumersOfOneProducerAreIndependent(): void - { - $this->producer->produce('a'); - - $one = $this->consumer('one'); - $two = $this->consumer('two'); - - $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null)); - - $this->producer->produce('b'); - - $this->assertSame(2, $two->consume(fn (CloudEvent $event) => null), 'The second consumer starts from the beginning'); - $this->assertSame(1, $one->consume(fn (CloudEvent $event) => null), 'The first only sees what is new to it'); - } -} diff --git a/tests/Feed/Unit/ServerTest.php b/tests/Feed/Unit/ServerTest.php index 443e71a..e9bc3a7 100644 --- a/tests/Feed/Unit/ServerTest.php +++ b/tests/Feed/Unit/ServerTest.php @@ -5,364 +5,17 @@ namespace Utopia\Tests\Unit; use PHPUnit\Framework\TestCase; -use Utopia\Feed\Store\Memory; -use Utopia\Feed\Store\None; -use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Batch; -use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Exception\Unsupported; use Utopia\Feed\Server; -use Utopia\Feed\Producer; -use Utopia\Feed\Readable; -use Utopia\Feed\Id; -use Utopia\Tests\Unit\Support\MidPollStore; +use Utopia\Feed\Store\None; +/** + * The server behaviour no working adapter can show: serving a feed whose + * backend was never configured. Everything a real adapter can exercise lives + * in {@see \Utopia\Tests\Server\Base} instead. + */ class ServerTest extends TestCase { - private Memory $store; - - private Server $server; - - private Producer $producer; - - protected function setUp(): void - { - $this->store = new Memory('edge'); - $this->producer = new Producer($this->store, 'urn:appwrite:cloud:fra'); - $this->server = new Server($this->store); - } - - /** @return list */ - private static function events(Batch $batch): array - { - return \array_values(\iterator_to_array($batch)); - } - - public function testReadsBackWhatWasAppended(): void - { - $this->producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); - - $events = self::events($this->server->read()); - - $this->assertCount(1, $events); - $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); - $this->assertSame('example.com', $events[0]->subject); - $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); - } - - public function testEventsComeBackOldestFirst(): void - { - foreach (['a', 'b', 'c'] as $type) { - $this->producer->produce($type); - } - - $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($this->server->read()))); - } - - public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void - { - $ids = []; - for ($i = 0; $i < 50; $i++) { - $ids[] = $this->producer->produce('test'); - } - - $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); - - for ($i = 1; $i < \count($ids); $i++) { - $this->assertGreaterThan(Id::decode($ids[$i - 1]), Id::decode($ids[$i]), 'Positions must increase'); - } - } - - public function testReadsStrictlyAfterTheGivenPosition(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - - $events = self::events($this->server->read($first)); - - $this->assertCount(1, $events); - $this->assertSame('b', $events[0]->type); - } - - public function testReadFromTheLastEventIsEmpty(): void - { - $this->producer->produce('a'); - $last = $this->producer->produce('b'); - - $this->assertCount(0, $this->server->read($last)); - } - - public function testNullPositionReadsFromTheOldestRetainedEvent(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->assertCount(2, $this->server->read(null)); - } - - public function testHonoursTheLimit(): void - { - foreach (\range(1, 10) as $i) { - $this->producer->produce('test'); - } - - $this->assertCount(3, $this->server->read(null, 3)); - } - - /** - * `limit` arrives from a consumer, so it is clamped rather than rejected β€” - * failing the read would stall a feed over something the producer can just - * decide. - */ - public function testClampsTheLimitToTheMaximum(): void - { - $this->producer->produce('test'); - - $this->assertCount(1, $this->server->read(null, Readable::MAX_BATCH * 10)); - $this->assertCount(1, $this->server->read(null, 0)); - $this->assertCount(1, $this->server->read(null, -5)); - } - - public function testRejectsAPositionThatIsNotAFeedId(): void - { - $this->expectException(Invalid::class); - - $this->server->read('not-a-position'); - } - - /** - * A producer that attaches a `traceparent` means it to reach the consumer. - * Stamping the event on publish rebuilds it, and storing it flattens it, so - * either step could quietly drop an attribute this library does not model. - */ - public function testExtensionAttributesSurviveAppendAndRead(): void - { - $this->producer->publish(new CloudEvent( - id: '', - type: 'test', - source: '', - extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2], - )); - - $event = self::events($this->server->read())[0]; - - $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); - $this->assertSame(2, $event->extensions['retrycount']); - } - - /** - * An extension name of only digits is legal β€” the spec allows `[a-z0-9]+` β€” - * and PHP stores such a name as an integer key. Anything that merges the - * extensions back in with a spread, or with `array_merge()`, renumbers that - * key and silently loses the attribute. - */ - public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void - { - $this->producer->publish(new CloudEvent( - id: '', - type: 'test', - source: '', - // @phpstan-ignore argument.type ('123' is an integer key in PHP) - extensions: ['123' => 'digits', 'trace' => 'ok'], - )); - - $event = self::events($this->server->read())[0]; - - // @phpstan-ignore offsetAccess.notFound - $this->assertSame('digits', $event->extensions['123']); - $this->assertSame('ok', $event->extensions['trace']); - } - - public function testDataschemaSurvivesAppendAndRead(): void - { - $this->producer->publish(new CloudEvent( - id: '', - type: 'test', - source: '', - dataschema: 'https://example.com/schema.json', - )); - - $this->assertSame('https://example.com/schema.json', self::events($this->server->read())[0]->dataschema); - } - - /** - * @return array - */ - public static function payloads(): array - { - return [ - 'map' => [['tags' => ['domain' => 'example.com']]], - 'list' => [['a', 'b', 'c']], - 'nested list' => [[['x' => 1], ['x' => 2]]], - 'string' => ['a string'], - 'number' => [42], - 'float' => [1.5], - 'boolean' => [true], - 'null' => [null], - 'empty' => [[]], - ]; - } - - /** - * The JSON event format leaves `data` unrestricted, so a list or a scalar - * has to survive as itself β€” a list must not come back as a map. - * - * @dataProvider payloads - */ - public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void - { - $this->producer->produce('test', $data); - - $this->assertSame($data, self::events($this->server->read())[0]->data); - } - - /** - * CloudEvents models an absent subject as null rather than an empty string, - * so a caller checking for one must check for null. - */ - public function testAnEventWithNoSubjectHasANullSubject(): void - { - $this->producer->produce('test'); - - $this->assertNull(self::events($this->server->read())[0]->subject); - } - - public function testASubjectSurvivesAppendAndRead(): void - { - $this->producer->produce('test', [], 'example.com'); - - $this->assertSame('example.com', self::events($this->server->read())[0]->subject); - } - - public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void - { - $this->producer->produce('test'); - - $started = \microtime(true); - $events = $this->server->poll(null, 10, 2000); - - $this->assertCount(1, $events); - $this->assertLessThan(1, \microtime(true) - $started); - } - - public function testPollGivesUpAtTheTimeoutWithAnEmptyBatch(): void - { - $started = \microtime(true); - $events = $this->server->poll(null, 10, 600); - $elapsed = \microtime(true) - $started; - - $this->assertCount(0, $events); - $this->assertGreaterThanOrEqual(0.4, $elapsed, 'Must actually wait'); - $this->assertLessThan(3.0, $elapsed, 'Must not wait far past the timeout'); - } - - public function testPollWithoutATimeoutIsAPlainRead(): void - { - $started = \microtime(true); - - $this->assertCount(0, $this->server->poll()); - $this->assertLessThan(0.4, \microtime(true) - $started); - } - - public function testAShorterPollIntervalDeliversAMidPollEventSooner(): void - { - $store = new MidPollStore('edge', pollInterval: 20); - - $started = \microtime(true); - $events = $store->poll(null, 10, 5_000); - $elapsed = \microtime(true) - $started; - - $this->assertCount(1, $events); - $this->assertLessThan(0.4, $elapsed, 'A 20ms interval must beat the default 500ms floor'); - } - - /** - * The overshoot fix: the loop must sleep the remaining time when that is - * less than the interval, not a full interval past the deadline. - */ - public function testPollHonoursATimeoutShorterThanTheInterval(): void - { - $store = new Memory('edge', pollInterval: 500); - - $started = \microtime(true); - $events = $store->poll(null, 10, 100); - $elapsed = \microtime(true) - $started; - - $this->assertSame([], $events); - $this->assertGreaterThanOrEqual(0.08, $elapsed, 'Must actually wait out the timeout'); - $this->assertLessThan(0.3, $elapsed, 'Must not sleep a full interval past the deadline'); - } - - public function testRejectsAZeroPollInterval(): void - { - $this->expectException(Invalid::class); - - new Memory('edge', pollInterval: 0); - } - - public function testRejectsANegativePollInterval(): void - { - $this->expectException(Invalid::class); - - new Memory('edge', pollInterval: -5); - } - - /** - * @dataProvider notRetentions - */ - public function testRejectsARetentionThatKeepsNothing(int $maxSize): void - { - $this->expectException(Invalid::class); - - new Memory('edge', maxSize: $maxSize); - } - - /** - * @return array - */ - public static function notRetentions(): array - { - return [ - 'zero' => [0], - 'negative' => [-5], - ]; - } - - public function testRetentionIsBoundedAndTrimsTheOldest(): void - { - $store = new Memory('small', maxSize: 3); - $producer = new Producer($store, 'urn:appwrite:cloud:fra'); - $server = new Server($store); - - foreach (['a', 'b', 'c', 'd', 'e'] as $type) { - $producer->produce($type); - } - - $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($server->read()))); - } - - /** - * The one case a caller has to design for: a consumer that fell behind the - * trim horizon gets what is left, not an error and not a gap it can detect. - */ - public function testAPositionBelowTheTrimHorizonReadsWhatIsLeft(): void - { - $store = new Memory('small', maxSize: 2); - $producer = new Producer($store, 'urn:appwrite:cloud:fra'); - $server = new Server($store); - - $first = $producer->produce('a'); - $producer->produce('b'); - $producer->produce('c'); - - $this->assertSame(['b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($server->read($first)))); - } - - public function testExposesTheFeedItReads(): void - { - $this->assertSame('edge', $this->server->getName()); - } - public function testAFeedWithNoBackendCannotBeRead(): void { $server = new Server(new None('edge')); @@ -371,127 +24,4 @@ public function testAFeedWithNoBackendCannotBeRead(): void $server->read(); } - - public function testTipIsTheNewestEventsId(): void - { - $this->assertNull($this->server->tip()); - - $this->producer->produce('a'); - $last = $this->producer->produce('b'); - - $this->assertSame($last, $this->server->tip()); - } - - public function testReadingFromTheTipSentinelSkipsTheBacklog(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->assertCount(0, $this->server->read(Readable::TIP)); - } - - public function testServeAppliesTheDefaultsWhenNoParametersArrive(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->assertCount(2, $this->server->serve([])); - } - - public function testServeCoercesTheStringValuesARouteHands(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - $this->producer->produce('c'); - - $batch = $this->server->serve([ - 'lastEventId' => $first, - 'limit' => '1', - 'timeout' => '0', - ]); - - $this->assertSame(['b'], \array_map(fn (CloudEvent $e): string => $e->type, self::events($batch))); - } - - public function testServeTreatsAnEmptyLastEventIdAsAbsent(): void - { - $this->producer->produce('a'); - - $this->assertCount(1, $this->server->serve(['lastEventId' => ''])); - } - - public function testServeRejectsALastEventIdThatIsNotAPosition(): void - { - $this->expectException(Invalid::class); - - $this->server->serve(['lastEventId' => 'not-a-position']); - } - - public function testServeLetsTheTipSentinelThrough(): void - { - $this->producer->produce('a'); - - $this->assertCount(0, $this->server->serve(['lastEventId' => Readable::TIP])); - } - - public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $this->assertCount(2, $this->server->serve(['limit' => 'lots', 'timeout' => 'soon'])); - } - - /** - * The trap the old API set: a route that passed the raw request limit to - * the caching rule while the read was clamped to less would mark a full - * batch `no-store` β€” or worse. The batch carries the limit it was actually - * built with, so a full batch under an oversized request is still - * recognized as settled history. - */ - public function testAnOversizedLimitStillYieldsAnHonestCacheControl(): void - { - foreach (\range(1, Readable::MAX_BATCH) as $i) { - $this->producer->produce('event-' . $i); - } - - $batch = $this->server->serve(['limit' => '5000']); - - $this->assertCount(Readable::MAX_BATCH, $batch); - $this->assertSame('public, max-age=31536000', $batch->cacheControl(public: true)); - } - - public function testAShortBatchIsNotCacheable(): void - { - $this->producer->produce('a'); - - $this->assertSame('no-store', $this->server->serve([])->cacheControl()); - } - - public function testAnEmptyBatchIsNotCacheable(): void - { - $this->assertSame('no-store', $this->server->serve([])->cacheControl()); - } - - public function testRejectsAnEmptyFeedName(): void - { - $this->expectException(Invalid::class); - - new Memory(''); - } - - public function testAcceptsTheSmallestUsefulRetentionCap(): void - { - $store = new Memory('edge', maxSize: 1); - $producer = new Producer($store, 'urn:appwrite:cloud:fra'); - $server = new Server($store); - - $producer->produce('a'); - $producer->produce('b'); - - $events = self::events($server->read()); - - $this->assertCount(1, $events); - $this->assertSame('b', $events[0]->type); - } } diff --git a/tests/Feed/Unit/StoreCacheTest.php b/tests/Feed/Unit/StoreCacheTest.php deleted file mode 100644 index 2373731..0000000 --- a/tests/Feed/Unit/StoreCacheTest.php +++ /dev/null @@ -1,170 +0,0 @@ -cache = new UtopiaCache(new CacheMemory()); - $this->store = new Cache($this->cache, 'edge'); - $this->producer = new Producer($this->store, 'urn:test'); - } - - public function testRoundTripsAnEventThroughTheServer(): void - { - $id = $this->producer->produce('io.appwrite.edge.invalidate-rule', ['tags' => ['domain' => 'example.com']], 'example.com'); - - $events = \array_values(\iterator_to_array((new Server($this->store))->read())); - - $this->assertCount(1, $events); - $this->assertSame($id, $events[0]->id); - $this->assertSame('io.appwrite.edge.invalidate-rule', $events[0]->type); - $this->assertSame('example.com', $events[0]->subject); - $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); - } - - public function testEventsComeBackOldestFirst(): void - { - foreach (['a', 'b', 'c'] as $type) { - $this->producer->produce($type); - } - - $this->assertSame(['a', 'b', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $this->store->read(null, 10))); - } - - public function testIdsAreStrictlyIncreasingEvenWithinAMillisecond(): void - { - $ids = []; - for ($i = 0; $i < 50; $i++) { - $ids[] = $this->producer->produce('test'); - } - - $this->assertSame($ids, \array_unique($ids), 'Positions must be unique'); - - for ($i = 1; $i < \count($ids); $i++) { - $this->assertGreaterThan(Id::decode($ids[$i - 1]), Id::decode($ids[$i]), 'Positions must increase'); - } - } - - public function testReadsStrictlyAfterTheGivenPosition(): void - { - $first = $this->producer->produce('a'); - $this->producer->produce('b'); - - $events = $this->store->read($first, 10); - - $this->assertCount(1, $events); - $this->assertSame('b', $events[0]->type); - $this->assertCount(0, $this->store->read($events[0]->id, 10)); - } - - public function testHonoursTheLimit(): void - { - foreach (\range(1, 10) as $i) { - $this->producer->produce('test'); - } - - $this->assertCount(3, $this->store->read(null, 3)); - } - - public function testRetentionIsBoundedAndTrimsTheOldest(): void - { - $store = new Cache($this->cache, 'small', maxSize: 3); - $producer = new Producer($store, 'urn:test'); - - foreach (['a', 'b', 'c', 'd', 'e'] as $type) { - $producer->produce($type); - } - - $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 10))); - } - - public function testTipIsTheNewestEventsId(): void - { - $this->assertNull($this->store->tip(), 'An empty feed has no tip'); - - $this->producer->produce('a'); - $last = $this->producer->produce('b'); - - $this->assertSame($last, $this->store->tip()); - } - - /** - * The feed lives in the cache, not in the store object: a second store - * over the same cache β€” another request handling the same feed β€” reads - * what the first appended. - */ - public function testTheFeedSurvivesTheStoreThatWroteIt(): void - { - $id = $this->producer->produce('a'); - - $events = (new Cache($this->cache, 'edge'))->read(null, 10); - - $this->assertCount(1, $events); - $this->assertSame($id, $events[0]->id); - } - - /** - * A cache is allowed to forget, and it may also hold a foreign value under - * the feed's key. Both read as an empty feed β€” a replay, never a fault. - */ - public function testAForeignValueUnderTheFeedsKeyReadsAsEmpty(): void - { - $this->cache->save('feed:edge', ['not' => 'a feed']); - - $this->assertCount(0, $this->store->read(null, 10)); - $this->assertNull($this->store->tip()); - } - - public function testExtensionAttributesSurviveTheTrip(): void - { - $this->producer->publish(new CloudEvent( - id: '', - type: 'test', - source: '', - extensions: ['traceparent' => '00-abc-def-01'], - )); - - $this->assertSame('00-abc-def-01', $this->store->read(null, 10)[0]->extensions['traceparent']); - } - - public function testAConsumerDrainsACacheFedFeed(): void - { - $this->producer->produce('a'); - $this->producer->produce('b'); - - $seen = []; - $consumer = new Consumer($this->store, new MemoryCursor(), 'invalidator'); - - $this->assertSame(2, $consumer->consume(function (CloudEvent $event) use (&$seen): void { - $seen[] = $event->type; - })); - $this->assertSame(['a', 'b'], $seen); - } -} From 58cf7acb03318da0ece7579a21b7e7b90e4d8896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 11:51:33 +0200 Subject: [PATCH 38/68] Depend on a released utopia-php/cloudevents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The library required `dev-feat-constructor-validation`, a feature branch of utopia-php/cloudevents locked to a single commit. That is a release blocker: the branch can be force-pushed or deleted at any time (it since has been β€” the locked commit `8a89ab2` no longer resolves), and any project requiring utopia-php/feed inherits a `dev-*` constraint that fails to resolve under the default `minimum-stability: stable`, so the package could not actually be installed from Packagist. Switch to `0.1.*` and relock. The released version carries the same constructor validation the branch was pinned for, so no source changes are needed; the whole suite and PHPStan stay green. Co-Authored-By: Claude Opus 5 (1M context) --- composer.json | 2 +- composer.lock | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/composer.json b/composer.json index 5f260f4..5d40923 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ "utopia-php/client": "^0.2", "utopia-php/pools": "1.*", "utopia-php/psr7": "^0.2", - "utopia-php/cloudevents": "dev-feat-constructor-validation" + "utopia-php/cloudevents": "0.1.*" }, "require-dev": { "laravel/pint": "^1.21", diff --git a/composer.lock b/composer.lock index 433413d..2b73b36 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "64da6e0642c2891229b50497a4cfef29", + "content-hash": "935a7e23baaac15e7abd66b7e9926cdb", "packages": [ { "name": "brick/math", @@ -2052,16 +2052,16 @@ }, { "name": "utopia-php/cloudevents", - "version": "dev-feat-constructor-validation", + "version": "0.1.0", "source": { "type": "git", "url": "https://github.com/utopia-php/cloudevents.git", - "reference": "8a89ab2365a70919f2e9a4c3742c28ac9020fb57" + "reference": "a1cd71180be54b73e7a16469a900dd85d993882d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cloudevents/zipball/8a89ab2365a70919f2e9a4c3742c28ac9020fb57", - "reference": "8a89ab2365a70919f2e9a4c3742c28ac9020fb57", + "url": "https://api.github.com/repos/utopia-php/cloudevents/zipball/a1cd71180be54b73e7a16469a900dd85d993882d", + "reference": "a1cd71180be54b73e7a16469a900dd85d993882d", "shasum": "" }, "require": { @@ -2098,9 +2098,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cloudevents/issues", - "source": "https://github.com/utopia-php/cloudevents/tree/feat-constructor-validation" + "source": "https://github.com/utopia-php/cloudevents/tree/0.1.0" }, - "time": "2026-07-30T11:07:39+00:00" + "time": "2026-07-31T10:57:20+00:00" }, { "name": "utopia-php/pools", @@ -4245,9 +4245,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/cloudevents": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 5ae89e42a94bfc1791cec27e15f0f8f478e7811e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 11:54:26 +0200 Subject: [PATCH 39/68] Fail loudly when the cursor cache cannot be written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Cursor\Cache::save()` ignored the adapter's return value. `Utopia\Cache\Adapter\Redis::save()` catches every Throwable internally and answers `false`, so against a cache backend that is down the write silently did nothing while the caller β€” and the abstract `Cursor::save()` docblock, and the README's exception table β€” was promised a `Transport`. Two operator-visible consequences: `Consumer::seek($poisonEventId)`, documented as persisted immediately, quietly moved nothing, so the consumer was blocked on the same poison event after the next restart; and every position save inside `consume()` failed unnoticed, so a long-running consumer replayed its whole retained backlog on each restart. `Store\Cache ::append()` already checks `$saved === false` and raises `Transport` β€” the cursor twin now matches it. `reset()` deliberately does not follow suit: `purge()` answers `false` both for a write it could not do and for a key that was never there, and the second is the ordinary case (a consumer with no saved position yet), so the return value carries no failure signal to act on. Comments in both methods record which is which. Tested with a `BrokenCache` adapter that fails the way the Redis one does β€” returning false rather than raising β€” at the cursor's own API, at `seek()`, and at `reset()` (which must stay harmless). Co-Authored-By: Claude Opus 5 (1M context) --- src/Feed/Cursor/Cache.php | 14 +++++- tests/Feed/Support/BrokenCache.php | 65 ++++++++++++++++++++++++++++ tests/Feed/Unit/CursorTest.php | 69 ++++++++++++++++++++++++++++-- 3 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 tests/Feed/Support/BrokenCache.php diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php index 98a9e30..c90ffc9 100644 --- a/src/Feed/Cursor/Cache.php +++ b/src/Feed/Cursor/Cache.php @@ -6,6 +6,7 @@ use Utopia\Cache\Cache as UtopiaCache; use Utopia\Feed\Cursor; +use Utopia\Feed\Exception\Transport; class Cache extends Cursor { @@ -27,11 +28,22 @@ public function load(string $feed, string $consumer): ?string public function save(string $feed, string $consumer, string $eventId): void { - $this->cache->save($this->key($feed, $consumer), $eventId); + $saved = $this->cache->save($this->key($feed, $consumer), $eventId); + + // A cache adapter reports a failed write by returning false rather + // than raising β€” swallowing that would let a position silently not + // persist, so the consumer replays its backlog on the next restart + // and a seek past a poison event quietly does nothing. + if ($saved === false) { + throw new Transport("Failed to save the {$consumer} cursor on the {$feed} feed"); + } } public function reset(string $feed, string $consumer): void { + // Unlike save(), purge()'s false is not a failure signal: it is also + // what an adapter answers for a key that was never there, which is + // the ordinary case for a consumer that has not saved a position yet. $this->cache->purge($this->key($feed, $consumer)); } } diff --git a/tests/Feed/Support/BrokenCache.php b/tests/Feed/Support/BrokenCache.php new file mode 100644 index 0000000..29972f6 --- /dev/null +++ b/tests/Feed/Support/BrokenCache.php @@ -0,0 +1,65 @@ +load($feed, $consumer); } + + /** + * A cache adapter answers a failed write with false rather than raising, + * so a cursor that ignores the return value reports a position as saved + * that was never stored β€” and the consumer replays its whole retained + * backlog on the next restart with nothing to explain why. + */ + public function testACacheThatCannotBeWrittenRaisesTransport(): void + { + $cursor = new CacheCursor(new UtopiaCache(new BrokenCache())); + + $this->expectException(Transport::class); + + $cursor->save('edge', 'invalidator', '1-0'); + } + + /** + * The operational consequence, through the API an operator actually + * reaches for: seek() is documented as persisting immediately, so a store + * that cannot hold the new position must not let the operator believe + * they stepped past a poison event. + */ + public function testASeekThroughAnUnwritableCacheFailsLoudly(): void + { + $store = new MemoryStore('edge'); + $poison = (new Producer($store, 'urn:test'))->produce('poison'); + + $consumer = new Consumer($store, new CacheCursor(new UtopiaCache(new BrokenCache())), 'invalidator'); + + try { + $consumer->seek($poison); + $this->fail('The store failure should have been raised'); + } catch (Transport) { + // Expected. + } + + $this->assertNull($consumer->position(), 'The in-memory position must not move on a failed seek'); + } + + /** + * Resetting is not held to the same rule: an adapter answers false both + * for a write it could not do and for a key that was never there, and the + * second is the ordinary case β€” a consumer that has not saved a position + * yet. Only a raising backend is a failure to report. + */ + public function testResettingAPositionThatWasNeverSavedIsHarmless(): void + { + $consumer = new Consumer(new MemoryStore('edge'), new CacheCursor(new UtopiaCache(new BrokenCache())), 'invalidator'); + + $consumer->reset(); + + $this->assertNull($consumer->position()); + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + } } From f2ab5c65625659ddae52df09a2aff6944984c06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 11:55:56 +0200 Subject: [PATCH 40/68] Keep datacontenttype across a store round trip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Store::encode()` flattened an event to type, source, subject, dataschema, time, data and extensions β€” `datacontenttype` was missing, and `decode()` never restored it. The rest of the pipeline models the attribute: `Producer::publish()` copies it into the rebuilt event, `Remote::event()` reads it off the wire, and `Remote::ATTRIBUTES` lists it. Only the local store round trip lost it, so a producer that encoded its payload as anything but JSON published events whose data no consumer could interpret, with nothing to signal the loss. Fixed in the one place both halves live, so every store adapter β€” Memory, Cache, Redis, Pool β€” picks it up at once. Also documented what `encode()` deliberately leaves out: `id` (the store assigns it) and `specversion` (restored as 1.0 because that is the only version `CloudEvent::fromArray()` accepts, so storing another would make the entry unreadable), and that an empty string is the absent-attribute sentinel β€” safe because the spec has no null attribute values. Two cases pinned in the shared producer suite: an explicit `application/xml` survives, and an event with no `datacontenttype` reads back with none rather than with the empty string the store flattens it to. Co-Authored-By: Claude Opus 5 (1M context) --- src/Feed/Store.php | 16 ++++++++++++++-- tests/Feed/Producer/Base.php | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/Feed/Store.php b/src/Feed/Store.php index f5f1fc0..d298caf 100644 --- a/src/Feed/Store.php +++ b/src/Feed/Store.php @@ -67,13 +67,25 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array } } - /** @return array */ + /** + * Every attribute a CloudEvent carries, flattened to strings. + * + * `specversion` is not among them: it is restored as `1.0` because that is + * the only version {@see CloudEvent::fromArray()} accepts, so storing + * another one would make the entry permanently unreadable. `id` is not + * either β€” the store assigns it and hands it back to {@see self::decode()}. + * An empty string stands for an absent optional attribute; the spec has no + * null attribute values, so it cannot collide with a real one. + * + * @return array + */ protected static function encode(CloudEvent $event): array { return [ 'type' => $event->type, 'source' => $event->source, 'subject' => $event->subject ?? '', + 'datacontenttype' => $event->datacontenttype ?? '', 'dataschema' => $event->dataschema ?? '', 'time' => $event->time ?? '', 'data' => self::json($event->data, 'data'), @@ -94,7 +106,7 @@ protected static function decode(string $id, array $fields): CloudEvent 'data' => \json_decode(self::field($fields, 'data'), true), ]; - foreach (['subject', 'dataschema', 'time'] as $optional) { + foreach (['subject', 'datacontenttype', 'dataschema', 'time'] as $optional) { $value = self::field($fields, $optional); if ($value !== '') { diff --git a/tests/Feed/Producer/Base.php b/tests/Feed/Producer/Base.php index 02d9bd7..8746482 100644 --- a/tests/Feed/Producer/Base.php +++ b/tests/Feed/Producer/Base.php @@ -211,6 +211,41 @@ public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void $this->assertSame('ok', $event->extensions['trace']); } + /** + * A producer that encodes its payload as something other than JSON says so + * with `datacontenttype`, and losing it leaves a consumer holding data it + * can no longer interpret. Nothing about the flattening the store does is + * visible to the caller, so only a round trip can show the attribute made + * it through. + */ + public function testDatacontenttypeSurvivesAppendAndRead(): void + { + $this->producer->publish(new CloudEvent( + id: '', + type: 'test', + source: '', + datacontenttype: 'application/xml', + data: '', + )); + + $event = $this->events()[0]; + + $this->assertSame('application/xml', $event->datacontenttype); + $this->assertSame('', $event->data); + } + + /** + * CloudEvents treats an absent `datacontenttype` as meaning the data is + * JSON, so "unset" has to come back unset rather than as the empty string + * the store flattens it to. + */ + public function testAnEventWithNoDatacontenttypeReadsBackWithNone(): void + { + $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', datacontenttype: null)); + + $this->assertNull($this->events()[0]->datacontenttype); + } + public function testDataschemaSurvivesAppendAndRead(): void { $this->producer->publish(new CloudEvent( From 2ba9acd4d40ece5e3ccdcd4abb1ad9197790760f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:00:25 +0200 Subject: [PATCH 41/68] Wrap cache backend failures as Transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README promises `Exception\Transport` when "the backend or network failed: Redis errors, HTTP failures, a cursor store that is down", and `Store\Redis`/`Cursor\Redis` honour it by wrapping `\RedisException`. The cache-backed twins wrapped nothing. `Utopia\Cache\Adapter\Redis` rethrows `\RedisException` once its internal retries are exhausted, so over a Redis-backed cache that is down, `Store\Cache::read()/tip()/append()` and all three `Cursor\Cache` methods let a raw `\RedisException` escape. That breaks the base-class promise that every error this library raises extends `Utopia\Feed\Exception`, and it breaks the README's canonical consume loop: `catch (Transport)` does not catch it, so the consumer process dies on a backend blip instead of retrying β€” the exact failure mode the Transport contract exists to prevent. Both adapters now wrap the cache calls, following the Redis adapters' pattern. The try blocks stay narrow on purpose: `Cursor::key()` and the event decoding raise `Invalid`, which is the caller's bug rather than the backend's failure, and catching `\Throwable` around them would erase that distinction. A test pins it β€” an unusable name stays `Invalid` even when the backend behind the cursor is also down. `BrokenCache` grew a `raises` mode for this, so it now covers both ways a cache adapter fails: answering `false` and letting the backend's error out. It raises a plain exception rather than a `\RedisException` so the service-free suites stay service-free; what the wrapping cares about is only that the error is not one of ours. Co-Authored-By: Claude Opus 5 (1M context) --- src/Feed/Cursor/Cache.php | 39 +++++++++++++++---- src/Feed/Store/Cache.php | 19 +++++++-- tests/Feed/Producer/CacheTest.php | 62 ++++++++++++++++++++++++++++++ tests/Feed/Support/BrokenCache.php | 34 +++++++++++++--- tests/Feed/Unit/CursorTest.php | 54 ++++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 16 deletions(-) diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php index c90ffc9..cecdc5b 100644 --- a/src/Feed/Cursor/Cache.php +++ b/src/Feed/Cursor/Cache.php @@ -20,15 +20,33 @@ public function __construct( public function load(string $feed, string $consumer): ?string { - /** @var mixed $cursor */ - $cursor = $this->cache->load($this->key($feed, $consumer), $this->ttl); + // The key is shaped outside the try: an unusable name is the caller's + // bug (Invalid), not the backend's failure (Transport). + $key = $this->key($feed, $consumer); + + try { + /** @var mixed $cursor */ + $cursor = $this->cache->load($key, $this->ttl); + } catch (\Throwable $error) { + // A cache adapter over a backend that is down raises whatever that + // backend raises β€” a raw \RedisException, say. Every error this + // library reports extends Utopia\Feed\Exception, and a consumer + // retrying on Transport must not crash on a backend blip instead. + throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } return \is_string($cursor) && $cursor !== '' ? $cursor : null; } public function save(string $feed, string $consumer, string $eventId): void { - $saved = $this->cache->save($this->key($feed, $consumer), $eventId); + $key = $this->key($feed, $consumer); + + try { + $saved = $this->cache->save($key, $eventId); + } catch (\Throwable $error) { + throw new Transport("Failed to save the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } // A cache adapter reports a failed write by returning false rather // than raising β€” swallowing that would let a position silently not @@ -41,9 +59,16 @@ public function save(string $feed, string $consumer, string $eventId): void public function reset(string $feed, string $consumer): void { - // Unlike save(), purge()'s false is not a failure signal: it is also - // what an adapter answers for a key that was never there, which is - // the ordinary case for a consumer that has not saved a position yet. - $this->cache->purge($this->key($feed, $consumer)); + $key = $this->key($feed, $consumer); + + try { + // Unlike save(), purge()'s false is not a failure signal: it is + // also what an adapter answers for a key that was never there, + // which is the ordinary case for a consumer that has not saved a + // position yet. Only a raising backend is a failure to report. + $this->cache->purge($key); + } catch (\Throwable $error) { + throw new Transport("Failed to reset the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } } } diff --git a/src/Feed/Store/Cache.php b/src/Feed/Store/Cache.php index 7219c7c..2876732 100644 --- a/src/Feed/Store/Cache.php +++ b/src/Feed/Store/Cache.php @@ -41,7 +41,11 @@ public function append(CloudEvent $event): string $entries = \array_slice($entries, -$this->maxSize); } - $saved = $this->cache->save($this->key(), $entries); + try { + $saved = $this->cache->save($this->key(), $entries); + } catch (\Throwable $error) { + throw new Transport("Failed to append to the {$this->name} feed: {$error->getMessage()}", previous: $error); + } if ($saved === false) { throw new Transport("Failed to append to the {$this->name} feed"); @@ -82,11 +86,20 @@ public function read(?string $lastEventId, int $limit): array /** * @return list}> + * @throws Transport When the cache backend cannot be reached. */ private function load(): array { - /** @var mixed $stored */ - $stored = $this->cache->load($this->key(), $this->ttl); + try { + /** @var mixed $stored */ + $stored = $this->cache->load($this->key(), $this->ttl); + } catch (\Throwable $error) { + // A cache adapter over a backend that is down raises whatever that + // backend raises β€” a raw \RedisException, say. Every error this + // library reports extends Utopia\Feed\Exception, and a consumer + // retrying on Transport must not crash on a backend blip instead. + throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); + } if (!\is_array($stored)) { return []; diff --git a/tests/Feed/Producer/CacheTest.php b/tests/Feed/Producer/CacheTest.php index e321798..cb34c9f 100644 --- a/tests/Feed/Producer/CacheTest.php +++ b/tests/Feed/Producer/CacheTest.php @@ -4,9 +4,14 @@ namespace Utopia\Tests\Producer; +use Utopia\Cache\Cache as UtopiaCache; use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Appendable; +use Utopia\Feed\Exception\Transport; use Utopia\Feed\Producer; +use Utopia\Feed\Store; use Utopia\Feed\Store\Cache as CacheStore; +use Utopia\Tests\Support\BrokenCache; use Utopia\Tests\Support\UsesCache; class CacheTest extends Base @@ -66,4 +71,61 @@ public function testAForeignValueUnderTheFeedsKeyReadsAsEmpty(): void $this->assertCount(0, $this->store->read(null, 10)); $this->assertNull($this->store->tip()); } + + /** A store over a cache backend that is down, as an operator would meet it. */ + private function unreachable(): Store&Appendable + { + return new CacheStore(new UtopiaCache(new BrokenCache(raises: true)), $this->name); + } + + /** + * A cache adapter lets the backend's own error out once its internal + * retries are exhausted, so without wrapping a raw \RedisException escapes + * this library entirely. The canonical consume loop retries on Transport + * and would crash on a backend blip instead β€” the exact failure mode the + * Transport contract exists to prevent. + * + * @dataProvider operations + * @param callable(Store&Appendable): void $operation + */ + public function testABackendThatIsDownRaisesTransport(callable $operation): void + { + $store = $this->unreachable(); + + $this->expectException(Transport::class); + + $operation($store); + } + + /** + * @return array + */ + public static function operations(): array + { + return [ + 'read' => [static function (Store&Appendable $store): void { + $store->read(null, 10); + }], + 'tip' => [static function (Store&Appendable $store): void { + $store->tip(); + }], + 'append' => [static function (Store&Appendable $store): void { + $store->append(new CloudEvent(id: '', type: 'test', source: 'urn:test')); + }], + ]; + } + + /** + * A cache that answers a write with false rather than raising must be + * caught too β€” the entry is not in the feed, so reporting the position it + * would have had would invent an event no consumer can ever read. + */ + public function testACacheThatRejectsTheWriteRaisesTransport(): void + { + $store = new CacheStore(new UtopiaCache(new BrokenCache()), $this->name); + + $this->expectException(Transport::class); + + $store->append(new CloudEvent(id: '', type: 'test', source: 'urn:test')); + } } diff --git a/tests/Feed/Support/BrokenCache.php b/tests/Feed/Support/BrokenCache.php index 29972f6..cd96b2e 100644 --- a/tests/Feed/Support/BrokenCache.php +++ b/tests/Feed/Support/BrokenCache.php @@ -7,23 +7,36 @@ use Utopia\Cache\Adapter; /** - * A cache backend that cannot be written, for testing that the cache-backed - * store and cursor notice. + * A cache backend that is down, for testing that the cache-backed store and + * cursor notice. * - * It fails the way {@see \Utopia\Cache\Adapter\Redis} does rather than the way - * a test double would find convenient: `save()` catches everything internally - * and answers `false`, so a caller that only looks at the absence of an - * exception sees a successful write. + * It fails the two ways {@see \Utopia\Cache\Adapter\Redis} actually fails + * rather than the way a test double would find convenient: + * + * - `save()` catches everything internally and answers `false`, so a caller + * that only looks at the absence of an exception sees a successful write. + * - `load()` and `purge()` let the backend's own error out once the adapter's + * internal retries are exhausted β€” a raw `\RedisException` in production, + * stood in for here by a plain exception so the service-free suites stay + * service-free. What matters is that it is not a `Utopia\Feed\Exception`. */ class BrokenCache implements Adapter { + public function __construct(private readonly bool $raises = false) + { + } + public function load(string $key, int $ttl, string $hash = ''): mixed { + $this->fail(); + return false; } public function save(string $key, array|string $data, string $hash = ''): bool|string|array { + $this->fail(); + return false; } @@ -40,6 +53,8 @@ public function list(string $key): array public function purge(string $key, string $hash = ''): bool { + $this->fail(); + return false; } @@ -62,4 +77,11 @@ public function getName(?string $key = null): string { return 'broken'; } + + private function fail(): void + { + if ($this->raises) { + throw new \RuntimeException('read error on connection to redis:6379'); + } + } } diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index 8caaaf2..1c651a1 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -8,6 +8,7 @@ use Utopia\Cache\Cache as UtopiaCache; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Consumer; +use Utopia\Feed\Cursor; use Utopia\Feed\Cursor\Cache as CacheCursor; use Utopia\Feed\Cursor\None; use Utopia\Feed\Exception\Invalid; @@ -119,4 +120,57 @@ public function testResettingAPositionThatWasNeverSavedIsHarmless(): void $this->assertNull($consumer->position()); $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); } + + /** + * The other way a cache backend fails: it lets its own error out once the + * adapter's internal retries are exhausted β€” a raw \RedisException in + * production. Unwrapped, that escapes this library entirely, so the + * canonical consume loop retrying on Transport crashes on a backend blip + * instead, which is what the Transport contract exists to prevent. + * + * @dataProvider operations + * @param callable(Cursor): void $operation + */ + public function testABackendThatIsDownRaisesTransport(callable $operation): void + { + $cursor = new CacheCursor(new UtopiaCache(new BrokenCache(raises: true))); + + $this->expectException(Transport::class); + + $operation($cursor); + } + + /** + * @return array + */ + public static function operations(): array + { + return [ + 'load' => [static function (Cursor $cursor): void { + $cursor->load('edge', 'invalidator'); + }], + 'save' => [static function (Cursor $cursor): void { + $cursor->save('edge', 'invalidator', '1-0'); + }], + 'reset' => [static function (Cursor $cursor): void { + $cursor->reset('edge', 'invalidator'); + }], + ]; + } + + /** + * An unusable name is the caller's bug, not the backend's failure, and it + * stays Invalid even when the backend behind the cursor is also down β€” + * otherwise wrapping the store call would swallow the distinction. + * + * @dataProvider unusableNames + */ + public function testAnUnusableNameIsStillInvalidOnABackendThatIsDown(string $feed, string $consumer): void + { + $cursor = new CacheCursor(new UtopiaCache(new BrokenCache(raises: true))); + + $this->expectException(Invalid::class); + + $cursor->load($feed, $consumer); + } } From a6e244a8a5db10eefcc25f3abc45ae82eb85ea13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:04:56 +0200 Subject: [PATCH 42/68] Make ext-redis a suggestion rather than a requirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redis is one of several store adapters. `Store\Memory`, `Store\Cache` and consuming a feed over HTTP work without it, and the only references to `\Redis` are in type hints of classes a Redis-less service never instantiates. Declaring `ext-redis` under `require` claimed otherwise, so this package's own manifest overstated what its code needs. Moved to `suggest` with a note naming the four adapters that need it. One thing the review that prompted this assumed does not hold, and the README now says so plainly rather than implying the problem is solved: this change does not free a downstream install. `utopia-php/cache` β€” a hard dependency β€” requires `ext-redis` and `ext-memcached` itself, so a machine without them still needs `--ignore-platform-req` (verified against a scratch project: the resolver now stops at the cache package's extensions, not at ours). That constraint is not this repo's to remove. The README's development instructions traded the blanket `--ignore-platform-reqs` for three targeted `--ignore-platform-req=ext-*` flags. The blanket flag also skipped the PHP version check, which for a library requiring PHP 8.5 is the one platform requirement that must not be waived silently. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 21 +++++++++++++++++---- composer.json | 4 +++- composer.lock | 5 ++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6406c38..4664915 100644 --- a/README.md +++ b/README.md @@ -342,12 +342,17 @@ whole contract. Producer and Server run per store (`memory`, `cache`, `redis`, provoke: a cursor store that is down, a body that is not a batch, a backend that was never configured. -The service-free suites need no Redis, but dependencies declare extensions the -suites never touch (`ext-redis`, `ext-memcached`, `ext-protobuf`), so install -past them: +The service-free suites need no Redis, but `utopia-php/cache` declares +`ext-redis` and `ext-memcached` and `utopia-php/telemetry` declares +`ext-protobuf`, none of which those suites touch. Install past exactly those +rather than with a blanket `--ignore-platform-reqs`, which would also skip the +PHP version check this library actually depends on: ```bash -composer install --ignore-platform-reqs +composer install \ + --ignore-platform-req=ext-redis \ + --ignore-platform-req=ext-memcached \ + --ignore-platform-req=ext-protobuf composer test # unit + memory + cache + http ``` @@ -373,6 +378,14 @@ Utopia Feed requires PHP 8.5 or later. This library is maintained by the [Utopia Framework](https://github.com/utopia-php/framework), it is dependency light and works standalone with any PHP project. +`ext-redis` is a suggestion, not a requirement: it is needed by +`Store\Redis`, `Store\Pool`, `Cursor\Redis` and `Cursor\Pool`, and a service +using the memory or cache adapters, or consuming a feed over HTTP, never loads +those classes. Note that `utopia-php/cache` requires `ext-redis` and +`ext-memcached` itself, so a machine without them still needs +`--ignore-platform-req` to install β€” that constraint comes from the cache +package rather than from this one. + ## Copyright and license The MIT License (MIT) [http://www.opensource.org/licenses/mit-license.php](http://www.opensource.org/licenses/mit-license.php) diff --git a/composer.json b/composer.json index 5d40923..0e32ff4 100644 --- a/composer.json +++ b/composer.json @@ -40,7 +40,6 @@ "require": { "php": ">=8.5", "ext-json": "*", - "ext-redis": "*", "psr/http-client": "^1.0", "psr/http-message": "^2.0", "utopia-php/cache": "3.*", @@ -49,6 +48,9 @@ "utopia-php/psr7": "^0.2", "utopia-php/cloudevents": "0.1.*" }, + "suggest": { + "ext-redis": "Required by the Store\\Redis, Store\\Pool, Cursor\\Redis and Cursor\\Pool adapters. The other adapters, and consuming a feed over HTTP, work without it" + }, "require-dev": { "laravel/pint": "^1.21", "phpstan/phpstan": "^2.2", diff --git a/composer.lock b/composer.lock index 2b73b36..5a7b463 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "935a7e23baaac15e7abd66b7e9926cdb", + "content-hash": "022655166a94f83c4c6d0213ede8aeda", "packages": [ { "name": "brick/math", @@ -4250,8 +4250,7 @@ "prefer-lowest": false, "platform": { "php": ">=8.5", - "ext-json": "*", - "ext-redis": "*" + "ext-json": "*" }, "platform-dev": {}, "plugin-api-version": "2.9.0" From d7c2fcbee0a351bfe5cf3c4cf6c8f0e0ed815d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:08:04 +0200 Subject: [PATCH 43/68] Escape names when building backend keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feed and cursor keys were built by joining names with `:` β€” a character a name is allowed to contain, since names are only validated as non-empty. Joined raw, the mapping is not injective, and both collisions corrupt data rather than losing it: - A feed named `edge:cursor:x` keyed to `feed:edge:cursor:x`, which is also the cursor key of consumer `x` on feed `edge`. `Cursor\Redis::save()` then `SET`s over an `XADD` stream. The new Redis test shows this is not the harmless WRONGTYPE it looks like β€” the whole feed is gone afterwards. - Two distinct pairs could share one position: (`a:cursor:b`, `c`) and (`a`, `b:cursor:c`) both joined to `feed:a:cursor:b:cursor:c`, giving two unrelated consumers the "sharing a name" hazard the README warns about without anyone sharing a name. Percent-encoding `:`, and `%` itself so the encoding stays reversible, makes the collisions unexpressible. Escaping rather than rejecting is deliberate: `Remote` reads third-party feeds whose names are arbitrary path segments, and refusing one over a detail of how this library stores positions would make that feed unconsumable. A name containing neither character comes through untouched, so the layout the README documents β€” `feed:`, `feed::cursor:` β€” still reads from a shell for every name anyone would actually pick. Both key shapes now live in one `Key` class. That also removes the duplicated `'feed:' . $this->name` in `Store\Redis` and `Store\Cache`, which was the reason a store could drift from the escaping the cursor does. Co-Authored-By: Claude Opus 5 (1M context) --- src/Feed/Cursor.php | 5 +- src/Feed/Key.php | 48 ++++++++++++++++ src/Feed/Store.php | 6 ++ src/Feed/Store/Cache.php | 5 -- src/Feed/Store/Redis.php | 6 +- tests/Feed/Producer/RedisTest.php | 19 +++++++ tests/Feed/Unit/KeyTest.php | 91 +++++++++++++++++++++++++++++++ 7 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 src/Feed/Key.php create mode 100644 tests/Feed/Unit/KeyTest.php diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index c81b115..444e616 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -24,7 +24,8 @@ abstract public function save(string $feed, string $consumer, string $eventId): abstract public function reset(string $feed, string $consumer): void; /** - * The one place a cursor key is shaped, so every store agrees on it. + * The gate every cursor operation goes through, so no adapter can build a + * key of its own and skip the check or diverge from the layout. * * @throws Invalid When either name is empty. */ @@ -34,6 +35,6 @@ protected function key(string $feed, string $consumer): string throw new Invalid('Cursor requires a feed and a consumer name'); } - return 'feed:' . $feed . ':cursor:' . $consumer; + return Key::cursor($feed, $consumer); } } diff --git a/src/Feed/Key.php b/src/Feed/Key.php new file mode 100644 index 0000000..f5de463 --- /dev/null +++ b/src/Feed/Key.php @@ -0,0 +1,48 @@ +name; } + /** The backend key this feed's events live under. */ + protected function key(): string + { + return Key::feed($this->name); + } + /** @return list */ abstract public function read(?string $lastEventId, int $limit): array; diff --git a/src/Feed/Store/Cache.php b/src/Feed/Store/Cache.php index 2876732..dcc2070 100644 --- a/src/Feed/Store/Cache.php +++ b/src/Feed/Store/Cache.php @@ -121,9 +121,4 @@ private function load(): array return $entries; } - - private function key(): string - { - return 'feed:' . $this->name; - } } diff --git a/src/Feed/Store/Redis.php b/src/Feed/Store/Redis.php index 9b0d227..01f16ec 100644 --- a/src/Feed/Store/Redis.php +++ b/src/Feed/Store/Redis.php @@ -24,7 +24,7 @@ public function __construct( public function append(CloudEvent $event): string { try { - $id = $this->redis->xAdd('feed:' . $this->name, '*', self::encode($event), $this->maxSize, true); + $id = $this->redis->xAdd($this->key(), '*', self::encode($event), $this->maxSize, true); } catch (\RedisException $error) { throw new Transport("Failed to append to the {$this->name} feed: {$error->getMessage()}", previous: $error); } @@ -39,7 +39,7 @@ public function append(CloudEvent $event): string public function tip(): ?string { try { - $entries = $this->redis->xRevRange('feed:' . $this->name, '+', '-', 1); + $entries = $this->redis->xRevRange($this->key(), '+', '-', 1); } catch (\RedisException $error) { throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); } @@ -58,7 +58,7 @@ public function read(?string $lastEventId, int $limit): array $start = $lastEventId === null ? '-' : Id::after($lastEventId); try { - $entries = $this->redis->xRange('feed:' . $this->name, $start, '+', $limit); + $entries = $this->redis->xRange($this->key(), $start, '+', $limit); } catch (\RedisException $error) { throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); } diff --git a/tests/Feed/Producer/RedisTest.php b/tests/Feed/Producer/RedisTest.php index 5d011ab..310f0ce 100644 --- a/tests/Feed/Producer/RedisTest.php +++ b/tests/Feed/Producer/RedisTest.php @@ -24,6 +24,25 @@ public function testEventsLiveInAStreamUnderTheFeedsKey(): void $this->assertSame(2, $this->redis()->xLen('feed:' . $this->name)); } + /** + * Feeds and cursors share one Redis keyspace, and both keys are built by + * joining names with `:`. Joined raw, a feed named `:cursor:x` takes + * the key consumer `x`'s position on `` occupies, so a cursor `SET` + * lands on a stream β€” a WRONGTYPE at best, and at worst one silently + * destroying the other. + */ + public function testAFeedNamedLikeACursorKeyDoesNotCollideWithOne(): void + { + $store = $this->store($this->name . ':cursor:x'); + (new Producer($store, 'urn:test'))->produce('a'); + + $cursor = $this->cursor(); + $cursor->save($this->name, 'x', '1-0'); + + $this->assertSame('1-0', $cursor->load($this->name, 'x'), 'The position is readable back'); + $this->assertCount(1, $store->read(null, 10), 'And the feed still holds its event'); + } + /** Trimming must happen on the server, not only in what read() returns. */ public function testTheStreamItselfIsTrimmed(): void { diff --git a/tests/Feed/Unit/KeyTest.php b/tests/Feed/Unit/KeyTest.php new file mode 100644 index 0000000..c14c1f2 --- /dev/null +++ b/tests/Feed/Unit/KeyTest.php @@ -0,0 +1,91 @@ +assertSame('feed:edge', Key::feed('edge')); + $this->assertSame('feed:edge:cursor:invalidator', Key::cursor('edge', 'invalidator')); + } + + /** + * The collision that corrupts data rather than losing it: the stream key + * of a feed named `edge:cursor:x` used to be the cursor key of consumer + * `x` on feed `edge`, so a cursor `SET` landed on an `XADD` stream. + */ + public function testAFeedNameCannotCollideWithACursorKey(): void + { + $this->assertNotSame(Key::feed('edge:cursor:x'), Key::cursor('edge', 'x')); + } + + /** + * And the collision that silently shares one position between two + * unrelated consumers β€” the "two processes sharing a name" hazard the + * README warns about, arrived at without anyone sharing a name. + */ + public function testTwoDistinctPairsCannotShareACursorKey(): void + { + $this->assertNotSame(Key::cursor('a:cursor:b', 'c'), Key::cursor('a', 'b:cursor:c')); + } + + /** + * Escaping is only injective if the escape character is escaped too: + * without that, `a:b` and `a%3Ab` would map to the same key and the fix + * would just move the collision somewhere less obvious. + */ + public function testTheEscapeCharacterIsItselfEscaped(): void + { + $this->assertNotSame(Key::feed('a:b'), Key::feed('a%3Ab')); + $this->assertNotSame(Key::cursor('a:b', 'c'), Key::cursor('a%3Ab', 'c')); + } + + /** + * @return array + */ + public static function names(): array + { + return [ + 'separator' => ['a:b'], + 'escape character' => ['a%b'], + 'both' => ['a%3A:b'], + 'the cursor infix' => ['edge:cursor:x'], + 'unicode' => ['ΓΌnΓ―cΓΈde'], + 'slashes' => ['a/b'], + ]; + } + + /** + * @dataProvider names + */ + public function testEveryNameKeepsItsOwnKeys(string $name): void + { + $others = \array_column(self::names(), 0); + + foreach ($others as $other) { + if ($other === $name) { + continue; + } + + $this->assertNotSame(Key::feed($other), Key::feed($name)); + $this->assertNotSame(Key::cursor($other, 'c'), Key::cursor($name, 'c')); + $this->assertNotSame(Key::cursor('f', $other), Key::cursor('f', $name)); + } + } +} From f9fd9fc2a102781f4f1f00658db9708c8670e907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:11:06 +0200 Subject: [PATCH 44/68] Pin the pooled store's borrow-per-read property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most of what this task asked for already landed with the test rework: `Producer/PoolTest`, `Server/PoolTest` and `Consumer/PoolTest` run the shared scenarios over `Store\Pool` and `Cursor\Pool`, as their own CI job, so the delegates, their return values and the inner store's constructor validation are all exercised. One claim the review named was genuinely unpinned, and it is the one that matters most: `Store\Pool` exists because a long poll borrows per read rather than once around the whole loop, so a held poll never ties up a connection. Nothing tested that. Hoisting the borrow β€” an inviting "optimization", since it looks like it saves pool traffic β€” would have removed the class's entire reason to exist with the whole suite still green, and would only surface in production as a pool exhausted by idle consumers holding connections for up to 30 seconds each. `CountingStack` records releases so the property is asserted directly: a ~500ms poll on a 50ms interval must release many times, not once. Confirmed by hoisting the borrow β€” the test fails with "1 is greater than 2" and nothing else in the suite notices. Two leak tests come with it, on the store and cursor sides: after a round of reads, tips, seeks and resets the pool is whole again. A leaked connection fails nothing functionally until the pool runs dry, which in a service is minutes into production rather than here. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Consumer/PoolTest.php | 22 ++++++++++++++ tests/Feed/Server/PoolTest.php | 45 ++++++++++++++++++++++++++++ tests/Feed/Support/CountingStack.php | 27 +++++++++++++++++ tests/Feed/Support/UsesPool.php | 35 ++++++++++++++-------- 4 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 tests/Feed/Support/CountingStack.php diff --git a/tests/Feed/Consumer/PoolTest.php b/tests/Feed/Consumer/PoolTest.php index b650cf3..bd95bd2 100644 --- a/tests/Feed/Consumer/PoolTest.php +++ b/tests/Feed/Consumer/PoolTest.php @@ -4,9 +4,31 @@ namespace Utopia\Tests\Consumer; +use Utopia\CloudEvents\CloudEvent; use Utopia\Tests\Support\UsesPool; class PoolTest extends Base { use UsesPool; + + /** + * Every cursor operation borrows a connection and must give it back β€” + * including `reset()` and a `seek()` that the shared scenarios drive but + * never watch the pool through. A leak here drains the pool over a + * consumer's lifetime rather than failing anything outright. + */ + public function testEveryCursorOperationReturnsItsConnection(): void + { + $this->producer->produce('a'); + $second = $this->producer->produce('b'); + + $consumer = $this->consumer(); + + $consumer->consume(fn (CloudEvent $event) => null); + $consumer->position(); + $consumer->seek($second); + $consumer->reset(); + + $this->assertSame(self::POOL_SIZE, $this->pool()->count(), 'The pool is whole again'); + } } diff --git a/tests/Feed/Server/PoolTest.php b/tests/Feed/Server/PoolTest.php index 19a8013..2dcd6b7 100644 --- a/tests/Feed/Server/PoolTest.php +++ b/tests/Feed/Server/PoolTest.php @@ -4,9 +4,54 @@ namespace Utopia\Tests\Server; +use Utopia\Feed\Server; +use Utopia\Feed\Store\Pool as PoolStore; +use Utopia\Tests\Support\CountingStack; use Utopia\Tests\Support\UsesPool; class PoolTest extends Base { use UsesPool; + + /** + * The property the pooled store exists for, and the only one the shared + * scenarios cannot show: a long poll is a loop of reads, and the store + * borrows for each read rather than once around the whole loop. Held for + * the wait, one poll would tie up a connection for up to 30 seconds, so a + * handful of idle consumers would exhaust the pool. + * + * Asserted as a count of releases, so an "optimization" that hoists the + * borrow out of the loop fails here instead of silently removing the + * class's entire reason to exist. + */ + public function testAHeldPollBorrowsPerReadRatherThanForTheWholeWait(): void + { + $adapter = new CountingStack(); + $store = new PoolStore(self::poolOver($adapter), $this->name, pollInterval: 50); + + $events = (new Server($store))->poll(null, 10, 500); + + $this->assertCount(0, $events, 'The feed is empty, so the poll waits out its timeout'); + $this->assertGreaterThan( + 2, + $adapter->releases, + 'A ~500ms poll on a 50ms interval reads many times; borrowing once for the whole wait would release once', + ); + } + + /** + * The other half of the same property: every borrow is given back. A leak + * would not fail a functional test until the pool ran dry, which in a + * service is minutes into production rather than here. + */ + public function testEveryBorrowIsReturnedToThePool(): void + { + $this->producer->produce('a'); + + $this->server->read(); + $this->server->tip(); + $this->server->serve([]); + + $this->assertSame(self::POOL_SIZE, $this->pool()->count(), 'The pool is whole again'); + } } diff --git a/tests/Feed/Support/CountingStack.php b/tests/Feed/Support/CountingStack.php new file mode 100644 index 0000000..528676b --- /dev/null +++ b/tests/Feed/Support/CountingStack.php @@ -0,0 +1,27 @@ +releases++; + + return parent::push($connection); + } +} diff --git a/tests/Feed/Support/UsesPool.php b/tests/Feed/Support/UsesPool.php index 969e924..7c3cf66 100644 --- a/tests/Feed/Support/UsesPool.php +++ b/tests/Feed/Support/UsesPool.php @@ -20,25 +20,36 @@ */ trait UsesPool { + /** The pool's connection count, and so the number of concurrent borrows it allows. */ + protected const int POOL_SIZE = 4; + /** @var UtopiaPool<\Redis|\RedisCluster>|null */ private ?UtopiaPool $pool = null; - /** @return UtopiaPool<\Redis|\RedisCluster> */ - protected function pool(): UtopiaPool + /** + * A pool over the suite's Redis, on the adapter given β€” the default + * {@see Stack} for ordinary use, or one that records borrows for a test + * that asserts on how the store uses the pool rather than on what it reads. + * + * @return UtopiaPool<\Redis|\RedisCluster> + */ + protected static function poolOver(Stack $adapter): UtopiaPool { - if ($this->pool === null) { - /** @var UtopiaPool<\Redis|\RedisCluster> $pool */ - $pool = new UtopiaPool(new Stack(), 'feed-tests', 4, static function (): \Redis { - $redis = new \Redis(); - $redis->connect((string) (\getenv('REDIS_HOST') ?: 'redis'), (int) (\getenv('REDIS_PORT') ?: 6379)); + /** @var UtopiaPool<\Redis|\RedisCluster> $pool */ + $pool = new UtopiaPool($adapter, 'feed-tests', self::POOL_SIZE, static function (): \Redis { + $redis = new \Redis(); + $redis->connect((string) (\getenv('REDIS_HOST') ?: 'redis'), (int) (\getenv('REDIS_PORT') ?: 6379)); - return $redis; - }); + return $redis; + }); - $this->pool = $pool; - } + return $pool; + } - return $this->pool; + /** @return UtopiaPool<\Redis|\RedisCluster> */ + protected function pool(): UtopiaPool + { + return $this->pool ??= self::poolOver(new Stack()); } protected function store(string $name, int $maxSize = 100_000, int $pollInterval = 500): Store&Appendable From 36bd329696023ec1f4cd7f9a612f6baa2b5e9a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:15:46 +0200 Subject: [PATCH 45/68] Stop the cache store loading the feed on every poll tick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Store\Cache` keeps the whole feed under one key, so `read()` fetched and re-validated every retained entry even to return nothing. Under the inherited poll loop that ran per `pollInterval` tick, per waiting consumer, for up to 30 seconds a request β€” and a caught-up consumer on a quiet feed is the common case, not the exception. The newest id now also lives under a small second key, so a poll that is provably caught up answers from that instead of loading the feed. Two properties keep it honest: - The marker is written *before* the feed, so it can be ahead but never behind. Behind, it would report a caught-up consumer and the event would never arrive; ahead only costs a read that finds nothing. - It is only ever used to skip a read, never to answer one. A missing, expired or unparseable marker falls straight through to the real read, and `tip()` still reads the feed rather than trusting it. Also lowered the adapter's default `maxSize` from the 100 000 it inherited from the Redis store to 1 000. That number was chosen for a backend where trimming is server-side and reads are ranged; here retention is also the size of every append's read-modify-write, so at the inherited default a single `produce()` moved megabytes through the cache in both directions. This is a behaviour change β€” a feed relying on 100 000 entries of retention now has to ask for it β€” which is why it is happening before the first release rather than after. Appends are still O(feed size): that is inherent to one key per feed, and the retention default is now scaled to it rather than hiding it. The README says all of this where the adapter is introduced. Costs are asserted, not just described: `CountingCache` records reads per key, and a 400ms caught-up poll must read the marker repeatedly and the feed exactly zero times. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 13 +++++ src/Feed/Key.php | 9 ++++ src/Feed/Store/Cache.php | 67 +++++++++++++++++++++++-- tests/Feed/Producer/CacheTest.php | 75 ++++++++++++++++++++++++++++ tests/Feed/Support/CountingCache.php | 34 +++++++++++++ 5 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 tests/Feed/Support/CountingCache.php diff --git a/README.md b/README.md index 4664915..0b6b049 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,19 @@ store that retains nothing is a misconfiguration, so the constructor throws | `Store\Memory` | Tests and single-process development | | `Store\None` | No backend configured β€” throws on use, so a misconfigured service fails loudly instead of dropping events | +`Store\Cache` keeps the whole feed under one key, which is what makes it cheap +to adopt and what bounds how far it scales: an append is a read-modify-write of +the entire retained feed, so retention is also the cost of producing one event. +Its default `maxSize` is therefore 1 000 rather than the 100 000 the Redis store +keeps, where trimming happens server-side and reads are ranged. Raising it is a +fine choice for a low-rate feed β€” just a deliberate one. + +Long polls do not pay that cost: the newest id is kept under a second, tiny key +(`feed::tip`), so a caught-up consumer waiting out a 30-second poll checks +that marker each tick instead of loading the feed. The marker is written before +the feed and is only ever used to skip a read, never to answer one, so a missing +or stale marker costs a wasted read rather than a missed event. + ### Cursors A cursor is keyed by feed and consumer name, so one store serves every feed a diff --git a/src/Feed/Key.php b/src/Feed/Key.php index f5de463..a0ce4ed 100644 --- a/src/Feed/Key.php +++ b/src/Feed/Key.php @@ -35,6 +35,15 @@ public static function cursor(string $feed, string $consumer): string return 'feed:' . self::escape($feed) . ':cursor:' . self::escape($consumer); } + /** + * The key a feed's newest id lives under, for backends that cannot answer + * "is there anything after this position?" without reading the feed. + */ + public static function tip(string $name): string + { + return self::feed($name) . ':tip'; + } + /** * Percent-encode the separator, and the escape character itself so the * encoding stays reversible. A name with neither is left exactly as it diff --git a/src/Feed/Store/Cache.php b/src/Feed/Store/Cache.php index dcc2070..3bbd07d 100644 --- a/src/Feed/Store/Cache.php +++ b/src/Feed/Store/Cache.php @@ -8,6 +8,7 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Exception\Transport; use Utopia\Feed\Id; +use Utopia\Feed\Key; use Utopia\Feed\Appendable; use Utopia\Feed\Store; @@ -15,6 +16,17 @@ class Cache extends Store implements Appendable { public const int TTL = 30 * 24 * 60 * 60; // 30 days + /** + * Deliberately far below the {@see Store::MAX_SIZE} the Redis store + * inherits, where trimming happens server-side and reads are ranged. + * Here the whole feed lives under one key, so retention is also the size + * of every append's read-modify-write: at 100 000 entries a single + * `produce()` moves megabytes through the cache both ways. A larger cap + * is a fine choice for a low-rate feed, but it should be one somebody + * made rather than one inherited from a backend with other costs. + */ + protected const int MAX_SIZE = 1_000; // entries + public function __construct( protected readonly UtopiaCache $cache, string $name, @@ -41,8 +53,24 @@ public function append(CloudEvent $event): string $entries = \array_slice($entries, -$this->maxSize); } + // The tip marker goes first, so it is never behind the feed. A crash + // between the two writes leaves it ahead, which only costs a read that + // finds nothing; behind, it would report a caught-up consumer and the + // event would never be delivered. + $this->write(Key::tip($this->name), $id); + $this->write($this->key(), $entries); + + return $id; + } + + /** + * @param string|array $value + * @throws Transport When the write fails, either way a cache adapter can. + */ + private function write(string $key, string|array $value): void + { try { - $saved = $this->cache->save($this->key(), $entries); + $saved = $this->cache->save($key, $value); } catch (\Throwable $error) { throw new Transport("Failed to append to the {$this->name} feed: {$error->getMessage()}", previous: $error); } @@ -50,8 +78,6 @@ public function append(CloudEvent $event): string if ($saved === false) { throw new Transport("Failed to append to the {$this->name} feed"); } - - return $id; } public function tip(): ?string @@ -65,6 +91,10 @@ public function read(?string $lastEventId, int $limit): array { $lastEventId = $this->resolve($lastEventId); + if ($lastEventId !== null && $this->caughtUp($lastEventId)) { + return []; + } + $after = $lastEventId === null ? null : Id::decode($lastEventId); $events = []; @@ -84,6 +114,37 @@ public function read(?string $lastEventId, int $limit): array return $events; } + /** + * Whether the feed provably holds nothing after $lastEventId, decided from + * the tip marker alone. + * + * The whole feed lives under one key, so answering this by reading it + * costs the entire retained feed β€” every poll tick, per waiting consumer, + * for up to 30 seconds a request. The marker turns the common case, a + * caught-up consumer waiting on a quiet feed, into one small read. + * + * Only ever used to skip work, never to invent an answer: the marker is + * written before the feed, so it is never behind, and a missing or + * unreadable one falls through to the real read. + * + * @throws Transport When the cache backend cannot be reached. + */ + private function caughtUp(string $lastEventId): bool + { + try { + /** @var mixed $tip */ + $tip = $this->cache->load(Key::tip($this->name), $this->ttl); + } catch (\Throwable $error) { + throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); + } + + if (!\is_string($tip) || !Id::isValid($tip)) { + return false; + } + + return Id::decode($tip) <= Id::decode($lastEventId); + } + /** * @return list}> * @throws Transport When the cache backend cannot be reached. diff --git a/tests/Feed/Producer/CacheTest.php b/tests/Feed/Producer/CacheTest.php index cb34c9f..d7dde4c 100644 --- a/tests/Feed/Producer/CacheTest.php +++ b/tests/Feed/Producer/CacheTest.php @@ -8,10 +8,13 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Appendable; use Utopia\Feed\Exception\Transport; +use Utopia\Feed\Key; use Utopia\Feed\Producer; +use Utopia\Feed\Server; use Utopia\Feed\Store; use Utopia\Feed\Store\Cache as CacheStore; use Utopia\Tests\Support\BrokenCache; +use Utopia\Tests\Support\CountingCache; use Utopia\Tests\Support\UsesCache; class CacheTest extends Base @@ -72,6 +75,78 @@ public function testAForeignValueUnderTheFeedsKeyReadsAsEmpty(): void $this->assertNull($this->store->tip()); } + /** + * The whole feed lives under one key, so answering "anything new?" by + * reading it costs the entire retained feed β€” every poll tick, per waiting + * consumer, for up to 30 seconds a request. A caught-up consumer on a + * quiet feed is the common case, and it must not pay that. + */ + public function testACaughtUpPollReadsTheTipMarkerRatherThanTheFeed(): void + { + $adapter = new CountingCache(); + $store = new CacheStore(new UtopiaCache($adapter), $this->name, pollInterval: 50); + + $last = (new Producer($store, 'urn:test'))->produce('a'); + $adapter->forget(); + + $events = (new Server($store))->poll($last, 10, 400); + + $this->assertCount(0, $events, 'The consumer is caught up, so the poll waits out its timeout'); + $this->assertGreaterThan(2, $adapter->reads(Key::tip($this->name)), 'Each tick checks the marker'); + $this->assertSame(0, $adapter->reads(Key::feed($this->name)), 'And never loads the feed to learn nothing'); + } + + /** + * The marker is only ever allowed to skip work, never to invent an answer. + * A cache is free to drop one key and keep another, so a marker that is + * gone must fall through to the real read rather than read as "caught up" + * and strand the consumer. + */ + public function testAMissingTipMarkerFallsBackToReadingTheFeed(): void + { + $first = $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->cache()->purge(Key::tip($this->name)); + + $this->assertCount(1, $this->store->read($first, 10)); + } + + /** + * And the gate must not be sticky: a consumer told it was caught up has to + * see the next event, or a quiet feed would stay quiet forever. + */ + public function testAnEventAppendedAfterACaughtUpReadIsStillDelivered(): void + { + $first = $this->producer->produce('a'); + + $this->assertCount(0, $this->store->read($first, 10), 'Caught up'); + + $this->producer->produce('b'); + + $this->assertCount(1, $this->store->read($first, 10), 'And no longer'); + } + + /** + * Retention is also the size of every append's read-modify-write here, so + * the cache store keeps a far smaller default than the Redis store, whose + * trimming is server-side and whose reads are ranged. + */ + public function testTheDefaultRetentionIsScaledToWhatAnAppendCosts(): void + { + $store = new CacheStore($this->cache(), $this->name); + $producer = new Producer($store, 'urn:test'); + + foreach (\range(1, 1_001) as $i) { + $producer->produce('event-' . $i); + } + + $events = $store->read(null, 2_000); + + $this->assertCount(1_000, $events); + $this->assertSame('event-2', $events[0]->type, 'The oldest went first'); + } + /** A store over a cache backend that is down, as an operator would meet it. */ private function unreachable(): Store&Appendable { diff --git a/tests/Feed/Support/CountingCache.php b/tests/Feed/Support/CountingCache.php new file mode 100644 index 0000000..802fd48 --- /dev/null +++ b/tests/Feed/Support/CountingCache.php @@ -0,0 +1,34 @@ + Reads per key. */ + public array $loads = []; + + public function load(string $key, int $ttl, string $hash = ''): mixed + { + $this->loads[$key] = ($this->loads[$key] ?? 0) + 1; + + return parent::load($key, $ttl, $hash); + } + + public function forget(): void + { + $this->loads = []; + } + + public function reads(string $key): int + { + return $this->loads[$key] ?? 0; + } +} From db8e293c802b967723e91fc8a8e70622de3a949f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:16:42 +0200 Subject: [PATCH 46/68] Run CI on main, and pin the linter's PHP version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three workflows triggered on `pull_request` only, so nothing ever ran on the default branch. The README's build badge points at `tests.yml`, a workflow with no runs on `main` β€” it would have shown "no status" indefinitely. More usefully, a merge that breaks `main` through a semantic conflict with an earlier merge stayed invisible until it surfaced inside somebody else's unrelated PR. All three now also run on push to `main`. The linter ran pint inside the unpinned `composer` image. This codebase uses PHP 8.5 syntax β€” `new Stream\Factory()->createStream(...)` without parens, typed class constants β€” which pint can only parse on a new enough runtime, so whichever PHP `composer:latest` happened to ship was an unstated build dependency: the job would break, or quietly under-lint, whenever the image moved. Everything else here pins carefully (`composer:2.7`, `redis:7.2-alpine`, `utopia-base:php-8.5-2.1.0`, checkout by SHA); this was the one exception. It now runs against the project's own test image, the way the analysis job already does, which pins the runtime and drops the duplicate dependency install path with it. Verified by running the job's exact commands. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/analysis.yml | 5 ++++- .github/workflows/linter.yml | 18 ++++++++++++++---- .github/workflows/tests.yml | 5 ++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 3a34d76..2f898d4 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -1,6 +1,9 @@ name: "Static analysis" -on: [pull_request] +on: + pull_request: + push: + branches: [main] jobs: check: diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index bdc8e8e..82454df 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -1,6 +1,9 @@ name: "Linter" -on: [pull_request] +on: + pull_request: + push: + branches: [main] jobs: lint: @@ -11,7 +14,14 @@ jobs: - name: Checkout repository uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - name: Run Linter + # Run against the test image rather than a bare composer container, as + # the analysis job does: the codebase uses PHP 8.5 syntax that pint can + # only parse on a new enough runtime, and whatever version `composer:latest` + # happens to ship would otherwise be an unstated build dependency. + - name: Build run: | - docker run --rm -v $PWD:/app composer sh -c \ - "composer install --profile --ignore-platform-reqs && composer lint" + docker compose build tests + docker compose up -d tests + + - name: Run Linter + run: docker compose exec -T tests composer lint diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 850bdb0..3a9d0fb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,9 @@ name: "Tests" -on: [pull_request] +on: + pull_request: + push: + branches: [main] jobs: tests: From d71568c5e565c74db243d7cc273caa33ce6f29fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:18:59 +0200 Subject: [PATCH 47/68] Let seek() take the ids a third-party feed actually uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The consuming side is deliberately interoperable: `Remote::event()` accepts any non-empty id, http-feeds endpoints commonly use UUIDs, and `consume()` tracks such a position and sends it back as `lastEventId` without complaint. But `seek()` gated on `Id::isValid()`, which describes the `{ms}-{seq}` shape a Redis stream mints β€” a producer-side implementation detail. Against a third-party feed everything therefore worked right up until the day an operator needed the documented poison-event escape hatch, and then `seek($poisonEventId)` rejected the very id the consumer had just handled and saved. There was no workaround short of writing to the cursor store by hand. The same inconsistency from the other side: `consume()` saved those ids happily, so "positions must be well formed" was only ever enforced on the manual path. The rule is now the feed's to state rather than this library's. A local `Store` mints its positions and pages by decoding them, so a malformed id is a caller's mistake and is still refused. Any other `Readable` β€” a `Remote`, or someone's own implementation β€” is the authority on its ids, so any non-empty one is accepted. Both still refuse the tip sentinel, which stands for wherever the feed ends when the request arrives: a start, not a position, and already expressible as `reset()` on a START_TIP consumer. The shared consumer scenarios split accordingly: ids no feed could use stay asserted everywhere, shape judgements run only where the feed owns the shape, and `Consumer/HttpTest` gains the case this is all about β€” a UUID seek that persists. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 11 +++++-- src/Feed/Consumer.php | 23 +++++++++++++-- tests/Feed/Consumer/Base.php | 49 +++++++++++++++++++++++++++++--- tests/Feed/Consumer/HttpTest.php | 25 ++++++++++++++++ 4 files changed, 100 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0b6b049..2e15a9b 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,15 @@ fails with a 4xx `Transport` error rather than silently replaying the backlog. retained event (or the tip, for a `Consumer::START_TIP` consumer). - `seek($eventId)` β€” treat `$eventId` as the last event handled; the next run starts strictly *after* it. Persisted immediately; a store failure surfaces - as `Transport`. The id must be well formed but need not still exist in the - feed. + as `Transport`. The id need not still exist in the feed. + +What counts as a usable id is the feed's to say. On a local store β€” which mints +`{ms}-{seq}` positions and pages by decoding them β€” anything else is rejected +as `Invalid`. On a feed read over HTTP the producer is the authority, so any +non-empty id is accepted: http-feeds endpoints commonly use UUIDs, and refusing +one would take the escape hatch below away from precisely the consumers that +have no way around it. Both refuse the tip sentinel `$`, which stands for a +start rather than a position. Either is safe to call from inside a handler: a run that finishes after the move keeps its own progress to itself rather than saving over the newer diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index 6e9fc42..dbde968 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -117,12 +117,31 @@ public function position(): ?string } /** - * @throws Exception\Invalid When $eventId is not a feed position (the tip sentinel included) + * Move the position by hand. + * + * What counts as a usable id is the feed's to say, not this library's. + * A local {@see Store} mints `{ms}-{seq}` positions and pages by decoding + * them, so anything else is a caller's mistake and is rejected here rather + * than on the next read. A feed reached over HTTP is another producer's: + * http-feeds endpoints commonly use UUIDs, `Remote` accepts any non-empty + * id, and `consume()` already tracks and sends one back β€” so refusing one + * here would take the documented poison-event escape hatch away from + * exactly the consumers that cannot work around it. + * + * The tip sentinel is refused either way: it stands for "wherever the feed + * ends when the request arrives", which is a start, not a position. That + * is what `reset()` on a {@see self::START_TIP} consumer expresses. + * + * @throws Exception\Invalid When $eventId cannot be a position on this feed * @throws Exception When the cursor store cannot be written */ public function seek(string $eventId): void { - if (!Id::isValid($eventId)) { + $usable = $this->feed instanceof Store + ? Id::isValid($eventId) + : $eventId !== '' && $eventId !== Readable::TIP; + + if (!$usable) { throw new Exception\Invalid('Invalid feed event id: ' . $eventId); } diff --git a/tests/Feed/Consumer/Base.php b/tests/Feed/Consumer/Base.php index 9bcf332..edc8b80 100644 --- a/tests/Feed/Consumer/Base.php +++ b/tests/Feed/Consumer/Base.php @@ -487,9 +487,17 @@ public function testAResetMadeInsideAHandlerIsNotOverwritten(): void } /** - * @dataProvider notPositions + * Whether the feed under test mints the positions it pages by, and so is + * the authority on their shape. A local store is; another producer's feed, + * read over HTTP, is not. */ - public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void + protected function ownsItsIdFormat(): bool + { + return true; + } + + /** A rejected seek must leave both positions exactly where they were. */ + private function assertSeekRejected(string $id): void { $first = $this->producer->produce('a'); $this->cursor->save($this->name, 'invalidator', $first); @@ -508,15 +516,48 @@ public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void } /** + * @dataProvider noPositions + */ + public function testSeekRejectsAnIdNoFeedCouldUse(string $id): void + { + $this->assertSeekRejected($id); + } + + /** + * Rejected whatever the feed is: an empty string names nothing, and the + * tip sentinel stands for wherever the feed ends when the request arrives + * β€” a start rather than a position. + * * @return array */ - public static function notPositions(): array + public static function noPositions(): array { return [ 'empty' => [''], + 'the tip sentinel' => ['$'], + ]; + } + + /** + * @dataProvider notPositions + */ + public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void + { + if (!$this->ownsItsIdFormat()) { + $this->markTestSkipped('Only the feed that mints its positions can judge their shape'); + } + + $this->assertSeekRejected($id); + } + + /** + * @return array + */ + public static function notPositions(): array + { + return [ 'not an id' => ['abc'], 'too many parts' => ['1-2-3'], - 'the tip sentinel' => ['$'], ]; } diff --git a/tests/Feed/Consumer/HttpTest.php b/tests/Feed/Consumer/HttpTest.php index e54f457..e6e48c2 100644 --- a/tests/Feed/Consumer/HttpTest.php +++ b/tests/Feed/Consumer/HttpTest.php @@ -34,6 +34,31 @@ protected function source(Store&Appendable $store): Adapter|Readable return $this->endpoint = new FeedServer(new Server($store)); } + /** The feed belongs to another producer, which mints its own ids. */ + protected function ownsItsIdFormat(): bool + { + return false; + } + + /** + * http-feeds endpoints commonly use UUIDs, and `Remote` reads them fine β€” + * a consumer tracks such a position and sends it back as `lastEventId` + * without complaint. Judging the shape here as well would take the + * poison-event escape hatch away from exactly the consumers that have no + * way around it: the id `seek()` refused would be one the consumer itself + * had just handled and saved. + */ + public function testSeekAcceptsTheOpaqueIdARemoteFeedMayUse(): void + { + $opaque = '550e8400-e29b-41d4-a716-446655440000'; + + $consumer = $this->consumer(); + $consumer->seek($opaque); + + $this->assertSame($opaque, $consumer->position()); + $this->assertSame($opaque, $this->cursor->load($this->name, 'invalidator'), 'And it is persisted, so a restart resumes from it'); + } + public function testTheProducerCachesFullBatchesAndNothingElse(): void { foreach (\range(1, 5) as $i) { From 2a91682891c7d5da02283379987c7cd19245d132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:20:14 +0200 Subject: [PATCH 48/68] Reject a lastEventId that is not a string in serve() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docblock promised `Invalid` when `lastEventId` is present but is neither a position nor the tip sentinel, but the coercion ran first and folded every non-string into `null`. PHP parses `?lastEventId[]=1-0` into an array, so that request was *present*, was *not* a position, and read as "from the oldest retained event". The result was the worst possible answer to a malformed parameter: instead of a 400, the caller got a full replay of the retained feed β€” the most expensive response the endpoint has β€” and a caught-up consumer would read it as a sudden flood of new events rather than as the error it was. `limit` and `timeout` deliberately keep falling back to their defaults for the same input. Both are the producer's to decide and neither can cause a wrong answer, which is the existing "garbage limit" policy; a comment and a test now say that the difference is a choice rather than an oversight. Co-Authored-By: Claude Opus 5 (1M context) --- src/Feed/Server.php | 18 +++++++++++++-- tests/Feed/Server/Base.php | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/Feed/Server.php b/src/Feed/Server.php index a25b279..1292b2f 100644 --- a/src/Feed/Server.php +++ b/src/Feed/Server.php @@ -51,13 +51,27 @@ public function poll(?string $lastEventId = null, int $limit = Readable::MAX_BAT */ public function serve(array $query): Batch { + /** @var mixed $lastEventId */ $lastEventId = $query[self::PARAM_LAST_EVENT_ID] ?? null; - $lastEventId = \is_string($lastEventId) && $lastEventId !== '' ? $lastEventId : null; - if ($lastEventId !== null && $lastEventId !== Readable::TIP && !Id::isValid($lastEventId)) { + // Absent and empty both mean "from the oldest retained event". Anything + // else present has to be a position or the sentinel β€” including values + // that are not strings at all: PHP parses `?lastEventId[]=1-0` into an + // array, and reading that as absent would answer a malformed parameter + // with a full replay of the retained feed. That is the most expensive + // response there is, and a caught-up consumer would take it for a + // sudden flood of new events rather than for the 400 it should be. + if ($lastEventId === null || $lastEventId === '') { + $lastEventId = null; + } elseif (!\is_string($lastEventId)) { + throw new Exception\Invalid('Invalid lastEventId: expected a string, got ' . \get_debug_type($lastEventId)); + } elseif ($lastEventId !== Readable::TIP && !Id::isValid($lastEventId)) { throw new Exception\Invalid('Invalid lastEventId: ' . $lastEventId); } + // `limit` and `timeout` stay forgiving: both are the producer's to + // decide, so a garbage value falls back to the default rather than + // stalling a feed over something that does not affect correctness. $limit = $query[self::PARAM_LIMIT] ?? null; $timeout = $query[self::PARAM_TIMEOUT] ?? null; diff --git a/tests/Feed/Server/Base.php b/tests/Feed/Server/Base.php index b7800cb..4323dd7 100644 --- a/tests/Feed/Server/Base.php +++ b/tests/Feed/Server/Base.php @@ -262,6 +262,52 @@ public function testServeRejectsALastEventIdThatIsNotAPosition(): void $this->server->serve(['lastEventId' => 'not-a-position']); } + /** + * PHP parses `?lastEventId[]=1-0` into an array, so a `lastEventId` that + * is present need not be a string. Coercing one to "absent" would answer + * a malformed parameter with a full replay of the retained feed β€” the + * most expensive response the endpoint has, and one a caught-up consumer + * would read as a sudden flood of new events rather than as the 400 it is. + * + * @dataProvider notStrings + */ + public function testServeRejectsALastEventIdThatIsNotEvenAString(mixed $lastEventId): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->expectException(Invalid::class); + + $this->server->serve(['lastEventId' => $lastEventId]); + } + + /** + * @return array + */ + public static function notStrings(): array + { + return [ + 'a repeated parameter' => [['1-0']], + 'an empty array' => [[]], + 'a nested map' => [['a' => '1-0']], + 'a boolean' => [true], + ]; + } + + /** + * `limit` and `timeout` stay forgiving where `lastEventId` does not: both + * are the producer's to decide, so garbage falls back to the default + * rather than stalling a feed over something that cannot cause a wrong + * answer. An array is garbage like any other. + */ + public function testServeFallsBackToTheDefaultsOnArrayLimitsAndTimeouts(): void + { + $this->producer->produce('a'); + $this->producer->produce('b'); + + $this->assertCount(2, $this->server->serve(['limit' => ['5'], 'timeout' => ['0']])); + } + public function testServeLetsTheTipSentinelThrough(): void { $this->producer->produce('a'); From 37214241baa36031b3073fa871bd2cfdd681e26c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:21:45 +0200 Subject: [PATCH 49/68] Document what publish() stamps over MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `publish()` takes a "prepared `CloudEvent`" and rebuilds it with the producer's own `source`, discarding whatever the caller set. That is deliberate and tested, but nothing said so: the docblock listed only the exceptions, and the README did not mention `publish()` at all. A caller relaying an event received from another feed would reasonably expect a prepared event to be published as prepared, and instead had its origin silently rewritten. The docblock and the README now name all three attributes the producer owns β€” `source`, `id` (the store assigns it, since it is also the event's position) and a missing `time` β€” and say what to do about the relay case: keep the original origin in an extension attribute. Writing it down surfaced a fourth. `specversion` is passed through by `publish()` but silently normalised to `1.0` by the store round trip, since that is the only version `CloudEvent::fromArray()` accepts β€” keeping another one would leave an entry in the feed that nothing could read. Documented and tested rather than left as a claim, along with the source replacement itself. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 10 ++++++++++ src/Feed/Producer.php | 18 ++++++++++++++++++ tests/Feed/Producer/Base.php | 25 +++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/README.md b/README.md index 2e15a9b..b5186c9 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,16 @@ held at most 30s), and throws `Exception\Invalid` on a malformed `lastEventId` position in the feed. Subclass `Producer` to give callers a typed vocabulary instead of raw type strings. +`publish()` takes a prepared `CloudEvent` for anything `produce()` cannot +express β€” a `dataschema`, a `traceparent`, a non-JSON `datacontenttype`. Three +attributes are the producer's rather than the caller's and are replaced +whatever the event arrived with: `source` becomes the producer's own, `id` is +assigned by the store (it is also the event's position, so only the store can +order it), and a missing `time` is stamped as now. That matters most when +relaying: an event received from another feed is republished as *this* +service's event, so keep the original origin in an extension attribute if +consumers need it. + The `Batch` that `serve()` (and `Server::read()`/`poll()`) returns counts and iterates as its events; `cacheControl()` marks a full batch as immutable history and everything shorter `no-store`, using the limit the batch was diff --git a/src/Feed/Producer.php b/src/Feed/Producer.php index 0becf38..d1aa25c 100644 --- a/src/Feed/Producer.php +++ b/src/Feed/Producer.php @@ -41,6 +41,24 @@ public function produce(string $type, mixed $data = [], string $subject = ''): s } /** + * Publish a prepared event. + * + * Three attributes are the producer's, not the caller's, and are replaced + * whatever the event arrived with: + * + * - `source` becomes this producer's. It records where the event happened, + * and a producer can only speak for itself β€” an event relayed from + * another feed is published as this service's event, not as the original + * producer's. Keep the origin in an extension attribute if it matters. + * - `id` is assigned by the store, since it is also the event's position + * in the feed and only the store can order it. + * - `time` is stamped as now when the event carries none. + * + * Everything else β€” `subject`, `datacontenttype`, `dataschema`, `data` + * and extensions β€” is published as prepared. `specversion` reads back as + * `1.0`: it is the only version the stored form can be decoded as, so a + * store that kept another one would hold an entry nothing could read. + * * @throws Exception\Invalid When the event has no type or cannot be encoded. * @throws Exception When the backend rejects the event. */ diff --git a/tests/Feed/Producer/Base.php b/tests/Feed/Producer/Base.php index 8746482..535ac66 100644 --- a/tests/Feed/Producer/Base.php +++ b/tests/Feed/Producer/Base.php @@ -106,6 +106,31 @@ public function testPublishStampsAPreparedEvent(): void $this->assertSame(['a' => 'b'], $event->data); } + /** + * `source` records where an event happened, and a producer can only speak + * for itself β€” so an event relayed from another feed is republished as + * this service's event. Documented rather than merely tested, because a + * caller handing over a "prepared" event would reasonably expect it to be + * published as prepared. + */ + public function testPublishReplacesTheCallersSourceWithTheProducersOwn(): void + { + $this->producer->publish(new CloudEvent(id: '', type: 'test', source: 'urn:somebody:else')); + + $this->assertSame('urn:test', $this->events()[0]->source); + } + + /** + * The stored form can only be decoded as CloudEvents 1.0, so keeping + * another version would leave an entry in the feed that nothing can read. + */ + public function testPublishNormalisesTheSpecVersion(): void + { + $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', specversion: '1.1')); + + $this->assertSame('1.0', $this->events()[0]->specversion); + } + public function testPublishKeepsATimeTheCallerSet(): void { $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', time: '2020-01-01T00:00:00.000Z')); From 9f8e93c2784a234c2c039bc855f7a1b85e4200b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:23:06 +0200 Subject: [PATCH 50/68] Declare the batch media type once, on Readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `application/cloudevents-batch+json` was written out on both sides of the wire: `Batch::MEDIA_TYPE` for the `Content-Type` a feed response carries, `Remote::MEDIA_TYPE` for the `Accept` every read sends. They are one contract β€” one side's Content-Type is literally the other side's Accept β€” and `Readable` is where this library already keeps what the serving and consuming sides share. Editing one copy, say for a parameterized variant, would have desynchronized the pair silently; only a test comparing the two literals would have caught it, and there was none. The value now lives on `Readable`. `Remote` implements the interface so it inherits the constant outright, and `Batch::MEDIA_TYPE` stays as an alias of it: both call sites keep the name that reads naturally where they are used, while the drift is structurally impossible rather than merely tested. The README's route example uses `Batch::MEDIA_TYPE` instead of repeating the literal a third time. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 3 ++- src/Feed/Batch.php | 8 ++++++-- src/Feed/Readable.php | 7 +++++++ src/Feed/Remote.php | 5 +++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b5186c9..590812e 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ The server side is three objects over one store β€” where the events live, here a capped Redis stream: ```php +use Utopia\Feed\Batch; use Utopia\Feed\Producer; use Utopia\Feed\Server; use Utopia\Feed\Store; @@ -56,7 +57,7 @@ $server = new Server($store); $batch = $server->serve($request->getParams()); $response - ->addHeader('Content-Type', 'application/cloudevents-batch+json') + ->addHeader('Content-Type', Batch::MEDIA_TYPE) // application/cloudevents-batch+json ->addHeader('Cache-Control', $batch->cacheControl(public: true)) ->json($batch->toArray()); ``` diff --git a/src/Feed/Batch.php b/src/Feed/Batch.php index ff64d19..8337ae8 100644 --- a/src/Feed/Batch.php +++ b/src/Feed/Batch.php @@ -12,8 +12,12 @@ */ final class Batch implements \Countable, \IteratorAggregate { - /** The batch media type on the wire β€” what a feed response's Content-Type carries. */ - public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; + /** + * What a feed response's `Content-Type` carries β€” the serving side's name + * for {@see Readable::MEDIA_TYPE}, which is where the value lives so the + * two sides of the wire cannot drift apart. + */ + public const string MEDIA_TYPE = Readable::MEDIA_TYPE; private const string CACHE_IMMUTABLE = 'max-age=31536000'; private const string CACHE_NONE = 'no-store'; diff --git a/src/Feed/Readable.php b/src/Feed/Readable.php index cda635b..44bdbb1 100644 --- a/src/Feed/Readable.php +++ b/src/Feed/Readable.php @@ -16,6 +16,13 @@ interface Readable */ public const string TIP = '$'; + /** + * The batch media type on the wire. One side's `Content-Type` is the + * other side's `Accept`, so it lives here with the rest of the contract + * the serving and consuming sides share rather than once on each. + */ + public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; + /** The most events one batch may carry β€” producers clamp `limit` to this. */ public const int MAX_BATCH = 1000; diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index 9d8303c..44bacfc 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -16,8 +16,9 @@ class Remote implements Readable { - /** The batch media type on the wire β€” sent as `Accept` with every read. */ - public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; + // MEDIA_TYPE β€” sent as `Accept` with every read β€” is inherited from + // Readable, so the header this asks for and the one Batch is served with + // are the same value rather than two copies of it. // The request parameters of https://www.http-feeds.org/. private const string PARAM_LAST_EVENT_ID = 'lastEventId'; From 131e23c93582587da3448f7712516dfe251c007c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:26:11 +0200 Subject: [PATCH 51/68] Move the test suite from PHPUnit 9.6 to 13 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The library requires PHP >= 8.5 but tested with PHPUnit 9.6, which went end of life in early 2024 β€” four majors behind. A brand-new library on a brand-new PHP version has the longest runway to be broken by a point release, and the least chance of a fix arriving for a test framework nobody supports any more. The migration also only ever gets more expensive: `phpunit.xml` used PHPUnit-9-only attributes (`convertErrorsToExceptions`, `backupStaticAttributes`) and the tests carried metadata in doc comments (`@dataProvider`), which PHPUnit 10 dropped. Every test added before the move would have added to it. Doing it at the initial PR is as cheap as it will ever be. All 13 `@dataProvider` annotations became `#[DataProvider]` attributes β€” the providers were already `static`, so nothing else had to change β€” and `phpunit.xml` was regenerated via `--migrate-configuration`, then reformatted back to the file's own layout and given the schema reference so editors can validate it. `.phpunit.cache/`, the result cache's new home, is ignored. Test and assertion counts are unchanged across every suite, which is what says the data providers still feed: a provider PHPUnit no longer sees does not error, it just silently runs the test once with no arguments. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 1 + composer.json | 2 +- composer.lock | 1018 ++++++++++++++++------------- phpunit.xml | 11 +- tests/Feed/Consumer/Base.php | 9 +- tests/Feed/Producer/Base.php | 12 +- tests/Feed/Producer/CacheTest.php | 3 +- tests/Feed/Server/Base.php | 4 +- tests/Feed/Unit/CursorTest.php | 10 +- tests/Feed/Unit/IdTest.php | 5 +- tests/Feed/Unit/KeyTest.php | 5 +- tests/Feed/Unit/RemoteTest.php | 3 +- 12 files changed, 608 insertions(+), 475 deletions(-) diff --git a/.gitignore b/.gitignore index 6bf176b..5d31694 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /vendor/ /.idea/ +.phpunit.cache/ .phpunit.result.cache .DS_Store diff --git a/composer.json b/composer.json index 0e32ff4..63dfc9b 100644 --- a/composer.json +++ b/composer.json @@ -54,7 +54,7 @@ "require-dev": { "laravel/pint": "^1.21", "phpstan/phpstan": "^2.2", - "phpunit/phpunit": "^9.6", + "phpunit/phpunit": "^13.0", "swoole/ide-helper": "^6.0" }, "config": { diff --git a/composer.lock b/composer.lock index 5a7b463..d83cd93 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "022655166a94f83c4c6d0213ede8aeda", + "content-hash": "43c96f9feee2bb8d8dc20f8ac83a2db8", "packages": [ { "name": "brick/math", @@ -2297,75 +2297,6 @@ } ], "packages-dev": [ - { - "name": "doctrine/instantiator", - "version": "2.1.0", - "source": { - "type": "git", - "url": "https://github.com/doctrine/instantiator.git", - "reference": "23da848e1a2308728fe5fdddabf4be17ff9720c7" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/23da848e1a2308728fe5fdddabf4be17ff9720c7", - "reference": "23da848e1a2308728fe5fdddabf4be17ff9720c7", - "shasum": "" - }, - "require": { - "php": "^8.4" - }, - "require-dev": { - "doctrine/coding-standard": "^14", - "ext-pdo": "*", - "ext-phar": "*", - "phpbench/phpbench": "^1.2", - "phpstan/phpstan": "^2.1", - "phpstan/phpstan-phpunit": "^2.0", - "phpunit/phpunit": "^10.5.58" - }, - "type": "library", - "autoload": { - "psr-4": { - "Doctrine\\Instantiator\\": "src/Doctrine/Instantiator/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Marco Pivetta", - "email": "ocramius@gmail.com", - "homepage": "https://ocramius.github.io/" - } - ], - "description": "A small, lightweight utility to instantiate objects in PHP without invoking their constructors", - "homepage": "https://www.doctrine-project.org/projects/instantiator.html", - "keywords": [ - "constructor", - "instantiate" - ], - "support": { - "issues": "https://github.com/doctrine/instantiator/issues", - "source": "https://github.com/doctrine/instantiator/tree/2.1.0" - }, - "funding": [ - { - "url": "https://www.doctrine-project.org/sponsorship.html", - "type": "custom" - }, - { - "url": "https://www.patreon.com/phpdoctrine", - "type": "patreon" - }, - { - "url": "https://tidelift.com/funding/github/packagist/doctrine%2Finstantiator", - "type": "tidelift" - } - ], - "time": "2026-01-05T06:47:08+00:00" - }, { "name": "laravel/pint", "version": "v1.30.0", @@ -2737,35 +2668,35 @@ }, { "name": "phpunit/php-code-coverage", - "version": "9.2.32", + "version": "14.2.4", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5" + "reference": "048a5c12bdb4580f4767ce2761793a16b170fbe4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/85402a822d1ecf1db1096959413d35e1c37cf1a5", - "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/048a5c12bdb4580f4767ce2761793a16b170fbe4", + "reference": "048a5c12bdb4580f4767ce2761793a16b170fbe4", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", + "ext-mbstring": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.19.1 || ^5.1.0", - "php": ">=7.3", - "phpunit/php-file-iterator": "^3.0.6", - "phpunit/php-text-template": "^2.0.4", - "sebastian/code-unit-reverse-lookup": "^2.0.3", - "sebastian/complexity": "^2.0.3", - "sebastian/environment": "^5.1.5", - "sebastian/lines-of-code": "^1.0.4", - "sebastian/version": "^3.0.2", - "theseer/tokenizer": "^1.2.3" + "nikic/php-parser": "^5.8.0", + "php": ">=8.4", + "phpunit/php-text-template": "^6.0", + "sebastian/complexity": "^6.0", + "sebastian/environment": "^9.3.2", + "sebastian/git-state": "^1.0", + "sebastian/lines-of-code": "^5.0.1", + "sebastian/version": "^7.0", + "theseer/tokenizer": "^2.0.1" }, "require-dev": { - "phpunit/phpunit": "^9.6" + "phpunit/phpunit": "^13.2.2" }, "suggest": { "ext-pcov": "PHP extension that provides line coverage", @@ -2774,7 +2705,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "9.2.x-dev" + "dev-main": "14.2.x-dev" } }, "autoload": { @@ -2803,40 +2734,52 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.32" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/14.2.4" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/php-code-coverage", + "type": "tidelift" } ], - "time": "2024-08-22T04:23:01+00:00" + "time": "2026-07-30T17:01:07+00:00" }, { "name": "phpunit/php-file-iterator", - "version": "3.0.6", + "version": "7.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf" + "reference": "6e5aa1fb0a95b1703d83e721299ee18bb4e2de50" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", - "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/6e5aa1fb0a95b1703d83e721299ee18bb4e2de50", + "reference": "6e5aa1fb0a95b1703d83e721299ee18bb4e2de50", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "3.0-dev" + "dev-main": "7.0-dev" } }, "autoload": { @@ -2863,36 +2806,49 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-file-iterator/issues", - "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/3.0.6" + "security": "https://github.com/sebastianbergmann/php-file-iterator/security/policy", + "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/7.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/php-file-iterator", + "type": "tidelift" } ], - "time": "2021-12-02T12:48:52+00:00" + "time": "2026-02-06T04:33:26+00:00" }, { "name": "phpunit/php-invoker", - "version": "3.1.1", + "version": "7.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-invoker.git", - "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67" + "reference": "42e5c5cae0c65df12d1b1a3ab52bf3f50f244d88" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/5a10147d0aaf65b58940a0b72f71c9ac0423cc67", - "reference": "5a10147d0aaf65b58940a0b72f71c9ac0423cc67", + "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/42e5c5cae0c65df12d1b1a3ab52bf3f50f244d88", + "reference": "42e5c5cae0c65df12d1b1a3ab52bf3f50f244d88", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { "ext-pcntl": "*", - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.0" }, "suggest": { "ext-pcntl": "*" @@ -2900,7 +2856,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "3.1-dev" + "dev-main": "7.0-dev" } }, "autoload": { @@ -2926,40 +2882,53 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-invoker/issues", - "source": "https://github.com/sebastianbergmann/php-invoker/tree/3.1.1" + "security": "https://github.com/sebastianbergmann/php-invoker/security/policy", + "source": "https://github.com/sebastianbergmann/php-invoker/tree/7.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/php-invoker", + "type": "tidelift" } ], - "time": "2020-09-28T05:58:55+00:00" + "time": "2026-02-06T04:34:47+00:00" }, { "name": "phpunit/php-text-template", - "version": "2.0.4", + "version": "6.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-text-template.git", - "reference": "5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28" + "reference": "a47af19f93f76aa3368303d752aa5272ca3299f4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28", - "reference": "5da5f67fc95621df9ff4c4e5a84d6a8a2acf7c28", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/a47af19f93f76aa3368303d752aa5272ca3299f4", + "reference": "a47af19f93f76aa3368303d752aa5272ca3299f4", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "2.0-dev" + "dev-main": "6.0-dev" } }, "autoload": { @@ -2985,40 +2954,53 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-text-template/issues", - "source": "https://github.com/sebastianbergmann/php-text-template/tree/2.0.4" + "security": "https://github.com/sebastianbergmann/php-text-template/security/policy", + "source": "https://github.com/sebastianbergmann/php-text-template/tree/6.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/php-text-template", + "type": "tidelift" } ], - "time": "2020-10-26T05:33:50+00:00" + "time": "2026-02-06T04:36:37+00:00" }, { "name": "phpunit/php-timer", - "version": "5.0.3", + "version": "9.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2" + "reference": "a0e12065831f6ab0d83120dc61513eb8d9a966f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2", - "reference": "5a63ce20ed1b5bf577850e2c4e87f4aa902afbd2", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/a0e12065831f6ab0d83120dc61513eb8d9a966f6", + "reference": "a0e12065831f6ab0d83120dc61513eb8d9a966f6", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "5.0-dev" + "dev-main": "9.0-dev" } }, "autoload": { @@ -3044,32 +3026,44 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-timer/issues", - "source": "https://github.com/sebastianbergmann/php-timer/tree/5.0.3" + "security": "https://github.com/sebastianbergmann/php-timer/security/policy", + "source": "https://github.com/sebastianbergmann/php-timer/tree/9.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/php-timer", + "type": "tidelift" } ], - "time": "2020-10-26T13:16:10+00:00" + "time": "2026-02-06T04:37:53+00:00" }, { "name": "phpunit/phpunit", - "version": "9.6.35", + "version": "13.2.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "0edba2f3a0c48df3553cb9b640810b30df60302b" + "reference": "5d2afe181339a56348ef9a80fa7eb806b7eae508" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0edba2f3a0c48df3553cb9b640810b30df60302b", - "reference": "0edba2f3a0c48df3553cb9b640810b30df60302b", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/5d2afe181339a56348ef9a80fa7eb806b7eae508", + "reference": "5d2afe181339a56348ef9a80fa7eb806b7eae508", "shasum": "" }, "require": { - "doctrine/instantiator": "^1.5.0 || ^2", "ext-dom": "*", "ext-filter": "*", "ext-json": "*", @@ -3079,27 +3073,25 @@ "myclabs/deep-copy": "^1.13.4", "phar-io/manifest": "^2.0.4", "phar-io/version": "^3.2.1", - "php": ">=7.3", - "phpunit/php-code-coverage": "^9.2.32", - "phpunit/php-file-iterator": "^3.0.6", - "phpunit/php-invoker": "^3.1.1", - "phpunit/php-text-template": "^2.0.4", - "phpunit/php-timer": "^5.0.3", - "sebastian/cli-parser": "^1.0.2", - "sebastian/code-unit": "^1.0.8", - "sebastian/comparator": "^4.0.10", - "sebastian/diff": "^4.0.6", - "sebastian/environment": "^5.1.5", - "sebastian/exporter": "^4.0.8", - "sebastian/global-state": "^5.0.8", - "sebastian/object-enumerator": "^4.0.4", - "sebastian/resource-operations": "^3.0.4", - "sebastian/type": "^3.2.1", - "sebastian/version": "^3.0.2" - }, - "suggest": { - "ext-soap": "To be able to generate mocks based on WSDL files", - "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" + "php": ">=8.4.1", + "phpunit/php-code-coverage": "^14.2.3", + "phpunit/php-file-iterator": "^7.0.0", + "phpunit/php-invoker": "^7.0.0", + "phpunit/php-text-template": "^6.0.0", + "phpunit/php-timer": "^9.0.0", + "sebastian/cli-parser": "^5.0.0", + "sebastian/comparator": "^8.3.0", + "sebastian/diff": "^9.0", + "sebastian/environment": "^9.3.2", + "sebastian/exporter": "^8.1.1", + "sebastian/file-filter": "^1.0", + "sebastian/git-state": "^1.0", + "sebastian/global-state": "^9.0.1", + "sebastian/object-enumerator": "^8.0.0", + "sebastian/recursion-context": "^8.0.0", + "sebastian/type": "^7.0.1", + "sebastian/version": "^7.0.0", + "staabm/side-effects-detector": "^1.0.5" }, "bin": [ "phpunit" @@ -3107,7 +3099,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "9.6-dev" + "dev-main": "13.2-dev" } }, "autoload": { @@ -3139,7 +3131,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.35" + "source": "https://github.com/sebastianbergmann/phpunit/tree/13.2.6" }, "funding": [ { @@ -3147,32 +3139,32 @@ "type": "other" } ], - "time": "2026-07-06T14:48:07+00:00" + "time": "2026-07-28T14:00:09+00:00" }, { "name": "sebastian/cli-parser", - "version": "1.0.2", + "version": "5.0.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/cli-parser.git", - "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b" + "reference": "eeb759ad3146b7096fb59c3195d39e071cd409e3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/2b56bea83a09de3ac06bb18b92f068e60cc6f50b", - "reference": "2b56bea83a09de3ac06bb18b92f068e60cc6f50b", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/eeb759ad3146b7096fb59c3195d39e071cd409e3", + "reference": "eeb759ad3146b7096fb59c3195d39e071cd409e3", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.2.6" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.0-dev" + "dev-main": "5.0-dev" } }, "autoload": { @@ -3195,153 +3187,60 @@ "homepage": "https://github.com/sebastianbergmann/cli-parser", "support": { "issues": "https://github.com/sebastianbergmann/cli-parser/issues", - "source": "https://github.com/sebastianbergmann/cli-parser/tree/1.0.2" + "security": "https://github.com/sebastianbergmann/cli-parser/security/policy", + "source": "https://github.com/sebastianbergmann/cli-parser/tree/5.0.1" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" - } - ], - "time": "2024-03-02T06:27:43+00:00" - }, - { - "name": "sebastian/code-unit", - "version": "1.0.8", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/code-unit.git", - "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/1fc9f64c0927627ef78ba436c9b17d967e68e120", - "reference": "1fc9f64c0927627ef78ba436c9b17d967e68e120", - "shasum": "" - }, - "require": { - "php": ">=7.3" - }, - "require-dev": { - "phpunit/phpunit": "^9.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0-dev" - } - }, - "autoload": { - "classmap": [ - "src/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sebastian@phpunit.de", - "role": "lead" - } - ], - "description": "Collection of value objects that represent the PHP code units", - "homepage": "https://github.com/sebastianbergmann/code-unit", - "support": { - "issues": "https://github.com/sebastianbergmann/code-unit/issues", - "source": "https://github.com/sebastianbergmann/code-unit/tree/1.0.8" - }, - "funding": [ + }, { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], - "time": "2020-10-26T13:08:54+00:00" - }, - { - "name": "sebastian/code-unit-reverse-lookup", - "version": "2.0.3", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", - "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", - "reference": "ac91f01ccec49fb77bdc6fd1e548bc70f7faa3e5", - "shasum": "" - }, - "require": { - "php": ">=7.3" - }, - "require-dev": { - "phpunit/phpunit": "^9.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.0-dev" - } - }, - "autoload": { - "classmap": [ - "src/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "BSD-3-Clause" - ], - "authors": [ + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, { - "name": "Sebastian Bergmann", - "email": "sebastian@phpunit.de" - } - ], - "description": "Looks up which function or method a line of code belongs to", - "homepage": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/", - "support": { - "issues": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/issues", - "source": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/tree/2.0.3" - }, - "funding": [ + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, { - "url": "https://github.com/sebastianbergmann", - "type": "github" + "url": "https://tidelift.com/funding/github/packagist/sebastian/cli-parser", + "type": "tidelift" } ], - "time": "2020-09-28T05:30:19+00:00" + "time": "2026-08-01T04:27:14+00:00" }, { "name": "sebastian/comparator", - "version": "4.0.10", + "version": "8.3.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "e4df00b9b3571187db2831ae9aada2c6efbd715d" + "reference": "c025fc7604afab3f195fab7cdaf72327331af241" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/e4df00b9b3571187db2831ae9aada2c6efbd715d", - "reference": "e4df00b9b3571187db2831ae9aada2c6efbd715d", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/c025fc7604afab3f195fab7cdaf72327331af241", + "reference": "c025fc7604afab3f195fab7cdaf72327331af241", "shasum": "" }, "require": { - "php": ">=7.3", - "sebastian/diff": "^4.0", - "sebastian/exporter": "^4.0" + "ext-dom": "*", + "ext-mbstring": "*", + "php": ">=8.4", + "sebastian/diff": "^9.0", + "sebastian/exporter": "^8.1.0" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.2" + }, + "suggest": { + "ext-bcmath": "For comparing BcMath\\Number objects" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "4.0-dev" + "dev-main": "8.3-dev" } }, "autoload": { @@ -3380,7 +3279,8 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", - "source": "https://github.com/sebastianbergmann/comparator/tree/4.0.10" + "security": "https://github.com/sebastianbergmann/comparator/security/policy", + "source": "https://github.com/sebastianbergmann/comparator/tree/8.3.0" }, "funding": [ { @@ -3400,33 +3300,33 @@ "type": "tidelift" } ], - "time": "2026-01-24T09:22:56+00:00" + "time": "2026-06-05T03:06:45+00:00" }, { "name": "sebastian/complexity", - "version": "2.0.3", + "version": "6.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/complexity.git", - "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a" + "reference": "c5651c795c98093480df79350cb050813fc7a2f3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/25f207c40d62b8b7aa32f5ab026c53561964053a", - "reference": "25f207c40d62b8b7aa32f5ab026c53561964053a", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/c5651c795c98093480df79350cb050813fc7a2f3", + "reference": "c5651c795c98093480df79350cb050813fc7a2f3", "shasum": "" }, "require": { - "nikic/php-parser": "^4.18 || ^5.0", - "php": ">=7.3" + "nikic/php-parser": "^5.0", + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "2.0-dev" + "dev-main": "6.0-dev" } }, "autoload": { @@ -3449,41 +3349,54 @@ "homepage": "https://github.com/sebastianbergmann/complexity", "support": { "issues": "https://github.com/sebastianbergmann/complexity/issues", - "source": "https://github.com/sebastianbergmann/complexity/tree/2.0.3" + "security": "https://github.com/sebastianbergmann/complexity/security/policy", + "source": "https://github.com/sebastianbergmann/complexity/tree/6.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/complexity", + "type": "tidelift" } ], - "time": "2023-12-22T06:19:30+00:00" + "time": "2026-02-06T04:41:32+00:00" }, { "name": "sebastian/diff", - "version": "4.0.6", + "version": "9.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc" + "reference": "a3fb6a298a265ff487a91bbea46e03cd01dbb226" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/ba01945089c3a293b01ba9badc29ad55b106b0bc", - "reference": "ba01945089c3a293b01ba9badc29ad55b106b0bc", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/a3fb6a298a265ff487a91bbea46e03cd01dbb226", + "reference": "a3fb6a298a265ff487a91bbea46e03cd01dbb226", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3", - "symfony/process": "^4.2 || ^5" + "phpunit/phpunit": "^13.2", + "symfony/process": "^7.4.13" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "4.0-dev" + "dev-main": "9.0-dev" } }, "autoload": { @@ -3515,35 +3428,48 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/diff/issues", - "source": "https://github.com/sebastianbergmann/diff/tree/4.0.6" + "security": "https://github.com/sebastianbergmann/diff/security/policy", + "source": "https://github.com/sebastianbergmann/diff/tree/9.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/diff", + "type": "tidelift" } ], - "time": "2024-03-02T06:30:58+00:00" + "time": "2026-06-05T03:04:51+00:00" }, { "name": "sebastian/environment", - "version": "5.1.5", + "version": "9.3.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed" + "reference": "6c9e487c9eb706a8d258102a1c0b0a3e53e86c2e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", - "reference": "830c43a844f1f8d5b7a1f6d6076b784454d8b7ed", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/6c9e487c9eb706a8d258102a1c0b0a3e53e86c2e", + "reference": "6c9e487c9eb706a8d258102a1c0b0a3e53e86c2e", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.1.11" }, "suggest": { "ext-posix": "*" @@ -3551,7 +3477,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "5.1-dev" + "dev-main": "9.3-dev" } }, "autoload": { @@ -3570,7 +3496,7 @@ } ], "description": "Provides functionality to handle HHVM/PHP environments", - "homepage": "http://www.github.com/sebastianbergmann/environment", + "homepage": "https://github.com/sebastianbergmann/environment", "keywords": [ "Xdebug", "environment", @@ -3578,42 +3504,55 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/environment/issues", - "source": "https://github.com/sebastianbergmann/environment/tree/5.1.5" + "security": "https://github.com/sebastianbergmann/environment/security/policy", + "source": "https://github.com/sebastianbergmann/environment/tree/9.3.2" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/environment", + "type": "tidelift" } ], - "time": "2023-02-03T06:03:51+00:00" + "time": "2026-05-25T13:41:38+00:00" }, { "name": "sebastian/exporter", - "version": "4.0.8", + "version": "8.1.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c" + "reference": "cfaa77c750dcad6f44c9bac8f62ac486e1c82c26" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/14c6ba52f95a36c3d27c835d65efc7123c446e8c", - "reference": "14c6ba52f95a36c3d27c835d65efc7123c446e8c", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/cfaa77c750dcad6f44c9bac8f62ac486e1c82c26", + "reference": "cfaa77c750dcad6f44c9bac8f62ac486e1c82c26", "shasum": "" }, "require": { - "php": ">=7.3", - "sebastian/recursion-context": "^4.0" + "ext-mbstring": "*", + "php": ">=8.4", + "sebastian/recursion-context": "^8.0" }, "require-dev": { - "ext-mbstring": "*", - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.2.4" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "4.0-dev" + "dev-main": "8.1-dev" } }, "autoload": { @@ -3655,7 +3594,8 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/exporter/issues", - "source": "https://github.com/sebastianbergmann/exporter/tree/4.0.8" + "security": "https://github.com/sebastianbergmann/exporter/security/policy", + "source": "https://github.com/sebastianbergmann/exporter/tree/8.1.1" }, "funding": [ { @@ -3675,38 +3615,173 @@ "type": "tidelift" } ], - "time": "2025-09-24T06:03:27+00:00" + "time": "2026-07-13T11:35:11+00:00" + }, + { + "name": "sebastian/file-filter", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/file-filter.git", + "reference": "33a26f394330f6faa7684bb9cc73afb7727aae93" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/file-filter/zipball/33a26f394330f6faa7684bb9cc73afb7727aae93", + "reference": "33a26f394330f6faa7684bb9cc73afb7727aae93", + "shasum": "" + }, + "require": { + "php": ">=8.4" + }, + "require-dev": { + "phpunit/phpunit": "^13.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for filtering files", + "homepage": "https://github.com/sebastianbergmann/file-filter", + "support": { + "issues": "https://github.com/sebastianbergmann/file-filter/issues", + "security": "https://github.com/sebastianbergmann/file-filter/security/policy", + "source": "https://github.com/sebastianbergmann/file-filter/tree/1.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/file-filter", + "type": "tidelift" + } + ], + "time": "2026-04-22T07:20:04+00:00" + }, + { + "name": "sebastian/git-state", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/git-state.git", + "reference": "792a952e0eba55b6960a48aeceb9f371aad1f76b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/git-state/zipball/792a952e0eba55b6960a48aeceb9f371aad1f76b", + "reference": "792a952e0eba55b6960a48aeceb9f371aad1f76b", + "shasum": "" + }, + "require": { + "php": ">=8.4" + }, + "require-dev": { + "phpunit/phpunit": "^13.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for describing the state of a Git checkout", + "homepage": "https://github.com/sebastianbergmann/git-state", + "support": { + "issues": "https://github.com/sebastianbergmann/git-state/issues", + "security": "https://github.com/sebastianbergmann/git-state/security/policy", + "source": "https://github.com/sebastianbergmann/git-state/tree/1.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/git-state", + "type": "tidelift" + } + ], + "time": "2026-03-21T12:54:28+00:00" }, { "name": "sebastian/global-state", - "version": "5.0.8", + "version": "9.0.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6" + "reference": "ba68ba79da690cf7eddefd3ce5b78b20b9ba9945" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/b6781316bdcd28260904e7cc18ec983d0d2ef4f6", - "reference": "b6781316bdcd28260904e7cc18ec983d0d2ef4f6", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/ba68ba79da690cf7eddefd3ce5b78b20b9ba9945", + "reference": "ba68ba79da690cf7eddefd3ce5b78b20b9ba9945", "shasum": "" }, "require": { - "php": ">=7.3", - "sebastian/object-reflector": "^2.0", - "sebastian/recursion-context": "^4.0" + "php": ">=8.4", + "sebastian/object-reflector": "^6.0", + "sebastian/recursion-context": "^8.0" }, "require-dev": { "ext-dom": "*", - "phpunit/phpunit": "^9.3" - }, - "suggest": { - "ext-uopz": "*" + "phpunit/phpunit": "^13.1.13" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "5.0-dev" + "dev-main": "9.0-dev" } }, "autoload": { @@ -3725,13 +3800,14 @@ } ], "description": "Snapshotting of global state", - "homepage": "http://www.github.com/sebastianbergmann/global-state", + "homepage": "https://www.github.com/sebastianbergmann/global-state", "keywords": [ "global state" ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.8" + "security": "https://github.com/sebastianbergmann/global-state/security/policy", + "source": "https://github.com/sebastianbergmann/global-state/tree/9.0.1" }, "funding": [ { @@ -3751,33 +3827,33 @@ "type": "tidelift" } ], - "time": "2025-08-10T07:10:35+00:00" + "time": "2026-06-01T15:11:33+00:00" }, { "name": "sebastian/lines-of-code", - "version": "1.0.4", + "version": "5.0.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/lines-of-code.git", - "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5" + "reference": "d1b6f8fce682505dbd048977f1abedf1b8ad3ff8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/e1e4a170560925c26d424b6a03aed157e7dcc5c5", - "reference": "e1e4a170560925c26d424b6a03aed157e7dcc5c5", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/d1b6f8fce682505dbd048977f1abedf1b8ad3ff8", + "reference": "d1b6f8fce682505dbd048977f1abedf1b8ad3ff8", "shasum": "" }, "require": { - "nikic/php-parser": "^4.18 || ^5.0", - "php": ">=7.3" + "nikic/php-parser": "^5.8.0", + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.2.4" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.0-dev" + "dev-main": "5.0-dev" } }, "autoload": { @@ -3800,42 +3876,55 @@ "homepage": "https://github.com/sebastianbergmann/lines-of-code", "support": { "issues": "https://github.com/sebastianbergmann/lines-of-code/issues", - "source": "https://github.com/sebastianbergmann/lines-of-code/tree/1.0.4" + "security": "https://github.com/sebastianbergmann/lines-of-code/security/policy", + "source": "https://github.com/sebastianbergmann/lines-of-code/tree/5.0.2" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/lines-of-code", + "type": "tidelift" } ], - "time": "2023-12-22T06:20:34+00:00" + "time": "2026-07-09T08:42:34+00:00" }, { "name": "sebastian/object-enumerator", - "version": "4.0.4", + "version": "8.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-enumerator.git", - "reference": "5c9eeac41b290a3712d88851518825ad78f45c71" + "reference": "b39ab125fd9a7434b0ecbc4202eebce11a98cfc5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/5c9eeac41b290a3712d88851518825ad78f45c71", - "reference": "5c9eeac41b290a3712d88851518825ad78f45c71", + "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/b39ab125fd9a7434b0ecbc4202eebce11a98cfc5", + "reference": "b39ab125fd9a7434b0ecbc4202eebce11a98cfc5", "shasum": "" }, "require": { - "php": ">=7.3", - "sebastian/object-reflector": "^2.0", - "sebastian/recursion-context": "^4.0" + "php": ">=8.4", + "sebastian/object-reflector": "^6.0", + "sebastian/recursion-context": "^8.0" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "4.0-dev" + "dev-main": "8.0-dev" } }, "autoload": { @@ -3857,40 +3946,53 @@ "homepage": "https://github.com/sebastianbergmann/object-enumerator/", "support": { "issues": "https://github.com/sebastianbergmann/object-enumerator/issues", - "source": "https://github.com/sebastianbergmann/object-enumerator/tree/4.0.4" + "security": "https://github.com/sebastianbergmann/object-enumerator/security/policy", + "source": "https://github.com/sebastianbergmann/object-enumerator/tree/8.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/object-enumerator", + "type": "tidelift" } ], - "time": "2020-10-26T13:12:34+00:00" + "time": "2026-02-06T04:46:36+00:00" }, { "name": "sebastian/object-reflector", - "version": "2.0.4", + "version": "6.0.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/object-reflector.git", - "reference": "b4f479ebdbf63ac605d183ece17d8d7fe49c15c7" + "reference": "3ca042c2c60b0eab094f8a1b6a7093f4d4c72200" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/b4f479ebdbf63ac605d183ece17d8d7fe49c15c7", - "reference": "b4f479ebdbf63ac605d183ece17d8d7fe49c15c7", + "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/3ca042c2c60b0eab094f8a1b6a7093f4d4c72200", + "reference": "3ca042c2c60b0eab094f8a1b6a7093f4d4c72200", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "2.0-dev" + "dev-main": "6.0-dev" } }, "autoload": { @@ -3912,40 +4014,53 @@ "homepage": "https://github.com/sebastianbergmann/object-reflector/", "support": { "issues": "https://github.com/sebastianbergmann/object-reflector/issues", - "source": "https://github.com/sebastianbergmann/object-reflector/tree/2.0.4" + "security": "https://github.com/sebastianbergmann/object-reflector/security/policy", + "source": "https://github.com/sebastianbergmann/object-reflector/tree/6.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/object-reflector", + "type": "tidelift" } ], - "time": "2020-10-26T13:14:26+00:00" + "time": "2026-02-06T04:47:13+00:00" }, { "name": "sebastian/recursion-context", - "version": "4.0.6", + "version": "8.0.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/recursion-context.git", - "reference": "539c6691e0623af6dc6f9c20384c120f963465a0" + "reference": "32dba72f2b4642d6a93db22d6c0a9280ff2e3ca0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/539c6691e0623af6dc6f9c20384c120f963465a0", - "reference": "539c6691e0623af6dc6f9c20384c120f963465a0", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/32dba72f2b4642d6a93db22d6c0a9280ff2e3ca0", + "reference": "32dba72f2b4642d6a93db22d6c0a9280ff2e3ca0", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.3" + "phpunit/phpunit": "^13.2.6" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "4.0-dev" + "dev-main": "8.0-dev" } }, "autoload": { @@ -3975,7 +4090,8 @@ "homepage": "https://github.com/sebastianbergmann/recursion-context", "support": { "issues": "https://github.com/sebastianbergmann/recursion-context/issues", - "source": "https://github.com/sebastianbergmann/recursion-context/tree/4.0.6" + "security": "https://github.com/sebastianbergmann/recursion-context/security/policy", + "source": "https://github.com/sebastianbergmann/recursion-context/tree/8.0.1" }, "funding": [ { @@ -3995,32 +4111,32 @@ "type": "tidelift" } ], - "time": "2025-08-10T06:57:39+00:00" + "time": "2026-08-03T05:58:12+00:00" }, { - "name": "sebastian/resource-operations", - "version": "3.0.4", + "name": "sebastian/type", + "version": "7.0.1", "source": { "type": "git", - "url": "https://github.com/sebastianbergmann/resource-operations.git", - "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e" + "url": "https://github.com/sebastianbergmann/type.git", + "reference": "fee0309275847fefd7636167085e379c1dbf6990" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/05d5692a7993ecccd56a03e40cd7e5b09b1d404e", - "reference": "05d5692a7993ecccd56a03e40cd7e5b09b1d404e", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/fee0309275847fefd7636167085e379c1dbf6990", + "reference": "fee0309275847fefd7636167085e379c1dbf6990", "shasum": "" }, "require": { - "php": ">=7.3" + "php": ">=8.4" }, "require-dev": { - "phpunit/phpunit": "^9.0" + "phpunit/phpunit": "^13.1.10" }, "type": "library", "extra": { "branch-alias": { - "dev-main": "3.0-dev" + "dev-main": "7.0-dev" } }, "autoload": { @@ -4035,46 +4151,58 @@ "authors": [ { "name": "Sebastian Bergmann", - "email": "sebastian@phpunit.de" + "email": "sebastian@phpunit.de", + "role": "lead" } ], - "description": "Provides a list of PHP built-in functions that operate on resources", - "homepage": "https://www.github.com/sebastianbergmann/resource-operations", + "description": "Collection of value objects that represent the types of the PHP type system", + "homepage": "https://github.com/sebastianbergmann/type", "support": { - "source": "https://github.com/sebastianbergmann/resource-operations/tree/3.0.4" + "issues": "https://github.com/sebastianbergmann/type/issues", + "security": "https://github.com/sebastianbergmann/type/security/policy", + "source": "https://github.com/sebastianbergmann/type/tree/7.0.1" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/type", + "type": "tidelift" } ], - "time": "2024-03-14T16:00:52+00:00" + "time": "2026-05-20T06:49:11+00:00" }, { - "name": "sebastian/type", - "version": "3.2.1", + "name": "sebastian/version", + "version": "7.0.0", "source": { "type": "git", - "url": "https://github.com/sebastianbergmann/type.git", - "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7" + "url": "https://github.com/sebastianbergmann/version.git", + "reference": "ad37a5552c8e2b88572249fdc19b6da7792e021b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", - "reference": "75e2c2a32f5e0b3aef905b9ed0b179b953b3d7c7", + "url": "https://api.github.com/repos/sebastianbergmann/version/zipball/ad37a5552c8e2b88572249fdc19b6da7792e021b", + "reference": "ad37a5552c8e2b88572249fdc19b6da7792e021b", "shasum": "" }, "require": { - "php": ">=7.3" - }, - "require-dev": { - "phpunit/phpunit": "^9.5" + "php": ">=8.4" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "3.2-dev" + "dev-main": "7.0-dev" } }, "autoload": { @@ -4093,72 +4221,84 @@ "role": "lead" } ], - "description": "Collection of value objects that represent the types of the PHP type system", - "homepage": "https://github.com/sebastianbergmann/type", + "description": "Library that helps with managing the version number of Git-hosted PHP projects", + "homepage": "https://github.com/sebastianbergmann/version", "support": { - "issues": "https://github.com/sebastianbergmann/type/issues", - "source": "https://github.com/sebastianbergmann/type/tree/3.2.1" + "issues": "https://github.com/sebastianbergmann/version/issues", + "security": "https://github.com/sebastianbergmann/version/security/policy", + "source": "https://github.com/sebastianbergmann/version/tree/7.0.0" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/version", + "type": "tidelift" } ], - "time": "2023-02-03T06:13:03+00:00" + "time": "2026-02-06T04:52:52+00:00" }, { - "name": "sebastian/version", - "version": "3.0.2", + "name": "staabm/side-effects-detector", + "version": "1.0.5", "source": { "type": "git", - "url": "https://github.com/sebastianbergmann/version.git", - "reference": "c6c1022351a901512170118436c764e473f6de8c" + "url": "https://github.com/staabm/side-effects-detector.git", + "reference": "d8334211a140ce329c13726d4a715adbddd0a163" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/version/zipball/c6c1022351a901512170118436c764e473f6de8c", - "reference": "c6c1022351a901512170118436c764e473f6de8c", + "url": "https://api.github.com/repos/staabm/side-effects-detector/zipball/d8334211a140ce329c13726d4a715adbddd0a163", + "reference": "d8334211a140ce329c13726d4a715adbddd0a163", "shasum": "" }, "require": { - "php": ">=7.3" + "ext-tokenizer": "*", + "php": "^7.4 || ^8.0" }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.0-dev" - } + "require-dev": { + "phpstan/extension-installer": "^1.4.3", + "phpstan/phpstan": "^1.12.6", + "phpunit/phpunit": "^9.6.21", + "symfony/var-dumper": "^5.4.43", + "tomasvotruba/type-coverage": "1.0.0", + "tomasvotruba/unused-public": "1.0.0" }, + "type": "library", "autoload": { "classmap": [ - "src/" + "lib/" ] }, "notification-url": "https://packagist.org/downloads/", "license": [ - "BSD-3-Clause" + "MIT" ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sebastian@phpunit.de", - "role": "lead" - } + "description": "A static analysis tool to detect side effects in PHP code", + "keywords": [ + "static analysis" ], - "description": "Library that helps with managing the version number of Git-hosted PHP projects", - "homepage": "https://github.com/sebastianbergmann/version", "support": { - "issues": "https://github.com/sebastianbergmann/version/issues", - "source": "https://github.com/sebastianbergmann/version/tree/3.0.2" + "issues": "https://github.com/staabm/side-effects-detector/issues", + "source": "https://github.com/staabm/side-effects-detector/tree/1.0.5" }, "funding": [ { - "url": "https://github.com/sebastianbergmann", + "url": "https://github.com/staabm", "type": "github" } ], - "time": "2020-09-28T06:39:44+00:00" + "time": "2024-10-20T05:08:20+00:00" }, { "name": "swoole/ide-helper", @@ -4194,23 +4334,23 @@ }, { "name": "theseer/tokenizer", - "version": "1.3.1", + "version": "2.0.1", "source": { "type": "git", "url": "https://github.com/theseer/tokenizer.git", - "reference": "b7489ce515e168639d17feec34b8847c326b0b3c" + "reference": "7989e43bf381af0eac72e4f0ca5bcbfa81658be4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b7489ce515e168639d17feec34b8847c326b0b3c", - "reference": "b7489ce515e168639d17feec34b8847c326b0b3c", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/7989e43bf381af0eac72e4f0ca5bcbfa81658be4", + "reference": "7989e43bf381af0eac72e4f0ca5bcbfa81658be4", "shasum": "" }, "require": { "ext-dom": "*", "ext-tokenizer": "*", "ext-xmlwriter": "*", - "php": "^7.2 || ^8.0" + "php": "^8.1" }, "type": "library", "autoload": { @@ -4232,7 +4372,7 @@ "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", "support": { "issues": "https://github.com/theseer/tokenizer/issues", - "source": "https://github.com/theseer/tokenizer/tree/1.3.1" + "source": "https://github.com/theseer/tokenizer/tree/2.0.1" }, "funding": [ { @@ -4240,7 +4380,7 @@ "type": "github" } ], - "time": "2025-11-17T20:03:58+00:00" + "time": "2025-12-08T11:19:18+00:00" } ], "aliases": [], diff --git a/phpunit.xml b/phpunit.xml index fcea8da..795f6ef 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -1,10 +1,11 @@ - + diff --git a/tests/Feed/Consumer/Base.php b/tests/Feed/Consumer/Base.php index edc8b80..1b1614c 100644 --- a/tests/Feed/Consumer/Base.php +++ b/tests/Feed/Consumer/Base.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Consumer; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\Client\Adapter; use Utopia\CloudEvents\CloudEvent; @@ -515,9 +516,7 @@ private function assertSeekRejected(string $id): void $this->assertSame($first, $consumer->position()); } - /** - * @dataProvider noPositions - */ + #[DataProvider('noPositions')] public function testSeekRejectsAnIdNoFeedCouldUse(string $id): void { $this->assertSeekRejected($id); @@ -538,9 +537,7 @@ public static function noPositions(): array ]; } - /** - * @dataProvider notPositions - */ + #[DataProvider('notPositions')] public function testSeekRejectsAnIdThatIsNotAPosition(string $id): void { if (!$this->ownsItsIdFormat()) { diff --git a/tests/Feed/Producer/Base.php b/tests/Feed/Producer/Base.php index 535ac66..d58c9da 100644 --- a/tests/Feed/Producer/Base.php +++ b/tests/Feed/Producer/Base.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Producer; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Appendable; @@ -183,9 +184,8 @@ public static function payloads(): array /** * The JSON event format leaves `data` unrestricted, so a list or a scalar * has to survive as itself β€” a list must not come back as a map. - * - * @dataProvider payloads */ + #[DataProvider('payloads')] public function testAnyJsonPayloadSurvivesTheRoundTrip(mixed $data): void { $this->producer->produce('test', $data); @@ -356,9 +356,7 @@ public function testRejectsAnEmptyFeedName(): void $this->store(''); } - /** - * @dataProvider notRetentions - */ + #[DataProvider('notRetentions')] public function testRejectsARetentionThatKeepsNothing(int $maxSize): void { $this->expectException(Invalid::class); @@ -377,9 +375,7 @@ public static function notRetentions(): array ]; } - /** - * @dataProvider notIntervals - */ + #[DataProvider('notIntervals')] public function testRejectsAPollIntervalBelowAMillisecond(int $pollInterval): void { $this->expectException(Invalid::class); diff --git a/tests/Feed/Producer/CacheTest.php b/tests/Feed/Producer/CacheTest.php index d7dde4c..9044123 100644 --- a/tests/Feed/Producer/CacheTest.php +++ b/tests/Feed/Producer/CacheTest.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Producer; +use PHPUnit\Framework\Attributes\DataProvider; use Utopia\Cache\Cache as UtopiaCache; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Appendable; @@ -160,9 +161,9 @@ private function unreachable(): Store&Appendable * and would crash on a backend blip instead β€” the exact failure mode the * Transport contract exists to prevent. * - * @dataProvider operations * @param callable(Store&Appendable): void $operation */ + #[DataProvider('operations')] public function testABackendThatIsDownRaisesTransport(callable $operation): void { $store = $this->unreachable(); diff --git a/tests/Feed/Server/Base.php b/tests/Feed/Server/Base.php index 4323dd7..e7ad1e0 100644 --- a/tests/Feed/Server/Base.php +++ b/tests/Feed/Server/Base.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Server; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Appendable; @@ -268,9 +269,8 @@ public function testServeRejectsALastEventIdThatIsNotAPosition(): void * a malformed parameter with a full replay of the retained feed β€” the * most expensive response the endpoint has, and one a caught-up consumer * would read as a sudden flood of new events rather than as the 400 it is. - * - * @dataProvider notStrings */ + #[DataProvider('notStrings')] public function testServeRejectsALastEventIdThatIsNotEvenAString(mixed $lastEventId): void { $this->producer->produce('a'); diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index 1c651a1..1004ca6 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Unit; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\Cache\Cache as UtopiaCache; use Utopia\CloudEvents\CloudEvent; @@ -57,9 +58,7 @@ public static function unusableNames(): array ]; } - /** - * @dataProvider unusableNames - */ + #[DataProvider('unusableNames')] public function testTheNoneStoreStillRejectsEmptyNames(string $feed, string $consumer): void { $this->expectException(Invalid::class); @@ -128,9 +127,9 @@ public function testResettingAPositionThatWasNeverSavedIsHarmless(): void * canonical consume loop retrying on Transport crashes on a backend blip * instead, which is what the Transport contract exists to prevent. * - * @dataProvider operations * @param callable(Cursor): void $operation */ + #[DataProvider('operations')] public function testABackendThatIsDownRaisesTransport(callable $operation): void { $cursor = new CacheCursor(new UtopiaCache(new BrokenCache(raises: true))); @@ -162,9 +161,8 @@ public static function operations(): array * An unusable name is the caller's bug, not the backend's failure, and it * stays Invalid even when the backend behind the cursor is also down β€” * otherwise wrapping the store call would swallow the distinction. - * - * @dataProvider unusableNames */ + #[DataProvider('unusableNames')] public function testAnUnusableNameIsStillInvalidOnABackendThatIsDown(string $feed, string $consumer): void { $cursor = new CacheCursor(new UtopiaCache(new BrokenCache(raises: true))); diff --git a/tests/Feed/Unit/IdTest.php b/tests/Feed/Unit/IdTest.php index 3f3ea4a..3b6262a 100644 --- a/tests/Feed/Unit/IdTest.php +++ b/tests/Feed/Unit/IdTest.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Unit; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\Feed\Exception\Invalid; use Utopia\Feed\Id; @@ -30,9 +31,7 @@ public static function ids(): array ]; } - /** - * @dataProvider ids - */ + #[DataProvider('ids')] public function testValidatesIds(string $id, bool $valid): void { $this->assertSame($valid, Id::isValid($id)); diff --git a/tests/Feed/Unit/KeyTest.php b/tests/Feed/Unit/KeyTest.php index c14c1f2..52ec126 100644 --- a/tests/Feed/Unit/KeyTest.php +++ b/tests/Feed/Unit/KeyTest.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Unit; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\Feed\Key; @@ -71,9 +72,7 @@ public static function names(): array ]; } - /** - * @dataProvider names - */ + #[DataProvider('names')] public function testEveryNameKeepsItsOwnKeys(string $name): void { $others = \array_column(self::names(), 0); diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index 24e0168..9613e0c 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -4,6 +4,7 @@ namespace Utopia\Tests\Unit; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Psr\Http\Message\ResponseInterface; use Utopia\Client; @@ -272,9 +273,9 @@ public function testRejectsABodyThatIsNotABatch(): void * endpoint that moved. Reading one as an empty batch would leave a * consumer sitting quietly at a position that never advances again. * - * @dataProvider notBatches * @param array $payload */ + #[DataProvider('notBatches')] public function testARespondingEndpointThatIsNotAFeedIsNotMistakenForBeingCaughtUp(array $payload): void { [$remote] = $this->remote([FakeTransport::json($payload)]); From 56f4c33e63cfd57d7d5195f972927988278804b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:28:16 +0200 Subject: [PATCH 52/68] Test the server's timeout and limit clamps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Server::poll()` clamps with `max(0, min($timeout, MAX_TIMEOUT))` and `serve()` coerces the query parameter before forwarding it. Neither had a test. Every non-zero timeout in the suite went through `poll()` directly, and every `serve()` test passed `'0'` or garbage, so a refactor that dropped the third argument entirely would have left the whole suite green while long polling over HTTP silently became a plain read β€” invisible in what comes back, since a caught-up read answers with the same empty batch either way. The clamp itself is the security-relevant one: `timeout` arrives from an untrusted HTTP client, so a regression means `?timeout=86400000` holds a worker for as long as the caller asks. The one existing `MAX_TIMEOUT` assertion covers the *consumer's* clamp on its own configuration, which is a different clamp against a different caller. `RecordingStore` captures what the server asked for, since that is the only place clamping is observable, and `ServerTest` pins both parameters across the range: forwarded as given, capped at the protocol maximum, floored (negative timeout to 0, limit to 1), and falling back on garbage β€” through `serve()` and through `poll()`. A position is asserted to arrive untouched, so the clamping is all `serve()` does. The behavioural half runs per adapter in `Server/Base`: `serve()` with a 600ms timeout on an empty feed must actually wait. Only a lower bound, so it measures the code rather than the CI machine. Confirmed all three fail when `serve()` stops forwarding the timeout. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Server/Base.php | 17 +++++ tests/Feed/Support/RecordingStore.php | 33 ++++++++++ tests/Feed/Unit/ServerTest.php | 93 ++++++++++++++++++++++++++- 3 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 tests/Feed/Support/RecordingStore.php diff --git a/tests/Feed/Server/Base.php b/tests/Feed/Server/Base.php index e7ad1e0..87918bc 100644 --- a/tests/Feed/Server/Base.php +++ b/tests/Feed/Server/Base.php @@ -315,6 +315,23 @@ public function testServeLetsTheTipSentinelThrough(): void $this->assertCount(0, $this->server->serve(['lastEventId' => Readable::TIP])); } + /** + * The long-poll contract has to hold through the HTTP entry point, not + * only through `poll()`. A refactor that stopped forwarding the timeout + * would turn every long poll into a plain read β€” consumers would spin + * instead of waiting, and nothing that asserts on returned events could + * tell, because a caught-up read returns the same empty batch either way. + */ + public function testServeHoldsALongPollOnAnEmptyFeed(): void + { + $started = \microtime(true); + + $batch = $this->server->serve(['timeout' => '600']); + + $this->assertCount(0, $batch); + $this->assertGreaterThanOrEqual(0.4, \microtime(true) - $started, 'Must actually wait'); + } + public function testServeFallsBackToTheDefaultOnAGarbageLimit(): void { $this->producer->produce('a'); diff --git a/tests/Feed/Support/RecordingStore.php b/tests/Feed/Support/RecordingStore.php new file mode 100644 index 0000000..960fba1 --- /dev/null +++ b/tests/Feed/Support/RecordingStore.php @@ -0,0 +1,33 @@ +lastEventId = $lastEventId; + $this->limit = $limit; + $this->timeout = $timeout; + + // Answered without waiting: the point is what was asked for. + return parent::poll($lastEventId, $limit, 0); + } +} diff --git a/tests/Feed/Unit/ServerTest.php b/tests/Feed/Unit/ServerTest.php index e9bc3a7..fbbd6d2 100644 --- a/tests/Feed/Unit/ServerTest.php +++ b/tests/Feed/Unit/ServerTest.php @@ -4,15 +4,20 @@ namespace Utopia\Tests\Unit; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Utopia\Feed\Exception\Unsupported; +use Utopia\Feed\Readable; use Utopia\Feed\Server; use Utopia\Feed\Store\None; +use Utopia\Tests\Support\RecordingStore; /** - * The server behaviour no working adapter can show: serving a feed whose - * backend was never configured. Everything a real adapter can exercise lives - * in {@see \Utopia\Tests\Server\Base} instead. + * The server behaviours no working adapter can show: serving a feed whose + * backend was never configured, and what the server asks its store for. + * Clamping and coercion are invisible in the events that come back, so only a + * store that records its arguments can tell them from a server that dropped + * them entirely. Everything else lives in {@see \Utopia\Tests\Server\Base}. */ class ServerTest extends TestCase { @@ -24,4 +29,86 @@ public function testAFeedWithNoBackendCannotBeRead(): void $server->read(); } + + /** + * `timeout` arrives from an untrusted HTTP client, so this is the clamp + * that keeps `?timeout=86400000` from holding a worker for as long as the + * caller feels like asking. The consumer clamps its own inputs too, but + * that is a different clamp against a different caller. + * + * @param array $query + */ + #[DataProvider('timeouts')] + public function testServeClampsTheTimeoutItForwards(array $query, int $expected): void + { + $store = new RecordingStore('edge'); + + (new Server($store))->serve($query); + + $this->assertSame($expected, $store->timeout); + } + + /** + * @return array, int}> + */ + public static function timeouts(): array + { + return [ + 'forwarded as given' => [['timeout' => '2500'], 2_500], + 'capped at the protocol maximum' => [['timeout' => '120000'], Readable::MAX_TIMEOUT], + 'a negative timeout floors at zero' => [['timeout' => '-5000'], 0], + 'garbage falls back to no wait' => [['timeout' => 'soon'], 0], + 'absent means no wait' => [[], 0], + ]; + } + + /** + * `limit` is clamped for the same reason and against the same caller. + * + * @param array $query + */ + #[DataProvider('limits')] + public function testServeClampsTheLimitItForwards(array $query, int $expected): void + { + $store = new RecordingStore('edge'); + + (new Server($store))->serve($query); + + $this->assertSame($expected, $store->limit); + } + + /** + * @return array, int}> + */ + public static function limits(): array + { + return [ + 'forwarded as given' => [['limit' => '25'], 25], + 'capped at the protocol maximum' => [['limit' => '5000'], Readable::MAX_BATCH], + 'zero floors at one' => [['limit' => '0'], 1], + 'a negative limit floors at one' => [['limit' => '-5'], 1], + 'garbage falls back to the maximum' => [['limit' => 'lots'], Readable::MAX_BATCH], + ]; + } + + /** The same clamps apply when a route reaches poll() rather than serve(). */ + public function testPollClampsTheTimeoutAndLimitToo(): void + { + $store = new RecordingStore('edge'); + + (new Server($store))->poll(null, 5_000, 120_000); + + $this->assertSame(Readable::MAX_TIMEOUT, $store->timeout); + $this->assertSame(Readable::MAX_BATCH, $store->limit); + } + + /** A position reaches the store as given, so clamping is all serve() does. */ + public function testServeForwardsThePositionUntouched(): void + { + $store = new RecordingStore('edge'); + + (new Server($store))->serve(['lastEventId' => '1-7']); + + $this->assertSame('1-7', $store->lastEventId); + } } From 103e87b1ed431d1df88a0b27c91945057bc4522c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:31:41 +0200 Subject: [PATCH 53/68] Test the Redis adapters' Transport contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README promises `Exception\Transport` when "the backend or network failed: Redis errors, HTTP failures". The HTTP half was tested thoroughly; the Redis half β€” the flagship production adapter β€” was not tested at all. `grep RedisException tests/` found nothing, and `FailingCursor` only shows how a consumer reacts to a `Transport` it raises itself, so the actual `\RedisException` β†’ `Transport` wrapping in `Store\Redis::append/tip/read` and `Cursor\Redis::load/save/reset` never ran under test. A regression letting the raw exception out would have shipped green and crashed every consumer catching `Utopia\Feed\Exception` as documented. All six operations are now covered, plus the consumer on top of them. Confirmed by widening the catch so the exception escapes: seven failures, all "RedisException ... does not match Transport". The review suggested closing a connected client, which does not work here β€” phpredis 6.3 reconnects transparently on the next command, so every call succeeds. A client that was never connected raises `\RedisException` from the extension for every command instead, which is both deterministic and a real misconfiguration rather than a state only a test can reach. Two defensive branches the review listed as dead are now reachable too, and the pair documents a policy rather than an accident. A foreign value under the feed's key makes Redis answer every stream command with an error, returned rather than raised: `append()` must not report a position for an event that is not in the feed, so it raises `Transport`, while `read()` and `tip()` treat an unreadable feed as an empty one β€” a replay at worst, where failing would stall every consumer. That is the same policy the cache store already applies to a foreign value under its key. `Store\Cache::append`'s false branch, the third one listed, was covered when the cache adapters' wrapping landed. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Consumer/RedisTest.php | 58 ++++++++++++++++++++++++ tests/Feed/Producer/RedisTest.php | 75 +++++++++++++++++++++++++++++++ tests/Feed/Support/UsesRedis.php | 16 +++++++ 3 files changed, 149 insertions(+) diff --git a/tests/Feed/Consumer/RedisTest.php b/tests/Feed/Consumer/RedisTest.php index 76dae36..ac92020 100644 --- a/tests/Feed/Consumer/RedisTest.php +++ b/tests/Feed/Consumer/RedisTest.php @@ -4,6 +4,12 @@ namespace Utopia\Tests\Consumer; +use PHPUnit\Framework\Attributes\DataProvider; +use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Consumer; +use Utopia\Feed\Cursor; +use Utopia\Feed\Cursor\Redis as RedisCursor; +use Utopia\Feed\Exception\Transport; use Utopia\Tests\Support\UsesRedis; class RedisTest extends Base @@ -25,4 +31,56 @@ public function testThePositionIsStoredWhereOperatorsExpectIt(): void $this->assertSame($last, $this->redis()->get('feed:' . $this->name . ':cursor:invalidator')); } + + /** + * The cursor's half of the `Transport` contract. `FailingCursor` shows how + * a consumer reacts to a `Transport`, but it raises one itself β€” the + * `\RedisException` wrapping this adapter does was never run under test, + * so a regression letting the raw exception out would have shipped green. + * + * @param callable(Cursor): void $operation + */ + #[DataProvider('operations')] + public function testABackendThatCannotBeReachedRaisesTransport(callable $operation): void + { + $cursor = new RedisCursor(self::unreachableRedis()); + + $this->expectException(Transport::class); + + $operation($cursor); + } + + /** + * @return array + */ + public static function operations(): array + { + return [ + 'load' => [static function (Cursor $cursor): void { + $cursor->load('edge', 'invalidator'); + }], + 'save' => [static function (Cursor $cursor): void { + $cursor->save('edge', 'invalidator', '1-0'); + }], + 'reset' => [static function (Cursor $cursor): void { + $cursor->reset('edge', 'invalidator'); + }], + ]; + } + + /** + * And the consumer's own contract on top of it: a position that cannot be + * read stops the run, since reading from an unknown position would replay + * the retained feed rather than report the failure. + */ + public function testAConsumerOverAnUnreachableCursorStopsWithTransport(): void + { + $this->producer->produce('a'); + + $consumer = new Consumer($this->store, new RedisCursor(self::unreachableRedis()), 'invalidator', feed: $this->name); + + $this->expectException(Transport::class); + + $consumer->consume(fn (CloudEvent $event) => null); + } } diff --git a/tests/Feed/Producer/RedisTest.php b/tests/Feed/Producer/RedisTest.php index 310f0ce..ac4b65b 100644 --- a/tests/Feed/Producer/RedisTest.php +++ b/tests/Feed/Producer/RedisTest.php @@ -4,7 +4,14 @@ namespace Utopia\Tests\Producer; +use PHPUnit\Framework\Attributes\DataProvider; +use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Appendable; +use Utopia\Feed\Exception\Transport; +use Utopia\Feed\Key; use Utopia\Feed\Producer; +use Utopia\Feed\Store; +use Utopia\Feed\Store\Redis as RedisStore; use Utopia\Tests\Support\UsesRedis; class RedisTest extends Base @@ -55,4 +62,72 @@ public function testTheStreamItselfIsTrimmed(): void $this->assertLessThan(300, $this->redis()->xLen('feed:' . $this->name), 'The stream must be trimmed'); } + + /** + * The README promises `Transport` when "the backend or network failed: + * Redis errors, HTTP failures". The HTTP half of that promise is tested + * thoroughly; the Redis half β€” the flagship production adapter β€” was not + * tested at all, so a regression letting a raw `\RedisException` out would + * have shipped green and crashed every consumer catching + * `Utopia\Feed\Exception` per the README. + * + * @param callable(Store&Appendable): void $operation + */ + #[DataProvider('operations')] + public function testABackendThatCannotBeReachedRaisesTransport(callable $operation): void + { + $store = new RedisStore(self::unreachableRedis(), $this->name); + + $this->expectException(Transport::class); + + $operation($store); + } + + /** + * @return array + */ + public static function operations(): array + { + return [ + 'read' => [static function (Store&Appendable $store): void { + $store->read(null, 10); + }], + 'tip' => [static function (Store&Appendable $store): void { + $store->tip(); + }], + 'append' => [static function (Store&Appendable $store): void { + $store->append(new CloudEvent(id: '', type: 'test', source: 'urn:test')); + }], + ]; + } + + /** + * A foreign value under the feed's key β€” someone else's key collision, or + * a leftover from another tool β€” makes Redis answer every stream command + * with an error rather than raising. Appending must not report a position + * for an event that is not in the feed, so the reply is checked rather + * than trusted. + */ + public function testAppendingOverAForeignValueRaisesTransport(): void + { + $this->redis()->set(Key::feed($this->name), 'not a stream'); + + $this->expectException(Transport::class); + + $this->store->append(new CloudEvent(id: '', type: 'test', source: 'urn:test')); + } + + /** + * Reading past the same value is the opposite call: a feed nobody can read + * is an empty feed β€” a replay at worst β€” and failing the read instead + * would stall every consumer of it. Same policy the cache store applies to + * a foreign value under its key. + */ + public function testReadingPastAForeignValueIsAnEmptyFeed(): void + { + $this->redis()->set(Key::feed($this->name), 'not a stream'); + + $this->assertSame([], $this->store->read(null, 10)); + $this->assertNull($this->store->tip()); + } } diff --git a/tests/Feed/Support/UsesRedis.php b/tests/Feed/Support/UsesRedis.php index 3dc5a37..a4f28a3 100644 --- a/tests/Feed/Support/UsesRedis.php +++ b/tests/Feed/Support/UsesRedis.php @@ -30,6 +30,22 @@ protected function redis(): \Redis return $this->redis; } + /** + * A client that cannot reach a server, for the error contract. + * + * Every command on one raises `\RedisException` from the extension itself, + * so the wrapping runs for real rather than against a double that throws + * on cue. Closing a connected client would not do: phpredis reconnects + * transparently on the next command. + * + * It is also a real misconfiguration β€” a client handed to a store before + * anything connected it β€” rather than a state only a test can produce. + */ + protected static function unreachableRedis(): \Redis + { + return new \Redis(); + } + protected function store(string $name, int $maxSize = 100_000, int $pollInterval = 500): Store&Appendable { return new RedisStore($this->redis(), $name, $maxSize, $pollInterval); From c1243253445202a5c59b54ea136eb9e5604850b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:35:21 +0200 Subject: [PATCH 54/68] Measure and report code coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `phpunit.xml` had no include list, so `--coverage-text` could not produce a meaningful report even if run: there was no way to measure coverage on this branch, let alone track it. Every gap the review found β€” the untested server timeout clamp, the dead Redis error branches, the missing `datacontenttype` round trip β€” was invisible, and uncovered code in future PRs would land with nothing to notice it. `phpunit.xml` now scopes coverage to `src/`, so a report says what the suite exercises of the library rather than of its own fixtures. The test image carries pcov β€” line coverage only, which is all a report needs, at a fraction of Xdebug's cost β€” loaded but switched off, so every ordinary run is unaffected and only `composer coverage` turns it on. Two scripts, because the interesting number and the fast one differ: `coverage` runs the service-free suites (~8s), `coverage:all` adds Redis and pool. CI runs the latter in its own job: the Redis and pooled adapters are exactly the ones whose error paths are easiest to leave untested, so a number that excluded them would flatter the suite. Where it lands today: 97.56% of lines across all suites, 84.11% from the service-free ones alone. No threshold β€” the number is there to be read, and a badly chosen gate is worse than none. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/tests.yml | 20 ++++++++++++++++++++ .gitignore | 1 + Dockerfile | 10 ++++++++++ README.md | 12 ++++++++++++ composer.json | 2 ++ phpunit.xml | 10 ++++++++++ 6 files changed, 55 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3a9d0fb..3f4afe4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,3 +34,23 @@ jobs: - name: Run ${{ matrix.suite }} tests run: docker compose exec -T tests vendor/bin/phpunit --testsuite ${{ matrix.suite }} + + coverage: + name: Coverage + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Build + run: | + docker compose build + docker compose up -d + + # Every suite, so the number covers the Redis and pooled adapters too β€” + # the ones whose error paths are easiest to leave untested. Printed for + # a human to read; there is deliberately no threshold yet, since a bad + # one is worse than none. + - name: Report coverage + run: docker compose exec -T tests composer coverage:all diff --git a/.gitignore b/.gitignore index 5d31694..06d7f80 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /vendor/ /.idea/ +coverage.xml .phpunit.cache/ .phpunit.result.cache .DS_Store diff --git a/Dockerfile b/Dockerfile index 501a5e0..0fed373 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,16 @@ LABEL maintainer="team@appwrite.io" WORKDIR /code +# pcov rather than Xdebug: it only does line coverage, which is all a report +# needs, and costs a fraction of the run time. Loaded but switched off, so +# every ordinary run is unaffected β€” `composer coverage` turns it on for the +# one run that wants it. +RUN apk add --no-cache --virtual .pcov-build-deps $PHPIZE_DEPS \ + && pecl install pcov \ + && docker-php-ext-enable pcov \ + && echo 'pcov.enabled=0' >> /usr/local/etc/php/conf.d/docker-php-ext-pcov.ini \ + && apk del .pcov-build-deps + COPY --from=vendor /src/vendor /code/vendor COPY --from=composer:2.7 /usr/bin/composer /usr/bin/composer diff --git a/README.md b/README.md index 590812e..3a67ec3 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,18 @@ docker compose exec tests composer check CI runs every suite as its own job, so a failing adapter is visible by name. +Coverage is measured over `src/` with [pcov](https://github.com/krakjoe/pcov), +which the test image carries loaded but switched off so ordinary runs are +unaffected: + +```bash +docker compose exec tests composer coverage # service-free suites, ~8s +docker compose exec tests composer coverage:all # every suite, needs Redis +``` + +CI prints the full report on every run. There is no threshold: the number is +there to be read, and a badly chosen gate is worse than none. + To test another PHP version, build with `PHP_VERSION=8.6 docker compose build`, and add it to the `php-versions` matrix in `.github/workflows/tests.yml`. diff --git a/composer.json b/composer.json index 63dfc9b..642a00c 100644 --- a/composer.json +++ b/composer.json @@ -23,6 +23,8 @@ "test:http": "vendor/bin/phpunit --testsuite http", "test:redis": "vendor/bin/phpunit --testsuite redis", "test:pool": "vendor/bin/phpunit --testsuite pool", + "coverage": "php -d pcov.enabled=1 -d pcov.directory=src vendor/bin/phpunit --testsuite unit,memory,cache,http --coverage-text --coverage-clover coverage.xml", + "coverage:all": "php -d pcov.enabled=1 -d pcov.directory=src vendor/bin/phpunit --testsuite unit,memory,cache,http,redis,pool --coverage-text --coverage-clover coverage.xml", "check": "vendor/bin/phpstan analyse --memory-limit=1G", "lint": "vendor/bin/pint --test", "format": "vendor/bin/pint" diff --git a/phpunit.xml b/phpunit.xml index 795f6ef..7c15a33 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -15,6 +15,16 @@ means that adapter honours the whole contract. `unit` needs no services; `redis` and `pool` need the compose Redis. --> + + + + ./src + + + ./tests/Feed/Unit From 82b07a24c703cf3383b0a5f4d14a4c63b47f2e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:36:56 +0200 Subject: [PATCH 55/68] Assert datacontenttype on both sides of the wire `grep datacontenttype tests/` used to find nothing, which is exactly why the store round trip could drop it silently: the producer suite carefully round-tripped `subject`, `dataschema`, `time`, extensions and even digits-only extension names, and skipped this one. The bug and the test gap were the same finding from two sides, and fixing one without the other invites the regression straight back. The producer-side round trip landed with that fix. This closes the other two places the review named. `testRoundTripsAnEvent` now asserts the attribute `produce()` sets, so the everyday path carries it rather than only the deliberate `publish()` case. And the `Remote` decode path, which was untested despite `datacontenttype` being in its `ATTRIBUTES` list and read by `optional()`: one event on the wire carrying every context attribute this library models plus an extension, each asserted after decoding. `RemoteTest` previously checked `id` and `data` and nothing else, so a decode that quietly dropped `subject`, `time`, `dataschema` or `datacontenttype` would have passed. A second case pins the absence: a feed that sends no `datacontenttype` decodes without one, rather than having a value invented that the producer never claimed. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Producer/Base.php | 1 + tests/Feed/Unit/RemoteTest.php | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/tests/Feed/Producer/Base.php b/tests/Feed/Producer/Base.php index d58c9da..70a0a4a 100644 --- a/tests/Feed/Producer/Base.php +++ b/tests/Feed/Producer/Base.php @@ -67,6 +67,7 @@ public function testRoundTripsAnEvent(): void $this->assertSame('urn:test', $events[0]->source); $this->assertSame(['tags' => ['domain' => 'example.com']], $events[0]->data); $this->assertNotNull($events[0]->time); + $this->assertSame('application/json', $events[0]->datacontenttype, 'produce() encodes its payload as JSON and says so'); } public function testStampsTheSourceAndTime(): void diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index 9613e0c..cf98d96 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -373,6 +373,50 @@ public function testFailsWhenAnEventIsNotACloudEventAtAll(): void $remote->read(); } + /** + * Every context attribute this library models has to come off the wire, + * not only the ones a consumer happens to look at. An attribute silently + * dropped in decoding is the same class of bug as one dropped in storing, + * and neither shows up in a test that asserts on `id` and `data` alone. + */ + public function testEveryModelledAttributeComesOffTheWire(): void + { + [$remote] = $this->remote([FakeTransport::json([self::raw('1-0', 'io.appwrite.edge.invalidate-rule', [ + 'subject' => 'example.com', + 'time' => '2026-07-31T09:15:02.123Z', + 'datacontenttype' => 'application/xml', + 'dataschema' => 'https://example.com/schema.json', + 'data' => '', + 'traceparent' => '00-abc-def-01', + ])])]); + + $event = $remote->read()[0]; + + $this->assertSame('1-0', $event->id); + $this->assertSame('io.appwrite.edge.invalidate-rule', $event->type); + $this->assertSame('urn:test', $event->source); + $this->assertSame('1.0', $event->specversion); + $this->assertSame('example.com', $event->subject); + $this->assertSame('2026-07-31T09:15:02.123Z', $event->time); + $this->assertSame('application/xml', $event->datacontenttype); + $this->assertSame('https://example.com/schema.json', $event->dataschema); + $this->assertSame('', $event->data); + $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); + } + + /** + * CloudEvents reads an absent `datacontenttype` as "the data is JSON", so + * a feed that does not send one must decode without inventing it β€” the + * attribute stays unset rather than becoming a value the producer never + * claimed. + */ + public function testAnAbsentDatacontenttypeIsNotInvented(): void + { + [$remote] = $this->remote([FakeTransport::json([self::raw('1-0', 'a')])]); + + $this->assertNull($remote->read()[0]->datacontenttype); + } + /** * The forward-compatibility property a feed depends on: it is read by * consumers older than the producer by design, so a producer that adds an From f7933cd4151b597100d4b4bf4b24f1bd7366e2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:39:08 +0200 Subject: [PATCH 56/68] Set timing bounds from the regression, not the runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several tests asserted an upper bound on elapsed time just above what the correct code takes, which measures the CI runner rather than the code. `testPollHonoursATimeoutShorterThanThePollInterval` allowed 300ms for a 100ms timeout and runs in four adapter suites β€” in `redis` and `pool` each poll tick is a round trip into a container, on a shared GitHub runner. Two hundred milliseconds of headroom is one noisy neighbour away from red, and that failure mode is the worst kind: rare, unreproducible locally, and it teaches everyone to hit re-run, which is how real regressions get waved through. Each bound is now derived from the elapsed time the regression it guards would produce. Overshooting the deadline means sleeping a full 500ms interval, and ignoring a configured interval means falling back to the 500ms default, so 0.45 catches both while leaving 350–400ms of slack over the ~20–100ms the correct code takes. Same for the no-timeout read, where sleeping at all costs a whole interval. The reasoning is written next to each number so the next person changing one knows what it is protecting. Confirmed by reintroducing both bugs: each fails exactly the test that exists for it, at ~0.50s against the 0.45 bound. Lower bounds are untouched β€” a sleep cannot finish early, so they cannot flake. `testDelegatesLongPollingToTheProducer` keeps its clock check with a note that the request count next to it is the assertion actually doing the work. Not changed: the deliberate 600ms waits multiplied across five adapter suites. That is run time rather than flakiness, and removing it means a clock abstraction through `Store::poll()` β€” a change to production code to suit the tests, which is a trade worth making deliberately rather than in passing. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Server/Base.php | 20 +++++++++++++++++--- tests/Feed/Server/MemoryTest.php | 5 ++++- tests/Feed/Unit/RemoteTest.php | 4 ++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/Feed/Server/Base.php b/tests/Feed/Server/Base.php index 87918bc..979eb11 100644 --- a/tests/Feed/Server/Base.php +++ b/tests/Feed/Server/Base.php @@ -134,6 +134,15 @@ public function testRejectsAPositionThatIsNotAFeedId(): void $this->server->read('not-a-position'); } + /** + * Every upper bound below is set from the elapsed time the *regression* + * would produce, not from the time the correct code takes. A bound just + * above the expected duration measures the CI runner rather than the + * code, and fails rarely, unreproducibly, and only under load β€” which + * teaches everyone to hit re-run, which is how real regressions get + * waved through. Lower bounds are safe either way: a sleep cannot + * finish early. + */ public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void { $this->producer->produce('test'); @@ -142,7 +151,8 @@ public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void $events = $this->server->poll(null, 10, 2000); $this->assertCount(1, $events); - $this->assertLessThan(1, \microtime(true) - $started); + // Waiting out the 2s timeout despite a full batch is the regression. + $this->assertLessThan(1, \microtime(true) - $started, 'Waiting events must return at once, not at the timeout'); } public function testPollGivesUpAtTheTimeoutWithAnEmptyBatch(): void @@ -161,7 +171,8 @@ public function testPollWithoutATimeoutIsAPlainRead(): void $started = \microtime(true); $this->assertCount(0, $this->server->poll()); - $this->assertLessThan(0.4, \microtime(true) - $started); + // Sleeping at all with no timeout costs a whole 500ms interval. + $this->assertLessThan(0.45, \microtime(true) - $started, 'No timeout means no sleep'); } /** @@ -178,7 +189,10 @@ public function testPollHonoursATimeoutShorterThanThePollInterval(): void $this->assertCount(0, $events); $this->assertGreaterThanOrEqual(0.08, $elapsed, 'Must actually wait out the timeout'); - $this->assertLessThan(0.3, $elapsed, 'Must not sleep a full interval past the deadline'); + // Overshooting means sleeping the full 500ms interval past a 100ms + // deadline, so anything under 0.45 catches it β€” with 350ms of slack + // over the ~100ms this takes when correct. + $this->assertLessThan(0.45, $elapsed, 'Must not sleep a full interval past the deadline'); } public function testTipIsTheNewestEventsId(): void diff --git a/tests/Feed/Server/MemoryTest.php b/tests/Feed/Server/MemoryTest.php index 65daba1..131a5f9 100644 --- a/tests/Feed/Server/MemoryTest.php +++ b/tests/Feed/Server/MemoryTest.php @@ -37,6 +37,9 @@ public function testAShorterPollIntervalDeliversAMidPollEventSooner(): void $elapsed = \microtime(true) - $started; $this->assertCount(1, $events); - $this->assertLessThan(0.4, $elapsed, 'A 20ms interval must beat the default 500ms floor'); + // Ignoring the configured interval means falling back to the 500ms + // default, so anything under 0.45 catches it β€” with 400ms of slack + // over the ~20ms this takes when correct. + $this->assertLessThan(0.45, $elapsed, 'A 20ms interval must beat the default 500ms floor'); } } diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index cf98d96..49b9386 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -166,6 +166,10 @@ public function testDelegatesLongPollingToTheProducer(): void $started = \microtime(true); $remote->poll(null, 100, 5000); + // The request count is the real check β€” it catches a client-side loop + // whatever the machine is doing. The clock only guards against a wait + // that makes no request at all, and is bounded by the 5s a wrong + // implementation would take rather than the ~0s this one does. $this->assertLessThan(1, \microtime(true) - $started, 'Must not wait client-side'); $this->assertCount(1, $transport->recorder->requests, 'Must not poll in a loop'); $this->assertStringContainsString('timeout=5000', $transport->recorder->last()['uri']); From 5490feb61de04d2931e553f88b71e8d7c7e6c145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:42:49 +0200 Subject: [PATCH 57/68] Make both decode paths filter extensions the same way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Remote::event()` silently dropped any extension the spec cannot carry β€” a name outside `[a-z0-9]+`, a value that is not bool/int/string β€” and none of it was tested. One well-formed extension riding along was covered; what happened to a float, an array, or an uppercase name was nobody's decision, just whatever the code did. The two decode paths also disagreed, and neither side of the disagreement was covered, so nothing would have gone red when it mattered. Over HTTP a foreign `"ratio": 1.5` was filtered out and the event delivered. Read from a local store, the same event was merged verbatim into `CloudEvent::fromArray()`, which rejects it β€” and since a store read decodes every entry in the batch, that entry failed every read past it, permanently, for every consumer, until it fell off the trim horizon. The worst failure shape a feed has, arrived at by accident. Both now go through one `Extensions::filter()`. Filtering, rather than raising, is the version somebody would have chosen: a feed is read by consumers older than its producer by design, and one odd attribute must not cost the event and everything behind it. This is not hypothetical for a local store either β€” a Redis stream is writable by any tool, and this library's producer cannot be assumed the only writer, which is how the new Redis test stages it. Tests name each shape the filter drops, so dropping stays a choice: float, array, object and null values; uppercase, dashed and underscored names. And each shape it keeps, including a digits-only name. Fixing that last one turned up the same hazard inside the suite: `RemoteTest::raw()` built fixtures with `array_merge()`, which renumbers integer keys, so a digits-only extension was lost in the fixture before the code under test ever saw it. It unions now. Co-Authored-By: Claude Opus 5 (1M context) --- src/Feed/Extensions.php | 71 +++++++++++++++++++++++++++++ src/Feed/Remote.php | 27 +---------- src/Feed/Store.php | 6 ++- tests/Feed/Producer/RedisTest.php | 34 ++++++++++++++ tests/Feed/Unit/RemoteTest.php | 75 +++++++++++++++++++++++++++++-- 5 files changed, 182 insertions(+), 31 deletions(-) create mode 100644 src/Feed/Extensions.php diff --git a/src/Feed/Extensions.php b/src/Feed/Extensions.php new file mode 100644 index 0000000..fbab86f --- /dev/null +++ b/src/Feed/Extensions.php @@ -0,0 +1,71 @@ + $candidates A decoded event, or just its + * extension attributes β€” modelled attributes are dropped either way. + * @return array + */ + public static function filter(array $candidates): array + { + $extensions = []; + + /** @var mixed $value */ + foreach ($candidates as $name => $value) { + if (\in_array($name, self::ATTRIBUTES, true)) { + continue; + } + + // A digit-only name is legal per the spec and arrives as an + // integer key in PHP, so the name is compared as a string. + if (\preg_match('/^[a-z0-9]+$/', (string) $name) !== 1) { + continue; + } + + if (\is_bool($value) || \is_int($value) || \is_string($value)) { + $extensions[$name] = $value; + } + } + + return $extensions; + } +} diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index 44bacfc..b828ed9 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -33,19 +33,6 @@ class Remote implements Readable */ private const int TIMEOUT_MARGIN = 10_000; - /** The context attributes this library models; the rest are extensions. */ - private const array ATTRIBUTES = [ - 'specversion', - 'type', - 'source', - 'id', - 'subject', - 'time', - 'datacontenttype', - 'dataschema', - 'data', - ]; - private readonly RequestFactory $requests; public function __construct( @@ -190,19 +177,7 @@ private static function event(array $raw): CloudEvent } } - $extensions = []; - - /** @var mixed $value */ - foreach ($raw as $name => $value) { - if (\in_array($name, self::ATTRIBUTES, true)) { - continue; - } - - if (\preg_match('/^[a-z0-9]+$/', (string) $name) === 1 - && (\is_bool($value) || \is_int($value) || \is_string($value))) { - $extensions[$name] = $value; - } - } + $extensions = Extensions::filter($raw); return new CloudEvent( type: $raw['type'], diff --git a/src/Feed/Store.php b/src/Feed/Store.php index f9f80cd..f6ddff7 100644 --- a/src/Feed/Store.php +++ b/src/Feed/Store.php @@ -120,7 +120,11 @@ protected static function decode(string $id, array $fields): CloudEvent } } - $event += \is_array($extensions) ? $extensions : []; + // Filtered rather than merged verbatim, and for the same reason the + // wire path filters: a read decodes every entry in the batch, so one + // entry a foreign writer left an unusable attribute on would otherwise + // fail every read past it, permanently, for every consumer. + $event += Extensions::filter(\is_array($extensions) ? $extensions : []); try { // The docblock wants array, but a digit-only diff --git a/tests/Feed/Producer/RedisTest.php b/tests/Feed/Producer/RedisTest.php index ac4b65b..848831f 100644 --- a/tests/Feed/Producer/RedisTest.php +++ b/tests/Feed/Producer/RedisTest.php @@ -63,6 +63,40 @@ public function testTheStreamItselfIsTrimmed(): void $this->assertLessThan(300, $this->redis()->xLen('feed:' . $this->name), 'The stream must be trimmed'); } + /** + * A stream is a shared, writable thing: another tool can `XADD` into a + * feed, and this library's own producer cannot be the only writer assumed. + * An entry carrying an attribute a CloudEvent cannot hold must therefore + * decode without it, exactly as the same event would arriving over HTTP. + * + * The failure this prevents is the worst shape a feed has: a read decodes + * every entry in the batch, so one poisoned entry would fail every read + * past it β€” permanently, for every consumer, until it fell off the trim + * horizon. + */ + public function testAForeignWritersUnusableExtensionIsDroppedRatherThanWedgingTheFeed(): void + { + $this->producer->produce('a'); + + $this->redis()->xAdd(Key::feed($this->name), '*', [ + 'type' => 'foreign', + 'source' => 'urn:somebody:else', + 'subject' => '', + 'datacontenttype' => '', + 'dataschema' => '', + 'time' => '', + 'data' => '{"ok":true}', + 'extensions' => '{"ratio":1.5,"trace":"abc"}', + ]); + + $this->producer->produce('c'); + + $events = $this->store->read(null, 10); + + $this->assertSame(['a', 'foreign', 'c'], \array_map(fn (CloudEvent $e): string => $e->type, $events), 'Nothing behind it is lost'); + $this->assertSame(['trace' => 'abc'], $events[1]->extensions, 'Only the attribute it could not hold is gone'); + } + /** * The README promises `Transport` when "the backend or network failed: * Redis errors, HTTP failures". The HTTP half of that promise is tested diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index 49b9386..3c226dd 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -304,17 +304,22 @@ public static function notBatches(): array /** * One event as a producer would put it on the wire. * - * @param array $overrides - * @return array + * Overrides win, and are unioned rather than merged: a digits-only + * extension name is legal per the spec and an integer key in PHP, which + * `array_merge()` would silently renumber β€” losing the attribute inside + * the fixture, before the code under test ever saw it. + * + * @param array $overrides + * @return array */ private static function raw(string $id, string $type, array $overrides = []): array { - return \array_merge([ + return $overrides + [ 'specversion' => '1.0', 'id' => $id, 'type' => $type, 'source' => 'urn:test', - ], $overrides); + ]; } /** @@ -444,6 +449,68 @@ public function testSurvivesAProducerThatMovedAhead(): void $this->assertSame('00-abc-def-01', $events[0]->extensions['traceparent']); } + /** + * The spec is narrow about extensions β€” a name of lowercase letters and + * digits, a value that is a boolean, an integer or a string β€” and a feed + * is read by consumers older than its producer by design. So an attribute + * outside that is dropped and the event still delivered, rather than one + * odd attribute costing the whole event and everything behind it. + * + * Dropping is a choice, not an accident, which is why each shape it can + * take is named here. + * + * @param array $extension + */ + #[DataProvider('unusableExtensions')] + public function testAnExtensionTheSpecCannotCarryIsDroppedAndTheEventKept(array $extension): void + { + [$remote] = $this->remote([FakeTransport::json([ + self::raw('1-0', 'a', $extension + ['keeps' => 'this one']), + ])]); + + $events = $remote->read(); + + $this->assertCount(1, $events, 'The event is still delivered'); + $this->assertSame(['keeps' => 'this one'], $events[0]->extensions); + } + + /** + * @return array}> + */ + public static function unusableExtensions(): array + { + return [ + // A JSON number with a decimal point decodes as a float. + 'a float value' => [['ratio' => 1.5]], + 'an array value' => [['tags' => ['a', 'b']]], + 'an object value' => [['nested' => ['a' => 'b']]], + 'a null value' => [['missing' => null]], + 'an uppercase name' => [['traceParent' => '00-abc-def-01']], + 'a name with a dash' => [['trace-parent' => '00-abc-def-01']], + 'a name with an underscore' => [['trace_parent' => '00-abc-def-01']], + ]; + } + + /** + * The types the spec does allow, including a digits-only name β€” legal per + * the spec, and an integer key in PHP, which anything merging with + * `array_merge()` would silently renumber. + */ + public function testEveryExtensionTheSpecAllowsIsKept(): void + { + [$remote] = $this->remote([FakeTransport::json([ + self::raw('1-0', 'a', ['trace' => 'abc', 'retrycount' => 2, 'replayed' => true, '123' => 'digits']), + ])]); + + $extensions = $remote->read()[0]->extensions; + + $this->assertSame('abc', $extensions['trace']); + $this->assertSame(2, $extensions['retrycount']); + $this->assertTrue($extensions['replayed']); + // @phpstan-ignore offsetAccess.notFound ('123' is an integer key in PHP) + $this->assertSame('digits', $extensions['123']); + } + /** * The spec's optional compaction/deletion feature marks an event with a * `method` attribute. This library does not implement the feature, but a From 0aafe6d641ac3352df07d58074de117fda6a7d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:44:26 +0200 Subject: [PATCH 58/68] Move the exact-trim retention tests into the shared suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `testRetentionTrimsToExactlyTheCap` and `testAcceptsTheSmallestUsefulRetentionCap` were copy-pasted line for line into `Producer/MemoryTest` and `Producer/CacheTest`, against the suite's own design principle: one abstract scenario suite per component, extended by every adapter. Two copies drift β€” tighten one and the other silently stays weak β€” and a future adapter that trims exactly inherits nothing, having to opt in by copying again. Both now live in `Producer/Base` behind a `trimsExactly()` hook. The property is the default, since it is what every adapter does except the two backed by Redis, where `XADD ... MAXLEN ~` trims to a node boundary β€” which is the whole reason the shared scenario only asserts the loose bound. `Producer/RedisTest` and `Producer/PoolTest` declare `false` and skip. Confirmed by adapter: Memory and Cache run both, Redis and Pool report them skipped rather than passing vacuously. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Producer/Base.php | 45 ++++++++++++++++++++++++++++++ tests/Feed/Producer/CacheTest.php | 27 ------------------ tests/Feed/Producer/MemoryTest.php | 29 ------------------- tests/Feed/Producer/PoolTest.php | 6 ++++ tests/Feed/Producer/RedisTest.php | 6 ++++ 5 files changed, 57 insertions(+), 56 deletions(-) diff --git a/tests/Feed/Producer/Base.php b/tests/Feed/Producer/Base.php index 70a0a4a..79e12fb 100644 --- a/tests/Feed/Producer/Base.php +++ b/tests/Feed/Producer/Base.php @@ -350,6 +350,51 @@ public function testRetentionIsBoundedAndDropsTheOldestFirst(): void $this->assertSame('event-299', \end($types), 'The newest event is retained'); } + /** + * Whether this adapter trims to exactly the cap. Redis does not β€” `XADD` + * with `~` trims to a node boundary, which is the whole reason the + * scenario above only asserts the loose bound. + */ + protected function trimsExactly(): bool + { + return true; + } + + /** An adapter that trims exactly owes the tighter contract: the bound is the cap itself. */ + public function testRetentionTrimsToExactlyTheCap(): void + { + if (!$this->trimsExactly()) { + $this->markTestSkipped('This adapter trims approximately'); + } + + $store = $this->store($this->name, maxSize: 3); + $producer = new Producer($store, 'urn:test'); + + foreach (['a', 'b', 'c', 'd', 'e'] as $type) { + $producer->produce($type); + } + + $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 10))); + } + + public function testAcceptsTheSmallestUsefulRetentionCap(): void + { + if (!$this->trimsExactly()) { + $this->markTestSkipped('This adapter trims approximately'); + } + + $store = $this->store($this->name, maxSize: 1); + $producer = new Producer($store, 'urn:test'); + + $producer->produce('a'); + $producer->produce('b'); + + $events = $store->read(null, 10); + + $this->assertCount(1, $events); + $this->assertSame('b', $events[0]->type); + } + public function testRejectsAnEmptyFeedName(): void { $this->expectException(Invalid::class); diff --git a/tests/Feed/Producer/CacheTest.php b/tests/Feed/Producer/CacheTest.php index 9044123..eee2ac4 100644 --- a/tests/Feed/Producer/CacheTest.php +++ b/tests/Feed/Producer/CacheTest.php @@ -22,33 +22,6 @@ class CacheTest extends Base { use UsesCache; - /** The cache store trims exactly, so the bound is the cap itself. */ - public function testRetentionTrimsToExactlyTheCap(): void - { - $store = $this->store($this->name, maxSize: 3); - $producer = new Producer($store, 'urn:test'); - - foreach (['a', 'b', 'c', 'd', 'e'] as $type) { - $producer->produce($type); - } - - $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 10))); - } - - public function testAcceptsTheSmallestUsefulRetentionCap(): void - { - $store = $this->store($this->name, maxSize: 1); - $producer = new Producer($store, 'urn:test'); - - $producer->produce('a'); - $producer->produce('b'); - - $events = $store->read(null, 10); - - $this->assertCount(1, $events); - $this->assertSame('b', $events[0]->type); - } - /** * The property that justifies the adapter: the feed lives in the cache, not * in the store object, so a second store over the same cache β€” another diff --git a/tests/Feed/Producer/MemoryTest.php b/tests/Feed/Producer/MemoryTest.php index 1872fbb..482a4c6 100644 --- a/tests/Feed/Producer/MemoryTest.php +++ b/tests/Feed/Producer/MemoryTest.php @@ -4,38 +4,9 @@ namespace Utopia\Tests\Producer; -use Utopia\CloudEvents\CloudEvent; -use Utopia\Feed\Producer; use Utopia\Tests\Support\UsesMemory; class MemoryTest extends Base { use UsesMemory; - - /** Memory trims exactly, so the bound is the cap itself. */ - public function testRetentionTrimsToExactlyTheCap(): void - { - $store = $this->store($this->name, maxSize: 3); - $producer = new Producer($store, 'urn:test'); - - foreach (['a', 'b', 'c', 'd', 'e'] as $type) { - $producer->produce($type); - } - - $this->assertSame(['c', 'd', 'e'], \array_map(fn (CloudEvent $e): string => $e->type, $store->read(null, 10))); - } - - public function testAcceptsTheSmallestUsefulRetentionCap(): void - { - $store = $this->store($this->name, maxSize: 1); - $producer = new Producer($store, 'urn:test'); - - $producer->produce('a'); - $producer->produce('b'); - - $events = $store->read(null, 10); - - $this->assertCount(1, $events); - $this->assertSame('b', $events[0]->type); - } } diff --git a/tests/Feed/Producer/PoolTest.php b/tests/Feed/Producer/PoolTest.php index 9866c59..b64884e 100644 --- a/tests/Feed/Producer/PoolTest.php +++ b/tests/Feed/Producer/PoolTest.php @@ -9,4 +9,10 @@ class PoolTest extends Base { use UsesPool; + + /** The same Redis underneath, so the same approximate trim. */ + protected function trimsExactly(): bool + { + return false; + } } diff --git a/tests/Feed/Producer/RedisTest.php b/tests/Feed/Producer/RedisTest.php index 848831f..8752376 100644 --- a/tests/Feed/Producer/RedisTest.php +++ b/tests/Feed/Producer/RedisTest.php @@ -18,6 +18,12 @@ class RedisTest extends Base { use UsesRedis; + /** `XADD ... MAXLEN ~` trims to a node boundary, not to the cap. */ + protected function trimsExactly(): bool + { + return false; + } + /** * The wire format other tools rely on: `GET`-able keys named after the * feed, holding a plain Redis stream β€” which is what lets an operator From 4cb4a75ce90ea385aeacddd5061badc0122cb7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:46:53 +0200 Subject: [PATCH 59/68] Route on the feed name in the HTTP suite's endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `FeedServer` read only the query string. The request path β€” which carries the feed name `Remote` so carefully `rawurlencode()`s β€” was discarded, so the fixture answered any name with the one feed it held. A consumer pointed at feed `other` was served `edge` and read the right events by accident; the only name check in the suite was client-side, in `Consumer` rejecting a contradicting `feed:` argument. For a suite whose docblock claims it exercises "the whole contract", routing was the missing half. The endpoint now compares the decoded last path segment against the feed it serves and answers 404 otherwise, and three tests cover what that makes checkable: - A consumer pointed at another feed gets `Transport` with status 404 and records no position. Confirmed to fail without the routing. - A name needing encoding round-trips: `a b/c` is encoded into one path segment and decoded back to the name the producer knows. The existing encoding test asserts the URI string, which cannot show that anything decodes it. - The `Accept` the consumer sends is the `Content-Type` the producer answers with, checked against each other rather than each against a literal β€” the loop left open when the media type was declared once on `Readable`. `Recorder` captures the served content type for it. Splitting the path before decoding is deliberate and commented: a feed called `a/b` travels as `a%2Fb`, and decoding first would split it into a path nobody asked for. So is tolerating a path with no slash at all β€” a consumer built over a client with no base URI sends the bare name, which is exactly how this suite builds one. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Consumer/HttpTest.php | 60 +++++++++++++++++++++++++++++++ tests/Feed/Support/FakeClient.php | 1 + tests/Feed/Support/FeedServer.php | 28 +++++++++++++++ tests/Feed/Support/Recorder.php | 4 +-- 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/tests/Feed/Consumer/HttpTest.php b/tests/Feed/Consumer/HttpTest.php index e6e48c2..158b786 100644 --- a/tests/Feed/Consumer/HttpTest.php +++ b/tests/Feed/Consumer/HttpTest.php @@ -8,8 +8,10 @@ use Utopia\CloudEvents\CloudEvent; use Utopia\Feed\Appendable; use Utopia\Feed\Consumer; +use Utopia\Feed\Exception\Transport; use Utopia\Feed\Producer; use Utopia\Feed\Readable; +use Utopia\Feed\Remote; use Utopia\Feed\Server; use Utopia\Feed\Store; use Utopia\Tests\Support\FeedServer; @@ -59,6 +61,64 @@ public function testSeekAcceptsTheOpaqueIdARemoteFeedMayUse(): void $this->assertSame($opaque, $this->cursor->load($this->name, 'invalidator'), 'And it is persisted, so a restart resumes from it'); } + /** + * The feed name is the one thing on the wire only the serving side can + * check. `Remote` encodes it into the request path, and the consumer's own + * `feed:` check is client-side β€” so nothing confirmed end to end that the + * name reaching the endpoint is the one it holds. A consumer pointed at + * the wrong feed must fail rather than read the right events by accident. + */ + public function testAConsumerPointedAtAnotherFeedIsNotServedThisOne(): void + { + $this->producer->produce('a'); + + $consumer = new Consumer($this->source($this->store), $this->cursor, 'invalidator', feed: $this->name . '-other'); + + try { + $consumer->consume(fn (CloudEvent $event) => null); + $this->fail('The endpoint holds another feed, so the read should have failed'); + } catch (Transport $error) { + $this->assertSame(404, $error->getCode()); + } + + $this->assertNull($consumer->position(), 'And nothing was recorded as read'); + } + + /** + * A name that needs encoding survives the round trip: `Remote` percent- + * encodes it into one path segment and the endpoint decodes that segment + * back. Asserting the URI string alone, as the encoding test does, cannot + * show that anything decodes it to the name the producer knows. + */ + public function testAFeedNameThatNeedsEncodingStillRoutes(): void + { + $store = $this->store('a b/c'); + (new Producer($store, 'urn:test'))->produce('a'); + + $consumer = new Consumer($this->source($store), $this->cursor, 'invalidator', feed: 'a b/c'); + + $this->assertSame(['a'], $this->drain($consumer)); + $this->assertStringContainsString('a%20b%2Fc', $this->endpoint->recorder->last()['uri']); + } + + /** + * The two halves of the media type handshake, checked against each other + * rather than each against a literal: the `Accept` the consumer sends is + * the `Content-Type` the producer answers with. Both now alias one + * constant, so this is what would notice if they stopped. + */ + public function testTheAcceptSentIsTheContentTypeServed(): void + { + $this->producer->produce('a'); + + $this->drain($this->consumer()); + + $request = $this->endpoint->recorder->last(); + + $this->assertSame(Remote::MEDIA_TYPE, $request['headers']['Accept'] ?? null); + $this->assertSame(Remote::MEDIA_TYPE, $request['contentType'], 'The producer answers with what the consumer asked for'); + } + public function testTheProducerCachesFullBatchesAndNothingElse(): void { foreach (\range(1, 5) as $i) { diff --git a/tests/Feed/Support/FakeClient.php b/tests/Feed/Support/FakeClient.php index 15d7ca6..7b6e643 100644 --- a/tests/Feed/Support/FakeClient.php +++ b/tests/Feed/Support/FakeClient.php @@ -44,6 +44,7 @@ public function sendRequest(RequestInterface $request): ResponseInterface 'timeout' => $this->timeout, 'status' => $response->getStatusCode(), 'cacheControl' => $response->getHeaderLine(Header::CACHE_CONTROL), + 'contentType' => $response->getHeaderLine(Header::CONTENT_TYPE), ]; return $response; diff --git a/tests/Feed/Support/FeedServer.php b/tests/Feed/Support/FeedServer.php index 866e261..d8d74e4 100644 --- a/tests/Feed/Support/FeedServer.php +++ b/tests/Feed/Support/FeedServer.php @@ -19,6 +19,12 @@ * HTTP route would, so a consumer reading it exercises the whole contract β€” * parameters, body and caching β€” rather than a fixture written to match the * consumer. + * + * It routes on the request path too, minimally but honestly. The feed name is + * the one thing on the wire that only the serving side can check: `Remote` + * `rawurlencode()`s it into the path, and a fixture that ignored the path + * would answer any name with the one feed it holds β€” so a consumer pointed at + * the wrong feed would read the right events and no test could tell. */ class FeedServer extends FakeClient { @@ -29,6 +35,10 @@ public function __construct(private readonly Server $server, Recorder $recorder protected function respond(RequestInterface $request): ResponseInterface { + if (self::feed($request) !== $this->server->getName()) { + return new Response(404, body: new Stream\Factory()->createStream('{"message":"No such feed"}')); + } + $query = []; \parse_str($request->getUri()->getQuery(), $query); @@ -40,4 +50,22 @@ protected function respond(RequestInterface $request): ResponseInterface ->withHeader(Header::CONTENT_TYPE, Batch::MEDIA_TYPE) ->withHeader(Header::CACHE_CONTROL, $batch->cacheControl()); } + + /** + * The feed name the request asks for: the last path segment, decoded. + * + * Split before decoding, never after β€” `Remote` percent-encodes the name + * into one segment, so a feed called `a/b` arrives as `a%2Fb` and decoding + * first would split it into a path it never asked for. + * + * The path may be relative and hold no slash at all: a consumer built + * straight over a client with no base URI sends the bare name. + */ + private static function feed(RequestInterface $request): string + { + $path = $request->getUri()->getPath(); + $slash = \strrpos($path, '/'); + + return \rawurldecode($slash === false ? $path : \substr($path, $slash + 1)); + } } diff --git a/tests/Feed/Support/Recorder.php b/tests/Feed/Support/Recorder.php index df6c049..4e19a11 100644 --- a/tests/Feed/Support/Recorder.php +++ b/tests/Feed/Support/Recorder.php @@ -16,7 +16,7 @@ */ class Recorder { - /** @var list, timeout: float|null, status: int, cacheControl: string}> */ + /** @var list, timeout: float|null, status: int, cacheControl: string, contentType: string}> */ public array $requests = []; /** @@ -28,7 +28,7 @@ class Recorder public array $responses = []; /** - * @return array{uri: string, method: string, headers: array, timeout: float|null, status: int, cacheControl: string} + * @return array{uri: string, method: string, headers: array, timeout: float|null, status: int, cacheControl: string, contentType: string} */ public function last(): array { From 0dab9bbac59d0af2a62e82a969059326ccee4861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:48:11 +0200 Subject: [PATCH 60/68] Restore consumer coverage over a real Redis stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Consumer/Base::setUp()` always builds the feed as a `MemoryStore`; the adapter subclasses swap only the cursor. So `Consumer/RedisTest` ran a Redis *cursor* against a *memory* feed, and after the test rework nothing anywhere had a `Consumer` read a real Redis stream. The consumer's paging arithmetic β€” `Id::after`, strictly-after reads, the batch loop β€” was exercised only against ids this library mints itself, never against the ones `XADD` assigns or the approximate trimming `XRANGE` reads back. Added to `Consumer/RedisTest` rather than by deriving `Consumer/Base`'s store from the trait, which keeps the change local: the shared scenarios still run against the simplest store there is, so a failure in them is still the consuming side's rather than a store's. Four scenarios, three of them lost with the old `E2E/RedisTest`: consume and resume through a persisted cursor without replaying, two named consumers over one stream, and reset replaying what the stream retains. The fourth is new and is the one the gap was really about β€” paging a backlog in batches of three. `XADD` assigns ids within a millisecond by bumping the sequence, so a batch boundary regularly falls between two ids sharing a timestamp, which is exactly where paging by string comparison goes wrong. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Consumer/RedisTest.php | 89 +++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/tests/Feed/Consumer/RedisTest.php b/tests/Feed/Consumer/RedisTest.php index ac92020..50ce855 100644 --- a/tests/Feed/Consumer/RedisTest.php +++ b/tests/Feed/Consumer/RedisTest.php @@ -9,13 +9,102 @@ use Utopia\Feed\Consumer; use Utopia\Feed\Cursor; use Utopia\Feed\Cursor\Redis as RedisCursor; +use Utopia\Feed\Appendable; use Utopia\Feed\Exception\Transport; +use Utopia\Feed\Producer; +use Utopia\Feed\Store; use Utopia\Tests\Support\UsesRedis; class RedisTest extends Base { use UsesRedis; + /** + * A feed on a real Redis stream, and a producer over it. + * + * The shared scenarios run against a memory feed on purpose β€” a failure + * there is the consuming side's, not a store's. But that leaves the + * consumer's paging arithmetic (`Id::after`, strictly-after reads, the + * batch loop) exercised only against ids this library mints itself, never + * against the ones `XADD` assigns or the approximate trimming `XRANGE` + * reads back. The scenarios below fill exactly that gap and no more. + * + * @return array{Store&Appendable, Producer} + */ + private function stream(): array + { + $store = $this->store($this->name); + + return [$store, new Producer($store, 'urn:test')]; + } + + public function testConsumesAStreamThroughAPersistedCursor(): void + { + [$store, $producer] = $this->stream(); + + $producer->produce('a'); + $last = $producer->produce('b'); + + $this->assertSame(['a', 'b'], $this->drain($this->consumer(store: $store))); + $this->assertSame($last, $this->cursor->load($this->name, 'invalidator')); + + $producer->produce('c'); + + // A fresh Consumer over the same cursor store: a restart. + $this->assertSame(['c'], $this->drain($this->consumer(store: $store)), 'Resumes without replaying'); + } + + /** + * `XADD` assigns ids within one millisecond by bumping the sequence, so a + * batch boundary regularly falls between two ids sharing a timestamp β€” + * which is the case paging by string comparison would get wrong. + */ + public function testPagesAStreamInBatchesWithoutSkippingOrRepeating(): void + { + [$store, $producer] = $this->stream(); + + foreach (\range(1, 10) as $i) { + $producer->produce('event-' . $i); + } + + $consumer = $this->consumer(batch: 3, store: $store); + + $seen = []; + foreach (\range(1, 4) as $ignored) { + $seen = [...$seen, ...$this->drain($consumer)]; + } + + $this->assertSame(\array_map(static fn (int $i): string => 'event-' . $i, \range(1, 10)), $seen); + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null), 'And then it is caught up'); + } + + public function testASecondConsumerOfTheSameStreamGetsItsOwnPosition(): void + { + [$store, $producer] = $this->stream(); + + $producer->produce('a'); + + $this->assertSame(['a'], $this->drain($this->consumer('one', store: $store))); + $this->assertSame(['a'], $this->drain($this->consumer('two', store: $store)), 'The second reads it too'); + $this->assertSame([], $this->drain($this->consumer('one', store: $store)), 'The first stays caught up'); + } + + public function testResetReplaysWhatTheStreamStillRetains(): void + { + [$store, $producer] = $this->stream(); + + $producer->produce('a'); + $producer->produce('b'); + + $consumer = $this->consumer(store: $store); + $this->drain($consumer); + + $consumer->reset(); + + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + $this->assertSame(['a', 'b'], $this->drain($consumer)); + } + /** * The stored form is deliberately plain β€” the key is * `feed::cursor:` and the value the event id as a string β€” From cfa9496e0044ef8844b804dab24712fd0c6b9601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:49:06 +0200 Subject: [PATCH 61/68] Restore consumer coverage over a cache-fed feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Store\Cache` was exercised by the producer suite (store-level reads) and the server suite, but never by a `Consumer`: `Consumer/CacheTest` swapped only the cursor, so the feed it consumed was still a `MemoryStore`. The integration the adapter exists for β€” a service that already carries a cache keeping both the feed and the position there β€” was the one thing not covered. Added to `Consumer/CacheTest` rather than by deriving `Consumer/Base`'s store from the trait, matching the Redis change and keeping the shared scenarios pointed at the simplest store there is. The drain-and-restart case is the one lost with the old `StoreCacheTest`. The other two are for how this adapter reads differently from the rest: a caught-up poll tick answers from the tip marker instead of loading the feed, so a test has to prove "caught up" still means caught up and not "nothing more, ever" β€” and that paging a backlog in batches works over an adapter that scans the whole feed for each page. Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Consumer/CacheTest.php | 77 +++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/Feed/Consumer/CacheTest.php b/tests/Feed/Consumer/CacheTest.php index 26fb38c..d37645d 100644 --- a/tests/Feed/Consumer/CacheTest.php +++ b/tests/Feed/Consumer/CacheTest.php @@ -4,9 +4,86 @@ namespace Utopia\Tests\Consumer; +use Utopia\CloudEvents\CloudEvent; +use Utopia\Feed\Appendable; +use Utopia\Feed\Producer; +use Utopia\Feed\Store; use Utopia\Tests\Support\UsesCache; class CacheTest extends Base { use UsesCache; + + /** + * A feed in the cache, and a producer over it. + * + * The shared scenarios run against a memory feed on purpose, and swap only + * the cursor β€” so `Store\Cache` was exercised by the producer and server + * suites but never by a `Consumer`, which is the integration the adapter + * exists for: a service that already carries a cache keeping both the feed + * and the position there. + * + * @return array{Store&Appendable, Producer} + */ + private function fed(): array + { + $store = $this->store($this->name); + + return [$store, new Producer($store, 'urn:test')]; + } + + public function testAConsumerDrainsACacheFedFeed(): void + { + [$store, $producer] = $this->fed(); + + $producer->produce('a'); + $last = $producer->produce('b'); + + $this->assertSame(['a', 'b'], $this->drain($this->consumer(store: $store), $count)); + $this->assertSame(2, $count); + $this->assertSame($last, $this->cursor->load($this->name, 'invalidator'), 'The position lives in the same cache as the feed'); + + $producer->produce('c'); + + // A fresh Consumer over the same cache: a restart. + $this->assertSame(['c'], $this->drain($this->consumer(store: $store)), 'Resumes without replaying'); + } + + /** + * The consumer's poll loop over this adapter, which reads the feed + * differently from the others: a caught-up tick answers from the tip + * marker rather than by loading the feed, so "caught up" has to keep + * meaning caught up and not "nothing more, ever". + */ + public function testACaughtUpConsumerStillSeesTheNextEvent(): void + { + [$store, $producer] = $this->fed(); + + $producer->produce('a'); + + $consumer = $this->consumer(store: $store); + + $this->assertSame(['a'], $this->drain($consumer)); + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null), 'Caught up'); + + $producer->produce('b'); + + $this->assertSame(['b'], $this->drain($consumer), 'And no longer'); + } + + public function testAConsumerPagesACacheFedBacklogInBatches(): void + { + [$store, $producer] = $this->fed(); + + foreach (\range(1, 10) as $i) { + $producer->produce('event-' . $i); + } + + $consumer = $this->consumer(batch: 4, store: $store); + + $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(4, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(2, $consumer->consume(fn (CloudEvent $event) => null)); + $this->assertSame(0, $consumer->consume(fn (CloudEvent $event) => null)); + } } From 0988afa057bbc1b1064567d56668a91f6887a8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:50:34 +0200 Subject: [PATCH 62/68] Assert full event fidelity across the HTTP wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing checked `subject`, `source`, `time`, `dataschema` or extensions after an HTTP round trip. `Consumer/Base::drain()` records only `type`, `RemoteTest::testReadsAFeedOverHttp` asserted `id` and `data`, and `BatchTest::testToArrayIsTheWireEncoding` asserted `id` and `specversion`. The producer and server suites do round-trip those attributes, but through the *store* β€” so an encoder or decoder that dropped one on the wire specifically would pass the whole suite, and the wire is the one place a dropped attribute cannot be recovered from. `Consumer/HttpTest` now pushes one fully populated event through the real code in both directions β€” `Batch` encoding it in the endpoint, `Remote` decoding it on the way back β€” and asserts every attribute on the far side, including a nested payload and all three extension value types. That restores what the old `RoundTripTest` covered, and more. `BatchTest` gains the encode side in isolation: one event asserted as a whole array rather than attribute by attribute, so an attribute that stopped being encoded fails rather than going unnoticed. Plus the property that makes the wire format readable β€” an absent optional attribute is omitted, not sent as null, since the spec has no null attribute values and a consumer has to tell "not set" from "set to nothing". Co-Authored-By: Claude Opus 5 (1M context) --- tests/Feed/Consumer/HttpTest.php | 44 +++++++++++++++++++++++++++ tests/Feed/Unit/BatchTest.php | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/tests/Feed/Consumer/HttpTest.php b/tests/Feed/Consumer/HttpTest.php index 158b786..4f1e8da 100644 --- a/tests/Feed/Consumer/HttpTest.php +++ b/tests/Feed/Consumer/HttpTest.php @@ -119,6 +119,50 @@ public function testTheAcceptSentIsTheContentTypeServed(): void $this->assertSame(Remote::MEDIA_TYPE, $request['contentType'], 'The producer answers with what the consumer asked for'); } + /** + * One event through the real wire code in both directions β€” `Batch` + * encoding it in the endpoint, `Remote` decoding it on the way back β€” + * with every attribute asserted on the far side. + * + * The shared consumer scenarios record only `type`, and the producer and + * server suites round-trip the rest through the *store*. So an encoder or + * decoder that dropped `subject`, `source`, `dataschema` or an extension + * on the wire specifically would pass everything else in the suite. + */ + public function testAnEventSurvivesTheWireWithEveryAttribute(): void + { + $id = $this->producer->publish(new CloudEvent( + id: '', + type: 'io.appwrite.edge.invalidate-rule', + source: 'ignored, the producer stamps its own', + subject: 'example.com', + time: '2026-07-31T09:15:02.123Z', + datacontenttype: 'application/json', + data: ['tags' => ['domain' => 'example.com'], 'depth' => [1, 2, 3]], + dataschema: 'https://example.com/schema.json', + extensions: ['traceparent' => '00-abc-def-01', 'retrycount' => 2, 'replayed' => true], + )); + + $received = null; + $this->consumer()->consume(function (CloudEvent $event) use (&$received): void { + $received = $event; + }); + + $this->assertInstanceOf(CloudEvent::class, $received); + $this->assertSame($id, $received->id); + $this->assertSame('1.0', $received->specversion); + $this->assertSame('io.appwrite.edge.invalidate-rule', $received->type); + $this->assertSame('urn:test', $received->source); + $this->assertSame('example.com', $received->subject); + $this->assertSame('2026-07-31T09:15:02.123Z', $received->time); + $this->assertSame('application/json', $received->datacontenttype); + $this->assertSame('https://example.com/schema.json', $received->dataschema); + $this->assertSame(['tags' => ['domain' => 'example.com'], 'depth' => [1, 2, 3]], $received->data); + $this->assertSame('00-abc-def-01', $received->extensions['traceparent']); + $this->assertSame(2, $received->extensions['retrycount']); + $this->assertTrue($received->extensions['replayed']); + } + public function testTheProducerCachesFullBatchesAndNothingElse(): void { foreach (\range(1, 5) as $i) { diff --git a/tests/Feed/Unit/BatchTest.php b/tests/Feed/Unit/BatchTest.php index 7103356..ebbd71d 100644 --- a/tests/Feed/Unit/BatchTest.php +++ b/tests/Feed/Unit/BatchTest.php @@ -82,4 +82,56 @@ public function testToArrayIsTheWireEncoding(): void $this->assertSame('1.0', $payload[0]['specversion']); $this->assertSame([], (new Batch([], 100))->toArray()); } + + /** + * Everything an event carries has to reach the wire, not only what a + * consumer of this suite happens to look at. Asserting `id` and + * `specversion` alone would pass an encoder that dropped `subject` or an + * extension β€” and on the wire is the one place a dropped attribute cannot + * be recovered from. + */ + public function testEveryAttributeReachesTheWire(): void + { + $batch = new Batch([new CloudEvent( + id: '1-0', + type: 'io.appwrite.edge.invalidate-rule', + source: 'urn:appwrite:cloud:fra', + subject: 'example.com', + time: '2026-07-31T09:15:02.123Z', + datacontenttype: 'application/json', + data: ['tags' => ['domain' => 'example.com']], + dataschema: 'https://example.com/schema.json', + extensions: ['traceparent' => '00-abc-def-01'], + )], 100); + + $this->assertSame([ + 'specversion' => '1.0', + 'type' => 'io.appwrite.edge.invalidate-rule', + 'source' => 'urn:appwrite:cloud:fra', + 'id' => '1-0', + 'subject' => 'example.com', + 'time' => '2026-07-31T09:15:02.123Z', + 'datacontenttype' => 'application/json', + 'dataschema' => 'https://example.com/schema.json', + 'data' => ['tags' => ['domain' => 'example.com']], + 'traceparent' => '00-abc-def-01', + ], $batch->toArray()[0]); + } + + /** + * The spec has no null attribute values, so an absent optional attribute + * is omitted rather than sent as null β€” a consumer must be able to tell + * "not set" from "set to nothing". + */ + public function testAnAbsentOptionalAttributeIsOmittedRatherThanNulled(): void + { + $encoded = (new Batch([new CloudEvent( + id: '1-0', + type: 'a', + source: 'urn:test', + datacontenttype: null, + )], 100))->toArray()[0]; + + $this->assertSame(['specversion', 'type', 'source', 'id'], \array_keys($encoded)); + } } From 0c80b03c0703fb2b9b1ec312c50456ce3ebb14a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 12:52:34 +0200 Subject: [PATCH 63/68] Reinstate a cursor contract suite per adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reworked `CursorTest` kept only the `Cursor\None` cases. Most of what the old parameterized suite covered is now exercised indirectly β€” `Consumer/Base` runs against every cursor adapter and asserts on `load()` throughout β€” but two things are not reachable from there at all. `Consumer::__construct()` validates its own name before the cursor is ever touched, so the adapters' `Invalid` path had no test outside `None`. And nothing checked that each adapter routes `load`, `save` and `reset` through the shared `Cursor::key()` rather than building a key of its own β€” an adapter that did would skip the validation and drift from the documented `feed::cursor:` layout at the same time, and only Redis has a layout test. `tests/Feed/Cursor/Base.php` follows the same shape as the other suites, subclassed for Memory, Cache, Redis and Pool, each joining its adapter's existing CI job. It covers the direct API the consumer scenarios reach only sideways: unknown consumer, save/load, overwrite, isolation by consumer and by feed, reset, reset of a position that was never saved, and reset touching only the consumer it names. The key assertion is the last one β€” all three operations against all three unusable name combinations, nine cases per adapter. Confirmed load-bearing by making `Cursor\Memory` build its own key: all nine fail. Co-Authored-By: Claude Opus 5 (1M context) --- phpunit.xml | 4 + tests/Feed/Cursor/Base.php | 166 +++++++++++++++++++++++++++++++ tests/Feed/Cursor/CacheTest.php | 12 +++ tests/Feed/Cursor/MemoryTest.php | 12 +++ tests/Feed/Cursor/PoolTest.php | 12 +++ tests/Feed/Cursor/RedisTest.php | 12 +++ 6 files changed, 218 insertions(+) create mode 100644 tests/Feed/Cursor/Base.php create mode 100644 tests/Feed/Cursor/CacheTest.php create mode 100644 tests/Feed/Cursor/MemoryTest.php create mode 100644 tests/Feed/Cursor/PoolTest.php create mode 100644 tests/Feed/Cursor/RedisTest.php diff --git a/phpunit.xml b/phpunit.xml index 7c15a33..9cddfd9 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -33,11 +33,13 @@ ./tests/Feed/Producer/MemoryTest.php ./tests/Feed/Server/MemoryTest.php ./tests/Feed/Consumer/MemoryTest.php + ./tests/Feed/Cursor/MemoryTest.php ./tests/Feed/Producer/CacheTest.php ./tests/Feed/Server/CacheTest.php ./tests/Feed/Consumer/CacheTest.php + ./tests/Feed/Cursor/CacheTest.php ./tests/Feed/Consumer/HttpTest.php @@ -46,11 +48,13 @@ ./tests/Feed/Producer/RedisTest.php ./tests/Feed/Server/RedisTest.php ./tests/Feed/Consumer/RedisTest.php + ./tests/Feed/Cursor/RedisTest.php ./tests/Feed/Producer/PoolTest.php ./tests/Feed/Server/PoolTest.php ./tests/Feed/Consumer/PoolTest.php + ./tests/Feed/Cursor/PoolTest.php diff --git a/tests/Feed/Cursor/Base.php b/tests/Feed/Cursor/Base.php new file mode 100644 index 0000000..e53b5a0 --- /dev/null +++ b/tests/Feed/Cursor/Base.php @@ -0,0 +1,166 @@ +name = 'test-' . \bin2hex(\random_bytes(8)); + $this->cursor = $this->cursor(); + } + + public function testAConsumerThatHasNeverRunHasNoPosition(): void + { + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + } + + public function testAPositionComesBackAsItWasSaved(): void + { + $this->cursor->save($this->name, 'invalidator', '1690000000000-7'); + + $this->assertSame('1690000000000-7', $this->cursor->load($this->name, 'invalidator')); + } + + /** + * Saving is how a consumer advances, so it happens on every run β€” the last + * one has to win rather than the first being kept. + */ + public function testSavingAgainMovesThePosition(): void + { + $this->cursor->save($this->name, 'invalidator', '1-0'); + $this->cursor->save($this->name, 'invalidator', '2-0'); + + $this->assertSame('2-0', $this->cursor->load($this->name, 'invalidator')); + } + + public function testConsumersOfOneFeedDoNotShareAPosition(): void + { + $this->cursor->save($this->name, 'one', '1-0'); + $this->cursor->save($this->name, 'two', '2-0'); + + $this->assertSame('1-0', $this->cursor->load($this->name, 'one')); + $this->assertSame('2-0', $this->cursor->load($this->name, 'two')); + } + + /** + * One cursor store serves every feed a service consumes, so the feed name + * is part of the key β€” the same consumer name on another feed is another + * position entirely. + */ + public function testTheSameConsumerNameOnAnotherFeedIsAnotherPosition(): void + { + $this->cursor->save($this->name, 'invalidator', '1-0'); + $this->cursor->save($this->name . '-other', 'invalidator', '2-0'); + + $this->assertSame('1-0', $this->cursor->load($this->name, 'invalidator')); + $this->assertSame('2-0', $this->cursor->load($this->name . '-other', 'invalidator')); + } + + public function testResetForgetsThePosition(): void + { + $this->cursor->save($this->name, 'invalidator', '1-0'); + $this->cursor->reset($this->name, 'invalidator'); + + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + } + + /** Resetting a consumer that never saved anything is the ordinary case. */ + public function testResettingAPositionThatWasNeverSavedIsHarmless(): void + { + $this->cursor->reset($this->name, 'invalidator'); + + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + } + + public function testResetOnlyForgetsTheConsumerItNames(): void + { + $this->cursor->save($this->name, 'one', '1-0'); + $this->cursor->save($this->name, 'two', '2-0'); + + $this->cursor->reset($this->name, 'one'); + + $this->assertNull($this->cursor->load($this->name, 'one')); + $this->assertSame('2-0', $this->cursor->load($this->name, 'two'), 'The other consumer is untouched'); + } + + /** + * A name that cannot be keyed is refused by every operation, not only by + * whichever one a caller happens to reach first. This is the assertion + * that says the adapter goes through the shared key builder at all β€” + * `Consumer` validates its own name long before the cursor sees it, so + * nothing else here can. + * + * @param callable(Cursor, string, string): void $operation + */ + #[DataProvider('operationsAndNames')] + public function testEveryOperationRefusesAnUnusableName(callable $operation, string $feed, string $consumer): void + { + $this->expectException(Invalid::class); + + $operation($this->cursor, $feed, $consumer); + } + + /** + * @return array + */ + public static function operationsAndNames(): array + { + $operations = [ + 'load' => static function (Cursor $cursor, string $feed, string $consumer): void { + $cursor->load($feed, $consumer); + }, + 'save' => static function (Cursor $cursor, string $feed, string $consumer): void { + $cursor->save($feed, $consumer, '1-0'); + }, + 'reset' => static function (Cursor $cursor, string $feed, string $consumer): void { + $cursor->reset($feed, $consumer); + }, + ]; + + $names = [ + 'no feed' => ['', 'invalidator'], + 'no consumer' => ['edge', ''], + 'neither' => ['', ''], + ]; + + $cases = []; + + foreach ($operations as $operation => $callable) { + foreach ($names as $name => [$feed, $consumer]) { + $cases["{$operation} with {$name}"] = [$callable, $feed, $consumer]; + } + } + + return $cases; + } +} diff --git a/tests/Feed/Cursor/CacheTest.php b/tests/Feed/Cursor/CacheTest.php new file mode 100644 index 0000000..bb1df17 --- /dev/null +++ b/tests/Feed/Cursor/CacheTest.php @@ -0,0 +1,12 @@ + Date: Mon, 3 Aug 2026 12:54:24 +0200 Subject: [PATCH 64/68] Rewrite the changelog as one initial release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file had two `## 0.1.0` sections, the top one documenting breaking changes over the bottom one, which was labelled "Initial release". One version cannot both be the initial release and break it. The top section also documented APIs that do not exist. `Start::Tip` is `Consumer::START_TIP`, a string constant rather than an enum. `Protocol::MAX_BATCH`/`MAX_TIMEOUT` live on `Readable`. `Protocol::encode()`, `Protocol::decode()` and `Protocol::MEDIA_TYPE` have no class at all β€” those responsibilities are `Batch`'s and `Remote`'s β€” so the note about a route never needing to name `Protocol` named something a route could not have named anyway. And since this is the first release, migrations between versions nobody has ever run are not changelog material. Journal β†’ Store and Protocol β†’ Readable/Batch/Remote are design history, which is what git history is for; the reasoning that survived is already in the README. Collapsed into a single `## 0.1.0 β€” Initial release` describing what actually ships, using the real names, and written after the rest of this branch landed so it documents the API as released rather than as planned β€” including the seek relaxation, the media type on `Readable`, the cache store's tip marker and its smaller retention default. Every class and constant named here was checked against `src/`, as was every number: 1000, 30s, 500ms, 1 000 and 100 000 entries, 30 days. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 185 ++++++++++++++++++++++++++------------------------- 1 file changed, 96 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 437c624..f5eba8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,97 +1,104 @@ # Changelog -## 0.1.0 +## 0.1.0 β€” Initial release -- **Breaking (the three classes):** the library is now three main classes. - `Producer` writes events to a feed with `produce()` (formerly `append()`; - `publish()` still takes a prepared `CloudEvent`). `Server` (formerly - `Feed`) is the HTTP feed endpoint, with `serve()` as its main method. - `Consumer` reads a feed as a client with `consume()`. -- **Breaking:** `Journal` is now `Store` β€” `Store\Redis`, `Store\Pool`, - `Store\Memory`, `Store\None`, in the `Utopia\Feed\Store` namespace. The - `Producer` and `Server` are built over a store; `Appendable` and `Readable` - are unchanged as its contracts. -- **Breaking:** `Consumer` no longer takes an endpoint. It is built straight - over a [utopia-php/client](https://github.com/utopia-php/client) whose - base URI points at the feed endpoint (`withBaseUri()`), plus the feed's - name: `new Consumer($client, $cursor, name: 'invalidator', feed: 'edge')`. - A local store still drops in for the client, for consuming a feed the same - service produces; the cursor moved forward to the second parameter. - `Remote` accordingly lost its `endpoint` parameter β€” the feed name is sent - as a relative path and the client resolves it. -- Added `Store\Cache` β€” a feed on a [utopia-php/cache](https://github.com/utopia-php/cache) - `Cache`, for a producer whose service already carries a cache and does not - want another backend. The whole feed lives under one key (last-writer-wins - appends β€” run one producing process), trims to `maxSize`, and expires `ttl` - seconds after the last append (default 30 days). Both `Store` and `Cursor` - now have `Redis` and `Cache` adapters. -- `Server::read()` and `Server::poll()` return a `Batch` instead of a plain - event array. A batch counts and iterates as its events and carries the - limit it was actually built with, so `Batch::cacheControl()` can never be - fed a number the read did not use. `Batch::toArray()` is the wire encoding, - `Batch::lastId()` the position a stateless relay tracks. -- Added `Server::serve(array $query): Batch` β€” the whole HTTP request in one - call: extracts `lastEventId`, `limit` and `timeout` from the route's raw - query parameters, coerces and clamps them, and rejects a malformed - `lastEventId` with `Exception\Invalid`. A route never needs to name - `Protocol`, which is now documented as internal plumbing. -- Added `Start::Tip` β€” a consumer with no stored position can opt into - starting at the tip of the feed (only what happens from now on) instead of - draining the backlog. Rides a protocol extension: the `lastEventId` value - `$`, resolved by the producer to the newest event as the request arrives. - Also added `Server::tip()`, the id of the newest event in a local store. -- Added `Consumer::seek(string $eventId)` β€” set the position explicitly: the - id is treated as the last event handled, persisted immediately, and the - next `consume()` starts strictly after it. The operational escape hatch for - a poison event: seek to the failing event's own id to step past it - deliberately. -- The long-poll read interval is now a constructor option on the stores: - `pollInterval`, in milliseconds, default 500. An interval below 1 ms throws - `Exception\Invalid`. The poll loop also no longer oversleeps: it sleeps the - remaining time when less than an interval is left, so a timeout is honoured - to within scheduler precision instead of running up to one interval late. -- **Breaking (renames):** every user-facing name now belongs to exactly one - side of the wire. `Journal\Http` is gone; its replacement is - `Utopia\Feed\Remote` β€” another service's feed, over HTTP β€” a standalone - class implementing the new `Readable` interface (`read`, `poll`, `tip`, - `getName`) rather than posing as a store. `Consumer` clamps its own - `batch`/`timeout`. The protocol limits moved with the responsibility: - `Feed::MAX_BATCH`/`Feed::MAX_TIMEOUT` are now `Protocol::MAX_BATCH` and - `Protocol::MAX_TIMEOUT`. -- **Breaking (wire format):** a feed batch on the wire is now the plain JSON - array of CloudEvents that [http-feeds.org](https://www.http-feeds.org/) - defines β€” the `{total, events}` envelope is gone, and an empty feed - serializes to `[]`. `Protocol::encode()` returns the bare array, - `Protocol::decode()` expects one, and `Remote` asks for the spec's - `application/cloudevents-batch+json` media type (`Protocol::MEDIA_TYPE`). - Both sides of a feed must move together. +Pull-based HTTP event feeds ([http-feeds.org](https://www.http-feeds.org/)) for +PHP. A producer writes events to an ordered log; each consumer polls for what +happened after the last event it handled, and keeps that position itself. -## 0.1.0 +Requires PHP 8.5. `ext-redis` is a suggestion, needed only by the Redis and +pooled adapters. -Initial release. +### Producing -- `Producer` β€” appends events to a feed this service owns -- `Feed` β€” reads and long-polls a feed, local or remote +- `Producer` β€” `produce(type, data, subject)` for the common case, and + `publish(CloudEvent)` for a prepared event. Both return the event's id, which + is also its position in the feed. The producer stamps its own `source`, the + store assigns the `id`, and a missing `time` becomes now; everything else is + published as prepared. - Events are [utopia-php/cloudevents](https://github.com/utopia-php/cloudevents) - `CloudEvent` objects β€” this library defines no event type of its own, so a feed - event is accepted anywhere a `CloudEvent` is, and `dataschema` and extension - attributes survive an append and a read -- `Appendable` β€” the journals that own their events and can be appended to; - `Journal\Http` deliberately does not implement it -- `Journal\Redis`, `Journal\Pool` β€” Redis streams, directly or over a pool -- `Journal\Http` β€” another service's feed, read over the wire with - [utopia-php/client](https://github.com/utopia-php/client); takes any of its - adapters, so a pooled or Swoole coroutine transport drops straight in -- `Journal\Memory`, `Journal\None` β€” for tests, and for no backend configured -- `Consumer` β€” the pull loop, with at-least-once semantics and a durable position -- `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory`, `Cursor\None` β€” where that - position lives, keyed by feed and consumer name -- `Protocol` β€” the http-feeds wire contract, shared by producer and consumer, and - the one place the feed's decode policy lives: strict about `id` because it is - the consumer's position, tolerant of everything else so a consumer older than - the producer keeps working -- `Id` β€” feed positions, and the arithmetic for paging past one + `CloudEvent` objects. This library defines no event type of its own, so a feed + event is accepted anywhere a `CloudEvent` is, and `datacontenttype`, + `dataschema` and extension attributes survive an append and a read. -Requires PHP 8.5. CI builds one parameterized image per version in the -`php-versions` matrix of `.github/workflows/tests.yml`, which is the only place -versions are listed. +### Serving + +- `Server` β€” the feed endpoint. `serve(array $query)` is the whole HTTP request + in one call: it reads `lastEventId`, `limit` and `timeout` from the raw query + parameters, coerces and clamps them, and rejects a `lastEventId` that is + neither a position nor the tip sentinel with `Exception\Invalid`. `read()`, + `poll()` and `tip()` are there for a route that wants the pieces. +- `Batch` β€” one read of a feed. Counts and iterates as its events, and carries + the limit it was actually built with, so `cacheControl()` cannot be handed a + number the read did not use: a full batch is settled history and cacheable + forever, anything shorter is `no-store`. `toArray()` is the wire encoding and + `lastId()` the position a stateless relay tracks. + +### Consuming + +- `Consumer` β€” the pull loop, at-least-once, with a durable position. + `consume(callable)` handles a batch and commits after the last event that + succeeded; a failing handler blocks the events behind it by design. +- Built over a local `Readable` for a feed the same service produces, or + straight over a [utopia-php/client](https://github.com/utopia-php/client) + whose base URI points at the feed endpoint, plus the feed's name. +- `position()`, `reset()` and `seek(eventId)` move the position by hand. + `seek()` is the escape hatch for a poison event: seek to the failing event's + own id to step past it. Both are safe to call from inside a handler. +- `Consumer::START_TIP` starts a consumer with no stored position at the tip of + the feed instead of at the oldest retained event. It rides a protocol + extension β€” the `lastEventId` value `$`, which the producer resolves to its + newest event as the request arrives β€” so skipping the backlog costs no extra + round trip and still delivers what lands mid-poll. + +### Stores and cursors + +- `Store\Redis`, `Store\Pool` β€” a capped Redis stream, directly or over a + [pooled](https://github.com/utopia-php/pools) connection. The pooled store + borrows per read, so a held long poll never ties up a connection. +- `Store\Cache` β€” the feed on a [Utopia cache](https://github.com/utopia-php/cache), + for a service that already carries one. The whole feed lives under one key, + so an append rewrites it (last-writer-wins β€” run one producing process) and + retention is also the cost of producing an event, which is why its default + `maxSize` is 1 000 rather than the 100 000 the Redis store keeps. The newest + id is kept under a second, tiny key so a caught-up long poll does not load + the feed to learn nothing. +- `Store\Memory` β€” tests and single-process development. `Store\None` β€” no + backend configured; throws on use, so a misconfigured service fails loudly + instead of dropping events. +- `Cursor\Cache`, `Cursor\Redis`, `Cursor\Pool`, `Cursor\Memory`, `Cursor\None` + β€” where a consumer's position lives, keyed by feed and consumer name. The + stored form is deliberately plain (`feed::cursor:` holding + the id as a string), so positions carry across upgrades and an operator can + answer "where is this consumer?" from a shell. +- Retention (`maxSize`) and the long-poll read interval (`pollInterval`, in + milliseconds, default 500) are constructor options on every store. + +### The wire + +- A batch is the plain JSON array of CloudEvents the spec defines β€” no + envelope. An empty array means the consumer is caught up. The media type is + `Readable::MEDIA_TYPE`, `application/cloudevents-batch+json`, aliased as + `Batch::MEDIA_TYPE` for the serving side and `Remote::MEDIA_TYPE` for the + consuming one. +- `Readable` is the contract both sides share β€” `read`, `poll`, `tip`, + `getName`, plus `TIP`, `MAX_BATCH` (1000) and `MAX_TIMEOUT` (30s). + `Appendable` is the contract of a store that owns its events; + `Remote` deliberately does not implement it. +- `Remote` β€” another service's feed over HTTP, a `Readable` in its own right + rather than a store. Decoding is strict about `id`, because that is the + consumer's position, and tolerant of everything else, so a consumer older + than the producer keeps working: unknown attributes ride along as + extensions, and an entry that cannot be read ends the batch early rather + than discarding the usable events before it β€” unless it is the first, where + there is no progress to keep and the read fails loudly instead. +- `Id` β€” feed positions (`{ms}-{seq}`) and the arithmetic for paging past one. + `Key` shapes the backend keys, escaping names so a feed and a cursor cannot + collide in one keyspace. + +### Errors + +Everything this library raises extends `Utopia\Feed\Exception`: +`Exception\Invalid` for something the caller handed over, `Exception\Transport` +for a backend or network failure, `Exception\Unsupported` for something a +backend cannot do. From df19717ef316800383c671f796a4ffa9d173f279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 13:59:25 +0200 Subject: [PATCH 65/68] Cut the comments down across the branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commits from 58cf7ac to 20f0d7c explained themselves at length: several comments argued the case for a decision over three or four sentences where one would do, and a few restated what the line below already said. Every comment added across that range is now either one line, a short paragraph, or gone. What survives is the part a reader cannot get from the code β€” why `purge()`'s false is not checked, why the tip marker is written before the feed, why the key builder escapes rather than rejects, why a timing bound is the number it is. The rationale that only mattered while the change was being made lives in the commit messages, which is where it belongs. Net βˆ’322 lines, no behaviour change: all 669 tests, PHPStan and Pint stay green. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/linter.yml | 6 ++-- .github/workflows/tests.yml | 6 ++-- Dockerfile | 6 ++-- phpunit.xml | 5 +--- src/Feed/Batch.php | 6 +--- src/Feed/Consumer.php | 16 +++------- src/Feed/Cursor.php | 4 +-- src/Feed/Cursor/Cache.php | 18 +++--------- src/Feed/Extensions.php | 31 +++++--------------- src/Feed/Key.php | 28 ++++-------------- src/Feed/Producer.php | 20 ++++--------- src/Feed/Readable.php | 6 +--- src/Feed/Remote.php | 4 +-- src/Feed/Server.php | 15 ++++------ src/Feed/Store.php | 19 +++++------- src/Feed/Store/Cache.php | 36 +++++------------------ tests/Feed/Consumer/Base.php | 9 ++---- tests/Feed/Consumer/CacheTest.php | 17 +++-------- tests/Feed/Consumer/HttpTest.php | 39 +++++++------------------ tests/Feed/Consumer/PoolTest.php | 6 ++-- tests/Feed/Consumer/RedisTest.php | 28 +++++------------- tests/Feed/Cursor/Base.php | 16 +++------- tests/Feed/Producer/Base.php | 31 +++++--------------- tests/Feed/Producer/CacheTest.php | 34 +++++++--------------- tests/Feed/Producer/RedisTest.php | 41 +++++++++----------------- tests/Feed/Server/Base.php | 35 ++++++++-------------- tests/Feed/Server/MemoryTest.php | 4 +-- tests/Feed/Server/PoolTest.php | 18 +++--------- tests/Feed/Support/BrokenCache.php | 17 ++++------- tests/Feed/Support/CountingCache.php | 5 +--- tests/Feed/Support/CountingStack.php | 8 ++--- tests/Feed/Support/FeedServer.php | 17 ++++------- tests/Feed/Support/RecordingStore.php | 7 ++--- tests/Feed/Support/UsesPool.php | 5 ++-- tests/Feed/Support/UsesRedis.php | 12 ++------ tests/Feed/Unit/BatchTest.php | 13 ++------- tests/Feed/Unit/CursorTest.php | 31 ++++++-------------- tests/Feed/Unit/KeyTest.php | 28 ++++-------------- tests/Feed/Unit/RemoteTest.php | 42 +++++++-------------------- tests/Feed/Unit/ServerTest.php | 15 ++++------ 40 files changed, 191 insertions(+), 513 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 82454df..e98d6e4 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -14,10 +14,8 @@ jobs: - name: Checkout repository uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - # Run against the test image rather than a bare composer container, as - # the analysis job does: the codebase uses PHP 8.5 syntax that pint can - # only parse on a new enough runtime, and whatever version `composer:latest` - # happens to ship would otherwise be an unstated build dependency. + # The project's own image, as the analysis job uses: pint needs a PHP + # new enough to parse 8.5 syntax, which `composer:latest` does not pin. - name: Build run: | docker compose build tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f4afe4..9951936 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,9 +48,7 @@ jobs: docker compose build docker compose up -d - # Every suite, so the number covers the Redis and pooled adapters too β€” - # the ones whose error paths are easiest to leave untested. Printed for - # a human to read; there is deliberately no threshold yet, since a bad - # one is worse than none. + # Every suite, so the Redis and pooled adapters count too. No threshold + # yet β€” the number is there to be read. - name: Report coverage run: docker compose exec -T tests composer coverage:all diff --git a/Dockerfile b/Dockerfile index 0fed373..c394fe9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,10 +15,8 @@ LABEL maintainer="team@appwrite.io" WORKDIR /code -# pcov rather than Xdebug: it only does line coverage, which is all a report -# needs, and costs a fraction of the run time. Loaded but switched off, so -# every ordinary run is unaffected β€” `composer coverage` turns it on for the -# one run that wants it. +# pcov rather than Xdebug: line coverage only, at a fraction of the cost. +# Loaded but switched off, so only `composer coverage` pays for it. RUN apk add --no-cache --virtual .pcov-build-deps $PHPIZE_DEPS \ && pecl install pcov \ && docker-php-ext-enable pcov \ diff --git a/phpunit.xml b/phpunit.xml index 9cddfd9..d60e668 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -15,10 +15,7 @@ means that adapter honours the whole contract. `unit` needs no services; `redis` and `pool` need the compose Redis. --> - + ./src diff --git a/src/Feed/Batch.php b/src/Feed/Batch.php index 8337ae8..a632129 100644 --- a/src/Feed/Batch.php +++ b/src/Feed/Batch.php @@ -12,11 +12,7 @@ */ final class Batch implements \Countable, \IteratorAggregate { - /** - * What a feed response's `Content-Type` carries β€” the serving side's name - * for {@see Readable::MEDIA_TYPE}, which is where the value lives so the - * two sides of the wire cannot drift apart. - */ + /** What a feed response's `Content-Type` carries. */ public const string MEDIA_TYPE = Readable::MEDIA_TYPE; private const string CACHE_IMMUTABLE = 'max-age=31536000'; diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index dbde968..b5b8215 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -119,18 +119,10 @@ public function position(): ?string /** * Move the position by hand. * - * What counts as a usable id is the feed's to say, not this library's. - * A local {@see Store} mints `{ms}-{seq}` positions and pages by decoding - * them, so anything else is a caller's mistake and is rejected here rather - * than on the next read. A feed reached over HTTP is another producer's: - * http-feeds endpoints commonly use UUIDs, `Remote` accepts any non-empty - * id, and `consume()` already tracks and sends one back β€” so refusing one - * here would take the documented poison-event escape hatch away from - * exactly the consumers that cannot work around it. - * - * The tip sentinel is refused either way: it stands for "wherever the feed - * ends when the request arrives", which is a start, not a position. That - * is what `reset()` on a {@see self::START_TIP} consumer expresses. + * What counts as a usable id is the feed's to say: a local {@see Store} + * mints `{ms}-{seq}` positions and pages by decoding them, while a remote + * producer is the authority on its own (http-feeds endpoints commonly use + * UUIDs). Both refuse the tip sentinel, which is a start, not a position. * * @throws Exception\Invalid When $eventId cannot be a position on this feed * @throws Exception When the cursor store cannot be written diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 444e616..61a0e0d 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -24,8 +24,8 @@ abstract public function save(string $feed, string $consumer, string $eventId): abstract public function reset(string $feed, string $consumer): void; /** - * The gate every cursor operation goes through, so no adapter can build a - * key of its own and skip the check or diverge from the layout. + * The gate every cursor operation goes through, so no adapter builds a key + * of its own and skips the check. * * @throws Invalid When either name is empty. */ diff --git a/src/Feed/Cursor/Cache.php b/src/Feed/Cursor/Cache.php index cecdc5b..b49dd0a 100644 --- a/src/Feed/Cursor/Cache.php +++ b/src/Feed/Cursor/Cache.php @@ -20,18 +20,13 @@ public function __construct( public function load(string $feed, string $consumer): ?string { - // The key is shaped outside the try: an unusable name is the caller's - // bug (Invalid), not the backend's failure (Transport). + // Keyed outside the try: an unusable name is Invalid, not Transport. $key = $this->key($feed, $consumer); try { /** @var mixed $cursor */ $cursor = $this->cache->load($key, $this->ttl); } catch (\Throwable $error) { - // A cache adapter over a backend that is down raises whatever that - // backend raises β€” a raw \RedisException, say. Every error this - // library reports extends Utopia\Feed\Exception, and a consumer - // retrying on Transport must not crash on a backend blip instead. throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); } @@ -48,10 +43,7 @@ public function save(string $feed, string $consumer, string $eventId): void throw new Transport("Failed to save the {$consumer} cursor: {$error->getMessage()}", previous: $error); } - // A cache adapter reports a failed write by returning false rather - // than raising β€” swallowing that would let a position silently not - // persist, so the consumer replays its backlog on the next restart - // and a seek past a poison event quietly does nothing. + // A cache reports a failed write by returning false rather than raising. if ($saved === false) { throw new Transport("Failed to save the {$consumer} cursor on the {$feed} feed"); } @@ -62,10 +54,8 @@ public function reset(string $feed, string $consumer): void $key = $this->key($feed, $consumer); try { - // Unlike save(), purge()'s false is not a failure signal: it is - // also what an adapter answers for a key that was never there, - // which is the ordinary case for a consumer that has not saved a - // position yet. Only a raising backend is a failure to report. + // purge()'s false is not checked: it also means "was never there", + // which is the ordinary case for a consumer with no position yet. $this->cache->purge($key); } catch (\Throwable $error) { throw new Transport("Failed to reset the {$consumer} cursor: {$error->getMessage()}", previous: $error); diff --git a/src/Feed/Extensions.php b/src/Feed/Extensions.php index fbab86f..dc7abf5 100644 --- a/src/Feed/Extensions.php +++ b/src/Feed/Extensions.php @@ -5,27 +5,16 @@ namespace Utopia\Feed; /** - * The one place this library decides what an extension attribute is. + * The one place this library decides what an extension attribute is, so the + * wire and the store agree. * - * A feed is read by consumers older than the producer by design, so an event - * carrying something unexpected has to stay readable. The spec is narrow about - * what an extension may be β€” a name of lowercase letters and digits, a value - * that is a boolean, an integer or a string β€” and anything outside that is - * dropped so one odd attribute cannot cost the whole event. - * - * Both decode paths go through here. They used to disagree: over HTTP a - * foreign `"ratio": 1.5` was filtered out and the event delivered, while the - * same event read from a local store was handed to `CloudEvent::fromArray()` - * whole, which rejected it β€” and since a store read decodes every entry in the - * batch, one such entry made every read past it fail permanently. Agreeing was - * the only version of that anybody would have chosen on purpose. + * A feed is read by consumers older than its producer by design, so anything + * the spec cannot carry is dropped rather than raised: one odd attribute must + * not cost the whole event. */ final class Extensions { - /** - * The context attributes this library models. Anything else in an event is - * a candidate extension. - */ + /** The context attributes this library models; anything else is a candidate extension. */ private const array ATTRIBUTES = [ 'specversion', 'type', @@ -39,10 +28,7 @@ final class Extensions ]; /** - * Keep only what a CloudEvent can carry as an extension. - * - * @param array $candidates A decoded event, or just its - * extension attributes β€” modelled attributes are dropped either way. + * @param array $candidates A decoded event, or just its extension attributes. * @return array */ public static function filter(array $candidates): array @@ -55,8 +41,7 @@ public static function filter(array $candidates): array continue; } - // A digit-only name is legal per the spec and arrives as an - // integer key in PHP, so the name is compared as a string. + // A digits-only name is legal, and an integer key in PHP. if (\preg_match('/^[a-z0-9]+$/', (string) $name) !== 1) { continue; } diff --git a/src/Feed/Key.php b/src/Feed/Key.php index a0ce4ed..9ec1054 100644 --- a/src/Feed/Key.php +++ b/src/Feed/Key.php @@ -7,19 +7,9 @@ /** * The one place a backend key is shaped, so every adapter agrees on it. * - * Keys are built by joining names with `:`, which is also a character a name - * may contain β€” so joining them raw lets distinct things share a key. A feed - * named `edge:cursor:x` would key to `feed:edge:cursor:x`, the same key as - * consumer `x`'s position on feed `edge`, and a `SET` would land on an - * `XADD` stream. Two consumers could likewise share one position: - * (`a:cursor:b`, `c`) and (`a`, `b:cursor:c`) both joined to - * `feed:a:cursor:b:cursor:c`. - * - * Escaping the separator out of the names makes the mapping injective, so - * those collisions cannot be expressed. It is deliberately not validation: - * `Remote` reads third-party feeds whose names are arbitrary path segments, - * and rejecting one here would make a feed unconsumable over a detail of how - * this library happens to store positions. + * Names may contain the `:` the keys join on, so they are escaped rather than + * rejected β€” `Remote` reads third-party feeds whose names are arbitrary path + * segments. */ final class Key { @@ -35,21 +25,13 @@ public static function cursor(string $feed, string $consumer): string return 'feed:' . self::escape($feed) . ':cursor:' . self::escape($consumer); } - /** - * The key a feed's newest id lives under, for backends that cannot answer - * "is there anything after this position?" without reading the feed. - */ + /** The key a feed's newest id lives under, for backends that cannot answer "anything new?" cheaply. */ public static function tip(string $name): string { return self::feed($name) . ':tip'; } - /** - * Percent-encode the separator, and the escape character itself so the - * encoding stays reversible. A name with neither is left exactly as it - * was, which is what keeps the layout the README documents readable from - * a shell for every name anyone would actually pick. - */ + /** The escape character goes first, so the encoding stays reversible. */ private static function escape(string $name): string { return \str_replace(['%', ':'], ['%25', '%3A'], $name); diff --git a/src/Feed/Producer.php b/src/Feed/Producer.php index d1aa25c..00d248b 100644 --- a/src/Feed/Producer.php +++ b/src/Feed/Producer.php @@ -43,21 +43,11 @@ public function produce(string $type, mixed $data = [], string $subject = ''): s /** * Publish a prepared event. * - * Three attributes are the producer's, not the caller's, and are replaced - * whatever the event arrived with: - * - * - `source` becomes this producer's. It records where the event happened, - * and a producer can only speak for itself β€” an event relayed from - * another feed is published as this service's event, not as the original - * producer's. Keep the origin in an extension attribute if it matters. - * - `id` is assigned by the store, since it is also the event's position - * in the feed and only the store can order it. - * - `time` is stamped as now when the event carries none. - * - * Everything else β€” `subject`, `datacontenttype`, `dataschema`, `data` - * and extensions β€” is published as prepared. `specversion` reads back as - * `1.0`: it is the only version the stored form can be decoded as, so a - * store that kept another one would hold an entry nothing could read. + * Three attributes are replaced whatever the event arrived with: `source` + * (a producer only speaks for itself, so a relayed event is republished as + * this service's), `id` (the store assigns it β€” it is also the position) + * and a missing `time`. `specversion` reads back as `1.0`; the rest is + * published as prepared. * * @throws Exception\Invalid When the event has no type or cannot be encoded. * @throws Exception When the backend rejects the event. diff --git a/src/Feed/Readable.php b/src/Feed/Readable.php index 44bdbb1..0022b5d 100644 --- a/src/Feed/Readable.php +++ b/src/Feed/Readable.php @@ -16,11 +16,7 @@ interface Readable */ public const string TIP = '$'; - /** - * The batch media type on the wire. One side's `Content-Type` is the - * other side's `Accept`, so it lives here with the rest of the contract - * the serving and consuming sides share rather than once on each. - */ + /** The batch media type on the wire β€” one side's `Content-Type`, the other's `Accept`. */ public const string MEDIA_TYPE = 'application/cloudevents-batch+json'; /** The most events one batch may carry β€” producers clamp `limit` to this. */ diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index b828ed9..6d813f2 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -16,9 +16,7 @@ class Remote implements Readable { - // MEDIA_TYPE β€” sent as `Accept` with every read β€” is inherited from - // Readable, so the header this asks for and the one Batch is served with - // are the same value rather than two copies of it. + // MEDIA_TYPE, sent as `Accept` with every read, is inherited from Readable. // The request parameters of https://www.http-feeds.org/. private const string PARAM_LAST_EVENT_ID = 'lastEventId'; diff --git a/src/Feed/Server.php b/src/Feed/Server.php index 1292b2f..c77158a 100644 --- a/src/Feed/Server.php +++ b/src/Feed/Server.php @@ -54,13 +54,10 @@ public function serve(array $query): Batch /** @var mixed $lastEventId */ $lastEventId = $query[self::PARAM_LAST_EVENT_ID] ?? null; - // Absent and empty both mean "from the oldest retained event". Anything - // else present has to be a position or the sentinel β€” including values - // that are not strings at all: PHP parses `?lastEventId[]=1-0` into an - // array, and reading that as absent would answer a malformed parameter - // with a full replay of the retained feed. That is the most expensive - // response there is, and a caught-up consumer would take it for a - // sudden flood of new events rather than for the 400 it should be. + // Absent and empty mean "from the oldest retained event"; anything else + // must be a position or the sentinel, non-strings included. PHP parses + // `?lastEventId[]=1-0` into an array, and reading that as absent would + // answer a malformed parameter with a full replay of the feed. if ($lastEventId === null || $lastEventId === '') { $lastEventId = null; } elseif (!\is_string($lastEventId)) { @@ -69,9 +66,7 @@ public function serve(array $query): Batch throw new Exception\Invalid('Invalid lastEventId: ' . $lastEventId); } - // `limit` and `timeout` stay forgiving: both are the producer's to - // decide, so a garbage value falls back to the default rather than - // stalling a feed over something that does not affect correctness. + // `limit` and `timeout` stay forgiving: neither can cause a wrong answer. $limit = $query[self::PARAM_LIMIT] ?? null; $timeout = $query[self::PARAM_TIMEOUT] ?? null; diff --git a/src/Feed/Store.php b/src/Feed/Store.php index f6ddff7..71c1417 100644 --- a/src/Feed/Store.php +++ b/src/Feed/Store.php @@ -74,14 +74,11 @@ public function poll(?string $lastEventId, int $limit, int $timeout): array } /** - * Every attribute a CloudEvent carries, flattened to strings. - * - * `specversion` is not among them: it is restored as `1.0` because that is - * the only version {@see CloudEvent::fromArray()} accepts, so storing - * another one would make the entry permanently unreadable. `id` is not - * either β€” the store assigns it and hands it back to {@see self::decode()}. - * An empty string stands for an absent optional attribute; the spec has no - * null attribute values, so it cannot collide with a real one. + * Every attribute a CloudEvent carries, flattened to strings, with the + * empty string for an absent one β€” the spec has no null attribute values, + * so it cannot collide. `id` and `specversion` are left out: the store + * assigns the first, and `1.0` is the only version {@see self::decode()} + * can restore. * * @return array */ @@ -120,10 +117,8 @@ protected static function decode(string $id, array $fields): CloudEvent } } - // Filtered rather than merged verbatim, and for the same reason the - // wire path filters: a read decodes every entry in the batch, so one - // entry a foreign writer left an unusable attribute on would otherwise - // fail every read past it, permanently, for every consumer. + // Filtered, not merged verbatim: a read decodes every entry, so one a + // foreign writer poisoned would fail every read past it forever. $event += Extensions::filter(\is_array($extensions) ? $extensions : []); try { diff --git a/src/Feed/Store/Cache.php b/src/Feed/Store/Cache.php index 3bbd07d..c92380b 100644 --- a/src/Feed/Store/Cache.php +++ b/src/Feed/Store/Cache.php @@ -16,15 +16,7 @@ class Cache extends Store implements Appendable { public const int TTL = 30 * 24 * 60 * 60; // 30 days - /** - * Deliberately far below the {@see Store::MAX_SIZE} the Redis store - * inherits, where trimming happens server-side and reads are ranged. - * Here the whole feed lives under one key, so retention is also the size - * of every append's read-modify-write: at 100 000 entries a single - * `produce()` moves megabytes through the cache both ways. A larger cap - * is a fine choice for a low-rate feed, but it should be one somebody - * made rather than one inherited from a backend with other costs. - */ + /** Far below {@see Store::MAX_SIZE}: an append here rewrites the whole feed. */ protected const int MAX_SIZE = 1_000; // entries public function __construct( @@ -53,10 +45,8 @@ public function append(CloudEvent $event): string $entries = \array_slice($entries, -$this->maxSize); } - // The tip marker goes first, so it is never behind the feed. A crash - // between the two writes leaves it ahead, which only costs a read that - // finds nothing; behind, it would report a caught-up consumer and the - // event would never be delivered. + // The marker first, so it can be ahead of the feed but never behind: + // behind, it would report a caught-up consumer and lose the event. $this->write(Key::tip($this->name), $id); $this->write($this->key(), $entries); @@ -65,7 +55,7 @@ public function append(CloudEvent $event): string /** * @param string|array $value - * @throws Transport When the write fails, either way a cache adapter can. + * @throws Transport When the write fails, either way a cache can. */ private function write(string $key, string|array $value): void { @@ -115,17 +105,9 @@ public function read(?string $lastEventId, int $limit): array } /** - * Whether the feed provably holds nothing after $lastEventId, decided from - * the tip marker alone. - * - * The whole feed lives under one key, so answering this by reading it - * costs the entire retained feed β€” every poll tick, per waiting consumer, - * for up to 30 seconds a request. The marker turns the common case, a - * caught-up consumer waiting on a quiet feed, into one small read. - * - * Only ever used to skip work, never to invent an answer: the marker is - * written before the feed, so it is never behind, and a missing or - * unreadable one falls through to the real read. + * Whether the feed provably holds nothing after $lastEventId, from the tip + * marker alone β€” so a caught-up long poll does not load the whole feed per + * tick. Only ever used to skip a read, never to answer one. * * @throws Transport When the cache backend cannot be reached. */ @@ -155,10 +137,6 @@ private function load(): array /** @var mixed $stored */ $stored = $this->cache->load($this->key(), $this->ttl); } catch (\Throwable $error) { - // A cache adapter over a backend that is down raises whatever that - // backend raises β€” a raw \RedisException, say. Every error this - // library reports extends Utopia\Feed\Exception, and a consumer - // retrying on Transport must not crash on a backend blip instead. throw new Transport("Failed to read the {$this->name} feed: {$error->getMessage()}", previous: $error); } diff --git a/tests/Feed/Consumer/Base.php b/tests/Feed/Consumer/Base.php index 1b1614c..95fefa9 100644 --- a/tests/Feed/Consumer/Base.php +++ b/tests/Feed/Consumer/Base.php @@ -487,11 +487,7 @@ public function testAResetMadeInsideAHandlerIsNotOverwritten(): void $this->assertSame(['a', 'b'], $this->drain($consumer), 'The reset stands, so everything retained replays'); } - /** - * Whether the feed under test mints the positions it pages by, and so is - * the authority on their shape. A local store is; another producer's feed, - * read over HTTP, is not. - */ + /** Whether the feed under test mints its own positions, and so judges their shape. */ protected function ownsItsIdFormat(): bool { return true; @@ -524,8 +520,7 @@ public function testSeekRejectsAnIdNoFeedCouldUse(string $id): void /** * Rejected whatever the feed is: an empty string names nothing, and the - * tip sentinel stands for wherever the feed ends when the request arrives - * β€” a start rather than a position. + * tip sentinel is a start rather than a position. * * @return array */ diff --git a/tests/Feed/Consumer/CacheTest.php b/tests/Feed/Consumer/CacheTest.php index d37645d..864a54c 100644 --- a/tests/Feed/Consumer/CacheTest.php +++ b/tests/Feed/Consumer/CacheTest.php @@ -15,13 +15,9 @@ class CacheTest extends Base use UsesCache; /** - * A feed in the cache, and a producer over it. - * - * The shared scenarios run against a memory feed on purpose, and swap only - * the cursor β€” so `Store\Cache` was exercised by the producer and server - * suites but never by a `Consumer`, which is the integration the adapter - * exists for: a service that already carries a cache keeping both the feed - * and the position there. + * A feed in the cache, and a producer over it β€” the integration the + * adapter exists for, which the shared scenarios miss by swapping only + * the cursor. * * @return array{Store&Appendable, Producer} */ @@ -49,12 +45,7 @@ public function testAConsumerDrainsACacheFedFeed(): void $this->assertSame(['c'], $this->drain($this->consumer(store: $store)), 'Resumes without replaying'); } - /** - * The consumer's poll loop over this adapter, which reads the feed - * differently from the others: a caught-up tick answers from the tip - * marker rather than by loading the feed, so "caught up" has to keep - * meaning caught up and not "nothing more, ever". - */ + /** A caught-up tick answers from the tip marker, so it must not get stuck there. */ public function testACaughtUpConsumerStillSeesTheNextEvent(): void { [$store, $producer] = $this->fed(); diff --git a/tests/Feed/Consumer/HttpTest.php b/tests/Feed/Consumer/HttpTest.php index 4f1e8da..914681e 100644 --- a/tests/Feed/Consumer/HttpTest.php +++ b/tests/Feed/Consumer/HttpTest.php @@ -43,12 +43,9 @@ protected function ownsItsIdFormat(): bool } /** - * http-feeds endpoints commonly use UUIDs, and `Remote` reads them fine β€” - * a consumer tracks such a position and sends it back as `lastEventId` - * without complaint. Judging the shape here as well would take the - * poison-event escape hatch away from exactly the consumers that have no - * way around it: the id `seek()` refused would be one the consumer itself - * had just handled and saved. + * http-feeds endpoints commonly use UUIDs, which `Remote` reads and + * `consume()` already saves β€” so refusing one here would take the + * poison-event escape hatch away from the consumers that need it most. */ public function testSeekAcceptsTheOpaqueIdARemoteFeedMayUse(): void { @@ -63,10 +60,8 @@ public function testSeekAcceptsTheOpaqueIdARemoteFeedMayUse(): void /** * The feed name is the one thing on the wire only the serving side can - * check. `Remote` encodes it into the request path, and the consumer's own - * `feed:` check is client-side β€” so nothing confirmed end to end that the - * name reaching the endpoint is the one it holds. A consumer pointed at - * the wrong feed must fail rather than read the right events by accident. + * check β€” the consumer's own `feed:` check is client-side. Pointed at the + * wrong feed it must fail, not read the right events by accident. */ public function testAConsumerPointedAtAnotherFeedIsNotServedThisOne(): void { @@ -85,10 +80,8 @@ public function testAConsumerPointedAtAnotherFeedIsNotServedThisOne(): void } /** - * A name that needs encoding survives the round trip: `Remote` percent- - * encodes it into one path segment and the endpoint decodes that segment - * back. Asserting the URI string alone, as the encoding test does, cannot - * show that anything decodes it to the name the producer knows. + * A name that needs encoding survives the round trip. Asserting the URI + * string, as the encoding test does, cannot show that anything decodes it. */ public function testAFeedNameThatNeedsEncodingStillRoutes(): void { @@ -101,12 +94,7 @@ public function testAFeedNameThatNeedsEncodingStillRoutes(): void $this->assertStringContainsString('a%20b%2Fc', $this->endpoint->recorder->last()['uri']); } - /** - * The two halves of the media type handshake, checked against each other - * rather than each against a literal: the `Accept` the consumer sends is - * the `Content-Type` the producer answers with. Both now alias one - * constant, so this is what would notice if they stopped. - */ + /** The two halves of the handshake checked against each other, not against a literal. */ public function testTheAcceptSentIsTheContentTypeServed(): void { $this->producer->produce('a'); @@ -120,14 +108,9 @@ public function testTheAcceptSentIsTheContentTypeServed(): void } /** - * One event through the real wire code in both directions β€” `Batch` - * encoding it in the endpoint, `Remote` decoding it on the way back β€” - * with every attribute asserted on the far side. - * - * The shared consumer scenarios record only `type`, and the producer and - * server suites round-trip the rest through the *store*. So an encoder or - * decoder that dropped `subject`, `source`, `dataschema` or an extension - * on the wire specifically would pass everything else in the suite. + * One event through the real wire code both ways, every attribute + * asserted. Everything else round-trips through the store instead, so a + * drop on the wire specifically would pass the rest of the suite. */ public function testAnEventSurvivesTheWireWithEveryAttribute(): void { diff --git a/tests/Feed/Consumer/PoolTest.php b/tests/Feed/Consumer/PoolTest.php index bd95bd2..e06a8ec 100644 --- a/tests/Feed/Consumer/PoolTest.php +++ b/tests/Feed/Consumer/PoolTest.php @@ -12,10 +12,8 @@ class PoolTest extends Base use UsesPool; /** - * Every cursor operation borrows a connection and must give it back β€” - * including `reset()` and a `seek()` that the shared scenarios drive but - * never watch the pool through. A leak here drains the pool over a - * consumer's lifetime rather than failing anything outright. + * Every cursor operation borrows a connection and must give it back. A + * leak drains the pool over a consumer's lifetime rather than failing. */ public function testEveryCursorOperationReturnsItsConnection(): void { diff --git a/tests/Feed/Consumer/RedisTest.php b/tests/Feed/Consumer/RedisTest.php index 50ce855..c85d3ec 100644 --- a/tests/Feed/Consumer/RedisTest.php +++ b/tests/Feed/Consumer/RedisTest.php @@ -20,14 +20,9 @@ class RedisTest extends Base use UsesRedis; /** - * A feed on a real Redis stream, and a producer over it. - * - * The shared scenarios run against a memory feed on purpose β€” a failure - * there is the consuming side's, not a store's. But that leaves the - * consumer's paging arithmetic (`Id::after`, strictly-after reads, the - * batch loop) exercised only against ids this library mints itself, never - * against the ones `XADD` assigns or the approximate trimming `XRANGE` - * reads back. The scenarios below fill exactly that gap and no more. + * A feed on a real Redis stream, and a producer over it. The shared + * scenarios use a memory feed, so the consumer's paging arithmetic is + * otherwise never run against the ids `XADD` assigns. * * @return array{Store&Appendable, Producer} */ @@ -55,9 +50,8 @@ public function testConsumesAStreamThroughAPersistedCursor(): void } /** - * `XADD` assigns ids within one millisecond by bumping the sequence, so a - * batch boundary regularly falls between two ids sharing a timestamp β€” - * which is the case paging by string comparison would get wrong. + * `XADD` bumps the sequence within a millisecond, so a batch boundary + * regularly falls between two ids sharing a timestamp. */ public function testPagesAStreamInBatchesWithoutSkippingOrRepeating(): void { @@ -122,10 +116,8 @@ public function testThePositionIsStoredWhereOperatorsExpectIt(): void } /** - * The cursor's half of the `Transport` contract. `FailingCursor` shows how - * a consumer reacts to a `Transport`, but it raises one itself β€” the - * `\RedisException` wrapping this adapter does was never run under test, - * so a regression letting the raw exception out would have shipped green. + * The cursor's half of the `Transport` contract: `FailingCursor` raises + * one itself, so the wrapping here is otherwise never run. * * @param callable(Cursor): void $operation */ @@ -157,11 +149,7 @@ public static function operations(): array ]; } - /** - * And the consumer's own contract on top of it: a position that cannot be - * read stops the run, since reading from an unknown position would replay - * the retained feed rather than report the failure. - */ + /** A position that cannot be read stops the run rather than replaying the feed. */ public function testAConsumerOverAnUnreachableCursorStopsWithTransport(): void { $this->producer->produce('a'); diff --git a/tests/Feed/Cursor/Base.php b/tests/Feed/Cursor/Base.php index e53b5a0..9e7980a 100644 --- a/tests/Feed/Cursor/Base.php +++ b/tests/Feed/Cursor/Base.php @@ -14,13 +14,8 @@ * provides: a position is remembered, keyed by feed *and* consumer, forgotten * on reset, and every operation refuses a name it cannot key. * - * Most of this is also visible through {@see \Utopia\Tests\Consumer\Base}, - * which runs every cursor adapter β€” but not all of it. `Consumer` validates - * its own name before the cursor is ever touched, so the adapters' `Invalid` - * path is unreachable from there, and nothing else checks that each adapter - * routes all three operations through the shared {@see Cursor::key()} rather - * than building a key of its own. An adapter that did would skip the - * validation and diverge from the documented key layout at the same time. + * The consumer scenarios cover most of this sideways, but not the `Invalid` + * path β€” `Consumer` validates its own name before the cursor is touched. */ abstract class Base extends TestCase { @@ -114,11 +109,8 @@ public function testResetOnlyForgetsTheConsumerItNames(): void } /** - * A name that cannot be keyed is refused by every operation, not only by - * whichever one a caller happens to reach first. This is the assertion - * that says the adapter goes through the shared key builder at all β€” - * `Consumer` validates its own name long before the cursor sees it, so - * nothing else here can. + * Refused by every operation, not just the first one a caller reaches β€” + * which is what says the adapter goes through the shared key builder. * * @param callable(Cursor, string, string): void $operation */ diff --git a/tests/Feed/Producer/Base.php b/tests/Feed/Producer/Base.php index 79e12fb..5d18216 100644 --- a/tests/Feed/Producer/Base.php +++ b/tests/Feed/Producer/Base.php @@ -109,11 +109,8 @@ public function testPublishStampsAPreparedEvent(): void } /** - * `source` records where an event happened, and a producer can only speak - * for itself β€” so an event relayed from another feed is republished as - * this service's event. Documented rather than merely tested, because a - * caller handing over a "prepared" event would reasonably expect it to be - * published as prepared. + * A producer can only speak for itself, so an event relayed from another + * feed is republished as this service's β€” surprising enough to pin. */ public function testPublishReplacesTheCallersSourceWithTheProducersOwn(): void { @@ -122,10 +119,7 @@ public function testPublishReplacesTheCallersSourceWithTheProducersOwn(): void $this->assertSame('urn:test', $this->events()[0]->source); } - /** - * The stored form can only be decoded as CloudEvents 1.0, so keeping - * another version would leave an entry in the feed that nothing can read. - */ + /** Another version would leave an entry in the feed that nothing can read. */ public function testPublishNormalisesTheSpecVersion(): void { $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', specversion: '1.1')); @@ -238,11 +232,8 @@ public function testADigitsOnlyExtensionNameSurvivesAppendAndRead(): void } /** - * A producer that encodes its payload as something other than JSON says so - * with `datacontenttype`, and losing it leaves a consumer holding data it - * can no longer interpret. Nothing about the flattening the store does is - * visible to the caller, so only a round trip can show the attribute made - * it through. + * Losing `datacontenttype` leaves a consumer holding data it can no longer + * interpret, and only a round trip can show the attribute made it through. */ public function testDatacontenttypeSurvivesAppendAndRead(): void { @@ -260,11 +251,7 @@ public function testDatacontenttypeSurvivesAppendAndRead(): void $this->assertSame('', $event->data); } - /** - * CloudEvents treats an absent `datacontenttype` as meaning the data is - * JSON, so "unset" has to come back unset rather than as the empty string - * the store flattens it to. - */ + /** Absent means JSON, so it must not come back as the empty string the store flattens it to. */ public function testAnEventWithNoDatacontenttypeReadsBackWithNone(): void { $this->producer->publish(new CloudEvent(id: '', type: 'test', source: '', datacontenttype: null)); @@ -350,11 +337,7 @@ public function testRetentionIsBoundedAndDropsTheOldestFirst(): void $this->assertSame('event-299', \end($types), 'The newest event is retained'); } - /** - * Whether this adapter trims to exactly the cap. Redis does not β€” `XADD` - * with `~` trims to a node boundary, which is the whole reason the - * scenario above only asserts the loose bound. - */ + /** Whether this adapter trims to exactly the cap; Redis trims to a node boundary. */ protected function trimsExactly(): bool { return true; diff --git a/tests/Feed/Producer/CacheTest.php b/tests/Feed/Producer/CacheTest.php index eee2ac4..823c614 100644 --- a/tests/Feed/Producer/CacheTest.php +++ b/tests/Feed/Producer/CacheTest.php @@ -51,9 +51,7 @@ public function testAForeignValueUnderTheFeedsKeyReadsAsEmpty(): void /** * The whole feed lives under one key, so answering "anything new?" by - * reading it costs the entire retained feed β€” every poll tick, per waiting - * consumer, for up to 30 seconds a request. A caught-up consumer on a - * quiet feed is the common case, and it must not pay that. + * reading it costs the entire feed β€” per tick, per waiting consumer. */ public function testACaughtUpPollReadsTheTipMarkerRatherThanTheFeed(): void { @@ -71,10 +69,8 @@ public function testACaughtUpPollReadsTheTipMarkerRatherThanTheFeed(): void } /** - * The marker is only ever allowed to skip work, never to invent an answer. - * A cache is free to drop one key and keep another, so a marker that is - * gone must fall through to the real read rather than read as "caught up" - * and strand the consumer. + * A cache may drop one key and keep another, so a missing marker must fall + * through to the real read rather than strand the consumer. */ public function testAMissingTipMarkerFallsBackToReadingTheFeed(): void { @@ -86,10 +82,7 @@ public function testAMissingTipMarkerFallsBackToReadingTheFeed(): void $this->assertCount(1, $this->store->read($first, 10)); } - /** - * And the gate must not be sticky: a consumer told it was caught up has to - * see the next event, or a quiet feed would stay quiet forever. - */ + /** And the gate must not be sticky, or a quiet feed stays quiet forever. */ public function testAnEventAppendedAfterACaughtUpReadIsStillDelivered(): void { $first = $this->producer->produce('a'); @@ -101,11 +94,7 @@ public function testAnEventAppendedAfterACaughtUpReadIsStillDelivered(): void $this->assertCount(1, $this->store->read($first, 10), 'And no longer'); } - /** - * Retention is also the size of every append's read-modify-write here, so - * the cache store keeps a far smaller default than the Redis store, whose - * trimming is server-side and whose reads are ranged. - */ + /** Retention is also the size of every append here, so the default is far smaller. */ public function testTheDefaultRetentionIsScaledToWhatAnAppendCosts(): void { $store = new CacheStore($this->cache(), $this->name); @@ -128,11 +117,9 @@ private function unreachable(): Store&Appendable } /** - * A cache adapter lets the backend's own error out once its internal - * retries are exhausted, so without wrapping a raw \RedisException escapes - * this library entirely. The canonical consume loop retries on Transport - * and would crash on a backend blip instead β€” the exact failure mode the - * Transport contract exists to prevent. + * A cache lets the backend's own error out once its retries are exhausted. + * Unwrapped, the canonical consume loop crashes on a backend blip instead + * of retrying β€” what the Transport contract exists to prevent. * * @param callable(Store&Appendable): void $operation */ @@ -165,9 +152,8 @@ public static function operations(): array } /** - * A cache that answers a write with false rather than raising must be - * caught too β€” the entry is not in the feed, so reporting the position it - * would have had would invent an event no consumer can ever read. + * A write answered with false must be caught too, or the position reported + * names an event no consumer can ever read. */ public function testACacheThatRejectsTheWriteRaisesTransport(): void { diff --git a/tests/Feed/Producer/RedisTest.php b/tests/Feed/Producer/RedisTest.php index 8752376..c0a301a 100644 --- a/tests/Feed/Producer/RedisTest.php +++ b/tests/Feed/Producer/RedisTest.php @@ -38,11 +38,9 @@ public function testEventsLiveInAStreamUnderTheFeedsKey(): void } /** - * Feeds and cursors share one Redis keyspace, and both keys are built by - * joining names with `:`. Joined raw, a feed named `:cursor:x` takes - * the key consumer `x`'s position on `` occupies, so a cursor `SET` - * lands on a stream β€” a WRONGTYPE at best, and at worst one silently - * destroying the other. + * Feeds and cursors share one keyspace. Joined raw, a feed named + * `:cursor:x` takes the key consumer `x`'s position occupies, and + * the cursor `SET` lands on β€” and destroys β€” the stream. */ public function testAFeedNamedLikeACursorKeyDoesNotCollideWithOne(): void { @@ -70,15 +68,9 @@ public function testTheStreamItselfIsTrimmed(): void } /** - * A stream is a shared, writable thing: another tool can `XADD` into a - * feed, and this library's own producer cannot be the only writer assumed. - * An entry carrying an attribute a CloudEvent cannot hold must therefore - * decode without it, exactly as the same event would arriving over HTTP. - * - * The failure this prevents is the worst shape a feed has: a read decodes - * every entry in the batch, so one poisoned entry would fail every read - * past it β€” permanently, for every consumer, until it fell off the trim - * horizon. + * Any tool can `XADD` into a stream, so an entry carrying an attribute a + * CloudEvent cannot hold must decode without it β€” as over HTTP. A read + * decodes every entry, so otherwise it fails every read past it forever. */ public function testAForeignWritersUnusableExtensionIsDroppedRatherThanWedgingTheFeed(): void { @@ -104,12 +96,9 @@ public function testAForeignWritersUnusableExtensionIsDroppedRatherThanWedgingTh } /** - * The README promises `Transport` when "the backend or network failed: - * Redis errors, HTTP failures". The HTTP half of that promise is tested - * thoroughly; the Redis half β€” the flagship production adapter β€” was not - * tested at all, so a regression letting a raw `\RedisException` out would - * have shipped green and crashed every consumer catching - * `Utopia\Feed\Exception` per the README. + * The README promises `Transport` for a Redis failure. Untested, a raw + * `\RedisException` escaping would crash consumers catching + * `Utopia\Feed\Exception` as documented. * * @param callable(Store&Appendable): void $operation */ @@ -142,11 +131,9 @@ public static function operations(): array } /** - * A foreign value under the feed's key β€” someone else's key collision, or - * a leftover from another tool β€” makes Redis answer every stream command + * A foreign value under the feed's key makes Redis answer stream commands * with an error rather than raising. Appending must not report a position - * for an event that is not in the feed, so the reply is checked rather - * than trusted. + * for an event that is not in the feed, so the reply is checked. */ public function testAppendingOverAForeignValueRaisesTransport(): void { @@ -158,10 +145,8 @@ public function testAppendingOverAForeignValueRaisesTransport(): void } /** - * Reading past the same value is the opposite call: a feed nobody can read - * is an empty feed β€” a replay at worst β€” and failing the read instead - * would stall every consumer of it. Same policy the cache store applies to - * a foreign value under its key. + * Reading is the opposite call: a feed nobody can read is an empty one β€” a + * replay at worst β€” where failing would stall every consumer of it. */ public function testReadingPastAForeignValueIsAnEmptyFeed(): void { diff --git a/tests/Feed/Server/Base.php b/tests/Feed/Server/Base.php index 979eb11..89ccc31 100644 --- a/tests/Feed/Server/Base.php +++ b/tests/Feed/Server/Base.php @@ -136,12 +136,9 @@ public function testRejectsAPositionThatIsNotAFeedId(): void /** * Every upper bound below is set from the elapsed time the *regression* - * would produce, not from the time the correct code takes. A bound just - * above the expected duration measures the CI runner rather than the - * code, and fails rarely, unreproducibly, and only under load β€” which - * teaches everyone to hit re-run, which is how real regressions get - * waved through. Lower bounds are safe either way: a sleep cannot - * finish early. + * would produce, not from what the correct code takes β€” a tighter bound + * measures the CI runner. Lower bounds cannot flake: a sleep cannot end + * early. */ public function testPollReturnsImmediatelyWhenEventsAreWaiting(): void { @@ -189,9 +186,7 @@ public function testPollHonoursATimeoutShorterThanThePollInterval(): void $this->assertCount(0, $events); $this->assertGreaterThanOrEqual(0.08, $elapsed, 'Must actually wait out the timeout'); - // Overshooting means sleeping the full 500ms interval past a 100ms - // deadline, so anything under 0.45 catches it β€” with 350ms of slack - // over the ~100ms this takes when correct. + // Overshooting sleeps the full 500ms interval past a 100ms deadline. $this->assertLessThan(0.45, $elapsed, 'Must not sleep a full interval past the deadline'); } @@ -278,11 +273,9 @@ public function testServeRejectsALastEventIdThatIsNotAPosition(): void } /** - * PHP parses `?lastEventId[]=1-0` into an array, so a `lastEventId` that - * is present need not be a string. Coercing one to "absent" would answer - * a malformed parameter with a full replay of the retained feed β€” the - * most expensive response the endpoint has, and one a caught-up consumer - * would read as a sudden flood of new events rather than as the 400 it is. + * PHP parses `?lastEventId[]=1-0` into an array, so a present + * `lastEventId` need not be a string. Coercing one to "absent" answers a + * malformed parameter with a full replay of the feed instead of a 400. */ #[DataProvider('notStrings')] public function testServeRejectsALastEventIdThatIsNotEvenAString(mixed $lastEventId): void @@ -309,10 +302,8 @@ public static function notStrings(): array } /** - * `limit` and `timeout` stay forgiving where `lastEventId` does not: both - * are the producer's to decide, so garbage falls back to the default - * rather than stalling a feed over something that cannot cause a wrong - * answer. An array is garbage like any other. + * `limit` and `timeout` stay forgiving where `lastEventId` does not: + * neither can cause a wrong answer. An array is garbage like any other. */ public function testServeFallsBackToTheDefaultsOnArrayLimitsAndTimeouts(): void { @@ -330,11 +321,9 @@ public function testServeLetsTheTipSentinelThrough(): void } /** - * The long-poll contract has to hold through the HTTP entry point, not - * only through `poll()`. A refactor that stopped forwarding the timeout - * would turn every long poll into a plain read β€” consumers would spin - * instead of waiting, and nothing that asserts on returned events could - * tell, because a caught-up read returns the same empty batch either way. + * The long-poll contract has to hold through the HTTP entry point too. A + * dropped timeout turns every long poll into a plain read, which no + * assertion on returned events can see β€” the batch is empty either way. */ public function testServeHoldsALongPollOnAnEmptyFeed(): void { diff --git a/tests/Feed/Server/MemoryTest.php b/tests/Feed/Server/MemoryTest.php index 131a5f9..661de15 100644 --- a/tests/Feed/Server/MemoryTest.php +++ b/tests/Feed/Server/MemoryTest.php @@ -37,9 +37,7 @@ public function testAShorterPollIntervalDeliversAMidPollEventSooner(): void $elapsed = \microtime(true) - $started; $this->assertCount(1, $events); - // Ignoring the configured interval means falling back to the 500ms - // default, so anything under 0.45 catches it β€” with 400ms of slack - // over the ~20ms this takes when correct. + // Ignoring the configured interval falls back to the 500ms default. $this->assertLessThan(0.45, $elapsed, 'A 20ms interval must beat the default 500ms floor'); } } diff --git a/tests/Feed/Server/PoolTest.php b/tests/Feed/Server/PoolTest.php index 2dcd6b7..91f031b 100644 --- a/tests/Feed/Server/PoolTest.php +++ b/tests/Feed/Server/PoolTest.php @@ -14,15 +14,9 @@ class PoolTest extends Base use UsesPool; /** - * The property the pooled store exists for, and the only one the shared - * scenarios cannot show: a long poll is a loop of reads, and the store - * borrows for each read rather than once around the whole loop. Held for - * the wait, one poll would tie up a connection for up to 30 seconds, so a - * handful of idle consumers would exhaust the pool. - * - * Asserted as a count of releases, so an "optimization" that hoists the - * borrow out of the loop fails here instead of silently removing the - * class's entire reason to exist. + * The property the pooled store exists for: a long poll borrows per read, + * not once around the loop, so an idle consumer does not tie up a + * connection for 30 seconds. Counted, so hoisting the borrow fails here. */ public function testAHeldPollBorrowsPerReadRatherThanForTheWholeWait(): void { @@ -39,11 +33,7 @@ public function testAHeldPollBorrowsPerReadRatherThanForTheWholeWait(): void ); } - /** - * The other half of the same property: every borrow is given back. A leak - * would not fail a functional test until the pool ran dry, which in a - * service is minutes into production rather than here. - */ + /** Every borrow is given back β€” a leak fails nothing until the pool runs dry. */ public function testEveryBorrowIsReturnedToThePool(): void { $this->producer->produce('a'); diff --git a/tests/Feed/Support/BrokenCache.php b/tests/Feed/Support/BrokenCache.php index cd96b2e..83ce6a3 100644 --- a/tests/Feed/Support/BrokenCache.php +++ b/tests/Feed/Support/BrokenCache.php @@ -7,18 +7,11 @@ use Utopia\Cache\Adapter; /** - * A cache backend that is down, for testing that the cache-backed store and - * cursor notice. - * - * It fails the two ways {@see \Utopia\Cache\Adapter\Redis} actually fails - * rather than the way a test double would find convenient: - * - * - `save()` catches everything internally and answers `false`, so a caller - * that only looks at the absence of an exception sees a successful write. - * - `load()` and `purge()` let the backend's own error out once the adapter's - * internal retries are exhausted β€” a raw `\RedisException` in production, - * stood in for here by a plain exception so the service-free suites stay - * service-free. What matters is that it is not a `Utopia\Feed\Exception`. + * A cache backend that is down, failing the two ways + * {@see \Utopia\Cache\Adapter\Redis} does: `save()` answers `false` without + * raising, and the rest let the backend's own error out. That error stands in + * for a `\RedisException` so the service-free suites stay service-free β€” all + * that matters is that it is not a `Utopia\Feed\Exception`. */ class BrokenCache implements Adapter { diff --git a/tests/Feed/Support/CountingCache.php b/tests/Feed/Support/CountingCache.php index 802fd48..ff97949 100644 --- a/tests/Feed/Support/CountingCache.php +++ b/tests/Feed/Support/CountingCache.php @@ -6,10 +6,7 @@ use Utopia\Cache\Adapter\Memory; -/** - * A working cache that records how often each key was read, for asserting on - * what an operation costs the backend rather than only on what it returns. - */ +/** A working cache that records how often each key was read. */ class CountingCache extends Memory { /** @var array Reads per key. */ diff --git a/tests/Feed/Support/CountingStack.php b/tests/Feed/Support/CountingStack.php index 528676b..c58e94c 100644 --- a/tests/Feed/Support/CountingStack.php +++ b/tests/Feed/Support/CountingStack.php @@ -7,15 +7,11 @@ use Utopia\Pools\Adapter\Stack; /** - * A pool adapter that records how often a connection was handed back. - * - * Counting releases rather than acquisitions is deliberate: the pool creates - * its first connection instead of popping one, so acquisitions undercount the - * first borrow, while every completed `use()` pushes exactly once. + * A pool adapter that counts completed borrows. Releases rather than + * acquisitions: the pool creates its first connection instead of popping one. */ class CountingStack extends Stack { - /** Completed borrows β€” one per `Pool::use()` that ran to the end. */ public int $releases = 0; public function push(mixed $connection): static diff --git a/tests/Feed/Support/FeedServer.php b/tests/Feed/Support/FeedServer.php index d8d74e4..1c41624 100644 --- a/tests/Feed/Support/FeedServer.php +++ b/tests/Feed/Support/FeedServer.php @@ -20,11 +20,9 @@ * parameters, body and caching β€” rather than a fixture written to match the * consumer. * - * It routes on the request path too, minimally but honestly. The feed name is - * the one thing on the wire that only the serving side can check: `Remote` - * `rawurlencode()`s it into the path, and a fixture that ignored the path - * would answer any name with the one feed it holds β€” so a consumer pointed at - * the wrong feed would read the right events and no test could tell. + * It routes on the request path too: a fixture that ignored it would answer + * any feed name with the one feed it holds, so a consumer pointed at the + * wrong feed would read the right events and no test could tell. */ class FeedServer extends FakeClient { @@ -54,12 +52,9 @@ protected function respond(RequestInterface $request): ResponseInterface /** * The feed name the request asks for: the last path segment, decoded. * - * Split before decoding, never after β€” `Remote` percent-encodes the name - * into one segment, so a feed called `a/b` arrives as `a%2Fb` and decoding - * first would split it into a path it never asked for. - * - * The path may be relative and hold no slash at all: a consumer built - * straight over a client with no base URI sends the bare name. + * Split before decoding, never after β€” a feed called `a/b` arrives as + * `a%2Fb`. The path may also hold no slash at all, since a client with no + * base URI sends the bare name. */ private static function feed(RequestInterface $request): string { diff --git a/tests/Feed/Support/RecordingStore.php b/tests/Feed/Support/RecordingStore.php index 960fba1..519304b 100644 --- a/tests/Feed/Support/RecordingStore.php +++ b/tests/Feed/Support/RecordingStore.php @@ -7,11 +7,8 @@ use Utopia\Feed\Store\Memory; /** - * A memory feed that records what it was asked for. - * - * Clamping and coercion are only visible in the arguments the store receives β€” - * a caller that stopped forwarding them entirely returns the same events β€” so - * asserting on the answer cannot tell the two apart. + * A memory feed that records what it was asked for. Clamping and coercion are + * only visible in the arguments, never in the events that come back. */ class RecordingStore extends Memory { diff --git a/tests/Feed/Support/UsesPool.php b/tests/Feed/Support/UsesPool.php index 7c3cf66..b34937b 100644 --- a/tests/Feed/Support/UsesPool.php +++ b/tests/Feed/Support/UsesPool.php @@ -27,9 +27,8 @@ trait UsesPool private ?UtopiaPool $pool = null; /** - * A pool over the suite's Redis, on the adapter given β€” the default - * {@see Stack} for ordinary use, or one that records borrows for a test - * that asserts on how the store uses the pool rather than on what it reads. + * A pool over the suite's Redis: the default {@see Stack}, or one that + * records borrows for a test asserting on how the pool is used. * * @return UtopiaPool<\Redis|\RedisCluster> */ diff --git a/tests/Feed/Support/UsesRedis.php b/tests/Feed/Support/UsesRedis.php index a4f28a3..6056545 100644 --- a/tests/Feed/Support/UsesRedis.php +++ b/tests/Feed/Support/UsesRedis.php @@ -31,15 +31,9 @@ protected function redis(): \Redis } /** - * A client that cannot reach a server, for the error contract. - * - * Every command on one raises `\RedisException` from the extension itself, - * so the wrapping runs for real rather than against a double that throws - * on cue. Closing a connected client would not do: phpredis reconnects - * transparently on the next command. - * - * It is also a real misconfiguration β€” a client handed to a store before - * anything connected it β€” rather than a state only a test can produce. + * A client that cannot reach a server: every command raises + * `\RedisException` from the extension itself. Closing a connected one + * would not do β€” phpredis reconnects on the next command. */ protected static function unreachableRedis(): \Redis { diff --git a/tests/Feed/Unit/BatchTest.php b/tests/Feed/Unit/BatchTest.php index ebbd71d..c00564d 100644 --- a/tests/Feed/Unit/BatchTest.php +++ b/tests/Feed/Unit/BatchTest.php @@ -84,11 +84,8 @@ public function testToArrayIsTheWireEncoding(): void } /** - * Everything an event carries has to reach the wire, not only what a - * consumer of this suite happens to look at. Asserting `id` and - * `specversion` alone would pass an encoder that dropped `subject` or an - * extension β€” and on the wire is the one place a dropped attribute cannot - * be recovered from. + * Asserted as a whole array: `id` and `specversion` alone would pass an + * encoder that dropped `subject` or an extension. */ public function testEveryAttributeReachesTheWire(): void { @@ -118,11 +115,7 @@ public function testEveryAttributeReachesTheWire(): void ], $batch->toArray()[0]); } - /** - * The spec has no null attribute values, so an absent optional attribute - * is omitted rather than sent as null β€” a consumer must be able to tell - * "not set" from "set to nothing". - */ + /** The spec has no null attribute values, so an absent attribute is omitted. */ public function testAnAbsentOptionalAttributeIsOmittedRatherThanNulled(): void { $encoded = (new Batch([new CloudEvent( diff --git a/tests/Feed/Unit/CursorTest.php b/tests/Feed/Unit/CursorTest.php index 1004ca6..87fe119 100644 --- a/tests/Feed/Unit/CursorTest.php +++ b/tests/Feed/Unit/CursorTest.php @@ -67,10 +67,8 @@ public function testTheNoneStoreStillRejectsEmptyNames(string $feed, string $con } /** - * A cache adapter answers a failed write with false rather than raising, - * so a cursor that ignores the return value reports a position as saved - * that was never stored β€” and the consumer replays its whole retained - * backlog on the next restart with nothing to explain why. + * A cache answers a failed write with false rather than raising, so + * ignoring it reports a position as saved that was never stored. */ public function testACacheThatCannotBeWrittenRaisesTransport(): void { @@ -82,10 +80,8 @@ public function testACacheThatCannotBeWrittenRaisesTransport(): void } /** - * The operational consequence, through the API an operator actually - * reaches for: seek() is documented as persisting immediately, so a store - * that cannot hold the new position must not let the operator believe - * they stepped past a poison event. + * The consequence through the API an operator reaches for: seek() persists + * immediately, so a failed one must not look like it worked. */ public function testASeekThroughAnUnwritableCacheFailsLoudly(): void { @@ -105,10 +101,8 @@ public function testASeekThroughAnUnwritableCacheFailsLoudly(): void } /** - * Resetting is not held to the same rule: an adapter answers false both - * for a write it could not do and for a key that was never there, and the - * second is the ordinary case β€” a consumer that has not saved a position - * yet. Only a raising backend is a failure to report. + * Resetting is not held to the same rule: false also means "was never + * there", the ordinary case for a consumer with no position yet. */ public function testResettingAPositionThatWasNeverSavedIsHarmless(): void { @@ -121,11 +115,8 @@ public function testResettingAPositionThatWasNeverSavedIsHarmless(): void } /** - * The other way a cache backend fails: it lets its own error out once the - * adapter's internal retries are exhausted β€” a raw \RedisException in - * production. Unwrapped, that escapes this library entirely, so the - * canonical consume loop retrying on Transport crashes on a backend blip - * instead, which is what the Transport contract exists to prevent. + * The other way a cache fails: it lets the backend's own error out. + * Unwrapped, that escapes this library entirely. * * @param callable(Cursor): void $operation */ @@ -157,11 +148,7 @@ public static function operations(): array ]; } - /** - * An unusable name is the caller's bug, not the backend's failure, and it - * stays Invalid even when the backend behind the cursor is also down β€” - * otherwise wrapping the store call would swallow the distinction. - */ + /** An unusable name stays Invalid even when the backend is also down. */ #[DataProvider('unusableNames')] public function testAnUnusableNameIsStillInvalidOnABackendThatIsDown(string $feed, string $consumer): void { diff --git a/tests/Feed/Unit/KeyTest.php b/tests/Feed/Unit/KeyTest.php index 52ec126..95e6ae2 100644 --- a/tests/Feed/Unit/KeyTest.php +++ b/tests/Feed/Unit/KeyTest.php @@ -9,48 +9,32 @@ use Utopia\Feed\Key; /** - * The keys a feed and a cursor occupy in one backend keyspace. Names come - * from configuration and from third-party feeds, so the mapping has to stay + * The keys a feed and a cursor occupy in one keyspace. Names come from + * configuration and from third-party feeds, so the mapping has to stay * injective for names nobody vetted. */ class KeyTest extends TestCase { - /** - * The layout the README documents and an operator reads from a shell β€” - * unchanged for every name anyone would actually pick, which is what makes - * escaping cheap enough to always do. - */ + /** The layout the README documents, unchanged for any name anyone would pick. */ public function testAnOrdinaryNameIsLeftAlone(): void { $this->assertSame('feed:edge', Key::feed('edge')); $this->assertSame('feed:edge:cursor:invalidator', Key::cursor('edge', 'invalidator')); } - /** - * The collision that corrupts data rather than losing it: the stream key - * of a feed named `edge:cursor:x` used to be the cursor key of consumer - * `x` on feed `edge`, so a cursor `SET` landed on an `XADD` stream. - */ + /** A feed named `edge:cursor:x` used to take consumer `x`'s cursor key on `edge`. */ public function testAFeedNameCannotCollideWithACursorKey(): void { $this->assertNotSame(Key::feed('edge:cursor:x'), Key::cursor('edge', 'x')); } - /** - * And the collision that silently shares one position between two - * unrelated consumers β€” the "two processes sharing a name" hazard the - * README warns about, arrived at without anyone sharing a name. - */ + /** And the one that silently shares a position between two unrelated consumers. */ public function testTwoDistinctPairsCannotShareACursorKey(): void { $this->assertNotSame(Key::cursor('a:cursor:b', 'c'), Key::cursor('a', 'b:cursor:c')); } - /** - * Escaping is only injective if the escape character is escaped too: - * without that, `a:b` and `a%3Ab` would map to the same key and the fix - * would just move the collision somewhere less obvious. - */ + /** Without escaping the escape character, `a:b` and `a%3Ab` would still collide. */ public function testTheEscapeCharacterIsItselfEscaped(): void { $this->assertNotSame(Key::feed('a:b'), Key::feed('a%3Ab')); diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index 3c226dd..24145d4 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -166,10 +166,8 @@ public function testDelegatesLongPollingToTheProducer(): void $started = \microtime(true); $remote->poll(null, 100, 5000); - // The request count is the real check β€” it catches a client-side loop - // whatever the machine is doing. The clock only guards against a wait - // that makes no request at all, and is bounded by the 5s a wrong - // implementation would take rather than the ~0s this one does. + // The request count is the real check; the clock is bounded by the 5s + // a client-side loop would take, not by the ~0s this takes. $this->assertLessThan(1, \microtime(true) - $started, 'Must not wait client-side'); $this->assertCount(1, $transport->recorder->requests, 'Must not poll in a loop'); $this->assertStringContainsString('timeout=5000', $transport->recorder->last()['uri']); @@ -304,10 +302,8 @@ public static function notBatches(): array /** * One event as a producer would put it on the wire. * - * Overrides win, and are unioned rather than merged: a digits-only - * extension name is legal per the spec and an integer key in PHP, which - * `array_merge()` would silently renumber β€” losing the attribute inside - * the fixture, before the code under test ever saw it. + * Overrides win, and are unioned rather than merged: `array_merge()` + * renumbers a digits-only name, losing it inside the fixture. * * @param array $overrides * @return array @@ -383,10 +379,8 @@ public function testFailsWhenAnEventIsNotACloudEventAtAll(): void } /** - * Every context attribute this library models has to come off the wire, - * not only the ones a consumer happens to look at. An attribute silently - * dropped in decoding is the same class of bug as one dropped in storing, - * and neither shows up in a test that asserts on `id` and `data` alone. + * Every attribute this library models has to come off the wire, not only + * the ones a test asserting on `id` and `data` happens to look at. */ public function testEveryModelledAttributeComesOffTheWire(): void { @@ -413,12 +407,7 @@ public function testEveryModelledAttributeComesOffTheWire(): void $this->assertSame('00-abc-def-01', $event->extensions['traceparent']); } - /** - * CloudEvents reads an absent `datacontenttype` as "the data is JSON", so - * a feed that does not send one must decode without inventing it β€” the - * attribute stays unset rather than becoming a value the producer never - * claimed. - */ + /** An absent `datacontenttype` means JSON, so it must not be invented on decode. */ public function testAnAbsentDatacontenttypeIsNotInvented(): void { [$remote] = $this->remote([FakeTransport::json([self::raw('1-0', 'a')])]); @@ -450,14 +439,9 @@ public function testSurvivesAProducerThatMovedAhead(): void } /** - * The spec is narrow about extensions β€” a name of lowercase letters and - * digits, a value that is a boolean, an integer or a string β€” and a feed - * is read by consumers older than its producer by design. So an attribute - * outside that is dropped and the event still delivered, rather than one - * odd attribute costing the whole event and everything behind it. - * - * Dropping is a choice, not an accident, which is why each shape it can - * take is named here. + * An attribute the spec cannot carry is dropped and the event still + * delivered, rather than one odd attribute costing the whole event. + * Dropping is a choice, so each shape it takes is named here. * * @param array $extension */ @@ -491,11 +475,7 @@ public static function unusableExtensions(): array ]; } - /** - * The types the spec does allow, including a digits-only name β€” legal per - * the spec, and an integer key in PHP, which anything merging with - * `array_merge()` would silently renumber. - */ + /** The types the spec allows, including a digits-only name β€” an integer key in PHP. */ public function testEveryExtensionTheSpecAllowsIsKept(): void { [$remote] = $this->remote([FakeTransport::json([ diff --git a/tests/Feed/Unit/ServerTest.php b/tests/Feed/Unit/ServerTest.php index fbbd6d2..4a7d4a2 100644 --- a/tests/Feed/Unit/ServerTest.php +++ b/tests/Feed/Unit/ServerTest.php @@ -13,11 +13,10 @@ use Utopia\Tests\Support\RecordingStore; /** - * The server behaviours no working adapter can show: serving a feed whose - * backend was never configured, and what the server asks its store for. - * Clamping and coercion are invisible in the events that come back, so only a - * store that records its arguments can tell them from a server that dropped - * them entirely. Everything else lives in {@see \Utopia\Tests\Server\Base}. + * The server behaviours no working adapter can show: a backend that was never + * configured, and what the server asks its store for β€” clamping is invisible + * in the events that come back. Everything else lives in + * {@see \Utopia\Tests\Server\Base}. */ class ServerTest extends TestCase { @@ -31,10 +30,8 @@ public function testAFeedWithNoBackendCannotBeRead(): void } /** - * `timeout` arrives from an untrusted HTTP client, so this is the clamp - * that keeps `?timeout=86400000` from holding a worker for as long as the - * caller feels like asking. The consumer clamps its own inputs too, but - * that is a different clamp against a different caller. + * `timeout` arrives from an untrusted client, so this clamp keeps + * `?timeout=86400000` from holding a worker for as long as it asks. * * @param array $query */ From 8ab8fa4dfde741a6a88cc6d8b8f5e49cff8782da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 14:48:31 +0200 Subject: [PATCH 66/68] Improve redis issue behaviour --- src/Feed/Cursor/Redis.php | 6 ++++++ tests/Feed/Cursor/RedisTest.php | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 828f8b1..48ecdae 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -19,6 +19,12 @@ public function load(string $feed, string $consumer): ?string /** @var mixed $cursor */ $cursor = $this->redis->get($this->key($feed, $consumer)); } catch (\RedisException $error) { + // A key left in another format holds no readable position; the + // next save() overwrites it. + if (\str_contains($error->getMessage(), 'WRONGTYPE')) { + return null; + } + throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); } diff --git a/tests/Feed/Cursor/RedisTest.php b/tests/Feed/Cursor/RedisTest.php index f68cbe5..59c57dc 100644 --- a/tests/Feed/Cursor/RedisTest.php +++ b/tests/Feed/Cursor/RedisTest.php @@ -4,9 +4,30 @@ namespace Utopia\Tests\Cursor; +use Utopia\Feed\Key; use Utopia\Tests\Support\UsesRedis; class RedisTest extends Base { use UsesRedis; + + /** A key left in another format is no position, not a permanent failure. */ + public function testAKeyOfAnotherTypeLoadsAsNoPosition(): void + { + $this->redis()->xAdd(Key::cursor($this->name, 'invalidator'), '1-0', ['p' => '1']); + + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + } + + /** The first save overwrites a key of any type, converting it in place. */ + public function testSavingOverAKeyOfAnotherTypeConvertsIt(): void + { + $key = Key::cursor($this->name, 'invalidator'); + $this->redis()->xAdd($key, '1-0', ['p' => '1']); + + $this->cursor->save($this->name, 'invalidator', '2-0'); + + $this->assertSame(\Redis::REDIS_STRING, $this->redis()->type($key)); + $this->assertSame('2-0', $this->cursor->load($this->name, 'invalidator')); + } } From 5a2f22897e3f1296fb79a1c3a0c2e63b217af61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 15:02:19 +0200 Subject: [PATCH 67/68] add fallback to xRevRange --- README.md | 5 ++- src/Feed/Cursor/Redis.php | 63 ++++++++++++++++++++++++++++++--- tests/Feed/Cursor/RedisTest.php | 21 +++++++---- 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3a67ec3..32b5a9a 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,10 @@ drains the backlog instead of dropping it. **One process per consumer name.** Two processes sharing a name share one position, so the feed is split between them rather than delivered to both β€” and because each save is last-writer-wins, the shared position can also move -backwards and replay. Give every consumer its own name. +backwards and replay. The same goes for moving a position by hand: a `reset()` +or `seek()` made while another process with the same name is mid-run can be +saved over when that run finishes, so stop the other process first. Give every +consumer its own name. ## Reference diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index 48ecdae..fbd3923 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -15,19 +15,27 @@ public function __construct(protected readonly \Redis|\RedisCluster $redis) public function load(string $feed, string $consumer): ?string { + $key = $this->key($feed, $consumer); + try { + $this->redis->clearLastError(); + /** @var mixed $cursor */ - $cursor = $this->redis->get($this->key($feed, $consumer)); + $cursor = $this->redis->get($key); } catch (\RedisException $error) { - // A key left in another format holds no readable position; the - // next save() overwrites it. - if (\str_contains($error->getMessage(), 'WRONGTYPE')) { - return null; + if (self::wrongType($error->getMessage())) { + return $this->loadStream($key, $consumer); } throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); } + // phpredis reports a refusal either by throwing or by returning + // `false` with the text in getLastError(); check both paths. + if ($cursor === false && self::wrongType($this->lastError())) { + return $this->loadStream($key, $consumer); + } + return \is_string($cursor) && $cursor !== '' ? $cursor : null; } @@ -48,4 +56,49 @@ public function reset(string $feed, string $consumer): void throw new Transport("Failed to reset the {$consumer} cursor: {$error->getMessage()}", previous: $error); } } + + /** + * A position kept as a one-entry stream by an earlier version, with the + * position as the entry's id. The next {@see save()} converts the key. + * + * @throws Transport When the key holds neither a string nor a stream. + */ + private function loadStream(string $key, string $consumer): ?string + { + try { + $this->redis->clearLastError(); + + $entries = $this->redis->xRevRange($key, '+', '-', 1); + } catch (\RedisException $error) { + throw new Transport("Failed to load the {$consumer} cursor: {$error->getMessage()}", previous: $error); + } + + if ($entries === false) { + throw new Transport("Failed to load the {$consumer} cursor: " . ($this->lastError() ?: 'Redis command failed')); + } + + if (!\is_array($entries) || $entries === []) { + return null; + } + + // The position is the entry's id, not its payload. + $id = \array_key_first($entries); + + return \is_string($id) && $id !== '' ? $id : null; + } + + /** The last command's error text, cleared on the way out. */ + private function lastError(): string + { + $error = $this->redis->getLastError(); + $this->redis->clearLastError(); + + return \is_string($error) ? $error : ''; + } + + /** Whether Redis refused because the key holds another type. */ + private static function wrongType(string $error): bool + { + return \str_contains($error, 'WRONGTYPE'); + } } diff --git a/tests/Feed/Cursor/RedisTest.php b/tests/Feed/Cursor/RedisTest.php index 59c57dc..81ef33b 100644 --- a/tests/Feed/Cursor/RedisTest.php +++ b/tests/Feed/Cursor/RedisTest.php @@ -11,16 +11,16 @@ class RedisTest extends Base { use UsesRedis; - /** A key left in another format is no position, not a permanent failure. */ - public function testAKeyOfAnotherTypeLoadsAsNoPosition(): void + /** An earlier version kept cursors as one-entry streams, with the position as the entry's id. */ + public function testAPositionRetainedAsAStreamStillLoads(): void { - $this->redis()->xAdd(Key::cursor($this->name, 'invalidator'), '1-0', ['p' => '1']); + $this->redis()->xAdd(Key::cursor($this->name, 'invalidator'), '1690000000000-7', ['p' => '1']); - $this->assertNull($this->cursor->load($this->name, 'invalidator')); + $this->assertSame('1690000000000-7', $this->cursor->load($this->name, 'invalidator')); } - /** The first save overwrites a key of any type, converting it in place. */ - public function testSavingOverAKeyOfAnotherTypeConvertsIt(): void + /** The first save after the upgrade converts the key to a string in place. */ + public function testSavingOverAStreamPositionConvertsTheKey(): void { $key = Key::cursor($this->name, 'invalidator'); $this->redis()->xAdd($key, '1-0', ['p' => '1']); @@ -30,4 +30,13 @@ public function testSavingOverAKeyOfAnotherTypeConvertsIt(): void $this->assertSame(\Redis::REDIS_STRING, $this->redis()->type($key)); $this->assertSame('2-0', $this->cursor->load($this->name, 'invalidator')); } + + public function testResetForgetsAPositionRetainedAsAStream(): void + { + $this->redis()->xAdd(Key::cursor($this->name, 'invalidator'), '1-0', ['p' => '1']); + + $this->cursor->reset($this->name, 'invalidator'); + + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + } } From 33c00e9503f31043471881ef8ede29b996f6a167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Aug 2026 15:22:45 +0200 Subject: [PATCH 68/68] Fix stability --- CHANGELOG.md | 5 ++- README.md | 25 ++++++++---- src/Feed/Consumer.php | 11 +++++- src/Feed/Cursor.php | 24 ++++++++++++ src/Feed/Cursor/None.php | 8 ++++ src/Feed/Cursor/Pool.php | 5 +++ src/Feed/Cursor/Redis.php | 5 +++ src/Feed/Remote.php | 11 +++++- tests/Feed/Consumer/Base.php | 19 +++++----- tests/Feed/Cursor/Base.php | 31 +++++++++++++++ tests/Feed/Cursor/RedisTest.php | 13 +++++++ tests/Feed/Unit/ConsumerTest.php | 65 ++++++++++++++++++++++++++++++-- tests/Feed/Unit/RemoteTest.php | 14 +++++++ 13 files changed, 213 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5eba8b..e6d7031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,10 @@ pooled adapters. β€” where a consumer's position lives, keyed by feed and consumer name. The stored form is deliberately plain (`feed::cursor:` holding the id as a string), so positions carry across upgrades and an operator can - answer "where is this consumer?" from a shell. + answer "where is this consumer?" from a shell. A run's save is conditional + on the position it started from (read-compare-write), so an instance that + fell behind cannot undo newer progress, a seek, or a reset β€” it adopts the + newer position instead. - Retention (`maxSize`) and the long-poll read interval (`pollInterval`, in milliseconds, default 500) are constructor options on every store. diff --git a/README.md b/README.md index 32b5a9a..4230d14 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,9 @@ start rather than a position. Either is safe to call from inside a handler: a run that finishes after the move keeps its own progress to itself rather than saving over the newer -decision. +decision. The same holds across processes β€” a run's save lands only if the +position is still where that run started, so a move made through another +instance stands, and the instance whose save was refused adopts it. `seek()` is the escape hatch for a poison event. A handler that keeps failing blocks the feed by design, so stepping past it is a decision made in code: @@ -235,12 +237,13 @@ consumer opted into `Consumer::START_TIP`), so a consumer deployed after the pro drains the backlog instead of dropping it. **One process per consumer name.** Two processes sharing a name share one -position, so the feed is split between them rather than delivered to both β€” -and because each save is last-writer-wins, the shared position can also move -backwards and replay. The same goes for moving a position by hand: a `reset()` -or `seek()` made while another process with the same name is mid-run can be -saved over when that run finishes, so stop the other process first. Give every -consumer its own name. +position, so the feed is split between them rather than delivered to both. +The position itself holds: a save is conditional on where its run started, so +a process that fell behind cannot undo the other's progress, a `seek()`, or a +`reset()` β€” it adopts the newer position and continues. The check is +read-compare-write rather than atomic, so a race that lands inside one round +trip can still replay a batch: duplicates, which handlers absorb by the first +rule, never a gap. Give every consumer its own name. ## Reference @@ -286,6 +289,14 @@ always written, so positions carry across an upgrade instead of replaying the retained feed, and `GET feed:edge:cursor:notifier` answers "where is this consumer?" from a shell. +A consumer's save is conditional β€” it lands only if the stored position is +still the one its run started from ([`Cursor::advance()`](src/Feed/Cursor.php)), +so a stale instance concedes to a newer save, seek, or reset instead of +undoing it. The check is read-compare-write on every adapter, deliberately: +it needs nothing more than `load()` and `save()`, works on ids with no order, +and narrows a lost update to a single round trip, which at-least-once +delivery absorbs. + | Cursor | Use for | | --- | --- | | `Cursor\Cache` | A consumer with a [Utopia cache](https://github.com/utopia-php/cache) β€” the usual choice for a remote feed. Holds a position for 30 days (`Cursor\Cache::TTL`) unless saved again | diff --git a/src/Feed/Consumer.php b/src/Feed/Consumer.php index b5b8215..f09d5f9 100644 --- a/src/Feed/Consumer.php +++ b/src/Feed/Consumer.php @@ -90,8 +90,17 @@ public function consume(callable $handler): int } if ($processed !== null && $this->moved === $moved) { + $expected = $this->position; $this->position = $processed; - $this->cursor->save($this->feed->getName(), $this->name, $processed); + + // Conditional for the same reason as the $moved guard, but across + // instances: a save lands only if the position is still where this + // run started. Refused means another instance moved it β€” progress, + // a seek, or a reset β€” and that newer decision stands. + if (!$this->cursor->advance($this->feed->getName(), $this->name, $processed, $expected)) { + $this->position = null; + $this->restored = false; + } } if ($failure !== null) { diff --git a/src/Feed/Cursor.php b/src/Feed/Cursor.php index 61a0e0d..e2bd591 100644 --- a/src/Feed/Cursor.php +++ b/src/Feed/Cursor.php @@ -23,6 +23,30 @@ abstract public function save(string $feed, string $consumer, string $eventId): */ abstract public function reset(string $feed, string $consumer): void; + /** + * Save $eventId only if the stored position is still $expected β€” what the + * caller's run started from, or null for none. A refusal means another + * instance moved the position, and that newer decision stands. + * + * Equality is all it takes, so it works on ids with no order (a remote + * feed's UUIDs). The check is read-compare-write, not atomic: it narrows + * the window for a lost update from a whole run to one round trip, and + * what slips through costs a bounded replay, which at-least-once delivery + * absorbs anyway. + * + * @throws Exception When the store cannot be read or written. + */ + public function advance(string $feed, string $consumer, string $eventId, ?string $expected): bool + { + if ($this->load($feed, $consumer) !== $expected) { + return false; + } + + $this->save($feed, $consumer, $eventId); + + return true; + } + /** * The gate every cursor operation goes through, so no adapter builds a key * of its own and skips the check. diff --git a/src/Feed/Cursor/None.php b/src/Feed/Cursor/None.php index d5edb49..c8807a1 100644 --- a/src/Feed/Cursor/None.php +++ b/src/Feed/Cursor/None.php @@ -25,4 +25,12 @@ public function reset(string $feed, string $consumer): void { $this->key($feed, $consumer); } + + /** Nothing is stored, so nothing can conflict: the caller's own memory is the only record. */ + public function advance(string $feed, string $consumer, string $eventId, ?string $expected): bool + { + $this->key($feed, $consumer); + + return true; + } } diff --git a/src/Feed/Cursor/Pool.php b/src/Feed/Cursor/Pool.php index 363c830..d91f050 100644 --- a/src/Feed/Cursor/Pool.php +++ b/src/Feed/Cursor/Pool.php @@ -34,4 +34,9 @@ public function reset(string $feed, string $consumer): void (new Redis($redis))->reset($feed, $consumer); }); } + + public function advance(string $feed, string $consumer, string $eventId, ?string $expected): bool + { + return $this->pool->use(fn (\Redis|\RedisCluster $redis): bool => (new Redis($redis))->advance($feed, $consumer, $eventId, $expected)); + } } diff --git a/src/Feed/Cursor/Redis.php b/src/Feed/Cursor/Redis.php index fbd3923..61eca51 100644 --- a/src/Feed/Cursor/Redis.php +++ b/src/Feed/Cursor/Redis.php @@ -9,6 +9,11 @@ class Redis extends Cursor { + // advance() is inherited: its load() already reads a legacy stream key, + // and its save() converts one. Redis' own compare-and-set (WATCH/MULTI) + // is not used on purpose β€” this client is shared with the producer's + // store, and WATCH state does not survive a shared connection. + public function __construct(protected readonly \Redis|\RedisCluster $redis) { } diff --git a/src/Feed/Remote.php b/src/Feed/Remote.php index 6d813f2..2d5cc65 100644 --- a/src/Feed/Remote.php +++ b/src/Feed/Remote.php @@ -95,12 +95,21 @@ private function fetch(?string $lastEventId, int $limit, int $timeout): array ); } + $raw = (string) $response->getBody(); + try { - $body = \json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + $body = \json_decode($raw, true, flags: JSON_THROW_ON_ERROR); } catch (\JsonException $error) { throw new Transport("The {$this->name} feed returned a body that is not JSON: {$error->getMessage()}", previous: $error); } + // json_decode collapses `{}` and `[]` into the same empty array, and + // only one of them is a batch: an empty object is a non-feed answering, + // not "caught up". + if ($body === [] && !\str_starts_with(\ltrim($raw), '[')) { + throw new Invalid('Expected a feed batch, got an object'); + } + return self::decode($body); } diff --git a/tests/Feed/Consumer/Base.php b/tests/Feed/Consumer/Base.php index 95fefa9..da5ab0d 100644 --- a/tests/Feed/Consumer/Base.php +++ b/tests/Feed/Consumer/Base.php @@ -599,15 +599,16 @@ public function testTwoConsumersSharingANameSplitTheFeed(): void } /** - * And the cost of sharing a name, which no amount of coordination inside a - * single process can remove: each save is last-writer-wins, so a replica - * holding an older position drags the shared one backwards when it saves. - * At-least-once makes that a replay rather than a loss β€” but it is why - * every consumer gets its own name. + * And what sharing a name costs: a replica holding an older position + * re-handles events it polls (at-least-once), but its save is conditional + * on the position its run started from, so it cannot drag the shared + * position backwards β€” it concedes to the replica that got ahead. Every + * consumer still gets its own name; the fence bounds the damage, it does + * not split the feed cleanly. */ - public function testAStaleConsumerSharingANameDragsThePositionBackwards(): void + public function testAStaleConsumerSharingANameCannotDragThePositionBackwards(): void { - $first = $this->producer->produce('a'); + $this->producer->produce('a'); $second = $this->producer->produce('b'); $stale = $this->consumer(batch: 1); @@ -617,8 +618,8 @@ public function testAStaleConsumerSharingANameDragsThePositionBackwards(): void $this->assertSame(['a', 'b'], $this->drain($ahead)); $this->assertSame($second, $this->cursor->load($this->name, 'invalidator')); - $this->assertSame(['a'], $this->drain($stale), 'The stale replica polls from where it thought it was'); - $this->assertSame($first, $this->cursor->load($this->name, 'invalidator'), 'Its save wins, so the shared position regresses'); + $this->assertSame(['a'], $this->drain($stale), 'The stale replica re-handles from where it thought it was'); + $this->assertSame($second, $this->cursor->load($this->name, 'invalidator'), 'Its save is refused, so the shared position holds'); } public function testPositionIsNullBeforeTheFirstRun(): void diff --git a/tests/Feed/Cursor/Base.php b/tests/Feed/Cursor/Base.php index 9e7980a..d0bcf38 100644 --- a/tests/Feed/Cursor/Base.php +++ b/tests/Feed/Cursor/Base.php @@ -108,6 +108,34 @@ public function testResetOnlyForgetsTheConsumerItNames(): void $this->assertSame('2-0', $this->cursor->load($this->name, 'two'), 'The other consumer is untouched'); } + public function testAdvanceLandsWhileThePositionIsUnmoved(): void + { + $this->assertTrue($this->cursor->advance($this->name, 'invalidator', '1-0', null)); + $this->assertSame('1-0', $this->cursor->load($this->name, 'invalidator')); + + $this->assertTrue($this->cursor->advance($this->name, 'invalidator', '2-0', '1-0')); + $this->assertSame('2-0', $this->cursor->load($this->name, 'invalidator')); + } + + /** A save conditioned on a position that has since moved is refused: the newer decision stands. */ + public function testAdvanceIsRefusedWhenThePositionHasMoved(): void + { + $this->cursor->save($this->name, 'invalidator', '5-0'); + + $this->assertFalse($this->cursor->advance($this->name, 'invalidator', '3-0', '2-0')); + $this->assertFalse($this->cursor->advance($this->name, 'invalidator', '3-0', null), 'A run that started from nothing concedes too'); + $this->assertSame('5-0', $this->cursor->load($this->name, 'invalidator')); + } + + public function testAdvanceIsRefusedAfterAReset(): void + { + $this->cursor->save($this->name, 'invalidator', '5-0'); + $this->cursor->reset($this->name, 'invalidator'); + + $this->assertFalse($this->cursor->advance($this->name, 'invalidator', '6-0', '5-0')); + $this->assertNull($this->cursor->load($this->name, 'invalidator')); + } + /** * Refused by every operation, not just the first one a caller reaches β€” * which is what says the adapter goes through the shared key builder. @@ -137,6 +165,9 @@ public static function operationsAndNames(): array 'reset' => static function (Cursor $cursor, string $feed, string $consumer): void { $cursor->reset($feed, $consumer); }, + 'advance' => static function (Cursor $cursor, string $feed, string $consumer): void { + $cursor->advance($feed, $consumer, '1-0', null); + }, ]; $names = [ diff --git a/tests/Feed/Cursor/RedisTest.php b/tests/Feed/Cursor/RedisTest.php index 81ef33b..14dbf52 100644 --- a/tests/Feed/Cursor/RedisTest.php +++ b/tests/Feed/Cursor/RedisTest.php @@ -31,6 +31,19 @@ public function testSavingOverAStreamPositionConvertsTheKey(): void $this->assertSame('2-0', $this->cursor->load($this->name, 'invalidator')); } + /** advance() compares against a legacy stream position, and a landing one converts the key. */ + public function testAdvanceOverAStreamPositionComparesAndConverts(): void + { + $key = Key::cursor($this->name, 'invalidator'); + $this->redis()->xAdd($key, '1-0', ['p' => '1']); + + $this->assertFalse($this->cursor->advance($this->name, 'invalidator', '9-0', '0-5'), 'A stale expectation is refused against a stream too'); + + $this->assertTrue($this->cursor->advance($this->name, 'invalidator', '2-0', '1-0')); + $this->assertSame(\Redis::REDIS_STRING, $this->redis()->type($key)); + $this->assertSame('2-0', $this->cursor->load($this->name, 'invalidator')); + } + public function testResetForgetsAPositionRetainedAsAStream(): void { $this->redis()->xAdd(Key::cursor($this->name, 'invalidator'), '1-0', ['p' => '1']); diff --git a/tests/Feed/Unit/ConsumerTest.php b/tests/Feed/Unit/ConsumerTest.php index 4c97eb0..7ba4cb7 100644 --- a/tests/Feed/Unit/ConsumerTest.php +++ b/tests/Feed/Unit/ConsumerTest.php @@ -62,14 +62,71 @@ public function load(string $feed, string $consumer): ?string $consumer = $this->consumer($cursor); - // One read on the first pass to restore the position; the two - // caught-up polls after it must not touch the store at all, which is - // what keeps an idle consumer on a timer free. + // Two reads on the first pass β€” one restores the position, one is the + // conditional save's compare. The caught-up polls after it must not + // touch the store at all, which is what keeps an idle consumer on a + // timer free. $consumer->consume(fn (CloudEvent $event) => null); + $this->assertSame(2, $cursor->loads); + $consumer->consume(fn (CloudEvent $event) => null); $consumer->consume(fn (CloudEvent $event) => null); - $this->assertSame(1, $cursor->loads); + $this->assertSame(2, $cursor->loads, 'An idle poll costs no cursor reads'); + } + + /** + * Two instances sharing a name share one position. A save is conditional + * on the position the run started from, so the instance that fell behind + * cannot undo the other's progress β€” it adopts the newer position and + * continues from there. + */ + public function testAStaleInstanceCannotUndoAnotherInstancesProgress(): void + { + $one = $this->consumer(); + $two = $this->consumer(); + + // An empty first run restores "no position yet" on both instances... + $this->assertSame(0, $one->consume(fn (CloudEvent $event) => null)); + $this->assertSame(0, $two->consume(fn (CloudEvent $event) => null)); + + foreach (['a', 'b', 'c', 'd'] as $type) { + $this->producer->produce($type); + } + + // ...then the first instance gets ahead. + $one->consume(fn (CloudEvent $event) => null); + $ahead = $one->position(); + $this->assertNotNull($ahead); + + // The stale instance re-handles from the start (at-least-once), but + // its save is refused rather than moving the shared position back. + $two->consume(fn (CloudEvent $event) => null); + $this->assertSame($ahead, $this->cursor->load('edge', 'invalidator')); + + // Having conceded, it reloads the shared position and stays there. + $this->assertSame($ahead, $two->position()); + } + + /** A seek made through one instance survives another instance's in-flight run. */ + public function testAStaleInstanceCannotUndoAnotherInstancesSeek(): void + { + $running = $this->consumer(); + $operator = $this->consumer(); + + // An empty first run restores "no position yet" before the seek lands. + $this->assertSame(0, $running->consume(fn (CloudEvent $event) => null)); + + foreach (['a', 'b'] as $type) { + $this->producer->produce($type); + } + + $operator->seek('9999999999999-0'); + + $running->consume(fn (CloudEvent $event) => null); + + $this->assertSame('9999999999999-0', $this->cursor->load('edge', 'invalidator')); + $this->assertSame('9999999999999-0', $running->position(), 'The refused instance adopts the seek'); } /** diff --git a/tests/Feed/Unit/RemoteTest.php b/tests/Feed/Unit/RemoteTest.php index 24145d4..d4c8673 100644 --- a/tests/Feed/Unit/RemoteTest.php +++ b/tests/Feed/Unit/RemoteTest.php @@ -287,6 +287,20 @@ public function testARespondingEndpointThatIsNotAFeedIsNotMistakenForBeingCaught $remote->read(); } + /** + * The empty object is the one non-batch json_decode() cannot expose: `{}` + * and `[]` both decode to the same empty array, and only one of them + * means "caught up". + */ + public function testAnEmptyObjectIsNotMistakenForBeingCaughtUp(): void + { + [$remote] = $this->remote([FakeTransport::raw('{}')]); + + $this->expectException(Invalid::class); + + $remote->read(); + } + /** * @return array}> */