diff --git a/.github/actions/consul-start/action.yml b/.github/actions/consul-start/action.yml new file mode 100644 index 00000000..f6272129 --- /dev/null +++ b/.github/actions/consul-start/action.yml @@ -0,0 +1,13 @@ +name: "Start Consul" +runs: + using: "composite" + steps: + - uses: nahsi/setup-hashi-tool@v1 + if: github.repository == 'ordo-one/package-consul' || github.repository == 'ordo-one/package-distributed-system' + with: + name: consul + - name: Start consul + if: github.repository == 'ordo-one/package-consul' || github.repository == 'ordo-one/package-distributed-system' + shell: bash + run: | + consul agent -dev -log-level=warn & diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..4ef90ab2 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,13 @@ +## Description + +Include a summary of the change and which issue is fixed. Please also include relevant motivation and context. + +## How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. + +## Minimal checklist: + +- [ ] I have performed a self-review of my own code +- [ ] I have added `DocC` code-level documentation for any public interfaces exported by the package +- [ ] I have added unit and/or integration tests that prove my fix is effective or that my feature works diff --git a/.github/workflows/lint-pr.yaml b/.github/workflows/lint-pr.yaml new file mode 100644 index 00000000..c9f58cb6 --- /dev/null +++ b/.github/workflows/lint-pr.yaml @@ -0,0 +1,65 @@ +name: "PR title validation" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + name: Validate PR title +# runs-on: [self-hosted, ubuntu-latest] + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configure which types are allowed. + # Default: https://github.com/commitizen/conventional-commit-types + types: | + fix + feat + docs + style + refactor + perf + test + build + ci + chore + revert + # Configure which scopes are allowed. + scopes: | + patch + hotfix + minor + major + # Configure that a scope must always be provided. + requireScope: false + # Configure which scopes are disallowed in PR titles. For instance by setting + # the value below, `chore(release): ...` and `ci(e2e,release): ...` will be rejected. + disallowScopes: | + release + # Configure additional validation for the subject based on a regex. + # This example ensures the subject doesn't start with an uppercase character. + #subjectPattern: ^.*\[sc-[0-9]+\].*$ + #subjectPatternError: | + # The pull request title should contain Shortcut case number like '[sc-123]' + # If the PR contains one of these labels, the validation is skipped. + # Multiple labels can be separated by newlines. + # If you want to rerun the validation when labels change, you might want + # to use the `labeled` and `unlabeled` event triggers in your workflow. + ignoreLabels: | + bot + ignore-semantic-pull-request + # For work-in-progress PRs you can typically use draft pull requests + # from GitHub. However, private repositories on the free plan don't have + # this option and therefore this action allows you to opt-in to using the + # special "[WIP]" prefix to indicate this state. This will avoid the + # validation of the PR title and the pull request checks remain pending. + # Note that a second check will be reported if this is enabled. + wip: true diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/semantic-release.yml new file mode 100644 index 00000000..74c6ffad --- /dev/null +++ b/.github/workflows/semantic-release.yml @@ -0,0 +1,66 @@ +name: Semantic Release + +on: + workflow_dispatch: + push: + branches: [ main, next ] + +jobs: + semantic-release: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Semantic Release Config + run: | + echo " + branches: + - main + - name: 'next' + prerelease: 'ordoalpha' + preset: 'conventionalcommits' + tagFormat: '\${version}' + plugins: + - - '@semantic-release/commit-analyzer' + - releaseRules: + - breaking: true + release: major + - type: build + release: patch + - type: chore + release: false + - type: feat + release: false + - type: fix + release: false + + - scope: 'no-release' + release: false + - scope: 'hotfix' + release: patch + - scope: 'patch' + release: patch + - scope: 'minor' + release: minor + - scope: 'major' + release: major + - - '@semantic-release/release-notes-generator' + - - '@semantic-release/github' + - successComment: false + failTitle: false" > .releaserc.yml + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install semantic-release + run: | + npm install semantic-release@v24 conventional-changelog-conventionalcommits@v8 -D + npm list + - name: Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: npx semantic-release diff --git a/.github/workflows/swift-benchmark-delta.yml b/.github/workflows/swift-benchmark-delta.yml new file mode 100644 index 00000000..eba0be64 --- /dev/null +++ b/.github/workflows/swift-benchmark-delta.yml @@ -0,0 +1,91 @@ +name: Benchmark PR vs main + +on: + workflow_dispatch: + pull_request: + branches: [ main ] + +jobs: + benchmark-delta: + + runs-on: ${{ matrix.os }} + continue-on-error: true + + strategy: + matrix: + #os: [[Linux, benchmark-swift-latest, self-hosted]] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Homebrew Mac + if: ${{ runner.os == 'Macos' }} + run: | + echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH + brew install jemalloc + + - name: Ubuntu deps + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y libjemalloc-dev libsasl2-dev libcurl4-openssl-dev + + - name: Start consul + uses: ./.github/actions/consul-start + + - name: Git URL token override and misc + run: | + #git config --global url."https://ordo-ci:${{ secrets.CI_MACHINE_PAT }}@github.com".insteadOf "https://github.com" + #/usr/bin/ordo-performance + [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV + echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH + - name: Run benchmarks for PR branch + if: ${{ env.hasBenchmark == '1' }} + run: | + swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request --no-progress --quiet + - name: Switch to branch 'main' + if: ${{ env.hasBenchmark == '1' }} + run: | + git stash + git checkout main + - name: Run benchmarks for branch 'main' + if: ${{ env.hasBenchmark == '1' }} + run: | + swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main --no-progress --quiet + - name: Compare PR and main + if: ${{ env.hasBenchmark == '1' }} + id: benchmark + run: | + echo $(date) >> $GITHUB_STEP_SUMMARY + echo "exitStatus=1" >> $GITHUB_ENV + swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY + echo "exitStatus=0" >> $GITHUB_ENV + continue-on-error: true + - if: ${{ env.exitStatus == '0' }} + name: Pull request comment text success + id: prtestsuccess + run: | + echo 'PRTEST<> $GITHUB_ENV + echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/ordo-one/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - if: ${{ env.exitStatus == '1' }} + name: Pull request comment text failure + id: prtestfailure + run: | + echo 'PRTEST<> $GITHUB_ENV + echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/ordo-one/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV + echo "_Pull request had performance regressions_" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: Comment PR + if: ${{ env.hasBenchmark == '1' }} + uses: thollander/actions-comment-pull-request@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: ${{ env.PRTEST }} + comment_includes: "Pull request benchmark comparison [${{ matrix.os }}] with" + - name: Exit with correct status + run: | + #/usr/bin/ordo-performance powersave + exit ${{ env.exitStatus }} diff --git a/.github/workflows/swift-check-api-breaks.yml b/.github/workflows/swift-check-api-breaks.yml new file mode 100644 index 00000000..f9ecce3c --- /dev/null +++ b/.github/workflows/swift-check-api-breaks.yml @@ -0,0 +1,27 @@ +name: Swift check API breaks + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize] + +jobs: + analyze-api-breakage: + + runs-on: [ubuntu-latest] + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Ubuntu deps + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y libjemalloc-dev libsasl2-dev libcurl4-openssl-dev + - name: Build + run: swift build + - name: Analyze API breakage + run: swift package diagnose-api-breaking-changes origin/next diff --git a/.github/workflows/swift-code-coverage.yml b/.github/workflows/swift-code-coverage.yml new file mode 100644 index 00000000..28997e64 --- /dev/null +++ b/.github/workflows/swift-code-coverage.yml @@ -0,0 +1,48 @@ +name: Swift code coverage + +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main ] +jobs: + test-code-coverage: + runs-on: [ubuntu-22.04] + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + with: + submodules: 'true' + + - name: Ubuntu deps + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y libjemalloc-dev libsasl2-dev libcurl4-openssl-dev llvm-15 + + - name: Start consul + uses: ./.github/actions/consul-start + + - name: Run tests + continue-on-error: true + run: | + [ -d Tests ] && swift test --parallel --enable-code-coverage + + - name: Export code coverage + run: | + xctest_binary=".build/debug/${{ github.event.repository.name }}PackageTests.xctest" + if [ ! -f ${xctest_binary} ]; then + xctest_binary=$(find .build/debug/ -type f -name "*.xctest" | tail -1) + echo "Will llvm-cov '${xctest_binary}'" + fi + + if [ -f ${xctest_binary} ]; then + llvm-cov-15 export -format="lcov" ${xctest_binary} -instr-profile .build/debug/codecov/default.profdata > info.lcov + fi + + - name: Upload codecov + uses: codecov/codecov-action@v2 + with: + token: ${{ secrets.CODECOV_REPO_TOKEN }} + files: info.lcov + fail_ci_if_error: true diff --git a/.github/workflows/swift-lint.yml b/.github/workflows/swift-lint.yml new file mode 100644 index 00000000..210183be --- /dev/null +++ b/.github/workflows/swift-lint.yml @@ -0,0 +1,20 @@ +name: Swift lint + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/swiftlint.yml' + - '.swiftlint.yml' + - '**/*.swift' + +jobs: + SwiftLint: + timeout-minutes: 60 + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v3 + - name: GitHub Action for SwiftLint with --strict + uses: norio-nomura/action-swiftlint@3.2.1 + with: + args: --strict diff --git a/.github/workflows/swift-linux-build.yml b/.github/workflows/swift-linux-build.yml new file mode 100644 index 00000000..444e7483 --- /dev/null +++ b/.github/workflows/swift-linux-build.yml @@ -0,0 +1,44 @@ +name: Swift Linux build + +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main, next ] + +jobs: + build-linux: + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + swift: ["5.9"] + + + runs-on: ${{ matrix.os }} + steps: + - uses: fwal/setup-swift@v1 + if: ${{ false }} + with: + swift-version: ${{ matrix.swift }} + + - uses: actions/checkout@v3 + with: + submodules: 'true' + + - name: Start consul + uses: ./.github/actions/consul-start + + - name: Ubuntu deps + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y libjemalloc-dev libsasl2-dev libcurl4-openssl-dev + + - name: Swift version + run: swift --version + + - name: Run tests + continue-on-error: true + run: docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.2204.59.yaml run test diff --git a/.github/workflows/swift-macos-build.yml b/.github/workflows/swift-macos-build.yml new file mode 100644 index 00000000..dc736557 --- /dev/null +++ b/.github/workflows/swift-macos-build.yml @@ -0,0 +1,55 @@ +name: Swift macOS build + +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main, next ] + +jobs: + build-macos: + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: [macos-13] + swift: ["5.9"] + + runs-on: ${{ matrix.os }} + + steps: + - uses: fwal/setup-swift@v1 + with: + swift-version: ${{ matrix.swift }} + + - name: Homebrew Mac + if: ${{ runner.os == 'Macos' }} + run: | + echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH + echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV + brew install jemalloc + + - uses: actions/checkout@v3 + with: + submodules: 'true' + + - name: Start consul + uses: ./.github/actions/consul-start + + - name: Swift version + run: swift --version + - name: Build + run: swift build + - name: Run tests + if: ${{ false }} + run: | + if [ -d Tests ]; then + swift test --parallel + fi + - name: Run tests (release) + if: ${{ false }} + run: | + if [ -d Tests ]; then + swift test -c release --parallel + fi diff --git a/.github/workflows/swift-outdated-dependencies.yml b/.github/workflows/swift-outdated-dependencies.yml new file mode 100644 index 00000000..8d45ba5c --- /dev/null +++ b/.github/workflows/swift-outdated-dependencies.yml @@ -0,0 +1,28 @@ +name: Swift outdated dependencies + +on: + workflow_dispatch: + schedule: + - cron: '0 8 */100,1-7 * MON' # First Monday of the month + +jobs: + spm-dep-check: + runs-on: [ubuntu-latest] + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + - name: Check Swift package dependencies + id: spm-dep-check + uses: MarcoEidinger/swift-package-dependencies-check@2.3.4 + with: + isMutating: true + failWhenOutdated: false + - name: Create Pull Request + if: steps.spm-dep-check.outputs.outdatedDependencies == 'true' + uses: peter-evans/create-pull-request@v3 + with: + commit-message: 'chore: update package dependencies' + branch: updatePackageDepedencies + delete-branch: true + title: 'chore: update package dependencies' + body: ${{ steps.spm-dep-check.outputs.releaseNotes }} diff --git a/.github/workflows/swift-sanitizer-address.yml b/.github/workflows/swift-sanitizer-address.yml new file mode 100644 index 00000000..46ab1c5b --- /dev/null +++ b/.github/workflows/swift-sanitizer-address.yml @@ -0,0 +1,58 @@ +name: Address sanitizer + +on: + workflow_dispatch: +# push: +# branches: [ main ] +# pull_request: +# branches: [ main, next ] + +jobs: + address-sanitizer: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + swift: ["5.9"] + + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - uses: fwal/setup-swift@v1 + with: + swift-version: ${{ matrix.swift }} + + - name: Homebrew Mac + if: ${{ runner.os == 'Macos' }} + run: | + echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH + echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV + brew install jemalloc + + - name: Ubuntu deps + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y libjemalloc-dev libsasl2-dev libcurl4-openssl-dev + + - uses: actions/checkout@v3 + with: + submodules: 'true' + + - name: Start consul + uses: ./.github/actions/consul-start + + - name: Swift version + run: swift --version + + # Required to clean build directory before sanitizer! + - name: Clean before debug build sanitizier + run: swift package clean + + - name: Run address sanitizer + run: swift test --sanitize=address + + - name: Clean before release build sanitizier + run: swift package clean + + - name: Run address sanitizer on release build + run: swift test --sanitize=address -c release -Xswiftc -enable-testing diff --git a/.github/workflows/swift-sanitizer-thread.yml b/.github/workflows/swift-sanitizer-thread.yml new file mode 100644 index 00000000..50b5b050 --- /dev/null +++ b/.github/workflows/swift-sanitizer-thread.yml @@ -0,0 +1,60 @@ +name: Thread sanitizer + +on: + workflow_dispatch: +# push: +# branches: [ main ] +# pull_request: +# branches: [ main, next ] + +jobs: + thread-sanitizer: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + swift: ["5.9"] + + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - uses: fwal/setup-swift@v1 + with: + swift-version: ${{ matrix.swift }} + + + - name: Homebrew Mac + if: ${{ runner.os == 'Macos' }} + run: | + echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH + echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV + brew install jemalloc + + - name: Ubuntu deps + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y libjemalloc-dev libsasl2-dev libcurl4-openssl-dev + echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV + + - uses: actions/checkout@v3 + with: + submodules: 'true' + + - name: Start consul + uses: ./.github/actions/consul-start + + - name: Swift version + run: swift --version + + # Required to clean build directory before sanitizer! + - name: Clean before debug build thread sanitizier + run: swift package clean + + - name: Run thread sanitizer + run: swift test --sanitize=thread + + - name: Clean before release build sanitizier + run: swift package clean + + - name: Run thread sanitizer on release build + run: swift test --sanitize=thread -c release -Xswiftc -enable-testing diff --git a/.gitignore b/.gitignore index 734831f0..0dea6af5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,100 @@ -.DS_Store -/.build -/Packages -/*.xcodeproj +#################################################################### +# DO NOT EDIT THIS FILE MANUALLY +# This is a master file maintained in https://github.com/ordo-one/repository-templates + +# Xcode +# +# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore + +## User settings xcuserdata/ + +## Benchmark data +.benchmarkBaselines/ + +## DocC build directories +**/.docc-build/ + +## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) +*.xcscmblueprint +*.xccheckout + +## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) +build/ DerivedData/ -.netrc -Package.resolved -.*.sw? +*.moved-aside +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 + +## Obj-C/Swift specific +*.hmap + +## App packaging +*.ipa +*.dSYM.zip +*.dSYM + +## Playgrounds +timeline.xctimeline +playground.xcworkspace + +# Swift Package Manager +# +# Add this line if you want to avoid checking in source code from Swift Package Manager dependencies. +# Packages/ +# Package.pins +# Package.resolved +# *.xcodeproj +# +# Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata +# hence it is not needed unless you have added a package configuration file to your project .swiftpm +.DS_Store +.build/ + +# CocoaPods +# +# We recommend against adding the Pods directory to your .gitignore. However +# you should judge for yourself, the pros and cons are mentioned at: +# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control +# +# Pods/ +# +# Add this line if you want to avoid checking in source code from the Xcode workspace +# *.xcworkspace + +# Carthage +# +# Add this line if you want to avoid checking in source code from Carthage dependencies. +# Carthage/Checkouts + +Carthage/Build/ + +# Accio dependency management +Dependencies/ +.accio/ + +# fastlane +# +# It is recommended to not store the screenshots in the git repo. +# Instead, use fastlane to re-generate the screenshots whenever they are needed. +# For more information about the recommended setup visit: +# https://docs.fastlane.tools/best-practices/source-control/#source-control + +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots/**/*.png +fastlane/test_output + +# Code Injection +# +# After new code Injection tools there's a generated folder /iOSInjectionProject +# https://github.com/johnno1962/injectionforxcode + +iOSInjectionProject/ diff --git a/.gitmodules b/.gitmodules index 111e6a87..139cff53 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "Sources/Crdkafka/librdkafka"] path = Sources/Crdkafka/librdkafka - url = https://github.com/confluentinc/librdkafka + url = https://github.com/ordo-one/librdkafka diff --git a/.swift-version b/.swift-version new file mode 100644 index 00000000..64ff7ded --- /dev/null +++ b/.swift-version @@ -0,0 +1 @@ +5.7.1 diff --git a/.swiftformat b/.swiftformat index 4f7cf860..fcc6fc7e 100644 --- a/.swiftformat +++ b/.swiftformat @@ -1,25 +1,5 @@ -# file options - ---swiftversion 5.4 ---exclude .build - -# format options - ---self insert ---patternlet inline ---ranges nospace ---stripunusedargs unnamed-only ---ifdef no-indent ---extensionacl on-declarations ---disable typeSugar # https://github.com/nicklockwood/SwiftFormat/issues/636 ---disable andOperator +#################################################################### +# DO NOT EDIT THIS FILE +--decimalgrouping 3,4 --disable wrapMultilineStatementBraces ---disable enumNamespaces ---disable redundantExtensionACL ---disable redundantReturn ---disable preferKeyPath ---disable sortedSwitchCases ---disable hoistTry ---disable hoistAwait - -# rules +--disable trailingCommas diff --git a/.swiftlint.yml b/.swiftlint.yml new file mode 100644 index 00000000..347c8e55 --- /dev/null +++ b/.swiftlint.yml @@ -0,0 +1,93 @@ +#################################################################### +# DO NOT EDIT THIS FILE +# This is a master file maintained in https://github.com/ordo-one/public-repository-templates +# +# Add overrides into `swiftlint_refinement.yml` in the directory it self or +# .swiftlint.yml under the specific directory where the code is located +# +# Documentation is under https://github.com/realm/SwiftLint +#################################################################### + +included: + - Benchmarks + - Sources + - Tests +excluded: +analyzer_rules: + - unused_import +opt_in_rules: + - array_init + - attributes + - closure_end_indentation + - closure_spacing + - collection_alignment + - contains_over_filter_count + - contains_over_filter_is_empty + - contains_over_first_not_nil + - contains_over_range_nil_comparison + - discouraged_none_name + - discouraged_object_literal + - empty_collection_literal + - empty_count + - empty_string + - empty_xctest_method + - enum_case_associated_values_count + - explicit_init + - extension_access_modifier + - fallthrough + - fatal_error_message + - file_name + - first_where + - flatmap_over_map_reduce + - identical_operands + - joined_default_parameter + - last_where + - legacy_multiple + - literal_expression_end_indentation + - lower_acl_than_parent + - modifier_order + - nimble_operator + - nslocalizedstring_key + - number_separator + - object_literal + - operator_usage_whitespace + - overridden_super_call + - override_in_extension + - pattern_matching_keywords + - prefer_self_in_static_references + - prefer_self_type_over_type_of_self + - private_action + - private_outlet + - prohibited_interface_builder + - prohibited_super_call + - quick_discouraged_call + - quick_discouraged_focused_test + - quick_discouraged_pending_test + - reduce_into + - redundant_nil_coalescing + - redundant_type_annotation + - single_test_class + - sorted_first_last + - sorted_imports + - static_operator + - strong_iboutlet + - test_case_accessibility + - toggle_bool + - unavailable_function + - unneeded_parentheses_in_closure_argument + - unowned_variable_capture + - untyped_error_in_catch + - vertical_parameter_alignment_on_call + - vertical_whitespace_closing_braces + - vertical_whitespace_opening_braces + - xct_specific_matcher + - yoda_condition +line_length: + warning: 140 + error: 140 + ignores_comments: true + ignores_urls: true + ignores_function_declarations: true + ignores_interpolated_strings: true +identifier_name: + excluded: [id, i, j, k] diff --git a/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift b/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift new file mode 100644 index 00000000..ec0bff9a --- /dev/null +++ b/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift @@ -0,0 +1,345 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Crdkafka +import Dispatch +import struct Foundation.Date +import struct Foundation.UUID +import Kafka +import Logging +import ServiceLifecycle + +let benchmarks = { + var uniqueTestTopic: String! + let messageCount: UInt = 1000 + + Benchmark.defaultConfiguration = .init( + metrics: [ + .wallClock, + .cpuTotal, + .contextSwitches, + .throughput, + .allocatedResidentMemory, + ] + .arc, + warmupIterations: 0, + scalingFactor: .one, + maxDuration: .seconds(5), + maxIterations: 100, + thresholds: [ + .wallClock: .init(relative: [.p90: 35]), + .cpuTotal: .init(relative: [.p90: 35]), + .allocatedResidentMemory: .init(relative: [.p90: 20]), + .contextSwitches: .init(relative: [.p90: 35]), + .throughput: .init(relative: [.p90: 35]), + .objectAllocCount: .init(relative: [.p90: 20]), + .retainCount: .init(relative: [.p90: 20]), + .releaseCount: .init(relative: [.p90: 20]), + .retainReleaseDelta: .init(relative: [.p90: 20]), + ] + ) + + Benchmark.setup = { + uniqueTestTopic = try await prepareTopic(messagesCount: messageCount, partitions: 6) + } + + Benchmark.teardown = { + if let uniqueTestTopic { + try deleteTopic(uniqueTestTopic) + } + uniqueTestTopic = nil + } + + Benchmark("SwiftKafkaConsumer_basic_consumer_messages_\(messageCount)") { benchmark in + let uniqueGroupID = UUID().uuidString + var consumerConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group( + id: uniqueGroupID, + topics: [uniqueTestTopic] + ), + bootstrapBrokerAddresses: [brokerAddress] + ) + consumerConfig.autoOffsetReset = .beginning + consumerConfig.broker.addressFamily = .v4 + // We must specify it at least 10 otherwise CI will timeout + consumerConfig.pollInterval = .milliseconds(1) + + let consumer = try KafkaConsumer( + configuration: consumerConfig, + logger: .perfLogger + ) + + let serviceGroupConfiguration = ServiceGroupConfiguration(services: [consumer], gracefulShutdownSignals: [.sigterm, .sigint], logger: .perfLogger) + let serviceGroup = ServiceGroup(configuration: serviceGroupConfiguration) + + try await withThrowingTaskGroup(of: Void.self) { group in + benchLog("Start consuming") + defer { + benchLog("Finish consuming") + } + // Run Task + group.addTask { + try await serviceGroup.run() + } + + // Second Consumer Task + group.addTask { + var ctr: UInt64 = 0 + var tmpCtr: UInt64 = 0 + let interval: UInt64 = Swift.max(UInt64(messageCount / 20), 1) + let totalStartDate = Date.timeIntervalSinceReferenceDate + var totalBytes: UInt64 = 0 + + try await benchmark.withMeasurement { + for try await record in consumer.messages { + ctr += 1 + totalBytes += UInt64(record.value.readableBytes) + + tmpCtr += 1 + if tmpCtr >= interval { + benchLog("read \(ctr * 100 / UInt64(messageCount))%") + tmpCtr = 0 + } + if ctr >= messageCount { + break + } + } + } + + let timeIntervalTotal = Date.timeIntervalSinceReferenceDate - totalStartDate + let avgRateMb = Double(totalBytes) / timeIntervalTotal / 1024 + benchLog("All read up to ctr: \(ctr), avgRate: (\(Int(avgRateMb))KB/s), timePassed: \(Int(timeIntervalTotal))sec") + } + + // Wait for second Consumer Task to complete + try await group.next() + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + } + + Benchmark("SwiftKafkaConsumer_with_offset_commit_messages_\(messageCount)") { benchmark in + let uniqueGroupID = UUID().uuidString + var consumerConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group( + id: uniqueGroupID, + topics: [uniqueTestTopic] + ), + bootstrapBrokerAddresses: [brokerAddress] + ) + consumerConfig.autoOffsetReset = .beginning + consumerConfig.broker.addressFamily = .v4 + consumerConfig.isAutoCommitEnabled = false + // We must specify it at least 10 otherwise CI will timeout + consumerConfig.pollInterval = .milliseconds(1) + + let consumer = try KafkaConsumer( + configuration: consumerConfig, + logger: .perfLogger + ) + + let serviceGroupConfiguration = ServiceGroupConfiguration(services: [consumer], gracefulShutdownSignals: [.sigterm, .sigint], logger: .perfLogger) + let serviceGroup = ServiceGroup(configuration: serviceGroupConfiguration) + + try await withThrowingTaskGroup(of: Void.self) { group in + benchLog("Start consuming") + defer { + benchLog("Finish consuming") + } + // Run Task + group.addTask { + try await serviceGroup.run() + } + + // Second Consumer Task + group.addTask { + var ctr: UInt64 = 0 + var tmpCtr: UInt64 = 0 + let interval: UInt64 = Swift.max(UInt64(messageCount / 20), 1) + let totalStartDate = Date.timeIntervalSinceReferenceDate + var totalBytes: UInt64 = 0 + + try await benchmark.withMeasurement { + for try await record in consumer.messages { + try consumer.scheduleCommit(record) + + ctr += 1 + totalBytes += UInt64(record.value.readableBytes) + + tmpCtr += 1 + if tmpCtr >= interval { + benchLog("read \(ctr * 100 / UInt64(messageCount))%") + tmpCtr = 0 + } + if ctr >= messageCount { + break + } + } + } + + let timeIntervalTotal = Date.timeIntervalSinceReferenceDate - totalStartDate + let avgRateMb = Double(totalBytes) / timeIntervalTotal / 1024 + benchLog("All read up to ctr: \(ctr), avgRate: (\(Int(avgRateMb))KB/s), timePassed: \(Int(timeIntervalTotal))sec") + } + + // Wait for second Consumer Task to complete + try await group.next() + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + } + + Benchmark("librdkafka_basic_consumer_messages_\(messageCount)") { benchmark in + let uniqueGroupID = UUID().uuidString + let rdKafkaConsumerConfig: [String: String] = [ + "group.id": uniqueGroupID, + "bootstrap.servers": "\(brokerAddress.host):\(brokerAddress.port)", + "broker.address.family": "v4", + "auto.offset.reset": "beginning", + ] + + let configPointer: OpaquePointer = rd_kafka_conf_new() + for (key, value) in rdKafkaConsumerConfig { + precondition(rd_kafka_conf_set(configPointer, key, value, nil, 0) == RD_KAFKA_CONF_OK) + } + + let kafkaHandle = rd_kafka_new(RD_KAFKA_CONSUMER, configPointer, nil, 0) + guard let kafkaHandle else { + preconditionFailure("Kafka handle was not created") + } + defer { + rd_kafka_destroy(kafkaHandle) + } + + rd_kafka_poll_set_consumer(kafkaHandle) + let subscriptionList = rd_kafka_topic_partition_list_new(1) + defer { + rd_kafka_topic_partition_list_destroy(subscriptionList) + } + rd_kafka_topic_partition_list_add( + subscriptionList, + uniqueTestTopic, + RD_KAFKA_PARTITION_UA + ) + rd_kafka_subscribe(kafkaHandle, subscriptionList) + rd_kafka_poll(kafkaHandle, 0) + + var ctr: UInt64 = 0 + var tmpCtr: UInt64 = 0 + + let interval: UInt64 = Swift.max(UInt64(messageCount / 20), 1) + let totalStartDate = Date.timeIntervalSinceReferenceDate + var totalBytes: UInt64 = 0 + + benchmark.withMeasurement { + while ctr < messageCount { + guard let record = rd_kafka_consumer_poll(kafkaHandle, 10) else { + continue + } + defer { + rd_kafka_message_destroy(record) + } + ctr += 1 + totalBytes += UInt64(record.pointee.len) + + tmpCtr += 1 + if tmpCtr >= interval { + benchLog("read \(ctr * 100 / UInt64(messageCount))%") + tmpCtr = 0 + } + } + } + + rd_kafka_consumer_close(kafkaHandle) + + let timeIntervalTotal = Date.timeIntervalSinceReferenceDate - totalStartDate + let avgRateMb = Double(totalBytes) / timeIntervalTotal / 1024 + benchLog("All read up to ctr: \(ctr), avgRate: (\(Int(avgRateMb))KB/s), timePassed: \(Int(timeIntervalTotal))sec") + } + + Benchmark("librdkafka_with_offset_commit_messages_\(messageCount)") { benchmark in + let uniqueGroupID = UUID().uuidString + let rdKafkaConsumerConfig: [String: String] = [ + "group.id": uniqueGroupID, + "bootstrap.servers": "\(brokerAddress.host):\(brokerAddress.port)", + "broker.address.family": "v4", + "auto.offset.reset": "beginning", + "enable.auto.commit": "false", + ] + + let configPointer: OpaquePointer = rd_kafka_conf_new() + for (key, value) in rdKafkaConsumerConfig { + precondition(rd_kafka_conf_set(configPointer, key, value, nil, 0) == RD_KAFKA_CONF_OK) + } + + let kafkaHandle = rd_kafka_new(RD_KAFKA_CONSUMER, configPointer, nil, 0) + guard let kafkaHandle else { + preconditionFailure("Kafka handle was not created") + } + defer { + rd_kafka_destroy(kafkaHandle) + } + + rd_kafka_poll_set_consumer(kafkaHandle) + let subscriptionList = rd_kafka_topic_partition_list_new(1) + defer { + rd_kafka_topic_partition_list_destroy(subscriptionList) + } + rd_kafka_topic_partition_list_add( + subscriptionList, + uniqueTestTopic, + RD_KAFKA_PARTITION_UA + ) + rd_kafka_subscribe(kafkaHandle, subscriptionList) + rd_kafka_poll(kafkaHandle, 0) + + var ctr: UInt64 = 0 + var tmpCtr: UInt64 = 0 + + let interval: UInt64 = Swift.max(UInt64(messageCount / 20), 1) + let totalStartDate = Date.timeIntervalSinceReferenceDate + var totalBytes: UInt64 = 0 + + benchmark.withMeasurement { + while ctr < messageCount { + guard let record = rd_kafka_consumer_poll(kafkaHandle, 10) else { + continue + } + defer { + rd_kafka_message_destroy(record) + } + guard record.pointee.err != RD_KAFKA_RESP_ERR__PARTITION_EOF else { + continue + } + let result = rd_kafka_commit_message(kafkaHandle, record, 0) + precondition(result == RD_KAFKA_RESP_ERR_NO_ERROR) + + ctr += 1 + totalBytes += UInt64(record.pointee.len) + + tmpCtr += 1 + if tmpCtr >= interval { + benchLog("read \(ctr * 100 / UInt64(messageCount))%") + tmpCtr = 0 + } + } + } + + rd_kafka_consumer_close(kafkaHandle) + + let timeIntervalTotal = Date.timeIntervalSinceReferenceDate - totalStartDate + let avgRateMb = Double(totalBytes) / timeIntervalTotal / 1024 + benchLog("All read up to ctr: \(ctr), avgRate: (\(Int(avgRateMb))KB/s), timePassed: \(Int(timeIntervalTotal))sec") + } +} diff --git a/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/Utilities.swift b/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/Utilities.swift new file mode 100644 index 00000000..304dc1fb --- /dev/null +++ b/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/Utilities.swift @@ -0,0 +1,128 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2022 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import class Foundation.ProcessInfo +import struct Foundation.UUID +import Kafka +@_spi(Internal) import Kafka +import Logging +import ServiceLifecycle + +let brokerAddress = KafkaConfiguration.BrokerAddress( + host: ProcessInfo.processInfo.environment["KAFKA_HOST"] ?? "localhost", + port: 9092 +) + +extension Logger { + static let perfLogger = { + var logger = Logger(label: "perf logger") + logger.logLevel = .critical + return logger + }() +} + +// For perf tests debugging +func benchLog(_ log: @autoclosure () -> Logger.Message) { + #if DEBUG + Logger.perfLogger.info(log()) + #endif +} + +func createTopic(partitions: Int32) throws -> String { + var basicConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: "no-group", topics: []), + bootstrapBrokerAddresses: [brokerAddress] + ) + basicConfig.broker.addressFamily = .v4 + + let client = try RDKafkaClient.makeClientForTopics(config: basicConfig, logger: .perfLogger) + return try client._createUniqueTopic(partitions: partitions, timeout: 10 * 1000) +} + +func deleteTopic(_ topic: String) throws { + var basicConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: "no-group", topics: []), + bootstrapBrokerAddresses: [brokerAddress] + ) + basicConfig.broker.addressFamily = .v4 + + let client = try RDKafkaClient.makeClientForTopics(config: basicConfig, logger: .perfLogger) + try client._deleteTopic(topic, timeout: 10 * 1000) +} + +func prepareTopic(messagesCount: UInt, partitions: Int32 = -1, logger: Logger = .perfLogger) async throws -> String { + let uniqueTestTopic = try createTopic(partitions: partitions) + + benchLog("Created topic \(uniqueTestTopic)") + + benchLog("Generating \(messagesCount) messages") + let testMessages = _createTestMessages(topic: uniqueTestTopic, count: messagesCount) + benchLog("Finish generating \(messagesCount) messages") + + var producerConfig = KafkaProducerConfiguration(bootstrapBrokerAddresses: [brokerAddress]) + producerConfig.broker.addressFamily = .v4 + + let (producer, acks) = try KafkaProducer.makeProducerWithEvents(configuration: producerConfig, logger: logger) + + let serviceGroupConfiguration = ServiceGroupConfiguration(services: [producer], gracefulShutdownSignals: [.sigterm, .sigint], logger: logger) + let serviceGroup = ServiceGroup(configuration: serviceGroupConfiguration) + + try await withThrowingTaskGroup(of: Void.self) { group in + benchLog("Start producing \(messagesCount) messages") + defer { + benchLog("Finish producing") + } + // Run Task + group.addTask { + try await serviceGroup.run() + } + + // Producer Task + group.addTask { + try await _sendAndAcknowledgeMessages( + producer: producer, + events: acks, + messages: testMessages, + skipConsistencyCheck: true + ) + } + + // Wait for Producer Task to complete + try await group.next() + await serviceGroup.triggerGracefulShutdown() + } + + return uniqueTestTopic +} + +extension Benchmark { + @discardableResult + func withMeasurement(_ body: () throws -> T) rethrows -> T { + self.startMeasurement() + defer { + self.stopMeasurement() + } + return try body() + } + + @discardableResult + func withMeasurement(_ body: () async throws -> T) async rethrows -> T { + self.startMeasurement() + defer { + self.stopMeasurement() + } + return try await body() + } +} diff --git a/Benchmarks/Benchmarks/SwiftKafkaProducerBenchmarks/KafkaProducerBenchmark.swift b/Benchmarks/Benchmarks/SwiftKafkaProducerBenchmarks/KafkaProducerBenchmark.swift new file mode 100644 index 00000000..1971d9e0 --- /dev/null +++ b/Benchmarks/Benchmarks/SwiftKafkaProducerBenchmarks/KafkaProducerBenchmark.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Crdkafka +import Kafka + +let benchmarks = { + Benchmark.defaultConfiguration = .init( + metrics: [.wallClock, .cpuTotal, .allocatedResidentMemory, .contextSwitches, .throughput] + .arc, + warmupIterations: 0, + scalingFactor: .one, + maxDuration: .seconds(5), + maxIterations: 100, + thresholds: [ + // Thresholds are wild guess mostly. Have to adjust with time. + .wallClock: .init(relative: [.p90: 10]), + .cpuTotal: .init(relative: [.p90: 10]), + .allocatedResidentMemory: .init(relative: [.p90: 20]), + .contextSwitches: .init(relative: [.p90: 10]), + .throughput: .init(relative: [.p90: 10]), + .objectAllocCount: .init(relative: [.p90: 10]), + .retainCount: .init(relative: [.p90: 10]), + .releaseCount: .init(relative: [.p90: 10]), + .retainReleaseDelta: .init(relative: [.p90: 10]), + ] + ) + + Benchmark.setup = {} + + Benchmark.teardown = {} +} diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift new file mode 100644 index 00000000..4ea81f8c --- /dev/null +++ b/Benchmarks/Package.swift @@ -0,0 +1,51 @@ +// swift-tools-version: 5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import PackageDescription + +let package = Package( + name: "benchmarks", + platforms: [ + .macOS(.v13), + ], + dependencies: [ + .package(path: "../"), + .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.22.3"), + ], + targets: [ + .executableTarget( + name: "SwiftKafkaConsumerBenchmarks", + dependencies: [ + .product(name: "Benchmark", package: "package-benchmark"), + .product(name: "Kafka", package: "swift-kafka-client"), + ], + path: "Benchmarks/SwiftKafkaConsumerBenchmarks", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark"), + ] + ), + .executableTarget( + name: "SwiftKafkaProducerBenchmarks", + dependencies: [ + .product(name: "Benchmark", package: "package-benchmark"), + .product(name: "Kafka", package: "swift-kafka-client"), + ], + path: "Benchmarks/SwiftKafkaProducerBenchmarks", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark"), + ] + ), + ] +) diff --git a/Package.swift b/Package.swift index 8d463327..8490fec1 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.7 +// swift-tools-version: 5.9 //===----------------------------------------------------------------------===// // // This source file is part of the swift-kafka-client open source project @@ -29,8 +29,8 @@ let rdkafkaExclude = [ let package = Package( name: "swift-kafka-client", platforms: [ - .macOS(.v13), - .iOS(.v16), + .macOS(.v14), + .iOS(.v17), .watchOS(.v9), .tvOS(.v16), ], @@ -45,10 +45,11 @@ let package = Package( ), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.55.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.62.0"), .package(url: "https://github.com/apple/swift-nio-ssl", from: "2.25.0"), .package(url: "https://github.com/swift-server/swift-service-lifecycle.git", from: "2.1.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"), + .package(url: "https://github.com/apple/swift-metrics", from: "2.4.1"), // The zstd Swift package produces warnings that we cannot resolve: // https://github.com/facebook/zstd/issues/3328 .package(url: "https://github.com/facebook/zstd.git", from: "1.5.0"), @@ -71,8 +72,8 @@ let package = Package( .define("_GNU_SOURCE", to: "1"), // Fix build error for Swift 5.9 onwards ], linkerSettings: [ - .linkedLibrary("curl"), - .linkedLibrary("sasl2"), + .linkedLibrary("curl", .when(platforms: [.macOS, .linux])), + .linkedLibrary("sasl2", .when(platforms: [.macOS, .linux])), .linkedLibrary("z"), // zlib ] ), @@ -83,6 +84,7 @@ let package = Package( .product(name: "NIOCore", package: "swift-nio"), .product(name: "ServiceLifecycle", package: "swift-service-lifecycle"), .product(name: "Logging", package: "swift-log"), + .product(name: "Metrics", package: "swift-metrics"), ] ), .target( @@ -93,7 +95,10 @@ let package = Package( ), .testTarget( name: "KafkaTests", - dependencies: ["Kafka"] + dependencies: [ + "Kafka", + .product(name: "MetricsTestKit", package: "swift-metrics"), + ] ), .testTarget( name: "IntegrationTests", diff --git a/Sources/Crdkafka/custom/include/openssl/ssl.h b/Sources/Crdkafka/custom/include/openssl/ssl.h index 634d88d4..c399c7d3 100644 --- a/Sources/Crdkafka/custom/include/openssl/ssl.h +++ b/Sources/Crdkafka/custom/include/openssl/ssl.h @@ -13,3 +13,5 @@ //===----------------------------------------------------------------------===// #include "CNIOBoringSSL_ssl.h" + +#define RAND_priv_bytes(buf, num) RAND_bytes((buf), (num)) diff --git a/Sources/Crdkafka/librdkafka b/Sources/Crdkafka/librdkafka index c282ba24..f5811a7e 160000 --- a/Sources/Crdkafka/librdkafka +++ b/Sources/Crdkafka/librdkafka @@ -1 +1 @@ -Subproject commit c282ba2423b2694052393c8edb0399a5ef471b3f +Subproject commit f5811a7ed5bc5fb9eb268cbf0c623af01f8e444c diff --git a/Sources/Kafka/Configuration/KafkaConfiguration+Metrics.swift b/Sources/Kafka/Configuration/KafkaConfiguration+Metrics.swift new file mode 100644 index 00000000..e9878c99 --- /dev/null +++ b/Sources/Kafka/Configuration/KafkaConfiguration+Metrics.swift @@ -0,0 +1,146 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Metrics + +extension KafkaConfiguration { + // MARK: - Metrics + + /// Configuration for the consumer metrics emitted by `SwiftKafka`. + public struct ConsumerMetrics: Sendable { + internal var enabled: Bool { + self.updateInterval != nil && + (self.queuedOperation != nil || + self.totalKafkaBrokerRequests != nil || + self.totalKafkaBrokerBytesSent != nil || + self.totalKafkaBrokerResponses != nil || + self.totalKafkaBrokerResponsesSize != nil || + self.totalKafkaBrokerMessagesBytesRecieved != nil || + self.topicsInMetadataCache != nil) + } + + /// Update interval for statistics. + public var updateInterval: Duration? + + /// Number of operations (callbacks, events, etc) waiting in the queue. + public var queuedOperation: Gauge? + + /// Total number of requests sent to Kafka brokers. + public var totalKafkaBrokerRequests: Gauge? + /// Total number of bytes transmitted to Kafka brokers. + public var totalKafkaBrokerBytesSent: Gauge? + /// Total number of responses received from Kafka brokers. + public var totalKafkaBrokerResponses: Gauge? + /// Total number of bytes received from Kafka brokers. + public var totalKafkaBrokerResponsesSize: Gauge? + + /// Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers. + public var totalKafkaBrokerMessagesRecieved: Gauge? + /// Total number of message bytes (including framing) received from Kafka brokers. + public var totalKafkaBrokerMessagesBytesRecieved: Gauge? + + /// Number of topics in the metadata cache. + public var topicsInMetadataCache: Gauge? + + private static func record(_ value: T?, to: Gauge?) { + guard let value, + let to else { + return + } + to.record(value) + } + + internal func update(with rdKafkaStatistics: RDKafkaStatistics) { + Self.record(rdKafkaStatistics.queuedOperation, to: self.queuedOperation) + + Self.record(rdKafkaStatistics.totalKafkaBrokerRequests, to: self.totalKafkaBrokerRequests) + Self.record(rdKafkaStatistics.totalKafkaBrokerBytesSent, to: self.totalKafkaBrokerBytesSent) + Self.record(rdKafkaStatistics.totalKafkaBrokerResponses, to: self.totalKafkaBrokerResponses) + Self.record(rdKafkaStatistics.totalKafkaBrokerResponsesSize, to: self.totalKafkaBrokerResponsesSize) + + Self.record(rdKafkaStatistics.totalKafkaBrokerMessagesRecieved, to: self.totalKafkaBrokerMessagesRecieved) + Self.record(rdKafkaStatistics.totalKafkaBrokerMessagesBytesRecieved, to: self.totalKafkaBrokerMessagesBytesRecieved) + + Self.record(rdKafkaStatistics.topicsInMetadataCache, to: self.topicsInMetadataCache) + } + } + + /// Configuration for the producer metrics emitted by `SwiftKafka`. + public struct ProducerMetrics: Sendable { + internal var enabled: Bool { + self.updateInterval != nil && + (self.queuedOperation != nil || + self.queuedProducerMessages != nil || + self.queuedProducerMessagesSize != nil || + self.totalKafkaBrokerRequests != nil || + self.totalKafkaBrokerBytesSent != nil || + self.totalKafkaBrokerResponses != nil || + self.totalKafkaBrokerResponsesSize != nil || + self.totalKafkaBrokerMessagesSent != nil || + self.totalKafkaBrokerMessagesBytesSent != nil || + self.topicsInMetadataCache != nil) + } + + /// Update interval for statistics. + public var updateInterval: Duration? + + /// Number of operations (callbacks, events, etc) waiting in the queue. + public var queuedOperation: Gauge? + /// Current number of queued producer messages. + public var queuedProducerMessages: Gauge? + /// Current total size in bytes of queued producer messages. + public var queuedProducerMessagesSize: Gauge? + + /// Total number of requests sent to Kafka brokers. + public var totalKafkaBrokerRequests: Gauge? + /// Total number of bytes transmitted to Kafka brokers. + public var totalKafkaBrokerBytesSent: Gauge? + /// Total number of responses received from Kafka brokers. + public var totalKafkaBrokerResponses: Gauge? + /// Total number of bytes received from Kafka brokers. + public var totalKafkaBrokerResponsesSize: Gauge? + + /// Total number of messages transmitted (produced) to Kafka brokers. + public var totalKafkaBrokerMessagesSent: Gauge? + /// Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers. + public var totalKafkaBrokerMessagesBytesSent: Gauge? + + /// Number of topics in the metadata cache. + public var topicsInMetadataCache: Gauge? + + private static func record(_ value: T?, to: Gauge?) { + guard let value, + let to else { + return + } + to.record(value) + } + + internal func update(with rdKafkaStatistics: RDKafkaStatistics) { + Self.record(rdKafkaStatistics.queuedOperation, to: self.queuedOperation) + Self.record(rdKafkaStatistics.queuedProducerMessages, to: self.queuedProducerMessages) + Self.record(rdKafkaStatistics.queuedProducerMessagesSize, to: self.queuedProducerMessagesSize) + + Self.record(rdKafkaStatistics.totalKafkaBrokerRequests, to: self.totalKafkaBrokerRequests) + Self.record(rdKafkaStatistics.totalKafkaBrokerBytesSent, to: self.totalKafkaBrokerBytesSent) + Self.record(rdKafkaStatistics.totalKafkaBrokerResponses, to: self.totalKafkaBrokerResponses) + Self.record(rdKafkaStatistics.totalKafkaBrokerResponsesSize, to: self.totalKafkaBrokerResponsesSize) + + Self.record(rdKafkaStatistics.totalKafkaBrokerMessagesSent, to: self.totalKafkaBrokerMessagesSent) + Self.record(rdKafkaStatistics.totalKafkaBrokerMessagesBytesSent, to: self.totalKafkaBrokerMessagesBytesSent) + + Self.record(rdKafkaStatistics.topicsInMetadataCache, to: self.topicsInMetadataCache) + } + } +} diff --git a/Sources/Kafka/Configuration/KafkaConfiguration.swift b/Sources/Kafka/Configuration/KafkaConfiguration.swift index 2248eb88..92dee50b 100644 --- a/Sources/Kafka/Configuration/KafkaConfiguration.swift +++ b/Sources/Kafka/Configuration/KafkaConfiguration.swift @@ -283,4 +283,25 @@ public enum KafkaConfiguration { /// Use the IPv6 address family. public static let v6 = IPAddressFamily(description: "v6") } + + /// Minimum time between key refresh attempts. + public struct KeyRefreshAttempts: Sendable, Hashable { + internal let rawValue: UInt + + private init(rawValue: UInt) { + self.rawValue = rawValue + } + + /// (Lowest granularity is milliseconds) + public static func value(_ value: Duration) -> KeyRefreshAttempts { + precondition( + value.canBeRepresentedAsMilliseconds, + "Lowest granularity is milliseconds" + ) + return .init(rawValue: UInt(value.inMilliseconds)) + } + + /// Disable automatic key refresh by setting this property. + public static let disable: KeyRefreshAttempts = .init(rawValue: 0) + } } diff --git a/Sources/Kafka/Configuration/KafkaConsumerConfiguration.swift b/Sources/Kafka/Configuration/KafkaConsumerConfiguration.swift index 60215250..eec11333 100644 --- a/Sources/Kafka/Configuration/KafkaConsumerConfiguration.swift +++ b/Sources/Kafka/Configuration/KafkaConsumerConfiguration.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -import Crdkafka import struct Foundation.UUID public struct KafkaConsumerConfiguration { @@ -22,42 +21,27 @@ public struct KafkaConsumerConfiguration { /// Effectively controls the rate at which incoming events and messages are consumed. /// Default: `.milliseconds(100)` public var pollInterval: Duration = .milliseconds(100) - - /// A struct representing different back pressure strategies for consuming messages in ``KafkaConsumer``. - public struct BackPressureStrategy: Sendable, Hashable { - enum _BackPressureStrategy: Sendable, Hashable { - case watermark(low: Int, high: Int) - } - - let _internal: _BackPressureStrategy - - private init(backPressureStrategy: _BackPressureStrategy) { - self._internal = backPressureStrategy - } - - /// A back pressure strategy based on high and low watermarks. - /// - /// The consumer maintains a buffer size between a low watermark and a high watermark - /// to control the flow of incoming messages. - /// - /// - Parameter low: The lower threshold for the buffer size (low watermark). - /// - Parameter high: The upper threshold for the buffer size (high watermark). - public static func watermark(low: Int, high: Int) -> BackPressureStrategy { - return .init(backPressureStrategy: .watermark(low: low, high: high)) - } - } - - /// The backpressure strategy to be used for message consumption. - /// See ``KafkaConsumerConfiguration/BackPressureStrategy-swift.struct`` for more information. - public var backPressureStrategy: BackPressureStrategy = .watermark( - low: 10, - high: 50 - ) + + public var listenForRebalance: Bool = false + + public var enablePartitionEof: Bool = false /// A struct representing the different Kafka message consumption strategies. public struct ConsumptionStrategy: Sendable, Hashable { + public struct TopicPartition: Sendable, Hashable { + public let partition: KafkaPartition + public let topic: String + public let offset: KafkaOffset + + public init(partition: KafkaPartition, topic: String, offset: KafkaOffset) { + self.partition = partition + self.topic = topic + self.offset = offset + } + } + enum _ConsumptionStrategy: Sendable, Hashable { - case partition(topic: String, partition: KafkaPartition, offset: KafkaOffset) + case partitions(groupID: String?, [TopicPartition]) case group(groupID: String, topics: [String]) } @@ -72,14 +56,14 @@ public struct KafkaConsumerConfiguration { /// /// - Parameters: /// - partition: The partition of the topic to consume from. + /// - groupID: The ID of the consumer group to commit to. Defaults to no group ID. Specifying a group ID is useful if partitions assignment is manually managed but committed offsets should still be tracked in a consumer group. /// - topic: The name of the Kafka topic. /// - offset: The offset to start consuming from. Defaults to the end of the Kafka partition queue (meaning wait for the next produced message). - public static func partition( - _ partition: KafkaPartition, - topic: String, - offset: KafkaOffset = .end + public static func partitions( + groupID: String? = nil, + partitions: [TopicPartition] ) -> ConsumptionStrategy { - return .init(consumptionStrategy: .partition(topic: topic, partition: partition, offset: offset)) + return .init(consumptionStrategy: .partitions(groupID: groupID, partitions)) } /// A consumption strategy based on consumer group membership. @@ -238,10 +222,21 @@ public struct KafkaConsumerConfiguration { /// Reconnect options. public var reconnect: KafkaConfiguration.ReconnectOptions = .init() + /// Options for librdkafka metrics updates + public var metrics: KafkaConfiguration.ConsumerMetrics = .init() + /// Security protocol to use (plaintext, ssl, sasl_plaintext, sasl_ssl). /// Default: `.plaintext` public var securityProtocol: KafkaConfiguration.SecurityProtocol = .plaintext + public var isolationLevel: String? + + public var groupInstanceId: String? + + public var compression: String? + + public var rebalanceStrategy: String? + public init( consumptionStrategy: ConsumptionStrategy, bootstrapBrokerAddresses: [KafkaConfiguration.BrokerAddress] @@ -258,12 +253,16 @@ extension KafkaConsumerConfiguration { var resultDict: [String: String] = [:] switch self.consumptionStrategy._internal { - case .partition: - // Although an assignment is not related to a consumer group, - // librdkafka requires us to set a `group.id`. - // This is a known issue: - // https://github.com/edenhill/librdkafka/issues/3261 - resultDict["group.id"] = UUID().uuidString + case .partitions(let groupID, _): + if let groupID = groupID { + resultDict["group.id"] = groupID + } else { + // Although an assignment is not related to a consumer group, + // librdkafka requires us to set a `group.id`. + // This is a known issue: + // https://github.com/edenhill/librdkafka/issues/3261 + resultDict["group.id"] = UUID().uuidString + } case .group(groupID: let groupID, topics: _): resultDict["group.id"] = groupID } @@ -301,6 +300,28 @@ extension KafkaConsumerConfiguration { resultDict["broker.address.family"] = broker.addressFamily.description resultDict["reconnect.backoff.ms"] = String(reconnect.backoff.rawValue) resultDict["reconnect.backoff.max.ms"] = String(reconnect.maximumBackoff.inMilliseconds) + resultDict["queued.max.messages.kbytes"] = String(8 * 1024) // XX MB per partition // TODO: remove + + if let isolationLevel { + resultDict["isolation.level"] = isolationLevel + } + + if let groupInstanceId { + resultDict["group.instance.id"] = groupInstanceId + } + + if let compression { + resultDict["compression.codec"] = compression + } + + if let rebalanceStrategy { + resultDict["partition.assignment.strategy"] = rebalanceStrategy + } + + if self.metrics.enabled, + let updateInterval = self.metrics.updateInterval { + resultDict["statistics.interval.ms"] = String(updateInterval.inMilliseconds) + } // Merge with SecurityProtocol configuration dictionary resultDict.merge(securityProtocol.dictionary) { _, _ in diff --git a/Sources/Kafka/Configuration/KafkaGroupConfiguration.swift b/Sources/Kafka/Configuration/KafkaGroupConfiguration.swift new file mode 100644 index 00000000..ef597fa6 --- /dev/null +++ b/Sources/Kafka/Configuration/KafkaGroupConfiguration.swift @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2022 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +public struct KafkaGroupConfiguration { + // MARK: - Common Client Config Properties + + /// Initial list of brokers. + /// Default: `[]` + public var bootstrapBrokerAddresses: [KafkaConfiguration.BrokerAddress] = [] + + public init(bootstrapBrokerAddresses: [KafkaConfiguration.BrokerAddress]) { + self.bootstrapBrokerAddresses = bootstrapBrokerAddresses + } +} + +// MARK: - KafkaGroupConfiguration + Dictionary + +extension KafkaGroupConfiguration { + internal var dictionary: [String: String] { + var resultDict: [String: String] = [:] + resultDict["bootstrap.servers"] = bootstrapBrokerAddresses.map(\.description).joined(separator: ",") + return resultDict + } +} + +extension KafkaGroupConfiguration: Sendable {} diff --git a/Sources/Kafka/Configuration/KafkaProducerConfiguration.swift b/Sources/Kafka/Configuration/KafkaProducerConfiguration.swift index a774905b..fde7aa9f 100644 --- a/Sources/Kafka/Configuration/KafkaProducerConfiguration.swift +++ b/Sources/Kafka/Configuration/KafkaProducerConfiguration.swift @@ -161,9 +161,14 @@ public struct KafkaProducerConfiguration { /// Reconnect options. public var reconnect: KafkaConfiguration.ReconnectOptions = .init() + /// Options for librdkafka metrics updates + public var metrics: KafkaConfiguration.ProducerMetrics = .init() + /// Security protocol to use (plaintext, ssl, sasl_plaintext, sasl_ssl). /// Default: `.plaintext` public var securityProtocol: KafkaConfiguration.SecurityProtocol = .plaintext + + public var compression: String? public init( bootstrapBrokerAddresses: [KafkaConfiguration.BrokerAddress] @@ -212,6 +217,15 @@ extension KafkaProducerConfiguration { resultDict["reconnect.backoff.ms"] = String(self.reconnect.backoff.rawValue) resultDict["reconnect.backoff.max.ms"] = String(self.reconnect.maximumBackoff.inMilliseconds) + if let compression { + resultDict["compression.codec"] = compression + } + + if self.metrics.enabled, + let updateInterval = self.metrics.updateInterval { + resultDict["statistics.interval.ms"] = String(updateInterval.inMilliseconds) + } + // Merge with SecurityProtocol configuration dictionary resultDict.merge(self.securityProtocol.dictionary) { _, _ in fatalError("securityProtocol and \(#file) should not have duplicate keys") diff --git a/Sources/Kafka/Configuration/KafkaTransactionalProducerConfiguration.swift b/Sources/Kafka/Configuration/KafkaTransactionalProducerConfiguration.swift new file mode 100644 index 00000000..25e5c659 --- /dev/null +++ b/Sources/Kafka/Configuration/KafkaTransactionalProducerConfiguration.swift @@ -0,0 +1,263 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +public struct KafkaTransactionalProducerConfiguration { + // MARK: - Kafka-specific Config properties + + /// If the ``isAutoCreateTopicsEnabled`` option is set to `true`, + /// the broker will automatically generate topics when producing data to non-existent topics. + /// The configuration specified in this ``KafkaTopicConfiguration`` will be applied to the newly created topic. + /// Default: See default values of ``KafkaTopicConfiguration`` + public var topicConfiguration: KafkaTopicConfiguration = .init() + + /// The time between two consecutive polls. + /// Effectively controls the rate at which incoming events are consumed. + /// Default: `.milliseconds(100)` + public var pollInterval: Duration = .milliseconds(100) + + /// Maximum timeout for flushing outstanding produce requests when the ``KafkaProducer`` is shutting down. + /// Default: `10000` + public var flushTimeoutMilliseconds: Int = 10000 { + didSet { + precondition( + 0...Int(Int32.max) ~= self.flushTimeoutMilliseconds, + "Flush timeout outside of valid range \(0...Int32.max)" + ) + } + } + + // MARK: - Producer-specific Config Properties + + /// When set to true, the producer will ensure that messages are successfully produced exactly once and in the original produce order. + /// The following configuration properties are adjusted automatically (if not modified by the user) when idempotence is enabled: + /// ``KafkaProducerConfiguration/maximumInFlightRequestsPerConnection`` = `5` (must be less than or equal to 5), + /// ``KafkaProducerConfiguration/maximumMessageSendRetries`` = `UInt32.max` (must be greater than 0), + /// ``KafkaTopicConfiguration/requiredAcknowledgements`` = ``KafkaTopicConfiguration/RequiredAcknowledgments/all``, + /// queuing strategy = FIFO. + /// Producer instantiation will fail if the user-supplied configuration is incompatible. + /// Default: `false` + public var isIdempotenceEnabled: Bool = false + + /// Producer queue options. + public struct QueueConfiguration: Sendable, Hashable { + /// Maximum number of messages allowed on the producer queue. This queue is shared by all topics and partitions. + public struct MessageLimit: Sendable, Hashable { + internal let rawValue: Int + + private init(rawValue: Int) { + self.rawValue = rawValue + } + + public static func maximumLimit(_ value: Int) -> MessageLimit { + return .init(rawValue: value) + } + + /// No limit for the maximum number of messages allowed on the producer queue. + public static let unlimited: MessageLimit = .init(rawValue: 0) + } + + /// Maximum number of messages allowed on the producer queue. This queue is shared by all topics and partitions. + /// Default: `.maximumLimit(100_000)` + public var messageLimit: MessageLimit = .maximumLimit(100_000) + + /// Maximum total message size sum allowed on the producer queue. This queue is shared by all topics and partitions. + /// This property has higher priority than ``KafkaConfiguration/QueueOptions/messageLimit``. + /// Default: `1_048_576 * 1024` + public var maximumMessageBytes: Int = 1_048_576 * 1024 + + /// How long wait for messages in the producer queue to accumulate before constructing message batches (MessageSets) to transmit to brokers. + /// A higher value allows larger and more effective (less overhead, improved compression) batches of messages to accumulate at the expense of increased message delivery latency. + /// (Lowest granularity is milliseconds) + /// Default: `.milliseconds(5)` + public var maximumMessageQueueTime: Duration = .milliseconds(5) { + didSet { + precondition( + self.maximumMessageQueueTime.canBeRepresentedAsMilliseconds, + "Lowest granularity is milliseconds" + ) + } + } + + public init() {} + } + + /// Producer queue options. + public var queue: QueueConfiguration = .init() + + /// How many times to retry sending a failing Message. + /// + /// - Note: retrying may cause reordering unless ``KafkaProducerConfiguration/isIdempotenceEnabled`` is set to `true`. + /// Default: `2_147_483_647` + public var maximumMessageSendRetries: Int = 2_147_483_647 + + /// Allow automatic topic creation on the broker when producing to non-existent topics. + /// The broker must also be configured with ``isAutoCreateTopicsEnabled`` = `true` for this configuration to take effect. + /// Default: `true` + public var isAutoCreateTopicsEnabled: Bool = true + + // MARK: - Common Client Config Properties + + /// Client identifier. + /// Default: `"rdkafka"` + public var identifier: String = "rdkafka" + + /// Initial list of brokers. + /// Default: `[]` + public var bootstrapBrokerAddresses: [KafkaConfiguration.BrokerAddress] = [] + + /// Message options. + public var message: KafkaConfiguration.MessageOptions = .init() + + /// Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hiccups. + /// Default: `100_000_000` + public var maximumReceiveMessageBytes: Int = 100_000_000 + + /// Maximum number of in-flight requests per broker connection. + /// This is a generic property applied to all broker communication, however, it is primarily relevant to produce requests. + /// In particular, note that other mechanisms limit the number of outstanding consumer fetch requests per broker to one. + /// Default: `5` + public var maximumInFlightRequestsPerConnection: Int = 5 { + didSet { + precondition( + self.maximumInFlightRequestsPerConnection <= 5, + "Max in flight requests is 5 for TransactionalProducer" + ) + } + } + + /// Metadata cache max age. + /// (Lowest granularity is milliseconds) + /// Default: `.milliseconds(900_000)` + public var maximumMetadataAge: Duration = .milliseconds(900_000) { + didSet { + precondition( + self.maximumMetadataAge.canBeRepresentedAsMilliseconds, + "Lowest granularity is milliseconds" + ) + } + } + + /// Topic metadata options. + public var topicMetadata: KafkaConfiguration.TopicMetadataOptions = .init() + + /// Topic denylist. + /// Default: `[]` + public var topicDenylist: [String] = [] + + /// Debug options. + /// Default: `[]` + public var debugOptions: [KafkaConfiguration.DebugOption] = [] + + /// Socket options. + public var socket: KafkaConfiguration.SocketOptions = .init() + + /// Broker options. + public var broker: KafkaConfiguration.BrokerOptions = .init() + + /// Reconnect options. + public var reconnect: KafkaConfiguration.ReconnectOptions = .init() + + /// Options for librdkafka metrics updates + public var metrics: KafkaConfiguration.ProducerMetrics = .init() + + /// Security protocol to use (plaintext, ssl, sasl_plaintext, sasl_ssl). + /// Default: `.plaintext` + public var securityProtocol: KafkaConfiguration.SecurityProtocol = .plaintext + + // TODO: add Docc + var transactionalId: String + var transactionsTimeout: Duration = .seconds(60) { + didSet { + precondition( + self.maximumMetadataAge.canBeRepresentedAsMilliseconds, + "Lowest granularity is milliseconds" + ) + } + } + + public var compression: String? + + public init( + transactionalId: String, + bootstrapBrokerAddresses: [KafkaConfiguration.BrokerAddress] + ) { + self.transactionalId = transactionalId + self.bootstrapBrokerAddresses = bootstrapBrokerAddresses + } +} + +// MARK: - KafkaProducerConfiguration + Dictionary + +extension KafkaTransactionalProducerConfiguration { + internal var dictionary: [String: String] { + var resultDict: [String: String] = [:] + + resultDict["transactional.id"] = self.transactionalId + resultDict["transaction.timeout.ms"] = String(self.transactionsTimeout.totalMilliseconds) + resultDict["enable.idempotence"] = "true" + + resultDict["queue.buffering.max.messages"] = String(self.queue.messageLimit.rawValue) + resultDict["queue.buffering.max.kbytes"] = String(self.queue.maximumMessageBytes / 1024) + resultDict["queue.buffering.max.ms"] = String(self.queue.maximumMessageQueueTime.inMilliseconds) + resultDict["message.send.max.retries"] = String(self.maximumMessageSendRetries) + resultDict["allow.auto.create.topics"] = String(self.isAutoCreateTopicsEnabled) + + resultDict["client.id"] = self.identifier + resultDict["bootstrap.servers"] = self.bootstrapBrokerAddresses.map(\.description).joined(separator: ",") + resultDict["message.max.bytes"] = String(self.message.maximumBytes) + resultDict["message.copy.max.bytes"] = String(self.message.maximumBytesToCopy) + resultDict["receive.message.max.bytes"] = String(self.maximumReceiveMessageBytes) + resultDict["max.in.flight.requests.per.connection"] = String(self.maximumInFlightRequestsPerConnection) + resultDict["metadata.max.age.ms"] = String(self.maximumMetadataAge.inMilliseconds) + resultDict["topic.metadata.refresh.interval.ms"] = String(self.topicMetadata.refreshInterval.rawValue) + resultDict["topic.metadata.refresh.fast.interval.ms"] = String(self.topicMetadata.refreshFastInterval.inMilliseconds) + resultDict["topic.metadata.refresh.sparse"] = String(self.topicMetadata.isSparseRefreshingEnabled) + resultDict["topic.metadata.propagation.max.ms"] = String(self.topicMetadata.maximumPropagation.inMilliseconds) + resultDict["topic.blacklist"] = self.topicDenylist.joined(separator: ",") + if !self.debugOptions.isEmpty { + resultDict["debug"] = self.debugOptions.map(\.description).joined(separator: ",") + } + resultDict["socket.timeout.ms"] = String(self.socket.timeout.inMilliseconds) + resultDict["socket.send.buffer.bytes"] = String(self.socket.sendBufferBytes.rawValue) + resultDict["socket.receive.buffer.bytes"] = String(self.socket.receiveBufferBytes.rawValue) + resultDict["socket.keepalive.enable"] = String(self.socket.isKeepaliveEnabled) + resultDict["socket.nagle.disable"] = String(self.socket.isNagleDisabled) + resultDict["socket.max.fails"] = String(self.socket.maximumFailures.rawValue) + resultDict["socket.connection.setup.timeout.ms"] = String(self.socket.connectionSetupTimeout.inMilliseconds) + resultDict["broker.address.ttl"] = String(self.broker.addressTimeToLive.inMilliseconds) + resultDict["broker.address.family"] = self.broker.addressFamily.description + resultDict["reconnect.backoff.ms"] = String(self.reconnect.backoff.rawValue) + resultDict["reconnect.backoff.max.ms"] = String(self.reconnect.maximumBackoff.inMilliseconds) + + // Merge with SecurityProtocol configuration dictionary + resultDict.merge(self.securityProtocol.dictionary) { _, _ in + fatalError("securityProtocol and \(#file) should not have duplicate keys") + } + + if let compression { + resultDict["compression.codec"] = compression + } + + if self.metrics.enabled, + let updateInterval = self.metrics.updateInterval { + resultDict["statistics.interval.ms"] = String(updateInterval.inMilliseconds) + } + + return resultDict + } +} + +// MARK: - KafkaProducerConfiguration + Sendable + +extension KafkaTransactionalProducerConfiguration: Sendable {} diff --git a/Sources/Kafka/ForTesting/RDKafkaClient+Topic.swift b/Sources/Kafka/ForTesting/RDKafkaClient+Topic.swift new file mode 100644 index 00000000..4c45e0da --- /dev/null +++ b/Sources/Kafka/ForTesting/RDKafkaClient+Topic.swift @@ -0,0 +1,167 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2022 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Crdkafka +import struct Foundation.UUID +import Logging + +@_spi(Internal) +extension RDKafkaClient { + /// Create a topic with a unique name (`UUID`). + /// Blocks for a maximum of `timeout` milliseconds. + /// - Parameter partitions: Partitions in topic (default: -1 - default for broker) + /// - Parameter timeout: Timeout in milliseconds. + /// - Returns: Name of newly created topic. + /// - Throws: A ``KafkaError`` if the topic creation failed. + public func _createUniqueTopic(partitions: Int32 = -1, timeout: Int32) throws -> String { + let uniqueTopicName = UUID().uuidString + + try _createTopic(topicName: uniqueTopicName, partitions: partitions, timeout: timeout) + + return uniqueTopicName + } + + /// Create a topic with specified name + /// Blocks for a maximum of `timeout` milliseconds. + /// - Parameter partitions: Partitions in topic (default: -1 - default for broker) + /// - Parameter timeout: Timeout in milliseconds. + /// - Returns: Name of newly created topic. + /// - Throws: A ``KafkaError`` if the topic creation failed. + public func _createTopic(topicName: String, partitions: Int32 = -1, timeout: Int32) throws { + let errorChars = UnsafeMutablePointer.allocate(capacity: RDKafkaClient.stringSize) + defer { errorChars.deallocate() } + + guard let newTopic = rd_kafka_NewTopic_new( + topicName, + partitions, + -1, // use default replication_factor + errorChars, + RDKafkaClient.stringSize + ) else { + let errorString = String(cString: errorChars) + throw KafkaError.topicCreation(reason: errorString) + } + defer { rd_kafka_NewTopic_destroy(newTopic) } + + try self.withKafkaHandlePointer { kafkaHandle in + let resultQueue = rd_kafka_queue_new(kafkaHandle) + defer { rd_kafka_queue_destroy(resultQueue) } + + var newTopicsArray: [OpaquePointer?] = [newTopic] + rd_kafka_CreateTopics( + kafkaHandle, + &newTopicsArray, + 1, + nil, + resultQueue + ) + + guard let resultEvent = rd_kafka_queue_poll(resultQueue, timeout) else { + throw KafkaError.topicCreation(reason: "No CreateTopics result after 10s timeout") + } + defer { rd_kafka_event_destroy(resultEvent) } + + let resultCode = rd_kafka_event_error(resultEvent) + guard resultCode == RD_KAFKA_RESP_ERR_NO_ERROR else { + throw KafkaError.rdKafkaError(wrapping: resultCode) + } + + guard let topicsResultEvent = rd_kafka_event_CreateTopics_result(resultEvent) else { + throw KafkaError.topicCreation(reason: "Received event that is not of type rd_kafka_CreateTopics_result_t") + } + + var resultTopicCount = 0 + let topicResults = rd_kafka_CreateTopics_result_topics( + topicsResultEvent, + &resultTopicCount + ) + + guard resultTopicCount == 1, let topicResult = topicResults?[0] else { + throw KafkaError.topicCreation(reason: "Received less/more than one topic result") + } + + let topicResultError = rd_kafka_topic_result_error(topicResult) + guard topicResultError == RD_KAFKA_RESP_ERR_NO_ERROR else { + throw KafkaError.rdKafkaError(wrapping: topicResultError) + } + + let receivedTopicName = String(cString: rd_kafka_topic_result_name(topicResult)) + guard receivedTopicName == topicName else { + throw KafkaError.topicCreation(reason: "Received topic result for topic with different name") + } + } + } + + /// Delete a topic. + /// Blocks for a maximum of `timeout` milliseconds. + /// - Parameter topic: Topic to delete. + /// - Parameter timeout: Timeout in milliseconds. + /// - Throws: A ``KafkaError`` if the topic deletion failed. + public func _deleteTopic(_ topic: String, timeout: Int32) throws { + let deleteTopic = rd_kafka_DeleteTopic_new(topic) + defer { rd_kafka_DeleteTopic_destroy(deleteTopic) } + + try self.withKafkaHandlePointer { kafkaHandle in + let resultQueue = rd_kafka_queue_new(kafkaHandle) + defer { rd_kafka_queue_destroy(resultQueue) } + + var deleteTopicsArray: [OpaquePointer?] = [deleteTopic] + rd_kafka_DeleteTopics( + kafkaHandle, + &deleteTopicsArray, + 1, + nil, + resultQueue + ) + + guard let resultEvent = rd_kafka_queue_poll(resultQueue, timeout) else { + throw KafkaError.topicDeletion(reason: "No DeleteTopics result after 10s timeout") + } + defer { rd_kafka_event_destroy(resultEvent) } + + let resultCode = rd_kafka_event_error(resultEvent) + guard resultCode == RD_KAFKA_RESP_ERR_NO_ERROR else { + throw KafkaError.rdKafkaError(wrapping: resultCode) + } + + guard let topicsResultEvent = rd_kafka_event_DeleteTopics_result(resultEvent) else { + throw KafkaError.topicDeletion(reason: "Received event that is not of type rd_kafka_DeleteTopics_result_t") + } + + var resultTopicCount = 0 + let topicResults = rd_kafka_DeleteTopics_result_topics( + topicsResultEvent, + &resultTopicCount + ) + + guard resultTopicCount == 1, let topicResult = topicResults?[0] else { + throw KafkaError.topicDeletion(reason: "Received less/more than one topic result") + } + + let topicResultError = rd_kafka_topic_result_error(topicResult) + guard topicResultError == RD_KAFKA_RESP_ERR_NO_ERROR else { + throw KafkaError.rdKafkaError(wrapping: topicResultError) + } + + let receivedTopicName = String(cString: rd_kafka_topic_result_name(topicResult)) + guard receivedTopicName == topic else { + throw KafkaError.topicDeletion(reason: "Received topic result for topic with different name") + } + } + } + + public static func makeClientForTopics(config: KafkaConsumerConfiguration, logger: Logger) throws -> RDKafkaClient { + return try Self.makeClient(type: .consumer, configDictionary: config.dictionary, events: [], logger: logger) + } +} diff --git a/Sources/Kafka/ForTesting/TestMessages.swift b/Sources/Kafka/ForTesting/TestMessages.swift new file mode 100644 index 00000000..f9df6224 --- /dev/null +++ b/Sources/Kafka/ForTesting/TestMessages.swift @@ -0,0 +1,104 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2022 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +import struct Foundation.Date +import NIOCore + +@_spi(Internal) +public enum _TestMessagesError: Error { + case deliveryReportsIdsIncorrect + case deliveryReportsNotAllMessagesAcknoledged + case deliveryReportsIncorrect +} + +@_spi(Internal) +public func _createTestMessages( + topic: String, + headers: [KafkaHeader] = [], + count: UInt +) -> [KafkaProducerMessage] { + return Array(0..], + skipConsistencyCheck: Bool = false +) async throws { + var messageIDs = Set() + messageIDs.reserveCapacity(messages.count) + + for message in messages { + while true { + do { + messageIDs.insert(try producer.send(message)) + break + } catch let error as KafkaError where error.description.contains("Queue full") { + // That means we have to flush queue immediately but there is no interface for that + // producer.flush() + } + } + } + + var receivedDeliveryReports = Set() + receivedDeliveryReports.reserveCapacity(messages.count) + + for await event in events { + switch event { + case .deliveryReports(let deliveryReports): + for deliveryReport in deliveryReports { + receivedDeliveryReports.insert(deliveryReport) + } + default: + break // Ignore any other events + } + + if receivedDeliveryReports.count >= messages.count { + break + } + } + + guard Set(receivedDeliveryReports.map(\.id)) == messageIDs else { + throw _TestMessagesError.deliveryReportsIdsIncorrect + } + + let acknowledgedMessages: [KafkaAcknowledgedMessage] = receivedDeliveryReports.compactMap { + guard case .acknowledged(let receivedMessage) = $0.status else { + return nil + } + return receivedMessage + } + + guard messages.count == acknowledgedMessages.count else { + throw _TestMessagesError.deliveryReportsNotAllMessagesAcknoledged + } + if skipConsistencyCheck { + return + } + for message in messages { + guard acknowledgedMessages.contains(where: { $0.topic == message.topic }), + acknowledgedMessages.contains(where: { $0.key == ByteBuffer(string: message.key!) }), + acknowledgedMessages.contains(where: { $0.value == ByteBuffer(string: message.value) }) else { + throw _TestMessagesError.deliveryReportsIncorrect + } + } +} diff --git a/Sources/Kafka/KafkaConsumer.swift b/Sources/Kafka/KafkaConsumer.swift index 1163f862..19f3dbbc 100644 --- a/Sources/Kafka/KafkaConsumer.swift +++ b/Sources/Kafka/KafkaConsumer.swift @@ -26,7 +26,7 @@ internal struct KafkaConsumerEventsDelegate: Sendable { extension KafkaConsumerEventsDelegate: NIOAsyncSequenceProducerDelegate { func produceMore() { - return // No back pressure + return // no backpressure } func didTerminate() { @@ -34,23 +34,6 @@ extension KafkaConsumerEventsDelegate: NIOAsyncSequenceProducerDelegate { } } -// MARK: - KafkaConsumerMessagesDelegate - -/// `NIOAsyncSequenceProducerDelegate` for ``KafkaConsumerMessages``. -internal struct KafkaConsumerMessagesDelegate: Sendable { - let stateMachine: NIOLockedValueBox -} - -extension KafkaConsumerMessagesDelegate: NIOAsyncSequenceProducerDelegate { - func produceMore() { - self.stateMachine.withLockedValue { $0.produceMore() } - } - - func didTerminate() { - self.stateMachine.withLockedValue { $0.finishMessageConsumption() } - } -} - // MARK: - KafkaConsumerEvents /// `AsyncSequence` implementation for handling ``KafkaConsumerEvent``s emitted by Kafka. @@ -78,60 +61,87 @@ public struct KafkaConsumerEvents: Sendable, AsyncSequence { /// `AsyncSequence` implementation for handling messages received from the Kafka cluster (``KafkaConsumerMessage``). public struct KafkaConsumerMessages: Sendable, AsyncSequence { - let stateMachine: NIOLockedValueBox + typealias LockedMachine = NIOLockedValueBox + + let stateMachine: LockedMachine + let pollInterval: Duration + let enablePartitionEof: Bool public typealias Element = KafkaConsumerMessage - typealias BackPressureStrategy = NIOAsyncSequenceProducerBackPressureStrategies.HighLowWatermark - typealias WrappedSequence = NIOThrowingAsyncSequenceProducer< - Result, - Error, - BackPressureStrategy, - KafkaConsumerMessagesDelegate - > - let wrappedSequence: WrappedSequence /// `AsynceIteratorProtocol` implementation for handling messages received from the Kafka cluster (``KafkaConsumerMessage``). public struct AsyncIterator: AsyncIteratorProtocol { - let stateMachine: NIOLockedValueBox - var wrappedIterator: WrappedSequence.AsyncIterator? + private let stateMachine: MachineHolder + let pollInterval: Duration + let enablePartitionEof: Bool + + weak var cachedClient: RDKafkaClient? + var messagesCount: Int = 0 + + private final class MachineHolder: Sendable { // only for deinit + let stateMachine: LockedMachine + init(stateMachine: LockedMachine) { + self.stateMachine = stateMachine + } - public mutating func next() async throws -> Element? { - guard let result = try await self.wrappedIterator?.next() else { - self.deallocateIterator() - return nil + deinit { + self.stateMachine.withLockedValue { $0.finishMessageConsumption() } } + } + + init(stateMachine: LockedMachine, pollInterval: Duration, enablePartitionEof: Bool) { + self.stateMachine = .init(stateMachine: stateMachine) + self.pollInterval = pollInterval + self.enablePartitionEof = enablePartitionEof + } - switch result { - case .success(let message): - let action = self.stateMachine.withLockedValue { $0.storeOffset() } + public mutating func next() async throws -> Element? { + defer { + self.messagesCount += 1 + if self.messagesCount >= 100 { + self.cachedClient = nil + self.messagesCount = 0 + } + } + while !Task.isCancelled { + if let client = self.cachedClient, // fast path + let message = try client.consumerPoll() { + if !message.eof || self.enablePartitionEof { + return message + } else { + continue + } + } + let action = self.stateMachine.stateMachine.withLockedValue { $0.nextConsumerPollLoopAction() } switch action { - case .storeOffset(let client): - do { - try client.storeMessageOffset(message) - } catch { - self.deallocateIterator() - throw error + case .poll(let client): + let message = try client.consumerPoll() + guard let message else { + self.cachedClient = nil + self.messagesCount = 0 + try await Task.sleep(for: self.pollInterval) + continue + } + if message.eof && !self.enablePartitionEof { + continue } + self.cachedClient = client return message - case .terminateConsumerSequence: - self.deallocateIterator() + case .suspendPollLoop: + try await Task.sleep(for: self.pollInterval) + case .terminatePollLoop: return nil } - case .failure(let error): - self.deallocateIterator() - throw error } - } - - private mutating func deallocateIterator() { - self.wrappedIterator = nil + return nil } } public func makeAsyncIterator() -> AsyncIterator { return AsyncIterator( stateMachine: self.stateMachine, - wrappedIterator: self.wrappedSequence.makeAsyncIterator() + pollInterval: self.pollInterval, + enablePartitionEof: self.enablePartitionEof ) } } @@ -140,19 +150,19 @@ public struct KafkaConsumerMessages: Sendable, AsyncSequence { /// A ``KafkaConsumer `` can be used to consume messages from a Kafka cluster. public final class KafkaConsumer: Sendable, Service { - typealias Producer = NIOThrowingAsyncSequenceProducer< - Result, - Error, - NIOAsyncSequenceProducerBackPressureStrategies.HighLowWatermark, - KafkaConsumerMessagesDelegate + typealias ProducerEvents = NIOAsyncSequenceProducer< + KafkaConsumerEvent, + NIOAsyncSequenceProducerBackPressureStrategies.NoBackPressure, + KafkaConsumerEventsDelegate > + /// The configuration object of the consumer client. private let configuration: KafkaConsumerConfiguration /// A logger. private let logger: Logger /// State of the `KafkaConsumer`. private let stateMachine: NIOLockedValueBox - + /// An asynchronous sequence containing messages from the Kafka cluster. public let messages: KafkaConsumerMessages @@ -171,35 +181,23 @@ public final class KafkaConsumer: Sendable, Service { client: RDKafkaClient, stateMachine: NIOLockedValueBox, configuration: KafkaConsumerConfiguration, - logger: Logger + logger: Logger, + eventSource: ProducerEvents.Source? = nil ) throws { self.configuration = configuration self.stateMachine = stateMachine self.logger = logger - let sourceAndSequence = NIOThrowingAsyncSequenceProducer.makeSequence( - elementType: Result.self, - backPressureStrategy: { - switch configuration.backPressureStrategy._internal { - case .watermark(let lowWatermark, let highWatermark): - return NIOAsyncSequenceProducerBackPressureStrategies.HighLowWatermark( - lowWatermark: lowWatermark, - highWatermark: highWatermark - ) - } - }(), - delegate: KafkaConsumerMessagesDelegate(stateMachine: self.stateMachine) - ) - self.messages = KafkaConsumerMessages( stateMachine: self.stateMachine, - wrappedSequence: sourceAndSequence.sequence + pollInterval: configuration.pollInterval, + enablePartitionEof: configuration.enablePartitionEof ) self.stateMachine.withLockedValue { $0.initialize( client: client, - source: sourceAndSequence.source + eventSource: eventSource ) } } @@ -218,10 +216,14 @@ public final class KafkaConsumer: Sendable, Service { logger: Logger ) throws { var subscribedEvents: [RDKafkaEvent] = [.log] + // Only listen to offset commit events when autoCommit is false if configuration.isAutoCommitEnabled == false { subscribedEvents.append(.offsetCommit) } + if configuration.metrics.enabled { + subscribedEvents.append(.statistics) + } let client = try RDKafkaClient.makeClient( type: .consumer, @@ -262,23 +264,52 @@ public final class KafkaConsumer: Sendable, Service { if configuration.isAutoCommitEnabled == false { subscribedEvents.append(.offsetCommit) } + if configuration.metrics.enabled { + subscribedEvents.append(.statistics) + } +// NOTE: since now consumer is being polled with rd_kafka_consumer_poll, +// we have to listen for rebalance through callback, otherwise consumer may fail +// if configuration.listenForRebalance { +// subscribedEvents.append(.rebalance) +// } + + // we assign events once, so it is always thread safe -> @unchecked Sendable + // but before start of consumer + final class EventsInFutureWrapper: @unchecked Sendable { + weak var consumer: KafkaConsumer? = nil + } + + let wrapper = EventsInFutureWrapper() + + // as kafka_consumer_poll is used, we MUST define rebalance cb instead of listening to events + let rebalanceCallBackStorage: RDKafkaClient.RebalanceCallbackStorage? + if configuration.listenForRebalance { + rebalanceCallBackStorage = RDKafkaClient.RebalanceCallbackStorage { rebalanceEvent in + let action = wrapper.consumer?.stateMachine.withLockedValue { $0.nextEventPollLoopAction() } + switch action { + case .pollForEvents(_, let eventSource): + // FIXME: in fact, it is better to put to messages sequence + // but so far there is no particular design for rebalance + // so, let's put it to events as previously + _ = eventSource?.yield(.init(rebalanceEvent)) + default: + return + } + } + } else { + rebalanceCallBackStorage = nil + } let client = try RDKafkaClient.makeClient( type: .consumer, configDictionary: configuration.dictionary, events: subscribedEvents, - logger: logger + logger: logger, + rebalanceCallBackStorage: rebalanceCallBackStorage ) let stateMachine = NIOLockedValueBox(StateMachine()) - - let consumer = try KafkaConsumer( - client: client, - stateMachine: stateMachine, - configuration: configuration, - logger: logger - ) - + // Note: // It's crucial to initialize the `sourceAndSequence` variable AFTER `client`. // This order is important to prevent the accidental triggering of `KafkaConsumerCloseOnTerminate.didTerminate()`. @@ -287,8 +318,18 @@ public final class KafkaConsumer: Sendable, Service { let sourceAndSequence = NIOAsyncSequenceProducer.makeSequence( elementType: KafkaConsumerEvent.self, backPressureStrategy: NIOAsyncSequenceProducerBackPressureStrategies.NoBackPressure(), + finishOnDeinit: true, delegate: KafkaConsumerEventsDelegate(stateMachine: stateMachine) ) + + let consumer = try KafkaConsumer( + client: client, + stateMachine: stateMachine, + configuration: configuration, + logger: logger, + eventSource: sourceAndSequence.source + ) + wrapper.consumer = consumer let eventsSequence = KafkaConsumerEvents(wrappedSequence: sourceAndSequence.sequence) return (consumer, eventsSequence) @@ -300,6 +341,9 @@ public final class KafkaConsumer: Sendable, Service { /// - Throws: A ``KafkaError`` if subscribing to the topic list failed. private func subscribe(topics: [String]) throws { let action = self.stateMachine.withLockedValue { $0.setUpConnection() } + if topics.isEmpty { + return + } switch action { case .setUpConnection(let client): let subscription = RDKafkaTopicPartitionList() @@ -310,6 +354,8 @@ public final class KafkaConsumer: Sendable, Service { ) } try client.subscribe(topicPartitionList: subscription) + case .consumerClosed: + throw KafkaError.connectionClosed(reason: "Consumer deinitialized before setup") } } @@ -320,17 +366,81 @@ public final class KafkaConsumer: Sendable, Service { /// Defaults to the end of the Kafka partition queue (meaning wait for next produced message). /// - Throws: A ``KafkaError`` if the consumer could not be assigned to the topic + partition pair. private func assign( - topic: String, - partition: KafkaPartition, - offset: KafkaOffset - ) throws { + partitions: [KafkaConsumerConfiguration.ConsumptionStrategy.TopicPartition] + ) async throws { let action = self.stateMachine.withLockedValue { $0.setUpConnection() } switch action { case .setUpConnection(let client): let assignment = RDKafkaTopicPartitionList() - assignment.setOffset(topic: topic, partition: partition, offset: Int64(offset.rawValue)) - try client.assign(topicPartitionList: assignment) + for partition in partitions { + assignment.setOffset(topic: partition.topic, partition: partition.partition, offset: partition.offset) + } + try await client.assign(topicPartitionList: assignment) + case .consumerClosed: + throw KafkaError.connectionClosed(reason: "Consumer deinitialized before setup") + } + } + + /// Subscribe to the given list of `topics`. + /// The partition assignment happens automatically using `KafkaConsumer`'s consumer group. + /// - Parameter topics: An array of topic names to subscribe to. + /// - Throws: A ``KafkaError`` if subscribing to the topic list failed. + public func subscribeTopics(topics: [String]) throws { + if topics.isEmpty { + return + } + let client = try self.stateMachine.withLockedValue { try $0.client() } + let subscription = RDKafkaTopicPartitionList() + for topic in topics { + subscription.add(topic: topic, partition: KafkaPartition.unassigned) } + try client.subscribe(topicPartitionList: subscription) + } + + + public func assign(_ list: KafkaTopicList?) async throws { + let action = self.stateMachine.withLockedValue { $0.seekOrRebalance() } + switch action { + case .allowed(let client): + try await client.assign(topicPartitionList: list?.list) + case .denied(let err): + throw KafkaError.client(reason: err) + } + } + + public func incrementalAssign(_ list: KafkaTopicList) async throws { + let action = self.stateMachine.withLockedValue { $0.seekOrRebalance() } + switch action { + case .allowed(let client): + try await client.incrementalAssign(topicPartitionList: list.list) + case .denied(let err): + throw KafkaError.client(reason: err) + } + } + + public func incrementalUnassign(_ list: KafkaTopicList) async throws { + let action = self.stateMachine.withLockedValue { $0.seekOrRebalance() } + switch action { + case .allowed(let client): + try await client.incrementalUnassign(topicPartitionList: list.list) + case .denied(let err): + throw KafkaError.client(reason: err) + } + } + + // TODO: add docc: timeout = 0 -> async (no errors reported) + public func seek(_ list: KafkaTopicList, timeout: Duration) async throws { + let action = self.stateMachine.withLockedValue { $0.seekOrRebalance() } + switch action { + case .allowed(let client): + try await client.seek(topicPartitionList: list.list, timeout: timeout) + case .denied(let err): + throw KafkaError.client(reason: err) + } + } + + public func metadata() async throws -> KafkaMetadata { + try await client().metadata() } /// Start the ``KafkaConsumer``. @@ -346,118 +456,53 @@ public final class KafkaConsumer: Sendable, Service { private func _run() async throws { switch self.configuration.consumptionStrategy._internal { - case .partition(topic: let topic, partition: let partition, offset: let offset): - try self.assign(topic: topic, partition: partition, offset: offset) + case .partitions(_, let partitions): + try await self.assign(partitions: partitions) case .group(groupID: _, topics: let topics): try self.subscribe(topics: topics) } - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - try await self.eventRunLoop() - } - - group.addTask { - try await self.messageRunLoop() - } - - // Throw when one of the two child task throws - try await group.next() - try await group.next() - } + try await self.eventRunLoop() } /// Run loop polling Kafka for new events. private func eventRunLoop() async throws { + var pollInterval = configuration.pollInterval + var events = [RDKafkaClient.KafkaEvent]() + events.reserveCapacity(100) while !Task.isCancelled { let nextAction = self.stateMachine.withLockedValue { $0.nextEventPollLoopAction() } switch nextAction { - case .pollForEvents(let client): + case .pollForEvents(let client, let eventSource): // Event poll to serve any events queued inside of `librdkafka`. - _ = client.eventPoll() - try await Task.sleep(for: self.configuration.pollInterval) - case .terminatePollLoop: - return - } - } - } - - /// Run loop polling Kafka for new consumer messages. - private func messageRunLoop() async throws { - while !Task.isCancelled { - let nextAction = self.stateMachine.withLockedValue { $0.nextConsumerPollLoopAction() } - switch nextAction { - case .pollForAndYieldMessages(let client, let source): - // Poll for new consumer messages. - let messageResults = self.batchConsumerPoll(client: client) - if messageResults.isEmpty { - self.stateMachine.withLockedValue { $0.waitForNewMessages() } - } else { - let yieldResult = source.yield(contentsOf: messageResults) - switch yieldResult { - case .produceMore: - break - case .stopProducing: - self.stateMachine.withLockedValue { $0.stopProducing() } - case .dropped: - return + let shouldSleep = client.eventPoll(events: &events) + for event in events { + switch event { + case .statistics(let statistics): + self.configuration.metrics.update(with: statistics) + case .rebalance(let rebalance): + self.logger.info("rebalance received \(rebalance), source nil: \(eventSource == nil)") + if let eventSource { + _ = eventSource.yield(.rebalance(rebalance)) + } else { + try await client.assign(topicPartitionList: nil) // fallback + } + default: + break // Ignore } } - case .pollForMessagesIfAvailable(let client, let source): - let messageResults = self.batchConsumerPoll(client: client) - if messageResults.isEmpty { - // Still no new messages, so sleep. - try await Task.sleep(for: self.configuration.pollInterval) + if shouldSleep { + pollInterval = min(self.configuration.pollInterval, pollInterval * 2) + try await Task.sleep(for: pollInterval) } else { - // New messages were produced to the partition that we previously finished reading. - let yieldResult = source.yield(contentsOf: messageResults) - switch yieldResult { - case .produceMore: - break - case .stopProducing: - self.stateMachine.withLockedValue { $0.stopProducing() } - case .dropped: - return - } + pollInterval = max(pollInterval / 3, .microseconds(1)) + await Task.yield() } - case .suspendPollLoop: - try await Task.sleep(for: self.configuration.pollInterval) case .terminatePollLoop: return } } } - /// Read `maxMessages` consumer messages from Kafka. - /// - /// - Parameters: - /// - client: Client used for handling the connection to the Kafka cluster. - /// - maxMessages: Maximum amount of consumer messages to read in this invocation. - private func batchConsumerPoll( - client: RDKafkaClient, - maxMessages: Int = 100 - ) -> [Result] { - var messageResults = [Result]() - messageResults.reserveCapacity(maxMessages) - - for _ in 0..? - do { - if let message = try client.consumerPoll() { - result = .success(message) - } - } catch { - result = .failure(error) - } - - if let result { - messageResults.append(result) - } - } - - return messageResults - } - /// Mark all messages up to the passed message in the topic as read. /// Schedules a commit and returns immediately. /// Any errors encountered after scheduling the commit will be discarded. @@ -543,6 +588,10 @@ public final class KafkaConsumer: Sendable, Service { } } } + + func client() throws -> RDKafkaClient { + return try self.stateMachine.withLockedValue { try $0.client() } + } } // MARK: - KafkaConsumer + StateMachine @@ -550,25 +599,6 @@ public final class KafkaConsumer: Sendable, Service { extension KafkaConsumer { /// State machine representing the state of the ``KafkaConsumer``. struct StateMachine: Sendable { - /// State of the event loop fetching new consumer messages. - enum MessagePollLoopState { - /// The sequence can take more messages. - /// - /// - Parameter source: The source for yielding new messages. - case running(source: Producer.Source) - /// Sequence suspended due to back pressure. - /// - /// - Parameter source: The source for yielding new messages. - case suspended(source: Producer.Source) - /// We have read to the end of a partition and are now waiting for new messages - /// to be produced. - /// - /// - Parameter source: The source for yielding new messages. - case waitingForMessages(source: Producer.Source) - /// The sequence has finished, and no more messages will be produced. - case finished - } - /// The state of the ``StateMachine``. enum State: Sendable { /// The state machine has been initialized with init() but is not yet Initialized @@ -579,20 +609,22 @@ extension KafkaConsumer { /// /// - Parameter client: Client used for handling the connection to the Kafka cluster. /// - Parameter source: The source for yielding new messages. + /// - Parameter eventSource: ``NIOAsyncSequenceProducer/Source`` used for yielding new events. case initializing( client: RDKafkaClient, - source: Producer.Source + eventSource: ProducerEvents.Source? ) /// The ``KafkaConsumer`` is consuming messages. /// /// - Parameter client: Client used for handling the connection to the Kafka cluster. - /// - Parameter state: State of the event loop fetching new consumer messages. - case running(client: RDKafkaClient, messagePollLoopState: MessagePollLoopState) + /// - Parameter eventSource: ``NIOAsyncSequenceProducer/Source`` used for yielding new events. + case running(client: RDKafkaClient, eventSource: ProducerEvents.Source?) /// The ``KafkaConsumer/triggerGracefulShutdown()`` has been invoked. /// We are now in the process of commiting our last state to the broker. /// /// - Parameter client: Client used for handling the connection to the Kafka cluster. - case finishing(client: RDKafkaClient) + /// - Parameter eventSource: ``NIOAsyncSequenceProducer/Source`` used for yielding new events. + case finishing(client: RDKafkaClient, eventSource: ProducerEvents.Source?) /// The ``KafkaConsumer`` is closed. case finished } @@ -604,23 +636,26 @@ extension KafkaConsumer { /// not yet available when the normal initialization occurs. mutating func initialize( client: RDKafkaClient, - source: Producer.Source + eventSource: ProducerEvents.Source? ) { guard case .uninitialized = self.state else { fatalError("\(#function) can only be invoked in state .uninitialized, but was invoked in state \(self.state)") } self.state = .initializing( client: client, - source: source + eventSource: eventSource ) } /// Action to be taken when wanting to poll for a new message. enum EventPollLoopAction { - /// Serve any queued callbacks on the event queue. + /// The ``KafkaConsumer`` stopped consuming messages or + /// is in the process of shutting down. + /// Poll to serve any queued events and commit outstanding state to the broker. /// /// - Parameter client: Client used for handling the connection to the Kafka cluster. - case pollForEvents(client: RDKafkaClient) + /// - Parameter eventSource: ``NIOAsyncSequenceProducer/Source`` used for yielding new elements. + case pollForEvents(client: RDKafkaClient, eventSource: ProducerEvents.Source?) /// Terminate the poll loop. case terminatePollLoop } @@ -635,14 +670,14 @@ extension KafkaConsumer { fatalError("\(#function) invoked while still in state \(self.state)") case .initializing: fatalError("Subscribe to consumer group / assign to topic partition pair before reading messages") - case .running(let client, _): - return .pollForEvents(client: client) - case .finishing(let client): + case .running(let client, let eventSource): + return .pollForEvents(client: client, eventSource: eventSource) + case .finishing(let client, let eventSource): if client.isConsumerClosed { self.state = .finished return .terminatePollLoop } else { - return .pollForEvents(client: client) + return .pollForEvents(client: client, eventSource: eventSource) } case .finished: return .terminatePollLoop @@ -654,20 +689,7 @@ extension KafkaConsumer { /// Poll for a new ``KafkaConsumerMessage``. /// /// - Parameter client: Client used for handling the connection to the Kafka cluster. - /// - Parameter source: ``NIOAsyncSequenceProducer/Source`` used for yielding new elements. - case pollForAndYieldMessages( - client: RDKafkaClient, - source: Producer.Source - ) - /// Poll for a new ``KafkaConsumerMessage`` or sleep for ``KafkaConsumerConfiguration/pollInterval`` - /// if there are no new messages to read from the partition. - /// - /// - Parameter client: Client used for handling the connection to the Kafka cluster. - /// - Parameter source: ``NIOAsyncSequenceProducer/Source`` used for yielding new elements. - case pollForMessagesIfAvailable( - client: RDKafkaClient, - source: Producer.Source - ) + case poll(client: RDKafkaClient) /// Sleep for ``KafkaConsumerConfiguration/pollInterval``. case suspendPollLoop /// Terminate the poll loop. @@ -683,18 +705,9 @@ extension KafkaConsumer { case .uninitialized: fatalError("\(#function) invoked while still in state \(self.state)") case .initializing: - fatalError("Subscribe to consumer group / assign to topic partition pair before reading messages") - case .running(let client, let consumerState): - switch consumerState { - case .running(let source): - return .pollForAndYieldMessages(client: client, source: source) - case .suspended(source: _): - return .suspendPollLoop - case .waitingForMessages(let source): - return .pollForMessagesIfAvailable(client: client, source: source) - case .finished: - return .terminatePollLoop - } + return .suspendPollLoop + case .running(let client, _): + return .poll(client: client) case .finishing, .finished: return .terminatePollLoop } @@ -705,6 +718,8 @@ extension KafkaConsumer { /// Set up the connection through ``subscribe()`` or ``assign()``. /// - Parameter client: Client used for handling the connection to the Kafka cluster. case setUpConnection(client: RDKafkaClient) + /// The ``KafkaConsumer`` is closed. + case consumerClosed } /// Get action to be taken when wanting to set up the connection through ``subscribe()`` or ``assign()``. @@ -714,13 +729,15 @@ extension KafkaConsumer { switch self.state { case .uninitialized: fatalError("\(#function) invoked while still in state \(self.state)") - case .initializing(let client, let source): - self.state = .running(client: client, messagePollLoopState: .running(source: source)) + case .initializing(let client, let eventSource): + self.state = .running(client: client, eventSource: eventSource) return .setUpConnection(client: client) case .running: fatalError("\(#function) should not be invoked more than once") - case .finishing, .finished: + case .finishing: fatalError("\(#function) should only be invoked when KafkaConsumer is running") + case .finished: + return .consumerClosed } } @@ -793,90 +810,68 @@ extension KafkaConsumer { fatalError("\(#function) invoked while still in state \(self.state)") case .initializing: fatalError("Subscribe to consumer group / assign to topic partition pair before reading messages") - case .running(let client, _): - self.state = .finishing(client: client) + case .running(let client, let eventSource), + .finishing(let client, let eventSource): + self.state = .finishing(client: client, eventSource: eventSource) return .triggerGracefulShutdown(client: client) - case .finishing, .finished: + case .finished: return nil } } - - // MARK: - Consumer Messages Poll Loop Actions - - /// The partition that was previously finished reading has got new messages produced to it. - mutating func newMessagesProduced() { - guard case .running(let client, let consumerState) = self.state else { - fatalError("\(#function) invoked while still in state \(self.state)") - } - - switch consumerState { - case .running, .suspended, .finished: - fatalError("\(#function) should not be invoked in state \(self.state)") - case .waitingForMessages(let source): - self.state = .running(client: client, messagePollLoopState: .running(source: source)) - } + + enum RebalanceAction { + /// Rebalance is still possible + /// + /// - Parameter client: Client used for handling the connection to the Kafka cluster. + case allowed( + client: RDKafkaClient + ) + /// Throw an error. The ``KafkaConsumer`` is closed. + case denied(error: String) } - /// The consumer has read to the end of a partition and shall now go into a sleep loop until new messages are produced. - mutating func waitForNewMessages() { - guard case .running(let client, let consumerState) = self.state else { - fatalError("\(#function) invoked while still in state \(self.state)") - } - - switch consumerState { - case .running(let source): - self.state = .running(client: client, messagePollLoopState: .waitingForMessages(source: source)) - case .suspended, .waitingForMessages, .finished: + + func seekOrRebalance() -> RebalanceAction { + switch state { + case .uninitialized: + fatalError("\(#function) should not be invoked in state \(self.state)") + case .initializing: fatalError("\(#function) should not be invoked in state \(self.state)") + case .running(let client, _): + return .allowed(client: client) + case .finishing(let client, _): + return .allowed(client: client) + case .finished: + return .denied(error: "Consumer finished") } } - /// ``KafkaConsumerMessages``'s back pressure mechanism asked us to produce more messages. - mutating func produceMore() { + /// The ``KafkaConsumerMessages`` asynchronous sequence was terminated. + mutating func finishMessageConsumption() { switch self.state { case .uninitialized: fatalError("\(#function) invoked while still in state \(self.state)") case .initializing: - break // This case can be triggered by the KafkaConsumerMessagesDeletgate - case .running(let client, let consumerState): - switch consumerState { - case .running, .waitingForMessages, .finished: - break - case .suspended(let source): - self.state = .running(client: client, messagePollLoopState: .running(source: source)) - } + self.state = .finished + case .running(let client, let eventSource): + self.state = .finishing(client: client, eventSource: eventSource) case .finishing, .finished: break } } - - /// ``KafkaConsumerMessages``'s back pressure mechanism asked us to temporarily stop producing messages. - mutating func stopProducing() { - guard case .running(let client, let consumerState) = self.state else { - fatalError("\(#function) invoked while still in state \(self.state)") - } - - switch consumerState { - case .suspended, .finished: - break - case .running(let source): - self.state = .running(client: client, messagePollLoopState: .suspended(source: source)) - case .waitingForMessages(let source): - self.state = .running(client: client, messagePollLoopState: .suspended(source: source)) - } - } - - /// The ``KafkaConsumerMessages`` asynchronous sequence was terminated. - mutating func finishMessageConsumption() { + + func client() throws -> RDKafkaClient { switch self.state { case .uninitialized: fatalError("\(#function) invoked while still in state \(self.state)") - case .initializing: - fatalError("Subscribe to consumer group / assign to topic partition pair before reading messages") + case .initializing(let client, _): + return client case .running(let client, _): - self.state = .running(client: client, messagePollLoopState: .finished) - case .finishing, .finished: - break + return client + case .finishing(let client, _): + return client + case .finished: + throw KafkaError.client(reason: "Consumer is finished") } } } diff --git a/Sources/Kafka/KafkaConsumerEvent.swift b/Sources/Kafka/KafkaConsumerEvent.swift index f654797b..62ad313e 100644 --- a/Sources/Kafka/KafkaConsumerEvent.swift +++ b/Sources/Kafka/KafkaConsumerEvent.swift @@ -12,13 +12,103 @@ // //===----------------------------------------------------------------------===// +public struct KafkaTopicList { + let list: RDKafkaTopicPartitionList + + init(from: RDKafkaTopicPartitionList) { + self.list = from + } + + public init(size: Int32 = 1) { + self.list = RDKafkaTopicPartitionList(size: size) + } + + public func append(topic: TopicPartition) { + self.list.setOffset(topic: topic.topic, partition: topic.partition, offset: topic.offset) + } +} + +public struct TopicPartition { + public let topic: String + public let partition: KafkaPartition + public let offset: KafkaOffset + + public init(_ topic: String, _ partition: KafkaPartition, _ offset: KafkaOffset) { + self.topic = topic + self.partition = partition + self.offset = offset + } +} + +extension TopicPartition: Sendable {} +extension TopicPartition: Hashable {} + +extension KafkaTopicList : Sendable {} +extension KafkaTopicList : Hashable {} + +//extension KafkaTopicList : CustomDebugStringConvertible { +// public var debugDescription: String { +// list.debugDescription +// } +//} + +extension KafkaTopicList : Sequence { + public struct TopicPartitionIterator : IteratorProtocol { + private let list: RDKafkaTopicPartitionList + private var idx = 0 + + init(list: RDKafkaTopicPartitionList) { + self.list = list + } + + mutating public func next() -> TopicPartition? { + guard let topic = list.getByIdx(idx: idx) else { + return nil + } + idx += 1 + return topic + } + } + + public func makeIterator() -> TopicPartitionIterator { + TopicPartitionIterator(list: self.list) + } +} + +public enum KafkaRebalanceProtocol: Sendable, Hashable { + case cooperative + case eager + case none + + static func convert(from proto: String) -> KafkaRebalanceProtocol{ + switch proto { + case "COOPERATIVE": return .cooperative + case "EAGER": return .eager + default: return .none + } + } +} + + +public enum RebalanceAction : Sendable, Hashable { + case assign(KafkaRebalanceProtocol, KafkaTopicList) + case revoke(KafkaRebalanceProtocol, KafkaTopicList) + case error(KafkaRebalanceProtocol, KafkaTopicList, KafkaError) +} + /// An enumeration representing events that can be received through the ``KafkaConsumerEvents`` asynchronous sequence. public enum KafkaConsumerEvent: Sendable, Hashable { + /// Rebalance from librdkafka + case rebalance(RebalanceAction) /// - Important: Always provide a `default` case when switiching over this `enum`. case DO_NOT_SWITCH_OVER_THIS_EXHAUSITVELY internal init(_ event: RDKafkaClient.KafkaEvent) { switch event { + case .statistics: + fatalError("Cannot cast \(event) to KafkaConsumerEvent") + case .rebalance(let action): + self = .rebalance(action) case .deliveryReport: fatalError("Cannot cast \(event) to KafkaConsumerEvent") } diff --git a/Sources/Kafka/KafkaConsumerMessage.swift b/Sources/Kafka/KafkaConsumerMessage.swift index c4a6570a..6295a332 100644 --- a/Sources/Kafka/KafkaConsumerMessage.swift +++ b/Sources/Kafka/KafkaConsumerMessage.swift @@ -17,6 +17,14 @@ import NIOCore /// A message received from the Kafka cluster. public struct KafkaConsumerMessage { + /// Internal enum for EOF, required to allow empty message + internal enum MessageContent: Hashable, Sendable { + case buffer(ByteBuffer) + case eof + } + + internal var _value: MessageContent + /// The topic that the message was received from. public var topic: String /// The partition that the message was received from. @@ -26,10 +34,41 @@ public struct KafkaConsumerMessage { /// The key of the message. public var key: ByteBuffer? /// The body of the message. - public var value: ByteBuffer + public var value: ByteBuffer { + switch _value { + case .buffer(let byteBuffer): + return byteBuffer + case .eof: + return ByteBuffer() + } + } /// The offset of the message in its partition. public var offset: KafkaOffset + /// If ``true``, means it is not a message but partition EOF event + public var eof: Bool { + switch _value { + case .buffer: + return false + case .eof: + return true + } + } + + /// Initialize ``KafkaConsumerMessage`` as EOF from `rd_kafka_topic_partition_t` pointer. + /// - Throws: A ``KafkaError`` if the received message is an error message or malformed. +// internal init(topicPartitionPointer: UnsafePointer) { +// let topicPartition = topicPartitionPointer.pointee +// guard let topic = String(validatingUTF8: topicPartition.topic) else { +// fatalError("Received topic name that is non-valid UTF-8") +// } +// self.topic = topic +// self.partition = KafkaPartition(rawValue: Int(topicPartition.partition)) +// self.offset = KafkaOffset(rawValue: Int(topicPartition.offset)) +// self.value = ByteBuffer() +// self.headers = [KafkaHeader]() +// } + /// Initialize ``KafkaConsumerMessage`` from `rd_kafka_message_t` pointer. /// - Throws: A ``KafkaError`` if the received message is an error message or malformed. internal init(messagePointer: UnsafePointer) throws { @@ -41,7 +80,7 @@ public struct KafkaConsumerMessage { let valueBufferPointer = UnsafeRawBufferPointer(start: valuePointer, count: rdKafkaMessage.len) - guard rdKafkaMessage.err == RD_KAFKA_RESP_ERR_NO_ERROR else { + guard rdKafkaMessage.err == RD_KAFKA_RESP_ERR_NO_ERROR || rdKafkaMessage.err == RD_KAFKA_RESP_ERR__PARTITION_EOF else { var errorStringBuffer = ByteBuffer(bytes: valueBufferPointer) let errorString = errorStringBuffer.readString(length: errorStringBuffer.readableBytes) @@ -59,20 +98,26 @@ public struct KafkaConsumerMessage { self.partition = KafkaPartition(rawValue: Int(rdKafkaMessage.partition)) - self.headers = try Self.getHeaders(for: messagePointer) - - if let keyPointer = rdKafkaMessage.key { - let keyBufferPointer = UnsafeRawBufferPointer( - start: keyPointer, - count: rdKafkaMessage.key_len - ) - self.key = .init(bytes: keyBufferPointer) + if rdKafkaMessage.err != RD_KAFKA_RESP_ERR__PARTITION_EOF { + self.headers = try Self.getHeaders(for: messagePointer) + + if let keyPointer = rdKafkaMessage.key { + let keyBufferPointer = UnsafeRawBufferPointer( + start: keyPointer, + count: rdKafkaMessage.key_len + ) + self.key = .init(bytes: keyBufferPointer) + } else { + self.key = nil + } + + self._value = .buffer(ByteBuffer(bytes: valueBufferPointer)) } else { - self.key = nil + self._value = .eof + self.key = .init() + self.headers = .init() } - self.value = ByteBuffer(bytes: valueBufferPointer) - self.offset = KafkaOffset(rawValue: Int(rdKafkaMessage.offset)) } } diff --git a/Sources/Kafka/KafkaError.swift b/Sources/Kafka/KafkaError.swift index ddaf119a..07978168 100644 --- a/Sources/Kafka/KafkaError.swift +++ b/Sources/Kafka/KafkaError.swift @@ -44,6 +44,10 @@ public struct KafkaError: Error, CustomStringConvertible, @unchecked Sendable { private var line: UInt { self.backing.line } + + public var isFatal: Bool { + self.backing.isFatal + } public var description: String { "KafkaError.\(self.code): \(self.reason) \(self.file):\(self.line)" @@ -56,7 +60,7 @@ public struct KafkaError: Error, CustomStringConvertible, @unchecked Sendable { } static func rdKafkaError( - wrapping error: rd_kafka_resp_err_t, file: String = #fileID, line: UInt = #line + wrapping error: rd_kafka_resp_err_t, isFatal: Bool = false, file: String = #fileID, line: UInt = #line ) -> KafkaError { let errorMessage = String(cString: rd_kafka_err2str(error)) return KafkaError( @@ -135,6 +139,44 @@ public struct KafkaError: Error, CustomStringConvertible, @unchecked Sendable { ) ) } + + static func transactionAborted( + reason: String, file: String = #fileID, line: UInt = #line + ) -> KafkaError { + return KafkaError( + backing: .init( + code: .transactionAborted, reason: reason, file: file, line: line + ) + ) + } + + static func transactionIncomplete( + reason: String, file: String = #fileID, line: UInt = #line + ) -> KafkaError { + return KafkaError( + backing: .init( + code: .transactionIncomplete, reason: reason, file: file, line: line + ) + ) + } + + static func transactionOutOfAttempts( + numOfAttempts: UInt64, file: String = #fileID, line: UInt = #line + ) -> KafkaError { + return KafkaError( + backing: .init( + code: .transactionOutOfAttempts, reason: "Out of \(numOfAttempts) attempts", file: file, line: line + ) + ) + } + + static func partitionEOF(file: String = #fileID, line: UInt = #line) -> KafkaError { + return KafkaError( + backing: .init( + code: .partitionEOF, reason: "Partition EOF", file: file, line: line + ) + ) + } } extension KafkaError { @@ -153,6 +195,11 @@ extension KafkaError { case messageConsumption case topicCreation case topicDeletion + case transactionAborted + case transactionIncomplete + case notInTransaction // FIXME: maybe add subcode ? + case transactionOutOfAttempts + case partitionEOF } fileprivate var backingCode: BackingCode @@ -177,6 +224,14 @@ extension KafkaError { public static let topicCreationFailed = ErrorCode(.topicCreation) /// Deleting a topic failed. public static let topicDeletionFailed = ErrorCode(.topicDeletion) + /// Transaction was aborted (can be re-tried from scratch). + public static let transactionAborted = ErrorCode(.transactionAborted) + /// Transaction could not be completed + public static let transactionIncomplete = ErrorCode(.transactionIncomplete) + /// Out of provided number of attempts + public static let transactionOutOfAttempts = ErrorCode(.transactionOutOfAttempts) + /// Out of provided number of attempts + public static let partitionEOF = ErrorCode(.partitionEOF) public var description: String { return String(describing: self.backingCode) @@ -196,16 +251,20 @@ extension KafkaError { let line: UInt + let isFatal: Bool + fileprivate init( code: KafkaError.ErrorCode, reason: String, file: String, - line: UInt + line: UInt, + isFatal: Bool = false ) { self.code = code self.reason = reason self.file = file self.line = line + self.isFatal = isFatal } // Only the error code matters for equality. diff --git a/Sources/Kafka/KafkaGroup.swift b/Sources/Kafka/KafkaGroup.swift new file mode 100644 index 00000000..dd11f7e8 --- /dev/null +++ b/Sources/Kafka/KafkaGroup.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Crdkafka + +public struct KafkaGroup { + public let name: String + public let broker: KafkaMetadataBroker + public let state: String + public let protocolType: String + public let `protocol`: String +} + +extension KafkaGroup { + public static func list(configuration: KafkaGroupConfiguration, + group: String? = nil, + retries: Int = 5, + timeout: Duration = .seconds(5)) throws -> [KafkaGroup] { + let configDictionary = configuration.dictionary + let rdConfig = try RDKafkaConfig.createFrom(configDictionary: configDictionary) + + let errorChars = UnsafeMutablePointer.allocate(capacity: RDKafkaClient.stringSize) + defer { errorChars.deallocate() } + + guard let kafkaHandle = rd_kafka_new(RD_KAFKA_PRODUCER, rdConfig, errorChars, RDKafkaClient.stringSize) else { + rd_kafka_conf_destroy(rdConfig) + let errorString = String(cString: errorChars) + throw KafkaError.client(reason: errorString) + } + defer { rd_kafka_destroy(kafkaHandle) } + + let rdGroup = group?.cString(using: .utf8) + let timeoutMs = Int32(timeout.inMilliseconds) + var err = RD_KAFKA_RESP_ERR_NO_ERROR + var grplist: UnsafePointer? = nil + var retries = min(retries, 1) + while true { + err = rd_kafka_list_groups(kafkaHandle, rdGroup, &grplist, timeoutMs) + if err == RD_KAFKA_RESP_ERR_NO_ERROR { + break + } + else if (err == RD_KAFKA_RESP_ERR__TRANSPORT) || (err == RD_KAFKA_RESP_ERR_COORDINATOR_LOAD_IN_PROGRESS) { + retries -= 1 + if retries == 0 { + throw KafkaError.rdKafkaError(wrapping: err) + } + } else { + throw KafkaError.rdKafkaError(wrapping: err) + } + } + + defer { rd_kafka_group_list_destroy(grplist) } + + if let grplist { + var groups = [KafkaGroup]() + for idx in 0.. + + init(metadata: UnsafePointer) { + self.metadata = metadata + } + + deinit { + rd_kafka_metadata_destroy(metadata) + } + + public private(set) lazy var topics = { + (0.. /// The configuration object of the producer client. - private let configuration: KafkaProducerConfiguration + private let configuration: KafkaProducerSharedProperties /// Topic configuration that is used when a new topic has to be created by the producer. private let topicConfiguration: KafkaTopicConfiguration @@ -93,7 +121,7 @@ public final class KafkaProducer: Service, Sendable { /// - Throws: A ``KafkaError`` if initializing the producer failed. private init( stateMachine: NIOLockedValueBox, - configuration: KafkaProducerConfiguration, + configuration: KafkaProducerSharedProperties, topicConfiguration: KafkaTopicConfiguration ) { self.stateMachine = stateMachine @@ -113,13 +141,26 @@ public final class KafkaProducer: Service, Sendable { public convenience init( configuration: KafkaProducerConfiguration, logger: Logger + ) throws { + try self.init(configuration: configuration as KafkaProducerSharedProperties, logger: logger) + } + + internal convenience init( + configuration: KafkaProducerSharedProperties, + logger: Logger ) throws { let stateMachine = NIOLockedValueBox(StateMachine(logger: logger)) + var subscribedEvents: [RDKafkaEvent] = [.log] // No .deliveryReport here! + + if configuration.metrics.enabled { + subscribedEvents.append(.statistics) + } + let client = try RDKafkaClient.makeClient( type: .producer, configDictionary: configuration.dictionary, - events: [.log], // No .deliveryReport here! + events: subscribedEvents, logger: logger ) @@ -153,13 +194,27 @@ public final class KafkaProducer: Service, Sendable { public static func makeProducerWithEvents( configuration: KafkaProducerConfiguration, logger: Logger + ) throws -> (KafkaProducer, KafkaProducerEvents) { + return try self.makeProducerWithEvents(configuration: configuration as (any KafkaProducerSharedProperties), logger: logger) + } + + internal static func makeProducerWithEvents( + configuration: KafkaProducerSharedProperties, + topicConfig: KafkaTopicConfiguration = KafkaTopicConfiguration(), + logger: Logger ) throws -> (KafkaProducer, KafkaProducerEvents) { let stateMachine = NIOLockedValueBox(StateMachine(logger: logger)) + var subscribedEvents: [RDKafkaEvent] = [.log, .deliveryReport] + // Listen to statistics events when statistics enabled + if configuration.metrics.enabled { + subscribedEvents.append(.statistics) + } + let client = try RDKafkaClient.makeClient( type: .producer, configDictionary: configuration.dictionary, - events: [.log, .deliveryReport], + events: subscribedEvents, logger: logger ) @@ -177,6 +232,7 @@ public final class KafkaProducer: Service, Sendable { let sourceAndSequence = NIOAsyncSequenceProducer.makeSequence( elementType: KafkaProducerEvent.self, backPressureStrategy: NIOAsyncSequenceProducerBackPressureStrategies.NoBackPressure(), + finishOnDeinit: true, delegate: KafkaProducerCloseOnTerminate(stateMachine: stateMachine) ) @@ -203,27 +259,43 @@ public final class KafkaProducer: Service, Sendable { } private func _run() async throws { + var pollInterval = self.configuration.pollInterval + var events = [RDKafkaClient.KafkaEvent]() while !Task.isCancelled { let nextAction = self.stateMachine.withLockedValue { $0.nextPollLoopAction() } switch nextAction { case .pollWithoutYield(let client): // Drop any incoming events - let _ = client.eventPoll() + _ = client.eventPoll(events: &events) case .pollAndYield(let client, let source): - let events = client.eventPoll() + let shouldSleep = client.eventPoll(events: &events) for event in events { - let producerEvent = KafkaProducerEvent(event) - // Ignore YieldResult as we don't support back pressure in KafkaProducer - _ = source?.yield(producerEvent) + switch event { + case .deliveryReport(let reports): + // Ignore YieldResult as we don't support back pressure in KafkaProducer + _ = source?.yield(.deliveryReports(reports)) + case .statistics(let statistics): + self.configuration.metrics.update(with: statistics) + default: + fatalError("Cannot cast \(event) to KafkaProducerEvent") + } + } + if shouldSleep { + pollInterval = min(self.configuration.pollInterval, pollInterval * 2) + try await Task.sleep(for: pollInterval) + } else { + pollInterval = max(pollInterval / 3, .microseconds(1)) + await Task.yield() } - try await Task.sleep(for: self.configuration.pollInterval) case .flushFinishSourceAndTerminatePollLoop(let client, let source): precondition( 0...Int(Int32.max) ~= self.configuration.flushTimeoutMilliseconds, "Flush timeout outside of valid range \(0...Int32.max)" ) + defer { // we should finish source indefinetely of exception in client.flush() + source?.finish() + } try await client.flush(timeoutMilliseconds: Int32(self.configuration.flushTimeoutMilliseconds)) - source?.finish() return case .terminatePollLoop: return @@ -263,6 +335,46 @@ public final class KafkaProducer: Service, Sendable { return KafkaProducerMessageID(rawValue: newMessageID) } } + + @discardableResult + public func flush(timeout: Duration) async -> Bool { + do { + let client = try client() + try await client.flush(timeoutMilliseconds: Int32(timeout.inMilliseconds)) + } catch { + return false + } + return true + } + + public func partitionForKey(_ key: some KafkaContiguousBytes, in topic: String, partitionCount: Int) -> KafkaPartition? { + self.stateMachine.withLockedValue { (stateMachine) -> KafkaPartition? in + guard let topicHandles = stateMachine.topicHandles else { + return nil + } + + let partition: Int32? = try? topicHandles.withTopicHandlePointer(topic: topic, topicConfiguration: topicConfiguration) { topicHandle in + key.withUnsafeBytes { buffer in + switch topicConfiguration.partitioner { + case .random: nil + case .consistent: rd_kafka_msg_partitioner_consistent_random(topicHandle, buffer.baseAddress, buffer.count, Int32(partitionCount), nil, nil) + case .consistentRandom: buffer.count == 0 ? nil : rd_kafka_msg_partitioner_consistent_random(topicHandle, buffer.baseAddress, buffer.count, Int32(partitionCount), nil, nil) + case .murmur2: rd_kafka_msg_partitioner_murmur2(topicHandle, buffer.baseAddress, buffer.count, Int32(partitionCount), nil, nil) + case .murmur2Random: buffer.count == 0 ? nil : rd_kafka_msg_partitioner_murmur2_random(topicHandle, buffer.baseAddress, buffer.count, Int32(partitionCount), nil, nil) + case .fnv1a: rd_kafka_msg_partitioner_fnv1a(topicHandle, buffer.baseAddress, buffer.count, Int32(partitionCount), nil, nil) + case .fnv1aRandom: buffer.count == 0 ? nil : rd_kafka_msg_partitioner_fnv1a_random(topicHandle, buffer.baseAddress, buffer.count, Int32(partitionCount), nil, nil) + default: nil + } + } + } + + return partition.map { KafkaPartition(rawValue: Int($0)) } + } + } + + func client() throws -> RDKafkaClient { + try self.stateMachine.withLockedValue { try $0.client() } + } } // MARK: - KafkaProducer + StateMachine @@ -453,5 +565,27 @@ extension KafkaProducer { break } } + + func client() throws -> RDKafkaClient { + switch self.state { + case .uninitialized: + fatalError("\(#function) invoked while still in state \(self.state)") + case .started(let client, _, _, _): + return client + case .eventConsumptionFinished(let client): + return client + case .finishing(let client, _): + return client + case .finished: + throw KafkaError.connectionClosed(reason: "Client stopped") + } + } + + var topicHandles: RDKafkaTopicHandles? { + if case .started(_, _, _, let topicHandles) = self.state { + return topicHandles + } + return nil + } } } diff --git a/Sources/Kafka/KafkaProducerEvent.swift b/Sources/Kafka/KafkaProducerEvent.swift index 9684f146..3e2ee6b4 100644 --- a/Sources/Kafka/KafkaProducerEvent.swift +++ b/Sources/Kafka/KafkaProducerEvent.swift @@ -23,6 +23,10 @@ public enum KafkaProducerEvent: Sendable, Hashable { switch event { case .deliveryReport(results: let results): self = .deliveryReports(results) + case .rebalance(_): + fatalError("Cannot cast \(event) to KafkaProducerEvent") + case .statistics: + fatalError("Cannot cast \(event) to KafkaProducerEvent") } } } diff --git a/Sources/Kafka/KafkaProducerMessage.swift b/Sources/Kafka/KafkaProducerMessage.swift index f9bc1ec2..5b17d983 100644 --- a/Sources/Kafka/KafkaProducerMessage.swift +++ b/Sources/Kafka/KafkaProducerMessage.swift @@ -48,7 +48,7 @@ public struct KafkaProducerMessage(_ message: KafkaProducerMessage) throws -> KafkaProducerMessageID { + sendTries += 1 + let id = try self.producer.send(message) + totalBytes += message.value.withUnsafeBytes({ $0.count }) + msgNum += 1 + return id + } + + + public func flush(timeout: Duration) async { + self.logger.debug("Flushing transaction msgNum: \(msgNum), offsetSend: \(offsetSend), offsetNum: \(offsetNum), totalBytes: \(totalBytes), sendTries: \(sendTries)") + await self.producer.flush(timeout: timeout) + } + + + func commit() async throws { + self.logger.debug("Committing transaction msgNum: \(msgNum), offsetSend: \(offsetSend), offsetNum: \(offsetNum), totalBytes: \(totalBytes), sendTries: \(sendTries)") + try await self.client.commitTransaction(attempts: .max, timeout: .kafkaUntilEndOfTransactionTimeout) + } + + func abort() async throws { + self.logger.debug("Aborting transaction msgNum: \(msgNum), offsetSend: \(offsetSend), offsetNum: \(offsetNum), totalBytes: \(totalBytes), sendTries: \(sendTries)") + try await self.client.abortTransaction(attempts: .max, timeout: .kafkaUntilEndOfTransactionTimeout) + } +} diff --git a/Sources/Kafka/KafkaTransactionalProducer.swift b/Sources/Kafka/KafkaTransactionalProducer.swift new file mode 100644 index 00000000..bb427cb2 --- /dev/null +++ b/Sources/Kafka/KafkaTransactionalProducer.swift @@ -0,0 +1,116 @@ +import Logging +import ServiceLifecycle +import Atomics + +public final class KafkaTransactionalProducer: Service, Sendable { + private let producer: KafkaProducer + private let logger: Logger + + private let id: ManagedAtomic = .init(0) + + private init(producer: KafkaProducer, config: KafkaTransactionalProducerConfiguration, logger: Logger) async throws { + self.producer = producer + self.logger = logger + let client = try producer.client() + try await client.initTransactions(timeout: config.transactionsTimeout) + } + + /// Initialize a new ``KafkaTransactionalProducer``. + /// + /// This creates a producer without listening for events. + /// + /// - Parameter config: The ``KafkaProducerConfiguration`` for configuring the ``KafkaProducer``. + /// - Parameter topicConfig: The ``KafkaTopicConfiguration`` used for newly created topics. + /// - Parameter logger: A logger. + /// - Returns: The newly created ``KafkaProducer``. + /// - Throws: A ``KafkaError`` if initializing the producer failed. + public convenience init( + config: KafkaTransactionalProducerConfiguration, + logger: Logger + ) async throws { + let producer = try KafkaProducer(configuration: config, logger: logger) + try await self.init(producer: producer, config: config, logger: logger) + } + + /// Initialize a new ``KafkaTransactionalProducer`` and a ``KafkaProducerEvents`` asynchronous sequence. + /// + /// Use the asynchronous sequence to consume events. + /// + /// - Important: When the asynchronous sequence is deinited the producer will be shutdown and disallow sending more messages. + /// Additionally, make sure to consume the asynchronous sequence otherwise the events will be buffered in memory indefinitely. + /// + /// - Parameter config: The ``KafkaProducerConfiguration`` for configuring the ``KafkaProducer``. + /// - Parameter topicConfig: The ``KafkaTopicConfiguration`` used for newly created topics. + /// - Parameter logger: A logger. + /// - Returns: A tuple containing the created ``KafkaProducer`` and the ``KafkaProducerEvents`` + /// `AsyncSequence` used for receiving message events. + /// - Throws: A ``KafkaError`` if initializing the producer failed. + public static func makeTransactionalProducerWithEvents( + config: KafkaTransactionalProducerConfiguration, + logger: Logger + ) async throws -> (KafkaTransactionalProducer, KafkaProducerEvents) { + let (producer, events) = try KafkaProducer.makeProducerWithEvents( + configuration: config, + logger: logger + ) + + let transactionalProducer = try await KafkaTransactionalProducer(producer: producer, config: config, logger: logger) + + return (transactionalProducer, events) + } + + // + public func withTransaction(_ body: @Sendable (KafkaTransaction) async throws -> Void) async throws { + let id = id.loadThenWrappingIncrement(ordering: .relaxed) + var logger = Logger(label: "Transaction \(id)") + logger.logLevel = self.logger.logLevel + + logger.debug("Begin txn \(id)") + defer { + logger.debug("End txn \(id)") + } + let transaction = try KafkaTransaction( + client: try producer.client(), + producer: self.producer, + logger: logger + ) + + do { // need to think here a little bit how to abort transaction + logger.debug("Fill the transaction \(id)") + try await body(transaction) + logger.debug("Committing the transaction \(id)") + try await transaction.commit() + } catch let error as KafkaError where error.code == .transactionAborted { + logger.debug("Transaction aborted by librdkafa") + throw error // transaction already aborted + } catch { // FIXME: maybe catch AbortTransaction? + logger.debug("Caught error for transaction \(id): \(error), aborting") + do { + try await transaction.abort() + logger.debug("Transaction \(id) aborted") + } catch { + logger.debug("Failed to perform abort for transaction \(id): \(error)") + // FIXME: that some inconsistent state + // should we emit fatalError(..) + // or propagate error as exception with isFgsatal flag? + } + throw error + } + } + + public func run() async throws { + try await self.producer.run() + } + + /// Method to shutdown the ``KafkaTransactionalProducer``. + /// + /// This method flushes any buffered messages and waits until a callback is received for all of them. + /// Afterwards, it shuts down the connection to Kafka and cleans any remaining state up. + public func triggerGracefulShutdown() { + self.producer.triggerGracefulShutdown() + } + + public func partitionForKey(_ key: some KafkaContiguousBytes, in topic: String, partitionCount: Int) -> KafkaPartition? { + self.producer.partitionForKey(key, in: topic, partitionCount: partitionCount) + } +} diff --git a/Sources/Kafka/RDKafka/RDKafkaClient.swift b/Sources/Kafka/RDKafka/RDKafkaClient.swift index 623f2c34..5db71654 100644 --- a/Sources/Kafka/RDKafka/RDKafkaClient.swift +++ b/Sources/Kafka/RDKafka/RDKafkaClient.swift @@ -14,11 +14,13 @@ import Crdkafka import Dispatch +import class Foundation.JSONDecoder import Logging /// Base class for ``KafkaProducer`` and ``KafkaConsumer``, /// which is used to handle the connection to the Kafka ecosystem. -final class RDKafkaClient: Sendable { +@_spi(Internal) +public final class RDKafkaClient: Sendable { // Default size for Strings returned from C API static let stringSize = 1024 @@ -35,16 +37,26 @@ final class RDKafkaClient: Sendable { /// `librdkafka`'s `rd_kafka_queue_t` that events are received on. private let queue: OpaquePointer + + private let rebalanceCallBackStorage: RebalanceCallbackStorage? + + /// Queue for blocking calls outside of cooperative thread pool + private var gcdQueue: DispatchQueue { + // global concurrent queue + .global(qos: .default) // FIXME: maybe DispatchQueue(label: "com.swift.kafka.queue") + } // Use factory method to initialize private init( type: ClientType, kafkaHandle: OpaquePointer, - logger: Logger + logger: Logger, + rebalanceCallBackStorage: RebalanceCallbackStorage? = nil ) { self.kafkaHandle = kafkaHandle self.logger = logger self.queue = rd_kafka_queue_get_main(self.kafkaHandle) + self.rebalanceCallBackStorage = rebalanceCallBackStorage rd_kafka_set_log_queue(self.kafkaHandle, self.queue) } @@ -55,12 +67,22 @@ final class RDKafkaClient: Sendable { rd_kafka_destroy(kafkaHandle) } + typealias RebalanceCallback = @Sendable (KafkaEvent) -> () + final class RebalanceCallbackStorage: Sendable { + let rebalanceCallback: RebalanceCallback + + init(rebalanceCallback: @escaping RebalanceCallback) { + self.rebalanceCallback = rebalanceCallback + } + } + /// Factory method creating a new instance of a ``RDKafkaClient``. static func makeClient( type: ClientType, configDictionary: [String: String], events: [RDKafkaEvent], - logger: Logger + logger: Logger, + rebalanceCallBackStorage: RebalanceCallbackStorage? = nil ) throws -> RDKafkaClient { let rdConfig = try RDKafkaConfig.createFrom(configDictionary: configDictionary) // Manually override some of the configuration options @@ -69,6 +91,34 @@ final class RDKafkaClient: Sendable { // KafkaConsumer is manually storing read offsets if type == .consumer { try RDKafkaConfig.set(configPointer: rdConfig, key: "enable.auto.offset.store", value: "false") + try RDKafkaConfig.set(configPointer: rdConfig, key: "enable.partition.eof", value: "true") + if let rebalanceCallBackStorage { + let rebalanceCb = Unmanaged.passUnretained(rebalanceCallBackStorage) + rd_kafka_conf_set_opaque(rdConfig, rebalanceCb.toOpaque()) + rd_kafka_conf_set_rebalance_cb(rdConfig) { handle, code, partitions, rebalanceOpaqueCb in + let protoStringDef = String(cString: rd_kafka_rebalance_protocol(handle)) + let rebalanceProtocol = KafkaRebalanceProtocol.convert(from: protoStringDef) + guard let partitions else { + fatalError("No partitions in callback") + } + let list = KafkaTopicList(from: .init(from: partitions)) + var event: KafkaEvent + switch code { + case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: + event = .rebalance(.assign(rebalanceProtocol, list)) + case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: + event = .rebalance(.revoke(rebalanceProtocol, list)) + default: + event = .rebalance(.error(rebalanceProtocol, list, KafkaError.rdKafkaError(wrapping: code))) + } + if let rebalanceOpaqueCb { + let rebalanceCb = Unmanaged.fromOpaque(rebalanceOpaqueCb).takeUnretainedValue() + rebalanceCb.rebalanceCallback(event) + } else { + fatalError("Cannot find rebalance cb") + } + } + } } RDKafkaConfig.setEvents(configPointer: rdConfig, events: events) @@ -89,7 +139,7 @@ final class RDKafkaClient: Sendable { throw KafkaError.client(reason: errorString) } - return RDKafkaClient(type: type, kafkaHandle: handle, logger: logger) + return RDKafkaClient(type: type, kafkaHandle: handle, logger: logger, rebalanceCallBackStorage: rebalanceCallBackStorage) } /// Produce a message to the Kafka cluster. @@ -148,7 +198,7 @@ final class RDKafkaClient: Sendable { } if error != RD_KAFKA_RESP_ERR_NO_ERROR { - throw KafkaError.rdKafkaError(wrapping: rd_kafka_last_error()) + throw KafkaError.rdKafkaError(wrapping: error) } } @@ -295,14 +345,19 @@ final class RDKafkaClient: Sendable { /// Swift wrapper for events from `librdkafka`'s event queue. enum KafkaEvent { case deliveryReport(results: [KafkaDeliveryReport]) + case statistics(RDKafkaStatistics) + case rebalance(RebalanceAction) } /// Poll the event `rd_kafka_queue_t` for new events. /// /// - Parameter maxEvents:Maximum number of events to serve in one invocation. - func eventPoll(maxEvents: Int = 100) -> [KafkaEvent] { - var events = [KafkaEvent]() + func eventPoll(events: inout [KafkaEvent], maxEvents: Int = 100) -> Bool /* should sleep */ { +// var events = [KafkaEvent]() + events.removeAll(keepingCapacity: true) events.reserveCapacity(maxEvents) + + var shouldSleep = true for _ in 0.. KafkaEvent { + guard let partitions = rd_kafka_event_topic_partition_list(event) else { + fatalError("Must never happen") // TODO: remove + } + + + let code = rd_kafka_event_error(event) + + let protoStringDef = String(cString: rd_kafka_rebalance_protocol(kafkaHandle)) + let rebalanceProtocol = KafkaRebalanceProtocol.convert(from: protoStringDef) + let list = KafkaTopicList(from: .init(from: partitions)) + switch code { + case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: + return .rebalance(.assign(rebalanceProtocol, list)) + case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: + return .rebalance(.revoke(rebalanceProtocol, list)) + default: + return .rebalance(.error(rebalanceProtocol, list, KafkaError.rdKafkaError(wrapping: code))) + } + } + /// Handle event of type `RDKafkaEvent.statistics`. + /// + /// - Parameter event: Pointer to underlying `rd_kafka_event_t`. + private func handleStatistics(_ event: OpaquePointer?) -> KafkaEvent? { + let jsonStr = String(cString: rd_kafka_event_stats(event)) + do { + if let jsonData = jsonStr.data(using: .utf8) { + let json = try JSONDecoder().decode(RDKafkaStatistics.self, from: jsonData) + return .statistics(json) + } + } catch { + assertionFailure("Error occurred when decoding JSON statistics: \(error) when decoding \(jsonStr)") + } + return nil + } + /// Handle event of type `RDKafkaEvent.log`. /// /// - Parameter event: Pointer to underlying `rd_kafka_event_t`. @@ -360,6 +481,7 @@ final class RDKafkaClient: Sendable { var buffer: UnsafePointer? var level: Int32 = 0 if rd_kafka_event_log(event, &faculty, &buffer, &level) == 0 { +// rd_kafka_event_debug_contexts if let faculty, let buffer { // Mapping according to https://en.wikipedia.org/wiki/Syslog switch level { @@ -399,6 +521,15 @@ final class RDKafkaClient: Sendable { guard let opaquePointer = rd_kafka_event_opaque(event) else { fatalError("Could not resolve reference to catpured Swift callback instance") } + + /* + let opaquePointer = rd_kafka_event_opaque(event) + guard let opaquePointer else { + let count = rd_kafka_event_message_count(event) + let str = String(cString: rd_kafka_event_name(event)) + fatalError("Could not resolve reference to catpured Swift callback instance for count \(count) in event \(str)") + } + */ let opaque = Unmanaged.fromOpaque(opaquePointer).takeUnretainedValue() let actualCallback = opaque.closure @@ -430,12 +561,59 @@ final class RDKafkaClient: Sendable { // Reached the end of the topic+partition queue on the broker if messagePointer.pointee.err == RD_KAFKA_RESP_ERR__PARTITION_EOF { - return nil + return try KafkaConsumerMessage(messagePointer: messagePointer) } let message = try KafkaConsumerMessage(messagePointer: messagePointer) return message } + /// Atomic incremental assignment of partitions to consume. + /// - Parameter topicPartitionList: Pointer to a list of topics + partition pairs. + func incrementalAssign(topicPartitionList: RDKafkaTopicPartitionList) async throws { + let error = await performBlockingCall(queue: self.gcdQueue) { + topicPartitionList.withListPointer { rd_kafka_incremental_assign(self.kafkaHandle, $0) } + } + + defer { rd_kafka_error_destroy(error) } + let code = rd_kafka_error_code(error) + if code != RD_KAFKA_RESP_ERR_NO_ERROR { + throw KafkaError.rdKafkaError(wrapping: code) + } + } + + /// Atomic incremental unassignment of partitions to consume. + /// - Parameter topicPartitionList: Pointer to a list of topics + partition pairs. + func incrementalUnassign(topicPartitionList: RDKafkaTopicPartitionList) async throws { + let error = await performBlockingCall(queue: self.gcdQueue) { + topicPartitionList.withListPointer { rd_kafka_incremental_unassign(self.kafkaHandle, $0) } + } + + defer { rd_kafka_error_destroy(error) } + let code = rd_kafka_error_code(error) + if code != RD_KAFKA_RESP_ERR_NO_ERROR { + throw KafkaError.rdKafkaError(wrapping: code) + } + } + + /// Seek for partitions to consume. + /// - Parameter topicPartitionList: Pointer to a list of topics + partition pairs. + func seek(topicPartitionList: RDKafkaTopicPartitionList, timeout: Duration) async throws { + assert(timeout >= .zero, "Timeout must be positive") + + let doSeek = { + topicPartitionList.withListPointer { rd_kafka_seek_partitions(self.kafkaHandle, $0, Int32(max(timeout, .zero).inMilliseconds)) } + } + let error = + timeout == .zero + ? doSeek() // async when timeout is zero + : await performBlockingCall(queue: gcdQueue, body: doSeek) + + defer { rd_kafka_error_destroy(error) } + let code = rd_kafka_error_code(error) + if code != RD_KAFKA_RESP_ERR_NO_ERROR { + throw KafkaError.rdKafkaError(wrapping: code) + } + } /// Subscribe to topic set using balanced consumer groups. /// - Parameter topicPartitionList: Pointer to a list of topics + partition pairs. @@ -447,15 +625,28 @@ final class RDKafkaClient: Sendable { } } } + + // TODO: remove? + func doOrThrow(_ body: () -> rd_kafka_resp_err_t, isFatal: Bool = false, file: String = #fileID, line: UInt = #line) throws { + let result = body() + if result != RD_KAFKA_RESP_ERR_NO_ERROR { + throw KafkaError.rdKafkaError(wrapping: result, isFatal: isFatal, file: file, line: line) + } + } /// Atomic assignment of partitions to consume. /// - Parameter topicPartitionList: Pointer to a list of topics + partition pairs. - func assign(topicPartitionList: RDKafkaTopicPartitionList) throws { - try topicPartitionList.withListPointer { pointer in - let result = rd_kafka_assign(self.kafkaHandle, pointer) - if result != RD_KAFKA_RESP_ERR_NO_ERROR { - throw KafkaError.rdKafkaError(wrapping: result) + func assign(topicPartitionList: RDKafkaTopicPartitionList?) async throws { + let result = await performBlockingCall(queue: self.gcdQueue) { + if let topicPartitionList { + return topicPartitionList.withListPointer { pointer in + rd_kafka_assign(self.kafkaHandle, pointer) + } } + return rd_kafka_assign(self.kafkaHandle, nil) + } + if result != RD_KAFKA_RESP_ERR_NO_ERROR { + throw KafkaError.rdKafkaError(wrapping: result) } } @@ -470,40 +661,6 @@ final class RDKafkaClient: Sendable { } } - /// Store `message`'s offset for next auto-commit. - /// - /// - Important: `enable.auto.offset.store` must be set to `false` when using this API. - func storeMessageOffset(_ message: KafkaConsumerMessage) throws { - // The offset committed is always the offset of the next requested message. - // Thus, we increase the offset of the current message by one before committing it. - // See: https://github.com/edenhill/librdkafka/issues/2745#issuecomment-598067945 - let changesList = RDKafkaTopicPartitionList() - changesList.setOffset( - topic: message.topic, - partition: message.partition, - offset: Int64(message.offset.rawValue + 1) - ) - - let error = changesList.withListPointer { listPointer in - rd_kafka_offsets_store( - self.kafkaHandle, - listPointer - ) - } - - if error != RD_KAFKA_RESP_ERR_NO_ERROR { - // Ignore RD_KAFKA_RESP_ERR__STATE error. - // RD_KAFKA_RESP_ERR__STATE indicates an attempt to commit to an unassigned partition, - // which can occur during rebalancing or when the consumer is shutting down. - // See "Upgrade considerations" for more details: https://github.com/confluentinc/librdkafka/releases/tag/v1.9.0 - // Since Kafka Consumers are designed for at-least-once processing, failing to commit here is acceptable. - if error == RD_KAFKA_RESP_ERR__STATE { - return - } - throw KafkaError.rdKafkaError(wrapping: error) - } - } - /// Non-blocking "fire-and-forget" commit of a `message`'s offset to Kafka. /// Schedules a commit and returns immediately. /// Any errors encountered after scheduling the commit will be discarded. @@ -518,7 +675,7 @@ final class RDKafkaClient: Sendable { changesList.setOffset( topic: message.topic, partition: message.partition, - offset: Int64(message.offset.rawValue + 1) + offset: .init(rawValue: message.offset.rawValue + 1) ) let error = changesList.withListPointer { listPointer in @@ -557,7 +714,7 @@ final class RDKafkaClient: Sendable { changesList.setOffset( topic: message.topic, partition: message.partition, - offset: Int64(message.offset.rawValue + 1) + offset: .init(rawValue: message.offset.rawValue + 1) ) // Unretained pass because the reference that librdkafka holds to capturedClosure @@ -621,4 +778,210 @@ final class RDKafkaClient: Sendable { func withKafkaHandlePointer(_ body: (OpaquePointer) throws -> T) rethrows -> T { return try body(self.kafkaHandle) } + + /// Scoped accessor that enables safe access to the pointer of the client's Kafka handle with async closure. + /// - Warning: Do not escape the pointer from the closure for later use. + /// - Parameter body: The closure will use the Kafka handle pointer. + @discardableResult + func withKafkaHandlePointer(_ body: (OpaquePointer) async throws -> T) async rethrows -> T { + return try await body(self.kafkaHandle) + } + + func metadata() async throws -> KafkaMetadata { + let queue = DispatchQueue(label: "com.swift-server.swift-kafka.metadata") + return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + queue.async { + var metadata: UnsafePointer? + let error = rd_kafka_metadata(self.kafkaHandle, 1, nil, &metadata, -1) + guard error == RD_KAFKA_RESP_ERR_NO_ERROR, + let metadata else { + continuation.resume(throwing: KafkaError.rdKafkaError(wrapping: error)) + return + } + continuation.resume(returning: KafkaMetadata(metadata: metadata)) + } + } + } + + func initTransactions(timeout: Duration) async throws { + let result = await performBlockingCall(queue: gcdQueue) { + rd_kafka_init_transactions(self.kafkaHandle, timeout.totalMilliseconds) + } + + if result != nil { + let code = rd_kafka_error_code(result) + rd_kafka_error_destroy(result) + throw KafkaError.rdKafkaError(wrapping: code) + } + } + + func beginTransaction() throws { + let result = rd_kafka_begin_transaction(kafkaHandle) + if result != nil { + let code = rd_kafka_error_code(result) + rd_kafka_error_destroy(result) + throw KafkaError.rdKafkaError(wrapping: code) + } + } + + func send( + attempts: UInt64, + offsets: RDKafkaTopicPartitionList, + forConsumerKafkaHandle consumer: OpaquePointer, + timeout: Duration + ) async throws { + try await offsets.withListPointer { topicPartitionList in + + let consumerMetadata = rd_kafka_consumer_group_metadata(consumer) + defer { rd_kafka_consumer_group_metadata_destroy(consumerMetadata) } + + // TODO: actually it should be withing some timeout (like transaction timeout or session timeout) + for idx in 0..? + _ = rd_kafka_assignment($0, &partitions) +// if err == nil { +// +// } + defer { + rd_kafka_topic_partition_list_destroy(partitions) + } + rd_kafka_position($0, partitions) + + guard let partitions else { + fatalError("TODO") + } + + var str = String() + for idx in 0...allocate(capacity: size) + defer { configValue.deallocate() } + + if RD_KAFKA_CONF_OK == rd_kafka_conf_get(configPointer, key, configValue, &size) { + let sizeNoNullTerm = size - 1 + let wasVal = String(unsafeUninitializedCapacity: sizeNoNullTerm) { + let buf = UnsafeRawBufferPointer( + UnsafeMutableRawBufferPointer( + start: configValue, + count: sizeNoNullTerm + )) + _ = $0.initialize(from: buf) + return sizeNoNullTerm + } + if wasVal == value { + return // Values are equal, avoid changing (not mark config as modified) + } + } + + } + let errorChars = UnsafeMutablePointer.allocate(capacity: RDKafkaClient.stringSize) defer { errorChars.deallocate() } diff --git a/Sources/Kafka/RDKafka/RDKafkaStatistics.swift b/Sources/Kafka/RDKafka/RDKafkaStatistics.swift new file mode 100644 index 00000000..96ceb4b2 --- /dev/null +++ b/Sources/Kafka/RDKafka/RDKafkaStatistics.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-client open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +// MARK: - RDKafkaStatistics + +struct RDKafkaStatistics: Hashable, Codable { + let queuedOperation: Int? + let queuedProducerMessages: Int? + let queuedProducerMessagesSize: Int? + let topicsInMetadataCache: Int? + let totalKafkaBrokerRequests: Int? + let totalKafkaBrokerBytesSent: Int? + let totalKafkaBrokerResponses: Int? + let totalKafkaBrokerResponsesSize: Int? + let totalKafkaBrokerMessagesSent: Int? + let totalKafkaBrokerMessagesBytesSent: Int? + let totalKafkaBrokerMessagesRecieved: Int? + let totalKafkaBrokerMessagesBytesRecieved: Int? + + enum CodingKeys: String, CodingKey { + case queuedOperation = "replyq" + case queuedProducerMessages = "msg_cnt" + case queuedProducerMessagesSize = "msg_size" + case topicsInMetadataCache = "metadata_cache_cnt" + case totalKafkaBrokerRequests = "tx" + case totalKafkaBrokerBytesSent = "tx_bytes" + case totalKafkaBrokerResponses = "rx" + case totalKafkaBrokerResponsesSize = "rx_bytes" + case totalKafkaBrokerMessagesSent = "txmsgs" + case totalKafkaBrokerMessagesBytesSent = "txmsg_bytes" + case totalKafkaBrokerMessagesRecieved = "rxmsgs" + case totalKafkaBrokerMessagesBytesRecieved = "rxmsg_bytes" + } +} diff --git a/Sources/Kafka/RDKafka/RDKafkaTopicConfig.swift b/Sources/Kafka/RDKafka/RDKafkaTopicConfig.swift index 100e734e..ec65fefb 100644 --- a/Sources/Kafka/RDKafka/RDKafkaTopicConfig.swift +++ b/Sources/Kafka/RDKafka/RDKafkaTopicConfig.swift @@ -35,6 +35,26 @@ struct RDKafkaTopicConfig { /// - Parameter value: The new value of the configuration property to be changed. /// - Throws: A ``KafkaError`` if setting the value failed. static func set(configPointer: OpaquePointer, key: String, value: String) throws { + var size: Int = RDKafkaClient.stringSize + let configValue = UnsafeMutablePointer.allocate(capacity: size) + defer { configValue.deallocate() } + + if RD_KAFKA_CONF_OK == rd_kafka_topic_conf_get(configPointer, key, configValue, &size) { + let sizeNoNullTerm = size - 1 + let wasVal = String(unsafeUninitializedCapacity: sizeNoNullTerm) { + let buf = UnsafeRawBufferPointer( + UnsafeMutableRawBufferPointer( + start: configValue, + count: sizeNoNullTerm + )) + _ = $0.initialize(from: buf) + return sizeNoNullTerm + } + if wasVal == value { + return // Values are equal, avoid changing (not mark config as modified) + } + } + let errorChars = UnsafeMutablePointer.allocate(capacity: RDKafkaClient.stringSize) defer { errorChars.deallocate() } diff --git a/Sources/Kafka/RDKafka/RDKafkaTopicPartitionList.swift b/Sources/Kafka/RDKafka/RDKafkaTopicPartitionList.swift index fc663599..c0a99468 100644 --- a/Sources/Kafka/RDKafka/RDKafkaTopicPartitionList.swift +++ b/Sources/Kafka/RDKafka/RDKafkaTopicPartitionList.swift @@ -15,7 +15,7 @@ import Crdkafka /// Swift wrapper type for `rd_kafka_topic_partition_list_t`. -final class RDKafkaTopicPartitionList { +public final class RDKafkaTopicPartitionList { private let _internal: UnsafeMutablePointer /// Create a new topic+partition list. @@ -24,6 +24,10 @@ final class RDKafkaTopicPartitionList { init(size: Int32 = 1) { self._internal = rd_kafka_topic_partition_list_new(size) } + + init(from: UnsafePointer) { + self._internal = rd_kafka_topic_partition_list_copy(from) + } deinit { rd_kafka_topic_partition_list_destroy(self._internal) @@ -44,7 +48,7 @@ final class RDKafkaTopicPartitionList { } /// Manually set read offset for a given topic+partition pair. - func setOffset(topic: String, partition: KafkaPartition, offset: Int64) { + func setOffset(topic: String, partition: KafkaPartition, offset: KafkaOffset) { precondition( 0...Int(Int32.max) ~= partition.rawValue || partition == .unassigned, "Partition ID outside of valid range \(0...Int32.max)" @@ -57,7 +61,7 @@ final class RDKafkaTopicPartitionList { ) else { fatalError("rd_kafka_topic_partition_list_add returned invalid pointer") } - partitionPointer.pointee.offset = offset + partitionPointer.pointee.offset = Int64(offset.rawValue) } /// Scoped accessor that enables safe access to the pointer of the underlying `rd_kafka_topic_partition_t`. @@ -67,4 +71,65 @@ final class RDKafkaTopicPartitionList { func withListPointer(_ body: (UnsafeMutablePointer) throws -> T) rethrows -> T { return try body(self._internal) } + + /// Scoped accessor that enables safe access to the pointer of the underlying `rd_kafka_topic_partition_t`. + /// - Warning: Do not escape the pointer from the closure for later use. + /// - Parameter body: The closure will use the pointer. + @discardableResult + func withListPointer(_ body: (UnsafeMutablePointer) async throws -> T) async rethrows -> T { + return try await body(self._internal) + } + + func getByIdx(idx: Int) -> TopicPartition? { + withListPointer { list -> TopicPartition? in + guard list.pointee.cnt > idx else { + return nil + } + let elem = list.pointee.elems[idx] + let topicName = String(cString: elem.topic) + let partition = KafkaPartition(rawValue: Int(elem.partition)) + let offset = KafkaOffset(rawValue: Int(elem.offset)) + return TopicPartition(topicName, partition, offset) + } + } + + var count: Int { + withListPointer { list in + Int(list.pointee.cnt) + } + } +} + +extension RDKafkaTopicPartitionList: Sendable {} +extension RDKafkaTopicPartitionList: Hashable { + + public func hash(into hasher: inout Hasher) { + for idx in 0.. Bool { + if lhs.count != rhs.count { + return false + } + + for idx in 0..(queue: DispatchQueue, body: @escaping () -> T) async -> T { + await withCheckedContinuation { continuation in + queue.async { + continuation.resume(returning: body()) + } + } +} + +// performs blocking calls outside of cooperative thread pool +internal func performBlockingCall(queue: DispatchQueue, body: @escaping () throws -> T) async throws -> T { + try await withCheckedThrowingContinuation { continuation in + queue.async { + do { + continuation.resume(returning: try body()) + } catch { + continuation.resume(throwing: error) + } + } + } +} + +// TODO: remove +/* typealias Producer = NIOAsyncSequenceProducer< +KafkaProducerEvent, +NIOAsyncSequenceProducerBackPressureStrategies.NoBackPressure, +KafkaProducerCloseOnTerminate +>*/ +//@discardableResult +//internal func yield(message: Event, to continuation: NIOAsyncSequenceProducer.Source?) async -> Bool { +// guard let continuation else { +// return false +// } +// while true { +// let result = continuation.yield(message) +// switch result { +// case .dropped: +// // Stream is closed +// return false +// case .produceMore: +// // Here we can know how many slots remains in the stream +// return true +// case .dropped: +// // Here we can know what message has beed dropped +// await Task.yield() +// continue +// @unknown default: +// fatalError("Runtime error: unknown case in \(#function), \(#file):\(#line)") +// } +// } +//} +// +//@discardableResult +//internal func yield(message: Event, to continuation: NIOThrowingAsyncSequenceProducer.Source?) async -> Bool { +// guard let continuation else { +// return false +// } +// while true { +// let result = continuation.yield(message) +// switch result { +// case .terminated: +// // Stream is closed +// return false +// case .enqueued: +// // Here we can know how many slots remains in the stream +// return true +// case .dropped: +// // Here we can know what message has beed dropped +// await Task.yield() +// continue +// @unknown default: +// fatalError("Runtime error: unknown case in \(#function), \(#file):\(#line)") +// } +// } +//} diff --git a/Sources/Kafka/Utilities/KafkaStatisticsJsonModel.swift b/Sources/Kafka/Utilities/KafkaStatisticsJsonModel.swift new file mode 100644 index 00000000..4faa90eb --- /dev/null +++ b/Sources/Kafka/Utilities/KafkaStatisticsJsonModel.swift @@ -0,0 +1,177 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the swift-kafka-gsoc open source project +// +// Copyright (c) 2023 Apple Inc. and the swift-kafka-gsoc project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of swift-kafka-gsoc project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +// This file was generated from JSON Schema using quicktype, do not modify it directly. +// To parse the JSON, add this file to your project and do: +// +// let statistics = try? newJSONDecoder().decode(KafkaStatisticsJsonModel.self, from: jsonData) + +// MARK: - Statistics + +public struct KafkaStatisticsJson: Hashable, Codable { + public let name, clientID, type: String? + public let ts, time, age, replyq: Int? + public let msgCnt, msgSize, msgMax, msgSizeMax: Int? + public let simpleCnt, metadataCacheCnt: Int? + public let brokers: [String: Broker]? + public let topics: [String: Topic]? + public let cgrp: Cgrp? + public let tx, txBytes, rx, rxBytes: Int? + public let txmsgs, txmsgBytes, rxmsgs, rxmsgBytes: Int? + + enum CodingKeys: String, CodingKey { + case name + case clientID = "client_id" + case type, ts, time, age, replyq + case msgCnt = "msg_cnt" + case msgSize = "msg_size" + case msgMax = "msg_max" + case msgSizeMax = "msg_size_max" + case simpleCnt = "simple_cnt" + case metadataCacheCnt = "metadata_cache_cnt" + case brokers, topics, cgrp, tx + case txBytes = "tx_bytes" + case rx + case rxBytes = "rx_bytes" + case txmsgs + case txmsgBytes = "txmsg_bytes" + case rxmsgs + case rxmsgBytes = "rxmsg_bytes" + } +} + +// MARK: - Broker + +public struct Broker: Hashable, Codable { + public let name: String? + public let nodeid: Int? + public let nodename, source, state: String? + public let stateage, outbufCnt, outbufMsgCnt, waitrespCnt: Int? + public let waitrespMsgCnt, tx, txbytes, txerrs: Int? + public let txretries, txidle, reqTimeouts, rx: Int? + public let rxbytes, rxerrs, rxcorriderrs, rxpartial: Int? + public let rxidle, zbufGrow, bufGrow, wakeups: Int? + public let connects, disconnects: Int? + public let intLatency, outbufLatency, rtt, throttle: [String: Int]? + public let req: [String: Int]? + public let toppars: [String: Toppar]? + + enum CodingKeys: String, CodingKey { + case name, nodeid, nodename, source, state, stateage + case outbufCnt = "outbuf_cnt" + case outbufMsgCnt = "outbuf_msg_cnt" + case waitrespCnt = "waitresp_cnt" + case waitrespMsgCnt = "waitresp_msg_cnt" + case tx, txbytes, txerrs, txretries, txidle + case reqTimeouts = "req_timeouts" + case rx, rxbytes, rxerrs, rxcorriderrs, rxpartial, rxidle + case zbufGrow = "zbuf_grow" + case bufGrow = "buf_grow" + case wakeups, connects, disconnects + case intLatency = "int_latency" + case outbufLatency = "outbuf_latency" + case rtt, throttle, req, toppars + } +} + +// MARK: - Toppars + +public struct Toppar: Hashable, Codable { + public let topic: String? + public let partition: Int? + + enum CodingKeys: String, CodingKey { + case topic, partition + } +} + +// MARK: - Cgrp + +public struct Cgrp: Hashable, Codable { + public let state: String? + public let stateage: Int? + public let joinState: String? + public let rebalanceAge, rebalanceCnt: Int? + public let rebalanceReason: String? + public let assignmentSize: Int? + + enum CodingKeys: String, CodingKey { + case state, stateage + case joinState = "join_state" + case rebalanceAge = "rebalance_age" + case rebalanceCnt = "rebalance_cnt" + case rebalanceReason = "rebalance_reason" + case assignmentSize = "assignment_size" + } +} + +// MARK: - Topic + +public struct Topic: Hashable, Codable { + public let topic: String? + public let age, metadataAge: Int? + public let batchsize, batchcnt: [String: Int]? + public let partitions: [String: Partition]? + + enum CodingKeys: String, CodingKey { + case topic, age + case metadataAge = "metadata_age" + case batchsize, batchcnt, partitions + } +} + +// MARK: - Partition + +public struct Partition: Hashable, Codable { + public let partition, broker, leader: Int? + public let desired, unknown: Bool? + public let msgqCnt, msgqBytes, xmitMsgqCnt, xmitMsgqBytes: Int? + public let fetchqCnt, fetchqSize: Int? + public let fetchState: String? + public let queryOffset, nextOffset, appOffset, storedOffset: Int? + public let commitedOffset, committedOffset, eofOffset, loOffset: Int? + public let hiOffset, lsOffset, consumerLag, consumerLagStored: Int? + public let txmsgs, txbytes, rxmsgs, rxbytes: Int? + public let msgs, rxVerDrops, msgsInflight, nextACKSeq: Int? + public let nextErrSeq, ackedMsgid: Int? + + enum CodingKeys: String, CodingKey { + case partition, broker, leader, desired, unknown + case msgqCnt = "msgq_cnt" + case msgqBytes = "msgq_bytes" + case xmitMsgqCnt = "xmit_msgq_cnt" + case xmitMsgqBytes = "xmit_msgq_bytes" + case fetchqCnt = "fetchq_cnt" + case fetchqSize = "fetchq_size" + case fetchState = "fetch_state" + case queryOffset = "query_offset" + case nextOffset = "next_offset" + case appOffset = "app_offset" + case storedOffset = "stored_offset" + case commitedOffset = "commited_offset" + case committedOffset = "committed_offset" + case eofOffset = "eof_offset" + case loOffset = "lo_offset" + case hiOffset = "hi_offset" + case lsOffset = "ls_offset" + case consumerLag = "consumer_lag" + case consumerLagStored = "consumer_lag_stored" + case txmsgs, txbytes, rxmsgs, rxbytes, msgs + case rxVerDrops = "rx_ver_drops" + case msgsInflight = "msgs_inflight" + case nextACKSeq = "next_ack_seq" + case nextErrSeq = "next_err_seq" + case ackedMsgid = "acked_msgid" + } +} diff --git a/Tests/IntegrationTests/KafkaTests.swift b/Tests/IntegrationTests/KafkaTests.swift index e6cf82e5..cfcbdb45 100644 --- a/Tests/IntegrationTests/KafkaTests.swift +++ b/Tests/IntegrationTests/KafkaTests.swift @@ -12,11 +12,14 @@ // //===----------------------------------------------------------------------===// +import Atomics import struct Foundation.UUID @testable import Kafka +@_spi(Internal) import Kafka import NIOCore import ServiceLifecycle import XCTest +import Logging // For testing locally on Mac, do the following: // @@ -34,11 +37,12 @@ import XCTest final class KafkaTests: XCTestCase { // Read environment variables to get information about the test Kafka server - let kafkaHost: String = ProcessInfo.processInfo.environment["KAFKA_HOST"] ?? "localhost" + let kafkaHost: String = ProcessInfo.processInfo.environment["KAFKA_HOST"] ?? "linux-dev" let kafkaPort: Int = .init(ProcessInfo.processInfo.environment["KAFKA_PORT"] ?? "9092")! var bootstrapBrokerAddress: KafkaConfiguration.BrokerAddress! var producerConfig: KafkaProducerConfiguration! var uniqueTestTopic: String! + var uniqueTestTopic2: String! override func setUpWithError() throws { self.bootstrapBrokerAddress = KafkaConfiguration.BrokerAddress( @@ -49,20 +53,8 @@ final class KafkaTests: XCTestCase { self.producerConfig = KafkaProducerConfiguration(bootstrapBrokerAddresses: [self.bootstrapBrokerAddress]) self.producerConfig.broker.addressFamily = .v4 - var basicConfig = KafkaConsumerConfiguration( - consumptionStrategy: .group(id: "no-group", topics: []), - bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] - ) - basicConfig.broker.addressFamily = .v4 - - // TODO: ok to block here? How to make setup async? - let client = try RDKafkaClient.makeClient( - type: .consumer, - configDictionary: basicConfig.dictionary, - events: [], - logger: .kafkaTest - ) - self.uniqueTestTopic = try client._createUniqueTopic(timeout: 10 * 1000) + self.uniqueTestTopic = try createUniqueTopic(partitions: 1) + self.uniqueTestTopic2 = try createUniqueTopic(partitions: 1) } override func tearDownWithError() throws { @@ -79,7 +71,12 @@ final class KafkaTests: XCTestCase { events: [], logger: .kafkaTest ) - try client._deleteTopic(self.uniqueTestTopic, timeout: 10 * 1000) + if let uniqueTestTopic { + try client._deleteTopic(uniqueTestTopic, timeout: 10 * 1000) + } + if let uniqueTestTopic2 { + try client._deleteTopic(uniqueTestTopic2, timeout: 10 * 1000) + } self.bootstrapBrokerAddress = nil self.producerConfig = nil @@ -148,16 +145,72 @@ final class KafkaTests: XCTestCase { } } + #if false + func testTempNoConsumerGroupLeft() async throws { + // resources-snapshots-dc-1 + + var consumerConfig = KafkaConsumerConfiguration( + consumptionStrategy: .partitions(partitions: [ + .init(partition: .init(rawValue: 0), topic: "xxx-snapshots-dc-1", offset: .init(rawValue: 0)) + ]), + bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] + ) + consumerConfig.autoOffsetReset = .beginning // Always read topics from beginning + consumerConfig.broker.addressFamily = .v4 + consumerConfig.isAutoCommitEnabled = false + consumerConfig.debugOptions = [.all] + + let logger = { + var logger = Logger(label: "test") + logger.logLevel = .info + return logger + } () + let consumer = try KafkaConsumer( + configuration: consumerConfig, + logger: logger + ) + + let serviceGroupConfiguration = ServiceGroupConfiguration(services: [consumer], gracefulShutdownSignals: [.sigint, .sigterm], logger: .kafkaTest) + let serviceGroup = ServiceGroup(configuration: serviceGroupConfiguration) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + logger.info("Service task started") + defer { + logger.info("Service task exited") + } + try await serviceGroup.run() + } + + // Consumer Task + group.addTask { + logger.info("Consumer task started") + defer { + logger.info("Consumer task exited") + } + + for try await message in consumer.messages { + logger.info("\(message)") +// consumer.triggerGracefulShutdown() +// return + } + } + + try await group.waitForAll() + + } + + } + func testProduceAndConsumeWithAssignedTopicPartition() async throws { let testMessages = Self.createTestMessages(topic: self.uniqueTestTopic, count: 10) let (producer, events) = try KafkaProducer.makeProducerWithEvents(configuration: self.producerConfig, logger: .kafkaTest) var consumerConfig = KafkaConsumerConfiguration( - consumptionStrategy: .partition( - KafkaPartition(rawValue: 0), - topic: self.uniqueTestTopic, - offset: KafkaOffset(rawValue: 0) - ), + consumptionStrategy: .partitions(partitions: [ + .init(partition: .init(rawValue: 0), topic: self.uniqueTestTopic, offset: .init(rawValue: 0)) + ]), bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] ) consumerConfig.autoOffsetReset = .beginning // Always read topics from beginning @@ -167,6 +220,15 @@ final class KafkaTests: XCTestCase { configuration: consumerConfig, logger: .kafkaTest ) + + var cont: AsyncStream.Continuation! + let sequenceForAcks = AsyncStream( + bufferingPolicy: .bufferingOldest(1_000)) { + continuation in + cont = continuation + } + let continuation = cont + let serviceGroupConfiguration = ServiceGroupConfiguration(services: [producer, consumer], logger: .kafkaTest) let serviceGroup = ServiceGroup(configuration: serviceGroupConfiguration) @@ -187,6 +249,36 @@ final class KafkaTests: XCTestCase { } // Consumer Task +// group.addTask { +// logger1.info("Task for consumer 1 started") +// var consumedMessages = [KafkaConsumerMessage]() +// for try await messageResult in consumer.messages { +// if !consumerConfig.enableAutoCommit { +// try await consumer.commitSync(messageResult) +// } +// guard case let message = messageResult else { +// continue +// } +// consumedMessages.append(message) +// if consumedMessages.count % max(testMessages.count / 10, 1) == 0 { +// logger1.info("Got \(consumedMessages.count) out of \(testMessages.count)") +// } +// +// if consumedMessages.count >= testMessages.count { +// break +// } +// } +// +// logger1.info("Task for consumer 1 finished, fetched \(consumedMessages.count)") +//// XCTAssertEqual(testMessages.count, consumedMessages.count) +//// +//// for (index, consumedMessage) in consumedMessages.enumerated() { +//// XCTAssertEqual(testMessages[index].topic, consumedMessage.topic) +//// XCTAssertEqual(testMessages[index].key, consumedMessage.key) +//// XCTAssertEqual(testMessages[index].value, consumedMessage.value) +//// } +// } +// group.addTask { var consumedMessages = [KafkaConsumerMessage]() for try await message in consumer.messages { @@ -195,6 +287,11 @@ final class KafkaTests: XCTestCase { if consumedMessages.count >= testMessages.count { break } + continuation?.yield(messageResult) + aa += 1 + if aa % max(testMessages.count / 10, 1) == 0 { + logger2.info("Got \(aa) out of \(testMessages.count)") + } } XCTAssertEqual(testMessages.count, consumedMessages.count) @@ -206,13 +303,17 @@ final class KafkaTests: XCTestCase { } } +// try? await Task.sleep(for: .seconds(5)) + // Wait for Producer Task and Consumer Task to complete try await group.next() try await group.next() + try await group.next() // Shutdown the serviceGroup await serviceGroup.triggerGracefulShutdown() } } + #endif func testProduceAndConsumeWithScheduleCommit() async throws { let testMessages = Self.createTestMessages(topic: self.uniqueTestTopic, count: 10) @@ -599,65 +700,550 @@ final class KafkaTests: XCTestCase { } } + func testPartitionForKey() async throws { + let (producer, events) = try KafkaProducer.makeProducerWithEvents(configuration: self.producerConfig, logger: .kafkaTest) + + let serviceGroupConfiguration = ServiceGroupConfiguration(services: [producer], logger: .kafkaTest) + let serviceGroup = ServiceGroup(configuration: serviceGroupConfiguration) + + let numberOfPartitions = 6 + let expectedTopic = try createUniqueTopic(partitions: Int32(numberOfPartitions)) + let key = "key" + + let expectedPartition = producer.partitionForKey(key, in: expectedTopic, partitionCount: numberOfPartitions) + XCTAssertNotNil(expectedPartition) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup.run() + } + + let message = KafkaProducerMessage( + topic: expectedTopic, + key: key, + value: "Hello, World!" + ) + + let messageID = try producer.send(message) + + var receivedDeliveryReports = Set() + + for await event in events { + switch event { + case .deliveryReports(let deliveryReports): + for deliveryReport in deliveryReports { + receivedDeliveryReports.insert(deliveryReport) + } + default: + break // Ignore any other events + } + + if receivedDeliveryReports.count >= 1 { + break + } + } + + let receivedDeliveryReport = receivedDeliveryReports.first! + XCTAssertEqual(messageID, receivedDeliveryReport.id) + + guard case .acknowledged(let receivedMessage) = receivedDeliveryReport.status else { + XCTFail() + return + } + + XCTAssertEqual(expectedTopic, receivedMessage.topic) + XCTAssertEqual(expectedPartition, receivedMessage.partition) + XCTAssertEqual(ByteBuffer(string: message.key!), receivedMessage.key) + XCTAssertEqual(ByteBuffer(string: message.value), receivedMessage.value) + + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + } + + func testPartitionEof() async throws { + let (producer, events) = try KafkaProducer.makeProducerWithEvents(configuration: self.producerConfig, logger: .kafkaTest) + + let testMessages = Self.createTestMessages(topic: self.uniqueTestTopic, count: 10) + + let serviceGroupConfiguration = ServiceGroupConfiguration(services: [producer], logger: .kafkaTest) + let serviceGroup = ServiceGroup(configuration: serviceGroupConfiguration) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup.run() + } + + // Producer Task + group.addTask { + try await Self.sendAndAcknowledgeMessages( + producer: producer, + events: events, + messages: testMessages + ) + } + + try await group.next() + + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + + var consumerConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group( + id: "test", + topics: [self.uniqueTestTopic] + ), + bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] + ) + consumerConfig.autoOffsetReset = .beginning // Read topic from beginning + consumerConfig.broker.addressFamily = .v4 + consumerConfig.enablePartitionEof = true + + let consumer = try KafkaConsumer( + configuration: consumerConfig, + logger: .kafkaTest + ) + + let consumerServiceGroupConfiguration = ServiceGroupConfiguration(services: [consumer], logger: .kafkaTest) + let consumerServiceGroup = ServiceGroup(configuration: consumerServiceGroupConfiguration) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await consumerServiceGroup.run() + } + + group.addTask { + var messages = [KafkaConsumerMessage]() + for try await record in consumer.messages { + guard !record.eof else { + break + } + messages.append(record) + } + XCTAssertEqual(messages.count, testMessages.count) + } + + try await group.next() + + await consumerServiceGroup.triggerGracefulShutdown() + } + } + + func testDuplicatedMessagesOnRebalance() async throws { + let partitionsNumber: Int32 = 12 + do { + var basicConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: "no-group", topics: []), + bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] + ) + basicConfig.broker.addressFamily = .v4 + + let client = try RDKafkaClient.makeClient( + // TODO: ok to block here? How to make setup async? + type: .consumer, + configDictionary: basicConfig.dictionary, + events: [], + logger: .kafkaTest + ) + // cleanup default test topic and create with 12 partitions + try client._deleteTopic(self.uniqueTestTopic, timeout: 10 * 1000) + self.uniqueTestTopic = try client._createUniqueTopic(partitions: partitionsNumber, timeout: 10 * 1000) + } + + let numOfMessages: UInt = 1000 + let testMessages = Self.createTestMessages(topic: uniqueTestTopic, count: numOfMessages) + let (producer, acks) = try KafkaProducer.makeProducerWithEvents(configuration: producerConfig, logger: .kafkaTest) + + let producerServiceGroupConfiguration = ServiceGroupConfiguration(services: [producer], gracefulShutdownSignals: [.sigterm, .sigint], logger: .kafkaTest) + let producerServiceGroup = ServiceGroup(configuration: producerServiceGroupConfiguration) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await producerServiceGroup.run() + } + + // Producer Task + group.addTask { + try await Self.sendAndAcknowledgeMessages( + producer: producer, + events: acks, + messages: testMessages, + skipConsistencyCheck: true + ) + } + + // Wait for Producer Task to complete + try await group.next() + // Shutdown the serviceGroup + await producerServiceGroup.triggerGracefulShutdown() + } + + // MARK: Consumer + + let uniqueGroupID = UUID().uuidString + + var consumer1Config = KafkaConsumerConfiguration( + consumptionStrategy: .group( + id: uniqueGroupID, + topics: [uniqueTestTopic] + ), + bootstrapBrokerAddresses: [bootstrapBrokerAddress] + ) + consumer1Config.autoOffsetReset = .beginning + consumer1Config.broker.addressFamily = .v4 + consumer1Config.pollInterval = .milliseconds(1) + consumer1Config.isAutoCommitEnabled = false + + let consumer1 = try KafkaConsumer( + configuration: consumer1Config, + logger: .kafkaTest + ) + + var consumer2Config = KafkaConsumerConfiguration( + consumptionStrategy: .group( + id: uniqueGroupID, + topics: [uniqueTestTopic] + ), + bootstrapBrokerAddresses: [bootstrapBrokerAddress] + ) + consumer2Config.autoOffsetReset = .beginning + consumer2Config.broker.addressFamily = .v4 + consumer2Config.pollInterval = .milliseconds(1) + consumer2Config.isAutoCommitEnabled = false + + let consumer2 = try KafkaConsumer( + configuration: consumer2Config, + logger: .kafkaTest + ) + + let serviceGroupConfiguration1 = ServiceGroupConfiguration(services: [consumer1], gracefulShutdownSignals: [.sigterm, .sigint], logger: .kafkaTest) + let serviceGroup1 = ServiceGroup(configuration: serviceGroupConfiguration1) + + let serviceGroupConfiguration2 = ServiceGroupConfiguration(services: [consumer2], gracefulShutdownSignals: [.sigterm, .sigint], logger: .kafkaTest) + let serviceGroup2 = ServiceGroup(configuration: serviceGroupConfiguration2) + + let sharedCtr = ManagedAtomic(0) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task for 1st consumer + group.addTask { + try await serviceGroup1.run() + } + // Run Task for 2nd consumer + group.addTask { + try await Task.sleep(for: .seconds(20)) // wait a bit that first consumer would form a queue + try await serviceGroup2.run() + } + + // First Consumer Task + group.addTask { + // 6 partitions + for try await record in consumer1.messages { + sharedCtr.wrappingIncrement(ordering: .relaxed) + + try consumer1.scheduleCommit(record) // commit time to time + try await Task.sleep(for: .milliseconds(100)) // don't read all messages before 2nd consumer + } + } + + // Second Consumer Task + group.addTask { + // 6 partitions + for try await record in consumer2.messages { + sharedCtr.wrappingIncrement(ordering: .relaxed) + + try consumer2.scheduleCommit(record) // commit time to time + } + } + + // Monitoring task + group.addTask { + while true { + let currentCtr = sharedCtr.load(ordering: .relaxed) + guard currentCtr >= numOfMessages else { + try await Task.sleep(for: .seconds(5)) // wait if new messages come here + continue + } + try await Task.sleep(for: .seconds(5)) // wait for extra messages + await serviceGroup1.triggerGracefulShutdown() + await serviceGroup2.triggerGracefulShutdown() + break + } + } + + try await group.next() + try await group.next() + try await group.next() + + // Wait for second Consumer Task to complete + let totalCtr = sharedCtr.load(ordering: .relaxed) + XCTAssertEqual(totalCtr, Int(numOfMessages)) + } + } + // MARK: - Helpers + func createUniqueTopic(partitions: Int32 = -1 /* default num for cluster */) throws -> String { + // TODO: ok to block here? How to make setup async? + + var basicConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: "no-group", topics: []), + bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] + ) + basicConfig.broker.addressFamily = .v4 + + let client = try RDKafkaClient.makeClient( + type: .consumer, + configDictionary: basicConfig.dictionary, + events: [], + logger: .kafkaTest + ) + return try client._createUniqueTopic(partitions: partitions, timeout: 10 * 1000) + } + private static func createTestMessages( topic: String, headers: [KafkaHeader] = [], count: UInt ) -> [KafkaProducerMessage] { - return Array(0..] + messages: [KafkaProducerMessage], + skipConsistencyCheck: Bool = false ) async throws { - var messageIDs = Set() + return try await _sendAndAcknowledgeMessages(producer: producer, events: events, messages: messages, skipConsistencyCheck: skipConsistencyCheck) + } +/* + func testProduceAndConsumeWithTransaction() async throws { + let testMessages = Self.createTestMessages(topic: uniqueTestTopic, count: 10) + + let (producer, events) = try KafkaProducer.makeProducerWithEvents(configuration: self.producerConfig, logger: .kafkaTest) - for message in messages { - messageIDs.insert(try producer.send(message)) + let transactionConfigProducer = KafkaTransactionalProducerConfiguration( + transactionalId: "1234", + bootstrapBrokerAddresses: [self.bootstrapBrokerAddress]) + + let transactionalProducer = try await KafkaTransactionalProducer(config: transactionConfigProducer, logger: .kafkaTest) + + let makeConsumerConfig = { (topic: String) -> KafkaConsumerConfiguration in + var consumerConfig = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: "subscription-test-group-id", topics: [topic]), + bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] + ) + consumerConfig.autoOffsetReset = .beginning // Always read topics from beginning + consumerConfig.broker.addressFamily = .v4 + consumerConfig.isAutoCommitEnabled = false + return consumerConfig } - var receivedDeliveryReports = Set() + let consumer = try KafkaConsumer( + configuration: makeConsumerConfig(uniqueTestTopic), + logger: .kafkaTest + ) + + let consumerAfterTransaction = try KafkaConsumer( + configuration: makeConsumerConfig(uniqueTestTopic2), + logger: .kafkaTest + ) + + let serviceGroup = ServiceGroup( + services: [ + producer, + consumer, + transactionalProducer, + consumerAfterTransaction, + ], + configuration: ServiceGroupConfiguration(gracefulShutdownSignals: []), + logger: .kafkaTest + ) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup.run() + } + + // Producer Task + group.addTask { + try await Self.sendAndAcknowledgeMessages( + producer: producer, + events: events, + messages: testMessages + ) + } - for await event in events { - switch event { - case .deliveryReports(let deliveryReports): - for deliveryReport in deliveryReports { - receivedDeliveryReports.insert(deliveryReport) + // Consumer Task + group.addTask { + var count = 0 + for try await messageResult in consumer.messages { + guard case let message = messageResult else { + continue + } + count += 1 + try await transactionalProducer.withTransaction { transaction in + let newMessage = KafkaProducerMessage( + topic: self.uniqueTestTopic2, + value: message.value.description + "_updated" + ) + try transaction.send(newMessage) + let partitionlist = RDKafkaTopicPartitionList() + partitionlist.setOffset(topic: self.uniqueTestTopic, partition: message.partition, offset: message.offset) + try await transaction.send(offsets: partitionlist, forConsumer: consumer) + } + + if count >= testMessages.count { + break + } } - default: - break // Ignore any other events + print("Changed all messages \(count)") } - if receivedDeliveryReports.count >= messages.count { - break + group.addTask { + var count = 0 + for try await messageAfterTransaction in consumerAfterTransaction.messages { + let value = messageAfterTransaction.value.getString(at: 0, length: messageAfterTransaction.value.readableBytes) + XCTAssert(value?.contains("_updated") ?? false) + count += 1 + if count >= testMessages.count || Task.isCancelled { + break + } + } + XCTAssertEqual(count, testMessages.count) } + + // Wait for Producer Task and Consumer Task to complete + try await group.next() + try await group.next() + try await group.next() + + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() } + } + + + */ + #if false + func testOrdo() async throws { +// let testMessages = Self.createTestMessages(topic: self.uniqueTestTopic, count: 10) +// let firstConsumerOffset = testMessages.count / 2 +// let (producer, acks) = try KafkaProducer.makeProducerWithEvents(configuration: self.producerConfig, logger: .kafkaTest) +// +// // Important: both consumer must have the same group.id + let uniqueGroupID = UUID().uuidString - XCTAssertEqual(Set(receivedDeliveryReports.map(\.id)), messageIDs) + // MARK: First Consumer + var logger_ = Logger(label: "ordo") + logger_.logLevel = .debug + let logger = logger_ + + logger.info("unique group id \(uniqueGroupID)") + +// let groupID = uniqueGroupID + let groupID = "test_group_id_1" + + var consumer1Config = KafkaConsumerConfiguration( + consumptionStrategy: .group( + id: groupID, + topics: ["transactions-pending-dc-1"] //["transactions-snapshots-dc-1"] + ), + bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] + ) + consumer1Config.isAutoCommitEnabled = false + consumer1Config.autoOffsetReset = .beginning // Read topic from beginning + consumer1Config.broker.addressFamily = .v4 +// consumer1Config.debugOptions = [.all] + consumer1Config.groupInstanceId = groupID + "_instance" //"transactions-pending-dc-1-test-instance-id" + consumer1Config.statisticsInterval = .value(.milliseconds(250)) + + + let (consumer1, events) = try KafkaConsumer.makeConsumerWithEvents( + configuration: consumer1Config, + logger: logger + ) + + let serviceGroup1 = ServiceGroup( + services: [consumer1], + configuration: ServiceGroupConfiguration(gracefulShutdownSignals: []), + logger: logger + ) - let acknowledgedMessages: [KafkaAcknowledgedMessage] = receivedDeliveryReports.compactMap { - guard case .acknowledged(let receivedMessage) = $0.status else { - return nil + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup1.run() } - return receivedMessage - } - XCTAssertEqual(messages.count, acknowledgedMessages.count) - for message in messages { - XCTAssertTrue(acknowledgedMessages.contains(where: { $0.topic == message.topic })) - XCTAssertTrue(acknowledgedMessages.contains(where: { $0.key == ByteBuffer(string: message.key!) })) - XCTAssertTrue(acknowledgedMessages.contains(where: { $0.value == ByteBuffer(string: message.value) })) + // First Consumer Task + group.addTask { +// try consumer1.subscribeTopics(topics: ["transactions-pending-dc-1"]) + var count = 0 + var totalCount = 0 + var start = Date.now + var partitions = [Bool]() + for try await message in consumer1.messages { + if message.eof { + logger.info("Reached EOF for partition \(message.partition) and topic \(message.topic)! Read \(totalCount)") + + while partitions.count <= message.partition.rawValue { + partitions.append(false) + } + partitions[message.partition.rawValue] = true + let res = partitions.first { $0 == false } + if partitions.count == 6 && res == nil { + break + } + + continue + } +// logger.info("Got msg: \(message)") + try await consumer1.commitSync(message) + count += 1 + totalCount += 1 + let now = Date.now + if count > 100 && now > start { + let diff = -start.timeIntervalSinceNow + let rate = Double(count) / diff + logger.info("Rate is \(rate) for last \(diff)") + count = 0 + start = now + try await consumer1.commitSync(message) + } + } + logger.info("Finally read \(totalCount)") + } + + group.addTask { + for try await event in events { + switch event { + case .statistics(let stat): +// logger.info("stats: \(stat)") + if let lag = stat.lag, lag == 0 { + logger.info("In sync with lag = 0 with stat \(stat)") + +// await serviceGroup1.triggerGracefulShutdown() + return + } + + default: + break + } + } + } + + try await group.next() + + await serviceGroup1.triggerGracefulShutdown() + } } +#endif } diff --git a/Tests/IntegrationTests/Utilities.swift b/Tests/IntegrationTests/Utilities.swift index db86c0a0..3fae75c9 100644 --- a/Tests/IntegrationTests/Utilities.swift +++ b/Tests/IntegrationTests/Utilities.swift @@ -12,171 +12,13 @@ // //===----------------------------------------------------------------------===// -import Crdkafka import struct Foundation.UUID -@testable import Kafka import Logging extension Logger { static var kafkaTest: Logger { var logger = Logger(label: "kafka.test") - logger.logLevel = .info + logger.logLevel = .debug return logger } } - -extension RDKafkaClient { -// func createUniqueTopic(timeout: Int32 = 10000) async throws -> String { -// try await withCheckedThrowingContinuation { continuation in -// do { -// let uniqueTopic = try self._createUniqueTopic(timeout: timeout) -// continuation.resume(returning: uniqueTopic) -// } catch { -// continuation.resume(throwing: error) -// } -// } -// } - - /// Create a topic with a unique name (`UUID`). - /// Blocks for a maximum of `timeout` milliseconds. - /// - Parameter timeout: Timeout in milliseconds. - /// - Returns: Name of newly created topic. - /// - Throws: A ``KafkaError`` if the topic creation failed. - func _createUniqueTopic(timeout: Int32) throws -> String { - let uniqueTopicName = UUID().uuidString - - let errorChars = UnsafeMutablePointer.allocate(capacity: RDKafkaClient.stringSize) - defer { errorChars.deallocate() } - - guard let newTopic = rd_kafka_NewTopic_new( - uniqueTopicName, - -1, // use default num_partitions - -1, // use default replication_factor - errorChars, - RDKafkaClient.stringSize - ) else { - let errorString = String(cString: errorChars) - throw KafkaError.topicCreation(reason: errorString) - } - defer { rd_kafka_NewTopic_destroy(newTopic) } - - try self.withKafkaHandlePointer { kafkaHandle in - let resultQueue = rd_kafka_queue_new(kafkaHandle) - defer { rd_kafka_queue_destroy(resultQueue) } - - var newTopicsArray: [OpaquePointer?] = [newTopic] - rd_kafka_CreateTopics( - kafkaHandle, - &newTopicsArray, - 1, - nil, - resultQueue - ) - - guard let resultEvent = rd_kafka_queue_poll(resultQueue, timeout) else { - throw KafkaError.topicCreation(reason: "No CreateTopics result after 10s timeout") - } - defer { rd_kafka_event_destroy(resultEvent) } - - let resultCode = rd_kafka_event_error(resultEvent) - guard resultCode == RD_KAFKA_RESP_ERR_NO_ERROR else { - throw KafkaError.rdKafkaError(wrapping: resultCode) - } - - guard let topicsResultEvent = rd_kafka_event_CreateTopics_result(resultEvent) else { - throw KafkaError.topicCreation(reason: "Received event that is not of type rd_kafka_CreateTopics_result_t") - } - - var resultTopicCount = 0 - let topicResults = rd_kafka_CreateTopics_result_topics( - topicsResultEvent, - &resultTopicCount - ) - - guard resultTopicCount == 1, let topicResult = topicResults?[0] else { - throw KafkaError.topicCreation(reason: "Received less/more than one topic result") - } - - let topicResultError = rd_kafka_topic_result_error(topicResult) - guard topicResultError == RD_KAFKA_RESP_ERR_NO_ERROR else { - throw KafkaError.rdKafkaError(wrapping: topicResultError) - } - - let receivedTopicName = String(cString: rd_kafka_topic_result_name(topicResult)) - guard receivedTopicName == uniqueTopicName else { - throw KafkaError.topicCreation(reason: "Received topic result for topic with different name") - } - } - - return uniqueTopicName - } - -// func deleteTopic(_ topic: String, timeout: Int32 = 10000) async throws { -// try await withCheckedThrowingContinuation { continuation in -// do { -// try self._deleteTopic(topic, timeout: timeout) -// continuation.resume() -// } catch { -// continuation.resume(throwing: error) -// } -// } -// } - - /// Delete a topic. - /// Blocks for a maximum of `timeout` milliseconds. - /// - Parameter topic: Topic to delete. - /// - Parameter timeout: Timeout in milliseconds. - /// - Throws: A ``KafkaError`` if the topic deletion failed. - func _deleteTopic(_ topic: String, timeout: Int32) throws { - let deleteTopic = rd_kafka_DeleteTopic_new(topic) - defer { rd_kafka_DeleteTopic_destroy(deleteTopic) } - - try self.withKafkaHandlePointer { kafkaHandle in - let resultQueue = rd_kafka_queue_new(kafkaHandle) - defer { rd_kafka_queue_destroy(resultQueue) } - - var deleteTopicsArray: [OpaquePointer?] = [deleteTopic] - rd_kafka_DeleteTopics( - kafkaHandle, - &deleteTopicsArray, - 1, - nil, - resultQueue - ) - - guard let resultEvent = rd_kafka_queue_poll(resultQueue, timeout) else { - throw KafkaError.topicDeletion(reason: "No DeleteTopics result after 10s timeout") - } - defer { rd_kafka_event_destroy(resultEvent) } - - let resultCode = rd_kafka_event_error(resultEvent) - guard resultCode == RD_KAFKA_RESP_ERR_NO_ERROR else { - throw KafkaError.rdKafkaError(wrapping: resultCode) - } - - guard let topicsResultEvent = rd_kafka_event_DeleteTopics_result(resultEvent) else { - throw KafkaError.topicDeletion(reason: "Received event that is not of type rd_kafka_DeleteTopics_result_t") - } - - var resultTopicCount = 0 - let topicResults = rd_kafka_DeleteTopics_result_topics( - topicsResultEvent, - &resultTopicCount - ) - - guard resultTopicCount == 1, let topicResult = topicResults?[0] else { - throw KafkaError.topicDeletion(reason: "Received less/more than one topic result") - } - - let topicResultError = rd_kafka_topic_result_error(topicResult) - guard topicResultError == RD_KAFKA_RESP_ERR_NO_ERROR else { - throw KafkaError.rdKafkaError(wrapping: topicResultError) - } - - let receivedTopicName = String(cString: rd_kafka_topic_result_name(topicResult)) - guard receivedTopicName == topic else { - throw KafkaError.topicDeletion(reason: "Received topic result for topic with different name") - } - } - } -} diff --git a/Tests/KafkaTests/KafkaConsumerTests.swift b/Tests/KafkaTests/KafkaConsumerTests.swift index 212853c6..73058c2e 100644 --- a/Tests/KafkaTests/KafkaConsumerTests.swift +++ b/Tests/KafkaTests/KafkaConsumerTests.swift @@ -12,9 +12,12 @@ // //===----------------------------------------------------------------------===// +@testable import CoreMetrics // for MetricsSystem.bootstrapInternal import struct Foundation.UUID @testable import Kafka import Logging +import Metrics +import MetricsTestKit import ServiceLifecycle import XCTest @@ -33,6 +36,17 @@ import XCTest // zookeeper-server-start /usr/local/etc/kafka/zookeeper.properties & kafka-server-start /usr/local/etc/kafka/server.properties final class KafkaConsumerTests: XCTestCase { + var metrics: TestMetrics! = TestMetrics() + + override func setUp() async throws { + MetricsSystem.bootstrapInternal(self.metrics) + } + + override func tearDown() async throws { + self.metrics = nil + MetricsSystem.bootstrapInternal(NOOPMetricsHandler.instance) + } + func testConsumerLog() async throws { let recorder = LogEventRecorder() let mockLogger = Logger(label: "kafka.test.consumer.log") { @@ -82,4 +96,81 @@ final class KafkaConsumerTests: XCTestCase { ) } } + + func testConsumerStatistics() async throws { + let uniqueGroupID = UUID().uuidString + var config = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: uniqueGroupID, topics: ["this-topic-does-not-exist"]), + bootstrapBrokerAddresses: [] + ) + config.metrics.updateInterval = .milliseconds(100) + config.metrics.queuedOperation = .init(label: "operations") + + let consumer = try KafkaConsumer(configuration: config, logger: .kafkaTest) + + let svcGroupConfig = ServiceGroupConfiguration(services: [consumer], logger: .kafkaTest) + let serviceGroup = ServiceGroup(configuration: svcGroupConfig) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup.run() + } + + try await Task.sleep(for: .seconds(1)) + + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + let value = try metrics.expectGauge("operations").lastValue + XCTAssertNotNil(value) + } + + func testConsumerConstructDeinit() async throws { + let uniqueGroupID = UUID().uuidString + let config = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: uniqueGroupID, topics: ["this-topic-does-not-exist"]), + bootstrapBrokerAddresses: [] + ) + + _ = try KafkaConsumer(configuration: config, logger: .kafkaTest) // deinit called before run + _ = try KafkaConsumer.makeConsumerWithEvents(configuration: config, logger: .kafkaTest) + } + + func testConsumerMessagesReadCancelledBeforeRun() async throws { + let uniqueGroupID = UUID().uuidString + let config = KafkaConsumerConfiguration( + consumptionStrategy: .group(id: uniqueGroupID, topics: ["this-topic-does-not-exist"]), + bootstrapBrokerAddresses: [] + ) + + let consumer = try KafkaConsumer(configuration: config, logger: .kafkaTest) + + let svcGroupConfig = ServiceGroupConfiguration(services: [consumer], logger: .kafkaTest) + let serviceGroup = ServiceGroup(configuration: svcGroupConfig) + + // explicitly run and cancel message consuming task before serviceGroup.run() + let consumingTask = Task { + for try await record in consumer.messages { + XCTFail("Unexpected record \(record))") + } + } + + try await Task.sleep(for: .seconds(1)) + + // explicitly cancel message consuming task before serviceGroup.run() + consumingTask.cancel() + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup.run() + } + + try await Task.sleep(for: .seconds(1)) + + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + } } diff --git a/Tests/KafkaTests/KafkaProducerTests.swift b/Tests/KafkaTests/KafkaProducerTests.swift index f4124898..14b83303 100644 --- a/Tests/KafkaTests/KafkaProducerTests.swift +++ b/Tests/KafkaTests/KafkaProducerTests.swift @@ -12,8 +12,11 @@ // //===----------------------------------------------------------------------===// +@testable import CoreMetrics // for MetricsSystem.bootstrapInternal @testable import Kafka import Logging +import Metrics +import MetricsTestKit import NIOCore import ServiceLifecycle import XCTest @@ -38,6 +41,7 @@ final class KafkaProducerTests: XCTestCase { let kafkaPort: Int = .init(ProcessInfo.processInfo.environment["KAFKA_PORT"] ?? "9092")! var bootstrapBrokerAddress: KafkaConfiguration.BrokerAddress! var config: KafkaProducerConfiguration! + var metrics: TestMetrics! = TestMetrics() override func setUpWithError() throws { self.bootstrapBrokerAddress = KafkaConfiguration.BrokerAddress( @@ -49,11 +53,16 @@ final class KafkaProducerTests: XCTestCase { bootstrapBrokerAddresses: [self.bootstrapBrokerAddress] ) self.config.broker.addressFamily = .v4 + + MetricsSystem.bootstrapInternal(self.metrics) } override func tearDownWithError() throws { self.bootstrapBrokerAddress = nil self.config = nil + + self.metrics = nil + MetricsSystem.bootstrapInternal(NOOPMetricsHandler.instance) } func testSend() async throws { @@ -340,4 +349,72 @@ final class KafkaProducerTests: XCTestCase { XCTAssertNil(producerCopy) } + + func testProducerStatistics() async throws { + self.config.metrics.updateInterval = .milliseconds(100) + self.config.metrics.queuedOperation = .init(label: "operations") + + let producer = try KafkaProducer( + configuration: self.config, + logger: .kafkaTest + ) + + let svcGroupConfig = ServiceGroupConfiguration(services: [producer], logger: .kafkaTest) + let serviceGroup = ServiceGroup(configuration: svcGroupConfig) + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup.run() + } + + try await Task.sleep(for: .seconds(1)) + + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + + let value = try metrics.expectGauge("operations").lastValue + XCTAssertNotNil(value) + } + + func testProducerConstructDeinit() async throws { + let config = KafkaProducerConfiguration(bootstrapBrokerAddresses: []) + + _ = try KafkaProducer(configuration: config, logger: .kafkaTest) // deinit called before run + _ = try KafkaProducer.makeProducerWithEvents(configuration: config, logger: .kafkaTest) // deinit called before run + } + + func testProducerEventsReadCancelledBeforeRun() async throws { + let config = KafkaProducerConfiguration(bootstrapBrokerAddresses: []) + + let (producer, events) = try KafkaProducer.makeProducerWithEvents(configuration: config, logger: .kafkaTest) + + let svcGroupConfig = ServiceGroupConfiguration(services: [producer], logger: .kafkaTest) + let serviceGroup = ServiceGroup(configuration: svcGroupConfig) + + // explicitly run and cancel message consuming task before serviceGroup.run() + let producerEventsTask = Task { + for try await event in events { + XCTFail("Unexpected record \(event))") + } + } + + try await Task.sleep(for: .seconds(1)) + + // explicitly cancel message consuming task before serviceGroup.run() + producerEventsTask.cancel() + + try await withThrowingTaskGroup(of: Void.self) { group in + // Run Task + group.addTask { + try await serviceGroup.run() + } + + try await Task.sleep(for: .seconds(1)) + + // Shutdown the serviceGroup + await serviceGroup.triggerGracefulShutdown() + } + } } diff --git a/dev/test-benchmark-thresholds.sh b/dev/test-benchmark-thresholds.sh new file mode 100644 index 00000000..731c3e97 --- /dev/null +++ b/dev/test-benchmark-thresholds.sh @@ -0,0 +1,42 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the swift-kafka-client open source project +## +## Copyright (c) YEARS Apple Inc. and the swift-kafka-client project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +cd Benchmarks +swift package --disable-sandbox benchmark baseline update PR --no-progress +git checkout main +swift package --disable-sandbox benchmark baseline update main --no-progress + +swift package benchmark baseline check main PR +BENCHMARK_RESULT=$? + +echo "Retcode is $BENCHMARK_RESULT" + +if [ $BENCHMARK_RESULT -eq 0 ]; then + echo "Benchmark results are the same as for main" +fi + +if [ $BENCHMARK_RESULT -eq 4 ]; then + echo "Benchmark results are better as for main" +fi + +if [ $BENCHMARK_RESULT -eq 1 ]; then + echo "Benchmark failed" + exit 1 +fi + +if [ $BENCHMARK_RESULT -eq 2 ]; then + echo "Benchmark results are worse than main" + exit 1 +fi diff --git a/dev/update-benchmark-thresholds.sh b/dev/update-benchmark-thresholds.sh new file mode 100755 index 00000000..9dc2c850 --- /dev/null +++ b/dev/update-benchmark-thresholds.sh @@ -0,0 +1,28 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the swift-kafka-client open source project +## +## Copyright (c) 2023 Apple Inc. and the swift-kafka-client project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of swift-kafka-client project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu +set -o pipefail + +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +target_repo=${2-"$here/.."} + +for f in 57 58 59 510 -nightly; do + echo "swift$f" + + docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml") + + docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline +done diff --git a/docker/Dockerfile b/docker/Dockerfile index 31ae9a21..322781d9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG swift_version=5.7 +ARG swift_version=5.9 ARG ubuntu_version=jammy ARG base_image=swift:$swift_version-$ubuntu_version FROM $base_image @@ -15,6 +15,8 @@ ENV LANGUAGE en_US.UTF-8 # Dependencies RUN apt-get update RUN apt-get install libsasl2-dev -y +RUN apt-get install libjemalloc-dev -y + # tools RUN mkdir -p $HOME/.tools diff --git a/docker/docker-compose.2204.510.yaml b/docker/docker-compose.2204.510.yaml index be270f0b..17acb143 100644 --- a/docker/docker-compose.2204.510.yaml +++ b/docker/docker-compose.2204.510.yaml @@ -19,5 +19,10 @@ services: - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete # - SANITIZER_ARG=--sanitize=thread # TSan broken still + update-benchmark-baseline: + image: swift-kafka-client:22.04-5.10 + environment: + - SWIFT_VERSION=5.10 + shell: image: swift-kafka-client:22.04-5.10 diff --git a/docker/docker-compose.2204.57.yaml b/docker/docker-compose.2204.57.yaml index 04060a2c..a465a610 100644 --- a/docker/docker-compose.2204.57.yaml +++ b/docker/docker-compose.2204.57.yaml @@ -15,9 +15,15 @@ services: test: image: swift-kafka-client:22.04-5.7 environment: + - SWIFT_VERSION=5.7 - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete # - SANITIZER_ARG=--sanitize=thread # TSan broken still + update-benchmark-baseline: + image: swift-kafka-client:22.04-5.7 + environment: + - SWIFT_VERSION=5.7 + shell: image: swift-kafka-client:22.04-5.7 diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml index afa74b8d..47b02679 100644 --- a/docker/docker-compose.2204.58.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -15,10 +15,16 @@ services: test: image: swift-kafka-client:22.04-5.8 environment: + - SWIFT_VERSION=5.8 - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete # - SANITIZER_ARG=--sanitize=thread # TSan broken still + update-benchmark-baseline: + image: swift-kafka-client:22.04-5.8 + environment: + - SWIFT_VERSION=5.8 + shell: image: swift-kafka-client:22.04-5.8 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml index f238c9d3..34833fff 100644 --- a/docker/docker-compose.2204.59.yaml +++ b/docker/docker-compose.2204.59.yaml @@ -15,10 +15,17 @@ services: test: image: swift-kafka-client:22.04-5.9 environment: + # - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors + - SWIFT_VERSION=5.9 - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete # - SANITIZER_ARG=--sanitize=thread # TSan broken still + update-benchmark-baseline: + image: swift-kafka-client:22.04-5.9 + environment: + - SWIFT_VERSION=5.9 + shell: image: swift-kafka-client:22.04-5.9 diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml index 7ae20fd8..acac1a54 100644 --- a/docker/docker-compose.2204.main.yaml +++ b/docker/docker-compose.2204.main.yaml @@ -11,10 +11,16 @@ services: test: image: swift-kafka-client:22.04-main environment: + - SWIFT_VERSION=main - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - STRICT_CONCURRENCY_ARG=-Xswiftc -strict-concurrency=complete # - SANITIZER_ARG=--sanitize=thread # TSan broken still + update-benchmark-baseline: + image: swift-kafka-client:22.04-main + environment: + - SWIFT_VERSION=main + shell: image: swift-kafka-client:22.04-main diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ff95cc5c..de2c910f 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -34,8 +34,8 @@ services: depends_on: [runtime-setup] volumes: - ~/.ssh:/root/.ssh - - ..:/code:z - working_dir: /code + - ..:/swift-kafka-client:z + working_dir: /swift-kafka-client soundness: <<: *common @@ -50,6 +50,7 @@ services: <<: *common depends_on: [kafka, runtime-setup] environment: + SWIFT_VERSION: 5.9 KAFKA_HOST: kafka command: > /bin/bash -xcl " @@ -57,6 +58,23 @@ services: swift $${SWIFT_TEST_VERB-test} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} $${STRICT_CONCURRENCY_ARG-} " + benchmark: + <<: *common + depends_on: [kafka, runtime-setup] + environment: + KAFKA_HOST: kafka + command: > + /bin/bash -xcl " + cd Benchmarks && swift package --disable-sandbox benchmark + " + + update-benchmark-baseline: + <<: *common + depends_on: [kafka, runtime-setup] + environment: + KAFKA_HOST: kafka + command: /bin/bash -xcl "cd Benchmarks && swift package --disable-sandbox --scratch-path .build/$${SWIFT_VERSION-}/ --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/$${SWIFT_VERSION-}/ --no-progress" + # util shell: