Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 20 additions & 80 deletions .github/actions/node-npm/pre-merge/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ description: Node.js pre-merge testing github iggy actions

inputs:
task:
description: "Task to run (lint, test, build, e2e, e2e-vsr)"
description: "Task to run (lint, test, build, e2e)"
required: true

runs:
Expand All @@ -39,11 +39,11 @@ runs:
shell: bash

- name: Setup Rust with cache
if: inputs.task == 'e2e' || inputs.task == 'e2e-vsr'
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/setup-rust-with-cache

- name: Install netcat
if: inputs.task == 'e2e' || inputs.task == 'e2e-vsr'
if: inputs.task == 'e2e'
run: sudo apt-get update && sudo apt-get install -y netcat-openbsd
shell: bash

Expand Down Expand Up @@ -90,109 +90,49 @@ runs:
npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/unit npm run test:unit
shell: bash

# TODO(hubcio): change to iggy-server once legacy server is removed
# (core/server has VSR support)
- name: Start Iggy server
id: iggy
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
# Node e2e asserts full cluster metadata (2 nodes from
# config.toml's [[cluster.nodes]] list), so cluster mode must be
# on. --replica-id picks the current node out of that list.
replica-id: "0"
env:
IGGY_CLUSTER_ENABLED: "true"
cargo-bin: iggy-server-ng
cargo-features: vsr
wait-timeout-seconds: "90"

- name: E2E tests
if: inputs.task == 'e2e'
run: |
cd foreign/node
mkdir -p ../../reports/node-coverage/e2e
npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/e2e npm run test:e2e
env:
IGGY_SERVER_HOST: 127.0.0.1
IGGY_SERVER_TCP_PORT: 8090
shell: bash

- name: Start Iggy VSR server 0
id: iggy-vsr-0
if: inputs.task == 'e2e-vsr'
uses: ./.github/actions/utils/server-start
with:
cargo-bin: iggy-server-ng
cargo-features: vsr
replica-id: "0"
pid-file: ${{ runner.temp }}/iggy-node-vsr-0.pid
log-file: ${{ runner.temp }}/iggy-node-vsr-0.log
wait-timeout-seconds: "90"
env:
IGGY_CLUSTER_ENABLED: "true"
IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-node-vsr-0-data

- name: Start Iggy VSR server 1
id: iggy-vsr-1
if: inputs.task == 'e2e-vsr'
uses: ./.github/actions/utils/server-start
with:
cargo-bin: iggy-server-ng
cargo-features: vsr
replica-id: "1"
tcp_address: 127.0.0.1:8091
http_address: 127.0.0.1:3001
pid-file: ${{ runner.temp }}/iggy-node-vsr-1.pid
log-file: ${{ runner.temp }}/iggy-node-vsr-1.log
wait-timeout-seconds: "90"
env:
IGGY_CLUSTER_ENABLED: "true"
IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-node-vsr-1-data

- name: VSR E2E tests
if: inputs.task == 'e2e-vsr'
run: |
cd foreign/node
mkdir -p ../../reports/node-coverage/e2e-vsr
npx c8 --reporter=lcov \
--reports-dir=../../reports/node-coverage/e2e-vsr \
npm run test:e2e:vsr
env:
IGGY_TEST_PROTOCOL: vsr
shell: bash

- name: Stop Iggy VSR server 1
if: always() && inputs.task == 'e2e-vsr'
uses: ./.github/actions/utils/server-stop
with:
pid-file: ${{ steps.iggy-vsr-1.outputs.pid_file }}
log-file: ${{ steps.iggy-vsr-1.outputs.log_file }}

- name: Stop Iggy VSR server 0
if: always() && inputs.task == 'e2e-vsr'
- name: Stop Iggy server
if: always() && inputs.task == 'e2e'
uses: ./.github/actions/utils/server-stop
with:
pid-file: ${{ steps.iggy-vsr-0.outputs.pid_file }}
log-file: ${{ steps.iggy-vsr-0.outputs.log_file }}
pid-file: ${{ steps.iggy.outputs.pid_file }}
log-file: ${{ steps.iggy.outputs.log_file }}

- name: Upload VSR server logs
if: always() && inputs.task == 'e2e-vsr'
- name: Upload server logs
if: always() && inputs.task == 'e2e'
uses: actions/upload-artifact@v7
with:
name: iggy-node-vsr-server-logs
path: |
${{ steps.iggy-vsr-0.outputs.log_file }}
${{ steps.iggy-vsr-1.outputs.log_file }}
name: iggy-node-server-logs
path: ${{ steps.iggy.outputs.log_file }}
if-no-files-found: ignore

- name: Stop Iggy server (plain)
if: always() && inputs.task == 'e2e'
uses: ./.github/actions/utils/server-stop
with:
pid-file: ${{ steps.iggy.outputs.pid_file }}
log-file: ${{ steps.iggy.outputs.log_file }}

# TODO(hubcio): change to iggy-server once legacy server is removed
# (core/server has VSR support)
- name: Start Iggy server (TLS)
id: iggy-tls
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
cargo-bin: iggy-server-ng
cargo-features: vsr
pid-file: ${{ runner.temp }}/iggy-server-tls.pid
log-file: ${{ runner.temp }}/iggy-server-tls.log
env:
Expand Down
4 changes: 2 additions & 2 deletions .github/config/components.yml
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ components:
- "ci-infrastructure" # CI changes trigger full regression
paths:
- "foreign/node/**"
tasks: ["lint", "test", "build", "e2e", "e2e-vsr"]
tasks: ["lint", "test", "build", "e2e"]

sdk-go:
depends_on:
Expand Down Expand Up @@ -370,7 +370,7 @@ components:
paths:
- "bdd/node/**"
- "bdd/scenarios/**"
tasks: ["bdd-node", "bdd-node-vsr"]
tasks: ["bdd-node"]

bdd-csharp:
depends_on:
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,7 @@ jobs:
if: >-
inputs.component == 'sdk-node' &&
(inputs.task == 'test' ||
inputs.task == 'e2e' ||
inputs.task == 'e2e-vsr')
inputs.task == 'e2e')
uses: codecov/codecov-action@v7.0.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/_test_bdd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ jobs:
run: |
# The VSR lanes need the vsr feature on both the server and the CLI,
# otherwise the CLI cannot frame requests for the VSR wire protocol
# and the healthcheck ping fails. The Go SDK speaks only VSR, while
# the Python wheels, Java SDK, .NET SDK, and C++ bindings are
# vsr-built, so those suites are always on this branch.
# and the healthcheck ping fails. The Go SDK and the Node SDK speak
# only VSR, while the Python wheels, Java SDK, .NET SDK, and C++
# bindings are vsr-built, so those suites are always on this branch.
case "${{ inputs.task }}" in
bdd-rust-vsr|bdd-go|bdd-go-race|bdd-python|bdd-node-vsr|bdd-csharp|bdd-java|bdd-cpp)
bdd-rust-vsr|bdd-go|bdd-go-race|bdd-python|bdd-node|bdd-csharp|bdd-java|bdd-cpp)
# TODO: change to iggy-server once legacy server is removed (core/server has VSR support)
SERVER_BIN="iggy-server-ng"
echo "Building the VSR server binary and CLI (--features vsr) for BDD tests..."
Expand Down Expand Up @@ -91,12 +91,12 @@ jobs:
if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
run: |
# Extract SDK name from task (format: bdd-<sdk>, or bdd-<sdk>-vsr
# for an explicit vsr lane). Python, C#, Java, and C++ have no
# legacy lane, so their plain task names run vsr.
# for an explicit vsr lane). Python, Node, C#, Java, and C++ have
# no legacy lane, so their plain task names run vsr.
SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/^bdd-//; s/-vsr$//')
EXTRA_FLAGS=()
case "${{ inputs.task }}" in
bdd-rust-vsr|bdd-python|bdd-node-vsr|bdd-csharp|bdd-cpp)
bdd-rust-vsr|bdd-python|bdd-node|bdd-csharp|bdd-cpp)
EXTRA_FLAGS+=(--vsr)
# TODO: change to iggy-server once legacy server is removed (core/server has VSR support)
export IGGY_SERVER_NG_PATH="target/debug/iggy-server-ng"
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/_test_examples.yml
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,11 @@ jobs:
echo "Building common binaries for all examples tests..."
echo "Current directory: $(pwd)"

# The Go SDK speaks only the VSR wire protocol, while the Python
# wheels, Java SDK, and .NET SDK are vsr-built, so those lanes need
# the VSR server. Every other language still runs against the legacy
# one until its own migration lands.
if [[ "${{ inputs.task }}" == "examples-go" || "${{ inputs.task }}" == "examples-python" || "${{ inputs.task }}" == "examples-csharp" || "${{ inputs.task }}" == "examples-java" ]]; then
# The Go SDK and the Node SDK speak only the VSR wire protocol,
# while the Python wheels, Java SDK, and .NET SDK are vsr-built, so
# those lanes need the VSR server. Every other language still runs
# against the legacy one until its own migration lands.
if [[ "${{ inputs.task }}" == "examples-go" || "${{ inputs.task }}" == "examples-python" || "${{ inputs.task }}" == "examples-csharp" || "${{ inputs.task }}" == "examples-java" || "${{ inputs.task }}" == "examples-node" ]]; then
# TODO: change to iggy-server once legacy server is removed (core/server has VSR support)
SERVER_BIN="iggy-server-ng"
echo "Building ${SERVER_BIN} (--features vsr)..."
Expand Down
11 changes: 6 additions & 5 deletions .github/workflows/coverage-baseline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -447,11 +447,15 @@ jobs:
cd foreign/node
npm ci --ignore-scripts

# TODO(hubcio): change to iggy-server once legacy server is removed
# (core/server has VSR support)
- name: Start Iggy server
id: iggy
uses: ./.github/actions/utils/server-start
env:
IGGY_CLUSTER_ENABLED: true
with:
cargo-bin: iggy-server-ng
cargo-features: vsr
wait-timeout-seconds: "90"

- name: Run unit tests with coverage
run: |
Expand All @@ -464,9 +468,6 @@ jobs:
cd foreign/node
mkdir -p ../../reports/node-coverage/e2e
npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/e2e npm run test:e2e
env:
IGGY_SERVER_HOST: 127.0.0.1
IGGY_SERVER_TCP_PORT: 8090

- name: Stop Iggy server
if: always()
Expand Down
3 changes: 0 additions & 3 deletions codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,6 @@ flag_management:
- name: node-e2e
paths:
- foreign/node/
- name: node-e2e-vsr
paths:
- foreign/node/
- name: go
paths:
- foreign/go/
Expand Down
5 changes: 4 additions & 1 deletion examples/node/src/tcp-tls/consumer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@
//
// Prerequisites:
// Start the Iggy server with TLS enabled:
// TODO(hubcio): change to iggy-server once legacy server is removed
// (core/server has VSR support)
// IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \
// IGGY_TCP_TLS_ENABLED=true \
// IGGY_TCP_TLS_CERT_FILE=core/certs/iggy_cert.pem \
// IGGY_TCP_TLS_KEY_FILE=core/certs/iggy_key.pem \
// cargo r --bin iggy-server
// cargo r --bin iggy-server-ng --features vsr
//
// Run this example (from examples/node/):
// DEBUG=iggy:* npx tsx src/tcp-tls/consumer.ts
Expand Down
20 changes: 17 additions & 3 deletions examples/node/src/tcp-tls/producer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,28 @@
//
// Prerequisites:
// Start the Iggy server with TLS enabled:
// TODO(hubcio): change to iggy-server once legacy server is removed
// (core/server has VSR support)
// IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \
// IGGY_TCP_TLS_ENABLED=true \
// IGGY_TCP_TLS_CERT_FILE=core/certs/iggy_cert.pem \
// IGGY_TCP_TLS_KEY_FILE=core/certs/iggy_key.pem \
// cargo r --bin iggy-server
// cargo r --bin iggy-server-ng --features vsr
//
// Run this example (from examples/node/):
// DEBUG=iggy:* npx tsx src/tcp-tls/producer.ts

import { readFileSync } from 'node:fs';
import { Client, Partitioning } from 'apache-iggy';
import { BATCHES_LIMIT, cleanup, initSystem, log, MESSAGES_PER_BATCH, sleep } from '../utils';
import {
BATCHES_LIMIT,
cleanup,
initSystem,
log,
MESSAGES_PER_BATCH,
PARTITION_ID,
sleep
} from '../utils';

async function produceMessages(
client: Client,
Expand Down Expand Up @@ -62,11 +73,14 @@ async function produceMessages(
});

try {
// The VSR client routes each send to an explicit partition.
// TODO(hubcio): Balanced partitioning to be implemented; not decided
// yet whether it'll be on server side or client side.
await client.message.send({
streamId,
topicId,
messages,
partition: Partitioning.Balanced,
partition: Partitioning.PartitionId(PARTITION_ID),
});
} catch (error) {
log('Error sending messages: %o', error);
Expand Down
Loading
Loading