From 176f93f5c8508a23132fafb7bc6bf8ca0c151840 Mon Sep 17 00:00:00 2001 From: georgiano Date: Wed, 21 Sep 2022 16:24:40 +0400 Subject: [PATCH 1/6] Prepared for release --- README.md | 4 ++-- gradle.properties | 2 +- .../th2/mstore/MessageProcessorMetrics.java | 15 +++++++++++++++ .../exactpro/th2/mstore/SessionBatchHolder.java | 4 +++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b0d4312..876c4aa 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Overview (4.0.0) +# Overview (4.1.0) Message store (mstore) is an important th2 component responsible for storing raw messages into Cradle. Please refer to [Cradle repository] (https://github.com/th2-net/cradleapi/blob/master/README.md) for more details. This component has a pin for listening messages via MQ. @@ -74,7 +74,7 @@ spec: This is a list of supported features provided by libraries. Please see more details about this feature via [link](https://github.com/th2-net/th2-common-j#configuration-formats). -## 4.0.x +## 4.1.0 + Added metrics collection for Prometheus + Limiting simultaneously processed message batches by number and content size + Retrying storing message batches in case of failure diff --git a/gradle.properties b/gradle.properties index c774bf7..7374912 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -release_version=4.0.0 +release_version=4.1.0 diff --git a/src/main/java/com/exactpro/th2/mstore/MessageProcessorMetrics.java b/src/main/java/com/exactpro/th2/mstore/MessageProcessorMetrics.java index 3ce0add..26883d2 100644 --- a/src/main/java/com/exactpro/th2/mstore/MessageProcessorMetrics.java +++ b/src/main/java/com/exactpro/th2/mstore/MessageProcessorMetrics.java @@ -1,3 +1,18 @@ +/* + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.exactpro.th2.mstore; import io.prometheus.client.Histogram; diff --git a/src/main/java/com/exactpro/th2/mstore/SessionBatchHolder.java b/src/main/java/com/exactpro/th2/mstore/SessionBatchHolder.java index 77174a4..e2ffcca 100644 --- a/src/main/java/com/exactpro/th2/mstore/SessionBatchHolder.java +++ b/src/main/java/com/exactpro/th2/mstore/SessionBatchHolder.java @@ -1,9 +1,11 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at + * * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 772291734fb6ec86c4efd575c9fa573b38a72951 Mon Sep 17 00:00:00 2001 From: georgiano Date: Thu, 13 Oct 2022 15:21:21 +0400 Subject: [PATCH 2/6] updated version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 7374912..94dbb78 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -release_version=4.1.0 +release_version=4.1.1 From 58cb841ff22e70f5c2b82af8016123d429388b42 Mon Sep 17 00:00:00 2001 From: Luka Tchumburidze Date: Fri, 21 Oct 2022 17:08:18 +0400 Subject: [PATCH 3/6] [TH2-4296] updated cradle version, fixed EventBatchDuration cache bug in cradle --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c539862..5cc5029 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { ext { sharedDir = file("${project.rootDir}/shared") - cradleVersion = '3.1.3' + cradleVersion = '3.1.4-dev-version-3.1-3295225943-fb2c8b3-SNAPSHOT' } group = 'com.exactpro.th2' From 0c4f85d446ade1289ed065d921b064e31d02c61f Mon Sep 17 00:00:00 2001 From: georgiano Date: Tue, 1 Nov 2022 11:36:09 +0400 Subject: [PATCH 4/6] [TH2-4394] Changed default configuration values --- src/main/java/com/exactpro/th2/mstore/Configuration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/exactpro/th2/mstore/Configuration.java b/src/main/java/com/exactpro/th2/mstore/Configuration.java index 11d59fc..ca00ea0 100644 --- a/src/main/java/com/exactpro/th2/mstore/Configuration.java +++ b/src/main/java/com/exactpro/th2/mstore/Configuration.java @@ -70,8 +70,8 @@ public static Builder builder() { public static final class Builder { private static final long DEFAULT_DRAIN_INTERVAL = 1000L; private static final long DEFAULT_WAIT_TIMEOUT = 5000L; - private static final int DEFAULT_MAX_TASK_RETRIES = 3; - private static final int DEFAULT_MAX_TASK_COUNT = 1024; + private static final int DEFAULT_MAX_TASK_RETRIES = 1000000; + private static final int DEFAULT_MAX_TASK_COUNT = 256; private static final long DEFAULT_RETRY_DELAY_BASEM_MS = 5000; @JsonProperty("drain-interval") From ef107e192c20951d41cbed71622817e985928519 Mon Sep 17 00:00:00 2001 From: georgiano Date: Tue, 1 Nov 2022 11:42:12 +0400 Subject: [PATCH 5/6] updated github action files --- .github/workflows/ci-unwelcome-words.yml | 28 ++++++------- .github/workflows/dev-docker-publish.yml | 28 ++++++------- .github/workflows/docker-publish.yml | 52 ++++++++++++------------ 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci-unwelcome-words.yml b/.github/workflows/ci-unwelcome-words.yml index cd7adcf..b8b2a91 100644 --- a/.github/workflows/ci-unwelcome-words.yml +++ b/.github/workflows/ci-unwelcome-words.yml @@ -7,17 +7,17 @@ jobs: test: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.sha }} - - name: Checkout tool - uses: actions/checkout@v2 - with: - repository: exactpro-th2/ci-github-action - ref: master - token: ${{ secrets.PAT_CI_ACTION }} - path: ci-github-action - - name: Run CI action - uses: ./ci-github-action - with: - ref: ${{ github.sha }} + - uses: actions/checkout@v3 + with: + ref: ${{ github.sha }} + - name: Checkout tool + uses: actions/checkout@v3 + with: + repository: exactpro-th2/ci-github-action + ref: master + token: ${{ secrets.PAT_CI_ACTION }} + path: ci-github-action + - name: Run CI action + uses: ./ci-github-action + with: + ref: ${{ github.sha }} diff --git a/.github/workflows/dev-docker-publish.yml b/.github/workflows/dev-docker-publish.yml index 4913fe7..7ad5102 100644 --- a/.github/workflows/dev-docker-publish.yml +++ b/.github/workflows/dev-docker-publish.yml @@ -3,8 +3,8 @@ name: Dev build and publish Docker distributions to Github Container Registry gh on: push: branches-ignore: - - master - - version-* + - master + - version-* # paths: # - gradle.properties # - package_info.json @@ -13,38 +13,38 @@ jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # Prepare custom build version - name: Get branch name id: branch - run: echo ::set-output name=branch_name::${GITHUB_REF#refs/*/} + run: echo "branch_name=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - name: Get SHA of the commit id: sha - run: echo ::set-output name=sha_short::$(git rev-parse --short HEAD) + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Get release_version id: ver - uses: christian-draeger/read-properties@1.0.1 + uses: christian-draeger/read-properties@1.1.1 with: path: gradle.properties - property: release_version + properties: release_version - name: Build custom release version id: release_ver - run: echo ::set-output name=value::"${{ steps.ver.outputs.value }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}-${{ steps.sha.outputs.sha_short }}" + run: echo value="${{ steps.ver.outputs.release_version }}-${{ steps.branch.outputs.branch_name }}-${{ github.run_id }}-${{ steps.sha.outputs.sha_short }}" >> $GITHUB_OUTPUT - name: Show custom release version run: echo ${{ steps.release_ver.outputs.value }} # Build and publish image - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 + uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: echo "REPOSITORY_NAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" >> $GITHUB_OUTPUT id: meta - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: push: true tags: ghcr.io/${{ github.repository }}:${{ steps.release_ver.outputs.value }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a77183c..74da11d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -3,35 +3,35 @@ name: Build and publish Docker distributions to Github Container Registry ghcr.i on: push: branches: - - master - - version-* + - master + - version-* paths: - - gradle.properties + - gradle.properties jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - run: echo "::set-output name=REPOSITORY_NAME::$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" - id: meta - - name: Read version from gradle.properties - id: read_property - uses: christian-draeger/read-properties@1.0.1 - with: - path: ./gradle.properties - property: release_version - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ steps.read_property.outputs.value }} - labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.read_property.outputs.value }} + - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: echo "REPOSITORY_NAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" >> $GITHUB_OUTPUT + id: meta + - name: Read version from gradle.properties + id: read_property + uses: christian-draeger/read-properties@1.1.1 + with: + path: ./gradle.properties + properties: release_version + - name: Build and push + id: docker_build + uses: docker/build-push-action@v3 + with: + push: true + tags: ghcr.io/${{ github.repository }}:${{ steps.read_property.outputs.release_version }} + labels: com.exactpro.th2.${{ steps.meta.outputs.REPOSITORY_NAME }}=${{ steps.read_property.outputs.release_version }} From 9f7a92fc0f27879a7f185f9c25fb21474106acad Mon Sep 17 00:00:00 2001 From: "giorgi.mania" Date: Thu, 10 Nov 2022 13:46:34 +0400 Subject: [PATCH 6/6] Prepared for release --- README.md | 17 ++++++++++++++--- build.gradle | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 876c4aa..a5566ff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Overview (4.1.0) +# Overview (4.1.1) Message store (mstore) is an important th2 component responsible for storing raw messages into Cradle. Please refer to [Cradle repository] (https://github.com/th2-net/cradleapi/blob/master/README.md) for more details. This component has a pin for listening messages via MQ. @@ -19,9 +19,9 @@ session alias, direction and sequence number are a **compound unique identifier* { "drain-interval": 1000, "termination-timeout": 5000, - "maxTaskCount" : 1024, + "maxTaskCount" : 256, "maxTaskDataSize" : 133169152, - "maxRetryCount" : 3, + "maxRetryCount" : 1000000, "retryDelayBase" : 5000 } ``` @@ -74,6 +74,17 @@ spec: This is a list of supported features provided by libraries. Please see more details about this feature via [link](https://github.com/th2-net/th2-common-j#configuration-formats). +## 4.1.1 + ++ Using Cradle 3.1.4. No changes related to message persistence ++ Changed default configuration to +```json +{ + "maxTaskCount" : 256, + "maxRetryCount" : 1000000, + "retryDelayBase" : 5000 +} +``` ## 4.1.0 + Added metrics collection for Prometheus + Limiting simultaneously processed message batches by number and content size diff --git a/build.gradle b/build.gradle index 5cc5029..369fbe8 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { ext { sharedDir = file("${project.rootDir}/shared") - cradleVersion = '3.1.4-dev-version-3.1-3295225943-fb2c8b3-SNAPSHOT' + cradleVersion = '3.1.4' } group = 'com.exactpro.th2'