Skip to content

Commit

Permalink
Merge branch 'unstable' into fix_flushdb
Browse files Browse the repository at this point in the history
  • Loading branch information
Mixficsol committed Mar 28, 2024
2 parents ac8e1dc + 187e0e1 commit ea1b0d0
Show file tree
Hide file tree
Showing 35 changed files with 2,167 additions and 517 deletions.
102 changes: 50 additions & 52 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,88 +14,86 @@ jobs:
strategy:
matrix:
include:
- arch: x86_64-unknown-linux-gnu
os: ubuntu-latest
file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-linux-amd64
file_ext: .tar.gz
- arch: aarch64-unknown-linux-gnu
os: ubuntu-latest
file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-linux-arm64
file_ext: .tar.gz
- arch: x86_64-apple-darwin
os: macos-latest
file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-darwin-amd64
file_ext: .tar.gz
- arch: aarch64-apple-darwin
os: macos-latest
file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-darwin-arm64
file_ext: .tar.gz
- os: ubuntu-latest
name: ${{ github.event.repository.name }}-${{ github.ref_name }}-ubuntu-amd64.tar.gz
- os: macos-latest
name: ${{ github.event.repository.name }}-${{ github.ref_name }}-macos-amd64.tar.gz

runs-on: ${{ matrix.os }}

steps:
- name: Checkout sources
uses: actions/checkout@v4

- name: Install Deps - linux
if: contains(matrix.arch, 'linux')
- name: Release build os - ${{ matrix.os }}
run: |
sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
sudo apt-get install -y clang-tidy-12
chmod +x ci/release-build.sh
./ci/release-build.sh install ${{ matrix.os }} ${{ env.BUILD_TYPE }}
- name: Install Deps - darwin
if: contains(matrix.os, 'macos')
- name: Calculate checksum and rename binary
run: |
brew update
brew install --overwrite python@3.12 autoconf protobuf llvm wget git
brew install gcc@10 automake cmake make binutils
chmod +x ci/release-build.sh
./ci/release-build.sh checksum ${{ github.event.repository.name }} ${{ matrix.name }}
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.name }}
path: build/${{ matrix.name }}

- name: Configure CMake - linux
if: contains(matrix.arch, 'linux')
run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS="-s" -DCMAKE_EXE_LINKER_FLAGS="-s"
- name: Upload checksum of artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.name }}.sha256sum
path: build/${{ matrix.name }}.sha256sum

- name: Configure CMake - darwin
if: contains(matrix.os, 'macos')
centos:
runs-on: ubuntu-latest
container:
image: centos:7
env:
name: ${{ github.event.repository.name }}-${{ github.ref_name }}-centos-amd64.tar.gz
steps:
- name: Install deps
run: |
yum install -y wget git autoconf centos-release-scl gcc perl-Digest-SHA
yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-make devtoolset-10-bin-util
yum install -y llvm-toolset-7 llvm-toolset-7-clang tcl which
- name: Install cmake
run: |
export CC=/usr/local/opt/gcc@10/bin/gcc-10
cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
wget https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.sh
bash ./cmake-3.26.4-linux-x86_64.sh --skip-license --prefix=/usr
- name: Cache Build - linux
if: contains(matrix.arch, 'linux')
uses: actions/cache@v3
id: cache-ubuntu
- name: Checkout sources
uses: actions/checkout@v3
with:
key: ${{ runner.os }}-build-ubuntu-${{ hashFiles('**/CMakeLists.txt') }}
path: |
${{ github.workspace }}/buildtrees
${{ github.workspace }}/deps
fetch-depth: 0

- name: Build
run: cmake --build build --config ${{ env.BUILD_TYPE }}
- name: Release build os - centos
run: |
chmod +x ci/release-build.sh
./ci/release-build.sh install centos ${{ env.BUILD_TYPE }} -xe
- name: Calculate checksum and rename binary
shell: bash
run: |
cd build/
chmod +x ${{ github.event.repository.name }}
tar -zcvf ${{ matrix.file_name }}${{ matrix.file_ext }} ${{ github.event.repository.name }}
echo $(shasum -a 256 ${{ matrix.file_name }}${{ matrix.file_ext }} | cut -f1 -d' ') > ${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum
run: ./ci/release-build.sh checksum ${{ github.event.repository.name }} ${{ env.name }}

- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.file_name }}${{ matrix.file_ext }}
path: build/${{ matrix.file_name }}${{ matrix.file_ext }}
name: ${{ env.name }}
path: build/${{ env.name }}

- name: Upload checksum of artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum
path: build/${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum
name: ${{ env.name }}.sha256sum
path: build/${{ env.name }}.sha256sum

release:
name: Release artifacts
needs: [ build ]
needs: [ build,centos ]
runs-on: ubuntu-latest
steps:
- name: Download artifacts
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ tags

# IDE
.vscode
.DS_Store

# generate
make_config.mk
Expand Down
44 changes: 42 additions & 2 deletions CHANGELOG.MD
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

- During the full replication process, the slave node of the pika service does not receive read traffic requests.[#2197](https://github.com/OpenAtomFoundation/pika/pull/2197) @[tedli](https://github.com/tedli)

- Pika cache adds bimap data type.[#2197](https://github.com/OpenAtomFoundation/pika/pull/2197) @[chejinge](https://github.com/chejinge)
- Pika cache adds bimap data type.[#2253](https://github.com/OpenAtomFoundation/pika/pull/2253) @[chejinge](https://github.com/chejinge)

- Delete the remaining Slots in Sharing mode. There is only DB under Pika, and there are multiple DBs under one Pika.[#2251](https://github.com/OpenAtomFoundation/pika/pull/2251) @[Mixficsol](https://github.com/Mixficsol)

Expand All @@ -24,7 +24,7 @@

- Pika supports large key analysis tools[#2195](https://github.com/OpenAtomFoundation/pika/pull/2195) @[sjcsjc123](https://github.com/sjcsjc123)

- Pika supports dynamic adjustment of Pika cache parameters[#2197](https://github.com/OpenAtomFoundation/pika/pull/2197) @[chejinge](https://github.com/chejinge)
- Pika supports dynamic adjustment of Pika cache parameters[#2253](https://github.com/OpenAtomFoundation/pika/pull/2253) @[chejinge](https://github.com/chejinge)

- Updated Pika benchmark tool to support more interface stress tests.[#2222](https://github.com/OpenAtomFoundation/pika/pull/2222)@[wangshao1](https://github.com/wangshao1)

Expand All @@ -40,6 +40,22 @@

- Pika distributed cluster supports automatic failover.[#2386](https://github.com/OpenAtomFoundation/pika/pull/2386)@[chengyu-l](https://github.com/chengyu-l)

- Pika supports redis rename-command function.[#2455](https://github.com/OpenAtomFoundation/pika/pull/2455)@[Mixficsol](https://github.com/Mixficsol)

- Optimize codis slot migration speed and support dynamic modification of migration thread and speed.[#2486](https://github.com/OpenAtomFoundation/pika/pull/2486) @[chejinge](https://github.com/chejinge)

- Pika supports dynamic adjustment of the max-conn-rbuf-size parameter.[#2434](https://github.com/OpenAtomFoundation/pika/pull/2434) @[HappyUncle](https://github.com/HappyUncle)

- Pika-operator supports namespace and can deploy different clusters under different namespaces.[#2480](https://github.com/OpenAtomFoundation/pika/pull/2480) @[Y-Rookie](https://github.com/Y-Rookie)

- Pika-operator supports monitoring indicator collection and automatically launches Pika-experter.[#2451](https://github.com/OpenAtomFoundation/pika/pull/2451) @[chengyu-l](https://github.com/chengyu-l)

- ACL forward compatible with userblacklist.[#2459](https://github.com/OpenAtomFoundation/pika/pull/2459) @[dingxiaoshuai](https://github.com/dingxiaoshuai)

- Enriched Pika TCL test set .[#2497](https://github.com/OpenAtomFoundation/pika/pull/2497) @[Mixficsol](https://github.com/Mixficsol)

- Enriched Pika Gotest test set.[#2502](https://github.com/OpenAtomFoundation/pika/pull/2502) @[Mixficsol](https://github.com/Mixficsol)

## bugfix

- Fixed an issue where Pika would accidentally delete dump files during full replication from the node.[#2377](https://github.com/OpenAtomFoundation/pika/pull/2377)@[wangshao1](https://github.com/wangshao1)
Expand Down Expand Up @@ -86,6 +102,30 @@

- Fix the problem of abnormal display of master and slave roles in Codis fe of pika.[#2387](https://github.com/OpenAtomFoundation/pika/pull/2387)@[chengyu-l](https://github.com/chengyu-l)

- Fix the problem of data inconsistency after migrating data.[#2485](https://github.com/OpenAtomFoundation/pika/pull/2485)@[chejinge](https://github.com/chejinge)

- Fix dbsize calculation error problem.[#2494](https://github.com/OpenAtomFoundation/pika/pull/2494)@[chejinge](https://github.com/chejinge)

- Fixed the issue of inaccurate display of the Codis-dashboard interface after scaling up or down or starting and stopping pods.[#2475](https://github.com/OpenAtomFoundation/pika/pull/2475)@[chengyu-l](https://github.com/chengyu-l)

- Fix the problem of repeated locking of DB layer.[#2372](https://github.com/OpenAtomFoundation/pika/pull/2372) @[Mixficsol](https://github.com/Mixficsol)

- Fixed the problem of data loss caused by failure to perform full copy.[#2439](https://github.com/OpenAtomFoundation/pika/pull/2439)@[wangshao1](https://github.com/wangshao1)

- Fixed the problem that during the master-slave replication process, the master instance did not correctly respond to the slave's synchronization request when executing bgsave.[#2437](https://github.com/OpenAtomFoundation/pika/pull/2437)@[wangshao1](https://github.com/wangshao1)

- During the full copy process, add data synchronization status to clarify the data synchronization progress.[#2430](https://github.com/OpenAtomFoundation/pika/pull/2430)@[baixin01](https://github.com/baixin01)

- Fixed the issue where the slave database did not lock the key of the operation when applying binlog, resulting in data inconsistency.[#2409](https://github.com/OpenAtomFoundation/pika/pull/2490) @[chejinge](https://github.com/chejinge)

- Fix the problem of master instance coredump during codis slot migration process.[#2415](https://github.com/OpenAtomFoundation/pika/pull/2415) @[chejinge](https://github.com/chejinge)

- Fixed the problem of deleting the dump file being used during the master-slave replication process.[#2377](https://github.com/OpenAtomFoundation/pika/pull/2377)@[wangshao1](https://github.com/wangshao1)

- Fixed the problem of rsync response error from slave instance during master-slave replication process.[#2319](https://github.com/OpenAtomFoundation/pika/pull/2319)@[wangshao1](https://github.com/wangshao1)

- Fixed the problem that in master-slave replication mode, after the master instance flushdb, the slave instance cannot be accessed due to deadlock.[#2372](https://github.com/OpenAtomFoundation/pika/pull/2372) @[Mixficsol](https://github.com/Mixficsol)


# v3.5.2

Expand Down
43 changes: 41 additions & 2 deletions CHANGELOG_CN.MD
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

- 在全量复制的过程中,pika 服务的从节点不接收读流量请求 [#2197](https://github.com/OpenAtomFoundation/pika/pull/2197) @[tedli](https://github.com/tedli)

- Pika cache 新增 bimap数据类型[#2197](https://github.com/OpenAtomFoundation/pika/pull/2197) @[chejinge](https://github.com/chejinge)
- Pika cache 新增 bimap 数据类型[#2253](https://github.com/OpenAtomFoundation/pika/pull/2253) @[chejinge](https://github.com/chejinge)

- 删除 Sharing 模式残留的 Slot,Pika 下只有 DB,一个 Pika 下有多个 DB[#2251](https://github.com/OpenAtomFoundation/pika/pull/2251) @[Mixficsol](https://github.com/Mixficsol)

Expand All @@ -24,7 +24,7 @@

- Pika 支持大 key 分析工具[#2195](https://github.com/OpenAtomFoundation/pika/pull/2195) @[sjcsjc123](https://github.com/sjcsjc123)

- Pika 支持动态调整 Pika cache 参数[#2197](https://github.com/OpenAtomFoundation/pika/pull/2197) @[chejinge](https://github.com/chejinge)
- Pika 支持动态调整 Pika cache 参数[#2253](https://github.com/OpenAtomFoundation/pika/pull/2253) @[chejinge](https://github.com/chejinge)

- 更新 Pika benchmark 工具支持更多的接口压测[#2222](https://github.com/OpenAtomFoundation/pika/pull/2222)@[wangshao1](https://github.com/wangshao1)

Expand All @@ -40,6 +40,22 @@

- Pika 分布式集群支持自动 failover[#2386](https://github.com/OpenAtomFoundation/pika/pull/2386)@[chengyu-l](https://github.com/chengyu-l)

- Pika 支持 redis rename-command 功能[#2455](https://github.com/OpenAtomFoundation/pika/pull/2455)@[Mixficsol](https://github.com/Mixficsol)

- 优化 codis slot 迁移速度,支持动态修改迁移线程和速度[#2486](https://github.com/OpenAtomFoundation/pika/pull/2486) @[chejinge](https://github.com/chejinge)

- Pika 支持动态调整 max-conn-rbuf-size 参数[#2434](https://github.com/OpenAtomFoundation/pika/pull/2434) @[HappyUncle](https://github.com/HappyUncle)

- Pika-operator 支持 namespace,可以在不同的 namespace 下部署不同的集群[#2480](https://github.com/OpenAtomFoundation/pika/pull/2480) @[Y-Rookie](https://github.com/Y-Rookie)

- Pika-operator 支持监控指标采集,自动拉起 Pika-expoter[#2451](https://github.com/OpenAtomFoundation/pika/pull/2451) @[chengyu-l](https://github.com/chengyu-l)

- ACL 向前兼容 userblacklist[#2459](https://github.com/OpenAtomFoundation/pika/pull/2459) @[dingxiaoshuai](https://github.com/dingxiaoshuail)

- 丰富了 Pika TCL 测试集[#2497](https://github.com/OpenAtomFoundation/pika/pull/2497) @[Mixficsol](https://github.com/Mixficsol)

- 丰富了 Pika Gotest 测试集[#2502](https://github.com/OpenAtomFoundation/pika/pull/2502) @[Mixficsol](https://github.com/Mixficsol)

## bugfix

- 修复 Pika 有从节点进行全量复制期间会误删除 dump 文件的问题[#2377](https://github.com/OpenAtomFoundation/pika/pull/2377)@[wangshao1](https://github.com/wangshao1)
Expand Down Expand Up @@ -78,6 +94,29 @@

- 修复 Codis fe pika 主从角色显示异常的问题[#2387](https://github.com/OpenAtomFoundation/pika/pull/2387)@[chengyu-l](https://github.com/chengyu-l)

- 修复迁移数据后数据不一致的问题[#2485](https://github.com/OpenAtomFoundation/pika/pull/2485)@[chejinge](https://github.com/chejinge)

- 修复dbsize 计算错误问题[#2494](https://github.com/OpenAtomFoundation/pika/pull/2494)@[chejinge](https://github.com/chejinge)

- 修复扩缩容或者 pod 起停后,Codis-dashboard 界面显示不准确的问题[#2475](https://github.com/OpenAtomFoundation/pika/pull/2475)@[chengyu-l](https://github.com/chengyu-l)

- 修复 DB 层重复上锁的问题[#2372](https://github.com/OpenAtomFoundation/pika/pull/2372) @[Mixficsol](https://github.com/Mixficsol)

- 修复全量复制失败后,未做处理导致数据丢失问题[#2439](https://github.com/OpenAtomFoundation/pika/pull/2439)@[wangshao1](https://github.com/wangshao1)

- 修复主从复制过程中,主实例执行 bgsave 过程中,没有正确回应从的同步请求[#2437](https://github.com/OpenAtomFoundation/pika/pull/2437)@[wangshao1](https://github.com/wangshao1)

- 全量复制过程中,添加数据同步状态,明确数据同步进度[#2430](https://github.com/OpenAtomFoundation/pika/pull/2430)@[baixin01](https://github.com/baixin01)

- 修复从库在 Apply binlog 时,没有对操作的 key 加锁,导致数据不一致的问题[#2409](https://github.com/OpenAtomFoundation/pika/pull/2490) @[chejinge](https://github.com/chejinge)

- 修复 codis slot 迁移过程中 master 实例 coredump 的问题[#2415](https://github.com/OpenAtomFoundation/pika/pull/2415) @[chejinge](https://github.com/chejinge)

- 修复在主从复制过程中,删除正在使用的 dump 文件的问题[#2377](https://github.com/OpenAtomFoundation/pika/pull/2377)@[wangshao1](https://github.com/wangshao1)

- 修复主从复制过程中从实例 rsync 响应错误的问题[#2319](https://github.com/OpenAtomFoundation/pika/pull/2319)@[wangshao1](https://github.com/wangshao1)

- 修复主从复制模式下,主实例flushdb后,从实例因死锁而无法访问的问题[#2372](https://github.com/OpenAtomFoundation/pika/pull/2372) @[Mixficsol](https://github.com/Mixficsol)

# v3.5.2

Expand Down
70 changes: 70 additions & 0 deletions ci/release-build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/bash

function install_deps() {
echo "install deps before ..."
if [[ $OS == *"macos"* ]]; then
brew update
brew install --overwrite python@3.12 autoconf protobuf llvm wget git
brew install gcc@10 automake cmake make binutils
elif [[ $OS == *"ubuntu"* ]]; then
sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
sudo apt-get install -y clang-tidy-12
else
echo "not support $OS"
fi
echo "install deps after success ..."
}

function configure_cmake() {
echo "configure cmake before ..."
if [[ $OS == *"macos"* ]]; then
export CC=/usr/local/opt/gcc@10/bin/gcc-10
cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE
elif [[ $OS == *"ubuntu"* ]]; then
cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS="-s" -DCMAKE_EXE_LINKER_FLAGS="-s"
elif [[ $OS == *"centos"* ]]; then
source /opt/rh/devtoolset-10/enable
cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
fi
echo "configure cmake after ..."
}

function build() {
echo "build before ..."
cmake --build build --config $BUILD_TYPE
echo "build after success ..."
}

function install() {
install_deps
configure_cmake
build
}

function checksum() {
cd build/ && chmod +x $REPO_NAME

mkdir bin && cp $REPO_NAME bin
mkdir conf && cp ../conf/pika.conf conf

tar -zcvf $PACKAGE_NAME bin/$REPO_NAME conf/pika.conf

echo $(shasum -a 256 $PACKAGE_NAME | cut -f1 -d' ') >${PACKAGE_NAME}.sha256sum
}

case $1 in
"install")
OS=$2
BUILD_TYPE=$3
install
;;
"checksum")
REPO_NAME=$2
PACKAGE_NAME=$3
checksum
;;
*)
echo "Invalid option"
echo "option $1"
;;
esac
Loading

0 comments on commit ea1b0d0

Please sign in to comment.