Merge pull request #34090 from trociny/wip-ceph-erasure-code-tool

tools/erasure-code: new tool to encode/decode files Reviewed-by: Josh Durgin <jdurgin@redhat.com> Reviewed-by: Kefu Chai <kchai@redhat.com>
ceph · Apr 27, 2020 · f0a4fc1 · f0a4fc1
2 parents fcf8bd7 + 56aaaf8
commit f0a4fc1
Show file tree

Hide file tree

Showing 11 changed files with 373 additions and 210 deletions.
diff --git a/alpine/APKBUILD.in b/alpine/APKBUILD.in
@@ -415,7 +415,6 @@ ceph_test() {
 		ceph_bench_log \
 		ceph_kvstorebench \
 		ceph_multi_stress_watch \
-		ceph_erasure_code \
 		ceph_erasure_code_benchmark \
 		ceph_omapbench \
 		ceph_objectstore_bench \

diff --git a/ceph.spec.in b/ceph.spec.in
@@ -1987,6 +1987,7 @@ fi
 %files osd
 %{_bindir}/ceph-clsinfo
 %{_bindir}/ceph-bluestore-tool
+%{_bindir}/ceph-erasure-code-tool
 %{_bindir}/ceph-objectstore-tool
 %{_bindir}/ceph-osdomap-tool
 %{_bindir}/ceph-osd
@@ -2211,7 +2212,6 @@ fi
 %{_bindir}/ceph_bench_log
 %{_bindir}/ceph_kvstorebench
 %{_bindir}/ceph_multi_stress_watch
-%{_bindir}/ceph_erasure_code
 %{_bindir}/ceph_erasure_code_benchmark
 %{_bindir}/ceph_omapbench
 %{_bindir}/ceph_objectstore_bench

diff --git a/debian/ceph-osd.install b/debian/ceph-osd.install
@@ -4,6 +4,7 @@ lib/systemd/system/ceph-osd*
 lib/systemd/system/ceph-volume@.service
 usr/bin/ceph-bluestore-tool
 usr/bin/ceph-clsinfo
+usr/bin/ceph-erasure-code-tool
 usr/bin/ceph-objectstore-tool
 usr/bin/ceph-osdomap-tool
 usr/bin/${CEPH_OSD_BASENAME} => /usr/bin/ceph-osd

diff --git a/debian/ceph-test.install b/debian/ceph-test.install
@@ -1,7 +1,6 @@
 usr/bin/ceph-client-debug
 usr/bin/ceph-coverage
 usr/bin/ceph_bench_log
-usr/bin/ceph_erasure_code
 usr/bin/ceph_erasure_code_benchmark
 usr/bin/ceph_kvstorebench
 usr/bin/ceph_multi_stress_watch

diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
@@ -487,7 +487,6 @@ add_dependencies(tests
   cls_lock
   ceph_test_objectstore
   ceph_erasure_code_non_regression
-  ceph_erasure_code
   cython_modules)
 if (WITH_CEPHFS)
   add_dependencies(tests ceph-mds)

diff --git a/src/test/ceph-erasure-code-tool/test_ceph-erasure-code-tool.sh b/src/test/ceph-erasure-code-tool/test_ceph-erasure-code-tool.sh
@@ -0,0 +1,43 @@
+#!/bin/sh -ex
+
+TMPDIR=/tmp/test_ceph-erasure-code-tool.$$
+mkdir $TMPDIR
+trap "rm -fr $TMPDIR" 0
+
+ceph-erasure-code-tool test-plugin-exists INVALID_PLUGIN && exit 1
+ceph-erasure-code-tool test-plugin-exists jerasure
+
+ceph-erasure-code-tool validate-profile \
+                       plugin=jerasure,technique=reed_sol_van,k=2,m=1
+
+test "$(ceph-erasure-code-tool validate-profile \
+          plugin=jerasure,technique=reed_sol_van,k=2,m=1 chunk_count)" = 3
+
+test "$(ceph-erasure-code-tool calc-chunk-size \
+          plugin=jerasure,technique=reed_sol_van,k=2,m=1 4194304)" = 2097152
+
+dd if="$(which ceph-erasure-code-tool)" of=$TMPDIR/data bs=770808 count=1
+cp $TMPDIR/data $TMPDIR/data.orig
+
+ceph-erasure-code-tool encode \
+                       plugin=jerasure,technique=reed_sol_van,k=2,m=1 \
+                       4096 \
+                       0,1,2 \
+                       $TMPDIR/data
+test -f $TMPDIR/data.0
+test -f $TMPDIR/data.1
+test -f $TMPDIR/data.2
+
+rm $TMPDIR/data
+
+ceph-erasure-code-tool decode \
+                       plugin=jerasure,technique=reed_sol_van,k=2,m=1 \
+                       4096 \
+                       0,2 \
+                       $TMPDIR/data
+
+size=$(stat -c '%s' $TMPDIR/data.orig)
+truncate -s "${size}" $TMPDIR/data # remove stripe width padding
+cmp $TMPDIR/data.orig $TMPDIR/data
+
+echo OK
diff --git a/src/test/erasure-code/CMakeLists.txt b/src/test/erasure-code/CMakeLists.txt
@@ -9,11 +9,6 @@ install(TARGETS ceph_erasure_code_benchmark
 add_executable(ceph_erasure_code_non_regression ceph_erasure_code_non_regression.cc)
 target_link_libraries(ceph_erasure_code_non_regression ceph-common Boost::program_options global ${CMAKE_DL_LIBS})
 
-add_executable(ceph_erasure_code ceph_erasure_code.cc)
-target_link_libraries(ceph_erasure_code ceph-common Boost::program_options global ${CMAKE_DL_LIBS})
-install(TARGETS ceph_erasure_code
-  DESTINATION bin)
-
 add_library(ec_example SHARED 
   ErasureCodePluginExample.cc
   $<TARGET_OBJECTS:erasure_code_objs>)

diff --git a/src/test/erasure-code/ceph_erasure_code.cc b/src/test/erasure-code/ceph_erasure_code.cc
diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
@@ -128,3 +128,4 @@ endif()
 
 add_subdirectory(immutable_object_cache)
 add_subdirectory(ceph-dencoder)
+add_subdirectory(erasure-code)
diff --git a/src/tools/erasure-code/CMakeLists.txt b/src/tools/erasure-code/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_executable(ceph-erasure-code-tool
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc
+  ceph-erasure-code-tool.cc)
+target_link_libraries(ceph-erasure-code-tool global ceph-common)
+install(TARGETS ceph-erasure-code-tool DESTINATION bin)