Add stub for building ColumnStore

.gitmodules

@@ -14,3 +14,6 @@
 [submodule "storage/maria/libmarias3"]
 	path = storage/maria/libmarias3
 	url = https://github.com/mariadb-corporation/libmarias3.git
+[submodule "storage/columnstore/columnstore"]
+	path = storage/columnstore/columnstore
+	url = https://github.com/mariadb-corporation/mariadb-columnstore-engine.git

cmake/make_dist.cmake.in

@@ -58,6 +58,14 @@ IF(GIT_EXECUTABLE)
   IF(NOT RESULT EQUAL 0)
     SET(GIT_EXECUTABLE)
   ENDIF()
+  EXECUTE_PROCESS(
+    COMMAND "${GIT_EXECUTABLE}" submodule foreach "${GIT_EXECUTABLE} checkout-index --all --prefix=${PACKAGE_DIR}/storage/columnstore/columnstore/$path/"
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore
+    RESULT_VARIABLE RESULT
+  )
+  IF(NOT RESULT EQUAL 0)
+    SET(GIT_EXECUTABLE)
+  ENDIF()
 ENDIF()
 
 CONFIGURE_FILE(${CMAKE_BINARY_DIR}/include/source_revision.h

debian/autobake-deb.sh

@@ -94,6 +94,14 @@ then
   sed -i -e "/Package: libmariadbd-dev/,/^$/d" debian/control
 fi
 
+# Don't package ColumnStore if it wasn't turned on to build
+if [[ $CMAKEFLAGS != *"COLUMNSTORE=YES"* ]]
+then
+    sed -i -e "/Package: mariadb-plugin-columnstore/,/^$/d" debian/control
+    sed -i -e "/Package: mariadb-columnstore-libs/,/^$/d" debian/control
+    sed -i -e "/Package: mariadb-columnstore-platform/,/^$/d" debian/control
+fi
+
 # Adjust changelog, add new version
 echo "Incrementing changelog and starting build scripts"
 

debian/control

@@ -650,6 +650,37 @@ Description: RocksDB storage engine for MariaDB
  at maximising storage efficiency while maintaining InnoDB-like performance.
  This package contains the RocksDB plugin for MariaDB.
 
+Package: mariadb-columnstore-libs
+Architecture: amd64
+Depends: ${misc:Depends},
+         ${shlibs:Depends}
+Description: Required libraries for the MariaDB ColumnStore storage engine
+
+Package: mariadb-columnstore-platform
+Architecture: amd64
+Depends: mariadb-columnstore-libs (= ${binary:Version}),
+         expect,
+         net-tools,
+         libsnappy1 | libsnappy1v5,
+         libjemalloc1 | libjemalloc2,
+         libboost-all-dev,
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: The platform binaries for MariaDB ColumnStore
+
+Package: mariadb-plugin-columnstore
+Architecture: amd64
+Depends: mariadb-server-10.5 (= ${binary:Version}),
+         mariadb-columnstore-libs (= ${binary:Version}),
+         mariadb-columnstore-platform (= ${binary:Version}),
+         ${misc:Depends},
+         ${shlibs:Depends}
+Recommends: python-mysqldb
+Description: MariaDB ColumnStore storage engine
+ The MariaDB ColumnStore storage engine is a high-performance columnar
+ analytical engine, aimed at rapid processing of analytical queries on very
+ large amounts of data.
+
 Package: mariadb-plugin-oqgraph
 Architecture: any
 Depends: libjudydebian1,

debian/mariadb-columnstore-libs.install

@@ -0,0 +1,35 @@
+usr/lib/x86_64-linux-gnu/libalarmmanager.so
+usr/lib/x86_64-linux-gnu/libbatchloader.so
+usr/lib/x86_64-linux-gnu/libbrm.so
+usr/lib/x86_64-linux-gnu/libcacheutils.so
+usr/lib/x86_64-linux-gnu/libcloudio.so
+usr/lib/x86_64-linux-gnu/libcommon.so
+usr/lib/x86_64-linux-gnu/libcompress.so
+usr/lib/x86_64-linux-gnu/libconfigcpp.so
+usr/lib/x86_64-linux-gnu/libdataconvert.so
+usr/lib/x86_64-linux-gnu/libddlcleanuputil.so
+usr/lib/x86_64-linux-gnu/libddlpackage.so
+usr/lib/x86_64-linux-gnu/libddlpackageproc.so
+usr/lib/x86_64-linux-gnu/libdmlpackage.so
+usr/lib/x86_64-linux-gnu/libdmlpackageproc.so
+usr/lib/x86_64-linux-gnu/libexecplan.so
+usr/lib/x86_64-linux-gnu/libfuncexp.so
+usr/lib/x86_64-linux-gnu/libidbdatafile.so
+usr/lib/x86_64-linux-gnu/libjoblist.so
+usr/lib/x86_64-linux-gnu/libjoiner.so
+usr/lib/x86_64-linux-gnu/liblibmysql_client.so
+usr/lib/x86_64-linux-gnu/libloggingcpp.so
+usr/lib/x86_64-linux-gnu/libmessageqcpp.so
+usr/lib/x86_64-linux-gnu/liboamcpp.so
+usr/lib/x86_64-linux-gnu/libquerystats.so
+usr/lib/x86_64-linux-gnu/libquerytele.so
+usr/lib/x86_64-linux-gnu/libregr.so
+usr/lib/x86_64-linux-gnu/librowgroup.so
+usr/lib/x86_64-linux-gnu/librwlock.so
+usr/lib/x86_64-linux-gnu/libthreadpool.so
+usr/lib/x86_64-linux-gnu/libthrift.so
+usr/lib/x86_64-linux-gnu/libudfsdk.so
+usr/lib/x86_64-linux-gnu/libwindowfunction.so
+usr/lib/x86_64-linux-gnu/libwriteengine.so
+usr/lib/x86_64-linux-gnu/libwriteengineclient.so
+usr/lib/x86_64-linux-gnu/libwriteengineredistribute.so

debian/mariadb-columnstore-platform.install

@@ -0,0 +1,106 @@
+etc/columnstore/AlarmConfig.xml
+etc/columnstore/Columnstore.xml
+etc/columnstore/ConsoleCmds.xml
+etc/columnstore/ErrorMessage.txt
+etc/columnstore/MessageFile.txt
+etc/columnstore/ProcessConfig.xml
+etc/columnstore/storagemanager.cnf
+usr/bin/DDLProc
+usr/bin/DMLProc
+usr/bin/ExeMgr
+usr/bin/MCSInstanceCmds.sh
+usr/bin/MCSVolumeCmds.sh
+usr/bin/MCSgetCredentials.sh
+usr/bin/PrimProc
+usr/bin/ProcMgr
+usr/bin/ProcMon
+usr/bin/ServerMonitor
+usr/bin/StorageManager
+usr/bin/WriteEngineServer
+usr/bin/alarmReport.sh
+usr/bin/autoConfigure
+usr/bin/bulklogReport.sh
+usr/bin/clearShm
+usr/bin/cleartablelock
+usr/bin/columnstore
+usr/bin/columnstore-post-install
+usr/bin/columnstore-pre-uninstall
+usr/bin/columnstoreClusterTester.sh
+usr/bin/columnstoreDBWrite
+usr/bin/columnstoreSupport
+usr/bin/columnstoreSyslogSetup.sh
+usr/bin/columnstore_installer
+usr/bin/columnstore_module_installer.sh
+usr/bin/columnstore_os_check.sh
+usr/bin/columnstore_run.sh
+usr/bin/colxml
+usr/bin/configReport.sh
+usr/bin/configxml.sh
+usr/bin/controllernode
+usr/bin/cpimport
+usr/bin/cpimport.bin
+usr/bin/cplogger
+usr/bin/dbbuilder
+usr/bin/dbrmctl
+usr/bin/ddlcleanup
+usr/bin/disable-rep-columnstore.sh
+usr/bin/editem
+usr/bin/getMySQLpw
+usr/bin/hardwareReport.sh
+usr/bin/idbmeminfo
+usr/bin/load_brm
+usr/bin/logReport.sh
+usr/bin/mariadb-command-line.sh
+usr/bin/master-rep-columnstore.sh
+usr/bin/mcsGetConfig
+usr/bin/mcsSetConfig
+usr/bin/mcs_module_installer.sh
+usr/bin/mcsadmin
+usr/bin/mycnfUpgrade
+usr/bin/os_detect.sh
+usr/bin/post-mysql-install
+usr/bin/post-mysqld-install
+usr/bin/postConfigure
+usr/bin/quick_installer_amazon.sh
+usr/bin/quick_installer_multi_server.sh
+usr/bin/quick_installer_single_server.sh
+usr/bin/remote_command.sh
+usr/bin/remote_command_verify.sh
+usr/bin/remote_scp_get.sh
+usr/bin/remote_scp_put.sh
+usr/bin/remotessh.exp
+usr/bin/reset_locks
+usr/bin/resourceReport.sh
+usr/bin/rollback
+usr/bin/rsync.sh
+usr/bin/save_brm
+usr/bin/slave-rep-columnstore.sh
+usr/bin/smcat
+usr/bin/smls
+usr/bin/smput
+usr/bin/smrm
+usr/bin/softwareReport.sh
+usr/bin/startupTests.sh
+usr/bin/viewtablelock
+usr/bin/workernode
+usr/lib/x86_64-linux-gnu/libmarias3.so
+usr/lib/x86_64-linux-gnu/libmarias3.so.3
+usr/lib/x86_64-linux-gnu/libmarias3.so.3.1.2
+usr/lib/x86_64-linux-gnu/libstoragemanager.so
+usr/share/columnstore/columnstore.service
+usr/share/columnstore/columnstoreAlias
+usr/share/columnstore/columnstoreLogRotate
+usr/share/columnstore/columnstoreSyslog
+usr/share/columnstore/columnstoreSyslog-ng
+usr/share/columnstore/columnstoreSyslog7
+usr/share/columnstore/columnstore_functions
+usr/share/columnstore/gitversionEngine
+usr/share/columnstore/mcstest-001.sh
+usr/share/columnstore/mcstest-002.sh
+usr/share/columnstore/mcstest-003.sh
+usr/share/columnstore/mcstest-004.sh
+usr/share/columnstore/myCnf-exclude-args.text
+usr/share/columnstore/myCnf-include-args.text
+usr/share/columnstore/releasenum
+var/lib/columnstore/local/module
+

debian/mariadb-columnstore-platform.postinst

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+columnstore-post-install

debian/mariadb-columnstore-platform.postrm

@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+if [ "$1" = "purge" ]; then
+    rm -rf /var/lib/columnstore
+fi
+

debian/mariadb-columnstore-platform.prerm

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+mcsadmin shutdown

debian/mariadb-plugin-columnstore.install

@@ -0,0 +1,11 @@
+etc/mysql/conf.d/columnstore.cnf
+usr/lib/mysql/plugin/ha_columnstore.so
+usr/lib/mysql/plugin/libregr_mysql.so
+usr/lib/mysql/plugin/libudf_mysql.so
+usr/sbin/install_mcs_mysql.sh
+usr/share/columnstore/calremoveuserpriority.sql
+usr/share/columnstore/calsetuserpriority.sql
+usr/share/columnstore/calshowprocesslist.sql
+usr/share/columnstore/columnstore_info.sql
+usr/share/columnstore/dumpcat_mysql.sql
+usr/share/columnstore/syscatalog_mysql.sql

storage/columnstore/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(PLUGIN_COLUMNSTORE "NO" CACHE STRING "Enable ColumnStore engine")
+
+if("NO" STREQUAL "${PLUGIN_COLUMNSTORE}")
+  return()
+endif()
+
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCOLUMNSTORE_MATURITY=MariaDB_PLUGIN_MATURITY_GAMMA")
+add_subdirectory(columnstore)
+
+# Needed to bump the component changes up to the main scope
+SET(CPACK_COMPONENTS_ALL ${CPACK_COMPONENTS_ALL} PARENT_SCOPE)
+IF (RPM)
+    SET(CPACK_RPM_columnstore-platform_USER_FILELIST ${CPACK_RPM_columnstore-platform_USER_FILELIST} PARENT_SCOPE)
+    SET(CPACK_RPM_columnstore-libs_USER_FILELIST ${CPACK_RPM_columnstore-libs_USER_FILELIST} PARENT_SCOPE)
+    SET(CPACK_RPM_columnstore-engine_USER_FILELIST ${CPACK_RPM_columnstore-engine_USER_FILELIST} PARENT_SCOPE)
+    SET(CPACK_RPM_columnstore-platform_PACKAGE_REQUIRES ${CPACK_RPM_columnstore-platform_PACKAGE_REQUIRES} PARENT_SCOPE)
+    SET(CPACK_RPM_columnstore-platform_PRE_INSTALL_SCRIPT_FILE ${CPACK_RPM_columnstore-platform_PRE_INSTALL_SCRIPT_FILE} PARENT_SCOPE)
+    SET(CPACK_RPM_columnstore-platform_POST_INSTALL_SCRIPT_FILE ${CPACK_RPM_columnstore-platform_POST_INSTALL_SCRIPT_FILE} PARENT_SCOPE)
+    SET(CPACK_RPM_columnstore-platform_PRE_UNINSTALL_SCRIPT_FILE ${CPACK_RPM_columnstore-platform_PRE_UNINSTALL_SCRIPT_FILE} PARENT_SCOPE)
+ENDIF ()

storage/columnstore/mysql-test/columnstore/include/cleanup_columnstore.inc

@@ -0,0 +1,4 @@
+--disable_query_log
+DROP FUNCTION IF EXISTS mcssystemready;
+--enable_query_log
+

storage/columnstore/mysql-test/columnstore/include/have_columnstore.inc

@@ -0,0 +1,13 @@
+if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'columnstore' AND support IN ('YES', 'DEFAULT', 'ENABLED')`)
+{
+  --skip Test requires engine ColumnStore.
+}
+
+--disable_query_log
+CREATE FUNCTION mcssystemready RETURNS INTEGER SONAME 'ha_columnstore.so';
+--enable_query_log
+
+if (`SELECT mcssystemready() = 0`)
+{
+  --skip Test requires ColumnStore to be running
+}

storage/columnstore/mysql-test/columnstore/r/basic.result

@@ -0,0 +1,17 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a INT, b VARCHAR(255)) ENGINE=columnstore;
+INSERT INTO t1 (a, b) VALUES (1, 'columnstore'), (2, 'test');
+SELECT * FROM t1;
+a	b
+1	columnstore
+2	test
+UPDATE t1 SET a=a+100;
+SELECT * FROM t1;
+a	b
+101	columnstore
+102	test
+DELETE FROM t1 WHERE a=101;
+SELECT * FROM t1;
+a	b
+102	test
+DROP TABLE t1;

storage/columnstore/mysql-test/columnstore/suite.opt

@@ -0,0 +1 @@
+--plugin-load-add=$HA_COLUMNSTORE_SO

storage/columnstore/mysql-test/columnstore/suite.pm

@@ -0,0 +1,7 @@
+package My::Suite::ColumnStore;
+
+use My::SysInfo;
+
+@ISA = qw(My::Suite);
+
+bless { };

storage/columnstore/mysql-test/columnstore/t/basic.test

@@ -0,0 +1,20 @@
+--source include/have_columnstore.inc
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (a INT, b VARCHAR(255)) ENGINE=columnstore;
+
+INSERT INTO t1 (a, b) VALUES (1, 'columnstore'), (2, 'test');
+SELECT * FROM t1;
+
+UPDATE t1 SET a=a+100;
+SELECT * FROM t1;
+
+DELETE FROM t1 WHERE a=101;
+SELECT * FROM t1;
+
+DROP TABLE t1;
+
+--source include/cleanup_columnstore.inc