Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add .NET Core API and NuGet Package (using swig) #142

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 19 additions & 0 deletions .gitignore
Expand Up @@ -84,6 +84,25 @@ build/
java/libjavamcsapi.so*
docs/example

#We need simutaneous builds for the dotnet NuGet package
build-win-x64
build-linux-x64

#Just about everything in dotnet is created, esp. noteable
# are the src/*.cs files which come from swig.
dotnet/dotnet_mcsapi_wrap.cxx
dotnet/src/bin/
dotnet/src/obj/
dotnet/src/*.cs
dotnet/test/bin
dotnet/test/obj
dotnet/example/bin
dotnet/example/obj
dotnet/lib-win-x64
dotnet/lib-linux-x64
dotnet/build-win-x64
dotnet/build-linux-x64

# QtCreator files
*.config
*.creator
Expand Down
67 changes: 67 additions & 0 deletions dotnet/CMakeLists.txt
@@ -0,0 +1,67 @@
cmake_minimum_required(VERSION 2.8.11)

include_directories(..)
#Find the packages and libraries
IF(WIN32)
SET(FIND_LIBRARY_USE_LIB64_PATHS ON)
SET(ENGINE_ARCH "x64")
ENDIF(WIN32)

IF(UNIX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILENAME__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"'")
ENDIF(UNIX)
IF(WIN32)
#TODO find a way to define the __FILENAME__ in Windows so that the debug output doesn't contain the absolute path.
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILENAME__=\"$(notdir $<)\"")
ENDIF(WIN32)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBUILDING_MYMCSAPI")

set(DOTNET_AVAILABLE 0)

execute_process(COMMAND dotnet --version OUTPUT_VARIABLE DOTNET_VERSION_AVAILABLE OUTPUT_STRIP_TRAILING_WHITESPACE)
IF(DOTNET_VERSION_AVAILABLE VERSION_GREATER 2.0)
set(DOTNET_AVAILABLE 1)
MESSAGE(STATUS "dotnet ${DOTNET_VERSION_AVAILABLE} Found")
ENDIF(DOTNET_VERSION_AVAILABLE VERSION_GREATER 2.0)

IF(NOT DOTNET_AVAILABLE)
MESSAGE(FATAL_ERROR "dotnet command line not available")
ENDIF()

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If not integrated into the continuous integration pipeline you have to add additional compile flags similar to the ones in the top level CMakeLists.txt to be compile-able on CentOS 7 which uses an older version of g++.

# Disable no-deprecated-declarations
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
if(CMAKE_COMPILER_IS_GNUCXX)
    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wunused -Wwrite-strings -Wno-strict-aliasing -Wextra -Werror -Wno-deprecated-declarations")
endif(CMAKE_COMPILER_IS_GNUCXX)

# Disable format-truncation since this triggers in mcsapi_types for something that isn't a problem
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
if(CMAKE_COMPILER_IS_GNUCXX)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat -Wformat-security -fstack-protector-all -fstack-check -Wno-format-truncation -Wno-unused-command-line-argument -Wno-unknown-warning-option -pie -fPIC")
endif(CMAKE_COMPILER_IS_GNUCXX)

#Checking for SWIG
find_package(SWIG 3 REQUIRED)

IF(DOTNET_AVAILABLE)
MESSAGE(STATUS "Have dotnet... generate wrapper code and makefile")

get_filename_component(INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" DIRECTORY)

set(SWIG_DOTNET_WRAPPER_SOURCE
"${CMAKE_CURRENT_SOURCE_DIR}/dotnet_mcsapi_wrap.cxx")


MESSAGE(STATUS "Have dotnet... Add generate wrapper code to build process ${SWIG_DOTNET_WRAPPER_SOURCE}")
MESSAGE(STATUS "${SWIG_EXECUTABLE} -c++ -csharp -I${INCLUDE_DIR} -namespace MariaDB.Data.ColumnStore -outdir ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/dotnet_mcsapi.i")

add_custom_command(OUTPUT ${SWIG_DOTNET_WRAPPER_SOURCE}
COMMAND ${SWIG_EXECUTABLE} -c++ -csharp -I${INCLUDE_DIR} -namespace MariaDB.Data.ColumnStore -outdir ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/dotnet_mcsapi.i
COMMENT "Generating the dotnet/c# wrapper source code")

add_library(dotnet_mcsapi SHARED "${SWIG_DOTNET_WRAPPER_SOURCE}")
set_target_properties(dotnet_mcsapi PROPERTIES OUTPUT_NAME "dotnet_mcsapi")
set_target_properties(dotnet_mcsapi PROPERTIES PREFIX "")
set_target_properties(dotnet_mcsapi PROPERTIES LIBRARY_OUTPUT_DIRECTORY "glue-lib")
IF(WIN32)
set_target_properties(dotnet_mcsapi PROPERTIES RUNTIME_OUTPUT_DIRECTORY "glue-lib")
target_link_libraries(dotnet_mcsapi ../lib-win-x64/mcsapi)
ENDIF(WIN32)
IF(UNIX)
target_link_libraries(dotnet_mcsapi mcsapi)
target_link_libraries(dotnet_mcsapi libmcsapi.a)
ENDIF(UNIX)

# Directory may not exist, so we'll copy the library in the csproj file.
# install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/dotnet_build/dotnet_mcsapi.so" DESTINATION "${CMAKE_CURRENT_SOURCE_DIR}/test/obj/Debug/netcoreapp2.1/")

ENDIF()

201 changes: 201 additions & 0 deletions dotnet/README.md
@@ -0,0 +1,201 @@
# MariaDB.Data.ColumnStore.ColumnStoreDriver

.NET Core wrapper on the ColumnStore c++ API for Linux (x64) and Windows (x64).

## Getting Started

Riffing off of [MariaDB ColumnStore Bulk Write SDK](https://mariadb.com/kb/en/library/columnstore-bulk-write-sdk/)...

First a simple table is created with the mcsmysql client:

```sql
MariaDB [test]> create table t1(i int, c char(3)) engine=columnstore;
```

Next, create a new console project

```shell
mkdir ColumnStoreTestProgram
cd ColumnStoreTestProgram
dotnet new console
```

Alter Program.cs

```cs
using System;
using MariaDB.Data.ColumnStore;

namespace ColumnStoreTestProgram
{
class Program
{
static void Main(string[] args)
{
// For windows, name the XML file location
var ColumnstoreXmlFilePath = @"C:\opt\mariadb\etc\Columnstore.xml";
var d = new ColumnStoreDriver(ColumnstoreXmlFilePath);
var b = d.createBulkInsert("mcsapi", "t1", 0, 0);
try
{
b.setColumn(0, 2);
b.setColumn(1, "c#!");
b.writeRow();
b.commit();
}
catch (Exception e)
{
b.rollback();
Console.WriteLine(e.StackTrace);
}
}
}
}
```

And run

```shell
dotnet run
```

Now back in mcsmysql verify the data is written:

```misc
MariaDB [test]> select * from t1;
+------+------+
| i | c |
+------+------+
| 2 | c#! |
+------+------+
```

### Windows

Note that when using this driver from windows, one must use the constructor that
takes a string pointing to the ```Columnstore.xml``` file, as shown above, otherwise
it will throw an exception since by default the constructor will look in UNIX-y type
locations.

```cs
using MariaDB.Data.ColumnStore;
...
var mcsConnection = new ColumnStoreDriver(@"C:\opt\mariadb\etc\Columnstore.xml");
```

### Further Examples

Also look at ```dotnet/test/ColumnStoreTest.cs``` for some hints on how to use this
package with some startup checks.

## Building the NuGet Package

Building the combination NuGet package requires a build environment for each
target platform. The simplest way to build the Linux + Win combination is by
using Windows 10 with gitbash coupled with the Windows Subsystem for Linux. (Of course
it is also possible to compile the wrapper on each platform and copy the needed
files over to a single system for the final NuGet pack.)

Requirements:

* Windows 10
* [Visual Studio Community Edition](https://visualstudio.microsoft.com/vs/community/)
* [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) (with a c++ development environment)
* [gitbash](https://git-scm.com/downloads)
* The other build requirements for the API

### Get the Source Code

For building the Linux shared object with WSL, the source code needs to be located
on a Windows partition because WSL can see windows files (under /mnt/DRIVE) but
not the other way around.

In Windows (gitbash):

```shell
mkdir ~/Code/external
cd ~/Code/external
git clone https://github.com/mariadb-corporation/mariadb-columnstore-api.git
cd ~/Code/external/mariadb-columnstore-api
```

### Build the Windows API DLL

The main README contains instructions on how to build the windows DLL.
Or try using [build-win-x64.sh](https://gist.github.com/TikiBill/3dc83d90e5337eb168da82c207f077e3)
from a windows gitbash prompt. Copy that file into mariadb-columnstore-api/extra and then run
it from gitbash.

### Copy the Windows DLLs

The resulting API DLL and LIB, along with the support DLLs (libxml2, libuv, libiconv), need to
be copied into dotnet/lib-win-x64. Assuming your build directory was build-win-x64:

```shell
cd ~/Code/external/mariadb-columnstore-api
mkdir dotnet/lib-win-x64
cp -v build-win-x64/src/RelWithDebInfo/mcsapi.dll dotnet/lib-win-x64
cp -v build-win-x64/src/RelWithDebInfo/mcsapi.lib dotnet/lib-win-x64
cp -v path/to/lib/libiconv.dll dotnet/lib-win-x64
cp -v path/to/lib/libuv.dll dotnet/lib-win-x64
cp -v path/to/lib/libxml2.dll dotnet/lib-win-x64
```

### Build the .NET Glue (Windows)

Run ```dotnet/build.sh``` to walk through building the windows glue dll and
c# files. This will put ```dotnet_mcsapi.dll``` in ```dotnet/lib-win-x64``` as
well as the .cs source files in ```dotnet/src```.

Note this script will also try to copy the DLLs outlined above if you have not
already done so.

### Build and Copy the Linux API Shared Object (.so)

Next, from WSL (or a linux box), follow the main README instructions to build
the shared mcsapi.so library. The commands would be (assuming the C: drive
contains your user directory, and your user name is USER):

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should further mention to execute the build.sh to build dotnet_mcsapi.so

```shell
cd /mnt/c/Users/USER/Code/external/mariadb-columnstore-api
mkdir dotnet/lib-linux-x64
mkdir build-linux-x64
cd build-linux-x64
cmake ..
make
cp src/libmcsapi.* ../dotnet/lib-linux-x64
```

### Build the NuGet Package

Now from either gitbash (Windows) or a WSL prompt, where ever you have the dotnet core SDK
installed, you can generate a NuGet package.

```shell
cd dotnet/src
dotnet pack -c Release
```

This NuGet package is self contained with the needed platform specific glue libraries, and
the needed DLLs for windows. That is, when using this package under windows, one does not
need to install mcsapi.so and the support libraries separately.

### Testing

Currently the tests are very basic and rely on the t1 table existing in the mcsapi database.
The easiest way to perform the tests is to run the MariaDB ColumnStore in the WSL instance, make
the mcsapi database and t1 table, and then run ```dotnet test```. If you copy over the
WSL Columnstore.xml file to e.g. /c/opt/mariadb/etc, then you can also run the tests from windows.

### Testing on Windows

To run the tests in dotnet/test in windows, one will need to set some environment variables, which are
mostly the same as outlined in the main README for windows testing:

* ```COLUMNSTORE_INSTALL_DIR``` -- Required for windows, points to the folder holding the etc/Columnstore.xml file.
* ```PATH``` -- This environment variable must also contain the dotnet/lib-win-x64 directory (as an absolute path).
Windows uses this to find the DLLs to use for testing.

## END OF LINE

[//]: # (cSpell:ignore cmake gitbash mkdir libxml libuv libiconv mcsapi mcsmysql mariadb columnstore libmcsapi nuget .)