From fa306dc936a0e4c4dc65ea46c86daa6d1f481b2d Mon Sep 17 00:00:00 2001 From: "L.Lin" <8199705@qq.com> Date: Mon, 10 Jun 2019 09:43:48 +0800 Subject: [PATCH] update --- .gitignore | 3 + README.md | 80 +- aws/src/AWSS3.cpp | 3 + aws/src/AWSS3.h | 3 +- hdf5/PluginHdf5.txt | 6 +- hdf5/PluginHdf5_win.txt | 5 +- hdf5/README_CN.md | 503 ++ hdf5/README_ENG.md | 508 ++ hdf5/h5file/SDScompound.h5 | Bin 0 -> 2208 bytes hdf5/h5file/array_mdatom.h5 | Bin 0 -> 16056 bytes hdf5/h5file/compound.h5 | Bin 0 -> 10016 bytes hdf5/h5file/float.h5 | Bin 0 -> 7764 bytes hdf5/h5file/named_type.h5 | Bin 0 -> 1168 bytes hdf5/h5file/scalar.h5 | Bin 0 -> 8294 bytes hdf5/h5file/smpl_enum.h5 | Bin 0 -> 2768 bytes hdf5/h5file/smpl_numeric.h5 | Bin 0 -> 5072 bytes hdf5/h5file/timestamp.h5 | Bin 0 -> 4840 bytes hdf5/h5file/vlen_str.h5 | Bin 0 -> 27296 bytes hdf5/src/hdf5_plugin.cpp | 121 +- hdf5/src/hdf5_plugin.h | 2 - hdf5/test/test_basic_simple_type.txt | 550 ++ include/CoreConcept.h | 423 +- include/SmartPointer.h | 2 - include/StreamEngine.h | 179 + include/SysIO.h | 6 +- mysql/README.md | 4 +- mysql/README_CN.md | 2 +- odbc/DolphinDBODBC.txt | 2 +- odbc/README.md | 4 +- odbc/src/DolphinDBODBC.cpp | 283 +- odbc/src/cvt.cpp | 58 + odbc/src/cvt.h | 32 + odbc/src/nanodbc.cpp | 969 ++-- odbc/src/nanodbc/nanodbc.h | 219 +- odbc/src/utfcpp/utf8.h | 38 + odbc/src/utfcpp/utf8/checked.h | 324 ++ odbc/src/utfcpp/utf8/core.h | 321 ++ odbc/src/utfcpp/utf8/cpp11.h | 103 + odbc/src/utfcpp/utf8/unchecked.h | 229 + odbc/test_setup/setup.txt | 4 + odbc/tests/README.md | 65 + odbc/tests/test_mssql.txt | 498 ++ odbc/tests/test_mysql.txt | 499 ++ odbc/tests/test_postgres.txt | 489 ++ odbc/utfcpp_LICENSE | 23 + odbc/windows mssql odbc guide/CMakeLists.txt | 14 + odbc/windows mssql odbc guide/nanodbc.cpp | 4981 ++++++++++++++++++ odbc/windows mssql odbc guide/nanodbc.h | 1880 +++++++ odbc/windows mssql odbc guide/readme.md | 5 + odbc/windows mssql odbc guide/test.txt | 500 ++ zlib/src/ZlibImpl.cpp | 4 +- zlib/src/ZlibImpl.h | 4 +- 52 files changed, 12944 insertions(+), 1004 deletions(-) create mode 100644 .gitignore create mode 100644 hdf5/README_CN.md create mode 100644 hdf5/README_ENG.md create mode 100644 hdf5/h5file/SDScompound.h5 create mode 100644 hdf5/h5file/array_mdatom.h5 create mode 100644 hdf5/h5file/compound.h5 create mode 100644 hdf5/h5file/float.h5 create mode 100644 hdf5/h5file/named_type.h5 create mode 100644 hdf5/h5file/scalar.h5 create mode 100644 hdf5/h5file/smpl_enum.h5 create mode 100644 hdf5/h5file/smpl_numeric.h5 create mode 100644 hdf5/h5file/timestamp.h5 create mode 100644 hdf5/h5file/vlen_str.h5 create mode 100644 hdf5/test/test_basic_simple_type.txt create mode 100644 include/StreamEngine.h create mode 100644 odbc/src/cvt.cpp create mode 100644 odbc/src/cvt.h create mode 100644 odbc/src/utfcpp/utf8.h create mode 100644 odbc/src/utfcpp/utf8/checked.h create mode 100644 odbc/src/utfcpp/utf8/core.h create mode 100644 odbc/src/utfcpp/utf8/cpp11.h create mode 100644 odbc/src/utfcpp/utf8/unchecked.h create mode 100644 odbc/test_setup/setup.txt create mode 100644 odbc/tests/README.md create mode 100644 odbc/tests/test_mssql.txt create mode 100644 odbc/tests/test_mysql.txt create mode 100644 odbc/tests/test_postgres.txt create mode 100644 odbc/utfcpp_LICENSE create mode 100644 odbc/windows mssql odbc guide/CMakeLists.txt create mode 100644 odbc/windows mssql odbc guide/nanodbc.cpp create mode 100644 odbc/windows mssql odbc guide/nanodbc.h create mode 100644 odbc/windows mssql odbc guide/readme.md create mode 100644 odbc/windows mssql odbc guide/test.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..932ed1bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ + +odbc/tests/mssql-employees.bak +odbc/tests/postgres.tar diff --git a/README.md b/README.md index b88feef8..2f980579 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,30 @@ -## DolphinDB introduction -DolphinDB is an extremely fast time series database and data analytical system with unlimited scaling capacity. It has built-in parallel and distributed computing supports. It comes with battery included for real time data processing and analytics across unlimited computing nodes. It is integrated with an easy-to-use fully featured programming language and a high-volume high-velocity streaming analytics system. DolphinDB offers operational simplicity, scalability, fault tolerance, and concurrency. To learn more about DolphinDB, please go to - -``` -Downloads: http://www.dolphindb.com/downloads.html -Manual: http://dolphindb.com/help/ -Examples: http://dolphindb.com/examples.html -``` - ## DolphinDB Plugin -DolphinDB supports dynamic loading of external plugins to extend system functionality. The plug-in only supports writing in C++, and it needs to be compiled into ".so" shared libraries or ".dll" shared library files. - +DolphinDB 支持动态的载入外部插件，拓展系统功能。插件仅支持使用C++编写，并且需要编译成so共享库或者dll共享库文件。 ## Directory Structures -* ```include```The directory contains the class declarations and some tool class declarations for the core data structures of DolphinDB. These classes are important basic tools for implementing plug-ins. -* ```demo```The directory contains a demo plug-in implementation. -* ```odbc```The directory contains an implementation of the odbc plugin. +* ```include```目录包含了DolphinDB的核心数据结构的类声明和一些工具类声明，这些类是实现插件的重要基础工具。 +* ```demo```目录包含了一个demo插件的实现。 +* ```odbc```目录包含了一个odbc插件的实现。 ## Loading Plugin -Use the ```loadPlugin``` function to load an external plugin that accepts a file path that describes the format of the plugin. +使用```loadPlugin```函数来加载外部插件，该函数接受一个文件路径，文件中描述了插件的格式。 ## DolphinDB Plugin Format - -DolphinDB uses a text file to describe the plugin. The file format is as follows: -The first line describes the plug-in name and shared library file name. -Each of the following lines describes the mapping between a shared library function and the dolphinDB function. +DolphinDB使用一个文本文件来描述插件,该文件格式如下: +首行描述了插件名字和共享库文件名。 +接下来的每一行都描述了一个共享库函数和dolphinDB函数的映射关系。 ``` module name，lib file function name in lib, function name in dolphindb, function type，minParamCount, maxParamCount, isAggregated ... ``` -**Explanation**： -* module name: plugin module name -* lib file: shared library file name -* function name in lib: The function name in the shared library -* function name in dolphindb: corresponding function name in dolphindb -* function type: operator or system -* minParamCount: the minimum number of parameters -* maxParamCount: the maximum number of parameters -* isAggregated: whether it is an aggregate function +**解释**： +* module name: 模块名 +* lib file: 共享文件库文件名 +* function name in lib: 共享库中导出的函数名 +* function name in dolphindb: dolphindb中的对应函数名 +* function type: operator或者system +* minParamCount: 最小参数个数 +* maxParamCount：最大参数个数 +* isAggregated: 是否为聚集函数 ## Example PluginDemo.txt: @@ -44,31 +33,26 @@ demo,libPluginDemo.so minmax,minmax,operator,1,1,0 foo,foo,system,1,1,0 ``` -The above description file defines a plugin named ```demo```. The shared library file is named ```libPluginDemo.so```. +上述的描述文件定义了一个名为```demo```的插件，共享库文件名为```libPluginDemo.so```. +插件导出了两个函数，第一个函数名字为```minmax```,该函数在dolphinDB中名字同样是```minmax```,operator类型，接受一个参数;第二个函数名字为```echo```,dolphinDB中名字同样是```echo```, system类型，接受一个参数. -The plug-in exports two functions. The first function is named ```minmax```. The name of the function is also ```minmax``` in dolphinDB. The function type is "operator" and accepts one parameter. The second function name is ```echo```, the name in dolphinDB is also ```echo```, the function type is "system" and accepts one argument. +写完描述文件之后，就可以开始编写插件了，内容请参考```demo```文件夹内容。 - -After writing the description file, you can start writing plugins. For content, please refer to ```demo``` folder contents. - - -The compiler needs to use DolphinDB's core library ```libDolphinDB.so```, which implements the classes declared in ```include``` directories. -The compilation steps are as follows +编译需要用到DolphinDB的核心库```libDolphinDB.so```,该核心库实现了```include```目录下声明的类。 +编译步骤如下 ``` cd demo g++ -DLINUX -fPIC -c src/Demo.cpp -I../include -o Demo.o g++ -fPIC -shared -o libPluginDemo.so Demo.o -lDolphinDB ``` +编译成功后，目录下会生成一个名为```libPluginDemo.so```的共享文件。 -After successful compilation, a shared library file named ```libPluginDemo.so``` will be generated under the directory. - - -Enter the following command in the dolphinb console to load the plugin and use it. +在dolphindb的控制台中输入下列命令加载插件并使用。 ``` ->loadPlugin(Path to PluginDemo.txt); // Load the plugin +>loadPlugin(PluginDemo.txt的路径); // 加载插件 (minmax,echo) ->use demo; // Import the plugin's namespace ->demo::minmax([12,3,4]); // You can also use minmax([12,3,4]) +>use demo; // 引入插件的命名空间 +>demo::minmax([12,3,4]); // 也可以使用minmax([12,3,4]) [3,12] >demo::echo("foo"); foo @@ -76,9 +60,11 @@ foo 1 ``` -For more complex plugin implementation please refer to the ```odbc``` directory. +更复杂的插件实现请参考```odbc```目录下的内容。 ## Tips -* It is recommended that you use the ```ld``` command to check if the compiler link is successful and there are undefined references in the so. If ```ld``` error, then DolphinDB can not load the plug-in correctly. -* If the program is crashed after loading the plug-in, you can try the following steps. - 1. Make sure that the ```include``` headers are consistent with the ```libDolphinDB.so``` implementation.Make sure that the version of ``gcc``` used to compile the plugin is consistent with the version of ``libBoardDB.so```, so as to avoid the incompatibilities between different versions of the compiler ABI. +* 建议使用```ld```命令检查下编译器链接是否成功，so中是否存在未定义的引用。如果```ld```报错，那么DolphinDB也无法正确加载插件。 +* 如果载入插件之后出现了crash，可以采取尝试以下步骤。 + 1. 确保```include```下的头文件和```libDolphinDB.so```实现保持一致. + 2. 确保用于编译插件的```gcc```版本和编译```libDolphinDB.so```的版本保持一致，以免出现不同版本的编译器ABI不兼容的问题。 +* 如果编译时出现链接问题（```undefined reference```），并且包含```std::__cxx11```字样，务必检查用于编译插件的```gcc```版本（```gcc 6.2.0 且有--disable-libstdcxx-dual-abi```） diff --git a/aws/src/AWSS3.cpp b/aws/src/AWSS3.cpp index e7e2b0e5..3b293479 100644 --- a/aws/src/AWSS3.cpp +++ b/aws/src/AWSS3.cpp @@ -464,6 +464,7 @@ void createS3Bucket(Heap* heap, vector& args) { Aws::Utils::Logging::ShutdownAWSLogging(); } +#if 0 ConstantSP createS3InputStream(Heap* heap, vector& args) { if(!args[0]->isDictionary()) { throw IllegalArgumentException("createS3InputStream", @@ -614,3 +615,5 @@ IO_ERR S3InputStream::fileStreamZlib(char *buf, size_t length, size_t& actualLen return CORRUPT; } } +#endif + diff --git a/aws/src/AWSS3.h b/aws/src/AWSS3.h index da662fb6..d8d25232 100644 --- a/aws/src/AWSS3.h +++ b/aws/src/AWSS3.h @@ -30,7 +30,7 @@ extern "C" void uploadS3Object(Heap* heap, vector& args); extern "C" ConstantSP listS3Bucket(Heap* heap, vector& args); extern "C" void deleteS3Bucket(Heap* heap, vector& args); extern "C" void createS3Bucket(Heap* heap, vector& args); - +#if 0 extern "C" ConstantSP createS3InputStream(Heap* heap, vector& args); class ZlibBuffer @@ -99,5 +99,6 @@ class S3InputStream : public DataInputStream }; zlibStruct* zStruct = nullptr; }; +#endif #endif /* AWSS3PLUGIN_H_ */ diff --git a/hdf5/PluginHdf5.txt b/hdf5/PluginHdf5.txt index b80a95ed..70bf8680 100644 --- a/hdf5/PluginHdf5.txt +++ b/hdf5/PluginHdf5.txt @@ -1,7 +1,7 @@ hdf5,libPluginHdf5.so h5ls,ls,operator,1,1,0 h5lsTable,lsTable,operator,1,1,0 -h5read,read,system,2,3,0 extractHDF5Schema,extractHDF5Schema,operator,2,2,0 -loadHDF5,loadHDF5,system,2,2,0 -loadHDF5Ex,loadHDF5Ex,system,3,3,0 +loadHDF5,loadHDF5,system,2,5,0 +loadHDF5Ex,loadHDF5Ex,system,5,8,0 +HDF5DS,HDF5DS,system,2,4,0 \ No newline at end of file diff --git a/hdf5/PluginHdf5_win.txt b/hdf5/PluginHdf5_win.txt index 65b129ae..a3820605 100644 --- a/hdf5/PluginHdf5_win.txt +++ b/hdf5/PluginHdf5_win.txt @@ -1,4 +1,7 @@ hdf5,libPluginHdf5.dll h5ls,ls,operator,1,1,0 h5lsTable,lsTable,operator,1,1,0 -h5read,read,system,2,3,0 \ No newline at end of file +extractHDF5Schema,extractHDF5Schema,operator,2,2,0 +loadHDF5,loadHDF5,system,2,5,0 +loadHDF5Ex,loadHDF5Ex,system,5,8,0 +HDF5DS,HDF5DS,system,2,4,0 \ No newline at end of file diff --git a/hdf5/README_CN.md b/hdf5/README_CN.md new file mode 100644 index 00000000..4ae7e4d0 --- /dev/null +++ b/hdf5/README_CN.md @@ -0,0 +1,503 @@ +# DolphinDB HDF5 Plugin + +DolphinDB的hdf5导入插件,该插件可將hdf5数据集导入进DolphinDB中,并且支持对数据类型转换 + + +* [Build](#安装构建) + * [build with cmake](#使用cmake构建) + * [build with makefile](#使用makefile构建) +* [User-Api](#用户接口) + * [hdf5::ls](#hdf5ls) + * [hdf5::lsTable](#hdf5lstable) + * [hdf5::extractHdf5Schema](#hdf5extracthdf5schema) + * [hdf5::loadHdf5](#hdf5loadhdf5) + * [hdf5::loadHdf5Ex](#hdf5loadhdf5ex) + * [hdf5::hdf5DS](#hdf5hdf5ds) +* [Data Type](#支持的数据类型) + * [integer](#integer) + * [float](#float) + * [time](#time) + * [string](#string) + * [enum](#enum) + * [compound](#compound-and-array) + * [array](#compound-and-array) +* [Table Struct](#表结构) + * [simple datatype table struct](#简单类型) + * [complex datatype table struct](#复杂类型) +* [performance](#性能数据) + +# 安装构建 + +## 在Linux下安装 + +### 使用cmake构建 +**Note:** [cmake](https://cmake.org/) 是一个流行的项目构建工具,可以帮你轻松的解决第三方依赖的问题 + +安装cmake +``` +sudo apt-get install cmake +``` +安装hdf5开发包 +``` +sudo apt-get install libhdf5-dev +``` +构建整个工程 +``` +mkdir build +cd build +cmake ../path_to_hdf5_plugin/ +make +``` + +### 使用makefile构建 +安装hdf5开发包 + +``` +sudo apt-get install libhdf5-dev +``` +执行make构建 +``` +make +``` + +**Remember:** 编译之前请确保`libDolphinDB.so`在gcc可搜索的路径中,可使用`LD_LIBRARY_PATH`指定其路径 + +编译之后目录下会产生libPluginHdf5.so文件 + +## 在Windows下安装 + +在Windows下安装，需要启用HDF5的`enable threadsafe`选项。这需要在编译HDF5时配置。 + +你可以使用我们预先编译的1.10.2版本的`hdf5.dll`文件。 + +你也可以自己选择HDF5的版本编译，在HDF5的[官方网站](https://www.hdfgroup.org/solutions/hdf5/)下载源代码，按照说明，如果采用configure方式编译，在配置时启用`--enable-threadsafe`, `--disable-cxx`, `--enable-shared`选项，如果采用CMake方式编译，在编译时启用`-DHDF5_ENABLE_THREADSAFE:BOOL=ON`, `-DHDF5_BUILD_CPP_LIB:BOOL=OFF`, `-DBUILD_SHARED_LIBS:BOOL=ON`参数。 + +# 用户接口 + +**Remember:** 使用api前需使用`loadPlugin("/path_to_PluginHdf5.txt/PluginHdf5.txt")`导入插件 + +## hdf5::ls + +### 语法 + +* `hdf5::ls(fileName)` + +### 参数 + +* `fileName`: hdf5文件名，类型为string。 + +### 详情 +* 列出一张表中的所有`hdf5对象`包括`数据集(dataset)`和`组(group)`以及`(命名类型)namedType`,对于数据集,我们会返回他的尺寸,尺寸是列优先的,对于比如DataSet{(7,3)}代表7列3行。 + +### 例子 +``` +hdf5::ls("/smpl_numeric.h5") + +output: + objName objType + -------------------- + / Group + /double DataSet{(7,3)} + /float DataSet{(7,3)} + /schar DataSet{(7,3)} + /sint DataSet{(7,3)} + /slong DataSet{(7,3)} + /sshort DataSet{(7,3)} + /uchar DataSet{(7,3)} + /uint DataSet{(7,3)} + /ulong DataSet{(1,1)} + /ushort DataSet{(7,3)} + +hdf5::ls("/named_type.h5") + +output: + objName objType + ---------------------- + / Group + /type_name NamedDataType + + +``` +## hdf5::lsTable + +### 语法 + +* `hdf5::lsTable(fileName)` + +### 参数 + +* `fileName`: hdf5文件名，类型为string。 + +### 详情 +* 列出一张表中的所有table信息，即hdf5`数据集(dataset)`对象信息，包括表名，表大小，表类型。 + +### 例子 +``` +hdf5::lsTable("/smpl_numeric.h5") + +output: + tableName tableDims tableType + /double 7,3 H5T_NATIVE_DOUBLE + /float 7,3 H5T_NATIVE_FLOAT + /schar 7,3 H5T_NATIVE_SCHAR + /sint 7,3 H5T_NATIVE_INT + /slong 7,3 H5T_NATIVE_LLONG + /sshort 7,3 H5T_NATIVE_SHORT + /uchar 7,3 H5T_NATIVE_UCHAR + /uint 7,3 H5T_NATIVE_UINT + /ulong 1,1 H5T_NATIVE_ULLONG + /ushort 7,3 H5T_NATIVE_USHORT +``` + +## hdf5::extractHdf5Schema + +### 语法 + +* `hdf5::extractHdf5Schema(fileName,datasetName)` + +### 参数 +* `fileName`: hdf5文件名，类型为字符串标量。 +* `datasetName`: dataset名称，即表名，可通过ls或lsTable获得，类型为字符串标量。 + +### 详情 +* 生成输入数据集的表的结构。表的结构有两列：列名和数据类型。 + +### 例子 +``` +hdf5::extractHdf5Schema("/smpl_numeric.h5","sint") + +output: + name type + col_0 INT + col_1 INT + col_2 INT + col_3 INT + col_4 INT + col_5 INT + col_6 INT + + +hdf5::extractHdf5Schema("/compound.h5","com") + +output: + name type + fs STRING + vs STRING + d DOUBLE + t TIMESTAMP + l LONG + f FLOAT + i INT + s SHORT + c CHAR +``` + +## hdf5::loadHdf5 + +### 语法 + +* `hdf5::loadHdf5(fileName,datasetName,[schema],[startRow],[rowNum])` + +### 参数 +* `fileName`: hdf5文件名，类型为字符串标量。 +* `datasetName`: dataset名称，即表名，可通过ls或lsTable获得，类型为字符串标量。 +* `schema`: 包含列名和列的数据类型的表。如果我们想要改变由系统自动决定的列的数据类型，需要在`schema`表中修改数据类型，并且把它作为`loadHdf5`函数的一个参数。 +* `startRow`: 读取hdf5数据集的起始行数，若不指定，默认从数据集起始位置读取。 +* `rowNum`: 读取hdf5数据集的行数，若不指定，默认读到数据集的结尾。 + +### 详情 +* 将hdf5文件中的数据集转换为DolphinDB数据库的内存表。 +* 读取的行数为hdf5文件中定义的行数，而不是读取结果中的DolphinDB表的行数。 +* 支持的数据类型,以及数据转化规则可见[数据类型](#支持的数据类型)章节。 + +### 例子 +``` +hdf5::loadHdf5("/smpl_numeric.h5","sint") + +output: + col_0 col_1 col_2 col_3 col_4 col_5 col_6 + (758) 8 (325,847) 87 687 45 90 + 61 0 28 77 546 789 45 + 799 5,444 325,847 678 90 54 0 + + +scm = table(`a`b`c`d`e`f`g as name, `CHAR`BOOL`SHORT`INT`LONG`DOUBLE`FLOAT as type) +hdf5::loadHdf5("../hdf5/h5file/smpl_numeric.h5","sint",scm,1,1) + +output: + a b c d e f g + '=' false 28 77 546 789 45 + +``` + +> **Note:** **数据集的dataspace维度必须小于等于2,只有二维或一维表可以被解析** + +## hdf5::loadHdf5Ex + +### 语法 + +* `hdf5::loadHdf5Ex(dbHandle,tableName,[partitionColumns],fileName,datasetName,[schema],[startRow],[rowNum])` + +### 参数 +* `dbHandle`和`tableName`: 如果我们要将输入数据文件保存在分布式数据库中，需要指定数据库句柄和表名。 +* `partitionColumns`: 字符串标量或向量，表示分区列。当分区数据库不是SEQ分区时，我们需要指定分区列。在组合分区中，partitionColumns是字符串向量。 +* `fileName`: hdf5文件名，类型为字符串标量。 +* `datasetName`: dataset名称，即表名，可通过ls或lsTable获得，类型为字符串标量。 +* `schema`: 包含列名和列的数据类型的表。如果我们想要改变由系统自动决定的列的数据类型，需要在`schema`表中修改数据类型，并且把它作为`loadHdf5Ex`函数的一个参数。 +* `startRow`: 读取hdf5数据集的起始行数，若不指定，默认从数据集起始位置读取。 +* `rowNum`: 读取hdf5数据集的行数，若不指定，默认读到数据集的结尾。 + +### 详情 +* 将hdf5文件中的数据集转换为DolphinDB数据库的分布式表，然后将表的元数据加载到内存中。 +* 读取的行数为hdf5文件中定义的行数，而不是读取结果中的DolphinDB表的行数。 +* 支持的数据类型,以及数据转化规则可见[数据类型](#支持的数据类型)章节。 + +### 例子 +* 磁盘上的SEQ分区表 +``` +db = database("seq_on_disk", SEQ, 16) +hdf5::loadHdf5Ex(db,`tb,,"/large_file.h5", "large_table") +``` + +* 内存中的SEQ分区表 +``` +db = database("", SEQ, 16) +hdf5::loadHdf5Ex(db,`tb,,"/large_file.h5", "large_table") +``` + +* 磁盘上的非SEQ分区表 +``` +db = database("non_seq_on_disk", RANGE, 0 500 1000) +hdf5::loadHdf5Ex(db,`tb,`col_4,"/smpl_numeric.h5","sint") +``` + +* 内存中的非SEQ分区表 +``` +db = database("", RANGE, 0 500 1000) +t0 = hdf5::loadHdf5Ex(db,`tb,`col_4,"/smpl_numeric.h5","sint") +``` + +## hdf5::hdf5DS + +### 语法 + +* `hdf5::hdf5DS(fileName,datasetName,[schema],[dsNum])` + +### 参数 +* `fileName`: hdf5文件名，类型为字符串标量。 +* `datasetName`: dataset名称，即表名，可通过ls或lsTable获得，类型为字符串标量。 +* `schema`: 包含列名和列的数据类型的表。如果我们想要改变由系统自动决定的列的数据类型，需要在`schema`表中修改数据类型，并且把它作为`hdf5DS`函数的一个参数。 +* `dsNum`: 需要生成的数据源数量。`hdf5DS`会将整个表均分为`dsNum`份作为结果。如果不指定，默认生成1个数据源。 + +### 详情 +* 根据输入的文件名和数据集名创建数据源列表。 + +### 例子 +``` +>ds = hdf5::hdf5DS(smpl_numeric.h5","sint") + +>size ds; +1 + +>ds[0]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 0, 3) > + +>ds = hdf5::hdf5DS(smpl_numeric.h5","sint",,3) + +>size ds; +3 + +>ds[0]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 0, 1) > + +>ds[1]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 1, 1) > + +>ds[2]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 2, 1) > +``` + +# 支持的数据类型 + +文件中的浮点和整数类型会被先转换为H5T_NATIVE_*类型(通过H5Tget_native_type) + +## integer +| type in hdf5 | corresponding c type | corresponding dolphindb type | +| ----------------- | :-------------------------- | :--------------------------- | +| H5T_NATIVE_CHAR | signed char / unsigned char | char/short | +| H5T_NATIVE_SCHAR | signed char | char | +| H5T_NATIVE_UCHAR | unsigned char | short | +| H5T_NATIVE_SHORT | short | short | +| H5T_NATIVE_USHORT | unsigned short | int | +| H5T_NATIVE_INT | int | int | +| H5T_NATIVE_UINT | unsigned int | long | +| H5T_NATIVE_LONG | long | int/long | +| H5T_NATIVE_ULONG | unsigned long | unsupported/long | +| H5T_NATIVE_LLONG | long long | long | +| H5T_NATIVE_ULLONG | unsigned long long | unsupported | + + +* dolphindb中数值类型都为有符号类型,为了防止溢出,所有无符号类型会被转化为```高一阶的有符号类型,64位无符号类型不予支持``` + +* H5T_NATIVE_CHAR 对应c中的char类型,而char是否有符号依赖与编译器及平台,若有符号,着在dolphindb中转化为char,否则为short + +* H5T_NATIVE_LONG以及H5T_NATIVE_ULONG 对应c中的long类型,long的大小依赖与编译器及平台,若long大小与int相同,着在转化过程中与int类型相同,若long大小与long long相同,着转化过程与long long相同 + +* 所有整数类型皆可以转化为dolphindb中的数值类型```(bool,char,short,int,long,float,double)```，若进行转化时会发生**溢出**，比如LONG->INT,则会返回一个int的最值 + +## float +| type in hdf5 | corresponding c type | corresponding dolphindb type | +| ----------------- | :------------------- | :--------------------------- | +| H5T_NATIVE_FLOAT | float | float | +| H5T_NATIVE_DOUBLE | double | double | + +注:IEEE754浮点数类型皆为有符号数 + +* 所有浮点数类型皆可以转化为dolphindb中的数值类型`(bool,char,short,int,long,float,double)`，若进行转化时会发生**溢出**，比如DOUBLE->FLOAT,则会返回一个float的最值 + +## time +| type in hdf5 | corresponding c type | corresponding dolphindb type | +| -------------- | :------------------- | :--------------------------- | +| H5T_UNIX_D32BE | 4 bytes integer | DT_TIMESTAMP | +| H5T_UNIX_D32LE | 4 bytes integer | DT_TIMESTAMP | +| H5T_UNIX_D64BE | 8 bytes integer | DT_TIMESTAMP | +| H5T_UNIX_D64LE | 8 bytes integer | DT_TIMESTAMP | + +* hdf5预定义的时间类型为**posix时间**,32位或者64位,hdf5的时间类型缺乏官方的定义，在此插件中，32位时间类型**代表距离1970年的秒数**，而64位则**精确到毫秒**。所有时间类型会被插件统一转化为一个64位整数,然后转化成dolphindb中的`timestamp`类型 + +* 以上类型皆可以转化为dolphindb中的时间相关类型`(date,month,time,minute,second,datetime,timestamp,nanotime,nanotimestamp)` + +## string +| type in hdf5 | corresponding c type | corresponding dolphindb type | +| ------------ | :------------------- | :--------------------------- | +| H5T_C_S1 | char* | DT_STRING | + +* H5T_C_S1,包括```固定长度(fixed-length)```字符串和```可变长度(variable-length)```字符串 + +* string类型可以转化为转化为dolphindb中的字符串相关类型```(string,symbol)``` + +## enum +| type in hdf5 | corresponding c type | corresponding dolphindb type | +| ------------ | :------------------- | :--------------------------- | +| ENUM | enum | DT_SYMBOL | + +* 枚举类型会被转化为dolphindb中的一个symbol变量，值得注意的是，每个字符串所对应的枚举值以及大小关系**并不会被保存**.即，定义了一个枚举类型 HDF5_ENUM{"a"=100，"b"=2000,"c"=30000},那么可能会被转化为 SYMBOL{"a"=3,"b"=1"c"=2} + +* enum类型可以转化为dolphindb中的字符串相关类型```(string,symbol)``` + +## compound and array +| type in hdf5 | corresponding c type | corresponding dolphindb type | +| ------------ | :------------------- | :--------------------------- | +| H5T_COMPOUND | struct | \ | +| H5T_ARRAY | array | \ | + +* 复合(compound)类型以及数组(array)类型,只要这些复杂类型中不包含不支持的类型,就可以被解析,而且支持嵌套 + +* 复杂类型的转化依赖与其内部子类型 + + + +# 表结构 + +## 简单类型 +对于简单类型,导入到dolphindb后的table与h5文件中的table会保持```相同``` + +### hdf5中的简单类型表 + +| | 1 | 2 | +| --- | :------ | :------ | +| 1 | int(10) | int(67) | +| 2 | int(20) | int(76) | + +### 导入进dolphindb后的简单类型表 + +| | col_1 | col_2 | +| --- | :---- | :---- | +| 1 | 10 | 67 | +| 2 | 20 | 76 | + +## 复杂类型 +对于复杂类型,dolphindb中的表的类型依赖与复杂类型的结构 + +### hdf5中的复合类型表 + +| | 1 | 2 | +| --- | :----------------------- | :----------------------- | +| 1 | struct{a:1 b:2 c:3.7} | struct{a:12 b:22 c:32.7} | +| 2 | struct{a:11 b:21 c:31.7} | struct{a:13 b:23 c:33.7} | + +### 导入进dolphindb后的复合类型表 +| | a | b | c | +| --- | :--- | :--- | :--- | +| 1 | 1 | 2 | 3.7 | +| 2 | 11 | 21 | 31.7 | +| 3 | 12 | 22 | 32.7 | +| 4 | 13 | 23 | 33.7 | + +### hdf5中的数组类型表 + +| | 1 | 2 | +| --- | :------------ | :-------------- | +| 1 | array(1,2,3) | array(4,5,6) | +| 2 | array(8,9,10) | array(15,16,17) | + +### 导入进dolphindb后的数组类型表 +| | array_1 | array_2 | array_3 | +| --- | :------ | :------ | :------ | +| 1 | 1 | 2 | 3 | +| 2 | 4 | 5 | 6 | +| 3 | 8 | 9 | 10 | +| 4 | 15 | 16 | 17 | + +对于嵌套的复杂类型,我们用一个```A```前缀代表这是个数组,```C```前缀代表这是个复合类型 + +### hdf5中的嵌套类型表 +| | 1 | 2 | +| --- | :-------------------------------------------------------- | :--------------------------------------------------------- | +| 1 | struct{a:array(1,2,3)
b:2
c:struct{d:"abc"}} | struct{a:array(7,8,9)
b:5
c:struct{d:"def"}} | +| 2 | struct{a:array(11,21,31)
b:0
c:struct{d:"opq"}} | struct{a:array(51,52,53)
b:24
c:struct{d:"hjk"}} | + + +### 导入进dolphindb后的嵌套类型表 + +| | Aa_1 | Aa_2 | Aa_3 | b | Cc_d | +| --- | :--- | :--- | :--- | :--- | :--- | +| 1 | 1 | 2 | 3 | 2 | abc | +| 2 | 7 | 8 | 9 | 5 | def | +| 3 | 11 | 21 | 31 | 0 | opq | +| 4 | 51 | 52 | 53 | 24 | hjk | + + +# 性能数据 + +## 环境 +* cpu: i7-7700 3.60GHZ +* ssd: read 460~500MB/S + +## 数据集导入性能 + +* 单一类型(int) + * 行数 1024 * 1024 * 16 + * 列数 64 + * 文件大小 4G + * 时间 8s +* 单一类型(usigned int) + * 行数 1024 * 1024 * 16 + * 列数 64 + * 文件大小 4G + * 时间 9s +* 单一类型(variable-length string) + * 行数 1024 * 1024 + * 列数 64 + * 文件大小 3.6G + * 时间 17s +* 复合类型 + * 子类型共9列,分别为 (str,str,double,int,long,float,int,short,char) + * 数据总量 1024 * 1024 * 62 + * 文件大小 3.9G + * 时间 10 s +* 数组复合类型 + * 子类型共72列,为(str,str,double,int,long,float,int,short,char) * 8 + * 数据总量 1024 * 128 * 62 + * 文件大小 3.9G + * 时间 15 s \ No newline at end of file diff --git a/hdf5/README_ENG.md b/hdf5/README_ENG.md new file mode 100644 index 00000000..e46912f0 --- /dev/null +++ b/hdf5/README_ENG.md @@ -0,0 +1,508 @@ +# DolphinDB HDF5 Plugin + +DolphinDB's HDF5 plugin imports HDF5 datasets into DolphinDB and supports data type conversions. + + +* [Build](#Build) + * [Build with cmake](#Build with cmake) + * [Build with makefile](#Build with makefile) +* [User-API](#User API) + * [hdf5::ls](#hdf5ls) + * [hdf5::lsTable](#hdf5lstable) + * [hdf5::extractHdf5Schema](#hdf5extracthdf5schema) + * [hdf5::loadHdf5](#hdf5loadhdf5) + * [hdf5::loadHdf5Ex](#hdf5loadhdf5ex) + * [hdf5::hdf5DS](#hdf5hdf5ds) +* [Data Types](#Data Types) + * [integer](#integer) + * [float](#float) + * [time](#time) + * [string](#string) + * [enum](#enum) + * [compound](#compound-and-array) + * [array](#compound-and-array) +* [Table Struct](#Table Struct) + * [Simple datatype table struct](#Simple datatype table struct) + * [complex datatype table struct](#complex datatype table struct) +* [Performance](#Performance) + +# Build + +## For Linux users + +### Build with cmake +**Note:** [cmake](https://cmake.org/) is a popular project build tool that can help you easily solve third-party dependencies. + +Install cmake +``` +sudo apt-get install cmake +``` + +Install HDF5 development kit +``` +sudo apt-get install libhdf5-dev +``` +Build the entire project + +``` +mkdir build +cd build +cmake ../path_to_hdf5_plugin/ +make +``` + +### Build with makefile + +Install HDF5 development kit + +``` +sudo apt-get install libhdf5-dev +``` +Execute "make" command + +``` +make +``` + +**Remember:** Before compiling, please make sure that `libDolphinDB.so` is in the gcc searchable path, you can use `LD_LIBRARY_PATH` to specify its path. + +The libPluginHdf5.so file will be generated after the compilation. + +## For Windows users + +The `enable threadsafe` configuration must be used for Windows. It should be set when compiling HDF5. + +You may use `hdf5.dll` file that have been prebuilt. + +You can also build HDF5 yourself. Download the source code from the [official website of HDF5](https://www.hdfgroup.org/solutions/hdf5/), and follow the instructions to build. If you are to build HDF5 with configure, the `--enable-threadsafe`, `--disable-cxx` and `--enable-shared` options must be used. If you are to build HDF5 with CMake, the `-DHDF5_ENABLE_THREADSAFE:BOOL=ON`, `-DHDF5_BUILD_CPP_LIB:BOOL=OFF`, `-DBUILD_SHARED_LIBS:BOOL=ON` must be added when compiling. + +# User-API + + +**Remember:** Use `loadPlugin("/path_to_PluginHdf5.txt/PluginHdf5.txt")` to import HDF5 plugin before using the API. + +## hdf5::ls + +### Syntax + +* `hdf5::ls(filename)` + +### Parameters + +* `filename`: a HDF5 file name of type `string`. + +### Details + +* List all the `HDF5 objects in a table including `dataset` and `group` and `(named type) namedType`. For the dataset, we will return the size (column size first followed by row size). For example, DataSet{(7,3)} represents 7 columns and 3 rows. + +### Example +``` +hdf5::ls("/smpl_numeric.h5") + +output: + objName objType + -------------------- + / Group + /double DataSet{(7,3)} + /float DataSet{(7,3)} + /schar DataSet{(7,3)} + /sint DataSet{(7,3)} + /slong DataSet{(7,3)} + /sshort DataSet{(7,3)} + /uchar DataSet{(7,3)} + /uint DataSet{(7,3)} + /ulong DataSet{(1,1)} + /ushort DataSet{(7,3)} + +hdf5::ls("/named_type.h5") + +output + objName objType + ---------------------- + / Group + /type_name NamedDataType + + +``` +## hdf5::lsTable + +### Syntax + +* `hdf5::lsTable(filename)` + +### Parameters + +* `filename`: a HDF5 file name of type `string`. + +### Details +* List all the table information in a table, that is, HDF5 `dataset` information, including table name, size, and type. + +### Example +``` +hdf5::lsTable("/smpl_numeric.h5") + +output: + tableName tableDims tableType + /double 7,3 H5T_NATIVE_DOUBLE + /float 7,3 H5T_NATIVE_FLOAT + /schar 7,3 H5T_NATIVE_SCHAR + /sint 7,3 H5T_NATIVE_INT + /slong 7,3 H5T_NATIVE_LLONG + /sshort 7,3 H5T_NATIVE_SHORT + /uchar 7,3 H5T_NATIVE_UCHAR + /uint 7,3 H5T_NATIVE_UINT + /ulong 1,1 H5T_NATIVE_ULLONG + /ushort 7,3 H5T_NATIVE_USHORT +``` + +## hdf5::extractHdf5Schema + +### Syntax + +* `hdf5::extractHdf5Schema(fileName,datasetName)` + +### Parameters +* `fileName`: a HDF5 file name of type `string`. +* `datasetName`: the dataset name, i.e., the table name of type `string`. It can be obtained by using `ls` or `lsTable`. + +### Details +* Generate the schema table for the input data file. The schema table has 2 columns: column names and their data types. + +### Example +``` +hdf5::extractHdf5Schema("/smpl_numeric.h5","sint") + +output: + name type + col_0 INT + col_1 INT + col_2 INT + col_3 INT + col_4 INT + col_5 INT + col_6 INT + + +hdf5::extractHdf5Schema("/compound.h5","com") + +output: + name type + fs STRING + vs STRING + d DOUBLE + t TIMESTAMP + l LONG + f FLOAT + i INT + s SHORT + c CHAR +``` + +## hdf5::loadHdf5 + +### Syntax + +* `hdf5::loadHdf5(fileName,datasetName,[schema],[startRow],[rowNum])` + +### Parameters +* `fileName`: a HDF5 file name of type `string`. +* `datasetName`: the dataset name, i.e., the table name of type `string`. It can be obtained by using `ls` or `lsTable`. +* `schema`: a table with column names and data types of columns. If there is a need to change the data type of a column that is automatically determined by the system, the schema table needs to be modified and used as an argument in `loadHdf5`. +* `startRow`: an integer indicating the start row to read. If not specified, the dataset will be read from the beginning. +* `rowNum`: an integer indicating the number of rows to read. If not specified, `loadHdf5` will read until the end of data. + +### Details +* Load an HDF5 file into a DolphinDB in-memory table. +* The number of rows to read is defined in the HDF5 file, rather than the number of rows in the output DolphinDB table. +* Supported data types, as well as data conversion rules are visible in the [Data Types] (#Data Types) section. + +### Example +``` +hdf5::loadHdf5("/smpl_numeric.h5","sint") + +output: + col_0 col_1 col_2 col_3 col_4 col_5 col_6 + (758) 8 (325,847) 87 687 45 90 + 61 0 28 77 546 789 45 + 799 5,444 325,847 678 90 54 0 + + +scm = table(`a`b`c`d`e`f`g as name, `CHAR`BOOL`SHORT`INT`LONG`DOUBLE`FLOAT as type) +hdf5::loadHdf5("../hdf5/h5file/smpl_numeric.h5","sint",scm,1,1) + +output: + a b c d e f g + '=' false 28 77 546 789 45 + +``` + +> **Note: the dimension of the dataset must be less than or equal to 2, only 2D or 1D tables can be parsed** + +## hdf5::loadHdf5Ex + +### Syntax + +* `hdf5::loadHdf5Ex(dbHandle,tableName,[partitionColumns],fileName,datasetName,[schema],[startRow],[rowNum])` + +### Parameters +* `dbHandle`和`tableName`: If the input data is to be saved into the distributed database, the database handle and table name should be specified. +* `partitionColumns`: a string scalar/vector indicating partitioning column(s). +* `fileName`: a HDF5 file name of type `string`. +* `datasetName`: the dataset name, i.e., the table name of type `string`. It can be obtained by using `ls` or `lsTable`. +* `schema`: a table with column names and data types of columns. If there is a need to change the data type of a column that is automatically determined by the system, the schema table needs to be modified and used as an argument in `loadHdf5Ex`. +* `startRow`: an integer indicating the start row to read. If not specified, the dataset will be read from the beginning. +* `rowNum`: an integer indicating the number of rows to read. If not specified, `loadHdf5` will read until the end of data. + +### Details +* Load an HDF5 file into a distributed table in a distributed database. The result is a table object with the loaded metadata. +* The number of rows to read is defined in the HDF5 file, rather than the number of rows in the output DolphinDB table. +* Supported data types, as well as data conversion rules are visible in the [Data Types] (#Data Types) section. + +### Example +* SEQ partitioned table on disk +``` +db = database("seq_on_disk", SEQ, 16) +hdf5::loadHdf5Ex(db,`tb,,"/large_file.h5", "large_table") +``` + +* SEQ in-memory partitioned table +``` +db = database("", SEQ, 16) +hdf5::loadHdf5Ex(db,`tb,,"/large_file.h5", "large_table") +``` + +* Non-SEQ partitioned table on disk +``` +db = database("non_seq_on_disk", RANGE, 0 500 1000) +hdf5::loadHdf5Ex(db,`tb,`col_4,"/smpl_numeric.h5","sint") +``` + +* Non-SEQ in-memory partitioned table +``` +db = database("", RANGE, 0 500 1000) +t0 = hdf5::loadHdf5Ex(db,`tb,`col_4,"/smpl_numeric.h5","sint") +``` + +## hdf5::hdf5DS + +### Syntax + +* `hdf5::hdf5DS(fileName,datasetName,[schema],[dsNum])` + +### Parameter +* `fileName`: a HDF5 file name of type `string`. +* `datasetName`: the dataset name, i.e., the table name of type `string`. It can be obtained by using `ls` or `lsTable`. +* `schema`: a table with column names and data types of columns. If there is a need to change the data type of a column that is automatically determined by the system, the schema table needs to be modified and used as an argument in `hdf5DS`. +* `dsNum`: the number of data sources to be generated. `hdf5DS` will divide the whole table equally into `dsNum` tables. If not specified, it will generate one data source. + +### Details +* Generate a tuple of data sources according to the input file name and dataset name. + +### Example +``` +>ds = hdf5::hdf5DS(smpl_numeric.h5","sint") + +>size ds; +1 + +>ds[0]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 0, 3) > + +>ds = hdf5::hdf5DS(smpl_numeric.h5","sint",,3) + +>size ds; +3 + +>ds[0]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 0, 1) > + +>ds[1]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 1, 1) > + +>ds[2]; +DataSource< loadHDF5("/smpl_numeric.h5", "sint", , 2, 1) > +``` + +# Data Types + +The floating point and integer types in the file are first converted to H5T_NATIVE_* type (via H5Tget_native_type) + +## integer +| type in HDF5 | corresponding c type | corresponding dolphindb type | +| ----------------- | :-------------------------- | :--------------------------- | +| H5T_NATIVE_CHAR | signed char / unsigned char | char/short | +| H5T_NATIVE_SCHAR | signed char | char | +| H5T_NATIVE_UCHAR | unsigned char | short | +| H5T_NATIVE_SHORT | short | short | +| H5T_NATIVE_USHORT | unsigned short | int | +| H5T_NATIVE_INT | int | int | +| H5T_NATIVE_UINT | unsigned int | long | +| H5T_NATIVE_LONG | long | int/long | +| H5T_NATIVE_ULONG | unsigned long | unsupported/long | +| H5T_NATIVE_LLONG | long long | long | +| H5T_NATIVE_ULLONG | unsigned long long | unsupported | + + +* The numeric types in dolphindb are all signed types. To prevent overflow, all unsigned types are converted to ```high-order signed types, 64-bit unsigned types are not supported ``` +* H5T_NATIVE_CHAR corresponds to the char type in c, and char has symbolic dependencies and compilers and platforms. If there is a symbol, it is converted to char in dolphindb, otherwise it is converted to a short in DolphinDB. +* H5T_NATIVE_LONG and H5T_NATIVE_ULONG correspond to the `long` type in c. The size of the `long` depends on the compiler and platform. If the size of the `long` is the same as the `int`, it is the same as the `int` in the conversion process. If the length is the same as that of the `long long`, it will be converted to `long long`. +* All integer types can be converted to the numeric type ```(bool,char,short,int,long,float,double)``` in dolphindb. **overflow** may occur. e.g. the maximum value of an int will be returned when converting LONG to INT. + +## float +| type in HDF5 | corresponding c type | corresponding dolphindb type | +| ----------------- | :------------------- | :--------------------------- | +| H5T_NATIVE_FLOAT | float | float | +| H5T_NATIVE_DOUBLE | double | double | + +Note: IEEE754 floating point types are all signed numbers. + +* All floating points types can be converted to the numeric type ```(bool,char,short,int,long,float,double)``` in dolphindb. **overflow** may occur. e.g. the maximum value of an `float` will be returned when converting DOUBLE to FLOAT. + +## time +| type in HDF5 | corresponding c type | corresponding dolphindb type | +| -------------- | :------------------- | :--------------------------- | +| H5T_UNIX_D32BE | 4 bytes integer | DT_TIMESTAMP | +| H5T_UNIX_D32LE | 4 bytes integer | DT_TIMESTAMP | +| H5T_UNIX_D64BE | 8 bytes integer | DT_TIMESTAMP | +| H5T_UNIX_D64LE | 8 bytes integer | DT_TIMESTAMP | + +* The predefined time type of HDF5 is **posix time**, 32-bit or 64-bit. The time type of HDF5 lacks the official definition. In this plug-in, the 32-bit time type ** represents the number of seconds from 1970**, The 64-bit ** is accurate to the millisecond**. All time types are uniformly converted by the plugin into a 64-bit integer and then converted to the `timestamp` type in dolphindb + +* All data types above can be converted to time-related types in dolphindb`(date,month,time,minute,second,datetime,timestamp,nanotime,nanotimestamp)` + +## string +| type in HDF5 | corresponding c type | corresponding dolphindb type | +| ------------ | :------------------- | :--------------------------- | +| H5T_C_S1 | char* | DT_STRING | + +* H5T_C_S1,包括```fixed-length```string和```variable-length```string + +* `string` type can be converted to a string-related type converted to dolphindb```(string,symbol)``` + +## enum +| type in HDF5 | corresponding c type | corresponding dolphindb type | +| ------------ | :------------------- | :--------------------------- | +| ENUM | enum | DT_SYMBOL | + +* Enum type will be converted to a symbol variable in dolphindb. It is worth noting that the enumeration value and size relationship ** of each string will not be saved**. For example, if an enum variable is defined as HDF5_ENUM{"a"=100,"b"=2000,"c"=30000}, it will be converted to SYMBOL{"a"=3,"b"=1"c"=2}. + + +## compound and array +| type in HDF5 | corresponding c type | corresponding dolphindb type | +| ------------ | :------------------- | :--------------------------- | +| H5T_COMPOUND | struct | \ | +| H5T_ARRAY | array | \ | + +* Compound types and array types, as long as these complex types do not contain unsupported types, nested parsing will be supported. + +* Complex type conversion depends on its internal sub data type. + + + +# Table Struct + +## Simple datatype table struct +For simple data types, the table imported into dolphindb from HDF5 file will keep ``` the same `` + +### Simple data types in HDF5 + +| | 1 | 2 | +| --- | :------ | :------ | +| 1 | int(10) | int(67) | +| 2 | int(20) | int(76) | + +### Converted data types in DolphinDB + +| | col_1 | col_2 | +| --- | :---- | :---- | +| 1 | 10 | 67 | +| 2 | 20 | 76 | + +## complex datatype table struct + +For complex data types, the data types in DolphinDB table depends on the structure of complex types + +### Table of compound type in HDF5 + +| | 1 | 2 | +| --- | :----------------------- | :----------------------- | +| 1 | struct{a:1 b:2 c:3.7} | struct{a:12 b:22 c:32.7} | +| 2 | struct{a:11 b:21 c:31.7} | struct{a:13 b:23 c:33.7} | + +### The corresponding table in DolphinDB after converting +| | a | b | c | +| --- | :--- | :--- | :--- | +| 1 | 1 | 2 | 3.7 | +| 2 | 11 | 21 | 31.7 | +| 3 | 12 | 22 | 32.7 | +| 4 | 13 | 23 | 33.7 | + +### Table of array in HDF5 + +| | 1 | 2 | +| --- | :------------ | :-------------- | +| 1 | array(1,2,3) | array(4,5,6) | +| 2 | array(8,9,10) | array(15,16,17) | + +### The correspoinding table in DolphinDB after converting + +| | array_1 | array_2 | array_3 | +| --- | :------ | :------ | :------ | +| 1 | 1 | 2 | 3 | +| 2 | 4 | 5 | 6 | +| 3 | 8 | 9 | 10 | +| 4 | 15 | 16 | 17 | + +For table of nested complex data types in HDF5, we use prefix ```A``` to represent this as an array and ```C``` as a compound type. + +### Table of nested compound types in HDF5 +| | 1 | 2 | +| --- | :-------------------------------------------------------- | :--------------------------------------------------------- | +| 1 | struct{a:array(1,2,3)
b:2
c:struct{d:"abc"}} | struct{a:array(7,8,9)
b:5
c:struct{d:"def"}} | +| 2 | struct{a:array(11,21,31)
b:0
c:struct{d:"opq"}} | struct{a:array(51,52,53)
b:24
c:struct{d:"hjk"}} | + + +### The corresponding table in DolphinDB after importing + +| | Aa_1 | Aa_2 | Aa_3 | b | Cc_d | +| --- | :--- | :--- | :--- | :--- | :--- | +| 1 | 1 | 2 | 3 | 2 | abc | +| 2 | 7 | 8 | 9 | 5 | def | +| 3 | 11 | 21 | 31 | 0 | opq | +| 4 | 51 | 52 | 53 | 24 | hjk | + + +# Performance + +## Environment + +* cpu: i7-7700 3.60GHZ +* ssd: read 460~500MB/S + +## Data import performance + +* int + * rows 1024 * 1024 * 16 + * cols 64 + * fileSize 4G + * time 8s +* usigned int + * rows 1024 * 1024 * 16 + * cols 64 + * fileSize 4G + * time 9s +* variable-length string + * rows 1024 * 1024 + * cols 64 + * fileSize 3.6G + * time 17s +* compound + * sub types include 9 data types: str,str,double,int,long,float,int,short, and char + * dataVolume: 1024 * 1024 * 62 + * fileSize 3.9G + * time 10 s +* compound array + * sub types include 72 data types: (str,str,double,int,long,float,int,short,char) * 8 + * dataVolume: 1024 * 128 * 62 + * fileSize 3.9G + * time 15s \ No newline at end of file diff --git a/hdf5/h5file/SDScompound.h5 b/hdf5/h5file/SDScompound.h5 new file mode 100644 index 0000000000000000000000000000000000000000..f2e70d1dbad862af7060296483b04e88fded7dbb GIT binary patch literal 2208 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@pf(0B95f~pPp8#brLg@}Dy@CnCU}OM61_lYJ zxFFPgbaf#?uC5F~l`!*RG*lZy01Jc#bp}j$lpY}=;Nj{R0P<=C)V~T)`UEtcLAeZ$ zMMa5~{%OG_MWx9lrA4X5PzhN2w16ojl4fLJ1*cyHr~!=7l+4Ho5rAYI1_l_-%&-Hj zUW}OmoYE8H^AdAY30o!rOxTbluE3BC)k7$)3o6(%K-4%em~b5DErY?IGD&z!4&U!5&i1G&s(h2?8hW zS-}z~93kqD+OvWA0#1`uJb~ztJv*51;snuuz#dXgG&so|F9f1J_MBjeMNSa&cG?2~ DC6Qt9 literal 0 HcmV?d00001 diff --git a/hdf5/h5file/array_mdatom.h5 b/hdf5/h5file/array_mdatom.h5 new file mode 100644 index 0000000000000000000000000000000000000000..4abbb532b1bdcf6e0f954df3c57ad829ece53bed GIT binary patch literal 16056 zcmeI3$8!@g6o=p1lu!Z*y<>WZ0HHSn34~q}da*+fozOxLy_XAjj?8f3$dM!e09XD7 zP7Dk%;d?8IIAOvV0vCAle5<$8TiN>??MQwr-K{8FQrNLyN6}PJAZ?^5Hp8L8r&se` zNjMMnS#Lx>zvJ`g31>v2ACW%3yx6zj+^=eVdAZ;j+7G$EOiJ1~tLd)`wU+kx0asBxQ7bK9&pgS}jcai;GhW@Ya-;(oRy^Kt&{}xFqsUZR0a-*QJmb z&|2J9lyyzWt4`h}&bs-Q)#t<8cPm7)xz&Hou3WvQERux(c9B%L*%rPrJg;@7EjU=Q zDPRixCknh@;BPeD^oFh6>0vsU4!KW;^#|(@)*nzHovVU~Jr_<7)4_DeeKM>+Sbz94 z{UMU3agT^Nho;DdJ%|9f)KZ?5k~n^$agi)&m_U6s_o2MFUdxJzaJaJ65AhX^gJ?=d zeFnsa)N9D&d&-T2JpGv5DiVi@zSo;!3YY?>fGJ=Km;(7L(E9Pe>3b0!=fn7~zd{G` zJjeav>hER|&eCJfGO~63cM_P)}i5w$P=F*_xW0% zYint!hu5G3dKyw{`=sywQ=n@t(xCs$-zT==rS7ebFZ?*a2r&lNe!laIy7B7&v-!oy zFE+n$8rjhlFa=BjQ@|8xodWre|2n2?(%6BfaNc~~rPeE8gPrhqA63YY?>!0#*2`tkoyuP>;Z>+U8UrW)~S?mXagJP*()h9y(L6fgx$ z0aL&f_+16`KYIAjuIvAF?_)b?54!2G1L($N-T2rU3ZV;hg(4`1ZqOZiKu_oey+KzE z_Jw}XA9V5HKo|srAq7KVC=3H#WH16o!YI(yj$=UAFOCC!NiYE>!Xzkx$)MXur-E*8 zoCecj2F!$6FdOE8PA=wwZll&sP-$2Qi(oO7K{+gerBDIOU^%e4Q&z$%SPg4HXCdog zJyb#!Y=Dih33N5Y7GPn7Y=iBv19n0NvQQ1XU^nc6y|54V!vUy)gK!8A!x1>e6?1%Q&!};0eSx2c{cGIrHMO;oL zCAUHg>bSovg@WYoCI3_mg@U6W=dMb7Ri6K|?!~iK%XuZ9k6bYKu@p*WG{pSBvOv4F z`IIL7Qf7@+`bE|mjM07bii=o}ZQS_7yx>Z_eg|jk`F41IG4wDc&pj`BzW%*H%0(|| z`n*39;e_n6zfWuZSoD+D{gUs5Y(jrwWM#MS%ari$gbDZSBMD?ueh$i-Ai=naXy+HM;!mcdc&*yygEY-v& zq**^%*?InCdy66`J&RI;g%zst8Z8hl5G@cb5G@cb5G@cb5H0XmTA;Jn-S71e21|Z) zNj(fP{FhAS)#lF12_KynzV*0yV`FA>#^0&Cw=Dh@FGn%({O!WRr~gWG|8vxlg%76* zqL{y6<>gZb_<1Xz9vry;i&mcO8xr%EsBD=(&LlE_KF?TqoRx*ITKf+ltBB(9*Q`7{ zoS?wdQQI_sb~r(Szd|L;{Mq3I1%81_7M>kWP~aD-WZ~K21O=YoN((OsSn&LL!NRk{ z)BY(mn8wczPx)ml&kj%dYgV2ep7PhNJUcw)SFAicJmpudyo6zo@%yu8<ao{o_1Igqdh8`yJ@yW*9(#pWkG(;w$6lb-Blc_c zi1k`MV!Kw4SgzG0c5C&B)mlAbvsRB-tkom-YW0Y>cIMmJ$=m6te0KIy;z|NY90%f% z(oL!MSr)GukJ!>*;D*I|i}e@VG%TP$fD!f&aC|>*aUqujL`rM?#O>0~nk^Zk3 z$mgc#X~JK5)&Ly`RS zwSuk>=Kv;GFUC0=M9%4q(~F(Wfmylo7uj!EgZvhzR|6#2j7t=0qNMm zIN^Ik2n>PCKC&f*hu{?Z-N6t5yY-^Nc53usT$;NJGSCqCIQwe@ApmywZe3l+J)0yA z!ww8@a{NAgMzltLU00v^k!sFPk-z2jqw-w*|ve#q?ex(nxH=z?Kj?^N;8F_^P$ zOoD;86S3EKQ7I9|M+7{Un06BBk#{Fcq*YuT=^|ETd?TeV_Qqu24aZAD?b z2|eWwQ5_bu+V=Tot$a50IG&#spjGFQqmIuXujw0quj7HOJfp>??|#Z>cN-tr3WKyI zZL78swIT*i7Xv-Z>tK$*$>u>NLCG!?p4Y|AnFem-UPk_%Ja?1%aG{ojT6aEv^cx<| z79ac&Axy{kJBN+|hOmUM2Swc|-#jS@fOkFIcxJr-==?4+*ZlqZch52ObuZ~8!^m>o z=u&0<9kt|Tm^bb}t}_90FCc1=Zz2G9Kfo8cA_gJ`A_gJ`A_gJ`j+cRM`MTr}a?azo zz4=%ShZ>%brTLeGiyag4=3}*w@e|L-mg>djVjZhesnm@dn`c7vCh^-bc%bvI5d;u_ zDLfpLv-<^o1>g~=kr~#ch+|4x!?~Q~UuWgl5xS}~xBb48KrZ=6)pYlymgg_QXq-O* Dl#%17 literal 0 HcmV?d00001 diff --git a/hdf5/h5file/named_type.h5 b/hdf5/h5file/named_type.h5 new file mode 100644 index 0000000000000000000000000000000000000000..a90dec1da0d642fd2f1b30c03562092a299a4638 GIT binary patch literal 1168 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@pf(a}T5f~pPp8#brLg@}Dy@CnCU}OM61_lYJ zxFFPgbaf#?uC5F~l`!*RG*lad2Q!4l0TURdM^p%SxH<-aJRAY_H7q@yfTlB8V3brA zq{inZ=B7dgVCm8Urhr5mJ^eBR(=RK~3=trPSPjW9!G8WOAhodcsQ^u{Bw9D(l>-3c CN-oI& literal 0 HcmV?d00001 diff --git a/hdf5/h5file/scalar.h5 b/hdf5/h5file/scalar.h5 new file mode 100644 index 0000000000000000000000000000000000000000..a6a1012fe8d679348d4595b9c5fcaa0d4a94d921 GIT binary patch literal 8294 zcmeH~%}&BV5XWa*FinjSJfWw31dd*f6ge0m;K5@tX@CT4g2XoMvcn!!?#Lq#pCDxdaj01y;1!UiD8Hj zqL%ubn*TWNVU%U$B7J_Zbs4p5VkO=I{jUi0vYR2_@T1uNbKr`%a0*FNx<+R%xt)dYs51AV^ObvE0VZs486r(Bb`x{K#|6p1_0_}Jvy z%0Ktp#XO(ak00JNY z0w4eae^21k;GWxg9dH3A8iZ=~ci|CAZ&%O7Y`{(`c@7#Oe zPJ64?%}UKp1rn(&4}UrxEaa+;#MV{rr|b)`PP-!)jMs^Fhk`In&A7?{Dik zcnF}AwSVt^EMb|ozrADs9xO{9%P25GvZ!N&2?XOzil|Vj6BLV3@N+psZ%_S+7FOLe z39NcqQA9^pj1aP`rru=HRgO@c&Ov_?@@dYpHvt^b%YQoASW{a?m~7>N=gx^egE}_1 zr^oh_cpp!S?YET8OHyL{x!C+JEw-P==G%;VZ14D49o5~T4Y-WH1PrI<$_Zk|U_GC*ukrPLiV4!ha zhB(BFzav3(To&6BhunoHi&Boj7Pz4}pa}{;1FTf4p+Lz|9q>SPLXLV>X#ywCJ#NTh zU(-WQ;aD|BuFsx0UNKI$qw-X4E2!&R&& ze_|#{JlQ!tjIZzsVT|z^EpZsXM|=ol!s=Hs*0Akw#TX|>a|3YcsWFeuvvdat3dLzO zkBbTthcWS(WX-CPGc+~gGYMRKc9b;nnuNGZLVP8W92X_Qb;{egf;dk?yeEk(T|-m=MOPV?*2kU*A`Orx>)w7Bom@BXdr9nJJn(9+Slvwim@esyx78S0tuZQ>gtQ=1<-KWtX8o#1&H$KU-- z;bOeUX@#_oZ8LnsUu!PDCCrik1~iTfqh40A;T)feMt)v4^zp7^$Wyve_Yc&#bR9{`xV+MpeSQ6lypyWC hmBd-^()Fu#!rp(1{UYPx8tUV@dAxf(-UA+Q^c(-G9|QmZ literal 0 HcmV?d00001 diff --git a/hdf5/h5file/timestamp.h5 b/hdf5/h5file/timestamp.h5 new file mode 100644 index 0000000000000000000000000000000000000000..34fb45a3ffb0c92d9e9508fe055f11d68570cd40 GIT binary patch literal 4840 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@pf)_#%5f~pPp8#brLg@}Dy@CnCU}OM61_lYJ zxFFPgbaf#?uC5F~l`!*RG*lad0SknM>N=S0C_UUmz{Axs0OZFAs7EcJ^a*G>gK`;4 zip@+wY!DrvnHQgvlb@Fk<{KM9#Ep$W;+c6RATBsR5q1kO@po5Q8{?IRwOKW=sGn1A-e63Q7urbwk|S zW-J@Uz)*l9PcKfepT7$uu#`}MS||W1IdEDJ2`vT&1#X;5z!FeTGbq3U5QlPD{;y!g zp3L35c=_R2z~lB zgx>rfLeKsTp_{)$=_cr_5O0phhl cybg%h1MvnR-U!5-fOs-#KaU=2rOV_%R^bH7<8e=2!xkGErbH@#HA)K-0F&@ zOFsh9#PBg#y7U|1`OnNLQ+h8g3yk3Z5bnIrnVJ52@9mv4r~NQKdi#9al{N#(W=*r{ zuvtCqdD&WWHCop8X`hLF`rfBsn_MO`_%o*4*LV8<59&Rd92+yDsr_r}%}j5z%k^pn z)BlRV_*n4)Uhrf8st0|#B@;=5m2H>! z<0W5>K7KAmPmOs|)qdQChxaE&qu?&FtrvF6z_G929ZFH~)&9krSOKP}>-9f9nWDgI z5+H$w3E=j*?JvG|$>Z|VG9C)T@>wb4KBw(xB#--$mh~Ezs)~g6&<*V|c4!aXa6Zrt z?OP7mw@UfW$i!Xrp^q&#FGbiv`uqEBsvi@2pC0NRw6_w~Q@giT+szYMwOd?t4VVK& z?QUFuZp)YpMEiduf-)FEw0=|sWo4z~-bAGc7(=xGk&zKwb`NO#^~1x%wjS>pMCVy7 zy0RXBUC2 zS=f(DI@2JE4q3pvHGOPedK+ESr*+X8-}3AJf*EKx1ADT_FO{7({KAJxPIO~Ym_1A> zXzPrxR_520HC>z1w()AdG*#O7Jp}4i*(6`}ir1*)?zL_hHhTXyt9w-c2^|yr$76i5q~;`1W14cNPq-LfCNZ@1W14cNZ?;2 zpo_ULpCQ)@vPzbJW3Vs2+k>1D4}mdHY|fZ-l4?C{eVpNDBmH2HKdB%17LL*nVgi!- zktP0-`oX$4sUNJrbv|kF5#Rt>Us;)7jj>XXZ3u$;NfO%oppQ91654H`g)?j|A2lW2mSYr?myJpyZr&(f51E;!TyKc zU}$vzA;I46pQP_U=r=swxZHnm9C#!F5+DH*AOR8}0TLhq5+DH*I2{D){eGUV_tVmh z{2m_s2uVJE5K8y+Py6rk6`)p;(9eJGWAk7C{k+OVm>~fYAOR8}fm2KX{`M{ZNsRr2 zunb>*SjOi({O_udXM(i-1NGxCLqt!XAJl(;BcIRr$oD^Vzd-fNa|`7K(bw-OX#LDw m+15*wl}{Hn$CLy}fCNZ@1W14cNPq-LfCNZ@1W4ex3H$*&;Rx*j literal 0 HcmV?d00001 diff --git a/hdf5/src/hdf5_plugin.cpp b/hdf5/src/hdf5_plugin.cpp index 2dd4ee3d..019ffd97 100644 --- a/hdf5/src/hdf5_plugin.cpp +++ b/hdf5/src/hdf5_plugin.cpp @@ -905,7 +905,7 @@ ConstantSP loadFromH5ToDatabase(Heap *heap, vector &arguments) if (diskSeqMode) { string id = db->getDomain()->getPartition(arguments[6]->getInt())->getPath(); string directory = db->getDatabaseDir() + "/" + id; - if (!DBFileIO::saveBasicTable(heap->currentSession(), directory, loadedTable.get(), tableName, db->getSymbolBaseManager(), NULL, true, 1, false)) + if (!DBFileIO::saveBasicTable(heap->currentSession(), directory, loadedTable.get(), tableName, NULL, true, 1, false)) throw RuntimeException("Failed to save the table to directory " + directory); return new Long(loadedTable->rows()); } @@ -945,7 +945,6 @@ vector generateH5Tasks(Heap* heap, const hid_t set, estimatedRows = std::min(rowNum, estimatedRows); int partitions = 0; - bool diskSeqMode = false; DomainSP domain = db->getDomain(); if (domain->getPartitionType() == SEQ) { @@ -954,7 +953,6 @@ vector generateH5Tasks(Heap* heap, const hid_t set, throw IOException("The database must have at least two partitions."); if ((estimatedRows / partitions) < 65536) throw IOException("The number of rows per partition is too small (<65,536) and the hdf5 file can't be partitioned."); - diskSeqMode = !db->getDatabaseDir().empty(); } if (partitions == 0) { long long fileSize = getDatasetSize(set); @@ -972,14 +970,6 @@ vector generateH5Tasks(Heap* heap, const hid_t set, int columns=colNames.size(); vector cols(columns); - bool containSymbolType = false; - for(int i=0; igetSymbolBaseManager()->findAndInsert(new SymbolBase()); - vector tasks; size_t rowIncrement = std::ceil(static_cast(estimatedRows) / partitions); @@ -1002,7 +992,7 @@ vector generateH5Tasks(Heap* heap, const hid_t set, } TableSP generateInMemoryParitionedTable(Heap *heap, const SystemHandleSP &db, - const TableSP &tables, const ConstantSP &partitionNames) + const ConstantSP &tables, const ConstantSP &partitionNames) { FunctionDefSP createPartitionedTable = heap->currentSession()->getFunctionDef("createPartitionedTable"); ConstantSP emptyString = new String(""); @@ -1010,7 +1000,7 @@ TableSP generateInMemoryParitionedTable(Heap *heap, const SystemHandleSP &db, return createPartitionedTable->call(heap, args); } -ConstantSP loadHDF5Ex(Heap* heap, const SystemHandleSP &db, const string &tableName, const ConstantSP &partitionColumnNames, +ConstantSP loadHDF5Ex(Heap *heap, const SystemHandleSP &db, const string &tableName, const ConstantSP &partitionColumns, const string &filename, const string &datasetName, const TableSP &schema, const size_t startRow, const size_t rowNum) { @@ -1022,7 +1012,7 @@ ConstantSP loadHDF5Ex(Heap* heap, const SystemHandleSP &db, const string &tableN StaticStageExecutor executor(true, false, false); executor.execute(heap, tasks); - for (int i = 0; i < partitions; ++i) { + for (int i = 0; i < partitions; i++) { const string &errMsg = tasks[i]->getErrorMessage(); if (!errMsg.empty()) throw RuntimeException(errMsg); @@ -1032,6 +1022,7 @@ ConstantSP loadHDF5Ex(Heap* heap, const SystemHandleSP &db, const string &tableN DomainSP domain = db->getDomain(); bool seqDomain = domain->getPartitionType() == SEQ; bool inMemory = db->getDatabaseDir().empty(); + ConstantSP tableName_ = new String(tableName); if (seqDomain) { if (inMemory) { @@ -1045,13 +1036,13 @@ ConstantSP loadHDF5Ex(Heap* heap, const SystemHandleSP &db, const string &tableN vector partitionColumnIndices(1, -1); vector baseIds; int baseId = -1; - db->getSymbolBaseManager()->getSymbolBases(baseIds); - if (!baseIds.empty()) { - string errMsg; - baseId = baseIds.back(); - if (!db->getSymbolBaseManager()->saveSymbolBase(db->getSymbolBaseManager()->findAndLoad(baseId), errMsg)) - throw IOException("Failed to save symbol base: " + errMsg); - } + // db->getSymbolBaseManager()->getSymbolBases(baseIds); + // if (!baseIds.empty()) { + // string errMsg; + // baseId = baseIds.back(); + // if (!db->getSymbolBaseManager()->saveSymbolBase(db->getSymbolBaseManager()->findAndLoad(baseId), errMsg)) + // throw IOException("Failed to save symbol base: " + errMsg); + // } string tableFile = db->getDatabaseDir() + "/" + tableName + ".tbl"; vector cols; @@ -1068,76 +1059,40 @@ ConstantSP loadHDF5Ex(Heap* heap, const SystemHandleSP &db, const string &tableN if (!DBFileIO::saveDatabase(db.get())) throw IOException("Failed to save database " + db->getDatabaseDir()); db->getDomain()->addTable(tableName, owner, cols, partitionColumnIndices); - - return DBFileIO::loadTable(heap->currentSession(), db.get(), tableName, nullSP, false, false); + vector loadTableArgs = {db, tableName_}; + return heap->currentSession()->getFunctionDef("loadTable")->call(heap, loadTableArgs); + // return DBFileIO::loadTable(heap->currentSession(), db.get(), tableName, nullSP, SEGTBL, false); } } else { - int columnSize = partitionColumnNames->size(); - if (domain->getPartitionDimensions() != columnSize) - throw RuntimeException("The number of partition columns doesn't match the database partition scheme."); - - TableSP emptyTable = DBFileIO::createEmptyTableFromSchema(convertedSchema); - vector cols; - DBFileIO::collectColumnDesc(emptyTable.get(), db->getSymbolBaseManager(), cols); - vector partitionColumnIndices(columnSize); - for (int i = 0; i < columnSize; ++i) { - partitionColumnIndices[i] = emptyTable->getColumnIndex(partitionColumnNames->getString(i)); - if (partitionColumnIndices[i] < 0) - throw RuntimeException("Invalid partition column " + partitionColumnNames->getString(i)); - domain->switchWorkingDimension(i); - if (!DBFileIO::checkPartitionColumnCompatibility(domain->getPartitionColumnType(), cols[partitionColumnIndices[i]].getType())) - throw RuntimeException("The data type of column [" + partitionColumnNames->getString(i) + "] doesn't match the database partition scheme."); - } - - if (inMemory) { - SmartPointer builder = new SegmentedInMemoryTableBuiler(heap, db, partitionColumnIndices, partitionColumnNames, convertedSchema); - for (int i = 0; i < partitions; i++) { - TableSP tmpTable = tasks[i]->getResultObject(); - builder->insert(tmpTable); - } - return builder->getSegmentedTable(); + string dbPath = db->getDatabaseDir(); + vector existsTableArgs = {new String(dbPath), tableName_}; + bool existsTable = heap->currentSession()->getFunctionDef("existsTable")->call(heap, existsTableArgs)->getBool(); + ConstantSP result; + + if (existsTable) { + vector loadTableArgs = {db, tableName_}; + result = heap->currentSession()->getFunctionDef("loadTable")->call(heap, loadTableArgs); } else { - bool fail = false; - IoTransactionSP trans = new IoTransaction(db->getDatabaseDir()); - string errMsg; - - if (!Util::exists(db->getDatabaseDir() + "/" + tableName + ".tbl")) { - try { - if (DBFileIO::saveTableHeader(owner, cols, partitionColumnIndices, 0, db->getDatabaseDir() + "/" + tableName + ".tbl", trans.get())) - db->getDomain()->addTable(tableName, owner, cols, partitionColumnIndices); - else - fail = true; - } - catch (exception &ex) { - errMsg = ex.what(); - fail = true; - } - if (fail) { - trans->rollback(); - if (errMsg.empty()) - errMsg = "Unknown error occurred when saving data to database [" + db->getDatabaseDir() + "]"; - throw RuntimeException(errMsg); - } - } - for (int i = 0; i < partitions; i++) { - TableSP tmpTable = tasks[i]->getResultObject(); - if (!DBFileIO::savePartitionedTable(heap->currentSession(), domain, tmpTable, tableName, trans.get(), 1)) - fail = true; - } + TableSP schema = extractHDF5Schema(filename, datasetName); + ConstantSP dummyTable = DBFileIO::createEmptyTableFromSchema(schema); + vector createTableArgs = {db, dummyTable, tableName_, partitionColumns}; + result = heap->currentSession()->getFunctionDef("createPartitionedTable")->call(heap, createTableArgs); + } - if (fail) { - trans->rollback(); - if (errMsg.empty()) - errMsg = "Unknown error occurred when saving data to database [" + db->getDatabaseDir() + "]"; - throw RuntimeException(errMsg); - } - else - return DBFileIO::loadTable(heap->currentSession(), db.get(), tableName, nullSP, false, false); + FunctionDefSP append = heap->currentSession()->getFunctionDef("append!"); + for (int i = 0; i < partitions; ++i) { + ConstantSP table = tasks[i]->getResultObject(); + vector appendArgs = {result, table}; + append->call(heap, appendArgs); } + if (!inMemory) { + vector loadTableArgs = {db, tableName_}; + result = heap->currentSession()->getFunctionDef("loadTable")->call(heap, loadTableArgs); + } + return result; } - return nullSP; } ConstantSP HDF5DS(const ConstantSP &filename, const ConstantSP &datasetName, diff --git a/hdf5/src/hdf5_plugin.h b/hdf5/src/hdf5_plugin.h index 9638340f..3416d03b 100644 --- a/hdf5/src/hdf5_plugin.h +++ b/hdf5/src/hdf5_plugin.h @@ -7,9 +7,7 @@ #include #include -#include #include -#include #include #include #include diff --git a/hdf5/test/test_basic_simple_type.txt b/hdf5/test/test_basic_simple_type.txt new file mode 100644 index 00000000..a478f9f7 --- /dev/null +++ b/hdf5/test/test_basic_simple_type.txt @@ -0,0 +1,550 @@ + +//set your path here +loadPlugin("/home/swang/dolphin_workspace/DolphinDBPlugin/hdf5/build/PluginHdf5.txt"); +H5_FILE_PATH="/home/swang/dolphin_workspace/DolphinDBPlugin/hdf5/h5file" + +@testing:case="test_fixed_string" +t=hdf5::read(H5_FILE_PATH+"/float.h5","fs") +assert 1,(size t.keys())==4 +assert 2,(size t) == 1 +assert 3,t[0,0]==`asd +assert 4,t[0,1]==`fds +assert 5,t[0,2]==`sdf +assert 6,t[0,3]==`eee +assert 7,(typestr t.values()[0])== "STRING VECTOR" +assert 8,(typestr t.values()[1])== "STRING VECTOR" +assert 9,(typestr t.values()[2])== "STRING VECTOR" +assert 10,(typestr t.values()[3])== "STRING VECTOR" + +@testing:case="test_fixed_string_to_symbol" +t=table(`a as a,`b as b,`c as c,`d as d) +hdf5::read(H5_FILE_PATH+"/float.h5","fs",t) +assert 1,(size t.keys())==4 +assert 2,(size t) == 2 +assert 3,t.values()[0]==`a`asd +assert 4,t.values()[1]==`b`fds +assert 5,t.values()[2]==`c`sdf +assert 6,t.values()[3]==`d`eee +assert 7,(typestr t.values()[0])== "STRING VECTOR" +assert 8,(typestr t.values()[1])== "STRING VECTOR" +assert 9,(typestr t.values()[2])== "STRING VECTOR" +assert 10,(typestr t.values()[3])== "STRING VECTOR" + +@testing:case="test_fixed_string_with_symbol_describe" +t=table(rand(`a`b ,1) as a,`b as b,`c as c,`d as d) +hdf5::read(H5_FILE_PATH+"/float.h5","fs",t) +assert 1,(size t.keys())==4 +assert 2,(size t) == 2 +assert 3,t.values()[0] ==`a`asd || t.values()[0]==`b`asd +assert 4,t.values()[1]==`b`fds +assert 5,t.values()[2]==`c`sdf +assert 6,t.values()[3]==`d`eee +assert 7,(typestr t.values()[0])== "FAST SYMBOL VECTOR"//string -> symbol +assert 8,(typestr t.values()[1])== "STRING VECTOR" +assert 9,(typestr t.values()[2])== "STRING VECTOR" +assert 10,(typestr t.values()[3])== "STRING VECTOR" + +@testing:case="test_fixed_string_with_unsupported_describe" ,exception=1 +t=table(7 as a,`b as b,`c as c,`d as d) +hdf5::read(H5_FILE_PATH+"/float.h5","fs",t) + +@testing:case="test_fixed_string_with_null" +t=table(rand(`a`b ,1) as a,`b as b,`c as c,`d as d,`e as ee,`f as f,`g as g,`h as h, `t as t ,`u as u) +hdf5::read(H5_FILE_PATH+"/float.h5","float_str",t) +assert 1,(size t.keys())==10 +assert 2,(size t) == 11 +assert 3,t.values()[0,0] in `a`b +assert 4,t.values()[0,1:(t.columns())] ==`44444444444444444444```````` +assert 5,t.values()[1]==`b`5````````` +assert 6,t.values()[2]==`c`5````````` +assert 7,t.values()[3]==`d`6````````` +assert 8,t.values()[4]== `e`7````````` +assert 9,t.values()[5]==`f`````````` +assert 10,(typestr t.values()[0])== "FAST SYMBOL VECTOR"//string -> symbol +assert 11,(typestr t.values()[1])== "STRING VECTOR" +assert 12,(typestr t.values()[2])== "STRING VECTOR" +assert 13,(typestr t.values()[3])== "STRING VECTOR" + +//test enum + +@testing:case="test_enum" +t=hdf5::read(H5_FILE_PATH+"/smpl_enum.h5","enum3") +assert 1,(size t.keys())==5 +assert 2,(size t) == 5 +assert 3,t.values()[0]==`R`R`R`R`R +assert 4,t.values()[1]==`R`R`R`G`R +assert 5,t.values()[2]==`B`R`B`R`R +assert 6,t.values()[3]==`R`G`R`R`R +assert 7,t.values()[4]==`R`R`R`R`R +assert 8,(typestr t.values()[0])== "FAST SYMBOL VECTOR" +assert 9,(typestr t.values()[1])== "FAST SYMBOL VECTOR" +assert 10,(typestr t.values()[2])== "FAST SYMBOL VECTOR" +assert 11,(typestr t.values()[3])== "FAST SYMBOL VECTOR" +assert 12,(typestr t.values()[4])== "FAST SYMBOL VECTOR" + +@testing:case="test_enum_to_string" +t=table(symbol([`a]) as a,`b as b,`c as c,`d as d,`e as ee) +t=hdf5::read(H5_FILE_PATH+"/smpl_enum.h5","enum3",t) +assert 1,(size t.keys())==5 +assert 2,(size t) == 6 +assert 3,t.values()[0]==`a`R`R`R`R`R +assert 4,t.values()[1]==`b`R`R`R`G`R +assert 5,t.values()[2]==`c`B`R`B`R`R +assert 6,t.values()[3]==`d`R`G`R`R`R +assert 7,t.values()[4]==`e`R`R`R`R`R +assert 8,(typestr t.values()[0])== "FAST SYMBOL VECTOR" +assert 9,(typestr t.values()[1])== "STRING VECTOR" +assert 10,(typestr t.values()[2])== "STRING VECTOR" +assert 11,(typestr t.values()[3])== "STRING VECTOR" +assert 12,(typestr t.values()[4])== "STRING VECTOR" + +@testing:case="test_enum_with_null_to_string" +t=table(symbol([`a]) as a as a,`b as b,`c as c) +t=hdf5::read(H5_FILE_PATH+"/smpl_enum.h5","enum4",t) +assert 1,(size t.keys())==3 +assert 2,(size t) == 4 +assert 3,t.values()[0]==`a`a``c +assert 4,t.values()[1]==`b``b` +assert 5,t.values()[2]==`c`a``c +assert 6,(typestr t.values()[0])== "FAST SYMBOL VECTOR" +assert 7,(typestr t.values()[1])== "STRING VECTOR" +assert 8,(typestr t.values()[2])== "STRING VECTOR" + +//test vlen_str + +@testing:case="test_vlen_str" +t=hdf5::read(H5_FILE_PATH+"/vlen_str.h5","vstr") +assert 1,(size t.keys())==4 +assert 2,(size t) == 3 +assert 3,t.values()[0]==["AA","1","~"] +assert 4,t.values()[1]==`BBB``gg +assert 5,t.values()[2]==["CCC","333","*&^"] +assert 6,t.values()[3]==`DDDD`9080`wwww +assert 7,(typestr t.values()[0])== "STRING VECTOR" +assert 8,(typestr t.values()[1])== "STRING VECTOR" +assert 9,(typestr t.values()[2])== "STRING VECTOR" +assert 10,(typestr t.values()[2])== "STRING VECTOR" + +@testing:case="test_vlen_str_to_symbol" +t=table(symbol([`a]) as a,`b as b,`c as c,`d as d) +t=hdf5::read(H5_FILE_PATH+"/vlen_str.h5","vstr",t) +assert 1,(size t.keys())==4 +assert 2,(size t) == 4 +assert 3,t.values()[0]==["a","AA","1","~"] +assert 4,t.values()[1]==`b`BBB``gg +assert 5,t.values()[2]==["c","CCC","333","*&^"] +assert 6,t.values()[3]==`d`DDDD`9080`wwww +assert 7,(typestr t.values()[0])== "FAST SYMBOL VECTOR" +assert 8,(typestr t.values()[1])== "STRING VECTOR" +assert 9,(typestr t.values()[2])== "STRING VECTOR" +assert 10,(typestr t.values()[2])== "STRING VECTOR" + + +//testing char +@testing:case="test_char" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","schar") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[45,-120,74] +assert 4,t.values()[1]==[77,-56,0] +assert 5,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 6,(typestr t.values()[1])== "FAST CHAR VECTOR" + +@testing:case="test_char_to_numericl" +t=table(1c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","schar",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[1,45,-120,74] +assert 4,t.values()[1]==[false,true,true,false] +assert 5,t.values()[2]==[5,23,98,-93] +assert 6,t.values()[3]==[6,54,59,-45] +assert 7,t.values()[4]==[7,67,-77,11] +assert 8,t.values()[5]==[8.6f,89,53,-67] +assert 9,t.values()[6]==[9.65,-98,26,0] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_uchar" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","uchar") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[0,57,237] +assert 4,t.values()[1]==[255,42,0] +assert 5,(typestr t.values()[0])== "FAST SHORT VECTOR" +assert 6,(typestr t.values()[1])== "FAST SHORT VECTOR" + +@testing:case="test_uchar_to_numericl" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","uchar",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,0,57,127] +assert 4,t.values()[1]==[false,true,true,false] +assert 5,t.values()[2]==[5,128,96,86] +assert 6,t.values()[3]==[6,250,32,54] +assert 7,t.values()[4]==[7,26,150,28] +assert 8,t.values()[5]==[8.6f,15,85,6] +assert 9,t.values()[6]==[9.65,89,96,7] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_sshort" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","sshort") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[-9834,-987,967] +assert 4,t.values()[1]==[2343,0,54] +assert 5,(typestr t.values()[0])== "FAST SHORT VECTOR" +assert 6,(typestr t.values()[1])== "FAST SHORT VECTOR" + +@testing:case="test_sshort_to_numericl" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","sshort",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,-127,-127,127] +assert 4,t.values()[1]==[false,true,false,true] +assert 5,t.values()[2]==[5,123,567,789] +assert 6,t.values()[3]==[6,87,77,678] +assert 7,t.values()[4]==[7,687,546,90] +assert 8,t.values()[5]==[8.6f,45,789,54] +assert 9,t.values()[6]==[9.65,90,45,0] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_ushort" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","ushort") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[9834,987,63357] +assert 4,t.values()[1]==[2343,0,65234] +assert 5,(typestr t.values()[0])== "FAST INT VECTOR" +assert 6,(typestr t.values()[1])== "FAST INT VECTOR" + +@testing:case="test_ushort_to_numericl" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","ushort",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,127,127,127] +assert 4,t.values()[1]==[false,true,false,true] +assert 5,t.values()[2]==[5,123,567,32767] +assert 6,t.values()[3]==[6,87,77,678] +assert 7,t.values()[4]==[7,687,546,90] +assert 8,t.values()[5]==[8.6f,45,789,54] +assert 9,t.values()[6]==[9.65,90,45,0] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_sint" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","sint") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[-758,61,799] +assert 4,t.values()[1]==[8,0,5444] +assert 5,(typestr t.values()[0])== "FAST INT VECTOR" +assert 6,(typestr t.values()[1])== "FAST INT VECTOR" + +@testing:case="test_sint_to_numericl" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","sint",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,-127,61,127] +assert 4,t.values()[1]==[false,true,false,true] +assert 5,t.values()[2]==[5,-32767,28,32767] +assert 6,t.values()[3]==[6,87,77,678] +assert 7,t.values()[4]==[7,687,546,90] +assert 8,t.values()[5]==[8.6f,45,789,54] +assert 9,t.values()[6]==[9.65,90,45,0] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_uint" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","uint") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[758,61,799] +assert 4,t.values()[2]==[325847,28,325847] +assert 5,(typestr t.values()[0])== "FAST LONG VECTOR" +assert 6,(typestr t.values()[1])== "FAST LONG VECTOR" + +@testing:case="test_uint_to_numericl" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","uint",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,127,61,127] +assert 4,t.values()[1]==[false,true,false,true] +assert 5,t.values()[2]==[5,32767,28,32767] +assert 6,t.values()[3]==[6,87,77,678] +assert 7,t.values()[4]==[7,687,546,90] +assert 8,t.values()[5]==[8.6f,45,789,54] +assert 9,t.values()[6]==[9.65,90,45,0] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_slong" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","slong") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[-758,61,799] +assert 4,t.values()[1]==[2345678910,0,5444] +assert 5,(typestr t.values()[0])== "FAST LONG VECTOR" +assert 6,(typestr t.values()[1])== "FAST LONG VECTOR" + +@testing:case="test_slong_to_numericl" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","slong",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,-127,61,127] +assert 4,t.values()[1]==[false,true,false,true] +assert 5,t.values()[2]==[5,-32767,28,32767] +assert 6,t.values()[3]==[6,-2147483647,23,2147483647] +assert 7,t.values()[4]==[7,687,546,90] +assert 8,t.values()[5]==[8.6f,45,789,54] +assert 9,t.values()[6]==[9.65,90,45,0] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_ulong",exception = 1 +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","ulong") + +@testing:case="test_float" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","float") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[-758,61,758] +assert 4,t.values()[1]==[1.0,0,1.0] +assert 5,(typestr t.values()[0])== "FAST FLOAT VECTOR" +assert 6,(typestr t.values()[1])== "FAST FLOAT VECTOR" + +@testing:case="test_float_to_numeric" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","float",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,-127,61,127] +assert 4,t.values()[1]==[false,true,false,true] +assert 5,t.values()[2]==[5,-32767,28,32767] +assert 6,t.values()[3]==[6,-2147483647,23,2147483647] +assert 7,t.values()[4]==[7,9223372036854775807l,159876,-9223372036854775807l] +assert 8,t.values()[5]==[8.6f,9.2854f,72.85f,5.896f] +assert 9,t.values()[6]==[9.65,90,45,0] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + +@testing:case="test_double" +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","double") +assert 1,(size t.keys())==7 +assert 2,(size t) == 3 +assert 3,t.values()[0]==[-758,61,758] +assert 4,t.values()[1]==[1.0,0,1.0] +assert 5,(typestr t.values()[0])== "FAST DOUBLE VECTOR" +assert 6,(typestr t.values()[1])== "FAST DOUBLE VECTOR" + +@testing:case="test_double_to_numericl" +t=table(-71c as a,false as b,5h as c,6 as d,7l as ee,8.6f as f,9.65 as g); +t=hdf5::read(H5_FILE_PATH+"/smpl_numeric.h5","double",t) +assert 1,(size t.keys())==7 +assert 2,(size t) == 4 +assert 3,t.values()[0]==[-71,-127,61,127] +assert 4,t.values()[1]==[false,true,false,true] +assert 5,t.values()[2]==[5,-32767,28,32767] +assert 6,t.values()[3]==[6,-2147483647,23,2147483647] +assert 7,t.values()[4]==[7,9223372036854775807l,159876,-9223372036854775807l] +assert 8,t.values()[5]==[8.6f,340282346638528860000000000000000000000f,72.85f,-340282326638528860000000000000000000000f] +assert 9,t.values()[6]==[9.65,8.8888e92,45.8,-8.8888e92] +assert 10,(typestr t.values()[0])== "FAST CHAR VECTOR" +assert 11,(typestr t.values()[1])== "FAST BOOL VECTOR" +assert 12,(typestr t.values()[2])== "FAST SHORT VECTOR" +assert 13,(typestr t.values()[3])== "FAST INT VECTOR" +assert 14,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 15,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 16,(typestr t.values()[6])== "FAST DOUBLE VECTOR" + + +@testing:case="test_timestamp64" +t=hdf5::read(H5_FILE_PATH+"/timestamp.h5","ts64") +t2=hdf5::read(H5_FILE_PATH+"/timestamp.h5","ts64_in_llong") +assert 1,(size t.keys())==9 +assert 2,(size t2.keys())==9 +assert 3,(size t)==3 +assert 4,(size t2)==3 +assert 5,cast(t.values()[0],LONG)==t2.values()[0] +assert 6,cast(t.values()[1],LONG)==t2.values()[1] +assert 7,cast(t.values()[2],LONG)==t2.values()[2] +assert 8,cast(t.values()[3],LONG)==t2.values()[3] +assert 9,cast(t.values()[4],LONG)==t2.values()[4] +assert 10,(typestr t.values()[0])== "FAST TIMESTAMP VECTOR" +assert 11,(typestr t.values()[1])== "FAST TIMESTAMP VECTOR" +assert 12,(typestr t.values()[2])== "FAST TIMESTAMP VECTOR" +assert 13,(typestr t.values()[3])== "FAST TIMESTAMP VECTOR" +assert 14,(typestr t.values()[4])== "FAST TIMESTAMP VECTOR" +assert 15,(typestr t.values()[5])== "FAST TIMESTAMP VECTOR" +assert 16,(typestr t.values()[6])== "FAST TIMESTAMP VECTOR" + +@testing:case="test_timestamp64_convert" +t=table(2013.06.13 as c_date ,2012.06M as c_month,13:30:10.008 as c_time, + 13:30m as c_minute,13:30:10 as c_second,2013.06.13T13:30:10 as c_datetime, + 2012.06.13T13:30:10.008 as c_timestamp,13:30:10.008007006 as c_nanotime,2012.06.13T13:30:10.008007006 as c_nanotimestamp); +t=hdf5::read(H5_FILE_PATH+"/timestamp.h5","ts64",t) +assert 1,(size t.keys())==9 +assert 2,(size t)==4 +assert 3,t.values()[0]==[2013.06.13,2017.11.28,2017.11.28,2017.11.28] +assert 4,t.values()[1]==[2012.06M,2017.11M,2017.11M,2017.11M] +assert 5,cast(t.values()[2],MINUTE)==[13:30m,09:59m,09:59m,09:59m] +assert 6,t.values()[3]==[13:30m,09:59m,09:59m,09:59m] +assert 5,cast(t.values()[4],MINUTE)==[13:30m,09:59m,09:59m,09:59m] +assert 10,(typestr t.values()[0])== "FAST DATE VECTOR" +assert 11,(typestr t.values()[1])== "FAST MONTH VECTOR" +assert 12,(typestr t.values()[2])== "FAST TIME VECTOR" +assert 13,(typestr t.values()[3])== "FAST MINUTE VECTOR" +assert 14,(typestr t.values()[4])== "FAST SECOND VECTOR" +assert 15,(typestr t.values()[5])== "FAST DATETIME VECTOR" +assert 16,(typestr t.values()[6])== "FAST TIMESTAMP VECTOR" +assert 17,(typestr t.values()[7])== "FAST NANOTIME VECTOR" +assert 18,(typestr t.values()[8])== "FAST NANOTIMESTAMP VECTOR" + +@testing:case="test_timestamp32" +t=hdf5::read(H5_FILE_PATH+"/timestamp.h5","ts32") +t2=hdf5::read(H5_FILE_PATH+"/timestamp.h5","ts32_in_int") +assert 1,(size t.keys())==9 +assert 2,(size t2.keys())==9 +assert 3,(size t)==3 +assert 4,(size t2)==3 +assert 5,cast(t.values()[0],LONG)==cast(t2.values()[0],LONG)*1000 +assert 6,cast(t.values()[1],LONG)==cast(t2.values()[1],LONG)*1000 +assert 7,cast(t.values()[2],LONG)==cast(t2.values()[2],LONG)*1000 +assert 8,cast(t.values()[3],LONG)==cast(t2.values()[3],LONG)*1000 +assert 9,cast(t.values()[4],LONG)==cast(t2.values()[4],LONG)*1000 +assert 10,(typestr t.values()[0])== "FAST TIMESTAMP VECTOR" +assert 11,(typestr t.values()[1])== "FAST TIMESTAMP VECTOR" +assert 12,(typestr t.values()[2])== "FAST TIMESTAMP VECTOR" +assert 13,(typestr t.values()[3])== "FAST TIMESTAMP VECTOR" +assert 14,(typestr t.values()[4])== "FAST TIMESTAMP VECTOR" +assert 15,(typestr t.values()[5])== "FAST TIMESTAMP VECTOR" +assert 16,(typestr t.values()[6])== "FAST TIMESTAMP VECTOR" + +@testing:case="test_timestamp32_convert" +t=table(2013.06.13 as c_date ,2012.06M as c_month,13:30:10.008 as c_time, + 13:30m as c_minute,13:30:10 as c_second,2013.06.13T13:30:10 as c_datetime, + 2012.06.13T13:30:10.008 as c_timestamp,13:30:10.008007006 as c_nanotime,2012.06.13T13:30:10.008007006 as c_nanotimestamp); +t=hdf5::read(H5_FILE_PATH+"/timestamp.h5","ts32",t) +assert 1,(size t.keys())==9 +assert 2,(size t)==4 +assert 3,t.values()[0]==[2013.06.13,2017.11.28,2017.11.28,2017.11.28] +assert 4,t.values()[1]==[2012.06M,2017.11M,2017.11M,2017.11M] +assert 5,cast(t.values()[2],MINUTE)==[13:30m,12:29m,12:29m,12:29m] +assert 6,t.values()[3]==[13:30m,12:29m,12:29m,12:29m] +assert 5,cast(t.values()[4],MINUTE)==[13:30m,12:29m,12:29m,12:29m] +assert 10,(typestr t.values()[0])== "FAST DATE VECTOR" +assert 11,(typestr t.values()[1])== "FAST MONTH VECTOR" +assert 12,(typestr t.values()[2])== "FAST TIME VECTOR" +assert 13,(typestr t.values()[3])== "FAST MINUTE VECTOR" +assert 14,(typestr t.values()[4])== "FAST SECOND VECTOR" +assert 15,(typestr t.values()[5])== "FAST DATETIME VECTOR" +assert 16,(typestr t.values()[6])== "FAST TIMESTAMP VECTOR" +assert 17,(typestr t.values()[7])== "FAST NANOTIME VECTOR" +assert 18,(typestr t.values()[8])== "FAST NANOTIMESTAMP VECTOR" + +//loadPlugin("/home/swang/dolphin_workspace/DolphinDBPlugin/build/PluginHdf5.txt"); +@testing:case="test_compound" +t=hdf5::read(H5_FILE_PATH+"/compound.h5","com") +assert 1,(size t.keys())==9 +assert 2,(size t)==27 +assert 3,t.values()[0][0]==`abcdefgh +assert 4,t.values()[1][0]=="this is a variable string" +assert 5,t.values()[2][0]==7.5 +assert 6,t.values()[3][0]==2017.11.28T15:47:21.000 +assert 7,t.values()[4][0]==23456789 +assert 8,t.values()[5][0]==8.45f +assert 9,t.values()[6][0]==7 +assert 10,t.values()[7][0]==23 +assert 11,t.values()[8][0]=='w' +assert 12,(typestr t.values()[0])== "STRING VECTOR" +assert 13,(typestr t.values()[1])== "STRING VECTOR" +assert 14,(typestr t.values()[2])== "FAST DOUBLE VECTOR" +assert 15,(typestr t.values()[3])== "FAST TIMESTAMP VECTOR" +assert 16,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 17,(typestr t.values()[5])== "FAST FLOAT VECTOR" +assert 18,(typestr t.values()[6])== "FAST INT VECTOR" +assert 19,(typestr t.values()[7])== "FAST SHORT VECTOR" +assert 20,(typestr t.values()[8])== "FAST CHAR VECTOR" + +@testing:case="test_nested_compound" +t=hdf5::read(H5_FILE_PATH+"/vlen_str.h5","compound_vstr") +assert 1,(size t.keys())==7 +assert 2,(size t)==4 +assert 3,t.values()[0]==[1,1,1,1] +assert 4,t.values()[1]==[9.8,9.8,9.8,9.8] +assert 5,t.values()[3]==["nest vlen 1","nest vlen 2","nest vlen 3","nest vlen 4"]; +assert 6,t.values()[6]==[ 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807] +assert 7,(typestr t.values()[0])== "FAST INT VECTOR" +assert 8,(typestr t.values()[1])== "FAST DOUBLE VECTOR" +assert 9,(typestr t.values()[2])== "FAST FLOAT VECTOR" +assert 10,(typestr t.values()[3])== "STRING VECTOR" +assert 11,(typestr t.values()[4])== "FAST LONG VECTOR" +assert 12,(typestr t.values()[5])== "STRING VECTOR" +assert 13,(typestr t.values()[6])== "FAST LONG VECTOR" + +@testing:case="test_array" +t=hdf5::read(H5_FILE_PATH+"/array_mdatom.h5","array_int_3_dims") +assert 1,(size t.keys())==24 +assert 2,(size t)==6 +assert 3,t.values()[0]==[0,24,48,72,96,120] +assert 4,t.values()[1]==[1,25,49,73,97,121] +assert 5,t.values()[2]==[2,26,50,74,98,122]; +assert 6,t.values()[3]==[3,27,51,75,99,123] +assert 7,(typestr t.values()[0])== "FAST INT VECTOR" +assert 8,(typestr t.values()[1])== "FAST INT VECTOR" +assert 9,(typestr t.values()[2])== "FAST INT VECTOR" +assert 10,(typestr t.values()[3])== "FAST INT VECTOR" + +@testing:case="test scalar" +t=hdf5::read(H5_FILE_PATH+"/scalar.h5","variable length string") +assert 1,(size t.keys())==1 +assert 2,(size t)==1 +assert 3,t.values()[0]=="Some string" diff --git a/include/CoreConcept.h b/include/CoreConcept.h index 6e674e72..5d5c7864 100644 --- a/include/CoreConcept.h +++ b/include/CoreConcept.h @@ -56,12 +56,9 @@ class ColumnRef; class SymbolBase; class SymbolBaseManager; class Output; -class Console; -class Session; -class ConstantMarshall; -class ConstantUnmarshall; class DebugContext; -class DomainSite; +class Session; +struct ClusterNodes; class DomainSitePool; class DomainPartition; class Domain; @@ -69,17 +66,13 @@ class PartitionGuard; struct TableUpdate; struct TableUpdateSizer; struct TableUpdateUrgency; -struct LocalTableUpdate; -struct TopicSubscribe; -class SessionThreadCallGuard; class ReducerContainer; class DistributedCall; class JobProperty; -struct JITCfgNode; -class Codegen; -struct JITCfgNode; -class Codegen; -class JITValue; +class IoTransaction; +class Decoder; +class VolumeMapper; +class SystemHandle; typedef SmartPointer AuthenticatedUserSP; typedef SmartPointer ByteArrayCodeBufferSP; @@ -102,31 +95,26 @@ typedef SmartPointer ColumnRefSP; typedef SmartPointer SymbolBaseSP; typedef SmartPointer SymbolBaseManagerSP; typedef SmartPointer OutputSP; -typedef SmartPointer ConsoleSP; -typedef SmartPointer SessionSP; -typedef SmartPointer ConstantMarshallSP; -typedef SmartPointer ConstantUnmarshallSP; typedef SmartPointer DebugContextSP; -typedef SmartPointer DomainSiteSP; +typedef SmartPointer SessionSP; +typedef SmartPointer ClusterNodesSP; typedef SmartPointer DomainSitePoolSP; typedef SmartPointer DomainPartitionSP; typedef SmartPointer DomainSP; typedef SmartPointer PartitionGuardSP; typedef SmartPointer TableUpdateSP; typedef SmartPointer > TableUpdateQueueSP; -typedef SmartPointer TopicSubscribeSP; -typedef SmartPointer SessionThreadCallGuardSP; typedef SmartPointer ReducerContainerSP; typedef SmartPointer DistributedCallSP; typedef SmartPointer JobPropertySP; -typedef SmartPointer JITCfgNodeSP; +typedef SmartPointer VolumeMapperSP; +typedef SmartPointer DecoderSP; typedef ConstantSP(*OptrFunc)(const ConstantSP&,const ConstantSP&); typedef ConstantSP(*SysFunc)(Heap* heap,vector& arguments); typedef ConstantSP(*TemplateOptr)(const ConstantSP&,const ConstantSP&,const string&, OptrFunc); typedef ConstantSP(*TemplateUserOptr)(Heap* heap, const ConstantSP&,const ConstantSP&, const FunctionDefSP&); typedef void (*SysProc)(Heap* heap,vector& arguments); -typedef std::function CFGTraversalFunc; class AuthenticatedUser{ public: @@ -408,8 +396,9 @@ class SymbolBase{ SymbolBase(const string& symbolFile, bool snapshot = false, bool supportOrder=true); SymbolBase(const string& symbolFile, const DataInputStreamSP& in, bool snapshot = false); SymbolBase* copy(); - bool saveSymbolBase(const string& symbolFile, string& errMsg); - bool saveSymbolBase(string& errMsg); + bool saveSymbolBase(string& errMsg, bool sync = false); + IO_ERR serialize(int offset, int length, Buffer& buf); + inline bool lastSaveSynchronized() const { return lastSaveSynchronized_;} inline int find(const string& symbol){ #ifndef LOCKFREE_SYMBASE LockGuard guard(&keyMutex_); @@ -454,6 +443,7 @@ class SymbolBase{ string dbPath_; int savingPoint_; bool modified_; + bool lastSaveSynchronized_; bool supportOrder_; DynamicArray key_; SmartPointer > ordinal_; @@ -1348,6 +1338,27 @@ class Output { virtual IO_ERR flush() = 0; }; +class DebugContext{ +public: + DebugContext(); + void waitForExecution(Heap* pHeap, Statement* pStatement); + void waitForStop(); + void continueExecution(int steps); + void decreaseSteps(); + int getSteps() const { return steps_;} + bool continueFlag() const { return continueFlag_;} + +private: + int steps_; + bool continueFlag_; + bool stopped_; + Mutex mutex_; + ConditionalVariable execCondition_; + ConditionalVariable stopCondition_; + Heap* lastHeap_; + Statement* lastStatement_; +}; + class Session { public: Session(const HeapSP& heap); @@ -1433,91 +1444,6 @@ class Session { int parallelism_; }; -class Console { -public: - Console(const SessionSP& session, const OutputSP& out); - virtual ~Console(){} - virtual void cancel(bool running){} - Output* getOutput(){return out_.get();} - Session* getSession(){return session_.get();} - long long getLastActiveTime() const { return lastActiveTime_;} - void setLastActiveTime(long long lastUpdate) { lastActiveTime_ = lastUpdate;} - inline void setJobId(const Guid& jobId); - inline const Guid& getJobId() const { return jobId_;} - inline const Guid& getTaskId() const { return jobId_;} - inline void setRootJobId(const Guid& rootJobId) { rootJobId_ = rootJobId;} - inline const Guid& getRootJobId() const { return rootJobId_;} - inline void setPriority(int priority) { priority_ = priority;} - inline int getPriority() const { return priority_;} - inline void setParallelism(int parallelism) { parallelism_ = parallelism;} - inline int getParallelism() const { return parallelism_;} - inline bool isCancellable() const {return cancellable_;} - inline void setCancellable(bool option){cancellable_ = option;} - void set(const Guid& rootJobId, const Guid& jobId, int parallelism, int priority); - inline void setFlag(long long flag) { flag_ = flag;} - inline long long getFlag() const { return flag_;} - inline bool isUrgent() const { return flag_ & 1;} - virtual IO_ERR readReady()=0; - virtual IO_ERR execute()=0; - virtual CONSOLE_TYPE getConsoleType() const = 0; - virtual void run() = 0; - virtual void getTaskDesc(string& type, string& desc) const = 0; - -protected: - Guid rootJobId_; - Guid jobId_; - int priority_; - int parallelism_; - bool cancellable_; - SessionSP session_; - OutputSP out_; - long long lastActiveTime_; - long long flag_; -}; - -class ConstantMarshall { -public: - virtual ~ConstantMarshall(){} - virtual bool start(const ConstantSP& target, bool blocking, IO_ERR& ret)=0; - virtual bool start(const char* requestHeader, size_t headerSize, const ConstantSP& target, bool blocking, IO_ERR& ret)=0; - virtual bool resume(IO_ERR& ret)=0; - virtual void reset() = 0; - virtual IO_ERR flush() = 0; -}; - -class ConstantUnmarshall{ -public: - virtual ~ConstantUnmarshall(){} - virtual bool start(short flag, bool blocking, IO_ERR& ret)=0; - virtual bool resume(IO_ERR& ret)=0; - virtual void reset() = 0; - ConstantSP getConstant(){return obj_;} - -protected: - ConstantSP obj_; -}; - -class DebugContext{ -public: - DebugContext(); - void waitForExecution(Heap* pHeap, Statement* pStatement); - void waitForStop(); - void continueExecution(int steps); - void decreaseSteps(); - int getSteps() const { return steps_;} - bool continueFlag() const { return continueFlag_;} - -private: - int steps_; - bool continueFlag_; - bool stopped_; - Mutex mutex_; - ConditionalVariable execCondition_; - ConditionalVariable stopCondition_; - Heap* lastHeap_; - Statement* lastStatement_; -}; - class Heap{ public: Heap():meta_(0), session_(0), size_(0), status_(0){} @@ -1571,94 +1497,9 @@ class Heap{ char status_; }; -struct JITCfgNode { - JITCfgNode() : visited_(false){} - string getInferredTypeCacheAsString() const { - string out = "{"; - for (auto kv: inferredTypeCache) { - out.append(kv.first + ": " +kv.second.getString() + ", "); - } - out.append("}"); - return out; - } - - // control flow graph: next block edge - vector cfgNexts; - // control flow graph: reverse edge to incoming block - vector cfgFroms; - unordered_map inferredTypeCache; - unordered_map> upstreamTypes; - bool visited_; -}; - -class Statement{ -public: - Statement(STATEMENT_TYPE type):breakpoint_(false), type_(type){} - virtual ~Statement(){} - STATEMENT_TYPE getType() const {return type_;} - virtual void execute(Heap* pHeap)=0; - virtual void debugExecute(Heap* pHeap, DebugContext* pContext); - virtual bool shouldReturn() const {return false;} - virtual bool shouldBreak() const {return false;} - virtual bool shouldContinue() const {return false;} - virtual string getScript(int indention) const = 0; - virtual IO_ERR serialize(Heap* pHeap, const ByteArrayCodeBufferSP& buffer) const = 0; - virtual void collectUserDefinedFunctions(unordered_map& functionDefs) const {} - void disableBreakpoint(); - void enableBreakpoint(); - - JITCfgNodeSP getCFGNode() const; - vector& getCFGNexts(); - vector& getCFGFroms(); - unordered_map> & getUpstreamTypes() { return cfgNode_->upstreamTypes; } - unordered_map & getInferredTypeCache(); - void addCFGNextBlock(const StatementSP& nextBlock); - void addCFGFromBlock(const StatementSP& fromBlock); - bool getCFGNodeVisited() const; - bool setCFGNodeVisited(bool visited = true); - string getInferredTypeCacheAsString() const; - virtual InferredType inferType(Heap* heap, unordered_set & vis, const string & varName); - virtual IO_ERR buildCFG(const StatementSP& self, std::unordered_map & context); - virtual string getInferredTypesDebugString(int indention) const; - virtual void traverseCFG(const StatementSP& self, unordered_set & visited, CFGTraversalFunc func); - virtual vector getVarNames() const; - -protected: - bool breakpoint_; - JITCfgNodeSP cfgNode_; - -private: - STATEMENT_TYPE type_; -}; - -class StatementFactory { -public: - virtual ~StatementFactory(){} - virtual Statement* readStatement(Session* session, const DataInputStreamSP& buffer) = 0; - virtual Statement* createReturnStatement(const ObjectSP& obj) = 0; -}; - -class DomainSite{ -public: - DomainSite(const string& host, int port, int index); - DomainSite(const string& host, int port, int index, const string& alias) ; - const string& getHost() const {return host_;} - int getPort() const {return port_;} - int getIndex() const {return index_;} - string getString() const; - const string& getAlias() const {return alias_;} - inline bool operator ==(const DomainSite& target) const { return host_ == target.host_ && port_ == target.port_;} - static DomainSite emptySite_; -private: - string host_; - int port_; - int index_; - string alias_; -}; - class DomainSitePool { public: - DomainSitePool() : lastSuccessfulSiteIndex_(-1), lastSiteIndex_(-1), nextSiteIndex_(-1), startSiteIndex_(-1){} + DomainSitePool() : lastSuccessfulSiteIndex_(-1), lastSiteIndex_(-1), nextSiteIndex_(-1), startSiteIndex_(-1), localSiteIndex_(-1){} DomainSitePool(const DomainPartitionSP& partition); void addSite(int siteIndex); int getLastSite() const; @@ -1667,6 +1508,7 @@ class DomainSitePool { int nextSite() const; inline int getNextSite() const { return sites_[nextSiteIndex_].first;} inline bool hasNextSite() const { return nextSiteIndex_ >= 0;} + inline bool containLocalSite() const { return localSiteIndex_ >= 0;} void initiateSite(bool useLastSuccessfulSite = false); void setIntialSite(int index); void resetInitialSite(); @@ -1679,6 +1521,7 @@ class DomainSitePool { mutable int lastSiteIndex_; mutable int nextSiteIndex_; int startSiteIndex_; + int localSiteIndex_; }; class DomainPartition{ @@ -1692,8 +1535,9 @@ class DomainPartition{ const int getKey() const {return key_;} virtual bool addSite(int siteIndex) {return false;} virtual int getSiteCount() const { return 0;} - virtual const DomainSite& getSite(int index) const { throw RuntimeException("DomainPartition::getSite method not supported.");} + virtual int getSiteIndex(int index) const { throw RuntimeException("DomainPartition::getSiteIndex method not supported.");} virtual bool isLocalPartition() const { return true;} + virtual bool containLocalCopy() const { return true;} string getString() const; inline const string& getPath() const {return path_;} inline const Guid& getId() const {return id_;} @@ -1843,41 +1687,6 @@ struct TableUpdateUrgency { } }; -struct TopicSubscribe { - TopicSubscribe(const string& topic, int hashValue, vector attributes, const FunctionDefSP& handler, const AuthenticatedUserSP& user, bool msgAsTable, int batchSize, int throttleTime) : msgAsTable_(msgAsTable), - hashValue_(hashValue), batchSize_(batchSize), throttleTime_(throttleTime), cumSize_(0), messageId_(-1), expired_(-1), topic_(topic), attributes_(attributes), handler_(handler), user_(user){} - bool append(long long msgId, const ConstantSP& msg, long long& outMsgId, ConstantSP& outMsg); - bool getMessage(long long now, long long& outMsgId, ConstantSP& outMsg); - bool updateSchema(const TableSP& emptyTable); - - const bool msgAsTable_; - const int hashValue_; - const int batchSize_; - const int throttleTime_; //in millisecond - int cumSize_; - std::atomic messageId_; - long long expired_; - const string topic_; - vector attributes_; - const FunctionDefSP handler_; - const AuthenticatedUserSP user_; - ConstantSP body_; - ConstantSP filter_; - Mutex mutex_; -}; - -class SessionThreadCallGuard { -public: - SessionThreadCallGuard() : session_(0){} - SessionThreadCallGuard(Session* session); - ~SessionThreadCallGuard(); - void setThreadCallMode(Session* session); - void releaseThreadCallMode(); - -private: - Session* session_; -}; - class ReducerContainer { public: ReducerContainer(Heap* heap, const FunctionDefSP& reducer) : heap_(heap), reducer_(reducer), objCount_(0){} @@ -1976,4 +1785,156 @@ struct JobProperty { bool cancellable_; }; +class StageExecutor { +public: + virtual ~StageExecutor(){} + virtual vector execute(Heap* heap, const vector& tasks) = 0; + virtual vector execute(Heap* heap, const vector& tasks, const JobProperty& jobProp) = 0; +}; + +class StaticStageExecutor : public StageExecutor{ +public: + StaticStageExecutor(bool parallel, bool reExecuteOnOOM, bool trackJobs, bool resumeOnError = false, bool scheduleRemoteSite = true) : parallel_(parallel), + reExecuteOnOOM_(reExecuteOnOOM), trackJobs_(trackJobs), resumeOnError_(resumeOnError), scheduleRemoteSite_(scheduleRemoteSite){} + virtual ~StaticStageExecutor(){} + virtual vector execute(Heap* heap, const vector& tasks); + virtual vector execute(Heap* heap, const vector& tasks, const JobProperty& jobProp); + +private: + void groupRemoteCalls(const vector& tasks, vector& groupedCalls, const ClusterNodesSP& clusterNodes); + +private: + bool parallel_; + bool reExecuteOnOOM_; + bool trackJobs_; + bool resumeOnError_; + bool scheduleRemoteSite_; +}; + +class PipelineStageExecutor : public StageExecutor { +public: + PipelineStageExecutor(vector& followingFunctors, bool trackJobs, int queueDepth = 2, int parallel = 1) : followingFunctors_(followingFunctors), trackJobs_(trackJobs), + queueDepth_(queueDepth), parallel_(parallel){} + virtual ~PipelineStageExecutor(){} + virtual vector execute(Heap* heap, const vector& tasks); + virtual vector execute(Heap* heap, const vector& tasks, const JobProperty& jobProp); + +private: + void parallelExecute(Heap* heap, vector& tasks); + +private: + vector followingFunctors_; + bool trackJobs_; + int queueDepth_; + int parallel_; +}; + +class Decoder { +public: + Decoder(int id, bool appendable) : id_(id), appendable_(appendable), codeSymbolAsString_(false){} + virtual ~Decoder(){} + virtual VectorSP code(const VectorSP& vec, bool lsnFlag) = 0; + virtual IO_ERR code(const VectorSP& vec, bool lsnFlag, const DataOutputStreamSP& out, int& checksum) = 0; + virtual IO_ERR decode(const VectorSP& vec, INDEX rowOffset, bool fullLoad, int checksum, const DataInputStreamSP& in, + long long byteSize, long long byteOffset, INDEX& postRows, long long& postByteOffset, long long& lsn) = 0; + inline int getID() const {return id_;} + inline bool isAppendable() const {return appendable_;} + inline bool codeSymbolAsString() const { return codeSymbolAsString_;} + void codeSymbolAsString(bool enabled) { codeSymbolAsString_ = enabled;} +private: + int id_; + bool appendable_; + bool codeSymbolAsString_; +}; + +class VolumeMapper { +public: + VolumeMapper(vector& volumes, int workers); + int getMappedDeviceId(const string& path); + +private: + int workers_; + unordered_map deviceMap_; +}; + +struct ColumnHeader{ + ColumnHeader(const char* header); + ColumnHeader(); + static int getHeaderSize(){ return 20;} + void serialize(ByteArrayCodeBuffer& buf); + void deserialize(const char* header); + inline bool isLittleEndian() const { return flag & 1;} + inline bool containChecksum() const {return flag & 2;} + inline bool lsnFlag() const{return flag & 4;} + inline void setLittleEndian(bool val){ if(val) flag |= 1; else flag &= ~1;} + inline void setChecksumOption(bool val){ if(val) flag |= 2; else flag &= ~2;} + inline void setLSNFlag(bool val){ if(val) flag |= 4; else flag &= ~4;} + + char version; + char flag; //bit0: littleEndian bit1: containChecksum + char charCode; + char compression; + char dataType; + char unitLength; + short reserved; + int extra; + int count; + int checksum; +}; + +class DBFileIO { +public: + static bool saveBasicTable(Session* session, const string& directory, Table* table, const string& tableName, IoTransaction* tran, bool append = false, int compressionMode = 0, bool saveSymbolBase = true); + static bool saveBasicTable(Session* session, SystemHandle* db, Table* table, const string& tableName, IoTransaction* tran, bool append = false, int compressionMode = 0, bool saveSymbolBase = true); + static bool saveBasicTable(Session* session, const string& directory, const string& tableDir, Table* table, const string& tableName, const vector& cols, SymbolBaseSP& symbase, IoTransaction* tran, bool chunkMode, bool append, int compressionMode, bool saveSymbolBase); + static bool saveBasicTable(const string& directory, const string& tableName, Table* table, const SymbolBaseSP& symBase, IoTransaction* tran, int compressionMode); + static bool saveBasicTable(Session* session, const string& tableDir, INDEX existingTblSize, Table* table, const vector& cols, const SymbolBaseSP& symBase, IoTransaction* tran, int compressionMode, bool saveSymbolBase, long long lsn); + static bool savePartitionedTable(Session* session, const DomainSP& domain, TableSP table, const string& tableName, IoTransaction* tran, int compressionMode = 0, bool saveSymbolBase = true ); + static bool saveDualPartitionedTable(Session* session, SystemHandle* db, const DomainSP& secDomain, TableSP table, const string& tableName, + const string& partitionColName, vector& secPartitionColNames, IoTransaction* tran, int compressionMode = 0); + static Table* loadTable(Session* session, const string& directory, const string& tableName, const SymbolBaseManagerSP& symbaseManager, const DomainSP& domain, const ConstantSP& partitions, TABLE_TYPE tableType, bool memoryMode); + static Table* loadTable(Session* session, SystemHandle* db, const string& tableName, const ConstantSP& partitions, TABLE_TYPE tableType, bool memoryMode); + static void removeTable(SystemHandle* db, const string& tableName); + static SystemHandle* openDatabase(const string& directory, const DomainSP& localDomain); + static bool saveDatabase(SystemHandle* db); + static bool removeDatabase(const string& dbDir); + + static ColumnHeader loadColumnHeader(const string& colFile); + static VectorSP loadColumn(const string& colFile, int devId, const SymbolBaseManagerSP& symbaseManager); + static VectorSP loadColumn(const string& colFile, int devId, const SymbolBaseSP& symbase); + static VectorSP loadColumn(const string& colFile, int devId, const SymbolBaseSP& symbase, int rows, long long& postFileOffset, bool& isLittleEndian, char& compressType); + static long long loadColumn(const string& colFile, long long fileOffset, bool isLittleEndian, char compressType, int devId, const SymbolBaseSP& symbase, INDEX rows, const VectorSP& col); + static Vector* loadTextVector(bool includeHeader, DATA_TYPE type, const string& path); + static bool saveColumn(const VectorSP& col, const string& colFile, int devId, INDEX existingTableSize, bool chunkNode, bool append, int compressionMode, IoTransaction* tran = NULL, long long lsn = -1); + static bool saveTableHeader(const string& owner, const vector& cols, vector& partitionColumnIndices, long long rows, const string& tableFile, IoTransaction* tran); + static bool loadTableHeader(const DataInputStreamSP& in, string& owner, vector& cols, vector& partitionColumnIndices); + static void removeBasicTable(const string& directory, const string& tableName); + static TableSP createEmptyTableFromSchema(const TableSP& schema); + static long long truncateColumnByLSN(const string& colFile, int devId, long long expectedLSN, bool sync=true); + static long long truncateColumnByRows(const string& colFile, int devId, INDEX rows, bool sync=true); + static void truncateSymbolBase(const string& symFile, int devId, INDEX rows, bool sync=true); + + static void checkTypeCompatibility(Table* table, vector& partitionColumns, vector& cols, vector& partitionColumnIndices); + static bool checkTypeCompatibility(DATA_TYPE type1, DATA_TYPE type2); + static bool checkPartitionColumnCompatibility(DATA_TYPE partitionSchemeType, DATA_TYPE partitionDataType); + static void saveSymbolBases(const SymbolBaseSP& symbase, IoTransaction* tran); + static void collectColumnDesc(Table* table, vector& cols); + static ConstantSP convertColumn(const ConstantSP& col, const ColumnDesc& desiredType, SymbolBaseSP& symbase); + static ConstantSP convertColumn(const ConstantSP& col, const ColumnDesc& desiredType, const SymbolBaseSP& symbase); + static VectorSP decompress(const VectorSP& col); + static VectorSP decompress(const VectorSP& col, const DecoderSP decoder); + static VectorSP compress(const VectorSP& col); + static TableSP compressTable(const TableSP& table); + static ConstantSP appendDataToFile(Heap* heap, vector& arguments); + static int getMappedDeviceId(const string& path); + static void setVolumeMapper(const VolumeMapperSP& volumeMapper) { volumeMapper_ = volumeMapper;} + static unsigned int checksum(FILE* fp, long long offset, long long length); + +private: + static VectorSP loadColumn(const string& colFile, int devId, const SymbolBaseManagerSP& symbaseManager, const SymbolBaseSP& symbase, + int rows, long long& postFileOffset, bool& isLittleEndian, char& compressType); + static inline DATA_TYPE getCompressedDataType(const VectorSP& vec){return (DATA_TYPE)vec->getChar(4);} + static VolumeMapperSP volumeMapper_; +}; + #endif /* CORECONCEPT_H_ */ diff --git a/include/SmartPointer.h b/include/SmartPointer.h index d65b56b2..4ec54ed5 100644 --- a/include/SmartPointer.h +++ b/include/SmartPointer.h @@ -60,7 +60,6 @@ class SmartPointer { return *((T*)tmp->p_); tmp->addRef(); - //TODO: the below operation is not thread-safe. But it is safe if there is only one writer and multiple readers. Counter* oldCounter = counterP_; counterP_= tmp; @@ -83,7 +82,6 @@ class SmartPointer { Counter* tmp = new Counter(0); tmp->addRef(); - //TODO: the below operation is not thread-safe. But it is safe if there is only one writer and multiple readers. Counter* oldCounter = counterP_; counterP_= tmp; diff --git a/include/StreamEngine.h b/include/StreamEngine.h new file mode 100644 index 00000000..4c8d8500 --- /dev/null +++ b/include/StreamEngine.h @@ -0,0 +1,179 @@ +// +// Created by jccai on 19-5-31. +// + +#ifndef DOLPHINDB_STREAMENGINE_H +#define DOLPHINDB_STREAMENGINE_H + +#include +#include +#include "CoreConcept.h" +#include "SmartPointer.h" + +class AbstractStreamEngine; +using AbstractStreamEngineSP = SmartPointer; + +class StreamEngineManager { +public: + StreamEngineManager(const StreamEngineManager &) = delete; + StreamEngineManager operator=(const StreamEngineManager &) = delete; + static StreamEngineManager& instance(); + + void insert(const AbstractStreamEngineSP &aggregator); + void remove(const string &name); + AbstractStreamEngineSP find(const string &name); + ConstantSP getStat(); +private: + StreamEngineManager() = default; + mutable Mutex mutex_; + unordered_map engines_; + unordered_set engineTypes_; +}; + +class AbstractStreamEngine : public Table { +public: + AbstractStreamEngine(const string &type, const string &name, const string &user); + + AbstractStreamEngine(const string &type, const string &name, const string &user, + const SmartPointer> &colNames); + + AbstractStreamEngine(const string &type, const string &name, const string &user, + const SmartPointer> &colNames, SmartPointer> colMap); + + ~AbstractStreamEngine() override; + + /// (optional) implement addMetrics if your engine support it + /// arguments[0]: meta code, new metrics + /// arguments[1]: table, indicating the scheme of the new metrics + virtual bool addMetrics(Heap *heap, vector &arguments) { return false; } + + /// (optional) called when removed from StreamEngineManager + /// should be non-blocking, e.g. set some flags and return immediately + virtual void finalize() {} + + /// (required) the scheme of the generated table must be consistent with the engineStat_ vector + /// generateEngineStatTable will be called asynchronously by StreamEngineManager, should be thread safe + /// e.g. + /// TableSP YourEngine::generateEngineTable() { + /// static vector colNames{"name", "user", "status", "lastErrMsg", ...}; + /// vector cols; + /// cols.push_back(Util::createVector(DT_STRING, 0)); + /// cols.push_back(Util::createVector(DT_STRING, 0)); + /// cols.push_back(Util::createVector(DT_STRING, 0)); + /// cols.push_back(Util::createVector(DT_STRING, 0)); + /// ... + /// return Util::createTable(colNames, cols); + /// } + virtual TableSP generateEngineStatTable() = 0; + + /// (required) initialize the engineStat_ which is used as a buffer + /// must be called immediately and only once after the engine's construction + /// e.g. + /// void YourEngine::initEngineStat() { + /// engineStat_.push_back(new String(engineName_)); + /// engineStat_.push_back(new String(engineUser_)); + /// engineStat_.push_back(new String(status_)); + /// engineStat_.push_back(new String(lastErrMsg_)); + /// ... + /// } + virtual void initEngineStat() = 0; + + /// (required) engine statues are represented by a Table + /// updateEngineStat will be called asynchronously by StreamEngineManager, should be thread safe + /// updateEngineStat should only update the changed fields in the engineStat_ to reduce overhead + /// e.g. + /// void YourEngine::updateEngineStat() { + /// LockGuard guard(&mutex_); + /// engineStat_[2]->setString(status_); + /// engineStat_[3]->setString(lastErrMsg_); + /// ... + /// } + virtual void updateEngineStat() = 0; + + string getEngineType(); + string getEngineName(); + string getEngineCreator(); + + vector &getEngineStatRef(); + + bool sizeable() const override { + return false; + } + + bool update(vector& values, const ConstantSP& indexSP, vector& colNames, string& errMsg) override { + errMsg = "StreamEngine doesn't support data update."; + return false; + } + + bool remove(const ConstantSP& indexSP, string& errMsg) override { + errMsg = "StreamEngine doesn't support data deletion."; + return false; + } + + ConstantSP getColumn(INDEX index) const override {return title_;} + const string &getColumnName(int index) const override { return colNames_->at(index); } + DATA_TYPE getColumnType(int index) const override { return DT_STRING; } + INDEX columns() const override { return colNames_->size(); } + INDEX size() const override { return 0; } + const string &getColumnQualifier(int index) const override { return name_; } + TABLE_TYPE getTableType() const override { return CUSTOMIZEDTBL; } + long long int getAllocatedMemory() const override { return 0; } + ConstantSP get(INDEX col, INDEX row) const override { throw RuntimeException("get not supported"); } + + ConstantSP getColumn(const string &name) const override; + ConstantSP getColumn(const string &qualifier, const string &name) const override; + ConstantSP getColumn(const string &name, const ConstantSP &rowFilter) const override; + ConstantSP getColumn(const string& qualifier, const string& name, const ConstantSP& rowFilter) const override; + ConstantSP getColumn(INDEX index, const ConstantSP& rowFilter) const override; + void setColumnName(int index, const string& name) override; + int getColumnIndex(const string& name) const override; + bool contain(const string& name) const override; + bool contain(const string& qualifier, const string& name) const override; + bool contain(ColumnRef* col) const override; + bool contain(const ColumnRefSP& col) const override; + bool containAll(const vector& cols) const override; + ConstantSP getColumnLabel() const override; + ConstantSP keys() const override; + ConstantSP values() const override; + string getString(INDEX index) const override; + string getString() const override; + bool set(INDEX index, const ConstantSP& value) override; + void setName(const string &name); + ConstantSP get(INDEX index) const override; + ConstantSP get(const ConstantSP &index) const override; + const string &getName() const override; + ConstantSP getInstance() const override; + ConstantSP getInstance(INDEX size) const override; + ConstantSP getValue() const override; + ConstantSP getValue(Heap *heap) override; + ConstantSP getValue(INDEX capacity) const override; + ConstantSP getReference(Heap *heap) override; + ConstantSP getWindow(INDEX colStart, int colLength, INDEX rowStart, int rowLength) const override; + ConstantSP getMember(const ConstantSP &key) const override; +protected: + const string engineType_; + const string engineName_; + const string engineUser_; + + /// Engine should update these fields. Basically a streaming engine should be confined to one thread + /// thus AbstractStreamEngine::append should be guarded by a mutex + /// these fields are recommended to be guarded by the same mutex + string status_ = "OK"; + string lastErrMsg_ = ""; + uint64_t cumMessages_ = 0; + + vector engineStat_; + + string name_; + + ConstantSP getInternal(INDEX index) const; + ConstantSP getInternal(const ConstantSP& index) const; + ConstantSP getMemberInternal(const ConstantSP& key) const; +private: + void initialize(); + ConstantSP title_; + SmartPointer> colNames_; + SmartPointer> colMap_; +}; + +#endif //DOLPHINDB_STREAMENGINE_H diff --git a/include/SysIO.h b/include/SysIO.h index 4e106596..4e8b03b0 100644 --- a/include/SysIO.h +++ b/include/SysIO.h @@ -10,7 +10,6 @@ #include #include -#include #include "SmartPointer.h" #include "Types.h" @@ -31,6 +30,7 @@ using std::string; +struct SSL; class Constant; class Socket; class UdpSocket; @@ -229,11 +229,11 @@ class DataOutputStream { FILE* getFile() const { return file_;} const char * getBuffer() const { return buf_;} size_t size() const { return size_;} - IO_ERR flush(); + IO_ERR flush(bool sync = false); IO_ERR close(); protected: - virtual IO_ERR internalFlush(size_t size); + virtual IO_ERR internalFlush(size_t size, bool sync = false); virtual IO_ERR internalClose(); virtual char* createBuffer(size_t& capacity); diff --git a/mysql/README.md b/mysql/README.md index 10ad0e6c..6780ba05 100644 --- a/mysql/README.md +++ b/mysql/README.md @@ -88,7 +88,7 @@ The file libPluginMySQL.so will be generated after the compilation. * Create a connection to MySQL server. Return a handle of MySQL connection, which will be used to access MySQL server later. -### Examples +### Example ``` conn = mysql::connect(`localhost, 3306, `root, `root, `DolphinDB) @@ -329,4 +329,4 @@ tb = loadTable("dfs://US", `tb) ## Time Consumed -160.5 seconds \ No newline at end of file +160.5 seconds diff --git a/mysql/README_CN.md b/mysql/README_CN.md index e58cc374..3e363fd1 100644 --- a/mysql/README_CN.md +++ b/mysql/README_CN.md @@ -329,4 +329,4 @@ tb = loadTable("dfs://US", `tb) ## 导入耗时 -160.5秒 \ No newline at end of file +160.5秒 diff --git a/odbc/DolphinDBODBC.txt b/odbc/DolphinDBODBC.txt index e21ad824..e16638a6 100644 --- a/odbc/DolphinDBODBC.txt +++ b/odbc/DolphinDBODBC.txt @@ -1,5 +1,5 @@ odbc,libPluginODBC.so -odbcQuery,query,system,2,3,0 +odbcQuery,query,system,2,4,0 odbcConnect,connect,system,1,1,0 odbcClose,close,system,1,1,0 odbcExecute,execute,system,2,3,0 \ No newline at end of file diff --git a/odbc/README.md b/odbc/README.md index 7a31a91d..54121b09 100644 --- a/odbc/README.md +++ b/odbc/README.md @@ -1,5 +1,7 @@ ## DolphinDB ODBC plugin +使用该插件可以方便的从ODBC支持的数据库中迁移数据至dolphinDB中. + With this plugin, you can easily pull your data from existing databases that support ODBC interface. ## Prerequisites @@ -54,4 +56,4 @@ The last argument is a optional user-provided table. If provided, query results You can ommit ```odbc::``` prefix by introducing obdc module namespace. ``` use odbc; -``` +``` \ No newline at end of file diff --git a/odbc/src/DolphinDBODBC.cpp b/odbc/src/DolphinDBODBC.cpp index 93a3a701..05f76ed4 100644 --- a/odbc/src/DolphinDBODBC.cpp +++ b/odbc/src/DolphinDBODBC.cpp @@ -9,41 +9,23 @@ #ifndef LINUX #include #endif -#include "nanodbc/nanodbc.h" #include #include #include #include -#include #include #include #include #include +#include "nanodbc/nanodbc.h" +#include "cvt.h" using namespace std; -ConstantSP minmax(const ConstantSP& a, const ConstantSP& b) { - if (!a->isScalar() && !a->isArray()) - throw IllegalArgumentException( - "minmax", - "The argument for minmax function must be a scalar or vector."); - ConstantSP result = Util::createVector(a->getType(), 2); - if (a->isScalar()) { - result->set(0, a); - result->set(1, a); - } else { - result->set(0, ((Vector*)a.get())->min()); - result->set(1, ((Vector*)a.get())->max()); - } - return result; -} static void odbcConnectionOnClose(Heap* heap, vector& args) { nanodbc::connection* cp = (nanodbc::connection*)(args[0]->getLong()); if (cp != nullptr) { - // printf("delete connection %p\n", cp); delete cp; args[0]->setLong(0); - } else { - // printf("connection %p already deleted", cp); } } @@ -84,8 +66,7 @@ static inline DATA_TYPE sqltype2DataType(int sqltype, long colsize) { } // fall-through default: - if ((sqltype == SQL_CHAR || sqltype == SQL_VARCHAR) && - colsize <= useStringSizeThreshold) { + if ((sqltype == SQL_CHAR || sqltype == SQL_VARCHAR) && colsize <= useStringSizeThreshold) { return DT_SYMBOL; } return DT_STRING; @@ -114,8 +95,7 @@ static bool compatible(DATA_TYPE dolphinType, int sqlType, int colsize) { case SQL_DECIMAL: return true; case SQL_CHAR: - if (colsize == 1) - return true; + if (colsize == 1) return true; default: return false; } @@ -152,27 +132,16 @@ static bool compatible(DATA_TYPE dolphinType, int sqlType, int colsize) { // creates a new native odbc connection if the first argument is a connection // string. returns the native odbc connection if the first argument is a // DT_RESOURCE type. -static ConstantSP odbcGetConnection(Heap* heap, - vector& args, - const string& funcName) { +static ConstantSP odbcGetConnection(Heap* heap, vector& args, const string& funcName) { if (args[0]->getType() == DT_STRING) { - string connStr = args[0]->getString(); + u16string connStr = utf8_to_utf16(args[0]->getString()); try { - unique_ptr cup( - new nanodbc::connection(connStr)); - - // printf("connected with driver %s\n", cup->driver_name().c_str()); - + unique_ptr cup(new nanodbc::connection(connStr)); const char* fmt = "odbc connection to [%s]"; vector descBuf(connStr.size() + strlen(fmt)); sprintf(descBuf.data(), fmt, connStr.c_str()); - - FunctionDefSP onClose(Util::createSystemProcedure( - "odbc connection onClose()", odbcConnectionOnClose, 1, 1)); - - Constant* res = - Util::createResource((long long)cup.release(), descBuf.data(), - onClose, heap->currentSession()); + FunctionDefSP onClose(Util::createSystemProcedure("odbc connection onClose()", odbcConnectionOnClose, 1, 1)); + Constant* res = Util::createResource((long long)cup.release(), descBuf.data(), onClose, heap->currentSession()); return res; } catch (const runtime_error& e) { const char* fmt = "error connecting to [%s]: %s"; @@ -183,52 +152,37 @@ static ConstantSP odbcGetConnection(Heap* heap, } else if (args[0]->getType() == DT_RESOURCE) { return args[0]; } else { - throw IllegalArgumentException(funcName, - "Unknown argument type, must be a " - "connection string or odbc connection " - "handle"); + throw IllegalArgumentException(funcName,"Unknown argument type, must be a connection string or odbc connection handle"); } - return nullptr; } -static TableSP odbcGetOrCreateTable(const Heap* heap, - const vector& args, - const nanodbc::result& results, - vector& columnVecs) { +static TableSP odbcGetOrCreateTable(const Heap* heap, const vector& args, const nanodbc::result& results, vector& columnVecs) { int symbolCount = 0; - if (args.size() >= 3) { + if (args.size() >= 3 && args[2]->getType() != DT_VOID) { if (args[2]->getForm() != DF_TABLE) { throw TableRuntimeException("The 3rd argument must be a table"); } TableSP tblSP(args[2]); INDEX columns = tblSP->columns(); if (columns != results.columns()) { - throw TableRuntimeException( - "The given table schema is incompatible with the returned " - "table from ODBC (different column count)"); + throw TableRuntimeException("The given table schema is incompatible with the returned table from ODBC (different column count)"); } for (INDEX i = 0; i < columns; ++i) { Vector* col = (Vector*)tblSP->getColumn(i).get(); DATA_TYPE dolphinType = col->getType(); - if (!compatible(dolphinType, results.column_datatype(i), - results.column_size(i))) { - const char* fmt = - "The given table schema is incompatible with the returned " - "table from ODBC at column %d[%s]"; + if (!compatible(dolphinType, results.column_datatype(i), results.column_size(i))) { + const char* fmt = "The given table schema is incompatible with the returned table from ODBC at column %d[%s]"; vector descBuf(10 + strlen(fmt) + results.column_name(i).size()); sprintf(descBuf.data(), fmt, i, results.column_name(i).c_str()); throw TableRuntimeException(descBuf.data()); } if (dolphinType == DT_SYMBOL) { - columnVecs[i] = - Util::createSymbolVector(col->getSymbolBase(), 0); + columnVecs[i] = Util::createSymbolVector(col->getSymbolBase(), 0); symbolCount++; } else { columnVecs[i] = Util::createVector(dolphinType, 0); } } - // printf("return table, %d symbol vector(s)\n", symbolCount); - // printf("return args[2]\n"); return tblSP; } else { short columns = results.columns(); @@ -236,13 +190,10 @@ static TableSP odbcGetOrCreateTable(const Heap* heap, vector columnTypes(columns); for (short i = 0; i < columns; ++i) { - columnNames[i] = results.column_name(i); - columnTypes[i] = sqltype2DataType(results.column_datatype(i), - results.column_size(i)); + columnNames[i] = utf16_to_utf8(results.column_name(i)); + columnTypes[i] = sqltype2DataType(results.column_datatype(i), results.column_size(i)); if (columnTypes[i] == DT_SYMBOL) { symbolCount++; - // printf("column %s has size %ld, use symbol\n", - // columnNames[i].c_str(), results.column_size(i)); } } // multiple columns share the same symbolbase @@ -257,7 +208,6 @@ static TableSP odbcGetOrCreateTable(const Heap* heap, columnVecs[i] = Util::createVector(columnTypes[i], 0); } } - // printf("Util::createTable, %d symbol vector(s)\n", symbolCount); return Util::createTable(columnNames, columnTypes, 0, 1); } } @@ -266,9 +216,7 @@ ConstantSP odbcClose(Heap* heap, vector& args) { assert(heap); assert(args.size() == 1); if (args[0]->getType() != DT_RESOURCE) { - throw IllegalArgumentException( - "odbc::close", - "Unknown argument type, must be an odbc connection handle"); + throw IllegalArgumentException("odbc::close", "Unknown argument type, must be an odbc connection handle"); } ConstantSP csp = odbcGetConnection(heap, args, "odbc::close"); nanodbc::connection* cp = (nanodbc::connection*)(csp->getLong()); @@ -280,9 +228,7 @@ ConstantSP odbcConnect(Heap* heap, vector& args) { assert(heap); assert(args.size() == 1); if (args[0]->getType() != DT_STRING) { - throw IllegalArgumentException( - "odbc::connect", - "Unknown argument type, must be an odbc connection string"); + throw IllegalArgumentException("odbc::connect", "Unknown argument type, must be an odbc connection string"); } return odbcGetConnection(heap, args, "odbc::connect"); } @@ -291,8 +237,21 @@ ConstantSP odbcQuery(Heap* heap, vector& args) { const static int nanodbc_rowset_size = 4096; assert(heap); assert(args.size() >= 2); + Encoding srcEncoding = Encoding::UTF8; +#ifdef WINDOWS + if(args.size() == 4 && args[3]->getType() != DT_VOID) { + if(args[3]->getType() != DT_STRING || args[3]->getForm() != DF_SCALAR) { + throw IllegalArgumentException("odbc::query", "The 4th argument must be the name of encoding, e.g.(GB2312)"); + } + if(Util::lower(args[3]->getString()) == "gb2312") { + srcEncoding = Encoding::GB2312; + } else { + throw IllegalArgumentException("odbc::query", "The 4th argument must be the name of encoding, e.g.(GB2312)"); + } + } +#endif - string querySql = args[1]->getString(); + u16string querySql = utf8_to_utf16(args[1]->getString()); ConstantSP csp = odbcGetConnection(heap, args, "odbc::query"); nanodbc::connection* cp = (nanodbc::connection*)(csp->getLong()); @@ -317,8 +276,7 @@ ConstantSP odbcQuery(Heap* heap, vector& args) { columnTypes[i] = arrCol[i]->getType(); } - vector> buffers = - vector>(columns, vector(Util::BUF_SIZE)); + vector> buffers = vector>(columns, vector(Util::BUF_SIZE)); int curLine = 0, rowIdx = 0; char charBuf[Util::BUF_SIZE]; short shortBuf[Util::BUF_SIZE]; @@ -327,150 +285,120 @@ ConstantSP odbcQuery(Heap* heap, vector& args) { while (hasNext) { for (short col = 0; col < columns; ++col) { // nanodbc stores numeric & decimal types as strings - if (results.column_datatype(col) == SQL_DECIMAL || - results.column_datatype(col) == SQL_NUMERIC) { - string str; - results.get_ref(col, "", str); + if (results.column_datatype(col) == SQL_DECIMAL || results.column_datatype(col) == SQL_NUMERIC) { + u16string _str; + results.get_ref(col, u"", _str); + string str = utf16_to_utf8(_str); if (str != "") - sscanf(str.c_str(), "%lf", - &(buffers[col][curLine].doubleVal)); + sscanf(str.c_str(), "%lf", &(buffers[col][curLine].doubleVal)); else buffers[col][curLine].doubleVal = DBL_NMIN; } else { switch (columnTypes[col]) { case DT_BOOL: - buffers[col][curLine].charVal = - results.get(col, CHAR_MIN); + buffers[col][curLine].charVal = results.get(col, CHAR_MIN); if (buffers[col][curLine].charVal != CHAR_MIN) { - buffers[col][curLine].charVal = - !!buffers[col][curLine].charVal; + buffers[col][curLine].charVal = !!buffers[col][curLine].charVal; } break; case DT_CHAR: - buffers[col][curLine].charVal = - results.get(col, CHAR_MIN); + buffers[col][curLine].charVal = results.get(col, CHAR_MIN); break; case DT_SHORT: - buffers[col][curLine].shortVal = - results.get(col, SHRT_MIN); + buffers[col][curLine].shortVal = results.get(col, SHRT_MIN); break; case DT_INT: - buffers[col][curLine].intVal = - results.get(col, INT_MIN); + buffers[col][curLine].intVal = results.get(col, INT_MIN); break; case DT_LONG: - buffers[col][curLine].longVal = - results.get(col, LLONG_MIN); + buffers[col][curLine].longVal = results.get(col, LLONG_MIN); break; case DT_FLOAT: - buffers[col][curLine].floatVal = - results.get(col, FLT_NMIN); + buffers[col][curLine].floatVal = results.get(col, FLT_NMIN); break; case DT_DOUBLE: - buffers[col][curLine].doubleVal = - results.get(col, DBL_NMIN); + buffers[col][curLine].doubleVal = results.get(col, DBL_NMIN); break; case DT_DATE: { nanodbc::date fallback = {0, 0, 0}; nanodbc::date date = {0, 0, 0}; results.get_ref(col, fallback, date); - if (memcmp(&date, &fallback, sizeof(nanodbc::date)) == - 0) { + if (memcmp(&date, &fallback, sizeof(nanodbc::date)) == 0) { buffers[col][curLine].intVal = INT_MIN; } else { - buffers[col][curLine].intVal = Util::countDays( - date.year, date.month, date.day); + buffers[col][curLine].intVal = Util::countDays(date.year, date.month, date.day); } } break; case DT_TIMESTAMP: { nanodbc::timestamp fallback = {0, 0, 0, 0, 0, 0, 0}; nanodbc::timestamp ts = {0, 0, 0, 0, 0, 0, 0}; results.get_ref(col, fallback, ts); - if (memcmp(&ts, &fallback, - sizeof(nanodbc::timestamp)) == 0) { + if (memcmp(&ts, &fallback, sizeof(nanodbc::timestamp)) == 0) { buffers[col][curLine].longVal = LLONG_MIN; } else { // ts.fract is the number of billionths of a second. // https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/c-data-types - buffers[col][curLine].longVal = - Util::countDays(ts.year, ts.month, ts.day) * - 86400000ll + - ((ts.hour * 60 + ts.min) * 60 + ts.sec) * 1000 + - ts.fract / 1000000; + buffers[col][curLine].longVal = Util::countDays(ts.year, ts.month, ts.day) * 86400000ll + ((ts.hour * 60 + ts.min) * 60 + ts.sec) * 1000 + ts.fract / 1000000; } } break; case DT_SECOND: { nanodbc::time fallback = {0, 0, 0}; nanodbc::time ts = {0, 0, 0}; results.get_ref(col, fallback, ts); - if (memcmp(&ts, &fallback, - sizeof(nanodbc::time)) == 0) { + if (memcmp(&ts, &fallback, sizeof(nanodbc::time)) == 0) { buffers[col][curLine].intVal = INT_MIN; } else { - buffers[col][curLine].intVal = - (ts.hour * 60 + ts.min) * 60 + ts.sec; + buffers[col][curLine].intVal = (ts.hour * 60 + ts.min) * 60 + ts.sec; } } break; case DT_TIME: { nanodbc::time fallback = {0, 0, 0}; nanodbc::time ts = {0, 0, 0}; results.get_ref(col, fallback, ts); - if (memcmp(&ts, &fallback, - sizeof(nanodbc::time)) == 0) { + if (memcmp(&ts, &fallback, sizeof(nanodbc::time)) == 0) { buffers[col][curLine].intVal = INT_MIN; } else { // ts.fract is the number of billionths of a second. // https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/c-data-types - buffers[col][curLine].intVal = - ((ts.hour * 60 + ts.min) * 60 + ts.sec) * 1000; + buffers[col][curLine].intVal = ((ts.hour * 60 + ts.min) * 60 + ts.sec) * 1000; } } break; case DT_NANOTIME: { nanodbc::timestamp fallback = {0, 0, 0, 0, 0, 0, 0}; nanodbc::timestamp ts = {0, 0, 0, 0, 0, 0, 0}; results.get_ref(col, fallback, ts); - if (memcmp(&ts, &fallback, - sizeof(nanodbc::timestamp)) == 0) { + if (memcmp(&ts, &fallback, sizeof(nanodbc::timestamp)) == 0) { buffers[col][curLine].longVal = LLONG_MIN; } else { // ts.fract is the number of billionths of a second. // https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/c-data-types - buffers[col][curLine].longVal = - ((ts.hour * 60 + ts.min) * 60 + ts.sec) * - 1000000000ll + - ts.fract; + buffers[col][curLine].longVal = ((ts.hour * 60 + ts.min) * 60 + ts.sec) * 1000000000ll + ts.fract; } } break; case DT_NANOTIMESTAMP: { nanodbc::timestamp fallback = {0, 0, 0, 0, 0, 0, 0}; nanodbc::timestamp ts = {0, 0, 0, 0, 0, 0, 0}; results.get_ref(col, fallback, ts); - if (memcmp(&ts, &fallback, - sizeof(nanodbc::timestamp)) == 0) { + if (memcmp(&ts, &fallback, sizeof(nanodbc::timestamp)) == 0) { buffers[col][curLine].longVal = LLONG_MIN; } else { // ts.fract is the number of billionths of a second. // https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/c-data-types - buffers[col][curLine].longVal = - Util::countDays(ts.year, ts.month, ts.day) * - 86400000000000ll + - ((ts.hour * 60 + ts.min) * 60 + ts.sec) * - 1000000000ll + - ts.fract; + buffers[col][curLine].longVal = Util::countDays(ts.year, ts.month, ts.day) * 86400000000000ll + ((ts.hour * 60 + ts.min) * 60 + ts.sec) * 1000000000ll + ts.fract; } } break; case DT_SYMBOL: { SymbolBaseSP symbolBase = arrCol[col]->getSymbolBase(); - string str; - results.get_ref(col, "", str); - buffers[col][curLine].intVal = - symbolBase->findAndInsert(str); + u16string _str; + results.get_ref(col, u"", _str); + string str = utf16_to_utf8(_str); + buffers[col][curLine].intVal = symbolBase->findAndInsert(str); } break; default: - string str; - results.get_ref(col, "", str); - buffers[col][curLine].pointer = - new char[str.size() + 1]; + u16string _str; + results.get_ref(col, u"", _str); + string str = utf16_to_utf8(_str); + buffers[col][curLine].pointer = new char[str.size() + 1]; if (str == "") { *(buffers[col][curLine].pointer) = '\0'; } else { @@ -490,77 +418,60 @@ ConstantSP odbcQuery(Heap* heap, vector& args) { switch (rawType) { case DT_BOOL: if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { - for (int j = 0; j < curLine; ++j) - charBuf[j] = (bool)colBuf[j].doubleVal; + for (int j = 0; j < curLine; ++j) charBuf[j] = (bool)colBuf[j].doubleVal; } else { - for (int j = 0; j < curLine; ++j) - charBuf[j] = colBuf[j].charVal; + for (int j = 0; j < curLine; ++j) charBuf[j] = colBuf[j].charVal; } arrCol[col]->appendBool(charBuf, curLine); break; case DT_CHAR: if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { - for (int j = 0; j < curLine; ++j) - charBuf[j] = (char)colBuf[j].doubleVal; + for (int j = 0; j < curLine; ++j) charBuf[j] = (char)colBuf[j].doubleVal; } else { - for (int j = 0; j < curLine; ++j) - charBuf[j] = colBuf[j].charVal; + for (int j = 0; j < curLine; ++j) charBuf[j] = colBuf[j].charVal; } arrCol[col]->appendChar(charBuf, curLine); break; case DT_SHORT: if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { - for (int j = 0; j < curLine; ++j) - shortBuf[j] = (short)colBuf[j].doubleVal; + for (int j = 0; j < curLine; ++j) shortBuf[j] = (short)colBuf[j].doubleVal; } else { - for (int j = 0; j < curLine; ++j) - shortBuf[j] = colBuf[j].shortVal; + for (int j = 0; j < curLine; ++j) shortBuf[j] = colBuf[j].shortVal; } arrCol[col]->appendShort(shortBuf, curLine); break; case DT_INT: if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { - for (int j = 0; j < curLine; ++j) - buf[j].intVal = (int)colBuf[j].doubleVal; + for (int j = 0; j < curLine; ++j) buf[j].intVal = (int)colBuf[j].doubleVal; } else { - for (int j = 0; j < curLine; ++j) - buf[j].intVal = colBuf[j].intVal; + for (int j = 0; j < curLine; ++j) buf[j].intVal = colBuf[j].intVal; } arrCol[col]->appendInt((int*)buf, curLine); break; case DT_LONG: if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { - for (int j = 0; j < curLine; ++j) - colBuf[j].longVal = - (long long)colBuf[j].doubleVal; + for (int j = 0; j < curLine; ++j) colBuf[j].longVal = (long long)colBuf[j].doubleVal; } - arrCol[col]->appendLong((long long*)(&colBuf[0]), - curLine); + arrCol[col]->appendLong((long long*)(&colBuf[0]), curLine); break; case DT_FLOAT: if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { - for (int j = 0; j < curLine; ++j) - buf[j].floatVal = - (long long)colBuf[j].doubleVal; + for (int j = 0; j < curLine; ++j) buf[j].floatVal = (long long)colBuf[j].doubleVal; } else { - for (int j = 0; j < curLine; ++j) - buf[j].floatVal = colBuf[j].floatVal; + for (int j = 0; j < curLine; ++j) buf[j].floatVal = colBuf[j].floatVal; } arrCol[col]->appendFloat((float*)buf, curLine); break; case DT_DOUBLE: - arrCol[col]->appendDouble((double*)(&colBuf[0]), - curLine); + arrCol[col]->appendDouble((double*)(&colBuf[0]), curLine); break; case DT_STRING: #ifdef BIT32 if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { for (int j = 0; j < curLine; ++j) { char strBuf[120]; - int nbytes = - snprintf(strBuf, sizeof(strBuf), "%lf", - colBuf[j].doubleVal); + int nbytes = snprintf(strBuf, sizeof(strBuf), "%lf", colBuf[j].doubleVal); if (colBuf[j].doubleVal != DBL_NMIN) { buf[j].pointer = new char[nbytes + 1]; strcpy(buf[j].pointer, strBuf); @@ -570,19 +481,15 @@ ConstantSP odbcQuery(Heap* heap, vector& args) { } } } else { - for (int j = 0; j < curLine; ++j) - buf[j].pointer = colBuf[j].pointer; + for (int j = 0; j < curLine; ++j) buf[j].pointer = colBuf[j].pointer; } arrCol[col]->appendString((char**)buf, curLine); - for (int j = 0; j < curLine; ++j) - delete[] buf[j].pointer; + for (int j = 0; j < curLine; ++j) delete[] buf[j].pointer; #else if (sqltype == SQL_DECIMAL || sqltype == SQL_NUMERIC) { for (int j = 0; j < curLine; ++j) { char strBuf[120]; - int nbytes = - snprintf(strBuf, sizeof(strBuf), "%lf", - colBuf[j].doubleVal); + int nbytes = snprintf(strBuf, sizeof(strBuf), "%lf", colBuf[j].doubleVal); if (colBuf[j].doubleVal != DBL_NMIN) { buffers[col][j].pointer = new char[nbytes + 1]; strcpy(buffers[col][j].pointer, strBuf); @@ -592,24 +499,19 @@ ConstantSP odbcQuery(Heap* heap, vector& args) { } } } - arrCol[col]->appendString((char**)(&buffers[col][0]), - curLine); - for (int j = 0; j < curLine; ++j) - delete[] buffers[col][j].pointer; + arrCol[col]->appendString((char**)(&buffers[col][0]), curLine); + for (int j = 0; j < curLine; ++j) delete[] buffers[col][j].pointer; #endif break; default: assert(false); } } - // printf("appended %d rows\n", curLine); curLine = 0; } } for (short i = 0; i < columns; ++i) { arrCol[i]->setNullFlag(arrCol[i]->hasNull()); - // printf("total %d rows, %d rows appended at column %s\n", rowIdx, - // arrCol[i]->size(), table->getColumnName(i).c_str()); } if (arrCol[0]->size() > 0) { @@ -623,19 +525,18 @@ ConstantSP odbcQuery(Heap* heap, vector& args) { return table; } -ConstantSP odbcExecute(Heap *heap, vector &args){ +ConstantSP odbcExecute(Heap* heap, vector& args) { assert(heap); assert(args.size() >= 2); - string querySql = args[1]->getString(); + u16string querySql = utf8_to_utf16(args[1]->getString()); ConstantSP csp = odbcGetConnection(heap, args, "odbc::query"); - nanodbc::connection *cp = (nanodbc::connection *)(csp->getLong()); + nanodbc::connection* cp = (nanodbc::connection*)(csp->getLong()); try { nanodbc::just_execute(*cp, querySql); - } - catch (const runtime_error &e){ - const char *fmt = "Executed query [%s]: %s"; + } catch (const runtime_error& e) { + const char* fmt = "Executed query [%s]: %s"; vector errorMsgBuf(querySql.size() + strlen(e.what()) + strlen(fmt)); sprintf(errorMsgBuf.data(), fmt, querySql.c_str(), e.what()); throw TableRuntimeException(errorMsgBuf.data()); diff --git a/odbc/src/cvt.cpp b/odbc/src/cvt.cpp new file mode 100644 index 00000000..f6300316 --- /dev/null +++ b/odbc/src/cvt.cpp @@ -0,0 +1,58 @@ +#include "cvt.h" +#include +#include + +#if defined(__GNUC__) && (__GNUC__ < 5) +#include "utfcpp/utf8.h" +#else +#include +#include +#endif + + +std::u16string utf8_to_utf16(const char* first, const char* last) { +#if defined(__GNUC__) && (__GNUC__ < 5) + std::u16string ret; + try { + utf8::utf8to16(first, last, std::back_inserter(ret)); + } catch(utf8::not_enough_room& ex) { + // pass + } + return ret; +#else + static thread_local std::wstring_convert, char16_t> conv; + return conv.from_bytes(first, last); +#endif +} + +std::u16string utf8_to_utf16(const std::string& utf8) { + return utf8_to_utf16(utf8.c_str(), utf8.c_str() + utf8.size()); +} + +std::string utf16_to_utf8(const char16_t* first, const char16_t* last) { +#if defined(__GNUC__) && (__GNUC__ < 5) + std::string ret; + utf8::utf16to8(first, last, std::back_inserter(ret)); + return ret; +#else + static thread_local std::wstring_convert, char16_t> conv; + return conv.to_bytes(first, last); +#endif +} + +std::string utf16_to_utf8(const std::u16string& utf16) { + return utf16_to_utf8(utf16.c_str(), utf16.c_str() + utf16.size()); +} + +#ifdef WINDOWS +int gb2312_to_utf8(const char* gb2312, char* utf8) { + int len = MultiByteToWideChar(CP_ACP, 0, gb2312, -1, NULL, 0); + wchar_t *wstr = new wchar_t[len + 1]; + memset(wstr, 0, len + 1); + MultiByteToWideChar(CP_ACP, 0, gb2312, -1, wstr, len); + len = WideCharToMultiByte(CP_UTF8, 0, wstr, -1, NULL, 0, NULL, NULL); + WideCharToMultiByte(CP_UTF8, 0, wstr, -1, utf8, len, NULL, NULL); + if (wstr) delete[] wstr; + return len; +} +#endif diff --git a/odbc/src/cvt.h b/odbc/src/cvt.h new file mode 100644 index 00000000..d963a3ab --- /dev/null +++ b/odbc/src/cvt.h @@ -0,0 +1,32 @@ +#ifndef CVT_H_ +#define CVT_H_ +#include +#ifdef WINDOWS +#include +#else +#include +#endif + +enum class Encoding { + UTF8, + GB2312 +}; + +std::u16string utf8_to_utf16(const std::string& utf8); +std::u16string utf8_to_utf16(const char* first, const char* last); +std::string utf16_to_utf8(const std::u16string& utf16); +std::string utf16_to_utf8(const char16_t* from, const char16_t* to); + + +int gb2312_to_utf8(const char* src, char* utf8); +template +int bytesToUTF8(const char* src, char* utf8); + +#ifdef WINDOWS +template <> +int bytesToUTF8(const char* src, char* utf8) { + return gb2312_to_utf8(src, utf8); +} +#endif + +#endif \ No newline at end of file diff --git a/odbc/src/nanodbc.cpp b/odbc/src/nanodbc.cpp index 41941ef6..10cae775 100644 --- a/odbc/src/nanodbc.cpp +++ b/odbc/src/nanodbc.cpp @@ -1,4 +1,4 @@ -/// \file nanodbc.cpp Implementation details. +/// \file nanodbc.cpp Implementation details. #ifndef DOXYGEN // ASCII art banners are helpful for code editors with a minimap display. @@ -63,7 +63,7 @@ // Microsoft has -150 thru -199 reserved for Microsoft SQL Server Native Client driver usage. // Originally, defined in sqlncli.h (old SQL Server Native Client driver) // and msodbcsql.h (new Microsoft ODBC Driver for SQL Server) -// See https://github.com/lexicalunit/nanodbc/issues/226 +// See https://github.com/nanodbc/nanodbc/issues/18 #ifndef SQL_SS_VARIANT #define SQL_SS_VARIANT (-150) #endif @@ -93,6 +93,16 @@ #define SQL_NVARCHAR (-10) #endif +// SQL_SS_LENGTH_UNLIMITED is used to describe the max length of +// VARCHAR(max), VARBINARY(max), NVARCHAR(max), and XML columns +#ifndef SQL_SS_LENGTH_UNLIMITED +#define SQL_SS_LENGTH_UNLIMITED (0) +#endif + +// Max length of DBVARBINARY and DBVARCHAR, etc. +1 for zero byte +// MSDN: Large value data types are those that exceed the maximum row size of 8 KB +#define SQLSERVER_DBMAXCHAR (8000 + 1) + // Default to ODBC version defined by NANODBC_ODBC_VERSION if provided. #ifndef NANODBC_ODBC_VERSION #ifdef SQL_OV_ODBC3_80 @@ -116,6 +126,10 @@ // MARK: Unicode - // clang-format on +// Import string types defined in header file, so we don't have to type nanodbc:: everywhere +using nanodbc::wide_char_t; +using nanodbc::wide_string; + #ifdef NANODBC_ENABLE_UNICODE #define NANODBC_FUNC(f) f##W #define NANODBC_SQLCHAR SQLWCHAR @@ -125,18 +139,14 @@ #endif #ifdef NANODBC_USE_IODBC_WIDE_STRINGS -typedef std::u32string wide_string; #define NANODBC_CODECVT_TYPE std::codecvt_utf8 #else #ifdef _MSC_VER -typedef std::wstring wide_string; #define NANODBC_CODECVT_TYPE std::codecvt_utf8_utf16 #else -typedef std::u16string wide_string; #define NANODBC_CODECVT_TYPE std::codecvt_utf8_utf16 #endif #endif -typedef wide_string::value_type wide_char_t; #if defined(_MSC_VER) #ifndef NANODBC_ENABLE_UNICODE @@ -203,6 +213,7 @@ typedef wide_string::value_type wide_char_t; // MARK: Error Handling - // clang-format on +#include "cvt.h" namespace { #ifdef NANODBC_ODBC_API_DEBUG @@ -239,84 +250,101 @@ inline bool success(RETCODE rc) return rc == SQL_SUCCESS || rc == SQL_SUCCESS_WITH_INFO; } -// Returns the array size. -template -inline std::size_t arrlen(T (&)[N]) +#if __cpp_lib_nonmember_container_access >= 201411 || _MSC_VER +using std::size; +#else +template +constexpr std::size_t size(const T (&array)[N]) noexcept { return N; } +#endif -// Operates like strlen() on a character array. -template -inline std::size_t strarrlen(T (&a)[N]) +template +inline std::size_t size(NANODBC_SQLCHAR const (&array)[N]) noexcept { - const T* s = &a[0]; - std::size_t i = 0; - while (*s++ && i < N) - i++; - return i; + auto const n = std::char_traits::length(array); + NANODBC_ASSERT(n < N); + return n < N ? n : N - 1; +} + +template +inline void convert(T const* beg, size_t n, std::basic_string& out) +{ + out.assign(beg, n); } -inline void convert(const wide_string& in, std::string& out) +inline void convert(wide_char_t const* beg, size_t n, std::string& out) { #ifdef NANODBC_ENABLE_BOOST using boost::locale::conv::utf_to_utf; - out = utf_to_utf(in.c_str(), in.c_str() + in.size()); -#elif defined(__GNUC__) && (__GNUC__ < 5) - std::vector characters(in.begin(), in.end()); - const wchar_t * source = characters.data(); - size_t size = wcsnrtombs(nullptr, &source, characters.size(), 0, nullptr); - if (size == std::string::npos) - throw std::range_error("UTF-16 -> UTF-8 conversion error"); - out.resize(size); - wcsnrtombs(&out[0], &source, characters.size(), out.length(), nullptr); -#elif defined(_MSC_VER) && (_MSC_VER >= 1900) - // Workaround for confirmed bug in VS2015 and VS2017 too - // See: https://connect.microsoft.com/VisualStudio/Feedback/Details/1403302 - auto p = reinterpret_cast(in.data()); - out = std::wstring_convert, unsigned short>().to_bytes( - p, p + in.size()); + out = utf_to_utf(beg, beg + n); +#elif defined(_MSC_VER) && (_MSC_VER == 1900) + // Workaround for confirmed bug in VS2015. See: + // https://connect.microsoft.com/VisualStudio/Feedback/Details/1403302 + // https://social.msdn.microsoft.com/Forums/en-US/8f40dcd8-c67f-4eba-9134-a19b9178e481/vs-2015-rc-linker-stdcodecvt-error + // Why static? http://stackoverflow.com/questions/26196686/utf8-utf16-codecvt-poor-performance + static thread_local std::wstring_convert, unsigned short> + converter; + out = converter.to_bytes( + reinterpret_cast(beg), + reinterpret_cast(beg + n)); #else - out = std::wstring_convert, wide_char_t>().to_bytes(in); + out = utf16_to_utf8(beg, beg + n); #endif } -#ifdef NANODBC_ENABLE_UNICODE -inline void convert(const std::string& in, wide_string& out) +inline void convert(char const* beg, size_t n, wide_string& out) { #ifdef NANODBC_ENABLE_BOOST using boost::locale::conv::utf_to_utf; - out = utf_to_utf(in.c_str(), in.c_str() + in.size()); -#elif defined(__GNUC__) && (__GNUC__ < 5) - size_t size = mbsnrtowcs(nullptr, in.data(), in.length(), 0, nullptr); - if (size == std::string::npos) - throw std::range_error("UTF-8 -> UTF-16 conversion error"); - std::vector characters(size); - const char * source = in.data(); - mbsnrtowcs(&characters[0], &source, in.length(), characters.size(), nullptr); - out = std::string(characters.begin(), characters.end()); -#elif defined(_MSC_VER) && (_MSC_VER >= 1900) - // Workaround for confirmed bug in VS2015 and VS2017 too - // See: https://connect.microsoft.com/VisualStudio/Feedback/Details/1403302 - auto s = - std::wstring_convert, unsigned short>().from_bytes(in); + out = utf_to_utf(beg, beg + n); +#elif defined(_MSC_VER) && (_MSC_VER == 1900) + // Workaround for confirmed bug in VS2015. See: + // https://connect.microsoft.com/VisualStudio/Feedback/Details/1403302 + // https://social.msdn.microsoft.com/Forums/en-US/8f40dcd8-c67f-4eba-9134-a19b9178e481/vs-2015-rc-linker-stdcodecvt-error + // Why static? http://stackoverflow.com/questions/26196686/utf8-utf16-codecvt-poor-performance + static thread_local std::wstring_convert, unsigned short> + converter; + auto s = converter.from_bytes(beg, beg + n); auto p = reinterpret_cast(s.data()); out.assign(p, p + s.size()); #else - out = std::wstring_convert, wide_char_t>().from_bytes(in); + +#ifdef WINDOWS + char* utf8 = new char[n*2]; + int len = bytesToUTF8(beg, utf8); + out = utf8_to_utf16(utf8, utf8 + len); + delete [] utf8; +#else + out = utf8_to_utf16(beg, beg + n); +#endif #endif } -inline void convert(const wide_string& in, wide_string& out) +template +inline void convert(char const* beg, std::basic_string& out) { - out = in; + convert(beg, std::strlen(beg), out); } -#else -inline void convert(const std::string& in, std::string& out) + +template +inline void convert(wchar_t const* beg, std::basic_string& out) { - out = in; + convert(beg, std::wcslen(beg), out); +} + +template +inline void convert(std::basic_string&& in, std::basic_string& out) +{ + out.assign(in); +} + +template +inline void convert(std::basic_string const& in, std::basic_string& out) +{ + convert(in.data(), in.size(), out); } -#endif // Attempts to get the most recent ODBC error as a string. // Always returns std::string, even in unicode mode. @@ -329,9 +357,9 @@ recent_error(SQLHANDLE handle, SQLSMALLINT handle_type, long& native, std::strin sql_message[0] = '\0'; SQLINTEGER i = 1; - SQLINTEGER native_error; - SQLSMALLINT total_bytes; - NANODBC_SQLCHAR sql_state[6]; + SQLINTEGER native_error = 0; + SQLSMALLINT total_bytes = 0; + NANODBC_SQLCHAR sql_state[6] = {0}; RETCODE rc; do @@ -349,7 +377,7 @@ recent_error(SQLHANDLE handle, SQLSMALLINT handle_type, long& native, std::strin &total_bytes); if (success(rc) && total_bytes > 0) - sql_message.resize(total_bytes + 1); + sql_message.resize(static_cast(total_bytes) + 1); if (rc == SQL_NO_DATA) break; @@ -368,7 +396,7 @@ recent_error(SQLHANDLE handle, SQLSMALLINT handle_type, long& native, std::strin if (!success(rc)) { - convert(result, rvalue); + convert(std::move(result), rvalue); return rvalue; } @@ -386,8 +414,17 @@ recent_error(SQLHANDLE handle, SQLSMALLINT handle_type, long& native, std::strin #endif } while (rc != SQL_NO_DATA); - convert(result, rvalue); - state = std::string(&sql_state[0], &sql_state[arrlen(sql_state) - 1]); + convert(std::move(result), rvalue); + if (size(sql_state) > 0) + { + state.clear(); + state.reserve(size(sql_state) - 1); + for (std::size_t idx = 0; idx != size(sql_state) - 1; ++idx) + { + state.push_back(static_cast(sql_state[idx])); + } + } + native = native_error; std::string status = state; status += ": "; @@ -402,6 +439,7 @@ recent_error(SQLHANDLE handle, SQLSMALLINT handle_type, long& native, std::strin } // namespace +#ifndef NANODBC_DISABLE_NANODBC_NAMESPACE_FOR_INTERNAL_TESTS namespace nanodbc { @@ -410,7 +448,7 @@ type_incompatible_error::type_incompatible_error() { } -const char* type_incompatible_error::what() const NANODBC_NOEXCEPT +const char* type_incompatible_error::what() const noexcept { return std::runtime_error::what(); } @@ -420,7 +458,7 @@ null_access_error::null_access_error() { } -const char* null_access_error::what() const NANODBC_NOEXCEPT +const char* null_access_error::what() const noexcept { return std::runtime_error::what(); } @@ -430,7 +468,7 @@ index_range_error::index_range_error() { } -const char* index_range_error::what() const NANODBC_NOEXCEPT +const char* index_range_error::what() const noexcept { return std::runtime_error::what(); } @@ -440,12 +478,12 @@ programming_error::programming_error(const std::string& info) { } -const char* programming_error::what() const NANODBC_NOEXCEPT +const char* programming_error::what() const noexcept { return std::runtime_error::what(); } -database_error::database_error(void* handle, short handle_type, const std::string& info) +database_error::database_error(SQLHANDLE handle, short handle_type, const std::string& info) : std::runtime_error(info) , native_error(0) , sql_state("00000") @@ -454,17 +492,17 @@ database_error::database_error(void* handle, short handle_type, const std::strin recent_error(handle, handle_type, native_error, sql_state); } -const char* database_error::what() const NANODBC_NOEXCEPT +const char* database_error::what() const noexcept { return message.c_str(); } -const long database_error::native() const NANODBC_NOEXCEPT +const long database_error::native() const noexcept { return native_error; } -const std::string database_error::state() const NANODBC_NOEXCEPT +const std::string database_error::state() const noexcept { return sql_state; } @@ -473,9 +511,14 @@ const std::string database_error::state() const NANODBC_NOEXCEPT // Throwing exceptions using NANODBC_THROW_DATABASE_ERROR enables file name // and line numbers to be inserted into the error message. Useful for debugging. +#ifdef NANODBC_THROW_NO_SOURCE_LOCATION +#define NANODBC_THROW_DATABASE_ERROR(handle, handle_type) \ + throw nanodbc::database_error(handle, handle_type, "ODBC database error: ") /**/ +#else #define NANODBC_THROW_DATABASE_ERROR(handle, handle_type) \ throw nanodbc::database_error( \ handle, handle_type, __FILE__ ":" NANODBC_STRINGIZE(__LINE__) ": ") /**/ +#endif // clang-format off // 8888888b. 888 d8b 888 @@ -592,23 +635,27 @@ struct sql_ctype }; template <> -struct sql_ctype +struct sql_ctype { -#ifdef NANODBC_ENABLE_UNICODE static const SQLSMALLINT value = SQL_C_WCHAR; -#else - static const SQLSMALLINT value = SQL_C_CHAR; -#endif }; template <> -struct sql_ctype +struct sql_ctype { -#ifdef NANODBC_ENABLE_UNICODE static const SQLSMALLINT value = SQL_C_WCHAR; -#else +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_CHAR; +}; + +template <> +struct sql_ctype +{ static const SQLSMALLINT value = SQL_C_CHAR; -#endif }; template <> @@ -675,10 +722,10 @@ struct bound_parameter { bound_parameter() = default; + SQLULEN size_ = 0; // SQL data size of column or expression inbytes or characters SQLUSMALLINT index_ = 0; // Zero-based index of parameter marker SQLSMALLINT iotype_ = 0; // Input/Output type of parameter SQLSMALLINT type_ = 0; // SQL data type of parameter - SQLULEN size_ = 0; // SQL data size of column or expression inbytes or characters SQLSMALLINT scale_ = 0; // decimal digits of column or expression }; @@ -699,9 +746,23 @@ struct bound_buffer std::size_t value_size_ = 0; // Size of single value (max size). Zero, if ignored. }; -// Allocates the native ODBC handles. -inline void allocate_environment_handle(SQLHENV& env) +inline void deallocate_handle(SQLHANDLE& handle, short handle_type) { + if (!handle) + return; + + RETCODE rc; + NANODBC_CALL_RC(SQLFreeHandle, rc, handle_type, handle); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(handle, handle_type); + handle = nullptr; +} + +inline void allocate_env_handle(SQLHENV& env) +{ + if (env) + return; + RETCODE rc; NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_ENV, SQL_NULL_HANDLE, &env); if (!success(rc)) @@ -721,18 +782,19 @@ inline void allocate_environment_handle(SQLHENV& env) } catch (...) { - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env); + deallocate_handle(env, SQL_HANDLE_ENV); throw; } } -inline void allocate_handle(SQLHENV& env, SQLHDBC& conn) +inline void allocate_dbc_handle(SQLHDBC& conn, SQLHENV env) { - allocate_environment_handle(env); + NANODBC_ASSERT(env); + if (conn) + return; try { - NANODBC_ASSERT(env); RETCODE rc; NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_DBC, env, &conn); if (!success(rc)) @@ -740,7 +802,7 @@ inline void allocate_handle(SQLHENV& env, SQLHDBC& conn) } catch (...) { - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env); + deallocate_handle(conn, SQL_HANDLE_DBC); throw; } } @@ -772,56 +834,55 @@ class connection::connection_impl connection_impl& operator=(const connection_impl&) = delete; connection_impl() - : env_(0) - , conn_(0) + : env_(nullptr) + , dbc_(nullptr) , connected_(false) , transactions_(0) , rollback_(false) { - allocate_handle(env_, conn_); } connection_impl(const string& dsn, const string& user, const string& pass, long timeout) - : env_(0) - , conn_(0) + : env_(nullptr) + , dbc_(nullptr) , connected_(false) , transactions_(0) , rollback_(false) { - allocate_handle(env_, conn_); + allocate(); + try { connect(dsn, user, pass, timeout); } catch (...) { - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_DBC, conn_); - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env_); + deallocate(); throw; } } connection_impl(const string& connection_string, long timeout) - : env_(0) - , conn_(0) + : env_(nullptr) + , dbc_(nullptr) , connected_(false) , transactions_(0) , rollback_(false) { - allocate_handle(env_, conn_); + allocate(); + try { connect(connection_string, timeout); } catch (...) { - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_DBC, conn_); - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env_); + deallocate(); throw; } } - ~connection_impl() NANODBC_NOEXCEPT + ~connection_impl() noexcept { try { @@ -831,48 +892,63 @@ class connection::connection_impl { // ignore exceptions thrown during disconnect } - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_DBC, conn_); - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env_); + deallocate(); + } + + void allocate() + { + allocate_env_handle(env_); + allocate_dbc_handle(dbc_, env_); + } + + void deallocate() + { + deallocate_handle(dbc_, SQL_HANDLE_DBC); + deallocate_handle(env_, SQL_HANDLE_ENV); } #if !defined(NANODBC_DISABLE_ASYNC) && defined(SQL_ATTR_ASYNC_DBC_EVENT) void enable_async(void* event_handle) { + NANODBC_ASSERT(dbc_); + RETCODE rc; NANODBC_CALL_RC( SQLSetConnectAttr, rc, - conn_, + dbc_, SQL_ATTR_ASYNC_DBC_FUNCTIONS_ENABLE, (SQLPOINTER)SQL_ASYNC_DBC_ENABLE_ON, SQL_IS_INTEGER); if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); NANODBC_CALL_RC( - SQLSetConnectAttr, rc, conn_, SQL_ATTR_ASYNC_DBC_EVENT, event_handle, SQL_IS_POINTER); + SQLSetConnectAttr, rc, dbc_, SQL_ATTR_ASYNC_DBC_EVENT, event_handle, SQL_IS_POINTER); if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); } void async_complete() { + NANODBC_ASSERT(dbc_); + RETCODE rc, arc; - NANODBC_CALL_RC(SQLCompleteAsync, rc, SQL_HANDLE_DBC, conn_, &arc); + NANODBC_CALL_RC(SQLCompleteAsync, rc, SQL_HANDLE_DBC, dbc_, &arc); if (!success(rc) || !success(arc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); connected_ = true; NANODBC_CALL_RC( SQLSetConnectAttr, rc, - conn_, + dbc_, SQL_ATTR_ASYNC_DBC_FUNCTIONS_ENABLE, (SQLPOINTER)SQL_ASYNC_DBC_ENABLE_OFF, SQL_IS_INTEGER); if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); } #endif // !NANODBC_DISABLE_ASYNC && SQL_ATTR_ASYNC_DBC_EVENT @@ -883,21 +959,28 @@ class connection::connection_impl long timeout, void* event_handle = nullptr) { + allocate_env_handle(env_); disconnect(); - RETCODE rc; - NANODBC_CALL_RC(SQLFreeHandle, rc, SQL_HANDLE_DBC, conn_); - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); - - NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_DBC, env_, &conn_); - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(env_, SQL_HANDLE_ENV); + deallocate_handle(dbc_, SQL_HANDLE_DBC); + allocate_dbc_handle(dbc_, env_); - NANODBC_CALL_RC( - SQLSetConnectAttr, rc, conn_, SQL_LOGIN_TIMEOUT, (SQLPOINTER)(std::intptr_t)timeout, 0); - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + RETCODE rc; + if (timeout != 0) + { + // Avoid to set the timeout to 0 (no timeout). + // This is a workaround for the Oracle ODBC Driver (11.1), as this + // operation is not supported by the Driver. + NANODBC_CALL_RC( + SQLSetConnectAttr, + rc, + dbc_, + SQL_LOGIN_TIMEOUT, + (SQLPOINTER)(std::intptr_t)timeout, + 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); + } #if !defined(NANODBC_DISABLE_ASYNC) && defined(SQL_ATTR_ASYNC_DBC_EVENT) if (event_handle != nullptr) @@ -907,7 +990,7 @@ class connection::connection_impl NANODBC_CALL_RC( NANODBC_FUNC(SQLConnect), rc, - conn_, + dbc_, (NANODBC_SQLCHAR*)dsn.c_str(), SQL_NTS, !user.empty() ? (NANODBC_SQLCHAR*)user.c_str() : 0, @@ -915,7 +998,7 @@ class connection::connection_impl !pass.empty() ? (NANODBC_SQLCHAR*)pass.c_str() : 0, SQL_NTS); if (!success(rc) && (event_handle == nullptr || rc != SQL_STILL_EXECUTING)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); connected_ = success(rc); @@ -925,21 +1008,28 @@ class connection::connection_impl RETCODE connect(const string& connection_string, long timeout, void* event_handle = nullptr) { + allocate_env_handle(env_); disconnect(); - RETCODE rc; - NANODBC_CALL_RC(SQLFreeHandle, rc, SQL_HANDLE_DBC, conn_); - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + deallocate_handle(dbc_, SQL_HANDLE_DBC); + allocate_dbc_handle(dbc_, env_); - NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_DBC, env_, &conn_); - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(env_, SQL_HANDLE_ENV); - - NANODBC_CALL_RC( - SQLSetConnectAttr, rc, conn_, SQL_LOGIN_TIMEOUT, (SQLPOINTER)(std::intptr_t)timeout, 0); - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + RETCODE rc; + if (timeout != 0) + { + // Avoid to set the timeout to 0 (no timeout). + // This is a workaround for the Oracle ODBC Driver (11.1), as this + // operation is not supported by the Driver. + NANODBC_CALL_RC( + SQLSetConnectAttr, + rc, + dbc_, + SQL_LOGIN_TIMEOUT, + (SQLPOINTER)(std::intptr_t)timeout, + 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); + } #if !defined(NANODBC_DISABLE_ASYNC) && defined(SQL_ATTR_ASYNC_DBC_EVENT) if (event_handle != nullptr) @@ -949,7 +1039,7 @@ class connection::connection_impl NANODBC_CALL_RC( NANODBC_FUNC(SQLDriverConnect), rc, - conn_, + dbc_, 0, (NANODBC_SQLCHAR*)connection_string.c_str(), SQL_NTS, @@ -958,7 +1048,7 @@ class connection::connection_impl nullptr, SQL_DRIVER_NOPROMPT); if (!success(rc) && (event_handle == nullptr || rc != SQL_STILL_EXECUTING)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); connected_ = success(rc); @@ -972,16 +1062,16 @@ class connection::connection_impl if (connected()) { RETCODE rc; - NANODBC_CALL_RC(SQLDisconnect, rc, conn_); + NANODBC_CALL_RC(SQLDisconnect, rc, dbc_); if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); } connected_ = false; } std::size_t transactions() const { return transactions_; } - void* native_dbc_handle() const { return conn_; } + void* native_dbc_handle() const { return dbc_; } void* native_env_handle() const { return env_; } @@ -1006,14 +1096,14 @@ class connection::connection_impl NANODBC_CALL_RC( NANODBC_FUNC(SQLGetConnectAttr), rc, - conn_, + dbc_, SQL_ATTR_CURRENT_CATALOG, name, sizeof(name) / sizeof(NANODBC_SQLCHAR), &length); if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); - return string(&name[0], &name[strarrlen(name)]); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); + return string(&name[0], &name[size(name)]); } std::size_t ref_transaction() { return ++transactions_; } @@ -1034,7 +1124,7 @@ class connection::connection_impl T get_info_impl(short info_type) const; HENV env_; - HDBC conn_; + HDBC dbc_; bool connected_; std::size_t transactions_; bool rollback_; // if true, this connection is marked for eventual transaction rollback @@ -1045,9 +1135,9 @@ T connection::connection_impl::get_info_impl(short info_type) const { T value; RETCODE rc; - NANODBC_CALL_RC(NANODBC_FUNC(SQLGetInfo), rc, conn_, info_type, &value, 0, nullptr); + NANODBC_CALL_RC(NANODBC_FUNC(SQLGetInfo), rc, dbc_, info_type, &value, 0, nullptr); if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); return value; } @@ -1060,14 +1150,14 @@ string connection::connection_impl::get_info_impl(short info_type) const NANODBC_CALL_RC( NANODBC_FUNC(SQLGetInfo), rc, - conn_, + dbc_, info_type, value, sizeof(value) / sizeof(NANODBC_SQLCHAR), &length); if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); - return string(&value[0], &value[strarrlen(value)]); + NANODBC_THROW_DATABASE_ERROR(dbc_, SQL_HANDLE_DBC); + return string(&value[0], &value[size(value)]); } string connection::connection_impl::dbms_name() const @@ -1141,7 +1231,7 @@ class transaction::transaction_impl conn_.ref_transaction(); } - ~transaction_impl() NANODBC_NOEXCEPT + ~transaction_impl() noexcept { if (!committed_) { @@ -1180,7 +1270,7 @@ class transaction::transaction_impl } } - void rollback() NANODBC_NOEXCEPT + void rollback() noexcept { if (committed_) return; @@ -1240,6 +1330,7 @@ class statement::statement_impl , open_(false) , conn_() , bind_len_or_null_() + , wide_string_data_() , string_data_() , binary_data_() #if defined(NANODBC_DO_ASYNC_IMPL) @@ -1256,6 +1347,7 @@ class statement::statement_impl , open_(false) , conn_() , bind_len_or_null_() + , wide_string_data_() , string_data_() , binary_data_() #if defined(NANODBC_DO_ASYNC_IMPL) @@ -1267,13 +1359,13 @@ class statement::statement_impl prepare(conn, query, timeout); } - ~statement_impl() NANODBC_NOEXCEPT + ~statement_impl() noexcept { if (open() && connected()) { NANODBC_CALL(SQLCancel, stmt_); reset_parameters(); - NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_STMT, stmt_); + deallocate_handle(stmt_, SQL_HANDLE_STMT); } } @@ -1284,7 +1376,12 @@ class statement::statement_impl NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_STMT, conn.native_dbc_handle(), &stmt_); open_ = success(rc); if (!open_) - NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + { + if (!stmt_) + NANODBC_THROW_DATABASE_ERROR(conn.native_dbc_handle(), SQL_HANDLE_DBC); + else + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } conn_ = conn; } @@ -1308,10 +1405,7 @@ class statement::statement_impl NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); reset_parameters(); - - NANODBC_CALL_RC(SQLFreeHandle, rc, SQL_HANDLE_STMT, stmt_); - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + deallocate_handle(stmt_, SQL_HANDLE_STMT); } open_ = false; @@ -1660,7 +1754,7 @@ class statement::statement_impl return cols; } - void reset_parameters() NANODBC_NOEXCEPT { NANODBC_CALL(SQLFreeStmt, stmt_, SQL_RESET_PARAMS); } + void reset_parameters() noexcept { NANODBC_CALL(SQLFreeStmt, stmt_, SQL_RESET_PARAMS); } short parameters() const { @@ -1770,7 +1864,51 @@ class statement::statement_impl } // calls actual ODBC bind parameter function - template + template ::value, int>::type = 0> + void bind_parameter(bound_parameter const& param, bound_buffer& buffer) + { + NANODBC_ASSERT(buffer.value_size_ > 0 || param.size_ > 0); + + auto value_size = buffer.value_size_; + if (value_size == 0) + value_size = param.size_; + + auto param_size = param.size_; + if (value_size > param_size) + { + // Parameter size reported by SQLDescribeParam for Large Objects: + // - For SQL VARBINARY(MAX), it is Zero which actually means SQL_SS_LENGTH_UNLIMITED. + // - For SQL UDT (eg. GEOMETRY), it may be driver-specific max limit (eg. SQL Server is + // DBMAXCHAR=8000 bytes). + // See MSDN for details + // https://docs.microsoft.com/en-us/sql/relational-databases/native-client/odbc/large-clr-user-defined-types-odbc + // + // If bound value is larger than parameter size, we force SQL_SS_LENGTH_UNLIMITED. + param_size = SQL_SS_LENGTH_UNLIMITED; + } + + RETCODE rc; + NANODBC_CALL_RC( + SQLBindParameter, + rc, + stmt_, // handle + param.index_ + 1, // parameter number + param.iotype_, // input or output type + sql_ctype::value, // value type + param.type_, // parameter type + param_size, // column size ignored for many types, but needed for strings + param.scale_, // decimal digits + (SQLPOINTER)buffer.values_, // parameter value + value_size, // buffer length + bind_len_or_null_[param.index_].data()); + + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + + // Supports code like: query.bind(0, std_string.c_str()) + // In this case, we need to pass nullptr to the final parameter of SQLBindParameter(). + template ::value, int>::type = 0> void bind_parameter(bound_parameter const& param, bound_buffer& buffer) { auto const buffer_size = buffer.value_size_ > 0 ? buffer.value_size_ : param.size_; @@ -1788,7 +1926,7 @@ class statement::statement_impl param.scale_, // decimal digits (SQLPOINTER)buffer.values_, // parameter value buffer_size, // buffer length - bind_len_or_null_[param.index_].data()); + (buffer.size_ <= 1 ? nullptr : bind_len_or_null_[param.index_].data())); if (!success(rc)) NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); @@ -1856,21 +1994,23 @@ class statement::statement_impl bind_parameter(param, buffer); } + template > void bind_strings( param_direction direction, short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, bool const* nulls = nullptr, - string::value_type const* null_sentry = nullptr); + T const* null_sentry = nullptr); + template > void bind_strings( param_direction direction, short param_index, - std::vector const& values, + std::vector const& values, bool const* nulls = nullptr, - string::value_type const* null_sentry = nullptr); + typename T::value_type const* null_sentry = nullptr); // handles multiple null values void bind_null(short param_index, std::size_t batch_size) @@ -1903,12 +2043,16 @@ class statement::statement_impl return lhs == rhs; } + template + std::vector& get_bound_string_data(short param_index); + private: HSTMT stmt_; bool open_; class connection conn_; std::map> bind_len_or_null_; - std::map> string_data_; + std::map> wide_string_data_; + std::map> string_data_; std::map> binary_data_; #if defined(NANODBC_DO_ASYNC_IMPL) @@ -1919,34 +2063,6 @@ class statement::statement_impl #endif }; -// Supports code like: query.bind(0, std_string.c_str()) -// In this case, we need to pass nullptr to the final parameter of SQLBindParameter(). -template <> -void statement::statement_impl::bind_parameter( - bound_parameter const& param, - bound_buffer& buffer) -{ - auto const buffer_size = buffer.value_size_ > 0 ? buffer.value_size_ : param.size_; - - RETCODE rc; - NANODBC_CALL_RC( - SQLBindParameter, - rc, - stmt_, // handle - param.index_ + 1, // parameter number - param.iotype_, // input or output type - sql_ctype::value, // value type - param.type_, // parameter type - param.size_, // column size ignored for many types, but needed for strings - param.scale_, // decimal digits - (SQLPOINTER)buffer.values_, // parameter value - buffer_size, // buffer length - (buffer.size_ <= 1 ? nullptr : bind_len_or_null_[param.index_].data())); - - if (!success(rc)) - NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); -} - template void statement::statement_impl::bind( param_direction direction, @@ -1975,13 +2091,16 @@ void statement::statement_impl::bind( bind_parameter(param, buffer); } +template void statement::statement_impl::bind_strings( param_direction direction, short param_index, - std::vector const& values, + std::vector const& values, bool const* nulls /*= nullptr*/, - string::value_type const* null_sentry /*= nullptr*/) + typename T::value_type const* null_sentry /*= nullptr*/) { + using string_vector = std::vector; + string_vector& string_data = get_bound_string_data(param_index); size_t const batch_size = values.size(); bound_parameter param; @@ -1995,32 +2114,24 @@ void statement::statement_impl::bind_strings( // add space for null terminator ++max_length; - string_data_[param_index] = std::vector(batch_size * max_length, 0); + string_data = string_vector(batch_size * max_length, 0); for (std::size_t i = 0; i < batch_size; ++i) { - std::copy( - values[i].begin(), - values[i].end(), - string_data_[param_index].data() + (i * max_length)); + std::copy(values[i].begin(), values[i].end(), string_data.data() + (i * max_length)); } bind_strings( - direction, - param_index, - string_data_[param_index].data(), - max_length, - batch_size, - nulls, - null_sentry); + direction, param_index, string_data.data(), max_length, batch_size, nulls, null_sentry); } +template void statement::statement_impl::bind_strings( param_direction direction, short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, bool const* nulls /*= nullptr*/, - string::value_type const* null_sentry /*= nullptr*/) + T const* null_sentry /*= nullptr*/) { bound_parameter param; prepare_bind(param_index, batch_size, direction, param); @@ -2029,21 +2140,11 @@ void statement::statement_impl::bind_strings( { for (std::size_t i = 0; i < batch_size; ++i) { - const string s_lhs(values + i * value_size, values + (i + 1) * value_size); - const string s_rhs(null_sentry); -#if NANODBC_ENABLE_UNICODE - std::string narrow_lhs; - narrow_lhs.reserve(s_lhs.size()); - convert(s_lhs, narrow_lhs); - std::string narrow_rhs; - narrow_rhs.reserve(s_rhs.size()); - convert(s_rhs, narrow_rhs); - if (std::strncmp(narrow_lhs.c_str(), narrow_rhs.c_str(), value_size) != 0) + const std::basic_string s_lhs( + values + i * value_size, values + (i + 1) * value_size); + const std::basic_string s_rhs(null_sentry); + if (!equals(s_lhs, s_rhs)) bind_len_or_null_[param_index][i] = SQL_NTS; -#else - if (std::strncmp(s_lhs.c_str(), s_rhs.c_str(), value_size) != 0) - bind_len_or_null_[param_index][i] = SQL_NTS; -#endif } } else if (nulls) @@ -2062,11 +2163,33 @@ void statement::statement_impl::bind_strings( } } - auto const buffer_length = value_size * sizeof(string::value_type); - bound_buffer buffer(values, batch_size, buffer_length); + auto const buffer_length = value_size * sizeof(T); + bound_buffer buffer(values, batch_size, buffer_length); bind_parameter(param, buffer); } +template <> +bool statement::statement_impl::equals(const std::string& lhs, const std::string& rhs) +{ + return std::strncmp(lhs.c_str(), rhs.c_str(), lhs.size()) == 0; +} + +template <> +bool statement::statement_impl::equals(const wide_string& lhs, const wide_string& rhs) +{ + // e6059ff3a79062f83256b9d1d3c9c8368798781e + // Functions like `swprintf()`, `wcsftime()`, `wcsncmp()` can not be used + // with `u16string` types. Instead, prefers to narrow unicode string to + // work with them, and then widen them after work has been completed. + std::string narrow_lhs; + narrow_lhs.reserve(lhs.size()); + convert(lhs, narrow_lhs); + std::string narrow_rhs; + narrow_rhs.reserve(rhs.size()); + convert(rhs, narrow_rhs); + return equals(narrow_lhs, narrow_rhs); +} + template <> bool statement::statement_impl::equals(const date& lhs, const date& rhs) { @@ -2087,6 +2210,20 @@ bool statement::statement_impl::equals(const timestamp& lhs, const timestamp& rh lhs.fract == rhs.fract; } +template <> +std::vector& +statement::statement_impl::get_bound_string_data(short param_index) +{ + return wide_string_data_[param_index]; +} + +template <> +std::vector& +statement::statement_impl::get_bound_string_data(short param_index) +{ + return string_data_[param_index]; +} + } // namespace nanodbc // clang-format off @@ -2150,7 +2287,7 @@ class result::result_impl auto_bind(); } - ~result_impl() NANODBC_NOEXCEPT { cleanup_bound_columns(); } + ~result_impl() noexcept { cleanup_bound_columns(); } void* native_statement_handle() const { return stmt_.native_statement_handle(); } @@ -2158,7 +2295,9 @@ class result::result_impl long affected_rows() const { return stmt_.affected_rows(); } - long rows() const NANODBC_NOEXCEPT + bool has_affected_rows() const { return stmt_.affected_rows() != -1; } + + long rows() const noexcept { NANODBC_ASSERT(row_count_ <= static_cast(std::numeric_limits::max())); return static_cast(row_count_); @@ -2265,7 +2404,7 @@ class result::result_impl return static_cast(pos) + rowset_position_; } - bool at_end() const NANODBC_NOEXCEPT + bool at_end() const noexcept { if (at_end_) return true; @@ -2289,7 +2428,7 @@ class result::result_impl bound_column& col = bound_columns_[column]; if (rowset_position_ >= rows()) throw index_range_error(); - return col.cbdata_[rowset_position_] == SQL_NULL_DATA; + return col.cbdata_[static_cast(rowset_position_)] == SQL_NULL_DATA; } bool is_null(const string& column_name) const @@ -2499,22 +2638,31 @@ class result::result_impl } private: - template + template ::value, int>::type = 0> + void get_ref_impl(short column, T& result) const; + + template ::value, int>::type = 0> void get_ref_impl(short column, T& result) const; - void before_move() NANODBC_NOEXCEPT + template ::value, int>::type = 0> + void get_ref_from_string_column(short column, T& result) const; + + template ::value, int>::type = 0> + void get_ref_from_string_column(short column, T& result) const; + + void before_move() noexcept { for (short i = 0; i < bound_columns_size_; ++i) { bound_column& col = bound_columns_[i]; - for (long j = 0; j < rowset_size_; ++j) + for (std::size_t j = 0; j < static_cast(rowset_size_); ++j) col.cbdata_[j] = 0; if (col.blob_ && col.pdata_) release_bound_resources(i); } } - void release_bound_resources(short column) NANODBC_NOEXCEPT + void release_bound_resources(short column) noexcept { NANODBC_ASSERT(column < bound_columns_size_); bound_column& col = bound_columns_[column]; @@ -2523,7 +2671,7 @@ class result::result_impl col.clen_ = 0; } - void cleanup_bound_columns() NANODBC_NOEXCEPT + void cleanup_bound_columns() noexcept { before_move(); delete[] bound_columns_; @@ -2671,7 +2819,7 @@ class result::result_impl case SQL_CHAR: case SQL_VARCHAR: case SQL_NVARCHAR: - col.ctype_ = SQL_C_CHAR; + col.ctype_ = sql_ctype::value; col.clen_ = (col.sqlsize_ + 1) * sizeof(SQLCHAR); if (is_blob) { @@ -2681,7 +2829,7 @@ class result::result_impl break; case SQL_WCHAR: case SQL_WVARCHAR: - col.ctype_ = SQL_C_WCHAR; + col.ctype_ = sql_ctype::value; col.clen_ = (col.sqlsize_ + 1) * sizeof(SQLWCHAR); if (is_blob) { @@ -2690,7 +2838,7 @@ class result::result_impl } break; case SQL_LONGVARCHAR: - col.ctype_ = SQL_C_CHAR; + col.ctype_ = sql_ctype::value; col.blob_ = true; col.clen_ = 0; break; @@ -2712,7 +2860,7 @@ class result::result_impl for (SQLSMALLINT i = 0; i < n_columns; ++i) { bound_column& col = bound_columns_[i]; - col.cbdata_ = new null_type[rowset_size_]; + col.cbdata_ = new null_type[static_cast(rowset_size_)]; if (col.blob_) { NANODBC_CALL_RC( @@ -2819,8 +2967,8 @@ inline void result::result_impl::get_ref_impl(short column, timestamp throw type_incompatible_error(); } -template <> -inline void result::result_impl::get_ref_impl(short column, string& result) const +template ::value, int>::type> +inline void result::result_impl::get_ref_impl(short column, T& result) const { bound_column& col = bound_columns_[column]; const SQLULEN column_size = col.sqlsize_; @@ -2858,27 +3006,28 @@ inline void result::result_impl::get_ref_impl(short column, string& resu buffer, // TargetValuePtr buffer_size, // BufferLength &ValueLenOrInd); // StrLen_or_IndPtr - if (ValueLenOrInd > 0) + if (ValueLenOrInd == SQL_NO_TOTAL) + out.append(buffer, col.ctype_ == SQL_C_BINARY ? buffer_size : buffer_size - 1); + else if (ValueLenOrInd > 0) out.append( buffer, std::min( ValueLenOrInd, col.ctype_ == SQL_C_BINARY ? buffer_size : buffer_size - 1)); else if (ValueLenOrInd == SQL_NULL_DATA) - col.cbdata_[rowset_position_] = (SQLINTEGER)SQL_NULL_DATA; + col.cbdata_[static_cast(rowset_position_)] = (SQLINTEGER)SQL_NULL_DATA; // Sequence of successful calls is: // SQL_NO_DATA or SQL_SUCCESS_WITH_INFO followed by SQL_SUCCESS. } while (rc == SQL_SUCCESS_WITH_INFO); if (rc == SQL_SUCCESS || rc == SQL_NO_DATA) - convert(out, result); + convert(std::move(out), result); else if (!success(rc)) NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); } else { const char* s = col.pdata_ + rowset_position_ * col.clen_; - const std::string::size_type str_size = std::strlen(s); - result.assign(s, s + str_size); + convert(s, result); } return; } @@ -2914,19 +3063,22 @@ inline void result::result_impl::get_ref_impl(short column, string& resu buffer, // TargetValuePtr buffer_size, // BufferLength &ValueLenOrInd); // StrLen_or_IndPtr - if (ValueLenOrInd > 0) + if (ValueLenOrInd == SQL_NO_TOTAL) + out.append(buffer, (buffer_size / sizeof(wide_char_t)) - 1); + else if (ValueLenOrInd > 0) out.append( buffer, std::min( ValueLenOrInd / sizeof(wide_char_t), (buffer_size / sizeof(wide_char_t)) - 1)); else if (ValueLenOrInd == SQL_NULL_DATA) - col.cbdata_[rowset_position_] = (SQLINTEGER)SQL_NULL_DATA; + col.cbdata_[static_cast(rowset_position_)] = + (SQLINTEGER)SQL_NULL_DATA; // Sequence of successful calls is: // SQL_NO_DATA or SQL_SUCCESS_WITH_INFO followed by SQL_SUCCESS. } while (rc == SQL_SUCCESS_WITH_INFO); if (rc == SQL_SUCCESS || rc == SQL_NO_DATA) - convert(out, result); + convert(std::move(out), result); else if (!success(rc)) NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); ; @@ -2934,11 +3086,13 @@ inline void result::result_impl::get_ref_impl(short column, string& resu else { // Type is unicode in the database, convert if necessary - const SQLWCHAR* s = + SQLWCHAR const* s = reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); - const string::size_type str_size = col.cbdata_[rowset_position_] / sizeof(SQLWCHAR); - wide_string temp(s, s + str_size); - convert(temp, result); + string::size_type const str_size = + col.cbdata_[static_cast(rowset_position_)] / sizeof(SQLWCHAR); + auto const us = reinterpret_cast( + s); // no-op or unsigned short to signed char16_t + convert(us, str_size, result); } return; } @@ -2952,74 +3106,47 @@ inline void result::result_impl::get_ref_impl(short column, string& resu case SQL_C_LONG: { - std::string buffer; - buffer.reserve(column_size + 2); // ensure terminating null & negative sign - buffer.resize(buffer.capacity()); - using std::fill; - fill(buffer.begin(), buffer.end(), '\0'); - const int32_t data = *reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); + std::string buffer(column_size + 1, 0); // ensure terminating null + const wide_char_t data = + *reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); const int bytes = - std::snprintf(const_cast(buffer.data()), column_size + 2, "%d", data); + std::snprintf(const_cast(buffer.data()), column_size + 1, "%d", data); if (bytes == -1) throw type_incompatible_error(); - else if ((SQLULEN)bytes < column_size + 1) - buffer.resize(bytes); - buffer.resize(std::strlen(buffer.data())); // drop any trailing nulls - result.reserve(buffer.size() * sizeof(string::value_type)); - convert(buffer, result); + convert(buffer.data(), result); // passing the C pointer drops trailing nulls return; } case SQL_C_SBIGINT: { - using namespace std; // in case intmax_t is in namespace std - std::string buffer; - buffer.reserve(column_size + 2); // ensure terminating null & negative sign - buffer.resize(buffer.capacity()); - using std::fill; - fill(buffer.begin(), buffer.end(), '\0'); + using namespace std; // in case intmax_t is in namespace std + std::string buffer(column_size + 1, 0); // ensure terminating null const intmax_t data = (intmax_t) * reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); const int bytes = - std::snprintf(const_cast(buffer.data()), column_size + 2, "%jd", data); + std::snprintf(const_cast(buffer.data()), column_size + 1, "%jd", data); if (bytes == -1) throw type_incompatible_error(); - else if ((SQLULEN)bytes < column_size + 1) - buffer.resize(bytes); - buffer.resize(std::strlen(buffer.data())); // drop any trailing nulls - result.reserve(buffer.size() * sizeof(string::value_type)); - convert(buffer, result); + convert(buffer.data(), result); // passing the C pointer drops trailing nulls return; } case SQL_C_FLOAT: { - std::string buffer; - buffer.reserve(column_size + 1); // ensure terminating null - buffer.resize(buffer.capacity()); - using std::fill; - fill(buffer.begin(), buffer.end(), '\0'); + std::string buffer(column_size + 1, 0); // ensure terminating null const float data = *reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); const int bytes = std::snprintf(const_cast(buffer.data()), column_size + 1, "%f", data); if (bytes == -1) throw type_incompatible_error(); - else if ((SQLULEN)bytes < column_size) - buffer.resize(bytes); - buffer.resize(std::strlen(buffer.data())); // drop any trailing nulls - result.reserve(buffer.size() * sizeof(string::value_type)); - convert(buffer, result); + convert(buffer.data(), result); // passing the C pointer drops trailing nulls return; } case SQL_C_DOUBLE: { - std::string buffer; const SQLULEN width = column_size + 2; // account for decimal mark and sign - buffer.reserve(width + 1); // ensure terminating null - buffer.resize(buffer.capacity()); - using std::fill; - fill(buffer.begin(), buffer.end(), '\0'); + std::string buffer(width + 1, 0); // ensure terminating null const double data = *reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); const int bytes = std::snprintf( const_cast(buffer.data()), @@ -3029,11 +3156,7 @@ inline void result::result_impl::get_ref_impl(short column, string& resu data); if (bytes == -1) throw type_incompatible_error(); - else if ((SQLULEN)bytes < column_size) - buffer.resize(bytes); - buffer.resize(std::strlen(buffer.data())); // drop any trailing nulls - result.reserve(buffer.size() * sizeof(string::value_type)); - convert(buffer, result); + convert(buffer.data(), result); // passing the C pointer drops trailing nulls return; } @@ -3140,7 +3263,7 @@ inline void result::result_impl::get_ref_impl>( out.insert(std::end(out), buffer, buffer + buffer_size_filled); } else if (ValueLenOrInd == SQL_NULL_DATA) - col.cbdata_[rowset_position_] = (SQLINTEGER)SQL_NULL_DATA; + col.cbdata_[static_cast(rowset_position_)] = (SQLINTEGER)SQL_NULL_DATA; // Sequence of successful calls is: // SQL_NO_DATA or SQL_SUCCESS_WITH_INFO followed by SQL_SUCCESS. } while (rc == SQL_SUCCESS_WITH_INFO); @@ -3161,7 +3284,75 @@ inline void result::result_impl::get_ref_impl>( throw type_incompatible_error(); } -template +namespace detail +{ +auto from_string(std::string const& s, float) -> decltype(std::stof(s)) +{ + return std::stof(s); +} + +auto from_string(std::string const& s, double) -> decltype(std::stod(s)) +{ + return std::stod(s); +} + +auto from_string(std::string const& s, long long) -> decltype(std::stoll(s)) +{ + return std::stoll(s); +} + +auto from_string(std::string const& s, unsigned long long) -> decltype(std::stoull(s)) +{ + return std::stoull(s); +} + +template ::value, int>::type = 0> +auto from_string(std::string const& s, R) -> R +{ + auto integer = from_string( + s, + typename std::conditional::value, long long, unsigned long long>::type{}); + if (integer > std::numeric_limits::max() || integer < std::numeric_limits::min()) + throw std::range_error("from_string argument out of range"); + return static_cast(integer); +} +} // namespace detail + +template +auto from_string(std::string const& s) -> R +{ + return detail::from_string(s, R{}); +} + +template ::value, int>::type> +void result::result_impl::get_ref_from_string_column(short column, T& result) const +{ + bound_column& col = bound_columns_[column]; + const char* s = col.pdata_ + rowset_position_ * col.clen_; + switch (col.ctype_) + { + case SQL_C_CHAR: + result = static_cast(*static_cast(s)); + return; + case SQL_C_WCHAR: + result = static_cast(*reinterpret_cast(s)); + return; + } + throw type_incompatible_error(); +} + +template ::value, int>::type> +void result::result_impl::get_ref_from_string_column(short column, T& result) const +{ + bound_column& col = bound_columns_[column]; + if (col.ctype_ != SQL_C_CHAR && col.ctype_ != SQL_C_WCHAR) + throw type_incompatible_error(); + std::string str; + get_ref_impl(col.column_, str); + result = from_string(str); +} + +template ::value, int>::type> void result::result_impl::get_ref_impl(short column, T& result) const { bound_column& col = bound_columns_[column]; @@ -3170,7 +3361,8 @@ void result::result_impl::get_ref_impl(short column, T& result) const switch (col.ctype_) { case SQL_C_CHAR: - result = (T) * (char*)(s); + case SQL_C_WCHAR: + get_ref_from_string_column(column, result); return; case SQL_C_SSHORT: result = (T) * (short*)(s); @@ -3228,18 +3420,18 @@ std::list list_drivers() SQLSMALLINT attrs_len_ret{0}; SQLUSMALLINT direction{SQL_FETCH_FIRST}; - HENV env{0}; - allocate_environment_handle(env); + connection env; // ensures handles RAII + env.allocate(); + NANODBC_ASSERT(env.native_env_handle()); std::list drivers; RETCODE rc{SQL_SUCCESS}; do { - NANODBC_ASSERT(env); NANODBC_CALL_RC( NANODBC_FUNC(SQLDrivers), rc, - env, + env.native_env_handle(), direction, // EnvironmentHandle descr, // DriverDescription sizeof(descr) / sizeof(NANODBC_SQLCHAR), // BufferLength1 @@ -3256,7 +3448,7 @@ std::list list_drivers() "incompatible SQLCHAR and string::value_type"); driver drv; - drv.name = string(&descr[0], &descr[strarrlen(descr)]); + drv.name = string(&descr[0], &descr[std::char_traits::length(descr)]); // Split "Key1=Value1\0Key2=Value2\0\0" into list of key-value pairs auto beg = &attrs[0]; @@ -3280,7 +3472,7 @@ std::list list_drivers() else { if (rc != SQL_NO_DATA) - NANODBC_THROW_DATABASE_ERROR(env, SQL_HANDLE_ENV); + NANODBC_THROW_DATABASE_ERROR(env.native_env_handle(), SQL_HANDLE_ENV); } } while (success(rc)); @@ -3356,11 +3548,10 @@ connection::connection(const connection& rhs) { } -#ifndef NANODBC_NO_MOVE_CTOR -connection::connection(connection&& rhs) NANODBC_NOEXCEPT : impl_(std::move(rhs.impl_)) +connection::connection(connection&& rhs) noexcept + : impl_(std::move(rhs.impl_)) { } -#endif connection& connection::operator=(connection rhs) { @@ -3368,7 +3559,7 @@ connection& connection::operator=(connection rhs) return *this; } -void connection::swap(connection& rhs) NANODBC_NOEXCEPT +void connection::swap(connection& rhs) noexcept { using std::swap; swap(impl_, rhs.impl_); @@ -3384,8 +3575,16 @@ connection::connection(const string& connection_string, long timeout) { } -connection::~connection() NANODBC_NOEXCEPT +connection::~connection() noexcept {} + +void connection::allocate() +{ + impl_->allocate(); +} + +void connection::deallocate() { + impl_->deallocate(); } void connection::connect(const string& dsn, const string& user, const string& pass, long timeout) @@ -3523,11 +3722,10 @@ transaction::transaction(const transaction& rhs) { } -#ifndef NANODBC_NO_MOVE_CTOR -transaction::transaction(transaction&& rhs) NANODBC_NOEXCEPT : impl_(std::move(rhs.impl_)) +transaction::transaction(transaction&& rhs) noexcept + : impl_(std::move(rhs.impl_)) { } -#endif transaction& transaction::operator=(transaction rhs) { @@ -3535,22 +3733,20 @@ transaction& transaction::operator=(transaction rhs) return *this; } -void transaction::swap(transaction& rhs) NANODBC_NOEXCEPT +void transaction::swap(transaction& rhs) noexcept { using std::swap; swap(impl_, rhs.impl_); } -transaction::~transaction() NANODBC_NOEXCEPT -{ -} +transaction::~transaction() noexcept {} void transaction::commit() { impl_->commit(); } -void transaction::rollback() NANODBC_NOEXCEPT +void transaction::rollback() noexcept { impl_->rollback(); } @@ -3602,11 +3798,10 @@ statement::statement(class connection& conn) { } -#ifndef NANODBC_NO_MOVE_CTOR -statement::statement(statement&& rhs) NANODBC_NOEXCEPT : impl_(std::move(rhs.impl_)) +statement::statement(statement&& rhs) noexcept + : impl_(std::move(rhs.impl_)) { } -#endif statement::statement(class connection& conn, const string& query, long timeout) : impl_(new statement_impl(conn, query, timeout)) @@ -3624,15 +3819,13 @@ statement& statement::operator=(statement rhs) return *this; } -void statement::swap(statement& rhs) NANODBC_NOEXCEPT +void statement::swap(statement& rhs) noexcept { using std::swap; swap(impl_, rhs.impl_); } -statement::~statement() NANODBC_NOEXCEPT -{ -} +statement::~statement() noexcept {} void statement::open(class connection& conn) { @@ -3788,7 +3981,7 @@ short statement::parameters() const return impl_->parameters(); } -void statement::reset_parameters() NANODBC_NOEXCEPT +void statement::reset_parameters() noexcept { impl_->reset_parameters(); } @@ -3807,8 +4000,27 @@ unsigned long statement::parameter_size(short param_index) const template void statement::bind( \ short, const type*, std::size_t, const bool*, param_direction) /* n-ary, flags */ +#define NANODBC_INSTANTIATE_BIND_STRINGS(type) \ + template void statement::bind_strings(short, std::vector const&, param_direction); \ + template void statement::bind_strings( \ + short, std::vector const&, type::value_type const*, param_direction); \ + template void statement::bind_strings( \ + short, std::vector const&, bool const*, param_direction); \ + template void statement::bind_strings( \ + short, const type::value_type*, std::size_t, std::size_t, param_direction); \ + template void statement::bind_strings( \ + short, \ + type::value_type const*, \ + std::size_t, \ + std::size_t, \ + type::value_type const*, \ + param_direction); \ + template void statement::bind_strings( \ + short, type::value_type const*, std::size_t, std::size_t, bool const*, param_direction) + // The following are the only supported instantiations of statement::bind(). -NANODBC_INSTANTIATE_BINDS(string::value_type); +NANODBC_INSTANTIATE_BINDS(std::string::value_type); +NANODBC_INSTANTIATE_BINDS(wide_string::value_type); NANODBC_INSTANTIATE_BINDS(short); NANODBC_INSTANTIATE_BINDS(unsigned short); NANODBC_INSTANTIATE_BINDS(int); @@ -3823,6 +4035,9 @@ NANODBC_INSTANTIATE_BINDS(date); NANODBC_INSTANTIATE_BINDS(time); NANODBC_INSTANTIATE_BINDS(timestamp); +NANODBC_INSTANTIATE_BIND_STRINGS(std::string); +NANODBC_INSTANTIATE_BIND_STRINGS(wide_string); + #undef NANODBC_INSTANTIATE_BINDS template @@ -3889,17 +4104,19 @@ void statement::bind( impl_->bind(direction, param_index, values, nullptr, null_sentry); } +template void statement::bind_strings( short param_index, - std::vector const& values, + std::vector const& values, param_direction direction) { impl_->bind_strings(direction, param_index, values); } +template void statement::bind_strings( short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, param_direction direction) @@ -3907,21 +4124,23 @@ void statement::bind_strings( impl_->bind_strings(direction, param_index, values, value_size, batch_size); } +template void statement::bind_strings( short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, - string::value_type const* null_sentry, + T const* null_sentry, param_direction direction) { impl_->bind_strings( direction, param_index, values, value_size, batch_size, nullptr, null_sentry); } +template void statement::bind_strings( short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, bool const* nulls, @@ -3930,18 +4149,20 @@ void statement::bind_strings( impl_->bind_strings(direction, param_index, values, value_size, batch_size, nulls); } +template void statement::bind_strings( short param_index, - std::vector const& values, - string::value_type const* null_sentry, + std::vector const& values, + typename T::value_type const* null_sentry, param_direction direction) { impl_->bind_strings(direction, param_index, values, nullptr, null_sentry); } +template void statement::bind_strings( short param_index, - std::vector const& values, + std::vector const& values, bool const* nulls, param_direction direction) { @@ -4365,8 +4586,6 @@ std::list catalog::list_catalogs() std::list catalog::list_schemas() { - // TODO: Possible to restrict list of schemas from a specified catalog? - // Special case for list of schemas: // all the other arguments must match empty string (""), // otherwise pattern-based lookup is performed returning @@ -4419,20 +4638,17 @@ result::result() { } -result::~result() NANODBC_NOEXCEPT -{ -} +result::~result() noexcept {} result::result(statement stmt, long rowset_size) : impl_(new result_impl(stmt, rowset_size)) { } -#ifndef NANODBC_NO_MOVE_CTOR -result::result(result&& rhs) NANODBC_NOEXCEPT : impl_(std::move(rhs.impl_)) +result::result(result&& rhs) noexcept + : impl_(std::move(rhs.impl_)) { } -#endif result::result(const result& rhs) : impl_(rhs.impl_) @@ -4445,7 +4661,7 @@ result& result::operator=(result rhs) return *this; } -void result::swap(result& rhs) NANODBC_NOEXCEPT +void result::swap(result& rhs) noexcept { using std::swap; swap(impl_, rhs.impl_); @@ -4456,7 +4672,7 @@ void* result::native_statement_handle() const return impl_->native_statement_handle(); } -long result::rowset_size() const NANODBC_NOEXCEPT +long result::rowset_size() const noexcept { return impl_->rowset_size(); } @@ -4466,7 +4682,12 @@ long result::affected_rows() const return impl_->affected_rows(); } -long result::rows() const NANODBC_NOEXCEPT +bool result::has_affected_rows() const +{ + return impl_->has_affected_rows(); +} + +long result::rows() const noexcept { return impl_->rows(); } @@ -4523,7 +4744,7 @@ unsigned long result::position() const return impl_->position(); } -bool result::at_end() const NANODBC_NOEXCEPT +bool result::at_end() const noexcept { return impl_->at_end(); } @@ -4657,7 +4878,8 @@ result::operator bool() const } // The following are the only supported instantiations of result::get_ref(). -template void result::get_ref(short, string::value_type&) const; +template void result::get_ref(short, std::string::value_type&) const; +template void result::get_ref(short, wide_string::value_type&) const; template void result::get_ref(short, short&) const; template void result::get_ref(short, unsigned short&) const; template void result::get_ref(short, int&) const; @@ -4674,7 +4896,8 @@ template void result::get_ref(short, time&) const; template void result::get_ref(short, timestamp&) const; template void result::get_ref(short, std::vector&) const; -template void result::get_ref(const string&, string::value_type&) const; +template void result::get_ref(const string&, std::string::value_type&) const; +template void result::get_ref(const string&, wide_string::value_type&) const; template void result::get_ref(const string&, short&) const; template void result::get_ref(const string&, unsigned short&) const; template void result::get_ref(const string&, int&) const; @@ -4692,7 +4915,10 @@ template void result::get_ref(const string&, timestamp&) const; template void result::get_ref(const string&, std::vector&) const; // The following are the only supported instantiations of result::get_ref() with fallback. -template void result::get_ref(short, const string::value_type&, string::value_type&) const; +template void +result::get_ref(short, const std::string::value_type&, std::string::value_type&) const; +template void +result::get_ref(short, const wide_string::value_type&, wide_string::value_type&) const; template void result::get_ref(short, const short&, short&) const; template void result::get_ref(short, const unsigned short&, unsigned short&) const; template void result::get_ref(short, const int&, int&) const; @@ -4710,7 +4936,10 @@ template void result::get_ref(short, const timestamp&, timestamp&) const; template void result::get_ref(short, const std::vector&, std::vector&) const; -template void result::get_ref(const string&, const string::value_type&, string::value_type&) const; +template void +result::get_ref(const string&, const std::string::value_type&, std::string::value_type&) const; +template void +result::get_ref(const string&, const wide_string::value_type&, wide_string::value_type&) const; template void result::get_ref(const string&, const short&, short&) const; template void result::get_ref(const string&, const unsigned short&, unsigned short&) const; template void result::get_ref(const string&, const int&, int&) const; @@ -4722,7 +4951,8 @@ template void result::get_ref(const string&, const unsigned long long int&, unsigned long long int&) const; template void result::get_ref(const string&, const float&, float&) const; template void result::get_ref(const string&, const double&, double&) const; -template void result::get_ref(const string&, const string&, string&) const; +template void result::get_ref(const string&, const std::string&, std::string&) const; +template void result::get_ref(const string&, const wide_string&, wide_string&) const; template void result::get_ref(const string&, const date&, date&) const; template void result::get_ref(const string&, const time&, time&) const; template void result::get_ref(const string&, const timestamp&, timestamp&) const; @@ -4730,7 +4960,8 @@ template void result::get_ref(const string&, const std::vector&, std::vector&) const; // The following are the only supported instantiations of result::get(). -template string::value_type result::get(short) const; +template std::string::value_type result::get(short) const; +template wide_string::value_type result::get(short) const; template short result::get(short) const; template unsigned short result::get(short) const; template int result::get(short) const; @@ -4741,13 +4972,15 @@ template long long int result::get(short) const; template unsigned long long int result::get(short) const; template float result::get(short) const; template double result::get(short) const; -template string result::get(short) const; +template std::string result::get(short) const; +template wide_string result::get(short) const; template date result::get(short) const; template time result::get(short) const; template timestamp result::get(short) const; template std::vector result::get(short) const; -template string::value_type result::get(const string&) const; +template std::string::value_type result::get(const string&) const; +template wide_string::value_type result::get(const string&) const; template short result::get(const string&) const; template unsigned short result::get(const string&) const; template int result::get(const string&) const; @@ -4758,14 +4991,16 @@ template long long int result::get(const string&) const; template unsigned long long int result::get(const string&) const; template float result::get(const string&) const; template double result::get(const string&) const; -template string result::get(const string&) const; +template std::string result::get(const string&) const; +template wide_string result::get(const string&) const; template date result::get(const string&) const; template time result::get(const string&) const; template timestamp result::get(const string&) const; template std::vector result::get(const string&) const; // The following are the only supported instantiations of result::get() with fallback. -template string::value_type result::get(short, const string::value_type&) const; +template std::string::value_type result::get(short, const std::string::value_type&) const; +template wide_string::value_type result::get(short, const wide_string::value_type&) const; template short result::get(short, const short&) const; template unsigned short result::get(short, const unsigned short&) const; template int result::get(short, const int&) const; @@ -4776,13 +5011,15 @@ template long long int result::get(short, const long long int&) const; template unsigned long long int result::get(short, const unsigned long long int&) const; template float result::get(short, const float&) const; template double result::get(short, const double&) const; -template string result::get(short, const string&) const; +template std::string result::get(short, const std::string&) const; +template wide_string result::get(short, const wide_string&) const; template date result::get(short, const date&) const; template time result::get(short, const time&) const; template timestamp result::get(short, const timestamp&) const; template std::vector result::get(short, const std::vector&) const; -template string::value_type result::get(const string&, const string::value_type&) const; +template std::string::value_type result::get(const string&, const std::string::value_type&) const; +template wide_string::value_type result::get(const string&, const wide_string::value_type&) const; template short result::get(const string&, const short&) const; template unsigned short result::get(const string&, const unsigned short&) const; template int result::get(const string&, const int&) const; @@ -4793,7 +5030,8 @@ template long long int result::get(const string&, const long long int&) const; template unsigned long long int result::get(const string&, const unsigned long long int&) const; template float result::get(const string&, const float&) const; template double result::get(const string&, const double&) const; -template string result::get(const string&, const string&) const; +template std::string result::get(const string&, const std::string&) const; +template wide_string result::get(const string&, const wide_string&) const; template date result::get(const string&, const date&) const; template time result::get(const string&, const time&) const; template timestamp result::get(const string&, const timestamp&) const; @@ -4801,6 +5039,7 @@ template std::vector result::get(const string&, const std::vector&) const; } // namespace nanodbc +#endif // NANODBC_DISABLE_NANODBC_NAMESPACE_FOR_INTERNAL_TESTS #undef NANODBC_THROW_DATABASE_ERROR #undef NANODBC_STRINGIZE diff --git a/odbc/src/nanodbc/nanodbc.h b/odbc/src/nanodbc/nanodbc.h index 2322a8d1..5ca02530 100644 --- a/odbc/src/nanodbc/nanodbc.h +++ b/odbc/src/nanodbc/nanodbc.h @@ -1,4 +1,4 @@ -/// \file nanodbc.h The entirety of nanodbc can be found within this file and nanodbc.cpp. +/// \file nanodbc.h The entirety of nanodbc can be found within this file and nanodbc.cpp. /// \mainpage /// @@ -6,7 +6,7 @@ /// This library provides a wrapper API for the native ODBC API. It aims to do everything ODBC does, /// but with a \b much nicer interface. Anything it doesn't (yet) do can be done by retrieving the /// native ODBC handles and dropping down to straight ODBC C API code. -/// For more propaganda, please see the project +/// For more propaganda, please see the project /// homepage. /// /// \section toc Table of Contents @@ -75,7 +75,7 @@ #ifndef NANODBC_H #define NANODBC_H - +#define NANODBC_ENABLE_UNICODE #include #include #include @@ -113,11 +113,18 @@ namespace nanodbc // clang-format on /// \addtogroup macros Macros -/// \brief Macros that nanodbc uses, can be overriden by users. +/// \brief Configuration and utility macros that nanodbc uses, can be overriden by users. /// /// @{ - #ifdef DOXYGEN + +/// \def NANODBC_THROW_NO_SOURCE_LOCATION +/// \brief Configures \c nanodbc::database_error message +/// +/// If defined, removes source file name and line number from \c nanodbc::database_error message +/// By default, nanodbc includes source location of exception in the error message. +#define NANODBC_THROW_NO_SOURCE_LOCATION 1 + /// \def NANODBC_ASSERT(expression) /// \brief Assertion. /// @@ -133,8 +140,8 @@ namespace nanodbc /// #endif /// \endcode #define NANODBC_ASSERT(expression) assert(expression) -#endif +#endif /// @} // You must explicitly request Unicode support by defining NANODBC_ENABLE_UNICODE at compile time. @@ -157,6 +164,18 @@ typedef std::string string; #define NANODBC_TEXT(s) s #endif +#ifdef NANODBC_USE_IODBC_WIDE_STRINGS +typedef std::u32string wide_string; +#else +#ifdef _MSC_VER +typedef std::wstring wide_string; +#else +typedef std::u16string wide_string; +#endif +#endif + +typedef wide_string::value_type wide_char_t; + #if defined(_WIN64) // LLP64 machine: Windows typedef std::int64_t null_type; @@ -188,12 +207,17 @@ typedef unspecified - type string; typedef unspecified - type null_type; #endif -#if defined(_MSC_VER) && _MSC_VER <= 1800 -// These versions of Visual C++ do not yet support \c noexcept or \c std::move. -#define NANODBC_NOEXCEPT -#define NANODBC_NO_MOVE_CTOR +#if __cplusplus >= 201402L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L) +// [[deprecated]] is only available in C++14 +#define NANODBC_DEPRECATED [[deprecated]] +#else +#ifdef __GNUC__ +#define NANODBC_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define NANODBC_DEPRECATED __declspec(deprecated) #else -#define NANODBC_NOEXCEPT noexcept +#define NANODBC_DEPRECATED +#endif #endif // clang-format off @@ -227,7 +251,7 @@ class type_incompatible_error : public std::runtime_error { public: type_incompatible_error(); - const char* what() const NANODBC_NOEXCEPT; + const char* what() const noexcept; }; /// \brief Accessed null data. @@ -236,7 +260,7 @@ class null_access_error : public std::runtime_error { public: null_access_error(); - const char* what() const NANODBC_NOEXCEPT; + const char* what() const noexcept; }; /// \brief Index out of range. @@ -245,7 +269,7 @@ class index_range_error : public std::runtime_error { public: index_range_error(); - const char* what() const NANODBC_NOEXCEPT; + const char* what() const noexcept; }; /// \brief Programming logic error. @@ -254,7 +278,7 @@ class programming_error : public std::runtime_error { public: explicit programming_error(const std::string& info); - const char* what() const NANODBC_NOEXCEPT; + const char* what() const noexcept; }; /// \brief General database error. @@ -267,9 +291,9 @@ class database_error : public std::runtime_error /// \param handle_type The native ODBC handle type code for the given handle. /// \param info Additional info that will be appended to the beginning of the error message. database_error(void* handle, short handle_type, const std::string& info = ""); - const char* what() const NANODBC_NOEXCEPT; - const long native() const NANODBC_NOEXCEPT; - const std::string state() const NANODBC_NOEXCEPT; + const char* what() const noexcept; + const long native() const noexcept; + const std::string state() const noexcept; private: long native_error; @@ -324,6 +348,28 @@ struct timestamp std::int32_t fract; ///< Fractional seconds. }; +/// \brief A type trait for testing if a type is a std::basic_string compatible with the current +/// nanodbc configuration +template +using is_string = std::integral_constant< + bool, + std::is_same::type, std::string>::value || + std::is_same::type, wide_string>::value>; + +/// \brief A type trait for testing if a type is a character compatible with the current nanodbc +/// configuration +template +using is_character = std::integral_constant< + bool, + std::is_same::type, std::string::value_type>::value || + std::is_same::type, wide_char_t>::value>; + +template +using enable_if_string = typename std::enable_if::value>::type; + +template +using enable_if_character = typename std::enable_if::value>::type; + /// \} /// \addtogroup mainc Main classes @@ -357,26 +403,24 @@ class transaction /// Copy constructor. transaction(const transaction& rhs); -#ifndef NANODBC_NO_MOVE_CTOR /// Move constructor. - transaction(transaction&& rhs) NANODBC_NOEXCEPT; -#endif + transaction(transaction&& rhs) noexcept; /// Assignment. transaction& operator=(transaction rhs); /// Member swap. - void swap(transaction& rhs) NANODBC_NOEXCEPT; + void swap(transaction& rhs) noexcept; /// \brief If this transaction has not been committed, will will rollback any modifying ops. - ~transaction() NANODBC_NOEXCEPT; + ~transaction() noexcept; /// \brief Commits transaction immediately. /// \throws database_error void commit(); /// \brief Marks this transaction for rollback. - void rollback() NANODBC_NOEXCEPT; + void rollback() noexcept; /// Returns the connection object. class connection& connection(); @@ -444,20 +488,18 @@ class statement /// \brief Copy constructor. statement(const statement& rhs); -#ifndef NANODBC_NO_MOVE_CTOR /// \brief Move constructor. - statement(statement&& rhs) NANODBC_NOEXCEPT; -#endif + statement(statement&& rhs) noexcept; /// \brief Assignment. statement& operator=(statement rhs); /// \brief Member swap. - void swap(statement& rhs) NANODBC_NOEXCEPT; + void swap(statement& rhs) noexcept; /// \brief Closes the statement. /// \see close() - ~statement() NANODBC_NOEXCEPT; + ~statement() noexcept; /// \brief Creates a statement for the given connection. /// \param conn The connection where the statement will be executed. @@ -614,8 +656,10 @@ class statement /// \see async_execute(), async_execute_direct() class result complete_execute(long batch_operations = 1); - /// left for backwards compatibility - class result async_complete(long batch_operations = 1); + /// \brief Completes a previously initiated asynchronous query execution, returning the result. + /// + /// \deprecated Use complete_execute instead. + NANODBC_DEPRECATED class result async_complete(long batch_operations = 1); /// undocumented - for internal use only (used from result_impl) void enable_async(void* event_handle); @@ -682,7 +726,7 @@ class statement short columns() const; /// \brief Resets all currently bound parameters. - void reset_parameters() NANODBC_NOEXCEPT; + void reset_parameters() noexcept; /// \brief Returns the number of parameters in the statement. /// \throws database_error @@ -807,9 +851,10 @@ class statement /// \brief Binds multiple string values. /// \see bind_strings + template > void bind_strings( short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, param_direction direction = PARAM_IN); @@ -820,59 +865,71 @@ class statement /// Longest string in the array determines maximum length of individual value. /// /// \see bind_strings + template > void bind_strings( short param_index, - std::vector const& values, + std::vector const& values, param_direction direction = PARAM_IN); /// \brief Binds multiple string values. /// \see bind_strings - template + template < + std::size_t BatchSize, + std::size_t ValueSize, + class T, + typename = enable_if_character> void bind_strings( short param_index, - string::value_type const (&values)[BatchSize][ValueSize], + T const (&values)[BatchSize][ValueSize], param_direction direction = PARAM_IN) { - auto param_values = reinterpret_cast(values); + auto param_values = reinterpret_cast(values); bind_strings(param_index, param_values, ValueSize, BatchSize, direction); } /// \brief Binds multiple string values. /// \see bind_strings + template > void bind_strings( short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, - string::value_type const* null_sentry, + T const* null_sentry, param_direction direction = PARAM_IN); /// \brief Binds multiple string values. /// \see bind_strings + template > void bind_strings( short param_index, - std::vector const& values, - string::value_type const* null_sentry, + std::vector const& values, + typename T::value_type const* null_sentry, param_direction direction = PARAM_IN); /// \brief Binds multiple string values. /// \see bind_strings - template + template < + std::size_t BatchSize, + std::size_t ValueSize, + class T, + typename = enable_if_character> void bind_strings( short param_index, - string::value_type const (&values)[BatchSize][ValueSize], - string::value_type const* null_sentry, + T const (&values)[BatchSize][ValueSize], + T const* null_sentry, param_direction direction = PARAM_IN) { - auto param_values = reinterpret_cast(values); + auto param_values = reinterpret_cast(values); bind_strings(param_index, param_values, ValueSize, BatchSize, null_sentry, direction); } /// \brief Binds multiple string values. /// \see bind_strings + template > void bind_strings( short param_index, - string::value_type const* values, + T const* values, std::size_t value_size, std::size_t batch_size, bool const* nulls, @@ -880,22 +937,27 @@ class statement /// \brief Binds multiple string values. /// \see bind_strings + template > void bind_strings( short param_index, - std::vector const& values, + std::vector const& values, bool const* nulls, param_direction direction = PARAM_IN); /// \brief Binds multiple string values. /// \see bind_strings - template + template < + std::size_t BatchSize, + std::size_t ValueSize, + class T, + typename = enable_if_character> void bind_strings( short param_index, - string::value_type const (&values)[BatchSize][ValueSize], + T const (&values)[BatchSize][ValueSize], bool const* nulls, param_direction direction = PARAM_IN) { - auto param_values = reinterpret_cast(values); + auto param_values = reinterpret_cast(values); bind_strings(param_index, param_values, ValueSize, BatchSize, nulls, direction); } @@ -949,19 +1011,20 @@ class connection /// Copy constructor. connection(const connection& rhs); -#ifndef NANODBC_NO_MOVE_CTOR /// Move constructor. - connection(connection&& rhs) NANODBC_NOEXCEPT; -#endif + connection(connection&& rhs) noexcept; /// Assignment. connection& operator=(connection rhs); /// Member swap. - void swap(connection&) NANODBC_NOEXCEPT; + void swap(connection&) noexcept; /// \brief Create new connection object and immediately connect to the given data source. - /// \param dsn The name of the data source. + /// + /// The function calls ODBC API SQLConnect. + /// + /// \param dsn The name of the data source name (DSN). /// \param user The username for authenticating to the data source. /// \param pass The password for authenticating to the data source. /// \param timeout Seconds before connection timeout. Default 0 meaning no timeout. @@ -971,6 +1034,9 @@ class connection /// \brief Create new connection object and immediately connect using the given connection /// string. + /// + /// The function calls ODBC API SQLDriverConnect. + /// /// \param connection_string The connection string for establishing a connection. /// \param timeout Seconds before connection timeout. Default is 0 indicating no timeout. /// \throws database_error @@ -981,7 +1047,21 @@ class connection /// /// Will not throw even if disconnecting causes some kind of error and raises an exception. /// If you explicitly need to know if disconnect() succeeds, call it directly. - ~connection() NANODBC_NOEXCEPT; + ~connection() noexcept; + + /// \brief Allocate environment and connection handles. + /// + /// Allows on-demand allocation of handles to configure the ODBC environment + /// and attributes, before database connection is established. + /// Typically, user does not have to make this call explicitly. + /// + /// \throws database_error + /// \see deallocate() + void allocate(); + + /// \brief Release environment and connection handles. + /// \see allocate() + void deallocate(); /// \brief Connect to the given data source. /// \param dsn The name of the data source. @@ -1130,34 +1210,41 @@ class result result(); /// \brief Free result set. - ~result() NANODBC_NOEXCEPT; + ~result() noexcept; /// \brief Copy constructor. result(const result& rhs); -#ifndef NANODBC_NO_MOVE_CTOR /// \brief Move constructor. - result(result&& rhs) NANODBC_NOEXCEPT; -#endif + result(result&& rhs) noexcept; /// \brief Assignment. result& operator=(result rhs); /// \brief Member swap. - void swap(result& rhs) NANODBC_NOEXCEPT; + void swap(result& rhs) noexcept; /// \brief Returns the native ODBC statement handle. void* native_statement_handle() const; /// \brief The rowset size for this result set. - long rowset_size() const NANODBC_NOEXCEPT; + long rowset_size() const noexcept; /// \brief Number of affected rows by the request or -1 if the affected rows is not available. /// \throws database_error long affected_rows() const; + /// \brief Reports if number of affected rows is available. + /// \return true if number of affected rows is known, regardless of the value; + /// false if the number is not available. + /// \throws database_error + /// \code{.cpp} + /// assert(r.has_affected_rows() == (r.affected_rows() >= 0)); + /// \endcode + bool has_affected_rows() const; + /// \brief Rows in the current rowset or 0 if the number of rows is not available. - long rows() const NANODBC_NOEXCEPT; + long rows() const noexcept; /// \brief Returns the number of columns in a result set. /// \throws database_error @@ -1210,7 +1297,7 @@ class result unsigned long position() const; /// \brief Returns true if there are no more results in the current result set. - bool at_end() const NANODBC_NOEXCEPT; + bool at_end() const noexcept; /// \brief Gets data from the given column of the current rowset. /// @@ -1409,7 +1496,7 @@ class result_iterator result_iterator() = default; /// Create result iterator for a given result set. - result_iterator(result& r) + explicit result_iterator(result& r) : result_(r) { ++(*this); @@ -1609,7 +1696,7 @@ class catalog }; /// \brief Creates catalog operating on database accessible through the specified connection. - catalog(connection& conn); + explicit catalog(connection& conn); /// \brief Creates result set with catalogs, schemas, tables, or table types. /// diff --git a/odbc/src/utfcpp/utf8.h b/odbc/src/utfcpp/utf8.h new file mode 100644 index 00000000..c2c85d6d --- /dev/null +++ b/odbc/src/utfcpp/utf8.h @@ -0,0 +1,38 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "utf8/checked.h" +#include "utf8/unchecked.h" + +#if __cplusplus >= 201103L // C++ 11 or later +#include "utf8/cpp11.h" +#endif // C++ 11 or later + +#endif // header guard diff --git a/odbc/src/utfcpp/utf8/checked.h b/odbc/src/utfcpp/utf8/checked.h new file mode 100644 index 00000000..c31861e0 --- /dev/null +++ b/odbc/src/utfcpp/utf8/checked.h @@ -0,0 +1,324 @@ +// Copyright 2006-2016 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" +#include + +namespace utf8 +{ + // Base for the exceptions that may be thrown from the library + class exception : public ::std::exception { + }; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public exception { + uint32_t cp; + public: + invalid_code_point(uint32_t codepoint) : cp(codepoint) {} + virtual const char* what() const throw() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + virtual const char* what() const throw() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + virtual const char* what() const throw() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public exception { + public: + virtual const char* what() const throw() { return "Not enough space"; } + }; + + /// The library API - functions intended to be called by the users + + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + out = utf8::append (replacement, out); + start = end; + break; + case internal::INVALID_LEAD: + out = utf8::append (replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append (replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } + } + return out; + } + + template + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + break; + case internal::NOT_ENOUGH_ROOM : + throw not_enough_room(); + case internal::INVALID_LEAD : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + throw invalid_utf8(*it); + case internal::INVALID_CODE_POINT : + throw invalid_code_point(cp); + } + return cp; + } + + template + uint32_t peek_next(octet_iterator it, octet_iterator end) + { + return utf8::next(it, end); + } + + template + uint32_t prior(octet_iterator& it, octet_iterator start) + { + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) + if (it == start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + return utf8::peek_next(it, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + const distance_type zero(0); + if (n < zero) { + // backward + for (distance_type i = n; i < zero; ++i) + utf8::prior(it, end); + } else { + // forward + for (distance_type i = zero; i < n; ++i) + utf8::next(it, end); + } + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::next(first, last); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); + + } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); + + result = utf8::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::next(start, end); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it, + const octet_iterator& rangestart, + const octet_iterator& rangeend) : + it(octet_it), range_start(rangestart), range_end(rangeend) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator == (const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + utf8::next(it, range_end); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator -- () + { + utf8::prior(it, range_start); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } + }; // class iterator + +} // namespace utf8 + +#endif //header guard + diff --git a/odbc/src/utfcpp/utf8/core.h b/odbc/src/utfcpp/utf8/core.h new file mode 100644 index 00000000..e007ca17 --- /dev/null +++ b/odbc/src/utfcpp/utf8/core.h @@ -0,0 +1,321 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +// Helper code - not intended to be directly called by the library users. May be changed at any time +namespace internal +{ + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint16_t LEAD_SURROGATE_MIN = 0xd800u; + const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; + const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const uint16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10) + const uint32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffffu; + + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((utf8::internal::mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_lead_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + } + + template + inline bool is_trail_surrogate(u16 cp) + { + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_code_point_valid(u32 cp) + { + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + } + + template + inline typename std::iterator_traits::difference_type + sequence_length(octet_iterator lead_it) + { + uint8_t lead = utf8::internal::mask8(*lead_it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + { + if (cp < 0x80) { + if (length != 1) + return true; + } + else if (cp < 0x800) { + if (length != 2) + return true; + } + else if (cp < 0x10000) { + if (length != 3) + return true; + } + + return false; + } + + enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; + + /// Helper for get_sequence_x + template + utf_error increase_safely(octet_iterator& it, octet_iterator end) + { + if (++it == end) + return NOT_ENOUGH_ROOM; + + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; + + return UTF8_OK; + } + + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + + /// get_sequence_x functions decode utf-8 sequences of the length x + template + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + return UTF8_OK; + } + + template + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + + return UTF8_OK; + } + + template + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + template + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR + + template + utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; + + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); + + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; + switch (length) { + case 0: + return INVALID_LEAD; + case 1: + err = utf8::internal::get_sequence_1(it, end, cp); + break; + case 2: + err = utf8::internal::get_sequence_2(it, end, cp); + break; + case 3: + err = utf8::internal::get_sequence_3(it, end, cp); + break; + case 4: + err = utf8::internal::get_sequence_4(it, end, cp); + break; + } + + if (err == UTF8_OK) { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { + if (!utf8::internal::is_overlong_sequence(cp, length)){ + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; + } + else + err = INVALID_CODE_POINT; + } + + // Failure branch - restore the original value of the iterator + it = original_it; + return err; + } + + template + inline utf_error validate_next(octet_iterator& it, octet_iterator end) { + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); + } + +} // namespace internal + + /// The library API - functions intended to be called by the users + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + if (err_code != internal::UTF8_OK) + return result; + } + return result; + } + + template + inline bool is_valid(octet_iterator start, octet_iterator end) + { + return (utf8::find_invalid(start, end) == end); + } + + template + inline bool starts_with_bom (octet_iterator it, octet_iterator end) + { + return ( + ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) + ); + } +} // namespace utf8 + +#endif // header guard + + diff --git a/odbc/src/utfcpp/utf8/cpp11.h b/odbc/src/utfcpp/utf8/cpp11.h new file mode 100644 index 00000000..d93961b0 --- /dev/null +++ b/odbc/src/utfcpp/utf8/cpp11.h @@ -0,0 +1,103 @@ +// Copyright 2018 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1 +#define UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1 + +#include "checked.h" +#include + +namespace utf8 +{ + + inline void append(char32_t cp, std::string& s) + { + append(uint32_t(cp), std::back_inserter(s)); + } + + inline std::string utf16to8(const std::u16string& s) + { + std::string result; + utf16to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u16string utf8to16(const std::string& s) + { + std::u16string result; + utf8to16(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::string utf32to8(const std::u32string& s) + { + std::string result; + utf32to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u32string utf8to32(const std::string& s) + { + std::u32string result; + utf8to32(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::size_t find_invalid(const std::string& s) + { + std::string::const_iterator invalid = find_invalid(s.begin(), s.end()); + return (invalid == s.end()) ? std::string::npos : (invalid - s.begin()); + } + + inline bool is_valid(const std::string& s) + { + return is_valid(s.begin(), s.end()); + } + + inline std::string replace_invalid(const std::string& s, char32_t replacement) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); + return result; + } + + inline std::string replace_invalid(const std::string& s) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline bool starts_with_bom(const std::string& s) + { + return starts_with_bom(s.begin(), s.end()); + } + +} // namespace utf8 + +#endif // header guard + diff --git a/odbc/src/utfcpp/utf8/unchecked.h b/odbc/src/utfcpp/utf8/unchecked.h new file mode 100644 index 00000000..c78419f6 --- /dev/null +++ b/odbc/src/utfcpp/utf8/unchecked.h @@ -0,0 +1,229 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" + +namespace utf8 +{ + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + switch (length) { + case 1: + break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } + ++it; + return cp; + } + + template + uint32_t peek_next(octet_iterator it) + { + return utf8::unchecked::next(it); + } + + template + uint32_t prior(octet_iterator& it) + { + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + + template + void advance (octet_iterator& it, distance_type n) + { + const distance_type zero(0); + if (n < zero) { + // backward + for (distance_type i = n; i < zero; ++i) + utf8::unchecked::prior(it); + } else { + // forward + for (distance_type i = zero; i < n; ++i) + utf8::unchecked::next(it); + } + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + } + result = utf8::unchecked::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::unchecked::next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::unchecked::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::unchecked::next(start); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it): it(octet_it) {} + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator == (const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator -- () + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } + }; // class iterator + + } // namespace utf8::unchecked +} // namespace utf8 + + +#endif // header guard + diff --git a/odbc/test_setup/setup.txt b/odbc/test_setup/setup.txt new file mode 100644 index 00000000..a9ab1770 --- /dev/null +++ b/odbc/test_setup/setup.txt @@ -0,0 +1,4 @@ +// assume unixODBC is installed and data source is properly configured +pluginCfgFile = "/home/ubuntu2/workspace/DolphinDBPlugin/odbc/DolphinDBODBC.txt"; +loadPlugin(pluginCfgFile); +PRECISIONLIMIT=3 diff --git a/odbc/tests/README.md b/odbc/tests/README.md new file mode 100644 index 00000000..ff631299 --- /dev/null +++ b/odbc/tests/README.md @@ -0,0 +1,65 @@ +# Tests for odbc plugin + +## Testing environment setup +### 1. install database instances +[https://www.digitalocean.com/community/tutorials/how-to-install-mysql-on-ubuntu-16-04](https://www.digitalocean.com/community/tutorials/how-to-install-mysql-on-ubuntu-16-04) +[https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04) +[https://www.microsoft.com/en-us/sql-server/sql-server-editions-express](https://www.microsoft.com/en-us/sql-server/sql-server-editions-express) + +### 2. importing testing database +* MySQL +``` +git clone https://github.com/zxjcarrot/test_db +cd test_db +mysql -u root -p < employees_partitioned.sql +``` +* Postgresql +use [pgadmin](https://www.pgadmin.org/) to restore backup file ```postgres.tar```. + +* SQL Server +use [SQL Server Management Studio](https://docs.microsoft.com/en-us/sql/ssms/download-sql-server-management-studio-ssms) to restore backup file ```mssql-employees.bak``` +### 3. Install ODBC drivers +``` +#install mysql odbc driver +apt-get install libmyodbc +#install postgresql odbc driver +apt-get install odbc-postgresql +``` +For ```mssql``` odbc drivers, check [this](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server) out. +> If you're having trouble installing ```libmyodbc``` on Ubuntu 16.04, try [this](https://www.datasunrise.com/blog/how-to-install-the-mysql-odbc-driver-on-ubuntu-16-04/). + +### 4. Configure ODBC datasource +Add follwing data source configurations to ```/etc/odbc.ini```. +``` +[mysql-employees] +Driver={Your driver name} +SERVER=127.0.0.1 +UID=root +PWD={Your Password} +PORT=3306 +DATABASE=employees + +[postgresql-employees] +Driver={Your driver name} +Servername=127.0.0.1 +Port=5432 +UserName=postgres +Password=postgres +Database=postgres + +[mssql-employees] +Driver={Your driver name} +Server=192.168.1.35 +Port=1433 +Username=sa +Password=sa +Database=employees +``` + +Look up ```/etc/odbcinst.ini``` for driver names and replace values to match your own environment. + +You can use follwing command to see if the setup is properly done. +``` +# connect to mysql-employees data source using isql tool +isql mysql-employees +``` diff --git a/odbc/tests/test_mssql.txt b/odbc/tests/test_mssql.txt new file mode 100644 index 00000000..2872704a --- /dev/null +++ b/odbc/tests/test_mssql.txt @@ -0,0 +1,498 @@ +//-----------------------------------mssql(express)---------------------------------------- +connStr="DSN=mssql-employees;Server=192.168.1.35;Uid=sa;Pwd=sa;Database=employees"; +databaseType="mssql"; + + +@testing:case=databaseType+"odbc_connect_normal"; +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; + +@testing:case=databaseType+"odbc_connect_fail", exception=1 +c=odbc::connect("gibberish"); +assert 1, isNull(c) == 0; + +@testing:case=databaseType+"odbc_query_connection_string" +assert 1, isNull(c) == 0; +t=odbc::query(connStr, "select * from employees"); +assert 2, t.rows() == 300024; +assert 3, t.cols() == 6; +t = select * from t order by emp_no +assert 4, t.emp_no[0] == 10001 +assert 5, typestr(t.emp_no[0]) == "INT" +assert 5, typestr(t.hire_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_query_connection_handle" +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; +t=odbc::query(c, "select * from employees"); +assert 2, t.rows() == 300024; +assert 3, t.cols() == 6; +t = select * from t order by emp_no +assert 4, t.emp_no[0] == 10001 +assert 5, typestr(t.emp_no[0]) == "INT" +assert 5, typestr(t.hire_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_query_after_close", exception=1 +c=odbc::connect(connStr); +t=odbc::query(c, "select * from innodb_index_stats"); +odbc::close(c); +t=odbc::query(c, "select * from innodb_index_stats"); + +@testing:case=databaseType+"odbc_close" +c=odbc::connect(connStr); +odbc::close(c); + +@testing:case=databaseType+"odbc_query_employees" +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; +t = odbc::query(c, "select * from employees"); +assert 2, (exec count(emp_no) from t) == 300024; +t = odbc::query(c, "select top(1025) * from employees"); +assert 3, (exec count(emp_no) from t) == 1025; +t = odbc::query(c, "select top(1023) * from employees"); +assert 4, (exec count(emp_no) from t) == 1023; +t = odbc::query(c, "select top(0) * from employees"); +assert 5, (exec count(emp_no) from t) == 0; + +// assume unixODBC is installed and data source is properly configured +@testing:case=databaseType+"odbc_query_employees_append" +c=odbc::connect(connStr); + +assert 1, isNull(c) == 0; +t = odbc::query(c, "select * from employees") +assert 2, (exec count(emp_no) from t) == 300024; +odbc::query(c, "select top(1000) * from employees", t) +assert 3, (exec count(emp_no) from t) == 301024; +odbc::query(c, "select top(1025) * from employees", t) +assert 4, (exec count(emp_no) from t) == 302049; +t = odbc::query(c, "select top(1000) * from employees", t) +assert 5, (exec count(emp_no) from t) == 303049; +t = odbc::query(c, "select top(0) * from employees", t) +assert 6, (exec count(emp_no) from t) == 303049; +t = odbc::query(c, "select * from salaries order by from_date"); +assert 7, (exec count(emp_no) from t) == 2844047; +assert 8, typestr(t.from_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_test_datatypes" +c=odbc::connect(connStr); +t = odbc::query(c, "select * from test_datatypes where id=1"); +assert 1, t.timestamp == 2017.06.15T13:29:34.001; +assert 2, t.date==2017.06.16 +assert 3, eqObj(t.float[0], double(21312.1), 1) +assert 4, eqObj(double(t.numeric[0]), double(1234213), 0) +assert 5, eqObj(t.double[0], double(5498498.498498498), 4) +assert 6, eqObj(t.int[0], 1) +assert 7, eqObj(t.varchar[0], "gibberish jabberish") +assert 8, eqObj(t.char[0], 'a') +assert 9, eqObj(t.tinyint[0], 127) +assert 10, eqObj(t.smallint[0], 32767) +assert 11, eqObj(t.bigint[0], 9223372036854775807l) + +t = odbc::query(c, "select * from test_datatypes where id=2"); +assert 12, t.timestamp[0] == 2017.06.15T13:30:41.000; +assert 13, t.date==0001.01.01 +assert 14, eqObj(t.float[0], double(-31231.2), 1) +assert 15, eqObj(double(t.numeric[0]), double(-224.5), 1) +assert 16, eqObj(t.double[0], double(-5498498.498498498), 4) +assert 17, eqObj(t.int[0], 2) +assert 18, eqObj(t.varchar[0], "asd") +assert 19, eqObj(t.char[0], 'b') +assert 20, eqObj(t.tinyint[0], short(-128)) +assert 21, isNull(t.smallint[0]) +assert 22, eqObj(t.bigint[0], -9223372036854775808l) + +t = odbc::query(c, "select * from test_datatypes where id=3"); +assert 23, t.timestamp[0] == 2017.06.15T11:47:37.989; +assert 24, t.date==date("1000.01.01") +assert 25, eqObj(t.float[0], double(0), 1) +assert 26, eqObj(double(t.numeric[0]), double(0), 0) +assert 27, eqObj(t.double[0], double(0), 4) +assert 28, eqObj(t.int[0], 3) +assert 29, eqObj(t.varchar[0], "ss121") +assert 30, eqObj(t.char[0], 'c') +assert 31, eqObj(t.tinyint[0], 0) +assert 32, eqObj(t.smallint[0], 0) +assert 33, eqObj(t.bigint[0], 0) + +t = odbc::query(c, "select * from test_datatypes") +assert 34, type(t.timestamp[0]) == 12 +assert 35, type(t.tinyint[0]) == 3 +assert 36, type(t.smallint[0]) == 3 +assert 37, type(t.int[0]) == 4 +assert 38, type(t.bigint[0]) == 5 +assert 39, type(t.float[0]) == 14 +assert 40, type(double(t.numeric[0])) == 14 +assert 41, type(t.double[0]) == 14 +assert 42, type(t.char[0]) == 2 +assert 43, type(t.varchar[0]) == 16 + +@testing:case=databaseType+"odbc_test_symbol" +c=odbc::connect(connStr); +t = odbc::query(c, "select * from test_symbols order by id"); +assert 1, type(t.symbol1) == 15 +assert 2, type(t.notSymbol1) == 16 +assert 3, type(t.symbol2) == 15 +assert 4, type(t.notSymbol2) == 16 +assert 5, trim(t.symbol1[0]) == "abc" +assert 6, trim(t.notSymbol1[0]) == "abcd" +assert 7, trim(t.symbol2[0]) == "aaaaaa" +assert 8, trim(t.notSymbol2[0]) == "bbbbbb" +assert 9, trim(t.symbol1[1]) == "def" +assert 10, trim(t.notSymbol1[1]) == "defdefdef" +assert 11, trim(t.symbol2[1]) == "ddd" +assert 12, trim(t.notSymbol2[1]) == "dddas" + + + +@testing:case=databaseType+"odbc_test_compatibility_symbol", exception=0 +c=odbc::connect(connStr); +id=[11121] +ts=[1]$timestamp +int1=[1] +tinyint=[1]$short +smallint=[1]$short +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], 1) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], 127) +assert 10, eqObj(t.smallint[1], 32767) +assert 11, eqObj(t.bigint[1], 9223372036854775807l) + +symbol1 = symbol(varchar); +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,symbol1,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.symbol1,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes", t) +assert 12, t.ts[1] == 2017.06.15T13:29:34.001; +assert 13, t.date1[1]==2017.06.16 +assert 14, eqObj(t.double1[1], double(21312.1), 1) +assert 15, eqObj(double(t.double2[1]), double(1234213), 0) +assert 16, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 17, eqObj(t.int1[1], 1) +assert 18, eqObj(t.symbol1[1], "gibberish jabberish") +assert 19, eqObj(t.char1[1], 'a') +assert 21, eqObj(t.tinyint[1], 127) +assert 22, eqObj(t.smallint[1], 32767) +assert 23, eqObj(t.bigint[1], 9223372036854775807l) + + +@testing:case=databaseType+"odbc_test_compatibility_upcast_int", exception=0 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$int +tinyint=[1]$int +smallint=[1]$int +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +t1 = odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], 1) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], 127) +assert 10, eqObj(t.smallint[1], 32767) +assert 11, eqObj(t.bigint[1], 9223372036854775807l) + +@testing:case=databaseType+"odbc_test_compatibility_upcast_long", exception=0 +id=[11121]$long +ts=[1]$timestamp +int1=[1]$long +tinyint=[1]$long +smallint=[1]$long +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], long(1)) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], long(127)) +assert 10, eqObj(t.smallint[1], long(32767)) +assert 11, eqObj(t.bigint[1], long(9223372036854775807l)) +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + +@testing:case=databaseType+"odbc_test_compatibility_upcast_double", exception=0 +id=[11121]$double +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], double(1), 4) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], double(127), 4) +assert 10, eqObj(t.smallint[1], double(32767), 4) +assert 11, eqObj(t.bigint[1], double(9223372036854775807l), 4) +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + +@testing:case=databaseType+"odbc_test_compatibility_upcast_timestamp", exception=0 +id=[11121]$double +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16T00:00:00.000 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], double(1), 4) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], double(127), 4) +assert 10, eqObj(t.smallint[1], double(32767), 4) +assert 11, eqObj(t.bigint[1], double(9223372036854775807l), 4) +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + + +@testing:case=databaseType+"odbc_test_compatibility_cast_string", exception=0 +id=[""] +ts=[""] +int1=[""] +tinyint=[""] +smallint=[""] +bigint=[""] +double1=[""] +double2=[""] +char1=[""] +varchar=[""] +double3=[""] +date1=[""] +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == "1"; +assert 1, t.ts[1] == "2017-06-15 13:29:34 +0000"; +assert 2, t.int1[1]== "1"; +assert 3, t.tinyint[1] == "127"; +assert 4, t.smallint[1] == "32767"; +assert 5, t.bigint[1] == "9223372036854775807"; +assert 6, eqObj(double(t.double1[1]), double("21312"), 0); +assert 7, eqObj(double(t.double2[1]), double("1234213.1"), 0); +assert 8, t.char1[1] == "a"; +assert 9, t.varchar[1] == "gibberish jabberish"; +assert 10, eqObj(double(t.double3[1]), double("5498498"), 0); +assert 11, t.date1[1] == "2017-06-16" +assert 12, t.id[2] == "2"; +assert 13, t.ts[2] == "2017-06-15 13:30:41 +0000"; +assert 14, t.int1[2]== "2"; +assert 15, t.tinyint[2] == "-128"; +assert 16, t.smallint[2] == "-32768"; +assert 17, t.bigint[2] == "-9223372036854775808"; +assert 18, eqObj(double(t.double1[2]), double("-31231"), 0); +assert 19, eqObj(double(t.double2[2]), double("-224.5"), 0); +assert 20, t.char1[2] == "b"; +assert 21, t.varchar[2] == "asd"; +assert 22, eqObj(double(t.double3[2]), double("-5498498"), 0); +assert 23, t.date1[2] == "1-01-01" + +@testing:case=databaseType+"odbc_test_compatibility_downcast_bool", exception=0 +id=[false]$bool +ts=[1]$timestamp +int1=[false]$bool +tinyint= [false]$bool +smallint=[false]$bool +bigint=[false]$bool +double1=[false]$bool +double2=[false]$bool +char1=[false]$bool +varchar=[""] +double3=[false]$bool +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == true +assert 2, t.int1[1] == true +assert 3, t.tinyint[1] == true +assert 4, t.smallint[1] == true +assert 5, t.bigint[1] == true +assert 6, t.double1[1] == true +assert 7, t.double3[1] == true +assert 8, t.tinyint[3] == false +assert 9, t.smallint[3] == false +assert 10, t.bigint[3] == false + +@testing:case=databaseType+"odbc_test_compatibility_downcast_short", exception=0 +id=[false]$short +ts=[1]$timestamp +int1=[false]$short +tinyint= [false]$short +smallint=[false]$short +bigint=[false]$short +double1=[false]$short +double2=[1]$short // nanodbc stores decimal & numeric values as chars +char1=[false]$short +varchar=[""] +double3=[false]$short +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == 1 +assert 2, t.int1[1] == 1 +assert 3, t.tinyint[1] == 127 +assert 4, t.smallint[1] == 32767 +assert 5, t.bigint[1] == -1 +//assert 6, t.double1[1] == 21312 // casting floats to ints is undefined behavior -- C99 spec, section 6.3.1.4 Real floating and integer. +//assert 8, t.double3[1] == -6526 + +@testing:case=databaseType+"odbc_test_compatibility_downcast_int", exception=0 +id=[0]$int +ts=[1]$timestamp +int1=[0]$int +tinyint= [0]$int +smallint=[0]$int +bigint=[0]$int +double1=[0]$int +double2=[1]$int +char1=[false]$int +varchar=[""] +double3=[false]$int +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == 1 +assert 2, t.int1[1] == 1 +assert 3, t.tinyint[1] == 127 +assert 4, t.smallint[1] == 32767 +assert 5, t.bigint[1] == -1 +//assert 6, t.double1[1] == 21312 // casting floats to ints is undefined behavior -- C99 spec, section 6.3.1.4 Real floating and integer. +//assert 8, t.double3[1] == 1618375405 + +@testing:case=databaseType+"odbc_test_compatibility_downcast_date", exception=0 +id=[0]$int +ts=[1]$date +int1=[0]$int +tinyint= [0]$int +smallint=[0]$int +bigint=[0]$int +double1=[0]$int +double2=[1]$int +char1=[false]$int +varchar=[""] +double3=[false]$int +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1]==2017.06.15 + +@testing:case=databaseType+"odbc_test_compatibility_wrong_column_count", exception=1 +t = table(1 as id); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type1", exception=1 +id=[11121]$int +ts=[1]$int // timestamp => int, should be incompatible +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type2", exception=1 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[1] // string => int, should be incompatible +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type3", exception=1 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[1]$timestamp // string => timestamp, should be incompatible +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + + +@testing:case=databaseType+"odbc_query_employees_append_empty",exception=1 +odbc::query(c, "", t); + +@testing:case=databaseType+"odbc_connect_invalid_arg",exception=1 +c=odbc::connect(connStr); +c=odbc::connect(c); diff --git a/odbc/tests/test_mysql.txt b/odbc/tests/test_mysql.txt new file mode 100644 index 00000000..d91c0b47 --- /dev/null +++ b/odbc/tests/test_mysql.txt @@ -0,0 +1,499 @@ +//-------------------------------mysql---------------------------------- +connStr="DSN=mysql-employees"; +databaseType="mysql" + +@testing:case=databaseType+"odbc_connect_normal"; +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; + +@testing:case=databaseType+"odbc_connect_fail", exception=1 +c=odbc::connect("gibberish"); +assert 1, isNull(c) == 0; + +@testing:case=databaseType+"odbc_query_connection_string" +assert 1, isNull(c) == 0; +t=odbc::query(connStr, "select * from employees"); +assert 2, t.rows() == 300024; +assert 3, t.cols() == 6; +t = select * from t order by emp_no +assert 4, t.emp_no[0] == 10001 +assert 5, typestr(t.emp_no[0]) == "INT" +assert 5, typestr(t.hire_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_query_connection_handle" +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; +t=odbc::query(c, "select * from employees"); +assert 2, t.rows() == 300024; +assert 3, t.cols() == 6; +t = select * from t order by emp_no +assert 4, t.emp_no[0] == 10001 +assert 5, typestr(t.emp_no[0]) == "INT" +assert 5, typestr(t.hire_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_query_after_close", exception=1 +c=odbc::connect(connStr); +t=odbc::query(c, "select * from innodb_index_stats"); +odbc::close(c); +t=odbc::query(c, "select * from innodb_index_stats"); + +@testing:case=databaseType+"odbc_close" +c=odbc::connect(connStr); +odbc::close(c); + +@testing:case=databaseType+"odbc_query_employees" +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; +t = odbc::query(c, "select * from employees"); +assert 2, (exec count(emp_no) from t) == 300024; +t = odbc::query(c, "select * from employees limit 0, 1025"); +assert 3, (exec count(emp_no) from t) == 1025; +t = odbc::query(c, "select * from employees limit 0, 1023"); +assert 4, (exec count(emp_no) from t) == 1023; +t = odbc::query(c, "select * from employees limit 0, 0"); +assert 5, (exec count(emp_no) from t) == 0; + +// assume unixODBC is installed and data source is properly configured +@testing:case=databaseType+"odbc_query_employees_append" +c=odbc::connect(connStr); + +assert 1, isNull(c) == 0; +t = odbc::query(c, "select * from employees") +assert 2, (exec count(emp_no) from t) == 300024; +odbc::query(c, "select * from employees limit 0, 1000", t) +assert 3, (exec count(emp_no) from t) == 301024; +odbc::query(c, "select * from employees limit 0, 1025", t) +assert 4, (exec count(emp_no) from t) == 302049; +t = odbc::query(c, "select * from employees limit 0, 1000", t) +assert 5, (exec count(emp_no) from t) == 303049; +t = odbc::query(c, "select * from employees limit 0, 0", t) +assert 6, (exec count(emp_no) from t) == 303049; +t = odbc::query(c, "select * from salaries order by from_date"); +assert 7, (exec count(emp_no) from t) == 2844047; +assert 8, typestr(t.from_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_test_datatypes" +c=odbc::connect(connStr); +t = odbc::query(c, "select * from test_datatypes where id=1"); +assert 1, t.timestamp == 2017.06.15T13:29:34.001; +assert 2, t.date==2017.06.16 +assert 3, eqObj(t.float[0], double(21312.1), 1) +assert 4, eqObj(double(t.numeric[0]), double(1234213), 0) +assert 5, eqObj(t.double[0], double(5498498.498498498), 4) +assert 6, eqObj(t.int[0], 1) +assert 7, eqObj(t.varchar[0], "gibberish jabberish") +assert 8, eqObj(t.char[0], 'a') +assert 9, eqObj(t.tinyint[0], 127) +assert 10, eqObj(t.smallint[0], 32767) +assert 11, eqObj(t.bigint[0], 9223372036854775807l) + +t = odbc::query(c, "select * from test_datatypes where id=2"); +assert 12, t.timestamp[0] == 2017.06.15T13:30:41.000; +assert 13, t.date==0001.01.01 +assert 14, eqObj(t.float[0], double(-31231.2), 1) +assert 15, eqObj(double(t.numeric[0]), double(-224.5), 1) +assert 16, eqObj(t.double[0], double(-5498498.498498498), 4) +assert 17, eqObj(t.int[0], 2) +assert 18, eqObj(t.varchar[0], "asd") +assert 19, eqObj(t.char[0], 'b') +assert 20, eqObj(t.tinyint[0], short(-128)) +assert 21, isNull(t.smallint[0]) +assert 22, eqObj(t.bigint[0], -9223372036854775808l) + +t = odbc::query(c, "select * from test_datatypes where id=3"); +assert 23, t.timestamp[0] == 2017.06.15T11:47:37.989; +assert 24, t.date==date("1000.01.01") +assert 25, eqObj(t.float[0], double(0), 1) +assert 26, eqObj(double(t.numeric[0]), double(0), 0) +assert 27, eqObj(t.double[0], double(0), 4) +assert 28, eqObj(t.int[0], 3) +assert 29, eqObj(t.varchar[0], "ss121") +assert 30, eqObj(t.char[0], 'c') +assert 31, eqObj(t.tinyint[0], 0) +assert 32, eqObj(t.smallint[0], 0) +assert 33, eqObj(t.bigint[0], 0) + +t = odbc::query(c, "select * from test_datatypes") +assert 34, type(t.timestamp[0]) == 12 +assert 35, type(t.tinyint[0]) == 3 +assert 36, type(t.smallint[0]) == 3 +assert 37, type(t.int[0]) == 4 +assert 38, type(t.bigint[0]) == 5 +assert 39, type(t.float[0]) == 14 +assert 40, type(double(t.numeric[0])) == 14 +assert 41, type(t.double[0]) == 14 +assert 42, type(t.char[0]) == 2 +assert 43, type(t.varchar[0]) == 16 + +@testing:case=databaseType+"odbc_test_symbol" +c=odbc::connect(connStr); +t = odbc::query(c, "select * from test_symbols order by id"); +assert 1, type(t.symbol1) == 15 +assert 2, type(t.notSymbol1) == 16 +assert 3, type(t.symbol2) == 15 +assert 4, type(t.notSymbol2) == 16 +assert 5, t.symbol1[0] == "abc" +assert 6, t.notSymbol1[0] == "abcd" +assert 7, t.symbol2[0] == "aaaaaa" +assert 8, t.notSymbol2[0] == "bbbbbb" +assert 9, t.symbol1[1] == "def" +assert 10, t.notSymbol1[1] == "defdefdef" +assert 11, t.symbol2[1] == "ddd" +assert 12, t.notSymbol2[1] == "dddas" + + +@testing:case=databaseType+"odbc_test_compatibility_symbol", exception=0 +c=odbc::connect(connStr); +id=[11121] +ts=[1]$timestamp +int1=[1] +tinyint=[1]$short +smallint=[1]$short +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], 1) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], 127) +assert 10, eqObj(t.smallint[1], 32767) +assert 11, eqObj(t.bigint[1], 9223372036854775807l) + +symbol1 = symbol(varchar); +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,symbol1,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.symbol1,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes", t) +assert 12, t.ts[1] == 2017.06.15T13:29:34.001; +assert 13, t.date1[1]==2017.06.16 +assert 14, eqObj(t.double1[1], double(21312.1), 1) +assert 15, eqObj(double(t.double2[1]), double(1234213), 0) +assert 16, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 17, eqObj(t.int1[1], 1) +assert 18, eqObj(t.symbol1[1], "gibberish jabberish") +assert 19, eqObj(t.char1[1], 'a') +assert 21, eqObj(t.tinyint[1], 127) +assert 22, eqObj(t.smallint[1], 32767) +assert 23, eqObj(t.bigint[1], 9223372036854775807l) + + +@testing:case=databaseType+"odbc_test_compatibility_upcast_int", exception=0 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$int +tinyint=[1]$int +smallint=[1]$int +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +t1 = odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], 1) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], 127) +assert 10, eqObj(t.smallint[1], 32767) +assert 11, eqObj(t.bigint[1], 9223372036854775807l) + +@testing:case=databaseType+"odbc_test_compatibility_upcast_long", exception=0 +id=[11121]$long +ts=[1]$timestamp +int1=[1]$long +tinyint=[1]$long +smallint=[1]$long +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], long(1)) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], long(127)) +assert 10, eqObj(t.smallint[1], long(32767)) +assert 11, eqObj(t.bigint[1], long(9223372036854775807l)) +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + +@testing:case=databaseType+"odbc_test_compatibility_upcast_double", exception=0 +id=[11121]$double +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], double(1), 4) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], double(127), 4) +assert 10, eqObj(t.smallint[1], double(32767), 4) +assert 11, eqObj(t.bigint[1], double(9223372036854775807l), 4) +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + +@testing:case=databaseType+"odbc_test_compatibility_upcast_timestamp", exception=0 +id=[11121]$double +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16T00:00:00.000 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], double(1), 4) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], double(127), 4) +assert 10, eqObj(t.smallint[1], double(32767), 4) +assert 11, eqObj(t.bigint[1], double(9223372036854775807l), 4) +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + + +@testing:case=databaseType+"odbc_test_compatibility_cast_string", exception=0 +id=[""] +ts=[""] +int1=[""] +tinyint=[""] +smallint=[""] +bigint=[""] +double1=[""] +double2=[""] +char1=[""] +varchar=[""] +double3=[""] +date1=[""] +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == "1"; +assert 1, t.ts[1] == "2017-06-15 13:29:34 +0000"; +assert 2, t.int1[1]== "1"; +assert 3, t.tinyint[1] == "127"; +assert 4, t.smallint[1] == "32767"; +assert 5, t.bigint[1] == "9223372036854775807"; +assert 6, t.double1[1] == "21312"; +assert 7, eqObj(double(t.double2[1]), double("1234213.1"), 0); +assert 8, t.char1[1] == "a"; +assert 9, t.varchar[1] == "gibberish jabberish"; +assert 10, t.double3[1] == "5498498" +assert 11, t.date1[1] == "2017-06-16" +assert 12, t.id[2] == "2"; +assert 13, t.ts[2] == "2017-06-15 13:30:41 +0000"; +assert 14, t.int1[2]== "2"; +assert 15, t.tinyint[2] == "-128"; +assert 16, t.smallint[2] == "-32768"; +assert 17, t.bigint[2] == "-9223372036854775808"; +assert 18, t.double1[2] == "-31231"; +assert 19, eqObj(double(t.double2[2]), double("-224.5"), 0); +assert 20, t.char1[2] == "b"; +assert 21, t.varchar[2] == "asd"; +assert 22, t.double3[2] == "-5498498" +assert 23, t.date1[2] == "1-01-01" +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + + +@testing:case=databaseType+"odbc_test_compatibility_downcast_bool", exception=0 +id=[false]$bool +ts=[1]$timestamp +int1=[false]$bool +tinyint= [false]$bool +smallint=[false]$bool +bigint=[false]$bool +double1=[false]$bool +double2=[false]$bool +char1=[false]$bool +varchar=[""] +double3=[false]$bool +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == true +assert 2, t.int1[1] == true +assert 3, t.tinyint[1] == true +assert 4, t.smallint[1] == true +assert 5, t.bigint[1] == true +assert 6, t.double1[1] == true +assert 7, t.double3[1] == true +assert 8, t.tinyint[3] == false +assert 9, t.smallint[3] == false +assert 10, t.bigint[3] == false + + +@testing:case=databaseType+"odbc_test_compatibility_downcast_short", exception=0 +id=[false]$short +ts=[1]$timestamp +int1=[false]$short +tinyint= [false]$short +smallint=[false]$short +bigint=[false]$short +double1=[false]$short +double2=[1]$short // nanodbc stores decimal & numeric values as chars +char1=[false]$short +varchar=[""] +double3=[false]$short +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == 1 +assert 2, t.int1[1] == 1 +assert 3, t.tinyint[1] == 127 +assert 4, t.smallint[1] == 32767 +assert 5, t.bigint[1] == -1 +//assert 6, t.double1[1] == 21312 // casting floats to ints is undefined behavior -- C99 spec, section 6.3.1.4 Real floating and integer. +//assert 8, t.double3[1] == -6526 + +@testing:case=databaseType+"odbc_test_compatibility_downcast_int", exception=0 +id=[0]$int +ts=[1]$timestamp +int1=[0]$int +tinyint= [0]$int +smallint=[0]$int +bigint=[0]$int +double1=[0]$int +double2=[1]$int +char1=[false]$int +varchar=[""] +double3=[false]$int +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == 1 +assert 2, t.int1[1] == 1 +assert 3, t.tinyint[1] == 127 +assert 4, t.smallint[1] == 32767 +assert 5, t.bigint[1] == -1 +//assert 6, t.double1[1] == 21312 // casting floats to ints is undefined behavior -- C99 spec, section 6.3.1.4 Real floating and integer. +//assert 8, t.double3[1] == 1618375405 + +@testing:case=databaseType+"odbc_test_compatibility_downcast_date", exception=0 +id=[0]$int +ts=[1]$date +int1=[0]$int +tinyint= [0]$int +smallint=[0]$int +bigint=[0]$int +double1=[0]$int +double2=[1]$int +char1=[false]$int +varchar=[""] +double3=[false]$int +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1]==2017.06.15 + +@testing:case=databaseType+"odbc_test_compatibility_wrong_column_count", exception=1 +t = table(1 as id); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type1", exception=1 +id=[11121]$int +ts=[1]$int // timestamp => int, should be incompatible +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type2", exception=1 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[1] // string => int, should be incompatible +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type3", exception=1 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[1]$timestamp // string => timestamp, should be incompatible +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + + +@testing:case=databaseType+"odbc_query_employees_append_empty",exception=1 +odbc::query(c, "", t); + +@testing:case=databaseType+"odbc_connect_invalid_arg",exception=1 +c=odbc::connect(connStr); +c=odbc::connect(c); diff --git a/odbc/tests/test_postgres.txt b/odbc/tests/test_postgres.txt new file mode 100644 index 00000000..efce73d6 --- /dev/null +++ b/odbc/tests/test_postgres.txt @@ -0,0 +1,489 @@ +//------------------------------------------postgresql------------------------------------------ + +use odbc; +connStr="DSN=postgresql-employees"; +databaseType="postgresql" + +@testing:case=databaseType+"odbc_connect_normalasdasdasd"; +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; + +@testing:case=databaseType+"odbc_query_connection_string" +assert 1, isNull(c) == 0; +t=odbc::query(connStr, "select * from employees"); +assert 2, t.rows() == 300024; +assert 3, t.cols() == 6; +t = select * from t order by emp_no +assert 4, t.emp_no[0] == 10001 +assert 5, typestr(t.emp_no[0]) == "INT" +assert 5, typestr(t.hire_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_query_connection_handle" +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; +t=odbc::query(c, "select * from employees"); +assert 2, t.rows() == 300024; +assert 3, t.cols() == 6; +t = select * from t order by emp_no +assert 4, t.emp_no[0] == 10001 +assert 5, typestr(t.emp_no[0]) == "INT" +assert 5, typestr(t.hire_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_query_after_close", exception=1 +c=odbc::connect(connStr); +t=odbc::query(c, "select * from innodb_index_stats"); +odbc::close(c); +t=odbc::query(c, "select * from innodb_index_stats"); + +@testing:case=databaseType+"odbc_close",exception=0 +c=odbc::connect(connStr); +odbc::close(c); + +@testing:case=databaseType+"odbc_query_employees" +c=odbc::connect(connStr); +assert 1, isNull(c) == 0; +t = odbc::query(c, "select * from employees"); +assert 2, (exec count(emp_no) from t) == 300024; +t = odbc::query(c, "select * from employees limit 1025"); +assert 3, (exec count(emp_no) from t) == 1025; +t = odbc::query(c, "select * from employees limit 1023"); +assert 4, (exec count(emp_no) from t) == 1023; +t = odbc::query(c, "select * from employees limit 0"); +assert 5, (exec count(emp_no) from t) == 0; + +// assume unixODBC is installed and data source is properly configured +@testing:case=databaseType+"odbc_query_employees_append" +c=odbc::connect(connStr); + +assert 1, isNull(c) == 0; +t = odbc::query(c, "select * from employees") +assert 2, (exec count(emp_no) from t) == 300024; +odbc::query(c, "select * from employees limit 1000", t) +assert 3, (exec count(emp_no) from t) == 301024; +odbc::query(c, "select * from employees limit 1025", t) +assert 4, (exec count(emp_no) from t) == 302049; +t = odbc::query(c, "select * from employees limit 1000", t) +assert 5, (exec count(emp_no) from t) == 303049; +t = odbc::query(c, "select * from employees limit 0", t) +assert 6, (exec count(emp_no) from t) == 303049; +t = odbc::query(c, "select * from salaries order by from_date"); +assert 7, (exec count(emp_no) from t) == 2844047; +assert 8, typestr(t.from_date[0]) == "DATE" + +@testing:case=databaseType+"odbc_test_datatypesasdas" +c=odbc::connect(connStr); +t = odbc::query(c, "select * from test_datatypes where id=1"); +assert 1, t.timestamp == 2017.06.15T13:29:34.001; +assert 2, t.date==2017.06.16 +assert 3, eqObj(t.float[0], double(21312.1), 1) +assert 4, eqObj(double(t.numeric[0]), double(1234213), 0) +assert 5, eqObj(t.double[0], double(5498498.498498498), 4) +assert 6, eqObj(t.int[0], 1) +assert 7, eqObj(t.varchar[0], "gibberish jabberish") +assert 8, eqObj(t.char[0], 'a') +assert 9, eqObj(t.tinyint[0], 127) +assert 10, eqObj(t.smallint[0], 32767) +assert 11, eqObj(t.bigint[0], 9223372036854775807l) + +t = odbc::query(c, "select * from test_datatypes where id=2"); +assert 12, t.timestamp == 2017.06.15T13:29:41.004 +assert 13, t.date==0001.01.01 +assert 14, eqObj(t.float[0], double(-31231.2), 1) +assert 15, eqObj(double(t.numeric[0]), double(-224.5), 1) +assert 16, eqObj(t.double[0], double(-5498498.498498498), 4) +assert 17, eqObj(t.int[0], 2) +assert 18, eqObj(t.varchar[0], "asd") +assert 19, eqObj(t.char[0], 'b') +assert 20, eqObj(t.tinyint[0], short(-128)) +assert 21, isNull(t.smallint[0]) +assert 22, eqObj(t.bigint[0], -9223372036854775808l) + +t = odbc::query(c, "select * from test_datatypes where id=3"); +assert 23, t.timestamp[0] == 2017.06.15T11:47:37.989; +assert 24, t.date==date("1000.01.01") +assert 25, eqObj(t.float[0], double(0), 1) +assert 26, eqObj(double(t.numeric[0]), double(0), 0) +assert 27, eqObj(t.double[0], double(0), 4) +assert 28, eqObj(t.int[0], 3) +assert 29, eqObj(t.varchar[0], "ss121") +assert 30, eqObj(t.char[0], 'c') +assert 31, eqObj(t.tinyint[0], 0) +assert 32, eqObj(t.smallint[0], 0) +assert 33, eqObj(t.bigint[0], 0) + +t = odbc::query(c, "select * from test_datatypes") +assert 34, type(t.timestamp[0]) == 14 +assert 35, type(t.tinyint[0]) == 3 +assert 36, type(t.smallint[0]) == 3 +assert 37, type(t.int[0]) == 4 +assert 38, type(t.bigint[0]) == 5 +assert 39, type(t.float[0]) == 16 +assert 40, type(double(t.numeric[0])) == 16 +assert 41, type(t.double[0]) == 16 +assert 42, type(t.char[0]) == 2 +assert 43, type(t.varchar[0]) == 18 + +@testing:case=databaseType+"odbc_test_symbol" +c=odbc::connect(connStr); +t = odbc::query(c, "select * from test_symbols order by id"); +assert 1, type(t.symbol1) == 17 +assert 2, type(t.notSymbol1) == 18 +assert 3, type(t.symbol2) == 17 +assert 4, type(t.notSymbol2) == 18 +assert 5, trim(t.symbol1[0]) == "abc" +assert 6, trim(t.notSymbol1[0]) == "abcd" +assert 7, trim(t.symbol2[0]) == "aaaaaa" +assert 8, trim(t.notSymbol2[0]) == "bbbbbb" +assert 9, trim(t.symbol1[1]) == "def" +assert 10, trim(t.notSymbol1[1]) == "defdefdef" +assert 11, trim(t.symbol2[1]) == "ddd" +assert 12, trim(t.notSymbol2[1]) == "dddas" + + +@testing:case=databaseType+"odbc_test_compatibility_symbol", exception=0 +c=odbc::connect(connStr); +id=[11121] +ts=[1]$timestamp +int1=[1] +tinyint=[1]$short +smallint=[1]$short +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], 1) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], 127) +assert 10, eqObj(t.smallint[1], 32767) +assert 11, eqObj(t.bigint[1], 9223372036854775807l) + +symbol1 = symbol(varchar); +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,symbol1,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 12, t.ts[1] == 2017.06.15T13:29:34.001; +assert 13, t.date1[1]==2017.06.16 +assert 14, eqObj(t.double1[1], double(21312.1), 1) +assert 15, eqObj(double(t.double2[1]), double(1234213), 0) +assert 16, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 17, eqObj(t.int1[1], 1) +assert 18, eqObj(t.symbol1[1], "gibberish jabberish") +assert 19, eqObj(t.char1[1], 'a') +assert 21, eqObj(t.tinyint[1], 127) +assert 22, eqObj(t.smallint[1], 32767) +assert 23, eqObj(t.bigint[1], 9223372036854775807l) + + +@testing:case=databaseType+"odbc_test_compatibility_upcast_int", exception=0 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$int +tinyint=[1]$int +smallint=[1]$int +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +t1 = odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], 1) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], 127) +assert 10, eqObj(t.smallint[1], 32767) +assert 11, eqObj(t.bigint[1], 9223372036854775807l) + +@testing:case=databaseType+"odbc_test_compatibility_upcast_long", exception=0 +id=[11121]$long +ts=[1]$timestamp +int1=[1]$long +tinyint=[1]$long +smallint=[1]$long +bigint=[1]$long +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], long(1)) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], long(127)) +assert 10, eqObj(t.smallint[1], long(32767)) +assert 11, eqObj(t.bigint[1], long(9223372036854775807l)) + +@testing:case=databaseType+"odbc_test_compatibility_upcast_double", exception=0 +id=[11121]$double +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], double(1), 4) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], double(127), 4) +assert 10, eqObj(t.smallint[1], double(32767), 4) +assert 11, eqObj(t.bigint[1], double(9223372036854775807l), 4) + +@testing:case=databaseType+"odbc_test_compatibility_upcast_timestamp", exception=0 +id=[11121]$double +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1] == 2017.06.15T13:29:34.001; +assert 2, t.date1[1]==2017.06.16T00:00:00.000 +assert 3, eqObj(t.double1[1], double(21312.1), 1) +assert 4, eqObj(double(t.double2[1]), double(1234213), 0) +assert 5, eqObj(t.double3[1], double(5498498.498498498), 4) +assert 6, eqObj(t.int1[1], double(1), 4) +assert 7, eqObj(t.varchar[1], "gibberish jabberish") +assert 8, eqObj(t.char1[1], 'a') +assert 9, eqObj(t.tinyint[1], double(127), 4) +assert 10, eqObj(t.smallint[1], double(32767), 4) +assert 11, eqObj(t.bigint[1], double(9223372036854775807l), 4) + +@testing:case=databaseType+"odbc_test_compatibility_cast_string", exception=0 +id=[""] +ts=[""] +int1=[""] +tinyint=[""] +smallint=[""] +bigint=[""] +double1=[""] +double2=[""] +char1=[""] +varchar=[""] +double3=[""] +date1=[""] +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == "1"; +assert 1, t.ts[1] == "2017-06-15 13:29:34 +0000"; +assert 2, t.int1[1]== "1"; +assert 3, t.tinyint[1] == "127"; +assert 4, t.smallint[1] == "32767"; +assert 5, t.bigint[1] == "9223372036854775807"; +assert 6, t.double1[1] == "21312"; +assert 7, eqObj(double(t.double2[1]), double("1234213.1"), 0); +assert 8, t.char1[1] == "a"; +assert 9, t.varchar[1] == "gibberish jabberish"; +assert 10, t.double3[1] == "5498498" +assert 11, t.date1[1] == "2017-06-16" +assert 12, t.id[2] == "2"; +assert 13, t.int1[2]== "2"; +assert 14, t.tinyint[2] == "-128"; +assert 15, t.smallint[2] == "-32768"; +assert 16, t.bigint[2] == "-9223372036854775808"; +assert 17, t.double1[2] == "-31231"; +assert 18, eqObj(double(t.double2[2]), double("-224.5"), 0); +assert 19, t.char1[2] == "b"; +assert 20, t.varchar[2] == "asd"; +assert 21, t.double3[2] == "-5498498" +assert 22, t.date1[2] == "1-01-01" + +@testing:case=databaseType+"odbc_test_compatibility_downcast_bool", exception=0 +id=[false]$bool +ts=[1]$timestamp +int1=[false]$bool +tinyint= [false]$bool +smallint=[false]$bool +bigint=[false]$bool +double1=[false]$bool +double2=[false]$bool +char1=[false]$bool +varchar=[""] +double3=[false]$bool +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == true +assert 2, t.int1[1] == true +assert 3, t.tinyint[1] == true +assert 4, t.smallint[1] == true +assert 5, t.bigint[1] == true +assert 6, t.double1[1] == true +assert 7, t.double3[1] == true +assert 8, t.tinyint[3] == false +assert 9, t.smallint[3] == false +assert 10, t.bigint[3] == false + + +@testing:case=databaseType+"odbc_test_compatibility_downcast_short", exception=0 +id=[false]$short +ts=[1]$timestamp +int1=[false]$short +tinyint= [false]$short +smallint=[false]$short +bigint=[false]$short +double1=[false]$short +double2=[1]$short // nanodbc stores decimal & numeric values as chars +char1=[false]$short +varchar=[""] +double3=[false]$short +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +[typestr t.id, typestr t.ts, typestr t.int1,typestr t.tinyint,typestr t.smallint,typestr t.bigint,typestr t.double1,typestr t.double2,typestr t.char1,typestr t.varchar,typestr t.double3,typestr t.date1]; + +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == 1 +assert 2, t.int1[1] == 1 +assert 3, t.tinyint[1] == 127 +assert 4, t.smallint[1] == 32767 +assert 5, t.bigint[1] == -1 +//assert 6, t.double1[1] == 21312 // casting floats to ints is undefined behavior -- C99 spec, section 6.3.1.4 Real floating and integer. +//assert 8, t.double3[1] == -6526 + +@testing:case=databaseType+"odbc_test_compatibility_downcast_int", exception=0 +id=[0]$int +ts=[1]$timestamp +int1=[0]$int +tinyint= [0]$int +smallint=[0]$int +bigint=[0]$int +double1=[0]$int +double2=[1]$int +char1=[false]$int +varchar=[""] +double3=[false]$int +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.id[1] == 1 +assert 2, t.int1[1] == 1 +assert 3, t.tinyint[1] == 127 +assert 4, t.smallint[1] == 32767 +assert 5, t.bigint[1] == -1 +//assert 6, t.double1[1] == 21312 // casting floats to ints is undefined behavior -- C99 spec, section 6.3.1.4 Real floating and integer. +//assert 8, t.double3[1] == 1618375405 + +@testing:case=databaseType+"odbc_test_compatibility_downcast_date", exception=0 +id=[0]$int +ts=[1]$date +int1=[0]$int +tinyint= [0]$int +smallint=[0]$int +bigint=[0]$int +double1=[0]$int +double2=[1]$int +char1=[false]$int +varchar=[""] +double3=[false]$int +date1=[1]$date +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) +assert 1, t.ts[1]==2017.06.15 + +@testing:case=databaseType+"odbc_test_compatibility_wrong_column_count", exception=1 +t = table(1 as id); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type1", exception=1 +id=[11121]$int +ts=[1]$int // timestamp => int, should be incompatible +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[""] +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type2", exception=1 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[1] // string => int, should be incompatible +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + +@testing:case=databaseType+"odbc_test_compatibility_wrong_type3", exception=1 +id=[11121]$int +ts=[1]$timestamp +int1=[1]$double +tinyint=[1]$double +smallint=[1]$double +bigint=[1]$double +double1=[1.1]$double +double2=[1.1]$double +char1=['c'] +varchar=[1]$timestamp // string => timestamp, should be incompatible +double3=[1.1]$double +date1=[1]$timestamp +t = table(id,ts,int1,tinyint,smallint,bigint,double1,double2,char1,varchar,double3,date1); +odbc::query(c, "select * from test_datatypes order by id", t) + + +@testing:case=databaseType+"odbc_query_employees_append_empty",exception=1 +odbc::query(c, "", t); + +@testing:case=databaseType+"odbc_connect_invalid_arg",exception=1 +c=odbc::connect(connStr); +c=odbc::connect(c); diff --git a/odbc/utfcpp_LICENSE b/odbc/utfcpp_LICENSE new file mode 100644 index 00000000..36b7cd93 --- /dev/null +++ b/odbc/utfcpp_LICENSE @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/odbc/windows mssql odbc guide/CMakeLists.txt b/odbc/windows mssql odbc guide/CMakeLists.txt new file mode 100644 index 00000000..1423701e --- /dev/null +++ b/odbc/windows mssql odbc guide/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.00) + +project(Odbc) + +include_directories("./include") +link_directories("C:/Users/zhu/Desktop/Odbc/") + +# aux_source_directory("./src" DolphinDB) + + +add_compile_options("-g" "-std=c++14" "-fPIC" "-DWINDOWS" "-Wall" "-D_WIN32_WINNT=0x0600" "-DWINVER=0x0600") + +add_library(PluginOdbc SHARED "./src/DolphinDBODBC.cpp " "./src/nanodbc.cpp") +target_link_libraries(PluginOdbc libDolphinDB.dll odbc32 odbccp32) diff --git a/odbc/windows mssql odbc guide/nanodbc.cpp b/odbc/windows mssql odbc guide/nanodbc.cpp new file mode 100644 index 00000000..1cecf400 --- /dev/null +++ b/odbc/windows mssql odbc guide/nanodbc.cpp @@ -0,0 +1,4981 @@ +/// \file nanodbc.cpp Implementation details. +#ifndef DOXYGEN + +// ASCII art banners are helpful for code editors with a minimap display. +// Generated with http://patorjk.com/software/taag/#p=display&v=0&f=Colossal + +#if defined(_MSC_VER) +#if _MSC_VER <= 1800 +// silence spurious Visual C++ warnings +#pragma warning(disable : 4244) // warning about integer conversion issues. +#pragma warning(disable : 4312) // warning about 64-bit portability issues. +#endif +#pragma warning(disable : 4996) // warning about deprecated declaration +#endif + +#include "nanodbc.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef __clang__ +#include +#endif + +// User may redefine NANODBC_ASSERT macro in nanodbc/nanodbc.h +#ifndef NANODBC_ASSERT +#include +#define NANODBC_ASSERT(expr) assert(expr) +#endif + +#ifdef NANODBC_ENABLE_BOOST +#include +#elif defined(__GNUC__) && (__GNUC__ < 5) +#include +#else +#include +#endif + +#ifdef __APPLE__ +// silence spurious OS X deprecation warnings +#define MAC_OS_X_VERSION_MIN_REQUIRED MAC_OS_X_VERSION_10_6 +#endif + +#ifdef _WIN32 +// needs to be included above sql.h for windows +#ifndef __MINGW32__ +#define NOMINMAX +#endif +#include +#endif + +#include +#include + +// Driver specific SQL data type defines. +// Microsoft has -150 thru -199 reserved for Microsoft SQL Server Native Client driver usage. +// Originally, defined in sqlncli.h (old SQL Server Native Client driver) +// and msodbcsql.h (new Microsoft ODBC Driver for SQL Server) +// See https://github.com/nanodbc/nanodbc/issues/18 +#ifndef SQL_SS_VARIANT +#define SQL_SS_VARIANT (-150) +#endif +#ifndef SQL_SS_XML +#define SQL_SS_XML (-152) +#endif +#ifndef SQL_SS_TABLE +#define SQL_SS_TABLE (-153) +#endif +#ifndef SQL_SS_TIME2 +#define SQL_SS_TIME2 (-154) +#endif +#ifndef SQL_SS_TIMESTAMPOFFSET +#define SQL_SS_TIMESTAMPOFFSET (-155) +#endif +// Large CLR User-Defined Types (ODBC) +// https://msdn.microsoft.com/en-us/library/bb677316.aspx +// Essentially, UDT is a varbinary type with additional metadata. +// Memory layout: SQLCHAR *(unsigned char *) +// C data type: SQL_C_BINARY +// Value: SQL_BINARY (-2) +#ifndef SQL_SS_UDT +#define SQL_SS_UDT (-151) // from sqlncli.h +#endif + +#ifndef SQL_NVARCHAR +#define SQL_NVARCHAR (-10) +#endif + +// SQL_SS_LENGTH_UNLIMITED is used to describe the max length of +// VARCHAR(max), VARBINARY(max), NVARCHAR(max), and XML columns +#ifndef SQL_SS_LENGTH_UNLIMITED +#define SQL_SS_LENGTH_UNLIMITED (0) +#endif + +// Max length of DBVARBINARY and DBVARCHAR, etc. +1 for zero byte +// MSDN: Large value data types are those that exceed the maximum row size of 8 KB +#define SQLSERVER_DBMAXCHAR (8000 + 1) + +// Default to ODBC version defined by NANODBC_ODBC_VERSION if provided. +#ifndef NANODBC_ODBC_VERSION +#ifdef SQL_OV_ODBC3_80 +// Otherwise, use ODBC v3.8 if it's available... +#define NANODBC_ODBC_VERSION SQL_OV_ODBC3_80 +#else +// or fallback to ODBC v3.x. +#define NANODBC_ODBC_VERSION SQL_OV_ODBC3 +#endif +#endif + +// clang-format off +// 888 888 d8b 888 +// 888 888 Y8P 888 +// 888 888 888 +// 888 888 88888b. 888 .d8888b .d88b. .d88888 .d88b. +// 888 888 888 "88b 888 d88P" d88""88b d88" 888 d8P Y8b +// 888 888 888 888 888 888 888 888 888 888 88888888 +// Y88b. .d88P 888 888 888 Y88b. Y88..88P Y88b 888 Y8b. +// "Y88888P" 888 888 888 "Y8888P "Y88P" "Y88888 "Y8888 +// MARK: Unicode - +// clang-format on + +// Import string types defined in header file, so we don't have to type nanodbc:: everywhere +using nanodbc::wide_char_t; +using nanodbc::wide_string; + +#ifdef NANODBC_ENABLE_UNICODE +#define NANODBC_FUNC(f) f##W +#define NANODBC_SQLCHAR SQLWCHAR +#else +#define NANODBC_FUNC(f) f +#define NANODBC_SQLCHAR SQLCHAR +#endif + +#ifdef NANODBC_USE_IODBC_WIDE_STRINGS +#define NANODBC_CODECVT_TYPE std::codecvt_utf8 +#else +#ifdef _MSC_VER +#define NANODBC_CODECVT_TYPE std::codecvt_utf8_utf16 +#else +#define NANODBC_CODECVT_TYPE std::codecvt_utf8_utf16 +#endif +#endif + +#if defined(_MSC_VER) +#ifndef NANODBC_ENABLE_UNICODE +// Disable unicode in sqlucode.h on Windows when NANODBC_ENABLE_UNICODE +// is not defined. This is required because unicode is enabled by +// default on many Windows systems. +#define SQL_NOUNICODEMAP +#endif +#endif + +// clang-format off +// .d88888b. 8888888b. 888888b. .d8888b. 888b d888 +// d88P" "Y88b 888 "Y88b 888 "88b d88P Y88b 8888b d8888 +// 888 888 888 888 888 .88P 888 888 88888b.d88888 +// 888 888 888 888 8888888K. 888 888Y88888P888 8888b. .d8888b 888d888 .d88b. .d8888b +// 888 888 888 888 888 "Y88b 888 888 Y888P 888 "88b d88P" 888P" d88""88b 88K +// 888 888 888 888 888 888 888 888 888 Y8P 888 .d888888 888 888 888 888 "Y8888b. +// Y88b. .d88P 888 .d88P 888 d88P Y88b d88P 888 " 888 888 888 Y88b. 888 Y88..88P X88 +// "Y88888P" 8888888P" 8888888P" "Y8888P" 888 888 "Y888888 "Y8888P 888 "Y88P" 88888P' +// MARK: ODBC Macros - +// clang-format on + +#define NANODBC_STRINGIZE_I(text) #text +#define NANODBC_STRINGIZE(text) NANODBC_STRINGIZE_I(text) + +// By making all calls to ODBC functions through this macro, we can easily get +// runtime debugging information of which ODBC functions are being called, +// in what order, and with what parameters by defining NANODBC_ODBC_API_DEBUG. +#ifdef NANODBC_ODBC_API_DEBUG +#include +#define NANODBC_CALL_RC(FUNC, RC, ...) \ + do \ + { \ + std::cerr << __FILE__ \ + ":" NANODBC_STRINGIZE(__LINE__) " " NANODBC_STRINGIZE(FUNC) "(" #__VA_ARGS__ ")" \ + << std::endl; \ + RC = FUNC(__VA_ARGS__); \ + } while (false) /**/ +#define NANODBC_CALL(FUNC, ...) \ + do \ + { \ + std::cerr << __FILE__ \ + ":" NANODBC_STRINGIZE(__LINE__) " " NANODBC_STRINGIZE(FUNC) "(" #__VA_ARGS__ ")" \ + << std::endl; \ + FUNC(__VA_ARGS__); \ + } while (false) /**/ +#else +#define NANODBC_CALL_RC(FUNC, RC, ...) RC = FUNC(__VA_ARGS__) +#define NANODBC_CALL(FUNC, ...) FUNC(__VA_ARGS__) +#endif + +// clang-format off +// 8888888888 888 888 888 888 d8b +// 888 888 888 888 888 Y8P +// 888 888 888 888 888 +// 8888888 888d888 888d888 .d88b. 888d888 8888888888 8888b. 88888b. .d88888 888 888 88888b. .d88b. +// 888 888P" 888P" d88""88b 888P" 888 888 "88b 888 "88b d88" 888 888 888 888 "88b d88P"88b +// 888 888 888 888 888 888 888 888 .d888888 888 888 888 888 888 888 888 888 888 888 +// 888 888 888 Y88..88P 888 888 888 888 888 888 888 Y88b 888 888 888 888 888 Y88b 888 +// 8888888888 888 888 "Y88P" 888 888 888 "Y888888 888 888 "Y88888 888 888 888 888 "Y88888 +// 888 +// Y8b d88P +// "Y88P" +// MARK: Error Handling - +// clang-format on + +namespace +{ +#ifdef NANODBC_ODBC_API_DEBUG +inline std::string return_code(RETCODE rc) +{ + switch (rc) + { + case SQL_SUCCESS: + return "SQL_SUCCESS"; + case SQL_SUCCESS_WITH_INFO: + return "SQL_SUCCESS_WITH_INFO"; + case SQL_ERROR: + return "SQL_ERROR"; + case SQL_INVALID_HANDLE: + return "SQL_INVALID_HANDLE"; + case SQL_NO_DATA: + return "SQL_NO_DATA"; + case SQL_NEED_DATA: + return "SQL_NEED_DATA"; + case SQL_STILL_EXECUTING: + return "SQL_STILL_EXECUTING"; + } + NANODBC_ASSERT(0); + return "unknown"; // should never make it here +} +#endif + +// Easy way to check if a return code signifies success. +inline bool success(RETCODE rc) +{ +#ifdef NANODBC_ODBC_API_DEBUG + std::cerr << "<-- rc: " << return_code(rc) << " | " << std::endl; +#endif + return rc == SQL_SUCCESS || rc == SQL_SUCCESS_WITH_INFO; +} + +#if __cpp_lib_nonmember_container_access >= 201411 || _MSC_VER +using std::size; +#else +template +constexpr std::size_t size(const T (&array)[N]) noexcept +{ + return N; +} +#endif + +template +inline std::size_t size(NANODBC_SQLCHAR const (&array)[N]) noexcept +{ + auto const n = std::char_traits::length(array); + NANODBC_ASSERT(n < N); + return n < N ? n : N - 1; +} + +inline void convert(const wide_string& in, std::string& out) +{ +#ifdef NANODBC_ENABLE_BOOST + using boost::locale::conv::utf_to_utf; + out = utf_to_utf(in.c_str(), in.c_str() + in.size()); +#elif defined(__GNUC__) && (__GNUC__ < 5) + std::vector characters(in.begin(), in.end()); + const wchar_t* source = characters.data(); + size_t size = wcsnrtombs(nullptr, &source, characters.size(), 0, nullptr); + if (size == std::string::npos) + throw std::range_error("UTF-16 -> UTF-8 conversion error"); + out.resize(size); + wcsnrtombs(&out[0], &source, characters.size(), out.length(), nullptr); +#elif defined(_MSC_VER) && (_MSC_VER >= 1900) + // Workaround for confirmed bug in VS2015 and VS2017 too + // See: https://connect.microsoft.com/VisualStudio/Feedback/Details/1403302 + auto p = reinterpret_cast(in.data()); + out = std::wstring_convert, unsigned short>().to_bytes( + p, p + in.size()); +#else + out = std::wstring_convert, wide_char_t>().to_bytes(in); +#endif +} + +inline void convert(const std::string& in, wide_string& out) +{ +#ifdef NANODBC_ENABLE_BOOST + using boost::locale::conv::utf_to_utf; + out = utf_to_utf(in.c_str(), in.c_str() + in.size()); +#elif defined(__GNUC__) && (__GNUC__ < 5) + const char* source = in.data(); + size_t size = mbsnrtowcs(nullptr, &source, in.length(), 0, nullptr); + if (size == std::string::npos) + throw std::range_error("UTF-8 -> UTF-16 conversion error"); + std::vector characters(size); + mbsnrtowcs(&characters[0], &source, in.length(), characters.size(), nullptr); + out = wide_string(characters.begin(), characters.end()); +#elif defined(_MSC_VER) && (_MSC_VER >= 1900) + // Workaround for confirmed bug in VS2015 and VS2017 too + // See: https://connect.microsoft.com/VisualStudio/Feedback/Details/1403302 + auto s = + std::wstring_convert, unsigned short>().from_bytes(in); + auto p = reinterpret_cast(s.data()); + out.assign(p, p + s.size()); +#else + out = std::wstring_convert, wide_char_t>().from_bytes(in); +#endif +} + +inline void convert(const wide_string& in, wide_string& out) +{ + out = in; +} + +inline void convert(const std::string& in, std::string& out) +{ + out = in; +} + +// Attempts to get the most recent ODBC error as a string. +// Always returns std::string, even in unicode mode. +inline std::string +recent_error(SQLHANDLE handle, SQLSMALLINT handle_type, long& native, std::string& state) +{ + nanodbc::string result; + std::string rvalue; + std::vector sql_message(SQL_MAX_MESSAGE_LENGTH); + sql_message[0] = '\0'; + + SQLINTEGER i = 1; + SQLINTEGER native_error; + SQLSMALLINT total_bytes; + NANODBC_SQLCHAR sql_state[6]; + RETCODE rc; + + do + { + NANODBC_CALL_RC( + NANODBC_FUNC(SQLGetDiagRec), + rc, + handle_type, + handle, + (SQLSMALLINT)i, + sql_state, + &native_error, + 0, + 0, + &total_bytes); + + if (success(rc) && total_bytes > 0) + sql_message.resize(static_cast(total_bytes) + 1); + + if (rc == SQL_NO_DATA) + break; + + NANODBC_CALL_RC( + NANODBC_FUNC(SQLGetDiagRec), + rc, + handle_type, + handle, + (SQLSMALLINT)i, + sql_state, + &native_error, + sql_message.data(), + (SQLSMALLINT)sql_message.size(), + &total_bytes); + + if (!success(rc)) + { + convert(result, rvalue); + return rvalue; + } + + if (!result.empty()) + result += ' '; + + result += nanodbc::string(sql_message.begin(), sql_message.end()); + i++; + +// NOTE: unixODBC using PostgreSQL and SQLite drivers crash if you call SQLGetDiagRec() +// more than once. So as a (terrible but the best possible) workaround just exit +// this loop early on non-Windows systems. +#ifndef _MSC_VER + break; +#endif + } while (rc != SQL_NO_DATA); + + convert(result, rvalue); + state = std::string(&sql_state[0], &sql_state[size(sql_state) - 1]); + native = native_error; + std::string status = state; + status += ": "; + status += rvalue; + + // some drivers insert \0 into error messages for unknown reasons + using std::replace; + replace(status.begin(), status.end(), '\0', ' '); + + return status; +} + +} // namespace + +namespace nanodbc +{ + +type_incompatible_error::type_incompatible_error() + : std::runtime_error("type incompatible") +{ +} + +const char* type_incompatible_error::what() const NANODBC_NOEXCEPT +{ + return std::runtime_error::what(); +} + +null_access_error::null_access_error() + : std::runtime_error("null access") +{ +} + +const char* null_access_error::what() const NANODBC_NOEXCEPT +{ + return std::runtime_error::what(); +} + +index_range_error::index_range_error() + : std::runtime_error("index out of range") +{ +} + +const char* index_range_error::what() const NANODBC_NOEXCEPT +{ + return std::runtime_error::what(); +} + +programming_error::programming_error(const std::string& info) + : std::runtime_error(info.c_str()) +{ +} + +const char* programming_error::what() const NANODBC_NOEXCEPT +{ + return std::runtime_error::what(); +} + +database_error::database_error(void* handle, short handle_type, const std::string& info) + : std::runtime_error(info) + , native_error(0) + , sql_state("00000") +{ + message = std::string(std::runtime_error::what()) + + recent_error(handle, handle_type, native_error, sql_state); +} + +const char* database_error::what() const NANODBC_NOEXCEPT +{ + return message.c_str(); +} + +const long database_error::native() const NANODBC_NOEXCEPT +{ + return native_error; +} + +const std::string database_error::state() const NANODBC_NOEXCEPT +{ + return sql_state; +} + +} // namespace nanodbc + +// Throwing exceptions using NANODBC_THROW_DATABASE_ERROR enables file name +// and line numbers to be inserted into the error message. Useful for debugging. +#define NANODBC_THROW_DATABASE_ERROR(handle, handle_type) \ + throw nanodbc::database_error( \ + handle, handle_type, __FILE__ ":" NANODBC_STRINGIZE(__LINE__) ": ") /**/ + +// clang-format off +// 8888888b. 888 d8b 888 +// 888 "Y88b 888 Y8P 888 +// 888 888 888 888 +// 888 888 .d88b. 888888 8888b. 888 888 .d8888b +// 888 888 d8P Y8b 888 "88b 888 888 88K +// 888 888 88888888 888 .d888888 888 888 "Y8888b. +// 888 .d88P Y8b. Y88b. 888 888 888 888 X88 +// 8888888P" "Y8888 "Y888 "Y888888 888 888 88888P' +// MARK: Details - +// clang-format on + +#if !defined(NANODBC_DISABLE_ASYNC) && defined(SQL_ATTR_ASYNC_STMT_EVENT) && \ + defined(SQL_API_SQLCOMPLETEASYNC) +#define NANODBC_DO_ASYNC_IMPL +#endif + +namespace +{ + +using namespace std; // if int64_t is in std namespace (in c++11) + +template +using is_integral8 = std::integral_constant< + bool, + std::is_integral::value && sizeof(T) == 1 && !std::is_same::value>; + +template +using is_integral16 = std::integral_constant< + bool, + std::is_integral::value && sizeof(T) == 2 && !std::is_same::value>; + +template +using is_integral32 = std::integral_constant< + bool, + std::is_integral::value && sizeof(T) == 4 && !std::is_same::value>; + +template +using is_integral64 = std::integral_constant::value && sizeof(T) == 8>; + +// A utility for calculating the ctype from the given type T. +// I essentially create a lookup table based on the MSDN ODBC documentation. +// See http://msdn.microsoft.com/en-us/library/windows/desktop/ms714556(v=vs.85).aspx +template +struct sql_ctype +{ +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_BINARY; +}; + +template +struct sql_ctype< + T, + typename std::enable_if::value && std::is_signed::value>::type> +{ + static const SQLSMALLINT value = SQL_C_SSHORT; +}; + +template +struct sql_ctype< + T, + typename std::enable_if::value && std::is_unsigned::value>::type> +{ + static const SQLSMALLINT value = SQL_C_USHORT; +}; + +template +struct sql_ctype< + T, + typename std::enable_if::value && std::is_signed::value>::type> +{ + static const SQLSMALLINT value = SQL_C_SLONG; +}; + +template +struct sql_ctype< + T, + typename std::enable_if::value && std::is_unsigned::value>::type> +{ + static const SQLSMALLINT value = SQL_C_ULONG; +}; + +template +struct sql_ctype< + T, + typename std::enable_if::value && std::is_signed::value>::type> +{ + static const SQLSMALLINT value = SQL_C_SBIGINT; +}; + +template +struct sql_ctype< + T, + typename std::enable_if::value && std::is_unsigned::value>::type> +{ + static const SQLSMALLINT value = SQL_C_UBIGINT; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_FLOAT; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_DOUBLE; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_WCHAR; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_WCHAR; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_CHAR; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_CHAR; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_DATE; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_TIME; +}; + +template <> +struct sql_ctype +{ + static const SQLSMALLINT value = SQL_C_TIMESTAMP; +}; + +// Encapsulates resources needed for column binding. +class bound_column +{ +public: + bound_column(const bound_column& rhs) = delete; + bound_column& operator=(bound_column rhs) = delete; + + bound_column() + : name_() + , column_(0) + , sqltype_(0) + , sqlsize_(0) + , scale_(0) + , ctype_(0) + , clen_(0) + , blob_(false) + , cbdata_(0) + , pdata_(0) + { + } + + ~bound_column() + { + delete[] cbdata_; + delete[] pdata_; + } + +public: + nanodbc::string name_; + short column_; + SQLSMALLINT sqltype_; + SQLULEN sqlsize_; + SQLSMALLINT scale_; + SQLSMALLINT ctype_; + SQLULEN clen_; + bool blob_; + nanodbc::null_type* cbdata_; + char* pdata_; +}; + +// Encapsulates properties of statement parameter. +// Parameter corresponds to parameter marker associated with a prepared SQL statement. +struct bound_parameter +{ + bound_parameter() = default; + + SQLULEN size_ = 0; // SQL data size of column or expression inbytes or characters + SQLUSMALLINT index_ = 0; // Zero-based index of parameter marker + SQLSMALLINT iotype_ = 0; // Input/Output type of parameter + SQLSMALLINT type_ = 0; // SQL data type of parameter + SQLSMALLINT scale_ = 0; // decimal digits of column or expression +}; + +// Encapsulates properties of buffer with data values bound to statement parameter. +template +struct bound_buffer +{ + bound_buffer() = default; + bound_buffer(T const* values, std::size_t size, std::size_t value_size = 0) + : values_(values) + , size_(size) + , value_size_(value_size) + { + } + + T const* values_ = nullptr; // Pointer to buffer for parameter's data + std::size_t size_ = 0; // Number of values (1 or length of array) + std::size_t value_size_ = 0; // Size of single value (max size). Zero, if ignored. +}; + +// Allocates the native ODBC handles. +inline void allocate_environment_handle(SQLHENV& env) +{ + RETCODE rc; + NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_ENV, SQL_NULL_HANDLE, &env); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(env, SQL_HANDLE_ENV); + + try + { + NANODBC_CALL_RC( + SQLSetEnvAttr, + rc, + env, + SQL_ATTR_ODBC_VERSION, + (SQLPOINTER)NANODBC_ODBC_VERSION, + SQL_IS_UINTEGER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(env, SQL_HANDLE_ENV); + } + catch (...) + { + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env); + throw; + } +} + +inline void allocate_handle(SQLHENV& env, SQLHDBC& conn) +{ + allocate_environment_handle(env); + + try + { + NANODBC_ASSERT(env); + RETCODE rc; + NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_DBC, env, &conn); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(env, SQL_HANDLE_ENV); + } + catch (...) + { + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env); + throw; + } +} + +} // namespace + +// clang-format off +// .d8888b. 888 d8b 8888888 888 +// d88P Y88b 888 Y8P 888 888 +// 888 888 888 888 888 +// 888 .d88b. 88888b. 88888b. .d88b. .d8888b 888888 888 .d88b. 88888b. 888 88888b.d88b. 88888b. 888 +// 888 d88""88b 888 "88b 888 "88b d8P Y8b d88P" 888 888 d88""88b 888 "88b 888 888 "888 "88b 888 "88b 888 +// 888 888 888 888 888 888 888 888 88888888 888 888 888 888 888 888 888 888 888 888 888 888 888 888 +// Y88b d88P Y88..88P 888 888 888 888 Y8b. Y88b. Y88b. 888 Y88..88P 888 888 888 888 888 888 888 d88P 888 +// "Y8888P" "Y88P" 888 888 888 888 "Y8888 "Y8888P "Y888 888 "Y88P" 888 888 8888888 888 888 888 88888P" 888 +// 888 +// 888 +// 888 +// MARK: Connection Impl - +// clang-format on + +namespace nanodbc +{ + +class connection::connection_impl +{ +public: + connection_impl(const connection_impl&) = delete; + connection_impl& operator=(const connection_impl&) = delete; + + connection_impl() + : env_(0) + , conn_(0) + , connected_(false) + , transactions_(0) + , rollback_(false) + { + allocate_handle(env_, conn_); + } + + connection_impl(const string& dsn, const string& user, const string& pass, long timeout) + : env_(0) + , conn_(0) + , connected_(false) + , transactions_(0) + , rollback_(false) + { + allocate_handle(env_, conn_); + try + { + connect(dsn, user, pass, timeout); + } + catch (...) + { + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_DBC, conn_); + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env_); + throw; + } + } + + connection_impl(const string& connection_string, long timeout) + : env_(0) + , conn_(0) + , connected_(false) + , transactions_(0) + , rollback_(false) + { + allocate_handle(env_, conn_); + try + { + connect(connection_string, timeout); + } + catch (...) + { + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_DBC, conn_); + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env_); + throw; + } + } + + ~connection_impl() NANODBC_NOEXCEPT + { + try + { + disconnect(); + } + catch (...) + { + // ignore exceptions thrown during disconnect + } + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_DBC, conn_); + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_ENV, env_); + } + +#if !defined(NANODBC_DISABLE_ASYNC) && defined(SQL_ATTR_ASYNC_DBC_EVENT) + void enable_async(void* event_handle) + { + RETCODE rc; + NANODBC_CALL_RC( + SQLSetConnectAttr, + rc, + conn_, + SQL_ATTR_ASYNC_DBC_FUNCTIONS_ENABLE, + (SQLPOINTER)SQL_ASYNC_DBC_ENABLE_ON, + SQL_IS_INTEGER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + + NANODBC_CALL_RC( + SQLSetConnectAttr, rc, conn_, SQL_ATTR_ASYNC_DBC_EVENT, event_handle, SQL_IS_POINTER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + } + + void async_complete() + { + RETCODE rc, arc; + NANODBC_CALL_RC(SQLCompleteAsync, rc, SQL_HANDLE_DBC, conn_, &arc); + if (!success(rc) || !success(arc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + + connected_ = true; + + NANODBC_CALL_RC( + SQLSetConnectAttr, + rc, + conn_, + SQL_ATTR_ASYNC_DBC_FUNCTIONS_ENABLE, + (SQLPOINTER)SQL_ASYNC_DBC_ENABLE_OFF, + SQL_IS_INTEGER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + } +#endif // !NANODBC_DISABLE_ASYNC && SQL_ATTR_ASYNC_DBC_EVENT + + RETCODE connect( + const string& dsn, + const string& user, + const string& pass, + long timeout, + void* event_handle = nullptr) + { + disconnect(); + + RETCODE rc; + NANODBC_CALL_RC(SQLFreeHandle, rc, SQL_HANDLE_DBC, conn_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + + NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_DBC, env_, &conn_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(env_, SQL_HANDLE_ENV); + + NANODBC_CALL_RC( + SQLSetConnectAttr, rc, conn_, SQL_LOGIN_TIMEOUT, (SQLPOINTER)(std::intptr_t)timeout, 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + +#if !defined(NANODBC_DISABLE_ASYNC) && defined(SQL_ATTR_ASYNC_DBC_EVENT) + if (event_handle != nullptr) + enable_async(event_handle); +#endif + + NANODBC_CALL_RC( + NANODBC_FUNC(SQLConnect), + rc, + conn_, + (NANODBC_SQLCHAR*)dsn.c_str(), + SQL_NTS, + !user.empty() ? (NANODBC_SQLCHAR*)user.c_str() : 0, + SQL_NTS, + !pass.empty() ? (NANODBC_SQLCHAR*)pass.c_str() : 0, + SQL_NTS); + if (!success(rc) && (event_handle == nullptr || rc != SQL_STILL_EXECUTING)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + + connected_ = success(rc); + + return rc; + } + + RETCODE + connect(const string& connection_string, long timeout, void* event_handle = nullptr) + { + disconnect(); + + RETCODE rc; + NANODBC_CALL_RC(SQLFreeHandle, rc, SQL_HANDLE_DBC, conn_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + + NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_DBC, env_, &conn_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(env_, SQL_HANDLE_ENV); + + NANODBC_CALL_RC( + SQLSetConnectAttr, rc, conn_, SQL_LOGIN_TIMEOUT, (SQLPOINTER)(std::intptr_t)timeout, 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + +#if !defined(NANODBC_DISABLE_ASYNC) && defined(SQL_ATTR_ASYNC_DBC_EVENT) + if (event_handle != nullptr) + enable_async(event_handle); +#endif + + NANODBC_CALL_RC( + NANODBC_FUNC(SQLDriverConnect), + rc, + conn_, + 0, + (NANODBC_SQLCHAR*)connection_string.c_str(), + SQL_NTS, + nullptr, + 0, + nullptr, + SQL_DRIVER_NOPROMPT); + if (!success(rc) && (event_handle == nullptr || rc != SQL_STILL_EXECUTING)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + + connected_ = success(rc); + + return rc; + } + + bool connected() const { return connected_; } + + void disconnect() + { + if (connected()) + { + RETCODE rc; + NANODBC_CALL_RC(SQLDisconnect, rc, conn_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + } + connected_ = false; + } + + std::size_t transactions() const { return transactions_; } + + void* native_dbc_handle() const { return conn_; } + + void* native_env_handle() const { return env_; } + + template + T get_info(short info_type) const + { + return get_info_impl(info_type); + } + string dbms_name() const; + + string dbms_version() const; + + string driver_name() const; + + string database_name() const; + + string catalog_name() const + { + NANODBC_SQLCHAR name[SQL_MAX_OPTION_STRING_LENGTH] = {0}; + SQLINTEGER length(0); + RETCODE rc; + NANODBC_CALL_RC( + NANODBC_FUNC(SQLGetConnectAttr), + rc, + conn_, + SQL_ATTR_CURRENT_CATALOG, + name, + sizeof(name) / sizeof(NANODBC_SQLCHAR), + &length); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + return string(&name[0], &name[size(name)]); + } + + std::size_t ref_transaction() { return ++transactions_; } + + std::size_t unref_transaction() + { + if (transactions_ > 0) + --transactions_; + return transactions_; + } + + bool rollback() const { return rollback_; } + + void rollback(bool onoff) { rollback_ = onoff; } + +private: + template + T get_info_impl(short info_type) const; + + HENV env_; + HDBC conn_; + bool connected_; + std::size_t transactions_; + bool rollback_; // if true, this connection is marked for eventual transaction rollback +}; + +template +T connection::connection_impl::get_info_impl(short info_type) const +{ + T value; + RETCODE rc; + NANODBC_CALL_RC(NANODBC_FUNC(SQLGetInfo), rc, conn_, info_type, &value, 0, nullptr); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + return value; +} + +template <> +string connection::connection_impl::get_info_impl(short info_type) const +{ + NANODBC_SQLCHAR value[1024] = {0}; + SQLSMALLINT length(0); + RETCODE rc; + NANODBC_CALL_RC( + NANODBC_FUNC(SQLGetInfo), + rc, + conn_, + info_type, + value, + sizeof(value) / sizeof(NANODBC_SQLCHAR), + &length); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_, SQL_HANDLE_DBC); + return string(&value[0], &value[size(value)]); +} + +string connection::connection_impl::dbms_name() const +{ + return get_info(SQL_DBMS_NAME); +} + +string connection::connection_impl::dbms_version() const +{ + return get_info(SQL_DBMS_VER); +} + +string connection::connection_impl::driver_name() const +{ + return get_info(SQL_DRIVER_NAME); +} + +string connection::connection_impl::database_name() const +{ + return get_info(SQL_DATABASE_NAME); +} + +template string connection::get_info(short info_type) const; +template unsigned short connection::get_info(short info_type) const; +template uint32_t connection::get_info(short info_type) const; +template uint64_t connection::get_info(short info_type) const; + +} // namespace nanodbc + +// clang-format off +// 88888888888 888 d8b 8888888 888 +// 888 888 Y8P 888 888 +// 888 888 888 888 +// 888 888d888 8888b. 88888b. .d8888b 8888b. .d8888b 888888 888 .d88b. 88888b. 888 88888b.d88b. 88888b. 888 +// 888 888P" "88b 888 "88b 88K "88b d88P" 888 888 d88""88b 888 "88b 888 888 "888 "88b 888 "88b 888 +// 888 888 .d888888 888 888 "Y8888b. .d888888 888 888 888 888 888 888 888 888 888 888 888 888 888 888 +// 888 888 888 888 888 888 X88 888 888 Y88b. Y88b. 888 Y88..88P 888 888 888 888 888 888 888 d88P 888 +// 888 888 "Y888888 888 888 88888P' "Y888888 "Y8888P "Y888 888 "Y88P" 888 888 8888888 888 888 888 88888P" 888 +// 888 +// 888 +// 888 +// MARK: Transaction Impl - +// clang-format on + +namespace nanodbc +{ + +class transaction::transaction_impl +{ +public: + transaction_impl(const transaction_impl&) = delete; + transaction_impl& operator=(const transaction_impl&) = delete; + + transaction_impl(const class connection& conn) + : conn_(conn) + , committed_(false) + { + if (conn_.transactions() == 0 && conn_.connected()) + { + RETCODE rc; + NANODBC_CALL_RC( + SQLSetConnectAttr, + rc, + conn_.native_dbc_handle(), + SQL_ATTR_AUTOCOMMIT, + (SQLPOINTER)SQL_AUTOCOMMIT_OFF, + SQL_IS_UINTEGER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_.native_dbc_handle(), SQL_HANDLE_DBC); + } + conn_.ref_transaction(); + } + + ~transaction_impl() NANODBC_NOEXCEPT + { + if (!committed_) + { + conn_.rollback(true); + conn_.unref_transaction(); + } + + if (conn_.transactions() == 0 && conn_.connected()) + { + if (conn_.rollback()) + { + NANODBC_CALL(SQLEndTran, SQL_HANDLE_DBC, conn_.native_dbc_handle(), SQL_ROLLBACK); + conn_.rollback(false); + } + + NANODBC_CALL( + SQLSetConnectAttr, + conn_.native_dbc_handle(), + SQL_ATTR_AUTOCOMMIT, + (SQLPOINTER)SQL_AUTOCOMMIT_ON, + SQL_IS_UINTEGER); + } + } + + void commit() + { + if (committed_) + return; + committed_ = true; + if (conn_.unref_transaction() == 0 && conn_.connected()) + { + RETCODE rc; + NANODBC_CALL_RC(SQLEndTran, rc, SQL_HANDLE_DBC, conn_.native_dbc_handle(), SQL_COMMIT); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(conn_.native_dbc_handle(), SQL_HANDLE_DBC); + } + } + + void rollback() NANODBC_NOEXCEPT + { + if (committed_) + return; + conn_.rollback(true); + } + + class connection& connection() { return conn_; } + + const class connection& connection() const { return conn_; } + +private: + class connection conn_; + bool committed_; +}; + +} // namespace nanodbc + +// clang-format off +// .d8888b. 888 888 888 8888888 888 +// d88P Y88b 888 888 888 888 888 +// Y88b. 888 888 888 888 888 +// "Y888b. 888888 8888b. 888888 .d88b. 88888b.d88b. .d88b. 88888b. 888888 888 88888b.d88b. 88888b. 888 +// "Y88b. 888 "88b 888 d8P Y8b 888 "888 "88b d8P Y8b 888 "88b 888 888 888 "888 "88b 888 "88b 888 +// "888 888 .d888888 888 88888888 888 888 888 88888888 888 888 888 888 888 888 888 888 888 888 +// Y88b d88P Y88b. 888 888 Y88b. Y8b. 888 888 888 Y8b. 888 888 Y88b. 888 888 888 888 888 d88P 888 +// "Y8888P" "Y888 "Y888888 "Y888 "Y8888 888 888 888 "Y8888 888 888 "Y888 8888888 888 888 888 88888P" 888 +// 888 +// 888 +// 888 +// MARK: Statement Impl - +// clang-format on + +namespace nanodbc +{ + +class statement::statement_impl +{ +public: + statement_impl(const statement_impl&) = delete; + statement_impl& operator=(const statement_impl&) = delete; + + statement_impl() + : stmt_(0) + , open_(false) + , conn_() + , bind_len_or_null_() +#if defined(NANODBC_DO_ASYNC_IMPL) + , async_(false) + , async_enabled_(false) + , async_event_(nullptr) +#endif + { + } + + statement_impl(class connection& conn) + : stmt_(0) + , open_(false) + , conn_() + , bind_len_or_null_() + , wide_string_data_() + , string_data_() + , binary_data_() +#if defined(NANODBC_DO_ASYNC_IMPL) + , async_(false) + , async_enabled_(false) + , async_event_(nullptr) +#endif + { + open(conn); + } + + statement_impl(class connection& conn, const string& query, long timeout) + : stmt_(0) + , open_(false) + , conn_() + , bind_len_or_null_() + , wide_string_data_() + , string_data_() + , binary_data_() +#if defined(NANODBC_DO_ASYNC_IMPL) + , async_(false) + , async_enabled_(false) + , async_event_(nullptr) +#endif + { + prepare(conn, query, timeout); + } + + ~statement_impl() NANODBC_NOEXCEPT + { + if (open() && connected()) + { + NANODBC_CALL(SQLCancel, stmt_); + reset_parameters(); + NANODBC_CALL(SQLFreeHandle, SQL_HANDLE_STMT, stmt_); + } + } + + void open(class connection& conn) + { + close(); + RETCODE rc; + NANODBC_CALL_RC(SQLAllocHandle, rc, SQL_HANDLE_STMT, conn.native_dbc_handle(), &stmt_); + open_ = success(rc); + if (!open_) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + conn_ = conn; + } + + bool open() const { return open_; } + + bool connected() const { return conn_.connected(); } + + const class connection& connection() const { return conn_; } + + class connection& connection() { return conn_; } + + void* native_statement_handle() const { return stmt_; } + + void close() + { + if (open() && connected()) + { + RETCODE rc; + NANODBC_CALL_RC(SQLCancel, rc, stmt_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + reset_parameters(); + + NANODBC_CALL_RC(SQLFreeHandle, rc, SQL_HANDLE_STMT, stmt_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + + open_ = false; + stmt_ = 0; + } + + void cancel() + { + RETCODE rc; + NANODBC_CALL_RC(SQLCancel, rc, stmt_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + + void prepare(class connection& conn, const string& query, long timeout) + { + open(conn); + prepare(query, timeout); + } + + RETCODE prepare(const string& query, long timeout, void* event_handle = nullptr) + { + if (!open()) + throw programming_error("statement has no associated open connection"); + +#if defined(NANODBC_DO_ASYNC_IMPL) + if (event_handle == nullptr) + disable_async(); + else + enable_async(event_handle); +#endif + + RETCODE rc; + NANODBC_CALL_RC( + NANODBC_FUNC(SQLPrepare), + rc, + stmt_, + (NANODBC_SQLCHAR*)query.c_str(), + (SQLINTEGER)query.size()); + if (!success(rc) && rc != SQL_STILL_EXECUTING) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + this->timeout(timeout); + + return rc; + } + + void timeout(long timeout) + { + RETCODE rc; + NANODBC_CALL_RC( + SQLSetStmtAttr, + rc, + stmt_, + SQL_ATTR_QUERY_TIMEOUT, + (SQLPOINTER)(std::intptr_t)timeout, + 0); + + // some drivers don't support timeout for statements, + // so only raise the error if a non-default timeout was requested. + if (!success(rc) && (timeout != 0)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + +#if defined(NANODBC_DO_ASYNC_IMPL) + void enable_async(void* event_handle) + { + RETCODE rc; + if (!async_enabled_) + { + NANODBC_CALL_RC( + SQLSetStmtAttr, + rc, + stmt_, + SQL_ATTR_ASYNC_ENABLE, + (SQLPOINTER)SQL_ASYNC_ENABLE_ON, + SQL_IS_INTEGER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + async_enabled_ = true; + } + + if (async_event_ != event_handle) + { + NANODBC_CALL_RC( + SQLSetStmtAttr, rc, stmt_, SQL_ATTR_ASYNC_STMT_EVENT, event_handle, SQL_IS_POINTER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + async_event_ = event_handle; + } + } + + void disable_async() const + { + if (async_enabled_) + { + RETCODE rc; + NANODBC_CALL_RC( + SQLSetStmtAttr, + rc, + stmt_, + SQL_ATTR_ASYNC_ENABLE, + (SQLPOINTER)SQL_ASYNC_ENABLE_OFF, + SQL_IS_INTEGER); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + async_enabled_ = false; + } + } + + bool async_helper(RETCODE rc) + { + if (rc == SQL_STILL_EXECUTING) + { + async_ = true; + return true; + } + else if (success(rc)) + { + async_ = false; + return false; + } + else + { + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + } + + bool async_prepare(const string& query, void* event_handle, long timeout) + { + return async_helper(prepare(query, timeout, event_handle)); + } + + bool async_execute_direct( + class connection& conn, + void* event_handle, + const string& query, + long batch_operations, + long timeout, + statement& statement) + { + return async_helper( + just_execute_direct(conn, query, batch_operations, timeout, statement, event_handle)); + } + + bool + async_execute(void* event_handle, long batch_operations, long timeout, statement& statement) + { + return async_helper(just_execute(batch_operations, timeout, statement, event_handle)); + } + + void call_complete_async() + { + if (async_) + { + RETCODE rc, arc; + NANODBC_CALL_RC(SQLCompleteAsync, rc, SQL_HANDLE_STMT, stmt_, &arc); + if (!success(rc) || !success(arc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + } + + result complete_execute(long batch_operations, statement& statement) + { + call_complete_async(); + + return result(statement, batch_operations); + } + + void complete_prepare() { call_complete_async(); } + +#endif + result execute_direct( + class connection& conn, + const string& query, + long batch_operations, + long timeout, + statement& statement) + { +#ifdef NANODBC_ENABLE_WORKAROUND_NODATA + const RETCODE rc = just_execute_direct(conn, query, batch_operations, timeout, statement); + if (rc == SQL_NO_DATA) + return result(); +#else + just_execute_direct(conn, query, batch_operations, timeout, statement); +#endif + return result(statement, batch_operations); + } + + RETCODE just_execute_direct( + class connection& conn, + const string& query, + long batch_operations, + long timeout, + statement&, // statement + void* event_handle = nullptr) + { + open(conn); + +#if defined(NANODBC_DO_ASYNC_IMPL) + if (event_handle == nullptr) + disable_async(); + else + enable_async(event_handle); +#endif + + RETCODE rc; + NANODBC_CALL_RC( + SQLSetStmtAttr, + rc, + stmt_, + SQL_ATTR_PARAMSET_SIZE, + (SQLPOINTER)(std::intptr_t)batch_operations, + 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + this->timeout(timeout); + + NANODBC_CALL_RC( + NANODBC_FUNC(SQLExecDirect), rc, stmt_, (NANODBC_SQLCHAR*)query.c_str(), SQL_NTS); + if (!success(rc) && rc != SQL_NO_DATA && rc != SQL_STILL_EXECUTING) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + return rc; + } + + result execute(long batch_operations, long timeout, statement& statement) + { +#ifdef NANODBC_ENABLE_WORKAROUND_NODATA + const RETCODE rc = just_execute(batch_operations, timeout, statement); + if (rc == SQL_NO_DATA) + return result(); +#else + just_execute(batch_operations, timeout, statement); +#endif + return result(statement, batch_operations); + } + + RETCODE just_execute( + long batch_operations, + long timeout, + statement& /*statement*/, + void* event_handle = nullptr) + { + RETCODE rc; + + if (open()) + { + // The ODBC cursor must be closed before subsequent executions, as described + // here + // http://msdn.microsoft.com/en-us/library/windows/desktop/ms713584%28v=vs.85%29.aspx + // + // However, we don't necessarily want to call SQLCloseCursor() because that + // will cause an invalid cursor state in the case that no cursor is currently open. + // A better solution is to use SQLFreeStmt() with the SQL_CLOSE option, which has + // the same effect without the undesired limitations. + NANODBC_CALL_RC(SQLFreeStmt, rc, stmt_, SQL_CLOSE); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + +#if defined(NANODBC_DO_ASYNC_IMPL) + if (event_handle == nullptr) + disable_async(); + else + enable_async(event_handle); +#endif + + NANODBC_CALL_RC( + SQLSetStmtAttr, + rc, + stmt_, + SQL_ATTR_PARAMSET_SIZE, + (SQLPOINTER)(std::intptr_t)batch_operations, + 0); + if (!success(rc) && rc != SQL_NO_DATA) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + this->timeout(timeout); + + NANODBC_CALL_RC(SQLExecute, rc, stmt_); + if (!success(rc) && rc != SQL_NO_DATA && rc != SQL_STILL_EXECUTING) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + return rc; + } + + result procedure_columns( + const string& catalog, + const string& schema, + const string& procedure, + const string& column, + statement& statement) + { + if (!open()) + throw programming_error("statement has no associated open connection"); + +#if defined(NANODBC_DO_ASYNC_IMPL) + disable_async(); +#endif + + RETCODE rc; + NANODBC_CALL_RC( + NANODBC_FUNC(SQLProcedureColumns), + rc, + stmt_, + (NANODBC_SQLCHAR*)(catalog.empty() ? nullptr : catalog.c_str()), + (catalog.empty() ? 0 : SQL_NTS), + (NANODBC_SQLCHAR*)(schema.empty() ? nullptr : schema.c_str()), + (schema.empty() ? 0 : SQL_NTS), + (NANODBC_SQLCHAR*)procedure.c_str(), + SQL_NTS, + (NANODBC_SQLCHAR*)(column.empty() ? nullptr : column.c_str()), + (column.empty() ? 0 : SQL_NTS)); + + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + return result(statement, 1); + } + + long affected_rows() const + { + SQLLEN rows; + RETCODE rc; + NANODBC_CALL_RC(SQLRowCount, rc, stmt_, &rows); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + NANODBC_ASSERT(rows <= static_cast(std::numeric_limits::max())); + return static_cast(rows); + } + + short columns() const + { + SQLSMALLINT cols; + RETCODE rc; + +#if defined(NANODBC_DO_ASYNC_IMPL) + disable_async(); +#endif + + NANODBC_CALL_RC(SQLNumResultCols, rc, stmt_, &cols); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + return cols; + } + + void reset_parameters() NANODBC_NOEXCEPT { NANODBC_CALL(SQLFreeStmt, stmt_, SQL_RESET_PARAMS); } + + short parameters() const + { + SQLSMALLINT params; + RETCODE rc; + +#if defined(NANODBC_DO_ASYNC_IMPL) + disable_async(); +#endif + + NANODBC_CALL_RC(SQLNumParams, rc, stmt_, ¶ms); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + return params; + } + + unsigned long parameter_size(short param_index) const + { + RETCODE rc; + SQLSMALLINT data_type; + SQLSMALLINT nullable; + SQLULEN parameter_size; + +#if defined(NANODBC_DO_ASYNC_IMPL) + disable_async(); +#endif + + NANODBC_CALL_RC( + SQLDescribeParam, + rc, + stmt_, + param_index + 1, + &data_type, + ¶meter_size, + 0, + &nullable); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + NANODBC_ASSERT( + parameter_size < static_cast(std::numeric_limits::max())); + return static_cast(parameter_size); + } + + static SQLSMALLINT param_type_from_direction(param_direction direction) + { + switch (direction) + { + case PARAM_IN: + return SQL_PARAM_INPUT; + break; + case PARAM_OUT: + return SQL_PARAM_OUTPUT; + break; + case PARAM_INOUT: + return SQL_PARAM_INPUT_OUTPUT; + break; + case PARAM_RETURN: + return SQL_PARAM_OUTPUT; + break; + default: + NANODBC_ASSERT(false); + throw programming_error("unrecognized param_direction value"); + } + } + + // initializes bind_len_or_null_ and gets information for bind + void prepare_bind( + short param_index, + std::size_t batch_size, + param_direction direction, + bound_parameter& param) + { + NANODBC_ASSERT(param_index >= 0); + +#if defined(NANODBC_DO_ASYNC_IMPL) + disable_async(); +#endif + + RETCODE rc; + SQLSMALLINT nullable; // unused + NANODBC_CALL_RC( + SQLDescribeParam, + rc, + stmt_, + param_index + 1, + ¶m.type_, + ¶m.size_, + ¶m.scale_, + &nullable); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + + param.index_ = param_index; + param.iotype_ = param_type_from_direction(direction); + + if (!bind_len_or_null_.count(param_index)) + bind_len_or_null_[param_index] = std::vector(); + std::vector().swap(bind_len_or_null_[param_index]); + + // ODBC weirdness: this must be at least 8 elements in size + const std::size_t indicator_size = batch_size > 8 ? batch_size : 8; + bind_len_or_null_[param_index].reserve(indicator_size); + bind_len_or_null_[param_index].assign(indicator_size, SQL_NULL_DATA); + + NANODBC_ASSERT(param.index_ == param_index); + NANODBC_ASSERT(param.iotype_ > 0); + } + + // calls actual ODBC bind parameter function + template ::value, int>::type = 0> + void bind_parameter(bound_parameter const& param, bound_buffer& buffer) + { + NANODBC_ASSERT(buffer.value_size_ > 0 || param.size_ > 0); + + auto value_size{buffer.value_size_}; + if (value_size == 0) + value_size = param.size_; + + auto param_size{param.size_}; + if (value_size > param_size) + { + // Parameter size reported by SQLDescribeParam for Large Objects: + // - For SQL VARBINARY(MAX), it is Zero which actually means SQL_SS_LENGTH_UNLIMITED. + // - For SQL UDT (eg. GEOMETRY), it may be driver-specific max limit (eg. SQL Server is + // DBMAXCHAR=8000 bytes). + // See MSDN for details + // https://docs.microsoft.com/en-us/sql/relational-databases/native-client/odbc/large-clr-user-defined-types-odbc + // + // If bound value is larger than parameter size, we force SQL_SS_LENGTH_UNLIMITED. + param_size = SQL_SS_LENGTH_UNLIMITED; + } + + RETCODE rc; + NANODBC_CALL_RC( + SQLBindParameter, + rc, + stmt_, // handle + param.index_ + 1, // parameter number + param.iotype_, // input or output type + sql_ctype::value, // value type + param.type_, // parameter type + param_size, // column size ignored for many types, but needed for strings + param.scale_, // decimal digits + (SQLPOINTER)buffer.values_, // parameter value + value_size, // buffer length + bind_len_or_null_[param.index_].data()); + + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + + // Supports code like: query.bind(0, std_string.c_str()) + // In this case, we need to pass nullptr to the final parameter of SQLBindParameter(). + template ::value, int>::type = 0> + void bind_parameter(bound_parameter const& param, bound_buffer& buffer) + { + auto const buffer_size = buffer.value_size_ > 0 ? buffer.value_size_ : param.size_; + + RETCODE rc; + NANODBC_CALL_RC( + SQLBindParameter, + rc, + stmt_, // handle + param.index_ + 1, // parameter number + param.iotype_, // input or output type + sql_ctype::value, // value type + param.type_, // parameter type + param.size_, // column size ignored for many types, but needed for strings + param.scale_, // decimal digits + (SQLPOINTER)buffer.values_, // parameter value + buffer_size, // buffer length + (buffer.size_ <= 1 ? nullptr : bind_len_or_null_[param.index_].data())); + + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + + template + void bind( + param_direction direction, + short param_index, + T const* values, + std::size_t batch_size, + bool const* nulls = nullptr, + T const* null_sentry = nullptr); + + // handles multiple binary values + void bind( + param_direction direction, + short param_index, + std::vector> const& values, + bool const* nulls = nullptr, + uint8_t const* null_sentry = nullptr) + { + std::size_t batch_size = values.size(); + bound_parameter param; + prepare_bind(param_index, batch_size, direction, param); + + size_t max_length = 0; + for (std::size_t i = 0; i < batch_size; ++i) + { + max_length = std::max(values[i].size(), max_length); + } + binary_data_[param_index] = std::vector(batch_size * max_length, 0); + for (std::size_t i = 0; i < batch_size; ++i) + { + std::copy( + values[i].begin(), + values[i].end(), + binary_data_[param_index].data() + (i * max_length)); + } + + if (null_sentry) + { + for (std::size_t i = 0; i < batch_size; ++i) + if (!std::equal(values[i].begin(), values[i].end(), null_sentry)) + { + bind_len_or_null_[param_index][i] = values[i].size(); + } + } + else if (nulls) + { + for (std::size_t i = 0; i < batch_size; ++i) + { + if (!nulls[i]) + bind_len_or_null_[param_index][i] = values[i].size(); // null terminated + } + } + else + { + for (std::size_t i = 0; i < batch_size; ++i) + { + bind_len_or_null_[param_index][i] = values[i].size(); + } + } + bound_buffer buffer(binary_data_[param_index].data(), batch_size, max_length); + bind_parameter(param, buffer); + } + + template > + void bind_strings( + param_direction direction, + short param_index, + T const* values, + std::size_t value_size, + std::size_t batch_size, + bool const* nulls = nullptr, + T const* null_sentry = nullptr); + + template > + void bind_strings( + param_direction direction, + short param_index, + std::vector const& values, + bool const* nulls = nullptr, + typename T::value_type const* null_sentry = nullptr); + + // handles multiple null values + void bind_null(short param_index, std::size_t batch_size) + { + bound_parameter param; + prepare_bind(param_index, batch_size, PARAM_IN, param); + + RETCODE rc; + NANODBC_CALL_RC( + SQLBindParameter, + rc, + stmt_, + param.index_ + 1, // parameter number + param.iotype_, // input or output typ, + SQL_C_CHAR, + param.type_, // parameter type + param.size_, // column size ignored for many types, but needed for string, + 0, // decimal digits + nullptr, // null value + 0, // buffe length + bind_len_or_null_[param.index_].data()); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_, SQL_HANDLE_STMT); + } + + // comparator for null sentry values + template + bool equals(const T& lhs, const T& rhs) + { + return lhs == rhs; + } + + template + std::vector& get_bound_string_data(short param_index); + +private: + HSTMT stmt_; + bool open_; + class connection conn_; + std::map> bind_len_or_null_; + std::map> wide_string_data_; + std::map> string_data_; + std::map> binary_data_; + +#if defined(NANODBC_DO_ASYNC_IMPL) + bool async_; // true if statement is currently in SQL_STILL_EXECUTING mode + mutable bool async_enabled_; // true if statement currently has SQL_ATTR_ASYNC_ENABLE = + // SQL_ASYNC_ENABLE_ON + void* async_event_; // currently active event handle for async notifications +#endif +}; + +template +void statement::statement_impl::bind( + param_direction direction, + short param_index, + T const* values, + std::size_t batch_size, + bool const* nulls /*= nullptr*/, + T const* null_sentry /*= nullptr*/) +{ + bound_parameter param; + prepare_bind(param_index, batch_size, direction, param); + + if (nulls || null_sentry) + { + for (std::size_t i = 0; i < batch_size; ++i) + if ((null_sentry && !equals(values[i], *null_sentry)) || (nulls && !nulls[i]) || !nulls) + bind_len_or_null_[param_index][i] = param.size_; + } + else + { + for (std::size_t i = 0; i < batch_size; ++i) + bind_len_or_null_[param_index][i] = param.size_; + } + + bound_buffer buffer(values, batch_size); + bind_parameter(param, buffer); +} + +template +void statement::statement_impl::bind_strings( + param_direction direction, + short param_index, + std::vector const& values, + bool const* nulls /*= nullptr*/, + typename T::value_type const* null_sentry /*= nullptr*/) +{ + using string_vector = std::vector; + string_vector& string_data = get_bound_string_data(param_index); + + size_t const batch_size = values.size(); + bound_parameter param; + prepare_bind(param_index, batch_size, direction, param); + + size_t max_length = 0; + for (std::size_t i = 0; i < batch_size; ++i) + { + max_length = std::max(values[i].length(), max_length); + } + // add space for null terminator + ++max_length; + + string_data = string_vector(batch_size * max_length, 0); + for (std::size_t i = 0; i < batch_size; ++i) + { + std::copy(values[i].begin(), values[i].end(), string_data.data() + (i * max_length)); + } + bind_strings( + direction, param_index, string_data.data(), max_length, batch_size, nulls, null_sentry); +} + +template +void statement::statement_impl::bind_strings( + param_direction direction, + short param_index, + T const* values, + std::size_t value_size, + std::size_t batch_size, + bool const* nulls /*= nullptr*/, + T const* null_sentry /*= nullptr*/) +{ + bound_parameter param; + prepare_bind(param_index, batch_size, direction, param); + + if (null_sentry) + { + for (std::size_t i = 0; i < batch_size; ++i) + { + const std::basic_string s_lhs( + values + i * value_size, values + (i + 1) * value_size); + const std::basic_string s_rhs(null_sentry); + if (!equals(s_lhs, s_rhs)) + bind_len_or_null_[param_index][i] = SQL_NTS; + } + } + else if (nulls) + { + for (std::size_t i = 0; i < batch_size; ++i) + { + if (!nulls[i]) + bind_len_or_null_[param_index][i] = SQL_NTS; // null terminated + } + } + else + { + for (std::size_t i = 0; i < batch_size; ++i) + { + bind_len_or_null_[param_index][i] = SQL_NTS; + } + } + + auto const buffer_length = value_size * sizeof(T); + bound_buffer buffer(values, batch_size, buffer_length); + bind_parameter(param, buffer); +} + +template <> +bool statement::statement_impl::equals(const std::string& lhs, const std::string& rhs) +{ + return std::strncmp(lhs.c_str(), rhs.c_str(), lhs.size()) == 0; +} + +template <> +bool statement::statement_impl::equals(const wide_string& lhs, const wide_string& rhs) +{ + // e6059ff3a79062f83256b9d1d3c9c8368798781e + // Functions like `swprintf()`, `wcsftime()`, `wcsncmp()` can not be used + // with `u16string` types. Instead, prefers to narrow unicode string to + // work with them, and then widen them after work has been completed. + std::string narrow_lhs; + narrow_lhs.reserve(lhs.size()); + convert(lhs, narrow_lhs); + std::string narrow_rhs; + narrow_rhs.reserve(rhs.size()); + convert(rhs, narrow_rhs); + return equals(narrow_lhs, narrow_rhs); +} + +template <> +bool statement::statement_impl::equals(const date& lhs, const date& rhs) +{ + return lhs.year == rhs.year && lhs.month == rhs.month && lhs.day == rhs.day; +} + +template <> +bool statement::statement_impl::equals(const time& lhs, const time& rhs) +{ + return lhs.hour == rhs.hour && lhs.min == rhs.min && lhs.sec == rhs.sec; +} + +template <> +bool statement::statement_impl::equals(const timestamp& lhs, const timestamp& rhs) +{ + return lhs.year == rhs.year && lhs.month == rhs.month && lhs.day == rhs.day && + lhs.hour == rhs.hour && lhs.min == rhs.min && lhs.sec == rhs.sec && + lhs.fract == rhs.fract; +} + +template <> +std::vector& +statement::statement_impl::get_bound_string_data(short param_index) +{ + return wide_string_data_[param_index]; +} + +template <> +std::vector& +statement::statement_impl::get_bound_string_data(short param_index) +{ + return string_data_[param_index]; +} + +} // namespace nanodbc + +// clang-format off +// 8888888b. 888 888 8888888 888 +// 888 Y88b 888 888 888 888 +// 888 888 888 888 888 888 +// 888 d88P .d88b. .d8888b 888 888 888 888888 888 88888b.d88b. 88888b. 888 +// 8888888P" d8P Y8b 88K 888 888 888 888 888 888 "888 "88b 888 "88b 888 +// 888 T88b 88888888 "Y8888b. 888 888 888 888 888 888 888 888 888 888 888 +// 888 T88b Y8b. X88 Y88b 888 888 Y88b. 888 888 888 888 888 d88P 888 +// 888 T88b "Y8888 88888P' "Y88888 888 "Y888 8888888 888 888 888 88888P" 888 +// 888 +// 888 +// 888 +// MARK: Result Impl - +// clang-format on + +namespace nanodbc +{ + +class result::result_impl +{ +public: + result_impl(const result_impl&) = delete; + result_impl& operator=(const result_impl&) = delete; + + result_impl(statement stmt, long rowset_size) + : stmt_(stmt) + , rowset_size_(rowset_size) + , row_count_(0) + , bound_columns_(0) + , bound_columns_size_(0) + , rowset_position_(0) + , bound_columns_by_name_() + , at_end_(false) +#if defined(NANODBC_DO_ASYNC_IMPL) + , async_(false) +#endif + { + RETCODE rc; + NANODBC_CALL_RC( + SQLSetStmtAttr, + rc, + stmt_.native_statement_handle(), + SQL_ATTR_ROW_ARRAY_SIZE, + (SQLPOINTER)(std::intptr_t)rowset_size_, + 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + + NANODBC_CALL_RC( + SQLSetStmtAttr, + rc, + stmt_.native_statement_handle(), + SQL_ATTR_ROWS_FETCHED_PTR, + &row_count_, + 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + + auto_bind(); + } + + ~result_impl() NANODBC_NOEXCEPT { cleanup_bound_columns(); } + + void* native_statement_handle() const { return stmt_.native_statement_handle(); } + + long rowset_size() const { return rowset_size_; } + + long affected_rows() const { return stmt_.affected_rows(); } + + long rows() const NANODBC_NOEXCEPT + { + NANODBC_ASSERT(row_count_ <= static_cast(std::numeric_limits::max())); + return static_cast(row_count_); + } + + short columns() const { return stmt_.columns(); } + + bool first() + { + rowset_position_ = 0; + return fetch(0, SQL_FETCH_FIRST); + } + + bool last() + { + rowset_position_ = 0; + return fetch(0, SQL_FETCH_LAST); + } + + bool next(void* event_handle = nullptr) + { + if (rows() && ++rowset_position_ < rowset_size_) + return rowset_position_ < rows(); + rowset_position_ = 0; + return fetch(0, SQL_FETCH_NEXT, event_handle); + } + +#if defined(NANODBC_DO_ASYNC_IMPL) + bool async_next(void* event_handle) + { + async_ = next(event_handle); + return async_; + } + + bool complete_next() + { + if (async_) + { + RETCODE rc, arc; + NANODBC_CALL_RC( + SQLCompleteAsync, rc, SQL_HANDLE_STMT, stmt_.native_statement_handle(), &arc); + if (arc == SQL_NO_DATA) + { + at_end_ = true; + return false; + } + if (!success(rc) || !success(arc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + async_ = false; + } + return !at_end_; + } +#endif + + bool prior() + { + if (rows() && --rowset_position_ >= 0) + return true; + rowset_position_ = 0; + return fetch(0, SQL_FETCH_PRIOR); + } + + bool move(long row) + { + rowset_position_ = 0; + return fetch(row, SQL_FETCH_ABSOLUTE); + } + + bool skip(long rows) + { + rowset_position_ += rows; + if (this->rows() && rowset_position_ < rowset_size_) + return rowset_position_ < this->rows(); + rowset_position_ = 0; + return fetch(rows, SQL_FETCH_RELATIVE); + } + + unsigned long position() const + { + SQLULEN pos = 0; // necessary to initialize to 0 + RETCODE rc; + NANODBC_CALL_RC( + SQLGetStmtAttr, + rc, + stmt_.native_statement_handle(), + SQL_ATTR_ROW_NUMBER, + &pos, + SQL_IS_UINTEGER, + 0); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + + // MSDN (https://msdn.microsoft.com/en-us/library/ms712631.aspx): + // If the number of the current row cannot be determined or + // there is no current row, the driver returns 0. + // Otherwise, valid row number is returned, starting at 1. + // + // NOTE: We try to address incorrect implementation in some drivers (e.g. SQLite ODBC) + // which instead of 0 return SQL_ROW_NUMBER_UNKNOWN(-2) . + if (pos == 0 || pos == static_cast(SQL_ROW_NUMBER_UNKNOWN)) + return 0; + + NANODBC_ASSERT(pos < static_cast(std::numeric_limits::max())); + return static_cast(pos) + rowset_position_; + } + + bool at_end() const NANODBC_NOEXCEPT + { + if (at_end_) + return true; + SQLULEN pos = 0; // necessary to initialize to 0 + RETCODE rc; + NANODBC_CALL_RC( + SQLGetStmtAttr, + rc, + stmt_.native_statement_handle(), + SQL_ATTR_ROW_NUMBER, + &pos, + SQL_IS_UINTEGER, + 0); + return (!success(rc) || rows() < 0 || pos - 1 > static_cast(rows())); + } + + bool is_null(short column) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + bound_column& col = bound_columns_[column]; + if (rowset_position_ >= rows()) + throw index_range_error(); + return col.cbdata_[static_cast(rowset_position_)] == SQL_NULL_DATA; + } + + bool is_null(const string& column_name) const + { + const short column = this->column(column_name); + return is_null(column); + } + + short column(const string& column_name) const + { + typedef std::map::const_iterator iter; + iter i = bound_columns_by_name_.find(column_name); + if (i == bound_columns_by_name_.end()) + throw index_range_error(); + return i->second->column_; + } + + string column_name(short column) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + return bound_columns_[column].name_; + } + + long column_size(short column) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + bound_column& col = bound_columns_[column]; + NANODBC_ASSERT(col.sqlsize_ <= static_cast(std::numeric_limits::max())); + return static_cast(col.sqlsize_); + } + + int column_size(const string& column_name) const + { + const short column = this->column(column_name); + return column_size(column); + } + + int column_decimal_digits(short column) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + bound_column& col = bound_columns_[column]; + return col.scale_; + } + + int column_decimal_digits(const string& column_name) const + { + const short column = this->column(column_name); + bound_column& col = bound_columns_[column]; + return col.scale_; + } + + int column_datatype(short column) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + bound_column& col = bound_columns_[column]; + return col.sqltype_; + } + + int column_datatype(const string& column_name) const + { + const short column = this->column(column_name); + bound_column& col = bound_columns_[column]; + return col.sqltype_; + } + + string column_datatype_name(short column) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + + NANODBC_SQLCHAR type_name[256] = {0}; + SQLSMALLINT len = 0; // total number of bytes + RETCODE rc; + NANODBC_CALL_RC( + SQLColAttribute, + rc, + stmt_.native_statement_handle(), + column + 1, + SQL_DESC_TYPE_NAME, + type_name, + sizeof(type_name) / sizeof(NANODBC_SQLCHAR), + &len, + nullptr); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + + NANODBC_ASSERT(len % sizeof(NANODBC_SQLCHAR) == 0); + len = len / sizeof(NANODBC_SQLCHAR); + return string(type_name, type_name + len); + } + + string column_datatype_name(const string& column_name) const + { + return column_datatype_name(this->column(column_name)); + } + + int column_c_datatype(short column) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + bound_column& col = bound_columns_[column]; + return col.ctype_; + } + + int column_c_datatype(const string& column_name) const + { + const short column = this->column(column_name); + bound_column& col = bound_columns_[column]; + return col.ctype_; + } + + bool next_result() + { + RETCODE rc; + +#if defined(NANODBC_DO_ASYNC_IMPL) + stmt_.disable_async(); +#endif + + NANODBC_CALL_RC(SQLMoreResults, rc, stmt_.native_statement_handle()); + if (rc == SQL_NO_DATA) + return false; + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + auto_bind(); + return true; + } + + template + void get_ref(short column, T& result) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + if (is_null(column)) + throw null_access_error(); + get_ref_impl(column, result); + } + + template + void get_ref(short column, const T& fallback, T& result) const + { + if (column >= bound_columns_size_) + throw index_range_error(); + if (is_null(column)) + { + result = fallback; + return; + } + get_ref_impl(column, result); + } + + template + void get_ref(const string& column_name, T& result) const + { + const short column = this->column(column_name); + if (is_null(column)) + throw null_access_error(); + get_ref_impl(column, result); + } + + template + void get_ref(const string& column_name, const T& fallback, T& result) const + { + const short column = this->column(column_name); + if (is_null(column)) + { + result = fallback; + return; + } + get_ref_impl(column, result); + } + + template + T get(short column) const + { + T result; + get_ref(column, result); + return result; + } + + template + T get(short column, const T& fallback) const + { + T result; + get_ref(column, fallback, result); + return result; + } + + template + T get(const string& column_name) const + { + T result; + get_ref(column_name, result); + return result; + } + + template + T get(const string& column_name, const T& fallback) const + { + T result; + get_ref(column_name, fallback, result); + return result; + } + +private: + template ::value, int>::type = 0> + void get_ref_impl(short column, T& result) const; + + template ::value, int>::type = 0> + void get_ref_impl(short column, T& result) const; + + template ::value, int>::type = 0> + void get_ref_from_string_column(short column, T& result) const; + + template ::value, int>::type = 0> + void get_ref_from_string_column(short column, T& result) const; + + void before_move() NANODBC_NOEXCEPT + { + for (short i = 0; i < bound_columns_size_; ++i) + { + bound_column& col = bound_columns_[i]; + for (std::size_t j = 0; j < static_cast(rowset_size_); ++j) + col.cbdata_[j] = 0; + if (col.blob_ && col.pdata_) + release_bound_resources(i); + } + } + + void release_bound_resources(short column) NANODBC_NOEXCEPT + { + NANODBC_ASSERT(column < bound_columns_size_); + bound_column& col = bound_columns_[column]; + delete[] col.pdata_; + col.pdata_ = 0; + col.clen_ = 0; + } + + void cleanup_bound_columns() NANODBC_NOEXCEPT + { + before_move(); + delete[] bound_columns_; + bound_columns_ = nullptr; + bound_columns_size_ = 0; + bound_columns_by_name_.clear(); + } + + // If event_handle is specified, fetch returns true iff the statement is still executing + bool fetch(long rows, SQLUSMALLINT orientation, void* event_handle = nullptr) + { + before_move(); + +#if defined(NANODBC_DO_ASYNC_IMPL) + if (event_handle == nullptr) + stmt_.disable_async(); + else + stmt_.enable_async(event_handle); +#endif // !NANODBC_DISABLE_ASYNC && SQL_ATTR_ASYNC_STMT_EVENT && SQL_API_SQLCOMPLETEASYNC + + RETCODE rc; + NANODBC_CALL_RC(SQLFetchScroll, rc, stmt_.native_statement_handle(), orientation, rows); + if (rc == SQL_NO_DATA) + { + at_end_ = true; + return false; + } +#if defined(NANODBC_DO_ASYNC_IMPL) + if (event_handle != nullptr) + return rc == SQL_STILL_EXECUTING; +#endif + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + return true; + } + + void auto_bind() + { + cleanup_bound_columns(); + + const short n_columns = columns(); + if (n_columns < 1) + return; + + NANODBC_ASSERT(!bound_columns_); + NANODBC_ASSERT(!bound_columns_size_); + bound_columns_ = new bound_column[n_columns]; + bound_columns_size_ = n_columns; + + RETCODE rc; + NANODBC_SQLCHAR column_name[1024]; + SQLSMALLINT sqltype = 0, scale = 0, nullable = 0, len = 0; + SQLULEN sqlsize = 0; + +#if defined(NANODBC_DO_ASYNC_IMPL) + stmt_.disable_async(); +#endif + + for (SQLSMALLINT i = 0; i < n_columns; ++i) + { + NANODBC_CALL_RC( + NANODBC_FUNC(SQLDescribeCol), + rc, + stmt_.native_statement_handle(), + i + 1, + (NANODBC_SQLCHAR*)column_name, + sizeof(column_name) / sizeof(NANODBC_SQLCHAR), + &len, + &sqltype, + &sqlsize, + &scale, + &nullable); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + + // Adjust the sqlsize parameter in case of "unlimited" data (varchar(max), + // nvarchar(max)). + bool is_blob = false; + + if (sqlsize == 0) + { + switch (sqltype) + { + case SQL_VARCHAR: + case SQL_WVARCHAR: + { + // Divide in half, due to sqlsize being 32-bit in Win32 (and 64-bit in x64) + // sqlsize = std::numeric_limits::max() / 2 - 1; + is_blob = true; + } + } + } + + bound_column& col = bound_columns_[i]; + col.name_ = reinterpret_cast(column_name); + col.column_ = i; + col.sqltype_ = sqltype; + col.sqlsize_ = sqlsize; + col.scale_ = scale; + bound_columns_by_name_[col.name_] = &col; + + using namespace std; // if int64_t is in std namespace (in c++11) + switch (col.sqltype_) + { + case SQL_BIT: + case SQL_TINYINT: + case SQL_SMALLINT: + case SQL_INTEGER: + case SQL_BIGINT: + col.ctype_ = SQL_C_SBIGINT; + col.clen_ = sizeof(int64_t); + break; + case SQL_DOUBLE: + case SQL_FLOAT: + case SQL_REAL: + col.ctype_ = SQL_C_DOUBLE; + col.clen_ = sizeof(double); + break; + case SQL_DECIMAL: + case SQL_NUMERIC: + col.ctype_ = SQL_C_CHAR; + // SQL column size defines number of digits without the decimal mark + // and without minus sign which may also occur. + // We need to adjust buffer length allow space for null-termination character + // as well as the fractional part separator and the minus sign. + col.clen_ = (col.sqlsize_ + 1 + 1 + 1) * sizeof(SQLCHAR); + break; + case SQL_DATE: + case SQL_TYPE_DATE: + col.ctype_ = SQL_C_DATE; + col.clen_ = sizeof(date); + break; + case SQL_TIME: + case SQL_TYPE_TIME: + case SQL_SS_TIME2: + col.ctype_ = SQL_C_TIME; + col.clen_ = sizeof(time); + break; + case SQL_TIMESTAMP: + case SQL_TYPE_TIMESTAMP: + case SQL_SS_TIMESTAMPOFFSET: + col.ctype_ = SQL_C_TIMESTAMP; + col.clen_ = sizeof(timestamp); + break; + case SQL_CHAR: + case SQL_VARCHAR: + case SQL_NVARCHAR: + col.ctype_ = sql_ctype::value; + col.clen_ = (col.sqlsize_ + 1) * sizeof(SQLCHAR); + if (is_blob) + { + col.clen_ = 0; + col.blob_ = true; + } + break; + case SQL_WCHAR: + case SQL_WVARCHAR: + col.ctype_ = sql_ctype::value; + col.clen_ = (col.sqlsize_ + 1) * sizeof(SQLWCHAR); + if (is_blob) + { + col.clen_ = 0; + col.blob_ = true; + } + break; + case SQL_LONGVARCHAR: + col.ctype_ = sql_ctype::value; + col.blob_ = true; + col.clen_ = 0; + break; + case SQL_BINARY: + case SQL_VARBINARY: + case SQL_LONGVARBINARY: + case SQL_SS_UDT: // MSDN: Essentially, UDT is a varbinary type with additional metadata. + col.ctype_ = SQL_C_BINARY; + col.blob_ = true; + col.clen_ = 0; + break; + default: + col.ctype_ = sql_ctype::value; + col.clen_ = 128; + break; + } + } + + for (SQLSMALLINT i = 0; i < n_columns; ++i) + { + bound_column& col = bound_columns_[i]; + col.cbdata_ = new null_type[static_cast(rowset_size_)]; + if (col.blob_) + { + NANODBC_CALL_RC( + SQLBindCol, + rc, + stmt_.native_statement_handle(), + i + 1, + col.ctype_, + 0, + 0, + col.cbdata_); + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + } + else + { + col.pdata_ = new char[rowset_size_ * col.clen_]; + NANODBC_CALL_RC( + SQLBindCol, + rc, + stmt_.native_statement_handle(), + i + 1, // ColumnNumber + col.ctype_, // TargetType + col.pdata_, // TargetValuePtr + col.clen_, // BufferLength + col.cbdata_); // StrLen_or_Ind + if (!success(rc)) + NANODBC_THROW_DATABASE_ERROR(stmt_.native_statement_handle(), SQL_HANDLE_STMT); + } + } + } + +private: + statement stmt_; + const long rowset_size_; + SQLULEN row_count_; + bound_column* bound_columns_; + short bound_columns_size_; + long rowset_position_; + std::map bound_columns_by_name_; + bool at_end_; +#if defined(NANODBC_DO_ASYNC_IMPL) + bool async_; // true if statement is currently in SQL_STILL_EXECUTING mode +#endif +}; + +template <> +inline void result::result_impl::get_ref_impl(short column, date& result) const +{ + bound_column& col = bound_columns_[column]; + switch (col.ctype_) + { + case SQL_C_DATE: + result = *reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); + return; + case SQL_C_TIMESTAMP: + { + timestamp stamp = *reinterpret_cast(col.pdata_ + rowset_position_ * col.clen_); + date d = {stamp.year, stamp.month, stamp.day}; + result = d; + return; + } + } + throw type_incompatible_error(); +} + +template <> +inline void result::result_impl::get_ref_impl