diff --git a/README.md b/README.md
index c7fd6873..123aeca1 100644
--- a/README.md
+++ b/README.md
@@ -27,8 +27,8 @@ See [embulk-output-sqlserver](embulk-output-sqlserver/).
### Overview
* **Plugin type**: output
-* **Load all or nothing**: depnds on the mode. see below.
-* **Resume supported**: depnds on the mode. see below.
+* **Load all or nothing**: depends on the mode. see below.
+* **Resume supported**: depends on the mode. see below.
### Configuration
diff --git a/embulk-output-oracle/README.md b/embulk-output-oracle/README.md
index 9a49543e..45b9399b 100644
--- a/embulk-output-oracle/README.md
+++ b/embulk-output-oracle/README.md
@@ -63,6 +63,9 @@ It requires Oracle JDBC driver too, but the version 12 driver doesn't work (the
"oci" means direct path insert using OCI(Oracle Call Interface). It is fastest.
It requires both Oracle JDBC driver and Oracle Instant Client (version 12.1.0.2.0).
You must set the library loading path to the OCI library.
+And it uses an optional native library (embulk-output-oracle-oci) written in cpp to improve performance furthermore.
+Not only the source codes of the library, but also the built libraries for Windows(x64) and Linux(x64) have bean committed.
+
### Supported types
@@ -123,6 +126,46 @@ out:
$ ./gradlew gem
```
+#### Build environment for native library
+
+For Windows (x64)
+
+(1) Install Microsoft Visual Studio (only 2010 is tested).
+
+(2) Install Oracle Instant Client SDK 11.1.0.6.0 for Microsoft Windows (x64).
+
+(3) Set environment variables.
+
+* OCI\_SDK_PATH ("sdk" directory of Oracle Instant Client)
+
+(4) Open src/main/cpp/win/embulk-output-oracle-oci.sln by Visual Studio and build.
+
+For Windows command line, the following are needed in addition to (1) - (4).
+
+(5) Set environment variables.
+
+* MSVC_PATH (ex. C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC)
+* MSSDK_PATH (ex. C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A)
+
+(6) Execute src/main/cpp/win/build.bat .
+
+
+For Linux (x64) (only Ubuntu Server 14.04 is tested)
+
+(1) Install gcc and g++ .
+
+(2) Install Oracle Instant Client Basic and SDK 11.1.0.6.0 for Linux (x64).
+
+(3) Create symbolic links of OCI libraries.
+
+ ln -s libocci.so.11.1 libocci.so
+ ln -s libclntsh.so.11.1 libclntsh.so
+
+(4) Set environment variables.
+
+* OCI_PATH (the directory of Oracle Instant Client Basic and the parent of the "sdk" directory)
+
+(5) Execute src/main/cpp/linux/build.sh .
***
is used to improve performance of embulk-output-oracle.
diff --git a/embulk-output-oracle/lib/embulk/native/x86_64-linux/libembulk-output-oracle-oci.so b/embulk-output-oracle/lib/embulk/native/x86_64-linux/libembulk-output-oracle-oci.so
new file mode 100644
index 00000000..2500279e
Binary files /dev/null and b/embulk-output-oracle/lib/embulk/native/x86_64-linux/libembulk-output-oracle-oci.so differ
diff --git a/embulk-output-oracle/lib/embulk/native/x86_64-windows/embulk-output-oracle-oci.dll b/embulk-output-oracle/lib/embulk/native/x86_64-windows/embulk-output-oracle-oci.dll
new file mode 100644
index 00000000..90f7c5e5
Binary files /dev/null and b/embulk-output-oracle/lib/embulk/native/x86_64-windows/embulk-output-oracle-oci.dll differ
diff --git a/embulk-output-oracle/src/main/cpp/common/embulk-output-oracle-oci.cpp b/embulk-output-oracle/src/main/cpp/common/embulk-output-oracle-oci.cpp
new file mode 100644
index 00000000..ff57a1dc
--- /dev/null
+++ b/embulk-output-oracle/src/main/cpp/common/embulk-output-oracle-oci.cpp
@@ -0,0 +1,27 @@
+#include
+
+
+extern "C"
+#ifdef WIN32
+ __declspec(dllexport)
+#endif
+sword embulk_output_oracle_OCIDirPathColArrayEntriesSet(
+ OCIDirPathColArray *dpca,
+ OCIError *errhp,
+ ub2 columnCount,
+ ub4 rowCount,
+ ub1* data,
+ ub2* sizes)
+{
+ for (ub4 row = 0; row < rowCount; row++) {
+ for (ub2 column = 0; column < columnCount; column++) {
+ ub2 size = *sizes++;
+ sword result = OCIDirPathColArrayEntrySet(dpca, errhp, row, column, data, size, OCI_DIRPATH_COL_COMPLETE);
+ if (result != OCI_SUCCESS) {
+ return result;
+ }
+ data += size;
+ }
+ }
+ return OCI_SUCCESS;
+}
diff --git a/embulk-output-oracle/src/main/cpp/linux/build.sh b/embulk-output-oracle/src/main/cpp/linux/build.sh
new file mode 100644
index 00000000..3fd5f114
--- /dev/null
+++ b/embulk-output-oracle/src/main/cpp/linux/build.sh
@@ -0,0 +1,15 @@
+# gcc, g++ and Oracle Instant Client Basic and SDK are requred.
+#
+# ln libocci.so.x.x libocci.so
+# ln libclntsh.so.x.x libclntsh.so
+#
+
+if [ "$OCI_PATH" = "" ]
+then
+ echo "You should set the environment variable 'OCI_PATH'."
+ exit 1
+fi
+
+mkdir -p ../../../../lib/embulk/native/x86_64-linux
+
+gcc -fPIC -I. -I"$OCI_PATH/sdk/include" -I../../../main/cpp/common -L"$OCI_PATH" -shared ../../../main/cpp/common/embulk-output-oracle-oci.cpp -locci -lclntsh -lstdc++ -o ../../../../lib/embulk/native/x86_64-linux/libembulk-output-oracle-oci.so
diff --git a/embulk-output-oracle/src/main/cpp/windows/build.bat b/embulk-output-oracle/src/main/cpp/windows/build.bat
new file mode 100644
index 00000000..54b4fb20
--- /dev/null
+++ b/embulk-output-oracle/src/main/cpp/windows/build.bat
@@ -0,0 +1,26 @@
+@ECHO OFF
+
+REM Visual Studio and Oracle Instant Client SDK are requred.
+REM You should set the environment variable 'PATH' to CL.exe(x86_amd64) of Visual Studio.
+
+IF "%OCI_SDK_PATH%" == "" (
+ ECHO "You should set the environment variable 'OCI_SDK_PATH'."
+ EXIT /B 1
+)
+
+IF "%MSVC_PATH%" == "" (
+ ECHO "You should set the environment variable 'MSVC_PATH'."
+ ECHO "For example : SET MSVC_PATH=C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC"
+ EXIT /B 1
+)
+
+IF "%MSSDK_PATH%" == "" (
+ ECHO "You should set the environment variable 'MSSDK_PATH'."
+ ECHO "For example : SET MSSDK_PATH=C:\Program Files (x86)\Microsoft SDKs\Windows\v7.0A"
+ EXIT /B 1
+)
+
+
+MKDIR ..\..\..\..\lib\embulk\native\x86_64-windows
+
+CL /I"%MSSDK_PATH%\Include" /I"%MSVC_PATH%\include" /I"%OCI_SDK_PATH%\include" /Zi /nologo /W3 /WX- /O2 /Oi /GL /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_USRDLL" /D "EMBULKOUTPUTORACLE_EXPORTS" /D "_WINDLL" /D "_UNICODE" /D "UNICODE" /Gm- /EHsc /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:queue ..\common\embulk-output-oracle-oci.cpp dllmain.cpp /link /LIBPATH:"%MSVC_PATH%\lib\amd64" /LIBPATH:"%MSSDK_PATH%\Lib\x64" /LIBPATH:"%OCI_SDK_PATH%\lib\msvc" /INCREMENTAL:NO /NOLOGO /LIBPATH:"%OCI_SDK_PATH%\lib\msvc" /OUT:"..\..\..\..\lib\embulk\native\x86_64-windows\embulk-output-oracle-oci.dll" /DLL "oci.lib" "kernel32.lib" "user32.lib" "gdi32.lib" "winspool.lib" "comdlg32.lib" "advapi32.lib" "shell32.lib" "ole32.lib" "oleaut32.lib" "uuid.lib" "odbc32.lib" "odbccp32.lib" /SUBSYSTEM:WINDOWS /OPT:REF /OPT:ICF /LTCG /TLBID:1 /DYNAMICBASE /NXCOMPAT /MACHINE:X64 /ERRORREPORT:QUEUE
\ No newline at end of file
diff --git a/embulk-output-oracle/src/main/cpp/windows/dllmain.cpp b/embulk-output-oracle/src/main/cpp/windows/dllmain.cpp
new file mode 100644
index 00000000..dcfcb4e9
--- /dev/null
+++ b/embulk-output-oracle/src/main/cpp/windows/dllmain.cpp
@@ -0,0 +1,25 @@
+#pragma once
+
+#include
+
+#define WIN32_LEAN_AND_MEAN
+
+#include
+
+
+BOOL APIENTRY DllMain( HMODULE hModule,
+ DWORD ul_reason_for_call,
+ LPVOID lpReserved
+ )
+{
+ switch (ul_reason_for_call)
+ {
+ case DLL_PROCESS_ATTACH:
+ case DLL_THREAD_ATTACH:
+ case DLL_THREAD_DETACH:
+ case DLL_PROCESS_DETACH:
+ break;
+ }
+ return TRUE;
+}
+
diff --git a/embulk-output-oracle/src/main/cpp/windows/embulk-output-oracle-oci.sln b/embulk-output-oracle/src/main/cpp/windows/embulk-output-oracle-oci.sln
new file mode 100644
index 00000000..5615e033
--- /dev/null
+++ b/embulk-output-oracle/src/main/cpp/windows/embulk-output-oracle-oci.sln
@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "embulk-output-oracle-oci", "embulk-output-oracle-oci.vcxproj", "{2284D821-588E-419F-8790-F5BCC12F93BD}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|x64 = Debug|x64
+ Release|x64 = Release|x64
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {2284D821-588E-419F-8790-F5BCC12F93BD}.Debug|x64.ActiveCfg = Debug|x64
+ {2284D821-588E-419F-8790-F5BCC12F93BD}.Debug|x64.Build.0 = Debug|x64
+ {2284D821-588E-419F-8790-F5BCC12F93BD}.Release|x64.ActiveCfg = Release|x64
+ {2284D821-588E-419F-8790-F5BCC12F93BD}.Release|x64.Build.0 = Release|x64
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/embulk-output-oracle/src/main/cpp/windows/embulk-output-oracle-oci.vcxproj b/embulk-output-oracle/src/main/cpp/windows/embulk-output-oracle-oci.vcxproj
new file mode 100644
index 00000000..1ea31ce1
--- /dev/null
+++ b/embulk-output-oracle/src/main/cpp/windows/embulk-output-oracle-oci.vcxproj
@@ -0,0 +1,171 @@
+
+
+
+
+ Debug
+ Win32
+
+
+ Debug
+ x64
+
+
+ Release
+ Win32
+
+
+ Release
+ x64
+
+
+
+ {2284D821-588E-419F-8790-F5BCC12F93BD}
+ Win32Proj
+ embulkoutputoracle
+ embulk-output-oracle-oci
+
+
+
+ DynamicLibrary
+ true
+ Unicode
+
+
+ DynamicLibrary
+ true
+ Unicode
+
+
+ DynamicLibrary
+ false
+ true
+ Unicode
+
+
+ DynamicLibrary
+ false
+ true
+ Unicode
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ $(SolutionDir)$(Configuration)\
+
+
+ true
+ $(SolutionDir)$(Configuration)\
+
+
+ false
+ $(SolutionDir)$(Configuration)\
+
+
+ false
+ $(SolutionDir)$(Configuration)\
+
+
+
+ NotUsing
+ Level3
+ Disabled
+ WIN32;_DEBUG;_WINDOWS;_USRDLL;EMBULKOUTPUTORACLE_EXPORTS;%(PreprocessorDefinitions)
+ $(OCI_SDK_PATH)\include;%(AdditionalIncludeDirectories)
+
+
+ Windows
+ true
+ $(OCI_SDK_PATH)\lib\msvc;%(AdditionalLibraryDirectories)
+ ..\..\..\..\lib\$(TargetName)$(TargetExt)
+
+
+
+
+ NotUsing
+ Level3
+ Disabled
+ WIN32;_DEBUG;_WINDOWS;_USRDLL;EMBULKOUTPUTORACLE_EXPORTS;%(PreprocessorDefinitions)
+ $(OCI_SDK_PATH)\include;%(AdditionalIncludeDirectories)
+
+
+ Windows
+ true
+ $(OCI_SDK_PATH)\lib\msvc;%(AdditionalLibraryDirectories)
+ oci.lib;%(AdditionalDependencies)
+ ..\..\..\..\lib\embulk\native\x86_64-windows\$(TargetName)$(TargetExt)
+
+
+
+
+ Level3
+ NotUsing
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_WINDOWS;_USRDLL;EMBULKOUTPUTORACLE_EXPORTS;%(PreprocessorDefinitions)
+ $(OCI_SDK_PATH)\include;%(AdditionalIncludeDirectories)
+
+
+ Windows
+ true
+ true
+ true
+ ..\..\..\..\lib\$(TargetName)$(TargetExt)
+
+
+
+
+ Level3
+ NotUsing
+ MaxSpeed
+ true
+ true
+ WIN32;NDEBUG;_WINDOWS;_USRDLL;EMBULKOUTPUTORACLE_EXPORTS;%(PreprocessorDefinitions)
+ $(OCI_SDK_PATH)\include;%(AdditionalIncludeDirectories)
+
+
+ Windows
+ true
+ true
+ true
+ $(OCI_SDK_PATH)\lib\msvc;%(AdditionalLibraryDirectories)
+ oci.lib;%(AdditionalDependencies)
+ ..\..\..\..\lib\embulk\native\x86_64-windows\$(TargetName)$(TargetExt)
+
+
+
+
+
+ false
+ false
+
+
+
+
+ false
+ false
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/embulk-output-oracle/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java
index b32bea3e..3c2dbb01 100644
--- a/embulk-output-oracle/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java
+++ b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java
@@ -21,16 +21,12 @@
import org.embulk.output.oracle.oci.OCIWrapper;
import org.embulk.output.oracle.oci.RowBuffer;
import org.embulk.output.oracle.oci.TableDefinition;
-import org.embulk.spi.Exec;
import org.embulk.spi.time.Timestamp;
-import org.slf4j.Logger;
public class DirectBatchInsert implements BatchInsert
{
private static OCIManager ociManager = new OCIManager();
- private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
-
private List ociKey;
private final String database;
private final String user;
@@ -40,9 +36,6 @@ public class DirectBatchInsert implements BatchInsert
private final OracleCharset nationalCharset;
private final int batchSize;
private RowBuffer buffer;
- private long totalRows;
- private int rowSize;
- private int batchWeight;
private boolean closed;
private DateFormat[] formats;
@@ -148,31 +141,23 @@ public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLExcepti
}
- rowSize = 0;
- for (ColumnDefinition column : columns) {
- rowSize += column.getDataSize();
- }
-
TableDefinition tableDefinition = new TableDefinition(schema, loadTable, columns);
ociKey = Arrays.asList(database, user, loadTable);
- ociManager.open(ociKey, database, user, password, tableDefinition);
+ OCIWrapper oci = ociManager.open(ociKey, database, user, password, tableDefinition, batchSize);
- buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8));
+ buffer = new RowBuffer(oci, tableDefinition);
}
@Override
public int getBatchWeight()
{
- return batchWeight;
+ // Automatically flushed in RowBuffer
+ return 0;
}
@Override
public void add() throws IOException, SQLException
{
- batchWeight += rowSize;
- if (buffer.isFull()) {
- flush();
- }
}
@Override
@@ -187,26 +172,7 @@ public void close() throws IOException, SQLException
@Override
public void flush() throws IOException, SQLException
{
- if (buffer.getRowCount() > 0) {
- try {
- logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
-
- long startTime = System.currentTimeMillis();
-
- OCIWrapper oci = ociManager.get(ociKey);
- synchronized (oci) {
- oci.loadBuffer(buffer);
- }
-
- totalRows += buffer.getRowCount();
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
- logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
-
- } finally {
- buffer.clear();
- batchWeight = 0;
- }
- }
+ buffer.flush();
}
@Override
diff --git a/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/BulkOCI.java b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/BulkOCI.java
new file mode 100644
index 00000000..909c6c2e
--- /dev/null
+++ b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/BulkOCI.java
@@ -0,0 +1,15 @@
+package org.embulk.output.oracle.oci;
+
+import jnr.ffi.Pointer;
+import jnr.ffi.types.u_int16_t;
+import jnr.ffi.types.u_int32_t;
+
+public interface BulkOCI {
+
+ short embulk_output_oracle_OCIDirPathColArrayEntriesSet(Pointer dpca,
+ Pointer errhp,
+ @u_int16_t short columnCount,
+ @u_int32_t int rowCount,
+ Pointer data,
+ Pointer sizes);
+}
diff --git a/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCI.java b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCI.java
index 0ae24079..f01a2bdd 100644
--- a/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCI.java
+++ b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCI.java
@@ -32,6 +32,7 @@ public interface OCI
static int OCI_ATTR_SCHEMA_NAME = 9;
static int OCI_ATTR_CHARSET_ID = 31;
static int OCI_ATTR_DATEFORMAT = 75;
+ static int OCI_ATTR_BUF_SIZE = 77;
static int OCI_ATTR_NUM_ROWS = 81;
static int OCI_ATTR_NUM_COLS = 102;
static int OCI_ATTR_LIST_COLUMNS = 103;
diff --git a/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCIManager.java b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCIManager.java
index 5c15a928..cba468be 100644
--- a/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCIManager.java
+++ b/embulk-output-oracle/src/main/java/org/embulk/output/oracle/oci/OCIManager.java
@@ -20,7 +20,8 @@ private static class OCIWrapperAndCounter {
private Map