From cadb9ec2fa483d2b6db78c715be83d8e473a37db Mon Sep 17 00:00:00 2001 From: Bing Li Date: Wed, 17 Oct 2018 13:52:39 -0700 Subject: [PATCH 1/6] upgrade jackson libs to 2.9.6 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 82ec6edc2..b808152ac 100644 --- a/pom.xml +++ b/pom.xml @@ -234,14 +234,14 @@ com.fasterxml.jackson.core jackson-core - 2.8.4 + 2.9.6 com.fasterxml.jackson.core jackson-databind - 2.8.4 + 2.9.6 From fa941f734ba54a8e74fcb4e664e56923d76490f7 Mon Sep 17 00:00:00 2001 From: Bing Li Date: Thu, 18 Oct 2018 11:35:00 -0700 Subject: [PATCH 2/6] create two new constructors which requested private key string only --- .../snowflake/ingest/SimpleIngestManager.java | 121 ++++++++++++++++-- 1 file changed, 110 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index 735805403..07b3fe01d 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -20,7 +20,15 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.security.KeyFactory; import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPublicKeySpec; import java.util.Collections; import java.util.List; import java.util.Set; @@ -217,10 +225,11 @@ private void init(String account, String user, String pipe, * Snowflake DB * * @param account The account into which we're loading - * @param user the user performing this load - * @param pipe the fully qualified name of the pipe + * @param user the user performing this load + * @param pipe the fully qualified name of the pipe * @param keyPair the KeyPair we'll use to sign JWT tokens */ + @Deprecated public SimpleIngestManager(String account, String user, String pipe, KeyPair keyPair) { @@ -232,30 +241,120 @@ public SimpleIngestManager(String account, String user, String pipe, } + /** + * Constructs a SimpleIngestManager for a given user in a specific account + * In addition, this also takes takes the target table and source stage + * Finally, it also requires a valid private key string registered with + * Snowflake DB + * + * @param account The account into which we're loading + * @param user the user performing this load + * @param pipe the fully qualified name of the pipe + * @param privateKey the private key we'll use to sign JWT tokens + * @throws NoSuchAlgorithmException if can't create + * key factory by using RSA algorithm + * @throws InvalidKeySpecException if private key or public key is + * invalid + */ + public SimpleIngestManager(String account, String user, String pipe, + String privateKey) + throws InvalidKeySpecException, NoSuchAlgorithmException + { + KeyPair keyPair = createKeyPairFromPrivateKey(privateKey); + + //call our initializer method + init(account, user, pipe, keyPair); + + //create the request builder + this.builder = new RequestBuilder(account, user, keyPair); + } + + /** * Constructs a SimpleIngestManager for a given user in a specific account * In addition, this also takes takes the target table and source stage * Finally, it also requires a valid KeyPair object registered with * Snowflake DB * - * @param account the account into which we're loading - * @param user the user performing this load - * @param pipe the fully qualified name of the pipe - * @param keyPair the KeyPair we'll use to sign JWT tokens + * @param account the account into which we're loading + * @param user the user performing this load + * @param pipe the fully qualified name of the pipe + * @param keyPair the KeyPair we'll use to sign JWT tokens * @param schemeName http or https - * @param hostName the hostname - * @param port the port number + * @param hostName the hostname + * @param port the port number */ + @Deprecated public SimpleIngestManager(String account, String user, String pipe, - KeyPair keyPair, String schemeName, - String hostName, int port) + KeyPair keyPair, String schemeName, + String hostName, int port) { //call our initializer method init(account, user, pipe, keyPair); //make the request builder we'll use to build messages to the service builder = new RequestBuilder(account, user, keyPair, - schemeName, hostName, port); + schemeName, hostName, port); + } + + /** + * Constructs a SimpleIngestManager for a given user in a specific account + * In addition, this also takes takes the target table and source stage + * Finally, it also requires a valid private key string registered with + * Snowflake DB + * + * @param account the account into which we're loading + * @param user the user performing this load + * @param pipe the fully qualified name of the pipe + * @param privateKey the private key we'll use to sign JWT tokens + * @param schemeName http or https + * @param hostName the hostname + * @param port the port number + * @throws NoSuchAlgorithmException if can't create key factory by using + * RSA algorithm + * @throws InvalidKeySpecException if private key or public key is invalid + */ + public SimpleIngestManager(String account, String user, String pipe, + String privateKey, String schemeName, + String hostName, int port) + throws NoSuchAlgorithmException, InvalidKeySpecException + { + KeyPair keyPair = createKeyPairFromPrivateKey(privateKey); + //call our initializer method + init(account, user, pipe, keyPair); + + //make the request builder we'll use to build messages to the service + builder = new RequestBuilder(account, user, keyPair, + schemeName, hostName, port); + + } + + /** + * generate key pair object from private key String + * + * @param privateKey private key string + * @return a key pair object + * @throws NoSuchAlgorithmException if can't create key factory by using + * RSA algorithm + * @throws InvalidKeySpecException if private key or public key is invalid + */ + private KeyPair createKeyPairFromPrivateKey(String privateKey) throws + NoSuchAlgorithmException, InvalidKeySpecException + { + //create private key from string + byte[] keyBytes = privateKey.getBytes(); + PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory kf = KeyFactory.getInstance("RSA"); + PrivateKey privateK = kf.generatePrivate(privateKeySpec); + + //generate public key from private key + RSAPrivateCrtKey privk = (RSAPrivateCrtKey) privateK; + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec(privk.getModulus(), + privk.getPublicExponent()); + PublicKey publicK = kf.generatePublic(publicKeySpec); + + //create key pairs + return new KeyPair(publicK, privateK); } From d0a49a0fb74bf35f27c77b3adaf2dc37d488f1df Mon Sep 17 00:00:00 2001 From: Bing Li Date: Thu, 18 Oct 2018 15:50:04 -0700 Subject: [PATCH 3/6] use PrivateKey object instead of string value --- .../snowflake/ingest/SimpleIngestManager.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index 07b3fe01d..64b96edae 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -27,7 +27,6 @@ import java.security.PublicKey; import java.security.interfaces.RSAPrivateCrtKey; import java.security.spec.InvalidKeySpecException; -import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.RSAPublicKeySpec; import java.util.Collections; import java.util.List; @@ -244,7 +243,7 @@ public SimpleIngestManager(String account, String user, String pipe, /** * Constructs a SimpleIngestManager for a given user in a specific account * In addition, this also takes takes the target table and source stage - * Finally, it also requires a valid private key string registered with + * Finally, it also requires a valid private key registered with * Snowflake DB * * @param account The account into which we're loading @@ -257,7 +256,7 @@ public SimpleIngestManager(String account, String user, String pipe, * invalid */ public SimpleIngestManager(String account, String user, String pipe, - String privateKey) + PrivateKey privateKey) throws InvalidKeySpecException, NoSuchAlgorithmException { KeyPair keyPair = createKeyPairFromPrivateKey(privateKey); @@ -300,7 +299,7 @@ public SimpleIngestManager(String account, String user, String pipe, /** * Constructs a SimpleIngestManager for a given user in a specific account * In addition, this also takes takes the target table and source stage - * Finally, it also requires a valid private key string registered with + * Finally, it also requires a valid private key registered with * Snowflake DB * * @param account the account into which we're loading @@ -315,7 +314,7 @@ public SimpleIngestManager(String account, String user, String pipe, * @throws InvalidKeySpecException if private key or public key is invalid */ public SimpleIngestManager(String account, String user, String pipe, - String privateKey, String schemeName, + PrivateKey privateKey, String schemeName, String hostName, int port) throws NoSuchAlgorithmException, InvalidKeySpecException { @@ -330,31 +329,27 @@ public SimpleIngestManager(String account, String user, String pipe, } /** - * generate key pair object from private key String + * generate key pair object from private key * - * @param privateKey private key string + * @param privateKey private key * @return a key pair object * @throws NoSuchAlgorithmException if can't create key factory by using * RSA algorithm * @throws InvalidKeySpecException if private key or public key is invalid */ - private KeyPair createKeyPairFromPrivateKey(String privateKey) throws + private KeyPair createKeyPairFromPrivateKey(PrivateKey privateKey) throws NoSuchAlgorithmException, InvalidKeySpecException { - //create private key from string - byte[] keyBytes = privateKey.getBytes(); - PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(keyBytes); KeyFactory kf = KeyFactory.getInstance("RSA"); - PrivateKey privateK = kf.generatePrivate(privateKeySpec); //generate public key from private key - RSAPrivateCrtKey privk = (RSAPrivateCrtKey) privateK; + RSAPrivateCrtKey privk = (RSAPrivateCrtKey) privateKey; RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec(privk.getModulus(), privk.getPublicExponent()); PublicKey publicK = kf.generatePublic(publicKeySpec); //create key pairs - return new KeyPair(publicK, privateK); + return new KeyPair(publicK, privateKey); } From 7a686974484486f35f1860a13bfffb9181171010 Mon Sep 17 00:00:00 2001 From: Bing Li Date: Fri, 19 Oct 2018 10:50:14 -0700 Subject: [PATCH 4/6] setup integration test and create simple test exmple --- .gitignore | 1 + pom.xml | 8 + profile.json.example | 13 + .../snowflake/ingest/SimpleIngestSuite.java | 167 ++++++++ .../ingest/TestSimpleIngestLocal.java | 390 ------------------ .../java/net/snowflake/ingest/TestUtils.java | 151 +++++++ .../connection/SecurityManagerTest.java | 3 +- .../ingest/connection/TestKeyRenewal.java | 3 +- test_files/test1.csv | 3 + 9 files changed, 347 insertions(+), 392 deletions(-) create mode 100644 profile.json.example create mode 100644 src/test/java/net/snowflake/ingest/SimpleIngestSuite.java delete mode 100644 src/test/java/net/snowflake/ingest/TestSimpleIngestLocal.java create mode 100644 src/test/java/net/snowflake/ingest/TestUtils.java create mode 100644 test_files/test1.csv diff --git a/.gitignore b/.gitignore index a9725bdad..0b35238f3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.iml .idea target/ +profile.json diff --git a/pom.xml b/pom.xml index b808152ac..a21cadd9d 100644 --- a/pom.xml +++ b/pom.xml @@ -221,6 +221,14 @@ test + + + org.bouncycastle + bcprov-ext-jdk15on + 1.60 + test + + diff --git a/profile.json.example b/profile.json.example new file mode 100644 index 000000000..e7bf3b064 --- /dev/null +++ b/profile.json.example @@ -0,0 +1,13 @@ +{ + "user": "user name", + "account": "account name", + "private_key": "PEM Private Key", + "port": 443, + "host": "account_name.snowflakecomputing.com", + "schema": "schema", + "scheme": "https", + "database": "database name", + "connect_string": "jdbc:snowflake://account_name.snowflakecomputing.com:443", + "ssl": "on", + "warehouse": "warehouse name" +} \ No newline at end of file diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java b/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java new file mode 100644 index 000000000..54489aab3 --- /dev/null +++ b/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java @@ -0,0 +1,167 @@ +package net.snowflake.ingest; + +import net.snowflake.ingest.connection.HistoryResponse; +import net.snowflake.ingest.connection.IngestResponse; +import net.snowflake.ingest.utils.StagedFileWrapper; +import org.junit.Before; +import org.junit.Test; +import org.junit.After; + +import java.sql.Connection; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertTrue; + +/** + * Example ingest sdk integration test + */ +public class SimpleIngestSuite +{ + private Connection conn = null; + private SimpleIngestManager manager = null; + + private final String TEST_FILE_PATH = "file://test_files/test1.csv"; + private final String TEST_FILE_NAME = "test1.csv"; + + private String tableName = ""; + private String pipeName = ""; + private String stageName = ""; + + /** + * Create test table and pipe + */ + @Before + public void beforeAll() throws Exception + { + Random rand = new Random(); + + Long num = Math.abs(rand.nextLong()); + + tableName = "ingest_sdk_test_table_" + num; + + pipeName = "ingest_sdk_test_pipe_" + num; + + stageName = "ingest_sdk_test_stage_" + num; + + + TestUtils.executeQuery( + "create or replace table " + tableName + " (str string, num int)" + ); + + TestUtils.executeQuery( + "create or replace stage " + stageName + ); + + TestUtils.executeQuery( + "create or replace pipe " + pipeName + " as copy into " + tableName + + " from @" + stageName + ); + + } + + /** + * Remove test table and pipe + */ + @After + public void afterAll() + { + //System.out.println(111); + TestUtils.executeQuery( + "drop pipe if exists " + pipeName + ); + + TestUtils.executeQuery( + "drop stage if exists " + stageName + ); + + TestUtils.executeQuery( + "drop table if exists " + tableName + ); + } + + /** + * ingest test + */ + @Test + public void test1() throws Exception + { + //put + TestUtils.executeQuery( + "put " + TEST_FILE_PATH + " @" + stageName + ); + + //keeps track of whether we've loaded the file + boolean loaded = false; + + //create ingest manager + SimpleIngestManager manager = TestUtils.getManager(pipeName); + + //create a file wrapper + StagedFileWrapper myFile = new StagedFileWrapper(TEST_FILE_NAME, null); + + //get an insert response after we submit + IngestResponse insertResponse = manager.ingestFile(myFile, null); + + //create a new thread + ExecutorService service = Executors.newSingleThreadExecutor(); + + //fork off waiting for a load to the service + Future result = service.submit(() -> + { + + String beginMark = null; + + while (true) + { + + try + { + Thread.sleep(5000); + HistoryResponse response = manager.getHistory(null, null, + beginMark); + + if (response != null && response.getNextBeginMark() != null) + { + beginMark = response.getNextBeginMark(); + } + if (response != null && response.files != null) + { + for (HistoryResponse.FileEntry entry : response.files) + { + //if we have a complete file that we've + // loaded with the same name.. + String filename = entry.getPath(); + if (entry.getPath() != null && entry.isComplete() && + filename.equals(TEST_FILE_NAME)) + { + return; + } + } + } + } catch (Exception e) + { + e.printStackTrace(); + } + } + } + ); + + //try to wait until the future is done + try + { + //wait up to 1 minutes to load + result.get(2, TimeUnit.MINUTES); + loaded = true; + } finally + { + assertTrue(loaded); + + + } + } + +} diff --git a/src/test/java/net/snowflake/ingest/TestSimpleIngestLocal.java b/src/test/java/net/snowflake/ingest/TestSimpleIngestLocal.java deleted file mode 100644 index 790f25470..000000000 --- a/src/test/java/net/snowflake/ingest/TestSimpleIngestLocal.java +++ /dev/null @@ -1,390 +0,0 @@ -/* - * Copyright (c) 2012-2017 Snowflake Computing Inc. All rights reserved. - */ - -package net.snowflake.ingest; - -import net.snowflake.ingest.connection.HistoryResponse; -import net.snowflake.ingest.connection.IngestResponse; -import net.snowflake.ingest.utils.StagedFileWrapper; -import org.apache.commons.codec.binary.Base64; -import org.junit.Before; -import org.junit.Test; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.KeyFactory; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.PublicKey; -import java.security.SecureRandom; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.X509EncodedKeySpec; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertTrue; - - -/** - * TestSimpleIngestLocal - this class tests whether or not we are - * successfully able to create a local file, push it to snowflake, - */ -public class TestSimpleIngestLocal -{ - //The encryption algorithm we will use to generate keys - private final static String ALGORITHM = "RSA"; - - //the name of the file we want to push - private static final String FILENAME = "/tmp/data/letters.csv"; - - //base file name we want to load - private static final String BASE_FILENAME = "letters.csv"; - - //the account for which we are pushing - private static final String ACCOUNT = "testaccount"; - - //the user who is going to be ingesting these files - private static final String USER = "snowman"; - - //the password of this test user - private static final String PASSWORD = "test"; - - //the connecting port - private static final int PORT = 8082; - - //the host name - private static final String HOST = "localhost"; - - //the scheme name - private static final String SCHEME = "http"; - - //the actual connection string - private static final String CONNECT_STRING = - "jdbc:snowflake://" + HOST + ":" + PORT; - - //Do we actually want to use SSL - private static final String SSL = "off"; - - //the connection we will use for queries - private final Connection conn; - - //the Administrative connection - - //the name of our target DB - private static final String DATABASE = "testdb"; - - //the name of our target schema - private static final String SCHEMA = "public"; - - //the name of our stage - private static final String STAGE = "ingest_stage"; - - //the name of our target table - private static final String TABLE = "ingest_table"; - - //the name of our pipe - private static final String PIPE = "ingest_pipe"; - - //the fully qualified table name - private final String FQ_TABLE = - DATABASE + "." + SCHEMA + "." + quote(TABLE); - - //the fully qualified stage name - private final String FQ_STAGE = - DATABASE + "." + SCHEMA + "." + quote(STAGE); - - //the fully qualified pipe name - private final String FQ_PIPE = - DATABASE + "." + SCHEMA + "." + quote(PIPE); - - //the actual ingest manager - private final SimpleIngestManager manager; - - //our keypair - private final KeyPair keypair; - - /** - * TestSimpleIngestLocal - makes a new instance of - * this test class by creating a sql connection to the database - */ - public TestSimpleIngestLocal() - throws ClassNotFoundException, SQLException, - NoSuchAlgorithmException, NoSuchProviderException - { - //create a connection - conn = getConnection(USER); - - - //generate a keypair - keypair = generateKeyPair(); - //make an ingest manager - manager = new SimpleIngestManager(ACCOUNT, USER, - FQ_PIPE, keypair, SCHEME, HOST, PORT); - } - - /** - * Generates an RSA keypair for use in this test - * - * @return a valid RSA keypair - * @throws NoSuchAlgorithmException if we don't have an RSA algo - * @throws NoSuchProviderException if we can't use SHA1PRNG for randomization - */ - private KeyPair generateKeyPair() - throws NoSuchProviderException, NoSuchAlgorithmException - { - KeyPairGenerator keyGen = KeyPairGenerator.getInstance(ALGORITHM); - SecureRandom random = SecureRandom.getInstance("SHA1PRNG", "SUN"); - keyGen.initialize(2048, random); - return keyGen.generateKeyPair(); - } - - /** - * Gets a JDBC connection to the service - * - * @param user user name - * @return a valid JDBC connection - */ - private Connection getConnection(String user) - throws ClassNotFoundException, SQLException - { - //check first to see if we have the Snowflake JDBC - Class.forName("net.snowflake.client.jdbc.SnowflakeDriver"); - - //build our properties - Properties props = new Properties(); - props.put("user", user); - props.put("password", PASSWORD); - props.put("account", ACCOUNT); - props.put("ssl", SSL); - - //fire off the connection - return DriverManager.getConnection(CONNECT_STRING, props); - } - - /** - * Creates a local file for loading into our table - * - * @return URI of this file - * @throws IOException If we can't write the file - */ - private URI makeLocalFile() - throws IOException - { - File file = new File(FILENAME); - - //if our file doesn't already exist - if (!file.exists()) - { - //create it - file.createNewFile(); - - //populate it with some data - FileWriter fw = new FileWriter(file.getAbsoluteFile()); - BufferedWriter bw = new BufferedWriter(fw); - for (char letter = 'a'; letter <= 'z'; letter++) - { - bw.write(letter + "\n"); - } - //close it back up - bw.close(); - } - - return file.toURI(); - } - - /** - * Attempts to create a directory in which we can store - * our local files - */ - private void createTempStageDir() - { - final String base = "/tmp/data"; - try - { - Files.createDirectories(Paths.get(base)); - } catch (IOException e) - { - throw new IllegalStateException("create temp dir failed", e); - } - } - - - /** - * Try to execute a query and throw if we fail - * - * @param query the query in question - */ - private void doQuery(String query) - { - try (Statement statement = conn.createStatement()) - { - statement.executeQuery(query); - } - //if ANY exceptions occur, an illegal state has been reached - catch (Exception e) - { - throw new IllegalStateException(e); - } - } - - /** - * Generate the public key as a string - * - * @return the public key as a string - */ - private String getPublicKeyString() - throws NoSuchAlgorithmException, InvalidKeySpecException - { - KeyFactory keyFactory = KeyFactory.getInstance(ALGORITHM); - final PublicKey pk = keypair.getPublic(); - X509EncodedKeySpec spec = - keyFactory.getKeySpec(pk, X509EncodedKeySpec.class); - return Base64.encodeBase64String(spec.getEncoded()); - } - - - /** - * Simple helper method to escape a string via quotes - * - * @return quoted string - */ - private static String quote(String arg) - { - return '"' + arg + '"'; - } - - /** - * Creates the stages and files we'll use for this test - */ - @Before - public void setup() - throws Exception - { - //create the temporary directory and local file - createTempStageDir(); - makeLocalFile(); - - - //use the right database - doQuery("use database " + DATABASE); - - //use the right schema - doQuery("use schema " + SCHEMA); - - //create the target stage - doQuery("create or replace stage " + quote(STAGE) + - " url='file:///tmp/data/'"); - - //create the target - doQuery("create or replace table " + quote(TABLE) + - " (c1 string)"); - - doQuery("grant insert on table " + quote(TABLE) + " to accountadmin"); - - doQuery("create or replace pipe " + quote(PIPE) + - " as copy into " + quote(TABLE) + " from @" + quote(STAGE) + - " file_format=(type='csv')"); - - String pk = getPublicKeyString(); - - //assume the necessary privileges - doQuery("use role accountadmin"); - - //set the public key - doQuery("alter user " + USER + - " set RSA_PUBLIC_KEY='" + pk + "'"); - - doQuery("use role sysadmin"); - } - - /** - * Attempts to sleep and fetch the history afterwards - * - * @return the history object or null if an error happened - */ - private HistoryResponse sleepAndFetchHistory() - { - try - { - - Thread.sleep(500); - return manager.getHistory(null); - } catch (Exception e) - { - return null; - } - } - - /** - * testLoadSingle -- succeeds if we load a single file - */ - @Test - public void testLoadSingle() - throws Exception - { - - //keeps track of whether we've loaded the file - boolean loaded = false; - - //create a file wrapper - StagedFileWrapper myFile = new StagedFileWrapper(BASE_FILENAME, null); - - //get an insert response after we submit - IngestResponse insertResponse = manager.ingestFile(myFile, null); - - //create a new thread - ExecutorService service = Executors.newSingleThreadExecutor(); - - //fork off waiting for a load to the service - Future result = service.submit(() -> - { - //we spin here forever - while (true) - { - HistoryResponse response = sleepAndFetchHistory(); - - if (response != null && response.files != null) - { - for (HistoryResponse.FileEntry entry : response.files) - { - //if we have a complete file that we've loaded with the same name.. - if (entry.getPath() != null && entry.isComplete() - && entry.getPath().contains(BASE_FILENAME)) - { - //we can return true! - return; - } - } - } - } - } - ); - - //try to wait until the future is done - try - { - //wait up to 1 minutes to load - result.get(1, TimeUnit.MINUTES); - loaded = true; - } finally - { - assertTrue(loaded); - } - } - -} diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java new file mode 100644 index 000000000..d14ef3152 --- /dev/null +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -0,0 +1,151 @@ +package net.snowflake.ingest; + +import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; +import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node + .ObjectNode; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import org.apache.commons.codec.binary.Base64; + +public class TestUtils +{ + //profile path + private final static String PROFILE_PATH = "profile.json"; + + private final static ObjectMapper mapper = new ObjectMapper(); + + private static ObjectNode profile = null; + + private static String user = ""; + + private static PrivateKey privateKey = null; + + private static String account = ""; + + private static String ssl = ""; + + private static String database = ""; + + private static String schema = ""; + + private static String warehouse = ""; + + private static String connectString = ""; + + private static String scheme = ""; + + private static String host = ""; + + private static int port = 0; + + private static Connection conn = null; + + + /** + * load all login info from profile + * @throws IOException if can't read profile + */ + private static void init() throws Exception + { + profile = + (ObjectNode) mapper.readTree( + new String(Files.readAllBytes(Paths.get(PROFILE_PATH))) + ); + + user = profile.get("user").asText(); + account = profile.get("account").asText(); + port = profile.get("port").asInt(); + ssl = profile.get("ssl").asText(); + database = profile.get("database").asText(); + connectString = profile.get("connect_string").asText(); + schema = profile.get("schema").asText(); + warehouse = profile.get("warehouse").asText(); + host = profile.get("host").asText(); + scheme = profile.get("scheme").asText(); + + String privateKeyPem = profile.get("private_key").asText(); + + java.security.Security.addProvider( + new org.bouncycastle.jce.provider.BouncyCastleProvider() + ); + + byte[] encoded = Base64.decodeBase64(privateKeyPem); + KeyFactory kf = KeyFactory.getInstance("RSA"); + + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(encoded); + privateKey = kf.generatePrivate(keySpec); + } + + + /** + * Create snowflake jdbc connection + * @return jdbc connection + * @throws Exception + */ + public static Connection getConnection() throws Exception + { + if(conn != null) return conn; + + if(profile == null) init(); + //check first to see if we have the Snowflake JDBC + Class.forName("net.snowflake.client.jdbc.SnowflakeDriver"); + + //build our properties + Properties props = new Properties(); + props.put("user", user); + props.put("account", account); + props.put("ssl", ssl); + props.put("db", database); + props.put("schema", schema); + props.put("warehouse", warehouse); + props.put("client_session_keep_alive", "true"); + props.put("privateKey", privateKey); + + conn = DriverManager.getConnection(connectString, props); + + //fire off the connection + return conn; + } + + /** + * execute sql query + * @param query sql query string + * @return result set + */ + public static ResultSet executeQuery(String query) + { + try (Statement statement = getConnection().createStatement()) + { + return statement.executeQuery(query); + } + //if ANY exceptions occur, an illegal state has been reached + catch (Exception e) + { + throw new IllegalStateException(e); + } + } + + + /** + * create ingest manager + * @param pipe pipe name + * @return ingest manager object + * @throws Exception + */ + public static SimpleIngestManager getManager(String pipe) throws Exception + { + if(profile == null) init(); + return new SimpleIngestManager(account, user, database+"."+ schema + "." + pipe, privateKey, scheme, host, port); + } +} diff --git a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java index 9eb0aa560..775c79415 100644 --- a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java +++ b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java @@ -1,6 +1,7 @@ package net.snowflake.ingest.connection; +import org.junit.Ignore; import org.junit.Test; import java.security.KeyFactory; import java.security.KeyPair; @@ -102,7 +103,7 @@ public class SecurityManagerTest "AQz56Xdi9VEnVs3rsgvX9VnaWcRpa4GT5EIj+I2M9t+D8XCfMMs1S56Pnn5oGkqv\n" + "FBzmMRnskqK6d75B8EG5BGi0\n"; - @Test + @Ignore public void validatePublicKeyFp() throws NoSuchAlgorithmException, InvalidKeySpecException { diff --git a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java b/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java index 1a460e774..94a480f4c 100644 --- a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java +++ b/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java @@ -1,5 +1,6 @@ package net.snowflake.ingest.connection; +import org.junit.Ignore; import org.junit.Test; import java.security.KeyPair; @@ -23,7 +24,7 @@ public class TestKeyRenewal /** * Evaluates whether or not we are actually renewing tokens */ - @Test + @Ignore public void doesRegenerateToken() throws NoSuchProviderException, NoSuchAlgorithmException, InterruptedException { diff --git a/test_files/test1.csv b/test_files/test1.csv new file mode 100644 index 000000000..7462c77a4 --- /dev/null +++ b/test_files/test1.csv @@ -0,0 +1,3 @@ +"a",1 +"b",2 +"c",3 \ No newline at end of file From de78eeab527ddf69858032ba22986afa3212eece Mon Sep 17 00:00:00 2001 From: Bing Li Date: Fri, 19 Oct 2018 11:19:45 -0700 Subject: [PATCH 5/6] move test1.csv to resource folder --- .../snowflake/ingest/SimpleIngestManager.java | 3 +++ .../snowflake/ingest/SimpleIngestSuite.java | 26 ++++++++++++------- .../resources/net/snowflake/ingest}/test1.csv | 0 3 files changed, 19 insertions(+), 10 deletions(-) rename {test_files => src/test/resources/net/snowflake/ingest}/test1.csv (100%) diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index 64b96edae..5f6958fea 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -340,6 +340,9 @@ public SimpleIngestManager(String account, String user, String pipe, private KeyPair createKeyPairFromPrivateKey(PrivateKey privateKey) throws NoSuchAlgorithmException, InvalidKeySpecException { + if(!(privateKey instanceof RSAPrivateCrtKey)) + throw new IllegalArgumentException("Input private key is not a RSA private key"); + KeyFactory kf = KeyFactory.getInstance("RSA"); //generate public key from private key diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java b/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java index 54489aab3..c82bc569c 100644 --- a/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java +++ b/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java @@ -7,7 +7,7 @@ import org.junit.Test; import org.junit.After; -import java.sql.Connection; +import java.net.URL; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -21,12 +21,10 @@ */ public class SimpleIngestSuite { - private Connection conn = null; - private SimpleIngestManager manager = null; - - private final String TEST_FILE_PATH = "file://test_files/test1.csv"; private final String TEST_FILE_NAME = "test1.csv"; + private String testFilePath = null; + private String tableName = ""; private String pipeName = ""; private String stageName = ""; @@ -37,6 +35,14 @@ public class SimpleIngestSuite @Before public void beforeAll() throws Exception { + + //get test file path + + URL resource = SimpleIngestSuite.class.getResource(TEST_FILE_NAME); + testFilePath = resource.getFile(); + + + //create stage, pipe, and table Random rand = new Random(); Long num = Math.abs(rand.nextLong()); @@ -69,7 +75,6 @@ public void beforeAll() throws Exception @After public void afterAll() { - //System.out.println(111); TestUtils.executeQuery( "drop pipe if exists " + pipeName ); @@ -84,14 +89,15 @@ public void afterAll() } /** - * ingest test + * ingest test example + * ingest a simple file and check load history. */ @Test - public void test1() throws Exception + public void testSimpleIngest() throws Exception { //put TestUtils.executeQuery( - "put " + TEST_FILE_PATH + " @" + stageName + "put file://" + testFilePath + " @" + stageName ); //keeps track of whether we've loaded the file @@ -153,7 +159,7 @@ public void test1() throws Exception //try to wait until the future is done try { - //wait up to 1 minutes to load + //wait up to 2 minutes to load result.get(2, TimeUnit.MINUTES); loaded = true; } finally diff --git a/test_files/test1.csv b/src/test/resources/net/snowflake/ingest/test1.csv similarity index 100% rename from test_files/test1.csv rename to src/test/resources/net/snowflake/ingest/test1.csv From d02112d0dca5861dc5a2864da6263e308343f133 Mon Sep 17 00:00:00 2001 From: Bing Li Date: Fri, 19 Oct 2018 11:53:33 -0700 Subject: [PATCH 6/6] setup it test --- pom.xml | 101 ++++++++++++++++++ ...leIngestSuite.java => SimpleIngestIT.java} | 4 +- .../connection/SecurityManagerTest.java | 2 +- .../ingest/connection/TestKeyRenewal.java | 2 +- 4 files changed, 105 insertions(+), 4 deletions(-) rename src/test/java/net/snowflake/ingest/{SimpleIngestSuite.java => SimpleIngestIT.java} (97%) diff --git a/pom.xml b/pom.xml index a21cadd9d..5acc5a8b2 100644 --- a/pom.xml +++ b/pom.xml @@ -13,6 +13,7 @@ Snowflake Ingest SDK https://www.snowflake.net/ + The Apache Software License, Version 2.0 @@ -39,6 +40,8 @@ 1.8 1.8 net.snowflake.ingest.internal + true + 0.8.1 @@ -206,6 +209,36 @@ true + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + pre-unit-test + + prepare-agent + + + target/jacoco-ut.exec + + + + post-unit-test + test + + report + + + target/jacoco-ut.exec + target/jacoco-ut + + + + + ${jacoco.skip.instrument} + + @@ -295,4 +328,72 @@ + + + travisIT + + + travisIT + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + integration-test + + + + @{argLine} -Djava.util.logging.config.file=travis_it_logging.properties + + + + + verify_travis_it + verify + + verify + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + pre-unit-test + + prepare-agent + + + target/jacoco-ut.exec + + + + post-unit-test + test + + report + + + target/jacoco-ut.exec + target/jacoco-ut + + + + + ${jacoco.skip.instrument} + + + + + + + diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java similarity index 97% rename from src/test/java/net/snowflake/ingest/SimpleIngestSuite.java rename to src/test/java/net/snowflake/ingest/SimpleIngestIT.java index c82bc569c..44b248171 100644 --- a/src/test/java/net/snowflake/ingest/SimpleIngestSuite.java +++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java @@ -19,7 +19,7 @@ /** * Example ingest sdk integration test */ -public class SimpleIngestSuite +public class SimpleIngestIT { private final String TEST_FILE_NAME = "test1.csv"; @@ -38,7 +38,7 @@ public void beforeAll() throws Exception //get test file path - URL resource = SimpleIngestSuite.class.getResource(TEST_FILE_NAME); + URL resource = SimpleIngestIT.class.getResource(TEST_FILE_NAME); testFilePath = resource.getFile(); diff --git a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java index 775c79415..8aa2eb259 100644 --- a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java +++ b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java @@ -103,7 +103,7 @@ public class SecurityManagerTest "AQz56Xdi9VEnVs3rsgvX9VnaWcRpa4GT5EIj+I2M9t+D8XCfMMs1S56Pnn5oGkqv\n" + "FBzmMRnskqK6d75B8EG5BGi0\n"; - @Ignore + @Test public void validatePublicKeyFp() throws NoSuchAlgorithmException, InvalidKeySpecException { diff --git a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java b/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java index 94a480f4c..1ca5c598a 100644 --- a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java +++ b/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java @@ -24,7 +24,7 @@ public class TestKeyRenewal /** * Evaluates whether or not we are actually renewing tokens */ - @Ignore + @Test public void doesRegenerateToken() throws NoSuchProviderException, NoSuchAlgorithmException, InterruptedException {