From b6d66ddd4e7140a3e3dafcc8a7dc0b5e10392d18 Mon Sep 17 00:00:00 2001 From: orellabac Date: Wed, 3 Jan 2024 15:03:59 -0600 Subject: [PATCH] Adding JDBC Data Reader --- extras/jdbc_read/.gitignore | 4 + extras/jdbc_read/README.md | 41 +++++++++ extras/jdbc_read/pom.xml | 37 ++++++++ .../src/main/java/JdbcDataReader.java | 86 ++++++++++++++++++ .../target/snowpark-jdbc-reader-0.0.1.jar | Bin 0 -> 4983 bytes 5 files changed, 168 insertions(+) create mode 100644 extras/jdbc_read/.gitignore create mode 100644 extras/jdbc_read/README.md create mode 100644 extras/jdbc_read/pom.xml create mode 100644 extras/jdbc_read/src/main/java/JdbcDataReader.java create mode 100644 extras/jdbc_read/target/snowpark-jdbc-reader-0.0.1.jar diff --git a/extras/jdbc_read/.gitignore b/extras/jdbc_read/.gitignore new file mode 100644 index 0000000..84716e7 --- /dev/null +++ b/extras/jdbc_read/.gitignore @@ -0,0 +1,4 @@ +target/classes +target/maven-status +target/maven-archiver + diff --git a/extras/jdbc_read/README.md b/extras/jdbc_read/README.md new file mode 100644 index 0000000..a73b248 --- /dev/null +++ b/extras/jdbc_read/README.md @@ -0,0 +1,41 @@ +# JDBC Data Reader for Snowpark + +See[ medium article for more details.](https://medium.com/@orellabac/ingest-external-data-into-snowflake-with-snowpark-and-jdbc-eb487b61078c) + +**Building** + +``` +mvn clean package +``` + +The target file will be at: `target/snowpark-jdbc-reader-0.0.1.jar` + +For convience there is a prebuilt-jar you can use. + +**Deployment** + +upload the jar to snowflake, you can use the [SnowSight UI to do that easily ](https://docs.snowflake.com/en/user-guide/data-load-local-file-system-stage-ui) + +and create an UDF like this: + +``` +CREATE OR REPLACE SECRET external_database_cred + TYPE = password + USERNAME = 'serveradmin' + PASSWORD = 'xxxxxxxxx'; + +CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION external_database_network_rule_ext_int + ALLOWED_NETWORK_RULES = (external_database_network_rule) + ALLOWED_AUTHENTICATION_SECRETS = (external_database_cred) + ENABLED = true; + + +CREATE OR REPLACE FUNCTION READ_JDBC(OPTION OBJECT, query STRING) + RETURNS TABLE(data OBJECT) + LANGUAGE JAVA +RUNTIME_VERSION='11' + IMPORTS = ('@mystage/snowpark-jdbc-reader-0.0.1.jar' ) -- Add the jar for any jdbc driver you need in this section + EXTERNAL_ACCESS_INTEGRATIONS = (external_database_network_rule_ext_int) + SECRETS = ('cred' = external_database_cred ) + HANDLER = 'JdbcDataReader'; +``` diff --git a/extras/jdbc_read/pom.xml b/extras/jdbc_read/pom.xml new file mode 100644 index 0000000..28794f1 --- /dev/null +++ b/extras/jdbc_read/pom.xml @@ -0,0 +1,37 @@ + + + + 4.0.0 + + com.snowflake + snowpark-jdbc-reader + 0.0.1 + + Simple JDBC reader + + https://medium.com/@orellabac/ingest-external-data-into-snowflake-with-snowpark-and-jdbc-eb487b61078c + + + UTF-8 + 11 + 11 + + + + + com.snowflake + snowpark + 1.9.0 + provided + + + junit + junit + 4.11 + test + + + + + \ No newline at end of file diff --git a/extras/jdbc_read/src/main/java/JdbcDataReader.java b/extras/jdbc_read/src/main/java/JdbcDataReader.java new file mode 100644 index 0000000..8cf51d7 --- /dev/null +++ b/extras/jdbc_read/src/main/java/JdbcDataReader.java @@ -0,0 +1,86 @@ +import java.io.PrintWriter; +import java.io.StringWriter; +import java.sql.*; +import java.util.*; +import java.util.stream.Stream; +import com.snowflake.snowpark_java.types.SnowflakeSecrets; + +public class JdbcDataReader { + + public static class OutputRow { + public Map data; + + public OutputRow(Map data) { + this.data = data; + } + } + + public static Class getOutputClass() { + return OutputRow.class; + } + + public Stream process(Map jdbcConfig, String query) { + String jdbcUrl = jdbcConfig.get("url"); + Properties properties = new Properties(); + + if ("true".equals(jdbcConfig.get("use_secrets"))) + { + SnowflakeSecrets sfSecrets = SnowflakeSecrets.newInstance(); + jdbcConfig.remove("use_secrets"); + var secret = sfSecrets.getUsernamePassword("cred"); + properties.setProperty("username", secret.getUsername()); + properties.setProperty("password", secret.getPassword()); + } + + try { + properties.putAll(jdbcConfig); + // Load the JDBC driver + Class.forName(jdbcConfig.get("driver")); + // Create a connection to the database + Connection connection = DriverManager.getConnection(jdbcUrl, properties); + // Create a statement for executing SQL queries + Statement statement = connection.createStatement(); + // Execute the query + ResultSet resultSet = statement.executeQuery(query); + // Get metadata about the result set + ResultSetMetaData metaData = resultSet.getMetaData(); + // Create a list of column names + List columnNames = new ArrayList<>(); + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + columnNames.add(metaData.getColumnName(i)); + } + // Convert the ResultSet to a Stream of OutputRow objects + Stream resultStream = Stream.generate(() -> { + try { + if (resultSet.next()) { + Map rowMap = new HashMap<>(); + for (String columnName : columnNames) { + String columnValue = resultSet.getString(columnName); + rowMap.put(columnName, columnValue); + } + return new OutputRow(rowMap); + } else { + // Close resources + resultSet.close(); + statement.close(); + connection.close(); + return null; + } + } catch (SQLException e) { + e.printStackTrace(); + return null; + } + }).takeWhile(Objects::nonNull); + return resultStream; + } catch (Exception e) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + String stackTrace = sw.toString(); + Map rowMap = new HashMap<>(); + rowMap.put("ERROR",e.getMessage() + "\n" + stackTrace); + return Stream.of(new OutputRow(rowMap)); + } + } +} \ No newline at end of file diff --git a/extras/jdbc_read/target/snowpark-jdbc-reader-0.0.1.jar b/extras/jdbc_read/target/snowpark-jdbc-reader-0.0.1.jar new file mode 100644 index 0000000000000000000000000000000000000000..11b40c32b118ebf417fe276017aeca0702b185b2 GIT binary patch literal 4983 zcmbVQ2UJt(whmQ50!jxdN;NeL|5o6ad zMIBi%L|utr1M5bGbtC%2?ey%Xrv~R_J-J2U-nXA4JrJ{Jc^kNb9TTf{$v z{`P-`UVu2kk+xtPOLGe_5&^eFAb;sxbdM3#4hI16!vz4Cv3O9drvls+uJwmA`yDq| zCpT9uM-M&=JGhHWn&w?qTv;lqY~xRpxw(k@ELRRP2~Vg9!uXYRp7{kGh}V z3z^-u$VXz4@t81Thd?CyH#BB{adhbnw_i^Ia zk-9aY!iZ5_7SoJTTk}GKJhXc0{nVO^nbK}ve!(^RA%irH2>H>&E&4u}acb{(^Rm}h z3nuF53_cTYzN1E8KW#MRj)=)PLeu%2Fejc>nN-S@uLL%M)e#?KQyjgoj0&(%vT9hF zX1dxX3B0m5$9X~;*sO$jgZB7jzy{|D8bRNdZ_wGYNIcWb_CdJ5iE~KHqNAYfl!-Z} z0F4LSYO`FTFDgF|m|#|=jI$dMkji}U9RC+!MzkHJpR2iorw{^x!AzB^S0#ArqY^|?>VbXxz05b+1j z&Ci6xj{pD?>uU z!chJ-)J3R;N13`e&mG-Gyew@j>{_ts3v!Ag}Uwe~4*AC7}t#G(hEm+Q2nxTv7*Z?1Q z5$TracL#J_8OAZ(FqkiTIYSo{Mhp?loXZR&vtGzyXAWx#9)x_|(tG`-LB}ada_6$v zw?_EXWB#|@iR_!bd(Ih^Ytq)!k4T!bvM!tH?g$ecT^@1lt4k{S;21&tKq}XI!gjcB zkuFl^l>{PpN*C9ytQw4aCOnpTV`w7fw}_^VRrmHLt=V)=j$vDl!vmUW^`31>%ywi> zes;l8SnFm_ubck1$boG87R3X)+f!4|XCSzO7WR!22h$I&j6{MZq*&j_h1+ugU(Yi2 z+FMQQ)VcVluW`?-*Sf!D+_K@BcQW}fq`X9CrlYe>$`PJpU#sKh?KsS^>8fUHce8j~ zs6kh$#=~L1W4e*PeNBzZeP2uGs%yhiPmkFWU3EbeLkcSDRxfi-gUYT?)Z^sJ)%l#1 zay8%O)c%1I*Og>YxmD@Q`mTKVUW1`xx`HpaphYS+~53ti=qEAiFo2Y}n)x$sc?h^wR&(MzkDFC##m>3x$@5VqK(k~)7> zPIItUJFf3t!}b}~BFKWSd-4Z`yIrrQ;|>&6LziLN2YjzJgHS-VRtJ8*HdIM+Z}$M$hB~SmpYa@}Oi|MX1o)tu<8Wln`i65Nx`CvZ3R! zOj~@J-g4=kNd>4l%7JCXc%M4cEl`GA54ap}P@DkPttcw4#?eO2X601`B~}C;TzPgE z*|t2xT0(^mP9^5C<%)0vg$5)oUE)nd>P%aS_DLB^G67#YC$*#+CA`|YzgZcyz2M~v zdEt_DJlX=fTng?dvtqI^F0tFqxwISYw{3fjmA`! zQe~l2M)KJ7o*)G;NK zhLB^G{U-rfcz-R9eMR~@-f9^=o zEp&S6+`dho{dY_NmoTFh6NTI{Nq1bHXR8Rca9)dvsU01mW*%`f(F4{HHrzaFpjoK| zn`s1b(dPE-L@6=^*mL?@^K?tVMp$6-V;h9mO?9Wo9 ztU%g^9gGR6g>7e6!M^X=F2@?Smzb>Hq?I^ey@@tM}EdulSmmoT17Up7#A z_4%UfcvieyyB22H!0|=ZUHf+sg{U%~mwCkRmTf5XFQUM(<}6YoObykHVFH&`+cf&?c1>8YSCo{AiQMe4G0Q~)V$XOu-?bSC zSI2;gTI&%#sh!X3^PEo1@oXXwb%0)J(xUlcVys5Q;b=)+6V_!URe0K~{TJuscdKuT z#R*H9Td}6}=Ci_~|aS_TB|o|U)JDH4qjX?gAGcS_zeyC+-pIReqTTOY*h@Wev5 zc)Eix2|9d5%&op})c0y>qnXKQA?s17cXVGk4A^+$Io|s`d;=01Q|6)mwA58Jq3^~x zcQ{UAuwXa5Jm1{cK!x{}Fy?i3WkPGF7#_$YfX9#pQH5Ssd`=`NULq&Ue2>fU{Lxr+ z%loQ&Mcv>Nn8h7FHo8&`o5G{YxXz-X*pH6gSEtwZE!Wr5{PGf{VI0YIi0et!jPh5X z2$dP67R)Qws$VTmHfRGudE?X0d3^=L{6H=>o!m2@V8y@X1s|{j;f+L5G|g0)F7ZiQ zLx~EzY4SLRSpDb}-?2~2_a>c;TvByewN*9F$i%Apr=D^p*rznrE8WSX1o4YNgK|w| z5aa^j3VxmxrB6bbDP81_iT=*m*?IGwy^Zlyx{~-KZ3T&I?;-EIw0z8^QzKPhk?du# z$ej_+xd5BRPiWAWCdp-{dIHH4_vsHpZWJwP_ujh4l-}pk4CKsRhQ`^gcpsBO_20DGGoTMZ7PHJQU0fTiv`iCQ1Xq4h0eL zDx1}nn%z^_1nCK_9ObuQ38%9pLYo@@?q>nRNXb5l!S{Dy| zbLS`sB9)_s2ul2l{H95|s5antI&Sfbs&C_Si?*`37f*aznyP{o@*cl#ph&S+3^`Ey z%056@OMi_dde=j30nIute@KAj;ne2|RCpvg!3Z&crq}fk)ukFo!hEJwM&0FW$4Q zHW(U9mLkd^FsX+%?Kj<}9Nxiu&}Yo&{y1VfW_#A5hk|xjB-Df@EqS$R+xcAFxil<` zoxDcGW8S>|>F|IH__~>;VQrav0onrp2r~YAH-i7FGFLDwl#y&DH4s#Ji(>t2tEj3J z^sErfT2+;6+pNN==AU-^n?--d;}~r*)~SbwZvx4?RVt^XpEnlam$z^jKnEVbiHhpa zN{)$XKN0P}cF&{G+V)M8H~RdB#5Xo!isARkiWPiy1O;`3mz+PQm1&GOEcgYC`tB1& zG8Zii$4qt8ekO#GX_TIl!H};L{4MhnUcOw18JZO1b2#fH(H^&@4d_jmkShz|!A+Xe7seJ`;@4Mb%w^>+8wKT` z-x7}Z1_quRVGoPK+{q-R$D~{6Y*#JpCCdkO{0QEgQa6cXV&)}7*ak=Yd!{{%P7IUS zZe=@%_U>_nN`JJ~n|bevHcK`aIOrM#v2KX`7?%s~@5QeJk;jgS0``{qugAp+>F9(& zx>_S#RC&33)Kt~@cGb$j1HJtGD!gyU`k@f$7#IpugMb&H{BQZe04V3m%DOLA&Jz3m zJY@U}8fvTMmG=y*^sDqp94zO@!KK3cYa)LkPkgs4fKAzp)lXu@McCiV75|O{I?bdyIlm2VVA!I|0+iOJ=BF9@!c++I(=e>M0&(wt)V`v2e5i!?uufrcsp;RPZw_Ugc@K517jKK%<}4QL_& literal 0 HcmV?d00001