java.lang.ArrayIndexOutOfBoundsException

[RustRw]

批量插入数据有异常：

java.lang.ArrayIndexOutOfBoundsException: 0
	at com.github.housepower.jdbc.stream.ValuesWithParametersInputFormat.quotedParameter(ValuesWithParametersInputFormat.java:62)
	at com.github.housepower.jdbc.stream.ValuesWithParametersInputFormat.next(ValuesWithParametersInputFormat.java:52)
	at com.github.housepower.jdbc.ClickHouseConnection.sendInsertRequest(ClickHouseConnection.java:144)
	at com.github.housepower.jdbc.statement.ClickHousePreparedInsertStatement.executeBatch(ClickHousePreparedInsertStatement.java:61)
	at com.dp.kuafu.sdk.SdkDau$$anonfun$1.apply(SdkDau.scala:140)
	at com.dp.kuafu.sdk.SdkDau$$anonfun$1.apply(SdkDau.scala:79)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
	at org.apache.spark.scheduler.Task.run(Task.scala:99)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

代码如下：

val sql = "INSERT INTO mk_dw.thirdappclientfunc(dhid, uhid, appId, productId, verCode, chanId, origChanId, " +
                              "lang, ip, createDt, imei, mac, capSsid, capBssid, capApRefId, longi, lati, mapSP, netModel, ts, " +
                              "funid, ext, cts, manuf, model, seq, vcode, vname, chid, dur, sid, sdk, province, pt, hour, minute) " +
                              "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"

                            println("sql:" + sql)

                            val pstmt = connection.prepareStatement(sql);

                            iter.foreach(sdkDauReq => {
                                pstmt.setString(1, sdkDauReq.dhid)
                                pstmt.setString(2, sdkDauReq.uhid)
                                pstmt.setString(3, sdkDauReq.appId)
                                pstmt.setInt(4, sdkDauReq.productId)
                                pstmt.setString(5, sdkDauReq.verCode)
                                pstmt.setString(6, sdkDauReq.chanId)
                                pstmt.setString(7, sdkDauReq.origChanId)
                                pstmt.setString(8, sdkDauReq.lang)
                                pstmt.setString(9, sdkDauReq.ip)
                                pstmt.setString(10, sdkDauReq.createDt)
                                pstmt.setString(11, sdkDauReq.imei)
                                pstmt.setString(12, sdkDauReq.mac)
                                pstmt.setString(13, sdkDauReq.capSsid)
                                pstmt.setString(14, sdkDauReq.capBssid)
                                pstmt.setString(15, sdkDauReq.capApRefId)
                                pstmt.setString(16, sdkDauReq.longi)
                                pstmt.setString(17, sdkDauReq.lati)
                                pstmt.setString(18, sdkDauReq.mapSP)
                                pstmt.setString(19, sdkDauReq.netModel)
                                pstmt.setString(20, sdkDauReq.ts)
                                pstmt.setString(21, sdkDauReq.funid)
                                pstmt.setString(22, sdkDauReq.ext)
                                pstmt.setString(23, sdkDauReq.cts)
                                pstmt.setString(24, sdkDauReq.manuf)
                                pstmt.setString(25, sdkDauReq.model)
                                pstmt.setString(26, sdkDauReq.seq)
                                pstmt.setString(27, sdkDauReq.vcode)
                                pstmt.setString(28, sdkDauReq.vname)
                                pstmt.setString(29, sdkDauReq.chid)
                                pstmt.setString(30, sdkDauReq.dur)
                                pstmt.setString(31, sdkDauReq.sid)
                                pstmt.setString(32, sdkDauReq.sdk)
                                pstmt.setString(33, sdkDauReq.province)
                                pstmt.setDate(34, sdkDauReq.pt)
                                pstmt.setString(35, sdkDauReq.hour)
                                pstmt.setString(36, sdkDauReq.minute)

                                pstmt.addBatch()
                            })
                            pstmt.executeBatch();

[pan3793]

Actually encounter different issue, see #155

[sundy-li]

Can't reproduce it in pure java, is it something different in spark?

[pan3793]

Close issue because 1. can't reproduce, 2. ValuesWithParametersInputFormat has been completely refactored.