Skip to content

Commit

Permalink
add semijoin to white list and use filter to create new iterator in L…
Browse files Browse the repository at this point in the history 
…eftSemiJoinBNL
  • Loading branch information
@adrian-wang
adrian-wang committed Jun 11, 2014
1 parent 0266a0c commit 27579a9
Show file tree
Hide file tree
Showing 52 changed files with 371 additions and 30 deletions.
34 changes: 4 additions & 30 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/joins.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,36 +184,10 @@ case class LeftSemiJoinHash(
}
}

new Iterator[Row] {
private[this] var currentStreamedRow: Row = _
private[this] var currentHashMatched: Boolean = false

private[this] val joinKeys = streamSideKeyGenerator()

override final def hasNext: Boolean =
streamIter.hasNext && fetchNext()

override final def next() = {
currentStreamedRow
}

/**
* Searches the streamed iterator for the next row that has at least one match in hashtable.
*
* @return true if the search is successful, and false the streamed iterator runs out of
* tuples.
*/
private final def fetchNext(): Boolean = {
currentHashMatched = false
while (!currentHashMatched && streamIter.hasNext) {
currentStreamedRow = streamIter.next()
if (!joinKeys(currentStreamedRow).anyNull) {
currentHashMatched = hashTable.contains(joinKeys.currentValue)
}
}
currentHashMatched
}
}
streamIter.filter(current => {
val joinKeys = streamSideKeyGenerator()
!joinKeys(current).anyNull && hashTable.contains(joinKeys.currentValue)
})
}
}
}
Expand Down
Empty file added sql/hive/src/test/resources/golden/semijoin-0-1631b71327abf75b96116036b977b26c
View file
Empty file.
11 changes: 11 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-1-e7c99e1f46d502edbb0925d75aab5f0c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
0 val_0
0 val_0
0 val_0
2 val_2
4 val_4
5 val_5
5 val_5
5 val_5
8 val_8
9 val_9
10 val_10
Empty file added sql/hive/src/test/resources/golden/semijoin-10-ffd4fb3a903a6725ccb97d5451a3fec6
View file
Empty file.
8 changes: 8 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-11-246a40dcafe077f02397e30d227c330
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
0 val_0
0 val_0
0 val_0
4 val_2
8 val_4
10 val_5
10 val_5
10 val_5
Empty file added sql/hive/src/test/resources/golden/semijoin-12-6d93a9d332ba490835b17f261a5467df
View file
Empty file.
Empty file added sql/hive/src/test/resources/golden/semijoin-13-18282d38b6efc0017089ab89b661764f
View file
Empty file.
Empty file added sql/hive/src/test/resources/golden/semijoin-14-19cfcefb10e1972bec0ffd421cd79de7
View file
Empty file.
11 changes: 11 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-15-1de6eb3f357bd1c4d02ab4d19d43589
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
val_0
val_0
val_0
val_10
val_2
val_4
val_5
val_5
val_5
val_8
val_9
Empty file added sql/hive/src/test/resources/golden/semijoin-16-d3a72a90515ac4a8d8e9ac923bcda3d
View file
Empty file.
3 changes: 3 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-17-f0f8720cfd11fd71af17b310dc2d1019
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
0 val_0
0 val_0
0 val_0
Empty file added sql/hive/src/test/resources/golden/semijoin-18-f7b2ce472443982e32d954cbb5c96765
View file
Empty file.
3 changes: 3 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-19-b1f1c7f701abe81c01e72fb98f0bd13f
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
val_10
val_8
val_9
Empty file added sql/hive/src/test/resources/golden/semijoin-2-deb9c3286ae8e851b1fdb270085b16bc
View file
Empty file.
Empty file added sql/hive/src/test/resources/golden/semijoin-20-b7a8ebaeb42b2eaba7d97cadc3fd96c1
View file
Empty file.
Empty file added sql/hive/src/test/resources/golden/semijoin-21-480418a0646cf7260b494b9eb4821bb6
View file
Empty file.
Empty file added sql/hive/src/test/resources/golden/semijoin-22-b6aebd98f7636cda7b24e0bf84d7ba41
View file
Empty file.
5 changes: 5 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-23-ed730ccdf552c07e7a82ba6b7fd3fbda
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
4 val_2
8 val_4
10 val_5
10 val_5
10 val_5
Empty file added sql/hive/src/test/resources/golden/semijoin-24-d16b37134de78980b2bf96029e8265c3
View file
Empty file.
19 changes: 19 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-25-be2bd011cc80b480b271a08dbf381e9b
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
0
0
0
0
0
0
2
4
4
5
5
5
8
8
9
10
10
10
10
Empty file added sql/hive/src/test/resources/golden/semijoin-26-f1d3bab29f1ebafa148dbe3816e1da25
View file
Empty file.
4 changes: 4 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-27-391c256254d171973f02e7f33672ce1d
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
0 val_0
0 val_0
0 val_0
8 val_8
Empty file added sql/hive/src/test/resources/golden/semijoin-28-b56400f6d9372f353cf7292a2182e963
View file
Empty file.
14 changes: 14 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-29-9efeef3d3c38e22a74d379978178c4f5
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
0 val_0 0 val_0
4 val_4 4 val_2
8 val_8 8 val_4
10 val_10 10 val_5
10 val_10 10 val_5
10 val_10 10 val_5
11 changes: 11 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-3-b4d4317dd3a10e18502f20f5c5250389
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
0 val_0
0 val_0
0 val_0
4 val_2
8 val_4
10 val_5
10 val_5
10 val_5
16 val_8
18 val_9
20 val_10
Empty file added sql/hive/src/test/resources/golden/semijoin-30-dd901d00fce5898b03a57cbc3028a70a
View file
Empty file.
14 changes: 14 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-31-e5dc4d8185e63e984aa4e3a2e08bd67
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
0 val_0
0 val_0
0 val_0
0 val_0
0 val_0
0 val_0
2 val_2
4 val_4
5 val_5
5 val_5
5 val_5
8 val_8
9 val_9
10 val_10
Empty file added sql/hive/src/test/resources/golden/semijoin-32-23017c7663f2710265a7e2a4a1606d39
View file
Empty file.
14 changes: 14 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-33-ed499f94c6e6ac847ef5187b3b43bbc5
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
0
0
0
0
0
0
4
4
8
8
10
10
10
10
Empty file added sql/hive/src/test/resources/golden/semijoin-34-5e1b832090ab73c141c1167d5b25a490
View file
Empty file.
26 changes: 26 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-35-8d5731f26232f6e26dd8012461b08d99
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
4
4
8
8
10
10
10
10
Empty file added sql/hive/src/test/resources/golden/semijoin-36-b1159823dca8025926407f8aa921238d
View file
Empty file.
29 changes: 29 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-37-a15b9074f999ce836be5329591b968d0
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
NULL
NULL
NULL
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
4
4
8
8
10
10
10
10
Empty file added sql/hive/src/test/resources/golden/semijoin-38-f37547c73a48ce3ba089531b176e6ba
View file
Empty file.
29 changes: 29 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-39-c22a6f11368affcb80a9c80e653a47a8
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
NULL
NULL
NULL
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
4
4
8
8
10
10
10
10
Empty file added sql/hive/src/test/resources/golden/semijoin-4-dfdad5a2742f93e8ea888191460809c0
View file
Empty file.
Empty file added sql/hive/src/test/resources/golden/semijoin-40-32071a51e2ba6e86b1c5e40de55aae63
View file
Empty file.
29 changes: 29 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-41-cf74f73a33b1af8902b7970a9350b092
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
4
4
8
8
10
10
10
10
16
18
20
Empty file added sql/hive/src/test/resources/golden/semijoin-42-6b4257a74fca627785c967c99547f4c0
View file
Empty file.
31 changes: 31 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-43-e8a166ac2e94bf8d1da0fe91b0db2c81
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
NULL
NULL
NULL
NULL
NULL
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
4
4
8
8
10
10
10
10
Empty file added sql/hive/src/test/resources/golden/semijoin-44-945aaa3a24359ef73acab1e99500d5ea
View file
Empty file.
42 changes: 42 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-45-3fd94ffd4f1eb6cf83dcc064599bf12b
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
NULL
NULL
NULL
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
2
4
4
5
5
5
8
8
9
10
10
10
10
10
10
10
10
10
10
10
10
Empty file added sql/hive/src/test/resources/golden/semijoin-46-620e01f81f6e5254b4bbe8fab4043ec0
View file
Empty file.
35 changes: 35 additions & 0 deletions sql/hive/src/test/resources/golden/semijoin-47-f0140e4ee92508ba241f91c157b7af9c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
4
4
8
8
10
10
10
10
10
10
10
10
10
10
16
18
20
Empty file added sql/hive/src/test/resources/golden/semijoin-48-8a04442e84f99a584c2882d0af8c25d8
View file
Empty file.
Empty file added sql/hive/src/test/resources/golden/semijoin-49-df1d6705d3624be72036318a6b42f04c
View file
Empty file.
Loading

0 comments on commit 27579a9

Please sign in to comment.