Skip to content

Commit

Permalink
Make partition aggregation examples more readable
Browse files Browse the repository at this point in the history
  • Loading branch information
k-jamroz committed Mar 18, 2024
1 parent 35f7271 commit d102584
Show file tree
Hide file tree
Showing 3 changed files with 231 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@

package com.hazelcast.jet.sql;

import com.hazelcast.jet.sql.impl.connector.map.model.Person;
import com.hazelcast.jet.sql.impl.connector.map.model.Order;
import com.hazelcast.jet.sql.impl.connector.map.model.OrderKey;
import com.hazelcast.map.IMap;
import com.hazelcast.sql.SqlResult;
import com.hazelcast.test.HazelcastSerialClassRunner;
import com.hazelcast.test.annotation.ParallelJVMTest;
import com.hazelcast.test.annotation.QuickTest;
Expand All @@ -27,12 +29,15 @@
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.util.List;
import java.util.stream.StreamSupport;

@RunWith(HazelcastSerialClassRunner.class)
@Category({QuickTest.class, ParallelJVMTest.class})
public class SqlPartitionPruningSingleTableAggregationTest extends SqlTestSupport {
private String mapName = "test_map"; //generateRandomString(16);
private String mapName = "orders"; //generateRandomString(16);

@BeforeClass
public static void beforeClass() {
Expand All @@ -41,15 +46,25 @@ public static void beforeClass() {

@Before
public void createTestTable() {
createMapping(mapName, Person.class, Integer.class);
createMapping(mapName, OrderKey.class, Order.class);
instance().getSql().execute("select get_ddl('relation', '" + mapName + "')")
.forEach(r -> System.out.println(r.<Object>getObject(0)));

IMap<Object, Object> map = instance().getMap(mapName);

for (int i = 0; i < 1000; i++) {
String key = "key" + i%3;
map.put(new Person(i, key), i);
final String[] countries = new String[] {"PL", "UA", "UK", "US"};
int orderId = 1000;
for (int custId = 0; custId < 10; ++custId) {
// create skewed data
for (int i = 0; i < 3*custId; i++) {
OrderKey key = new OrderKey("C" + custId, orderId++, countries[custId % countries.length]);
Order data = new Order();
data.setAmount(BigDecimal.valueOf(orderId + 5));
data.setOrderDate(LocalDateTime.now().minusYears(1).plusHours(orderId));
data.setPriority(orderId % 7 == 0 ? Order.Priority.URGENT : Order.Priority.NORMAL);
data.setDeliveryDate(orderId % 2 == 0 ? null : data.getOrderDate().plusDays(4));
map.put(key, data);
}
}
}

Expand All @@ -59,24 +74,46 @@ public void test_countNoFilter() {
}

@Test
public void test_countFilterKeyAttr() {
test_countPartitioned("name='key0'");
test_countPartitioned("id=10");
public void test_countFilterKeyPartitionAttr() {
test_countPartitioned("customerId='C2'");
}

@Test
public void test_countFilterKeyNonPartitionAttr() {
test_countPartitioned("country='PL'");
}

@Test
public void test_countFilterKeyPartitionAndNonPartitionAttr() {
// additional predicates should not confuse partition pruning - can be executed as residual filters
test_countPartitioned("customerId='C2' and country like 'U%'");
}

@Test
public void test_countFilterKeyPartitionAnValueAttr() {
// additional predicates should not confuse partition pruning - can be executed as residual filters
test_countPartitioned("customerId='C2' and cast(priority as varchar) = 'NORMAL'");
}

@Test
public void test_countMultiplePartitions() {
test_countPartitioned("customerId in ('C2', 'C3', 'C4')");
}

private void test_countPartitioned(String filter) {
String filterText = filter != null ? " WHERE " + filter : "";

//TODO: how is distinct different?
//TODO: order by after aggregation

// no grouping
analyzeQuery("select count(*) from " + mapName + filterText, null); //rows(1, 1000L));
analyzeQuery("select count(*), sum(amount) from " + mapName + filterText, null);
// group by key attr
analyzeQuery("select count(*), name from " + mapName + filterText + " group by name", null);
analyzeQuery("select count(*), sum(amount), customerId from " + mapName + filterText + " group by customerId", null);
// group by key attr function
analyzeQuery("select count(*), name from " + mapName + filterText + " group by name", null);
analyzeQuery("select count(*), sum(amount), lower(customerId) from " + mapName + filterText + " group by lower(customerId)", null);
// group by key attr and value (same for attr?)
analyzeQuery("select count(*), name, this from " + mapName + filterText + " group by name, this", null);
analyzeQuery("select count(*), sum(amount), customerId, priority from " + mapName + filterText + " group by customerId, priority", null);
}

private void analyzeQuery(String sql, List<Row> rows) {
Expand All @@ -85,7 +122,11 @@ private void analyzeQuery(String sql, List<Row> rows) {
assertRowsAnyOrder(sql, rows);
} else {
// instance().getSql().execute(sql).forEach(System.out::println);
instance().getSql().execute(sql).close();
try(SqlResult result = instance().getSql().execute(sql)) {
StreamSupport.stream(result.spliterator(), false)
.limit(10)
.forEach(System.out::println);
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright 2023 Hazelcast Inc.
*
* Licensed under the Hazelcast Community License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://hazelcast.com/hazelcast-community-license
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hazelcast.jet.sql.impl.connector.map.model;

import java.io.Serializable;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.util.Objects;

public class Order implements Serializable {
private BigDecimal amount;
private LocalDateTime orderDate;
private LocalDateTime deliveryDate;

public enum Priority {
NORMAL, URGENT
};

private Priority priority;

public BigDecimal getAmount() {
return amount;
}

public void setAmount(BigDecimal amount) {
this.amount = amount;
}

public LocalDateTime getOrderDate() {
return orderDate;
}

public void setOrderDate(LocalDateTime orderDate) {
this.orderDate = orderDate;
}

public LocalDateTime getDeliveryDate() {
return deliveryDate;
}

public void setDeliveryDate(LocalDateTime deliveryDate) {
this.deliveryDate = deliveryDate;
}

public Priority getPriority() {
return priority;
}

public void setPriority(Priority priority) {
this.priority = priority;
}

@Override
public String toString() {
return "Order{" +
"amount=" + amount +
", orderDate=" + orderDate +
", deliveryDate=" + deliveryDate +
", priority=" + priority +
'}';
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Order order = (Order) o;
return Objects.equals(amount, order.amount) && Objects.equals(orderDate, order.orderDate) && Objects.equals(deliveryDate, order.deliveryDate) && priority == order.priority;
}

@Override
public int hashCode() {
return Objects.hash(amount, orderDate, deliveryDate, priority);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Copyright 2023 Hazelcast Inc.
*
* Licensed under the Hazelcast Community License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://hazelcast.com/hazelcast-community-license
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hazelcast.jet.sql.impl.connector.map.model;

import java.io.Serializable;
import java.util.Objects;

public class OrderKey implements Serializable {
// FK, business partitioning key
private String customerId;
// PK, should be unique
private long orderId;
// attribute added for tests as a non-unique, non-partitioned attribute, not much practical use
private String country;

public OrderKey() {
}

public OrderKey(String customerId, long orderId, String country) {
this.customerId = customerId;
this.orderId = orderId;
this.country = country;
}

public String getCustomerId() {
return customerId;
}

public void setCustomerId(String customerId) {
this.customerId = customerId;
}

public long getOrderId() {
return orderId;
}

public void setOrderId(long orderId) {
this.orderId = orderId;
}

public String getCountry() {
return country;
}

public void setCountry(String country) {
this.country = country;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
OrderKey orderKey = (OrderKey) o;
return orderId == orderKey.orderId;
}

@Override
public int hashCode() {
return Objects.hash(orderId);
}

@Override
public String toString() {
return "OrderKey{" +
"customerId='" + customerId + '\'' +
", orderId=" + orderId +
", country='" + country + '\'' +
'}';
}


}

0 comments on commit d102584

Please sign in to comment.