Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PHOENIX-7155 Validate Partial Index support with JSON #1767

Merged
merged 5 commits into from
Dec 21, 2023
Merged

PHOENIX-7155 Validate Partial Index support with JSON #1767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a9ef832
IT to test Partial Index with JSON
ranganathg Dec 18, 2023
f445e0c
Added Partial Index tests using JSON Exists function.
ranganathg Dec 19, 2023
b54205a
Adding json specific tests in testWhereInclusion
ranganathg Dec 20, 2023
adf668d
Removing json file create for partial Index test and some clean up in…
ranganathg Dec 20, 2023
49f4464
Removing json files from check
ranganathg Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions phoenix-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@
<excludes>
<exclude>src/main/java/org/apache/phoenix/coprocessor/generated/*.java</exclude>
<exclude>src/main/resources/META-INF/services/java.sql.Driver</exclude>
<exclude>src/it/resources/json/*.json</exclude>
</excludes>
</configuration>
</plugin>
Expand Down
219 changes: 196 additions & 23 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,22 @@
*/
package org.apache.phoenix.end2end.index;

import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.phoenix.end2end.IndexToolIT;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.exception.PhoenixParserException;
import org.apache.phoenix.jdbc.PhoenixResultSet;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.*;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
Expand All @@ -55,6 +41,23 @@
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;

import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.*;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.*;

@Category(NeedsOwnMiniClusterTest.class)
@RunWith(Parameterized.class)
public class PartialIndexIT extends BaseTest {
Expand Down Expand Up @@ -182,7 +185,7 @@ public void testDDLWithAllDataTypes() throws Exception {
+ "S UNSIGNED_DATE, T UNSIGNED_TIMESTAMP, U CHAR(10), V BINARY(1024), "
+ "W VARBINARY, Y INTEGER ARRAY, Z VARCHAR ARRAY[10], AA DATE ARRAY, "
+ "AB TIMESTAMP ARRAY, AC UNSIGNED_TIME ARRAY, AD UNSIGNED_DATE ARRAY, "
+ "AE UNSIGNED_TIMESTAMP ARRAY "
+ "AE UNSIGNED_TIMESTAMP ARRAY, AF JSON "
+ "CONSTRAINT pk PRIMARY KEY (id,kp)) "
+ "MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0" );
String indexTableName = generateUniqueName();
Expand Down Expand Up @@ -764,4 +767,174 @@ public void testPartialIndexPreferredOverFullIndex() throws Exception {
assertFalse(rs.next());
}
}

@Test
public void testPartialIndexWithJson() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
String dataTableName = generateUniqueName();
conn.createStatement().execute("create table " + dataTableName +
" (id varchar not null primary key, " +
"A integer, B integer, C double, D varchar, jsoncol json)");
String indexTableName = generateUniqueName();
String json = "{\"info\":{\"age\": %s }}";
// Add rows to the data table before creating a partial index to test that the index
// will be built correctly by IndexTool
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id1', 25, 2, 3.14, 'a','" +
String.format(json, 25) + "')");

conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol)" +
" values ('id2', 100, 'b','" + String.format(json, 100) + "')");
conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") +
(local ? "LOCAL " : " ") + "INDEX " + indexTableName +
" on " + dataTableName + " (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) " +
(uncovered ? "" : "INCLUDE (B, C, D)") + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 50 ASYNC");

IndexToolIT.runIndexTool(false, null, dataTableName, indexTableName);

String selectSql =
"SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 60";
ResultSet rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is used
assertPlan((PhoenixResultSet) rs, "", indexTableName);
assertTrue(rs.next());
assertEquals("b", rs.getString(1));
assertFalse(rs.next());

selectSql =
"SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 50";
rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is not used
assertPlan((PhoenixResultSet) rs, "", dataTableName);

// Add more rows to test the index write path
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id3', 50, 2, 9.5, 'c','" + String.format(
json, 50) + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id4', 75, 2, 9.5, 'd','" + String.format(
json, 75) + "')");

// Verify that index table includes only the rows with A > 50
selectSql = "SELECT * from " + indexTableName;
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(75, rs.getInt(1));
assertTrue(rs.next());
assertEquals(100, rs.getInt(1));
assertFalse(rs.next());

// Overwrite an existing row that satisfies the index WHERE clause
// such that the new version of the row does not satisfy the index where clause
// anymore. This should result in deleting the index row.
String dml =
"UPSERT INTO " + dataTableName + " values ('id2', 0, 2, 9.5, 'd', JSON_MODIFY(jsoncol, '$.info.age', '0')) ";
conn.createStatement().execute(dml);
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(75, rs.getInt(1));
assertFalse(rs.next());

// Retrieve the updated row from the data table and verify that the index table is not used
selectSql =
"SELECT ID from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 0";
rs = conn.createStatement().executeQuery(selectSql);
assertPlan((PhoenixResultSet) rs, "", dataTableName);
assertTrue(rs.next());
assertEquals("id2", rs.getString(1));

// Test index verification and repair by IndexTool
verifyIndex(dataTableName, indexTableName);

try (Connection newConn = DriverManager.getConnection(getUrl())) {
PTable indexTable = PhoenixRuntime.getTableNoCache(newConn, indexTableName);
assertTrue(StringUtils.deleteWhitespace(indexTable.getIndexWhere())
.equals("CAST(TO_NUMBER(JSON_VALUE(JSONCOL,'$.info.age'))ASINTEGER)>50"));
}
}
}

@Test
public void testPartialIndexWithJsonExists() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
String dataTableName = generateUniqueName();
conn.createStatement().execute("create table " + dataTableName +
" (id varchar not null primary key, " +
"A integer, B integer, C double, D varchar, jsoncol json)" +
(salted ? " SALT_BUCKETS=4" : ""));
String indexTableName = generateUniqueName();
String jsonWithPathExists = "{\"info\":{\"address\":{\"exists\":true}}}";
String jsonWithoutPathExists = "{\"info\":{\"age\": 25 }}";
// Add rows to the data table before creating a partial index to test that the index
// will be built correctly by IndexTool
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id1', 70, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id2', 100, 'b','" + jsonWithoutPathExists + "')");
conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") +
(local ? "LOCAL " : " ") + "INDEX " + indexTableName + " on " + dataTableName + " (A) " +
(uncovered ? "" : "INCLUDE (B, C, D)") + " WHERE JSON_EXISTS(JSONCOL, '$.info.address.exists') ASYNC");
IndexToolIT.runIndexTool(false, null, dataTableName, indexTableName);

String selectSql =
"SELECT " + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") +
" A, D from " + dataTableName + " WHERE A > 60 AND JSON_EXISTS(jsoncol, '$.info.address.exists')";
ResultSet rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is used
assertPlan((PhoenixResultSet) rs, "", indexTableName);
assertTrue(rs.next());
assertEquals(70, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertFalse(rs.next());

// Add more rows to test the index write path
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id3', 20, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id4', 90, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id5', 150, 'b','" + jsonWithoutPathExists + "')");

// Verify that index table includes only the rows where jsonPath Exists
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(70, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertTrue(rs.next());
assertEquals(90, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertFalse(rs.next());

rs = conn.createStatement().executeQuery("SELECT Count(*) from " + dataTableName);
// Verify that the index table is not used
assertPlan((PhoenixResultSet) rs, "", dataTableName);
assertTrue(rs.next());
assertEquals(5, rs.getInt(1));

// Overwrite an existing row that satisfies the index WHERE clause such that
// the new version of the row does not satisfy the index where clause anymore. This
// should result in deleting the index row.
conn.createStatement().execute(
"upsert into " + dataTableName + " (ID, B, jsoncol) values ('id4', null, '" + jsonWithoutPathExists + "')");
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(70, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertFalse(rs.next());

// Test index verification and repair by IndexTool
verifyIndex(dataTableName, indexTableName);

try (Connection newConn = DriverManager.getConnection(getUrl())) {
PTable indexTable = PhoenixRuntime.getTableNoCache(newConn, indexTableName);
assertTrue(StringUtils.deleteWhitespace(indexTable.getIndexWhere())
.equals("JSON_EXISTS(JSONCOL,'$.info.address.exists')"));
}
}
}
}
37 changes: 1 addition & 36 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,14 @@
@Category(ParallelStatsDisabledTest.class)
public class JsonFunctionsIT extends ParallelStatsDisabledIT {
public static String BASIC_JSON = "json/json_functions_basic.json";
public static String FUNCTIONS_TEST_JSON = "json/json_functions_tests.json";
public static String DATA_TYPES_JSON = "json/json_datatypes.json";
String basicJson = "";
String dataTypesJson = "";
String functionsJson = "";

@Before
public void setup() throws IOException {
basicJson = getJsonString(BASIC_JSON, "$[0]");
dataTypesJson = getJsonString(DATA_TYPES_JSON);
functionsJson = getJsonString(FUNCTIONS_TEST_JSON);
}

@Test
Expand Down Expand Up @@ -158,39 +155,6 @@ public void testSimpleJsonModify() throws Exception {
}
}

@Test
public void testSimpleJsonValue2() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
String tableName = generateUniqueName();
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
String ddl = "create table if not exists " + tableName + " (pk integer primary key, col integer, jsoncol json)";
conn.createStatement().execute(ddl);
PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES (?,?,?)");
stmt.setInt(1, 1);
stmt.setInt(2, 2);
stmt.setString(3, functionsJson);
stmt.execute();
conn.commit();
ResultSet rs = conn.createStatement().executeQuery("SELECT JSON_VALUE(JSONCOL,'$.test'), " +
"JSON_VALUE(JSONCOL, '$.testCnt'), " +
"JSON_VALUE(JSONCOL, '$.infoTop[5].info.address.state')," +
"JSON_VALUE(JSONCOL, '$.infoTop[4].tags[1]'), " +
"JSON_QUERY(JSONCOL, '$.infoTop'), " +
"JSON_QUERY(JSONCOL, '$.infoTop[5].info'), " +
"JSON_QUERY(JSONCOL, '$.infoTop[5].friends') " +
"FROM " + tableName + " WHERE JSON_VALUE(JSONCOL, '$.test')='test1'");
assertTrue(rs.next());
assertEquals("test1", rs.getString(1));
assertEquals("SomeCnt1", rs.getString(2));
assertEquals("North Dakota", rs.getString(3));
assertEquals("sint", rs.getString(4));
compareJson(rs.getString(5), functionsJson, "$.infoTop");
compareJson(rs.getString(6), functionsJson, "$.infoTop[5].info");
compareJson(rs.getString(7), functionsJson, "$.infoTop[5].friends");
}
}

private void compareJson(String result, String json, String path) throws JsonProcessingException {
Configuration conf = Configuration.builder().jsonProvider(new GsonJsonProvider()).build();
Object read = JsonPath.using(conf).parse(json).read(path);
Expand Down Expand Up @@ -445,6 +409,7 @@ private void checkInvalidJsonIndexExpression(Properties props, String tableName,
private static String getJsonString(String jsonFilePath) throws IOException {
return getJsonString(jsonFilePath, "$");
}

private static String getJsonString(String jsonFilePath, String jsonPath) throws IOException {
URL fileUrl = JsonFunctionsIT.class.getClassLoader().getResource(jsonFilePath);
String json = FileUtils.readFileToString(new File(fileUrl.getFile()));
Expand Down