Skip to content

Commit

Permalink
PHOENIX-7155 Validate Partial Index support with JSON (apache#1767)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ranganathg
ranganathg committed Jan 3, 2024
1 parent 4d1237f commit fd71272
Show file tree
Hide file tree
Showing 10 changed files with 242 additions and 454 deletions.
3 changes: 3 additions & 0 deletions phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PDateArray;
import org.apache.phoenix.schema.types.PJson;
import org.apache.phoenix.schema.types.PNumericType;
import org.apache.phoenix.schema.types.PTime;
import org.apache.phoenix.schema.types.PTimeArray;
Expand Down Expand Up @@ -110,6 +111,8 @@ private String getValue(PDataType type) {
return "ARRAY[" + getValue(PDate.INSTANCE) + "]";
} else if (type instanceof PArrayDataType) {
return "ARRAY" + type.getSampleValue().toString();
} else if (type instanceof PJson) {
return "'{a:1}'";
} else {
return "0123";
}
Expand Down
219 changes: 196 additions & 23 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,22 @@
*/
package org.apache.phoenix.end2end.index;

import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.phoenix.end2end.IndexToolIT;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.exception.PhoenixParserException;
import org.apache.phoenix.jdbc.PhoenixResultSet;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.*;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
Expand All @@ -55,6 +41,23 @@
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;

import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.*;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.*;

@Category(NeedsOwnMiniClusterTest.class)
@RunWith(Parameterized.class)
public class PartialIndexIT extends BaseTest {
Expand Down Expand Up @@ -182,7 +185,7 @@ public void testDDLWithAllDataTypes() throws Exception {
+ "S UNSIGNED_DATE, T UNSIGNED_TIMESTAMP, U CHAR(10), V BINARY(1024), "
+ "W VARBINARY, Y INTEGER ARRAY, Z VARCHAR ARRAY[10], AA DATE ARRAY, "
+ "AB TIMESTAMP ARRAY, AC UNSIGNED_TIME ARRAY, AD UNSIGNED_DATE ARRAY, "
+ "AE UNSIGNED_TIMESTAMP ARRAY "
+ "AE UNSIGNED_TIMESTAMP ARRAY, AF JSON "
+ "CONSTRAINT pk PRIMARY KEY (id,kp)) "
+ "MULTI_TENANT=true, COLUMN_ENCODED_BYTES=0" );
String indexTableName = generateUniqueName();
Expand Down Expand Up @@ -764,4 +767,174 @@ public void testPartialIndexPreferredOverFullIndex() throws Exception {
assertFalse(rs.next());
}
}

@Test
public void testPartialIndexWithJson() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
String dataTableName = generateUniqueName();
conn.createStatement().execute("create table " + dataTableName +
" (id varchar not null primary key, " +
"A integer, B integer, C double, D varchar, jsoncol json)");
String indexTableName = generateUniqueName();
String json = "{\"info\":{\"age\": %s }}";
// Add rows to the data table before creating a partial index to test that the index
// will be built correctly by IndexTool
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id1', 25, 2, 3.14, 'a','" +
String.format(json, 25) + "')");

conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol)" +
" values ('id2', 100, 'b','" + String.format(json, 100) + "')");
conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") +
(local ? "LOCAL " : " ") + "INDEX " + indexTableName +
" on " + dataTableName + " (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) " +
(uncovered ? "" : "INCLUDE (B, C, D)") + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 50 ASYNC");

IndexToolIT.runIndexTool(false, null, dataTableName, indexTableName);

String selectSql =
"SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 60";
ResultSet rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is used
assertPlan((PhoenixResultSet) rs, "", indexTableName);
assertTrue(rs.next());
assertEquals("b", rs.getString(1));
assertFalse(rs.next());

selectSql =
"SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 50";
rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is not used
assertPlan((PhoenixResultSet) rs, "", dataTableName);

// Add more rows to test the index write path
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id3', 50, 2, 9.5, 'c','" + String.format(
json, 50) + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id4', 75, 2, 9.5, 'd','" + String.format(
json, 75) + "')");

// Verify that index table includes only the rows with A > 50
selectSql = "SELECT * from " + indexTableName;
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(75, rs.getInt(1));
assertTrue(rs.next());
assertEquals(100, rs.getInt(1));
assertFalse(rs.next());

// Overwrite an existing row that satisfies the index WHERE clause
// such that the new version of the row does not satisfy the index where clause
// anymore. This should result in deleting the index row.
String dml =
"UPSERT INTO " + dataTableName + " values ('id2', 0, 2, 9.5, 'd', JSON_MODIFY(jsoncol, '$.info.age', '0')) ";
conn.createStatement().execute(dml);
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(75, rs.getInt(1));
assertFalse(rs.next());

// Retrieve the updated row from the data table and verify that the index table is not used
selectSql =
"SELECT ID from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 0";
rs = conn.createStatement().executeQuery(selectSql);
assertPlan((PhoenixResultSet) rs, "", dataTableName);
assertTrue(rs.next());
assertEquals("id2", rs.getString(1));

// Test index verification and repair by IndexTool
verifyIndex(dataTableName, indexTableName);

try (Connection newConn = DriverManager.getConnection(getUrl())) {
PTable indexTable = PhoenixRuntime.getTableNoCache(newConn, indexTableName);
assertTrue(StringUtils.deleteWhitespace(indexTable.getIndexWhere())
.equals("CAST(TO_NUMBER(JSON_VALUE(JSONCOL,'$.info.age'))ASINTEGER)>50"));
}
}
}

@Test
public void testPartialIndexWithJsonExists() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
String dataTableName = generateUniqueName();
conn.createStatement().execute("create table " + dataTableName +
" (id varchar not null primary key, " +
"A integer, B integer, C double, D varchar, jsoncol json)" +
(salted ? " SALT_BUCKETS=4" : ""));
String indexTableName = generateUniqueName();
String jsonWithPathExists = "{\"info\":{\"address\":{\"exists\":true}}}";
String jsonWithoutPathExists = "{\"info\":{\"age\": 25 }}";
// Add rows to the data table before creating a partial index to test that the index
// will be built correctly by IndexTool
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id1', 70, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id2', 100, 'b','" + jsonWithoutPathExists + "')");
conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") +
(local ? "LOCAL " : " ") + "INDEX " + indexTableName + " on " + dataTableName + " (A) " +
(uncovered ? "" : "INCLUDE (B, C, D)") + " WHERE JSON_EXISTS(JSONCOL, '$.info.address.exists') ASYNC");
IndexToolIT.runIndexTool(false, null, dataTableName, indexTableName);

String selectSql =
"SELECT " + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") +
" A, D from " + dataTableName + " WHERE A > 60 AND JSON_EXISTS(jsoncol, '$.info.address.exists')";
ResultSet rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is used
assertPlan((PhoenixResultSet) rs, "", indexTableName);
assertTrue(rs.next());
assertEquals(70, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertFalse(rs.next());

// Add more rows to test the index write path
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id3', 20, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id4', 90, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id5', 150, 'b','" + jsonWithoutPathExists + "')");

// Verify that index table includes only the rows where jsonPath Exists
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(70, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertTrue(rs.next());
assertEquals(90, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertFalse(rs.next());

rs = conn.createStatement().executeQuery("SELECT Count(*) from " + dataTableName);
// Verify that the index table is not used
assertPlan((PhoenixResultSet) rs, "", dataTableName);
assertTrue(rs.next());
assertEquals(5, rs.getInt(1));

// Overwrite an existing row that satisfies the index WHERE clause such that
// the new version of the row does not satisfy the index where clause anymore. This
// should result in deleting the index row.
conn.createStatement().execute(
"upsert into " + dataTableName + " (ID, B, jsoncol) values ('id4', null, '" + jsonWithoutPathExists + "')");
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(70, rs.getInt(1));
assertEquals("a", rs.getString(2));
assertFalse(rs.next());

// Test index verification and repair by IndexTool
verifyIndex(dataTableName, indexTableName);

try (Connection newConn = DriverManager.getConnection(getUrl())) {
PTable indexTable = PhoenixRuntime.getTableNoCache(newConn, indexTableName);
assertTrue(StringUtils.deleteWhitespace(indexTable.getIndexWhere())
.equals("JSON_EXISTS(JSONCOL,'$.info.address.exists')"));
}
}
}
}
37 changes: 1 addition & 36 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,14 @@
@Category(ParallelStatsDisabledTest.class)
public class JsonFunctionsIT extends ParallelStatsDisabledIT {
public static String BASIC_JSON = "json/json_functions_basic.json";
public static String FUNCTIONS_TEST_JSON = "json/json_functions_tests.json";
public static String DATA_TYPES_JSON = "json/json_datatypes.json";
String basicJson = "";
String dataTypesJson = "";
String functionsJson = "";

@Before
public void setup() throws IOException {
basicJson = getJsonString(BASIC_JSON, "$[0]");
dataTypesJson = getJsonString(DATA_TYPES_JSON);
functionsJson = getJsonString(FUNCTIONS_TEST_JSON);
}

@Test
Expand Down Expand Up @@ -158,39 +155,6 @@ public void testSimpleJsonModify() throws Exception {
}
}

@Test
public void testSimpleJsonValue2() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
String tableName = generateUniqueName();
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
String ddl = "create table if not exists " + tableName + " (pk integer primary key, col integer, jsoncol json)";
conn.createStatement().execute(ddl);
PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES (?,?,?)");
stmt.setInt(1, 1);
stmt.setInt(2, 2);
stmt.setString(3, functionsJson);
stmt.execute();
conn.commit();
ResultSet rs = conn.createStatement().executeQuery("SELECT JSON_VALUE(JSONCOL,'$.test'), " +
"JSON_VALUE(JSONCOL, '$.testCnt'), " +
"JSON_VALUE(JSONCOL, '$.infoTop[5].info.address.state')," +
"JSON_VALUE(JSONCOL, '$.infoTop[4].tags[1]'), " +
"JSON_QUERY(JSONCOL, '$.infoTop'), " +
"JSON_QUERY(JSONCOL, '$.infoTop[5].info'), " +
"JSON_QUERY(JSONCOL, '$.infoTop[5].friends') " +
"FROM " + tableName + " WHERE JSON_VALUE(JSONCOL, '$.test')='test1'");
assertTrue(rs.next());
assertEquals("test1", rs.getString(1));
assertEquals("SomeCnt1", rs.getString(2));
assertEquals("North Dakota", rs.getString(3));
assertEquals("sint", rs.getString(4));
compareJson(rs.getString(5), functionsJson, "$.infoTop");
compareJson(rs.getString(6), functionsJson, "$.infoTop[5].info");
compareJson(rs.getString(7), functionsJson, "$.infoTop[5].friends");
}
}

private void compareJson(String result, String json, String path) throws JsonProcessingException {
Configuration conf = Configuration.builder().jsonProvider(new GsonJsonProvider()).build();
Object read = JsonPath.using(conf).parse(json).read(path);
Expand Down Expand Up @@ -445,6 +409,7 @@ private void checkInvalidJsonIndexExpression(Properties props, String tableName,
private static String getJsonString(String jsonFilePath) throws IOException {
return getJsonString(jsonFilePath, "$");
}

private static String getJsonString(String jsonFilePath, String jsonPath) throws IOException {
URL fileUrl = JsonFunctionsIT.class.getClassLoader().getResource(jsonFilePath);
String json = FileUtils.readFileToString(new File(fileUrl.getFile()));
Expand Down

0 comments on commit fd71272

Please sign in to comment.