Skip to content

Commit

Permalink
Removing json file create for partial Index test and some clean up in…
Browse files Browse the repository at this point in the history 
… JSON IT too.
  • Loading branch information
@ranganathg
ranganathg committed Dec 20, 2023
1 parent b54205a commit ce5045d
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 552 deletions.
77 changes: 42 additions & 35 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,15 @@
import java.util.Map;
import java.util.Properties;

import static org.apache.phoenix.end2end.json.JsonFunctionsIT.getJsonString;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.*;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_OLD_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REPAIR_EXTRA_UNVERIFIED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REPAIR_EXTRA_VERIFIED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.*;

Expand Down Expand Up @@ -779,43 +786,45 @@ public void testPartialIndexWithJson() throws Exception {
" (id varchar not null primary key, " +
"A integer, B integer, C double, D varchar, jsoncol json)");
String indexTableName = generateUniqueName();
String partialIndexJson = "json/json_partialindex_tests.json";
String json = "{\"info\":{\"age\": %s }}";
// Add rows to the data table before creating a partial index to test that the index
// will be built correctly by IndexTool
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id1', 25, 2, 3.14, 'a','" + getJsonString(
partialIndexJson, "$[0]") + "')");
"upsert into " + dataTableName + " values ('id1', 25, 2, 3.14, 'a','" +
String.format(json, 25) + "')");

conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id2', 100, 'b','" + getJsonString(
partialIndexJson, "$[3]") + "')");
"upsert into " + dataTableName + " (id, A, D, jsoncol)" +
" values ('id2', 100, 'b','" + String.format(json, 100) + "')");
conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") +
(local ? "LOCAL " : " ") + "INDEX "
+ indexTableName + " on " + dataTableName + " (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) " + (
uncovered ? "" : "INCLUDE (B, C, D)") + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 50 ASYNC");
(local ? "LOCAL " : " ") + "INDEX " + indexTableName +
" on " + dataTableName + " (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) " +
(uncovered ? "" : "INCLUDE (B, C, D)") + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 50 ASYNC");

IndexToolIT.runIndexTool(false, null, dataTableName, indexTableName);

String selectSql = "SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 60";
String selectSql =
"SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 60";
ResultSet rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is used
assertPlan((PhoenixResultSet) rs, "", indexTableName);
assertTrue(rs.next());
assertEquals("b", rs.getString(1));
assertFalse(rs.next());

selectSql = "SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 50";
selectSql =
"SELECT D from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 50";
rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is not used
assertPlan((PhoenixResultSet) rs, "", dataTableName);

// Add more rows to test the index write path
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id3', 50, 2, 9.5, 'c','" + getJsonString(
partialIndexJson, "$[1]") + "')");
"upsert into " + dataTableName + " values ('id3', 50, 2, 9.5, 'c','" + String.format(
json, 50) + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id4', 75, 2, 9.5, 'd','" + getJsonString(
partialIndexJson, "$[2]") + "')");
"upsert into " + dataTableName + " values ('id4', 75, 2, 9.5, 'd','" + String.format(
json, 75) + "')");

// Verify that index table includes only the rows with A > 50
selectSql = "SELECT * from " + indexTableName;
Expand All @@ -829,15 +838,17 @@ public void testPartialIndexWithJson() throws Exception {
// Overwrite an existing row that satisfies the index WHERE clause
// such that the new version of the row does not satisfy the index where clause
// anymore. This should result in deleting the index row.
String dml = "UPSERT INTO " + dataTableName + " values ('id2', 0, 2, 9.5, 'd', JSON_MODIFY(jsoncol, '$.info.age', '0')) "; //, JSON_MODIFY(jsoncol, '$.info.age', '"0"')
String dml =
"UPSERT INTO " + dataTableName + " values ('id2', 0, 2, 9.5, 'd', JSON_MODIFY(jsoncol, '$.info.age', '0')) ";
conn.createStatement().execute(dml);
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(75, rs.getInt(1));
assertFalse(rs.next());

// Retrieve the updated row from the data table and verify that the index table is not used
selectSql = "SELECT ID from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 0";
selectSql =
"SELECT ID from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 0";
rs = conn.createStatement().executeQuery(selectSql);
assertPlan((PhoenixResultSet) rs, "", dataTableName);
assertTrue(rs.next());
Expand All @@ -862,25 +873,25 @@ public void testPartialIndexWithJsonExists() throws Exception {
String dataTableName = generateUniqueName();
conn.createStatement().execute("create table " + dataTableName +
" (id varchar not null primary key, " +
"A integer, B integer, C double, D varchar, jsoncol json)"
+ (salted ? " SALT_BUCKETS=4" : ""));
"A integer, B integer, C double, D varchar, jsoncol json)" +
(salted ? " SALT_BUCKETS=4" : ""));
String indexTableName = generateUniqueName();
String partialIndexJson = "json/json_partialindex_tests.json";
String jsonWithPathExists = "{\"info\":{\"address\":{\"exists\":true}}}";
String jsonWithoutPathExists = "{\"info\":{\"age\": 25 }}";
// Add rows to the data table before creating a partial index to test that the index
// will be built correctly by IndexTool
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id1', 70, 2, 3.14, 'a','" + getJsonString(
partialIndexJson, "$[0]") + "')");
"upsert into " + dataTableName + " values ('id1', 70, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id2', 100, 'b','" + getJsonString(
partialIndexJson, "$[1]") + "')");
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id2', 100, 'b','" + jsonWithoutPathExists + "')");
conn.createStatement().execute("CREATE " + (uncovered ? "UNCOVERED " : " ") +
(local ? "LOCAL " : " ") + "INDEX " + indexTableName + " on " + dataTableName + " (A) " +
(uncovered ? "" : "INCLUDE (B, C, D)") + " WHERE JSON_EXISTS(JSONCOL, '$.info.address.exists') ASYNC");
IndexToolIT.runIndexTool(false, null, dataTableName, indexTableName);

String selectSql = "SELECT " + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") +
" A, D from " + dataTableName + " WHERE A > 60 AND JSON_EXISTS(jsoncol, '$.info.address.exists')";
String selectSql =
"SELECT " + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") +
" A, D from " + dataTableName + " WHERE A > 60 AND JSON_EXISTS(jsoncol, '$.info.address.exists')";
ResultSet rs = conn.createStatement().executeQuery(selectSql);
// Verify that the index table is used
assertPlan((PhoenixResultSet) rs, "", indexTableName);
Expand All @@ -891,14 +902,11 @@ public void testPartialIndexWithJsonExists() throws Exception {

// Add more rows to test the index write path
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id3', 20, 2, 3.14, 'a','" + getJsonString(
partialIndexJson, "$[2]") + "')");
"upsert into " + dataTableName + " values ('id3', 20, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " values ('id4', 90, 2, 3.14, 'a','" + getJsonString(
partialIndexJson, "$[4]") + "')");
"upsert into " + dataTableName + " values ('id4', 90, 2, 3.14, 'a','" + jsonWithPathExists + "')");
conn.createStatement().execute(
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id5', 150, 'b','" + getJsonString(
partialIndexJson, "$[3]") + "')");
"upsert into " + dataTableName + " (id, A, D, jsoncol) values ('id5', 150, 'b','" + jsonWithoutPathExists + "')");

// Verify that index table includes only the rows where jsonPath Exists
rs = conn.createStatement().executeQuery(selectSql);
Expand All @@ -920,8 +928,7 @@ public void testPartialIndexWithJsonExists() throws Exception {
// the new version of the row does not satisfy the index where clause anymore. This
// should result in deleting the index row.
conn.createStatement().execute(
"upsert into " + dataTableName + " (ID, B, jsoncol) values ('id4', null, '" + getJsonString(
partialIndexJson, "$[3]") + "')");
"upsert into " + dataTableName + " (ID, B, jsoncol) values ('id4', null, '" + jsonWithoutPathExists + "')");
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
assertEquals(70, rs.getInt(1));
Expand Down
40 changes: 2 additions & 38 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,14 @@
@Category(ParallelStatsDisabledTest.class)
public class JsonFunctionsIT extends ParallelStatsDisabledIT {
public static String BASIC_JSON = "json/json_functions_basic.json";
public static String FUNCTIONS_TEST_JSON = "json/json_functions_tests.json";
public static String DATA_TYPES_JSON = "json/json_datatypes.json";
String basicJson = "";
String dataTypesJson = "";
String functionsJson = "";

@Before
public void setup() throws IOException {
basicJson = getJsonString(BASIC_JSON, "$[0]");
dataTypesJson = getJsonString(DATA_TYPES_JSON);
functionsJson = getJsonString(FUNCTIONS_TEST_JSON);
}

@Test
Expand Down Expand Up @@ -158,39 +155,6 @@ public void testSimpleJsonModify() throws Exception {
}
}

@Test
public void testSimpleJsonValue2() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
String tableName = generateUniqueName();
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
String ddl = "create table if not exists " + tableName + " (pk integer primary key, col integer, jsoncol json)";
conn.createStatement().execute(ddl);
PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES (?,?,?)");
stmt.setInt(1, 1);
stmt.setInt(2, 2);
stmt.setString(3, functionsJson);
stmt.execute();
conn.commit();
ResultSet rs = conn.createStatement().executeQuery("SELECT JSON_VALUE(JSONCOL,'$.test'), " +
"JSON_VALUE(JSONCOL, '$.testCnt'), " +
"JSON_VALUE(JSONCOL, '$.infoTop[5].info.address.state')," +
"JSON_VALUE(JSONCOL, '$.infoTop[4].tags[1]'), " +
"JSON_QUERY(JSONCOL, '$.infoTop'), " +
"JSON_QUERY(JSONCOL, '$.infoTop[5].info'), " +
"JSON_QUERY(JSONCOL, '$.infoTop[5].friends') " +
"FROM " + tableName + " WHERE JSON_VALUE(JSONCOL, '$.test')='test1'");
assertTrue(rs.next());
assertEquals("test1", rs.getString(1));
assertEquals("SomeCnt1", rs.getString(2));
assertEquals("North Dakota", rs.getString(3));
assertEquals("sint", rs.getString(4));
compareJson(rs.getString(5), functionsJson, "$.infoTop");
compareJson(rs.getString(6), functionsJson, "$.infoTop[5].info");
compareJson(rs.getString(7), functionsJson, "$.infoTop[5].friends");
}
}

private void compareJson(String result, String json, String path) throws JsonProcessingException {
Configuration conf = Configuration.builder().jsonProvider(new GsonJsonProvider()).build();
Object read = JsonPath.using(conf).parse(json).read(path);
Expand Down Expand Up @@ -442,11 +406,11 @@ private void checkInvalidJsonIndexExpression(Properties props, String tableName,
}
}

public static String getJsonString(String jsonFilePath) throws IOException {
private static String getJsonString(String jsonFilePath) throws IOException {
return getJsonString(jsonFilePath, "$");
}

public static String getJsonString(String jsonFilePath, String jsonPath) throws IOException {
private static String getJsonString(String jsonFilePath, String jsonPath) throws IOException {
URL fileUrl = JsonFunctionsIT.class.getClassLoader().getResource(jsonFilePath);
String json = FileUtils.readFileToString(new File(fileUrl.getFile()));
Configuration conf = Configuration.builder().jsonProvider(new GsonJsonProvider()).build();
Expand Down

0 comments on commit ce5045d

Please sign in to comment.