From 5a567ed6abf3f3ba5e0beb33a50d793162c06f28 Mon Sep 17 00:00:00 2001 From: Niel Markwick Date: Tue, 14 May 2024 20:24:02 +0200 Subject: [PATCH] feat: Support for diffs on Create Search Index (#104) --- .../spannerddl/diff/AstTreeUtils.java | 14 + .../spannerddl/diff/DatbaseDefinition.java | 10 + .../solutions/spannerddl/diff/DdlDiff.java | 30 ++ .../diff/SchemaUpdateStatements.java | 41 +++ .../ASTalter_search_index_statement.java | 28 ++ .../spannerddl/parser/ASTcolumn_type.java | 1 + .../parser/ASTcreate_index_where_clause.java | 55 ++++ .../ASTcreate_search_index_statement.java | 276 ++++++++++++++++++ .../parser/ASTgeneration_clause.java | 9 +- .../spannerddl/parser/ASTkey_part.java | 15 +- .../solutions/spannerddl/parser/ASTname.java | 5 + .../spannerddl/parser/ASTorder_by_key.java | 50 ++++ .../spannerddl/parser/ASTpartition_key.java | 50 ++++ .../solutions/spannerddl/parser/ASTpath.java | 5 + .../spannerddl/parser/ASTsearch_index.java | 28 ++ .../solutions/spannerddl/parser/ASTtable.java | 5 + .../spannerddl/parser/ASTtoken_key_list.java | 51 ++++ src/main/jjtree-sources/ddl_parser.jjt | 69 ++++- .../spannerddl/diff/DdlDiffFromFilesTest.java | 4 +- src/test/resources/ddlParserUnsupported.txt | 8 + src/test/resources/ddlParserValidation.txt | 17 ++ src/test/resources/expectedDdlDiff.txt | 21 +- src/test/resources/newDdl.txt | 20 ++ src/test/resources/originalDdl.txt | 20 ++ 24 files changed, 821 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/diff/SchemaUpdateStatements.java create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_search_index_statement.java create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_index_where_clause.java create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_search_index_statement.java create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTorder_by_key.java create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpartition_key.java create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTsearch_index.java create mode 100644 src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtoken_key_list.java diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/diff/AstTreeUtils.java b/src/main/java/com/google/cloud/solutions/spannerddl/diff/AstTreeUtils.java index e61169b..aeed51e 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/diff/AstTreeUtils.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/diff/AstTreeUtils.java @@ -139,4 +139,18 @@ public static void validateChildrenClasses( } } } + + /** Verifies that each child is the specified class. */ + public static void validateChildrenClass( + Node[] children, Class validChildClass) { + for (Node child : children) { + if (validChildClass != child.getClass()) { + throw new IllegalArgumentException( + "Unexpected child node " + + child.getClass().getSimpleName() + + " in parent " + + child.jjtGetParent().getClass().getSimpleName()); + } + } + } } diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/diff/DatbaseDefinition.java b/src/main/java/com/google/cloud/solutions/spannerddl/diff/DatbaseDefinition.java index d1e2673..bb45676 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/diff/DatbaseDefinition.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/diff/DatbaseDefinition.java @@ -23,6 +23,7 @@ import com.google.cloud.solutions.spannerddl.parser.ASTcheck_constraint; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_change_stream_statement; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_index_statement; +import com.google.cloud.solutions.spannerddl.parser.ASTcreate_search_index_statement; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_table_statement; import com.google.cloud.solutions.spannerddl.parser.ASTddl_statement; import com.google.cloud.solutions.spannerddl.parser.ASTforeign_key; @@ -49,6 +50,7 @@ static DatbaseDefinition create(List statements) { // Use LinkedHashMap to preserve creation order in original DDL. LinkedHashMap tablesInCreationOrder = new LinkedHashMap<>(); LinkedHashMap indexes = new LinkedHashMap<>(); + LinkedHashMap searchIndexes = new LinkedHashMap<>(); LinkedHashMap constraints = new LinkedHashMap<>(); LinkedHashMap ttls = new LinkedHashMap<>(); LinkedHashMap changeStreams = new LinkedHashMap<>(); @@ -76,6 +78,11 @@ static DatbaseDefinition create(List statements) { createTable.getRowDeletionPolicyClause(); rowDeletionPolicyClause.ifPresent(rdp -> ttls.put(createTable.getTableName(), rdp)); break; + case DdlParserTreeConstants.JJTCREATE_SEARCH_INDEX_STATEMENT: + searchIndexes.put( + ((ASTcreate_search_index_statement) statement).getName(), + (ASTcreate_search_index_statement) statement); + break; case DdlParserTreeConstants.JJTCREATE_INDEX_STATEMENT: indexes.put( ((ASTcreate_index_statement) statement).getIndexName(), @@ -118,6 +125,7 @@ static DatbaseDefinition create(List statements) { } return new AutoValue_DatbaseDefinition( ImmutableMap.copyOf(tablesInCreationOrder), + ImmutableMap.copyOf(searchIndexes), ImmutableMap.copyOf(indexes), ImmutableMap.copyOf(constraints), ImmutableMap.copyOf(ttls), @@ -127,6 +135,8 @@ static DatbaseDefinition create(List statements) { abstract ImmutableMap tablesInCreationOrder(); + abstract ImmutableMap searchIndexes(); + abstract ImmutableMap indexes(); abstract ImmutableMap constraints(); diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java b/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java index c40d23b..5865425 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java @@ -27,6 +27,7 @@ import com.google.cloud.solutions.spannerddl.parser.ASTcolumn_type; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_change_stream_statement; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_index_statement; +import com.google.cloud.solutions.spannerddl.parser.ASTcreate_search_index_statement; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_table_statement; import com.google.cloud.solutions.spannerddl.parser.ASTddl_statement; import com.google.cloud.solutions.spannerddl.parser.ASTforeign_key; @@ -96,6 +97,7 @@ public class DdlDiff { private final MapDifference ttlDifferences; private final MapDifference alterDatabaseOptionsDifferences; private final MapDifference changeStreamDifferences; + private final MapDifference searchIndexDifferences; private final String databaseName; // for alter Database private DdlDiff(DatbaseDefinition originalDb, DatbaseDefinition newDb, String databaseName) @@ -113,6 +115,8 @@ private DdlDiff(DatbaseDefinition originalDb, DatbaseDefinition newDb, String da Maps.difference(originalDb.alterDatabaseOptions(), newDb.alterDatabaseOptions()); this.changeStreamDifferences = Maps.difference(originalDb.changeStreams(), newDb.changeStreams()); + this.searchIndexDifferences = + Maps.difference(originalDb.searchIndexes(), newDb.searchIndexes()); if (!alterDatabaseOptionsDifferences.areEqual() && Strings.isNullOrEmpty(databaseName)) { // should never happen, but... @@ -182,6 +186,14 @@ public List generateDifferenceStatements(Map options) } } + // drop deleted search indexes. + if (options.get(ALLOW_DROP_STATEMENTS_OPT)) { + for (String searchIndexName : searchIndexDifferences.entriesOnlyOnLeft().keySet()) { + LOG.info("Dropping deleted search index: {}", searchIndexName); + output.add("DROP SEARCH INDEX " + searchIndexName); + } + } + // Drop modified indexes that need to be re-created... for (ValueDifference difference : indexDifferences.entriesDiffering().values()) { @@ -215,6 +227,12 @@ public List generateDifferenceStatements(Map options) output.add("ALTER TABLE " + tableName + " DROP ROW DELETION POLICY"); } + // For each changed search index, apply the drop column statements + SchemaUpdateStatements searchIndexUpdateStatements = + ASTcreate_search_index_statement.generateAlterStatementsFor( + searchIndexDifferences.entriesDiffering(), options.get(ALLOW_DROP_STATEMENTS_OPT)); + output.addAll(searchIndexUpdateStatements.dropStatements()); + if (options.get(ALLOW_DROP_STATEMENTS_OPT)) { // Drop tables that have been deleted -- need to do it in reverse creation order. List reverseOrderedTableNames = @@ -370,6 +388,17 @@ public List generateDifferenceStatements(Map options) } } + for (ASTcreate_search_index_statement searchIndex : + searchIndexDifferences.entriesOnlyOnRight().values()) { + LOG.info("Creating new search index: {}", searchIndex.getName()); + output.add(searchIndex.toString()); + } + + // For each changed search index, apply the add column statements + output.addAll(searchIndexUpdateStatements.createStatements()); + + // Add all new search indexes + return output.build(); } @@ -722,6 +751,7 @@ static List parseDdl(String original) throws DdlDiffException case DdlParserTreeConstants.JJTCREATE_INDEX_STATEMENT: case DdlParserTreeConstants.JJTALTER_DATABASE_STATEMENT: case DdlParserTreeConstants.JJTCREATE_CHANGE_STREAM_STATEMENT: + case DdlParserTreeConstants.JJTCREATE_SEARCH_INDEX_STATEMENT: // no-op break; default: diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/diff/SchemaUpdateStatements.java b/src/main/java/com/google/cloud/solutions/spannerddl/diff/SchemaUpdateStatements.java new file mode 100644 index 0000000..8405dc1 --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/diff/SchemaUpdateStatements.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.solutions.spannerddl.diff; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import java.util.List; + +/** Simple container class to hold a set of drop, modify and create statements. */ +@AutoValue +public abstract class SchemaUpdateStatements { + + /** Creates an instance storing the specified statements. */ + public static SchemaUpdateStatements create( + List dropStatements, List modifyStatements, List createStatements) { + return new AutoValue_SchemaUpdateStatements( + ImmutableList.copyOf(dropStatements), + ImmutableList.copyOf(modifyStatements), + ImmutableList.copyOf(createStatements)); + } + + public abstract ImmutableList dropStatements(); + + public abstract ImmutableList modifyStatements(); + + public abstract ImmutableList createStatements(); +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_search_index_statement.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_search_index_statement.java new file mode 100644 index 0000000..02ea1fe --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTalter_search_index_statement.java @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.solutions.spannerddl.parser; + +public class ASTalter_search_index_statement extends SimpleNode { + public ASTalter_search_index_statement(int id) { + super(id); + throw new UnsupportedOperationException("Not Implemented"); + } + + public ASTalter_search_index_statement(DdlParser p, int id) { + super(p, id); + throw new UnsupportedOperationException("Not Implemented"); + } +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumn_type.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumn_type.java index 7a2ba00..7b8fb9e 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumn_type.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcolumn_type.java @@ -55,6 +55,7 @@ public String toString() { case "DATE": case "NUMERIC": case "JSON": + case "TOKENLIST": return typeName; case "STRING": case "BYTES": diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_index_where_clause.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_index_where_clause.java new file mode 100644 index 0000000..9fd277c --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_index_where_clause.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.solutions.spannerddl.parser; + +import static com.google.cloud.solutions.spannerddl.diff.AstTreeUtils.validateChildrenClass; + +import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils; +import com.google.common.base.Joiner; +import java.util.List; +import java.util.stream.Collectors; + +public class ASTcreate_index_where_clause extends SimpleNode { + public ASTcreate_index_where_clause(int id) { + super(id); + } + + public ASTcreate_index_where_clause(DdlParser p, int id) { + super(p, id); + } + + /** + * void create_index_where_clause() : {} { path() ( path() + * )* } + */ + @Override + public String toString() { + validateChildrenClass(children, ASTpath.class); + + // convert paths object to "pathName IS NOT NULL" to make joining easier. + List paths = + AstTreeUtils.getChildrenAssertType(children, ASTpath.class).stream() + .map((ASTpath path) -> path.toString() + " IS NOT NULL") + .collect(Collectors.toList()); + + return "WHERE " + Joiner.on(" AND ").join(paths); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_search_index_statement.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_search_index_statement.java new file mode 100644 index 0000000..8e10e35 --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_search_index_statement.java @@ -0,0 +1,276 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.solutions.spannerddl.parser; + +import static com.google.cloud.solutions.spannerddl.diff.AstTreeUtils.getChildByType; +import static com.google.cloud.solutions.spannerddl.diff.AstTreeUtils.getOptionalChildByType; + +import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils; +import com.google.cloud.solutions.spannerddl.diff.DdlDiffException; +import com.google.cloud.solutions.spannerddl.diff.SchemaUpdateStatements; +import com.google.common.base.Joiner; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MapDifference; +import com.google.common.collect.MapDifference.ValueDifference; +import com.google.common.collect.Maps; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ASTcreate_search_index_statement extends SimpleNode + implements Comparable { + + private static final Logger LOG = LoggerFactory.getLogger(ASTcreate_search_index_statement.class); + + public ASTcreate_search_index_statement(int id) { + super(id); + } + + public ASTcreate_search_index_statement(DdlParser p, int id) { + super(p, id); + } + + public String getName() { + return AstTreeUtils.tokensToString(getChildByType(children, ASTname.class)); + } + + private void validateChildren() { + AstTreeUtils.validateChildrenClasses( + children, + ImmutableSet.of( + ASTname.class, + ASTtable.class, + ASTtoken_key_list.class, + ASTstored_column_list.class, + ASTpartition_key.class, + ASTorder_by_key.class, + ASTcreate_index_where_clause.class, + ASTindex_interleave_clause.class, + ASToptions_clause.class)); + } + + /** Create string version, optionally including the IF NOT EXISTS clause */ + @Override + public String toString() { + validateChildren(); + ASTindex_interleave_clause interleave = + getOptionalChildByType(children, ASTindex_interleave_clause.class); + + return Joiner.on(" ") + .skipNulls() + .join( + "CREATE SEARCH INDEX", + getName(), + "ON", + getChildByType(children, ASTtable.class), + getChildByType(children, ASTtoken_key_list.class), + getOptionalChildByType(children, ASTstored_column_list.class), + getOptionalChildByType(children, ASTpartition_key.class), + getOptionalChildByType(children, ASTorder_by_key.class), + getOptionalChildByType(children, ASTcreate_index_where_clause.class), + (interleave != null ? "," : null), + interleave, + getOptionalChildByType(children, ASToptions_clause.class)); + } + + @Override + public int compareTo(ASTcreate_search_index_statement other) { + return this.getName().compareTo(other.getName()); + } + + @Override + public boolean equals(Object other) { + if (other instanceof ASTcreate_index_statement) { + return this.toString().equals(other.toString()); + } + return false; + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + + public static SchemaUpdateStatements generateAlterStatementsFor( + Map> searchIndexDifferences, + boolean allowDropColumnStatements) + throws DdlDiffException { + final ImmutableList.Builder dropStatements = ImmutableList.builder(); + final ImmutableList.Builder createStatements = ImmutableList.builder(); + + for (ValueDifference diff : searchIndexDifferences.values()) { + diff.leftValue() + .generateAlterStatementsFor( + diff.rightValue(), dropStatements, createStatements, allowDropColumnStatements); + } + return SchemaUpdateStatements.create( + dropStatements.build(), List.of(), createStatements.build()); + } + + private void generateAlterStatementsFor( + ASTcreate_search_index_statement other, + ImmutableList.Builder dropStatements, + ImmutableList.Builder createStatements, + boolean allowDropColumnStatements) + throws DdlDiffException { + // for simplicity/clarity + final ASTcreate_search_index_statement original = this; + + // Validate possible diffs + if (!original.getName().equals(other.getName())) { + throw new DdlDiffException( + "CREATE SEARCH INDEX name mismatch: " + original.getName() + " != " + other.getName()); + } + + if (!Objects.equal( + getOptionalChildByType(original.children, ASTpartition_key.class), + getOptionalChildByType(other.children, ASTpartition_key.class))) { + throw new DdlDiffException( + "Cannot generate diff for CREATE SEARCH INDEX: " + + original.getName() + + " PARTITION BY clause changed"); + } + + if (!Objects.equal( + getOptionalChildByType(original.children, ASTorder_by_key.class), + getOptionalChildByType(other.children, ASTorder_by_key.class))) { + throw new DdlDiffException( + "Cannot generate diff for CREATE SEARCH INDEX: " + + original.getName() + + " ORDER BY clause changed"); + } + + if (!Objects.equal( + getOptionalChildByType(original.children, ASTcreate_index_where_clause.class), + getOptionalChildByType(other.children, ASTcreate_index_where_clause.class))) { + throw new DdlDiffException( + "Cannot generate diff for CREATE SEARCH INDEX: " + + original.getName() + + " WHERE clause changed"); + } + + if (!Objects.equal( + getOptionalChildByType(original.children, ASTindex_interleave_clause.class), + getOptionalChildByType(other.children, ASTindex_interleave_clause.class))) { + throw new DdlDiffException( + "Cannot generate diff for CREATE SEARCH INDEX: " + + original.getName() + + " INTERLEAVE IN clause changed"); + } + + if (!Objects.equal( + getOptionalChildByType(original.children, ASToptions_clause.class), + getOptionalChildByType(other.children, ASToptions_clause.class))) { + throw new DdlDiffException( + "Cannot generate diff for CREATE SEARCH INDEX: " + + original.getName() + + " OPTIONS clause changed"); + } + + // Look for differences in tokenKeyList + // Easiest is to use Maps.difference, but first we need some maps, and we need to preserve order + // so convert the keyParts to String, and then add to a LinkedHashMap. + Map originalKeyParts = + getChildByType(original.children, ASTtoken_key_list.class).getKeyParts().stream() + .collect( + Collectors.toMap( + ASTkey_part::toString, Function.identity(), (x, y) -> y, LinkedHashMap::new)); + Map newKeyParts = + getChildByType(other.children, ASTtoken_key_list.class).getKeyParts().stream() + .collect( + Collectors.toMap( + ASTkey_part::toString, Function.identity(), (x, y) -> y, LinkedHashMap::new)); + MapDifference keyPartsDiff = + Maps.difference(originalKeyParts, newKeyParts); + + // Look for differences in storedColumnList + // Easiest is to use Maps.difference, but first we need some maps, and we need to preserve order + // so convert the keyParts to String, and then add to a LinkedHashMap. + Map originalStoredColumns = + getChildByType(original.children, ASTstored_column_list.class).getStoredColumns().stream() + .collect( + Collectors.toMap( + ASTstored_column::toString, + Function.identity(), + (x, y) -> y, + LinkedHashMap::new)); + Map newStoredColumns = + getChildByType(other.children, ASTstored_column_list.class).getStoredColumns().stream() + .collect( + Collectors.toMap( + ASTstored_column::toString, + Function.identity(), + (x, y) -> y, + LinkedHashMap::new)); + MapDifference storedColDiff = + Maps.difference(originalStoredColumns, newStoredColumns); + + if (allowDropColumnStatements) { + for (ASTkey_part droppedToken : keyPartsDiff.entriesOnlyOnLeft().values()) { + LOG.info( + "Dropping token colum {} for search index: {}", + droppedToken.getKeyPath(), + original.getName()); + + dropStatements.add( + "ALTER SEARCH INDEX " + + original.getName() + + " DROP COLUMN " + + droppedToken.getKeyPath()); + } + } + + for (ASTstored_column droppedStoredCol : storedColDiff.entriesOnlyOnLeft().values()) { + LOG.info( + "Dropping stored colum {} for search index: {}", + droppedStoredCol.toString(), + original.getName()); + + dropStatements.add( + "ALTER SEARCH INDEX " + + original.getName() + + " DROP STORED COLUMN " + + droppedStoredCol.toString()); + } + + for (ASTkey_part newToken : keyPartsDiff.entriesOnlyOnRight().values()) { + LOG.info( + "Adding token colum {} for search index: {}", newToken.getKeyPath(), original.getName()); + + createStatements.add( + "ALTER SEARCH INDEX " + original.getName() + " ADD COLUMN " + newToken.toString()); + } + + for (ASTstored_column newStoredCol : storedColDiff.entriesOnlyOnRight().values()) { + LOG.info( + "Adding stored colum {} for search index: {}", + newStoredCol.toString(), + original.getName()); + + createStatements.add( + "ALTER SEARCH INDEX " + + original.getName() + + " ADD STORED COLUMN " + + newStoredCol.toString()); + } + } +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTgeneration_clause.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTgeneration_clause.java index a35d7e8..51a3e34 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTgeneration_clause.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTgeneration_clause.java @@ -28,6 +28,13 @@ public ASTgeneration_clause(DdlParser p, int id) { @Override public String toString() { final ASTexpression exp = (ASTexpression) children[0]; - return "AS ( " + exp.toString() + " ) STORED"; + final String storedOpt = + children.length > 1 && children[1].getClass() == ASTstored.class ? " STORED" : ""; + return "AS ( " + exp.toString() + " )" + storedOpt; + } + + @Override + public int hashCode() { + return this.toString().hashCode(); } } diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTkey_part.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTkey_part.java index 4dd1458..568aa1b 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTkey_part.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTkey_part.java @@ -29,19 +29,24 @@ public ASTkey_part(DdlParser p, int id) { super(p, id); } + public String getKeyPath() { + if (children == null) { + return jjtGetFirstToken().toString(); + } + return ((ASTpath) children[0]).toString(); + } + @Override public String toString() { if (children == null) { return jjtGetFirstToken().toString(); } - if (children.length == 1) { - return ((ASTpath) children[0]).toString() + " ASC"; // key name without direction ; + if (children.length == 1) { + return getKeyPath() + " ASC"; // key name without direction ; } else { // key name and ASC/DESC - return ((ASTpath) children[0]).toString() - + " " - + children[1].toString().toUpperCase(Locale.ROOT); + return getKeyPath() + " " + children[1].toString().toUpperCase(Locale.ROOT); } } } diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTname.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTname.java index 25f7bc8..f3e97e7 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTname.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTname.java @@ -33,4 +33,9 @@ public ASTname(DdlParser p, int id) { public String toString() { return AstTreeUtils.tokensToString(this, false); } + + @Override + public int hashCode() { + return toString().hashCode(); + } } diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTorder_by_key.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTorder_by_key.java new file mode 100644 index 0000000..5fad4ff --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTorder_by_key.java @@ -0,0 +1,50 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.solutions.spannerddl.parser; + +import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils; +import com.google.common.base.Joiner; + +public class ASTorder_by_key extends SimpleNode { + public ASTorder_by_key(int id) { + super(id); + } + + public ASTorder_by_key(DdlParser p, int id) { + super(p, id); + } + + private void validateChildren() { + AstTreeUtils.validateChildrenClass(children, ASTkey_part.class); + } + + @Override + public String toString() { + validateChildren(); + return "ORDER BY " + + Joiner.on(", ").join(AstTreeUtils.getChildrenAssertType(children, ASTkey_part.class)); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + + @Override + public boolean equals(Object obj) { + return toString().equals(obj.toString()); + } +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpartition_key.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpartition_key.java new file mode 100644 index 0000000..30815c2 --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpartition_key.java @@ -0,0 +1,50 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.solutions.spannerddl.parser; + +import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils; +import com.google.common.base.Joiner; + +public class ASTpartition_key extends SimpleNode { + public ASTpartition_key(int id) { + super(id); + } + + public ASTpartition_key(DdlParser p, int id) { + super(p, id); + } + + private void validateChildren() { + AstTreeUtils.validateChildrenClass(children, ASTkey_part.class); + } + + @Override + public String toString() { + validateChildren(); + return "PARTITION BY " + + Joiner.on(", ").join(AstTreeUtils.getChildrenAssertType(children, ASTkey_part.class)); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + + @Override + public boolean equals(Object obj) { + return toString().equals(obj.toString()); + } +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpath.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpath.java index 67e5bf2..7525e1b 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpath.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTpath.java @@ -33,4 +33,9 @@ public ASTpath(DdlParser p, int id) { public String toString() { return AstTreeUtils.tokensToString(this, false); } + + @Override + public int hashCode() { + return toString().hashCode(); + } } diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTsearch_index.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTsearch_index.java new file mode 100644 index 0000000..f7a2be7 --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTsearch_index.java @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.solutions.spannerddl.parser; + +public class ASTsearch_index extends SimpleNode { + public ASTsearch_index(int id) { + super(id); + throw new UnsupportedOperationException("Not Implemented"); + } + + public ASTsearch_index(DdlParser p, int id) { + super(p, id); + throw new UnsupportedOperationException("Not Implemented"); + } +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtable.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtable.java index d215cec..8a7354c 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtable.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtable.java @@ -33,4 +33,9 @@ public ASTtable(DdlParser p, int id) { public String toString() { return AstTreeUtils.tokensToString(this); } + + @Override + public int hashCode() { + return toString().hashCode(); + } } diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtoken_key_list.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtoken_key_list.java new file mode 100644 index 0000000..7b5fa54 --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTtoken_key_list.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.solutions.spannerddl.parser; + +import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; +import java.util.List; + +public class ASTtoken_key_list extends SimpleNode { + public ASTtoken_key_list(int id) { + super(id); + } + + public ASTtoken_key_list(DdlParser p, int id) { + super(p, id); + } + + private void validateChildren() { + AstTreeUtils.validateChildrenClasses(children, ImmutableSet.of(ASTkey_part.class)); + } + + @Override + public String toString() { + validateChildren(); + return "( " + Joiner.on(", ").join(getKeyParts()) + " )"; + } + + public List getKeyParts() { + return AstTreeUtils.getChildrenAssertType(children, ASTkey_part.class); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } +} diff --git a/src/main/jjtree-sources/ddl_parser.jjt b/src/main/jjtree-sources/ddl_parser.jjt index b95d538..a94ae08 100644 --- a/src/main/jjtree-sources/ddl_parser.jjt +++ b/src/main/jjtree-sources/ddl_parser.jjt @@ -107,7 +107,7 @@ void create_statement() #void : | create_proto_bundle_statement() | create_table_statement() | create_index_statement() - + | create_search_index_statement() | create_or_replace_statement() | create_change_stream_statement() | create_sequence_statement() @@ -169,6 +169,17 @@ void alter_index_statement() : ( identifier() #column_name ) } +void alter_search_index_statement() : +{} +{ + (qualified_identifier() #name) + ( LOOKAHEAD(2) stored_column() #add_stored_column + | LOOKAHEAD(2) identifier() #drop_stored_column_name + | LOOKAHEAD(2) key_part() #add_token_column + | LOOKAHEAD(2) key_part() #drop_token_column + ) +} + void options_clause() : {} { @@ -251,7 +262,7 @@ void column_type() : | "." qualified_identifier() #pgtype | | "<" column_type() ">" - + | | ( "<" struct_fields() ">" | "<>" ) | dotted_path() } @@ -376,6 +387,57 @@ void create_index_statement() : [ LOOKAHEAD(2) "," index_interleave_clause() ] } +void create_search_index_statement() : +{} +{ + qualified_identifier() #name + qualified_identifier() #table + token_key_list() + [ stored_column_list() ] + [ partition_key() ] + [ order_by_key() ] + [ create_index_where_clause() ] + [ LOOKAHEAD(2) "," index_interleave_clause() ] + [ options_clause() ] +} + +void token_key_list() : +{} +{ + "(" + [ key_part() + ("," key_part() )* + ] + ")" +} + +void partition_key() : +{} +{ + key_part() + // The special {} inside the LOOKAHEAD here inserts a little bit of code into + // the generated parser, and makes parsing of the immediately following tokens + // conditional on that expression evaluating to true. Here, we peek ahead 3 + // tokens, and make sure it's not "IN" from the possible following + // INTERLEAVE IN clause. + + // Note for schema-diff-tool. This code has been converted from C++ to java. + ( LOOKAHEAD(3, {(getToken(3) == null || getToken(3).kind != IN)}) + "," key_part() )* +} + +void order_by_key() : +{} +{ + key_part() + // See partition_key() above for a detailed explanation of this + // unusual-looking LOOKAHEAD block. Here, we're avoiding INTERLEAVE IN being + // ambiguous, since INTERLEAVE is a pseudo-reserved word. + + // Note for schema-diff-tool. This code has been converted from C++ to java. + ( LOOKAHEAD(3, {(getToken(3) == null || getToken(3).kind != IN)}) + "," key_part() )* +} void create_index_where_clause() : {} @@ -625,7 +687,7 @@ void drop_statement() : ( ( #table [ if_exists() ] | #index [ if_exists() ] - + | #search_index [ if_exists() ] | #view [ if_exists() ] | #change_stream | #role @@ -646,6 +708,7 @@ void alter_statement() #void : | alter_table_statement() | alter_change_stream_statement() | alter_index_statement() + | alter_search_index_statement() | alter_proto_bundle_statement() | alter_sequence_statement() | alter_model_statement() diff --git a/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java b/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java index be4cfae..720605a 100644 --- a/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java +++ b/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java @@ -81,7 +81,9 @@ public void compareDddTextFiles() throws IOException { // build an expectedResults without any column or table drops. List expectedDiffNoDrops = expectedDiff.stream() - .filter(statement -> !statement.matches(".*DROP (TABLE|COLUMN|CHANGE STREAM).*")) + .filter( + statement -> + !statement.matches(".*DROP (TABLE|COLUMN|CHANGE STREAM|SEARCH INDEX).*")) .collect(Collectors.toList()); // remove any drop indexes from the expectedResults if they do not have an equivalent diff --git a/src/test/resources/ddlParserUnsupported.txt b/src/test/resources/ddlParserUnsupported.txt index 4a5bd44..d1bf7af 100644 --- a/src/test/resources/ddlParserUnsupported.txt +++ b/src/test/resources/ddlParserUnsupported.txt @@ -155,4 +155,12 @@ ALTER PROTO BUNDLE UPDATE (proto.path.name, other.proto.path) ALTER PROTO BUNDLE DELETE (proto.path.name, other.proto.path) +== Test 18a alter search index + +ALTER SEARCH INDEX AlbumsIndex ADD STORED COLUMN test + +== Test 18b alter search index + +ALTER SEARCH INDEX AlbumsIndex ADD COLUMN add_token_column + == diff --git a/src/test/resources/ddlParserValidation.txt b/src/test/resources/ddlParserValidation.txt index e7dcb38..904cfdf 100644 --- a/src/test/resources/ddlParserValidation.txt +++ b/src/test/resources/ddlParserValidation.txt @@ -65,6 +65,7 @@ CREATE TABLE test.test ( jsoncol JSON, pgcolumn PG.SOMETHING, generatedcol STRING(MAX) AS ( sizedstring + strstr ( maxstring, strpos ( maxstring, 'xxx' ), length ( maxstring ) ) + 2.0 ) STORED, + tokenlistCol TOKENLIST AS ( TOKENIZE_FULLTEXT ( maxstring ) ) HIDDEN, CONSTRAINT fk_col_remote FOREIGN KEY ( col1, col2 ) REFERENCES test.other_table ( other_col1, other_col2 ) ON DELETE CASCADE, CONSTRAINT fk_col_remote2 FOREIGN KEY ( col1 ) REFERENCES test.other_table ( other_col1 ) ON DELETE NO ACTION, CONSTRAINT check_some_value CHECK (( length ( sizedstring ) > 100 OR sizedstring = "xxx" ) AND boolcol = TRUE AND intcol > -123.4 AND numericcol < 1.5) @@ -133,4 +134,20 @@ CREATE CHANGE STREAM change_stream_name FOR ALL OPTIONS (retention_period='1d',v CREATE CHANGE STREAM change_stream_name FOR table1, table2 ( ), table3 ( col1, col2 ) OPTIONS (retention_period='7d',value_capture_type='NEW_ROW') +== Test 13a create search index full + +CREATE SEARCH INDEX AlbumsIndex +ON Albums ( AlbumTitle_Tokens ASC, Rating_Tokens ASC ) +STORING ( Genre, albumName ) +PARTITION BY SingerId ASC +ORDER BY ReleaseTimestamp DESC +WHERE Genre IS NOT NULL AND albumName IS NOT NULL , +INTERLEAVE IN Singers +OPTIONS (sort_order_sharding=TRUE) + +== Test 13b create search index Simple + +CREATE SEARCH INDEX AlbumsIndex +ON Albums ( AlbumTitle_Tokens ASC ) + == diff --git a/src/test/resources/expectedDdlDiff.txt b/src/test/resources/expectedDdlDiff.txt index db25ed6..a8e29cb 100644 --- a/src/test/resources/expectedDdlDiff.txt +++ b/src/test/resources/expectedDdlDiff.txt @@ -293,5 +293,24 @@ ALTER INDEX test4 DROP STORED COLUMN col2 ALTER INDEX test4 ADD STORED COLUMN col4 ALTER INDEX test4 ADD STORED COLUMN col5 -== +== TEST 55 Creating new search index - after table creation + +CREATE TABLE test2 ( col1 INT64 ) PRIMARY KEY (col1 ASC) + +CREATE SEARCH INDEX AlbumsIndex ON Albums ( AlbumTitle_Tokens ASC ) + +== TEST 56 Dropping search index - before table dropping + +DROP SEARCH INDEX AlbumsIndex +DROP TABLE test2 +== TEST 57 Changing search index - before and after table changes + +ALTER SEARCH INDEX AlbumsIndex DROP COLUMN col2 +ALTER SEARCH INDEX AlbumsIndex DROP STORED COLUMN scol2 +ALTER TABLE test1 DROP COLUMN col2 +ALTER TABLE test1 ADD COLUMN col3 INT64 +ALTER SEARCH INDEX AlbumsIndex ADD COLUMN col3 ASC +ALTER SEARCH INDEX AlbumsIndex ADD STORED COLUMN scol3 + +== diff --git a/src/test/resources/newDdl.txt b/src/test/resources/newDdl.txt index 74e3fe4..6f32467 100644 --- a/src/test/resources/newDdl.txt +++ b/src/test/resources/newDdl.txt @@ -469,4 +469,24 @@ CREATE UNIQUE NULL_FILTERED INDEX test5 ON test1 ( col1 ASC ); CREATE UNIQUE NULL_FILTERED INDEX test4 ON test1 ( col1 ASC ) STORING ( col3, col5, col4 ); +== TEST 55 Creating new search index - after table creation + +create table test1 ( col1 int64 ) primary key (col1); + +create table test2 ( col1 int64 ) primary key (col1); + +CREATE SEARCH INDEX AlbumsIndex ON Albums ( AlbumTitle_Tokens ); + +== TEST 56 Dropping search index - before table dropping + +create table test1 ( col1 int64 ) primary key (col1); + +== TEST 57 Changing search index - before and after table changes + +create table test1 ( col1 int64, col3 int64 ) primary key (col1); + +CREATE SEARCH INDEX AlbumsIndex +ON Albums (col1, col3) +STORING (scol1, scol3); + == diff --git a/src/test/resources/originalDdl.txt b/src/test/resources/originalDdl.txt index 7ae9df7..768f093 100644 --- a/src/test/resources/originalDdl.txt +++ b/src/test/resources/originalDdl.txt @@ -467,6 +467,26 @@ CREATE UNIQUE NULL_FILTERED INDEX test5 ON test1 ( col1 ASC ) STORING ( col2, co CREATE UNIQUE NULL_FILTERED INDEX test4 ON test1 ( col1 ASC ) STORING ( col2, col3 ); +== TEST 55 Creating new search index - after table creation + +create table test1 ( col1 int64 ) primary key (col1); + +== TEST 56 Dropping search index - before table dropping + +create table test1 ( col1 int64 ) primary key (col1); + +create table test2 ( col1 int64 ) primary key (col1); + +CREATE SEARCH INDEX AlbumsIndex ON Albums ( AlbumTitle_Tokens ASC, Rating_Tokens ASC ) + +== TEST 57 Changing search index - before and after table changes + +create table test1 ( col1 int64, col2 int64 ) primary key (col1); + +CREATE SEARCH INDEX AlbumsIndex +ON Albums (col1, col2) +STORING (scol1, scol2); + ==