diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ef2a703f..f6f71e983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## [3.9.4] - 2021-04-08 +### Fixed +* Support for '.' wildcards in database pattern calls. See [#216](https://github.com/HotelsDotCom/waggle-dance/issues/216) + ## [3.9.3] - 2021-03-15 ### Fixed * Null Pointer Exception when database name was null in `get_privilege_set` call. diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMapping.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMapping.java index 0bf8f8e05..f7f2f9825 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMapping.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMapping.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2019 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtils.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtils.java index 1f578e85a..2d3d459dc 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtils.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtils.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2019 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,15 +46,29 @@ static String[] splitPattern(String prefix, String pattern) { while (index >= 0) { String subPatternRegex = subPattern.replaceAll("\\*", ".*"); if (prefix.matches(subPatternRegex)) { - return new String[] { subPattern, pattern.substring(subPattern.length() - 1) }; + if (subPattern.endsWith("*")) { + // * is a multi character match so belongs to prefix and pattern. + return new String[] { subPattern, pattern.substring(subPattern.length() - 1) }; + } + // Dot is a one character x match so can't belong to the pattern anymore. + return new String[] { subPattern, pattern.substring(subPattern.length()) }; } - // Skip last * and find the next sub-pattern - if (subPattern.endsWith("*")) { + // Skip last * or . and find the next sub-pattern + if (subPattern.endsWith("*") || subPattern.endsWith(".")) { subPattern = subPattern.substring(0, subPattern.length() - 1); } - index = subPattern.lastIndexOf("*"); - if (index >= 0) { - subPattern = subPattern.substring(0, index + 1); + int lastStar = subPattern.lastIndexOf('*'); + int lastDot = subPattern.lastIndexOf('.'); + if (lastStar > lastDot) { + index = lastStar; + if (lastStar >= 0) { + subPattern = subPattern.substring(0, index + 1); + } + } else { + index = lastDot; + if (lastDot >= 0) { + subPattern = subPattern.substring(0, subPattern.length() - 1); + } } } return new String[] {}; diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutor.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutor.java index 10a172f89..f519f3132 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutor.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutor.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2019 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,9 +44,12 @@ public List executeRequests( List>> allRequests, long requestTimeout, String errorMessage) { + List allResults = new ArrayList<>(); + if (allRequests.isEmpty()) { + return allResults; + } ExecutorService executorService = Executors.newFixedThreadPool(allRequests.size()); try { - List allResults = new ArrayList<>(); List>> futures = Collections.emptyList(); Iterator>> iterator = allRequests.iterator(); diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java index 9447b9ce5..26d041a33 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2019 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,7 +72,6 @@ protected List getAllDatabases( mappingWithPattern.getValue(), filter); allRequests.add(databasesByPatternRequest); } - List result = getPanopticOperationExecutor() .executeRequests(allRequests, GET_DATABASES_TIMEOUT, "Can't fetch databases by pattern: {}"); return result; diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java index 3cfbd0fa6..8c733df91 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java @@ -219,7 +219,7 @@ public DatabaseMapping databaseMapping(@NotNull String databaseName) throws NoSu @Override public void checkTableAllowed(String databaseName, String tableName, DatabaseMapping mapping) - throws NoSuchObjectException { + throws NoSuchObjectException { String databasePrefix = mapping.getDatabasePrefix(); String transformedDbName = mapping.transformInboundDatabaseName(databaseName); if (!isTableAllowed(databasePrefix, transformedDbName, tableName)) { @@ -232,7 +232,7 @@ public List filterTables(String databaseName, List tableNames, D List allowedTables = new ArrayList<>(); String databasePrefix = mapping.getDatabasePrefix(); String transformedDb = mapping.transformInboundDatabaseName(databaseName); - for (String table: tableNames) { + for (String table : tableNames) { if (isTableAllowed(databasePrefix, transformedDb, table)) { allowedTables.add(table); } @@ -317,8 +317,8 @@ public PanopticOperationHandler getPanopticOperationHandler() { public List getTableMeta(String db_patterns, String tbl_patterns, List tbl_types) { Map databaseMappingsForPattern = databaseMappingsByDbPattern(db_patterns); - BiFunction filter = (tableMeta, mapping) -> - databaseAndTableAllowed(tableMeta.getDbName(), tableMeta.getTableName(), mapping); + BiFunction filter = (tableMeta, mapping) -> databaseAndTableAllowed( + tableMeta.getDbName(), tableMeta.getTableName(), mapping); return super.getTableMeta(tbl_patterns, tbl_types, databaseMappingsForPattern, filter); } diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java index 462ee7e73..1f14e7a9a 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2020 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java index bb65785f1..ad9611c25 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2020 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index 2c9ef53b3..2c62b56c3 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -15,8 +15,6 @@ */ package com.hotels.bdp.waggledance.mapping.model; -import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; - import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; @@ -25,6 +23,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; + import java.util.Arrays; import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl; diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtilsTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtilsTest.java index 28e946385..8cb162465 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtilsTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtilsTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2020 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -77,6 +77,41 @@ public void subPatternMatchesDatabaseAndAllSpecificTables() { assertThat(patternParts[1], is("*base")); } + @Test + public void splitPatternWithDot() { + String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle.dm*"); + assertThat(patternParts[0], is("waggle.")); + assertThat(patternParts[1], is("dm*")); + } + + @Test + public void splitPatternWithDots() { + String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle...dm"); + assertThat(patternParts[0], is("waggle.")); + assertThat(patternParts[1], is("..dm")); + } + + @Test + public void splitPatternWithDotsAndStarEnd() { + String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle...dm*"); + assertThat(patternParts[0], is("waggle.")); + assertThat(patternParts[1], is("..dm*")); + } + + @Test + public void splitPatternWithDotsAndStar() { + String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle.*.dm"); + assertThat(patternParts[0], is("waggle.*")); + assertThat(patternParts[1], is("*.dm")); + } + + @Test + public void splitPatternWithDotsInMiddle() { + String[] patternParts = GrammarUtils.splitPattern(PREFIX, "wa..le_dm"); + assertThat(patternParts[0], is("wa..le_")); + assertThat(patternParts[1], is("dm")); + } + @Test public void matchesWithNullPattern() { Map splits = GrammarUtils.selectMatchingPrefixes(ImmutableSet.of(PREFIX, "other_"), null); @@ -93,6 +128,13 @@ public void matchesWithWildcardPattern() { assertThat(splits.get("other_"), is("*")); } + @Test + public void matchesWithDotWildcardPattern() { + Map splits = GrammarUtils.selectMatchingPrefixes(ImmutableSet.of(PREFIX, "other_"), "other.dm"); + assertThat(splits.size(), is(1)); + assertThat(splits.get("other_"), is("dm")); + } + @Test public void doesNotMatchPatternSimpleDatabaseName() { Map splits = GrammarUtils.selectMatchingPrefixes(ImmutableSet.of(PREFIX, "other_"), "database"); diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutorTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutorTest.java index 3af90614e..bd6e1066c 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutorTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutorTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2020 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -119,6 +119,14 @@ public List call() throws Exception { assertThat(executeRequests.get(1), is("call3")); } + @Test + public void executeEmptyRequests() throws Exception { + PanopticConcurrentOperationExecutor executor = new PanopticConcurrentOperationExecutor(); + List allRequests = Lists.newArrayList(); + List executeRequests = executor.executeRequests(allRequests, REQUEST_TIMEOUT, "error in call: {}"); + assertThat(executeRequests.size(), is(0)); + } + private class DummyRequestCallable implements RequestCallable> { private final String callValue; diff --git a/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java b/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java index e5fd979b5..3194598f3 100644 --- a/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java +++ b/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hive.metastore.api.ResourceType; import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.thrift.TException; import org.junit.After; import org.junit.Before; @@ -1060,4 +1061,32 @@ public void get_privilege_set() throws Exception { assertNotNull(get_privilege_set); } + @Test + public void getTableMeta() throws Exception { + runner = WaggleDanceRunner + .builder(configLocation) + .databaseResolution(DatabaseResolution.PREFIXED) + .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY) + .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE) + .build(); + + runWaggleDance(runner); + HiveMetaStoreClient proxy = getWaggleDanceClient(); + + List tableMeta = proxy + .getTableMeta("waggle_remote_remote_database", "*", Lists.newArrayList("EXTERNAL_TABLE")); + assertThat(tableMeta.size(), is(1)); + assertThat(tableMeta.get(0).getDbName(), is("waggle_remote_remote_database")); + assertThat(tableMeta.get(0).getTableName(), is(REMOTE_TABLE)); + // use wildcards: '.' + tableMeta = proxy.getTableMeta("waggle_remote.remote_database", "*", Lists.newArrayList("EXTERNAL_TABLE")); + assertThat(tableMeta.size(), is(1)); + assertThat(tableMeta.get(0).getDbName(), is("waggle_remote_remote_database")); + assertThat(tableMeta.get(0).getTableName(), is(REMOTE_TABLE)); + tableMeta = proxy.getTableMeta("waggle.remote_remote_database", "*", Lists.newArrayList("EXTERNAL_TABLE")); + assertThat(tableMeta.size(), is(1)); + assertThat(tableMeta.get(0).getDbName(), is("waggle_remote_remote_database")); + assertThat(tableMeta.get(0).getTableName(), is(REMOTE_TABLE)); + } + }