Merge 08e02f3 into b468b15

CHANGELOG.md

@@ -1,3 +1,7 @@
+## [3.9.4] - 2021-04-08
+### Fixed
+* Support for '.' wildcards in database pattern calls. See [#216](https://github.com/HotelsDotCom/waggle-dance/issues/216)
+
 ## [3.9.3] - 2021-03-15
 ### Fixed
 * Null Pointer Exception when database name was null in `get_privilege_set` call.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMapping.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtils.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -46,15 +46,29 @@ static String[] splitPattern(String prefix, String pattern) {
     while (index >= 0) {
       String subPatternRegex = subPattern.replaceAll("\\*", ".*");
       if (prefix.matches(subPatternRegex)) {
-        return new String[] { subPattern, pattern.substring(subPattern.length() - 1) };
+        if (subPattern.endsWith("*")) {
+          // * is a multi character match so belongs to prefix and pattern.
+          return new String[] { subPattern, pattern.substring(subPattern.length() - 1) };
+        }
+        // Dot is a one character x match so can't belong to the pattern anymore.
+        return new String[] { subPattern, pattern.substring(subPattern.length()) };
       }
-      // Skip last * and find the next sub-pattern
-      if (subPattern.endsWith("*")) {
+      // Skip last * or . and find the next sub-pattern
+      if (subPattern.endsWith("*") || subPattern.endsWith(".")) {
         subPattern = subPattern.substring(0, subPattern.length() - 1);
       }
-      index = subPattern.lastIndexOf("*");
-      if (index >= 0) {
-        subPattern = subPattern.substring(0, index + 1);
+      int lastStar = subPattern.lastIndexOf('*');
+      int lastDot = subPattern.lastIndexOf('.');
+      if (lastStar > lastDot) {
+        index = lastStar;
+        if (lastStar >= 0) {
+          subPattern = subPattern.substring(0, index + 1);
+        }
+      } else {
+        index = lastDot;
+        if (lastDot >= 0) {
+          subPattern = subPattern.substring(0, subPattern.length() - 1);
+        }
       }
     }
     return new String[] {};

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutor.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -44,9 +44,12 @@ public <T> List<T> executeRequests(
       List<? extends RequestCallable<List<T>>> allRequests,
       long requestTimeout,
       String errorMessage) {
+    List<T> allResults = new ArrayList<>();
+    if (allRequests.isEmpty()) {
+      return allResults;
+    }
     ExecutorService executorService = Executors.newFixedThreadPool(allRequests.size());
     try {
-      List<T> allResults = new ArrayList<>();
       List<Future<List<T>>> futures = Collections.emptyList();
       Iterator<? extends RequestCallable<List<T>>> iterator = allRequests.iterator();
 

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -72,7 +72,6 @@ protected List<String> getAllDatabases(
           mappingWithPattern.getValue(), filter);
       allRequests.add(databasesByPatternRequest);
     }
-
     List<String> result = getPanopticOperationExecutor()
         .executeRequests(allRequests, GET_DATABASES_TIMEOUT, "Can't fetch databases by pattern: {}");
     return result;

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java

@@ -219,7 +219,7 @@ public DatabaseMapping databaseMapping(@NotNull String databaseName) throws NoSu
 
   @Override
   public void checkTableAllowed(String databaseName, String tableName, DatabaseMapping mapping)
-      throws NoSuchObjectException {
+    throws NoSuchObjectException {
     String databasePrefix = mapping.getDatabasePrefix();
     String transformedDbName = mapping.transformInboundDatabaseName(databaseName);
     if (!isTableAllowed(databasePrefix, transformedDbName, tableName)) {
@@ -232,7 +232,7 @@ public List<String> filterTables(String databaseName, List<String> tableNames, D
     List<String> allowedTables = new ArrayList<>();
     String databasePrefix = mapping.getDatabasePrefix();
     String transformedDb = mapping.transformInboundDatabaseName(databaseName);
-    for (String table: tableNames) {
+    for (String table : tableNames) {
       if (isTableAllowed(databasePrefix, transformedDb, table)) {
         allowedTables.add(table);
       }
@@ -317,8 +317,8 @@ public PanopticOperationHandler getPanopticOperationHandler() {
       public List<TableMeta> getTableMeta(String db_patterns, String tbl_patterns, List<String> tbl_types) {
         Map<DatabaseMapping, String> databaseMappingsForPattern = databaseMappingsByDbPattern(db_patterns);
 
-        BiFunction<TableMeta, DatabaseMapping, Boolean> filter = (tableMeta, mapping) ->
-            databaseAndTableAllowed(tableMeta.getDbName(), tableMeta.getTableName(), mapping);
+        BiFunction<TableMeta, DatabaseMapping, Boolean> filter = (tableMeta, mapping) -> databaseAndTableAllowed(
+            tableMeta.getDbName(), tableMeta.getTableName(), mapping);
 
         return super.getTableMeta(tbl_patterns, tbl_types, databaseMappingsForPattern, filter);
       }

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2020 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2020 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java

@@ -15,8 +15,6 @@
  */
 package com.hotels.bdp.waggledance.mapping.model;
 
-import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance;
-
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
@@ -25,6 +23,8 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance;
+
 import java.util.Arrays;
 
 import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/GrammarUtilsTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2020 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -77,6 +77,41 @@ public void subPatternMatchesDatabaseAndAllSpecificTables() {
     assertThat(patternParts[1], is("*base"));
   }
 
+  @Test
+  public void splitPatternWithDot() {
+    String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle.dm*");
+    assertThat(patternParts[0], is("waggle."));
+    assertThat(patternParts[1], is("dm*"));
+  }
+
+  @Test
+  public void splitPatternWithDots() {
+    String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle...dm");
+    assertThat(patternParts[0], is("waggle."));
+    assertThat(patternParts[1], is("..dm"));
+  }
+
+  @Test
+  public void splitPatternWithDotsAndStarEnd() {
+    String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle...dm*");
+    assertThat(patternParts[0], is("waggle."));
+    assertThat(patternParts[1], is("..dm*"));
+  }
+
+  @Test
+  public void splitPatternWithDotsAndStar() {
+    String[] patternParts = GrammarUtils.splitPattern(PREFIX, "waggle.*.dm");
+    assertThat(patternParts[0], is("waggle.*"));
+    assertThat(patternParts[1], is("*.dm"));
+  }
+
+  @Test
+  public void splitPatternWithDotsInMiddle() {
+    String[] patternParts = GrammarUtils.splitPattern(PREFIX, "wa..le_dm");
+    assertThat(patternParts[0], is("wa..le_"));
+    assertThat(patternParts[1], is("dm"));
+  }
+
   @Test
   public void matchesWithNullPattern() {
     Map<String, String> splits = GrammarUtils.selectMatchingPrefixes(ImmutableSet.of(PREFIX, "other_"), null);
@@ -93,6 +128,13 @@ public void matchesWithWildcardPattern() {
     assertThat(splits.get("other_"), is("*"));
   }
 
+  @Test
+  public void matchesWithDotWildcardPattern() {
+    Map<String, String> splits = GrammarUtils.selectMatchingPrefixes(ImmutableSet.of(PREFIX, "other_"), "other.dm");
+    assertThat(splits.size(), is(1));
+    assertThat(splits.get("other_"), is("dm"));
+  }
+
   @Test
   public void doesNotMatchPatternSimpleDatabaseName() {
     Map<String, String> splits = GrammarUtils.selectMatchingPrefixes(ImmutableSet.of(PREFIX, "other_"), "database");

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/PanopticConcurrentOperationExecutorTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2020 Expedia, Inc.
+ * Copyright (C) 2016-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -119,6 +119,14 @@ public List<String> call() throws Exception {
     assertThat(executeRequests.get(1), is("call3"));
   }
 
+  @Test
+  public void executeEmptyRequests() throws Exception {
+    PanopticConcurrentOperationExecutor executor = new PanopticConcurrentOperationExecutor();
+    List<DummyRequestCallable> allRequests = Lists.newArrayList();
+    List<String> executeRequests = executor.executeRequests(allRequests, REQUEST_TIMEOUT, "error in call: {}");
+    assertThat(executeRequests.size(), is(0));
+  }
+
   private class DummyRequestCallable implements RequestCallable<List<String>> {
 
     private final String callValue;

waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java

@@ -58,6 +58,7 @@
 import org.apache.hadoop.hive.metastore.api.ResourceType;
 import org.apache.hadoop.hive.metastore.api.ResourceUri;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
 import org.apache.thrift.TException;
 import org.junit.After;
 import org.junit.Before;
@@ -1060,4 +1061,32 @@ public void get_privilege_set() throws Exception {
     assertNotNull(get_privilege_set);
   }
 
+  @Test
+  public void getTableMeta() throws Exception {
+    runner = WaggleDanceRunner
+        .builder(configLocation)
+        .databaseResolution(DatabaseResolution.PREFIXED)
+        .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
+        .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
+        .build();
+
+    runWaggleDance(runner);
+    HiveMetaStoreClient proxy = getWaggleDanceClient();
+
+    List<TableMeta> tableMeta = proxy
+        .getTableMeta("waggle_remote_remote_database", "*", Lists.newArrayList("EXTERNAL_TABLE"));
+    assertThat(tableMeta.size(), is(1));
+    assertThat(tableMeta.get(0).getDbName(), is("waggle_remote_remote_database"));
+    assertThat(tableMeta.get(0).getTableName(), is(REMOTE_TABLE));
+    // use wildcards: '.'
+    tableMeta = proxy.getTableMeta("waggle_remote.remote_database", "*", Lists.newArrayList("EXTERNAL_TABLE"));
+    assertThat(tableMeta.size(), is(1));
+    assertThat(tableMeta.get(0).getDbName(), is("waggle_remote_remote_database"));
+    assertThat(tableMeta.get(0).getTableName(), is(REMOTE_TABLE));
+    tableMeta = proxy.getTableMeta("waggle.remote_remote_database", "*", Lists.newArrayList("EXTERNAL_TABLE"));
+    assertThat(tableMeta.size(), is(1));
+    assertThat(tableMeta.get(0).getDbName(), is("waggle_remote_remote_database"));
+    assertThat(tableMeta.get(0).getTableName(), is(REMOTE_TABLE));
+  }
+
 }