[GOBBLIN-204] Add a service that fetches GaaS flow configs from a git…

Closes #2055 from htran1/git_config
apache · Aug 16, 2017 · da382db · da382db
1 parent 56cecb2
commit da382db
Show file tree

Hide file tree

Showing 9 changed files with 876 additions and 18 deletions.
diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java
@@ -831,4 +831,18 @@ public class ConfigurationKeys {
    * Configuration related to ConfigStore based copy/retention
    */
   public static final String CONFIG_BASED_PREFIX = "gobblin.configBased";
+
+  /**
+   * Configuration related to the git flow config monitoring service
+   */
+  public static final String GIT_CONFIG_MONITOR_PREFIX = "gitConfigMonitor.";
+  public static final String GIT_CONFIG_MONITOR_REPO_URI = GIT_CONFIG_MONITOR_PREFIX + "repositoryUri";
+  public static final String GIT_CONFIG_MONITOR_REPO_DIR = GIT_CONFIG_MONITOR_PREFIX + "repositoryDirectory";
+  public static final String DEFAULT_GIT_CONFIG_MONITOR_REPO_DIR = "git-flow-config";
+  public static final String GIT_CONFIG_MONITOR_CONFIG_DIR = GIT_CONFIG_MONITOR_PREFIX + "configDirectory";
+  public static final String DEFAULT_GIT_CONFIG_MONITOR_CONFIG_DIR = "gobblin-config";
+  public static final String GIT_CONFIG_MONITOR_POLLING_INTERVAL = GIT_CONFIG_MONITOR_PREFIX + "pollingInterval";
+  public static final String GIT_CONFIG_MONITOR_BRANCH_NAME = GIT_CONFIG_MONITOR_PREFIX + "branchName";
+  public static final String DEFAULT_GIT_CONFIG_MONITOR_BRANCH_NAME = "master";
+  public static final int DEFAULT_GIT_CONFIG_MONITOR_POLLING_INTERVAL = 60;
 }
diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/spec_store/FSSpecStore.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/spec_store/FSSpecStore.java
@@ -17,11 +17,11 @@
 
 package org.apache.gobblin.runtime.spec_store;
 
+import com.google.common.io.ByteStreams;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Collection;
 
-import org.apache.commons.compress.utils.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -258,7 +258,7 @@ protected Spec readSpecFromFile(Path path) throws IOException {
     Spec spec = null;
 
     try (FSDataInputStream fis = fs.open(path);) {
-      spec = this.specSerDe.deserialize(IOUtils.toByteArray(fis));
+      spec = this.specSerDe.deserialize(ByteStreams.toByteArray(fis));
     }
 
     return spec;

diff --git a/gobblin-service/build.gradle b/gobblin-service/build.gradle
@@ -56,6 +56,7 @@ dependencies {
   compile externalDependency.jacksonCore
   compile externalDependency.jacksonMapper
   compile externalDependency.javaxInject
+  compile externalDependency.jgit
   compile externalDependency.jodaTime
   compile externalDependency.kafka08
   compile externalDependency.log4j

diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java
@@ -34,6 +34,7 @@ public class ServiceConfigKeys {
   public static final String GOBBLIN_SERVICE_ORCHESTRATOR_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "orchestrator.enabled";
   public static final String GOBBLIN_SERVICE_RESTLI_SERVER_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "restliServer.enabled";
   public static final String GOBBLIN_SERVICE_TOPOLOGY_SPEC_FACTORY_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "topologySpecFactory.enabled";
+  public static final String GOBBLIN_SERVICE_GIT_CONFIG_MONITOR_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "gitConfigMonitor.enabled";
 
   // Helix / ServiceScheduler Keys
   public static final String HELIX_CLUSTER_NAME_KEY = GOBBLIN_SERVICE_PREFIX + "helix.cluster.name";