Big query support

schedoscope-conf/src/main/resources/reference.conf

@@ -329,6 +329,69 @@
 
       salt = "vD75MqvaasIlCf7H"
 
+      #
+      # BigQuery exporter settings.
+      #
+
+      bigQuery {
+
+        #
+        # GCP project ID under which the exported BigQuery dataset will be created
+        #
+
+        projectId = ""
+
+        #
+        # GCP key in JSON format to use for authentication
+        #
+
+        gcpKey = ""
+
+        #
+        # Number of reducers to use for parallel writing to BigQuery.
+        #
+
+        numberOfReducers = 10
+
+        #
+        # GCP data storage location of exported data within BigQuery.
+        #
+
+        dataLocation = "EU"
+
+        #
+        # GCP Cloud Storage bucket for temporary storage to use for exporting to BigQuery.
+        #
+
+        exportStorageBucket = "schedoscope_bigquery_export"
+
+        #
+        # GCP Cloud Storage bucket folder prefix to apply to blobs when exporting to BigQuery
+        #
+
+        exportStorageBucketFolderPrefix = ""
+
+        #
+        # GCP Cloud Storage bucket region to use for exporting to BigQuery
+        #
+
+        exportStorageBucketRegion = "europe-west3"
+
+        #
+        # Host of proxy to use for GCP API access
+        #
+
+        proxyHost = ""
+
+        #
+        # Port of proxy to use for GCP API access
+        #
+
+        proxyPort = ""
+
+
+      }
+
       #
       # JDBC exporter settings.
       #

schedoscope-conf/src/main/scala/org/schedoscope/conf/BaseSettings.scala

@@ -247,10 +247,55 @@ class BaseSettings(val config: Config) {
   lazy val redisExportBatchSize = config.getInt("schedoscope.export.redis.insertBatchSize")
 
   /**
-    * Number of reducers to use for Redis export.
+    * Number of reducers to use for Kafka export.
     */
   lazy val kafkaExportNumReducers = config.getInt("schedoscope.export.kafka.numberOfReducers")
 
+  /**
+    * GCP project ID under which exported BigQuery dataset will be created. Defaults to the default project of the current user.
+    */
+  lazy val bigQueryExportProjectId = config.getString("schedoscope.export.bigQuery.projectId")
+
+  /**
+    * Number of reducers to use for BigQuery export.
+    */
+  lazy val bigQueryExportNumReducers = config.getInt("schedoscope.export.bigQuery.numberOfReducers")
+
+  /**
+    * GCP data storage location of exported data within BigQuery. Defaults to EU.
+    */
+  lazy val bigQueryExportDataLocation = config.getString("schedoscope.export.bigQuery.dataLocation")
+
+  /**
+    * GCP key in JSON format to use for authentication when exporting to BigQuery. If not set, the key of the current user is used.
+    */
+  lazy val bigQueryExportGcpKey = config.getString("schedoscope.export.bigQuery.gcpKey")
+
+  /**
+    * GCP Cloud Storage bucket to use for temporary storage while exporting to BigQuery. Defaults to "schedoscope_bigquery_export"
+    */
+  lazy val bigQueryExportStorageBucket = config.getString("schedoscope.export.bigQuery.exportStorageBucket")
+
+  /**
+    * Folder prefix to apply to blobs in the GCP Cloud Storage bucket while exporting to BigQuery. Defaults to ""
+    */
+  lazy val bigQueryExportStorageBucketFolderPrefix = config.getString("schedoscope.export.bigQuery.exportStorageBucketFolderPrefix")
+
+  /**
+    * GCP Cloud Storage bucket region to use for exporting to BigQuery. Defaults to europe-west3
+    */
+  lazy val bigQueryExportStorageBucketRegion = config.getString("schedoscope.export.bigQuery.exportStorageBucketRegion")
+
+  /**
+    * Host of proxy to use for GCP API access. Set to empty, i.e., no proxy to use.
+    */
+  lazy val bigQueryExportProxyHost = config.getString("schedoscope.export.bigQuery.proxyHost")
+
+  /**
+    * Port of proxy to use for GCP API access. Set to empty, i.e., no proxy to use.
+    */
+  lazy val bigQueryExportProxyPort = config.getString("schedoscope.export.bigQuery.proxyPort")
+
   /**
     * Number of reducers to use for (S)Ftp export.
     */

schedoscope-core/pom.xml

@@ -51,11 +51,6 @@
             <artifactId>schedoscope-conf</artifactId>
             <version>${schedoscope.version}</version>
         </dependency>
-        <dependency>
-            <artifactId>guava</artifactId>
-            <groupId>com.google.guava</groupId>
-            <version>11.0</version>
-        </dependency>
         <dependency>
             <groupId>joda-time</groupId>
             <artifactId>joda-time</artifactId>
@@ -232,10 +227,6 @@
                     <artifactId>slf4j-api</artifactId>
                     <groupId>org.slf4j</groupId>
                 </exclusion>
-                <exclusion>
-                    <artifactId>guava</artifactId>
-                    <groupId>com.google.guava</groupId>
-                </exclusion>
                 <exclusion>
                     <artifactId>jsp-api</artifactId>
                     <groupId>javax.servlet.jsp</groupId>