From eb047fa8859ce7513bc45d58e516121967d81a76 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com> Date: Fri, 3 Jan 2020 22:56:31 +0800 Subject: [PATCH 1/4] Use Nexus staging plugin for deployment (#394) * Use Nexus staging pluging for deployment * Fix Javadoc error * Hard coded parent version as variable substitution is not supported --- .prow/scripts/test-end-to-end-batch.sh | 4 +- .prow/scripts/test-end-to-end.sh | 7 ++- Makefile | 2 +- core/pom.xml | 6 +-- .../feast/core/config/MonitoringConfig.java | 2 +- .../java/feast/core/service/SpecService.java | 4 +- .../java/feast/core/util/PackageUtil.java | 3 +- infra/docker/core/Dockerfile | 2 +- infra/docker/serving/Dockerfile | 2 +- ingestion/pom.xml | 4 +- pom.xml | 51 +++++++++++++++---- sdk/java/pom.xml | 6 +-- serving/pom.xml | 4 +- 13 files changed, 66 insertions(+), 31 deletions(-) diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh index b370c5b045b..ba395fc6166 100755 --- a/.prow/scripts/test-end-to-end-batch.sh +++ b/.prow/scripts/test-end-to-end-batch.sh @@ -138,7 +138,7 @@ management: enabled: false EOF -nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \ +nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 30 @@ -191,7 +191,7 @@ spring: web-environment: false EOF -nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \ +nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/serving.warehouse.application.yml \ &> /var/log/feast-serving-warehouse.log & sleep 15 diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index 2c6f4a098f9..e8160444d39 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -75,6 +75,9 @@ Building jars for Feast # Build jars for Feast mvn --quiet --batch-mode --define skipTests=true clean package +ls -lh core/target/*jar +ls -lh serving/target/*jar + echo " ============================================================ Starting Feast Core @@ -121,7 +124,7 @@ management: enabled: false EOF -nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \ +nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 30 @@ -172,7 +175,7 @@ spring: web-environment: false EOF -nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \ +nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \ --spring.config.location=file:///tmp/serving.online.application.yml \ &> /var/log/feast-serving-online.log & sleep 15 diff --git a/Makefile b/Makefile index b7eb0edbade..9f0742b2e6c 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ build-cli: $(MAKE) -C cli build-all build-java: - mvn clean verify -Drevision=$(VERSION) + mvn clean verify build-docker: docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . diff --git a/core/pom.xml b/core/pom.xml index f6e4909260a..c0d889e2832 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,9 +21,9 @@ 4.0.0 - feast + dev.feast feast-parent - ${revision} + 0.3.6-SNAPSHOT Feast Core @@ -48,7 +48,7 @@ - feast + dev.feast feast-ingestion ${project.version} diff --git a/core/src/main/java/feast/core/config/MonitoringConfig.java b/core/src/main/java/feast/core/config/MonitoringConfig.java index fd20bed1ee8..53c9562c47c 100644 --- a/core/src/main/java/feast/core/config/MonitoringConfig.java +++ b/core/src/main/java/feast/core/config/MonitoringConfig.java @@ -66,7 +66,7 @@ public FeastResourceCollector feastResourceCollector( /** * Register custom Prometheus collector that exports metrics about JVM resource usage. * - * @return @{link {@link JVMResourceCollector}} + * @return {@link JVMResourceCollector} */ @Bean public JVMResourceCollector jvmResourceCollector() { diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 4ea2d288f2e..99862fd5dc0 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -82,7 +82,7 @@ public SpecService( * required. If the version is provided then it will be used for the lookup. If the version is * omitted then the latest version will be returned. * - * @param GetFeatureSetRequest containing the name and version of the feature set + * @param request containing the name and version of the feature set * @return GetFeatureSetResponse containing a single feature set */ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) @@ -141,7 +141,7 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) * *

The version filter is optional; If not provided, this method will return all featureSet * versions of the featureSet name provided. Valid version filters should optionally contain a - * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1 + * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1 * * @param filter filter containing the desired featureSet name and version filter * @return ListFeatureSetsResponse with list of featureSets found matching the filter diff --git a/core/src/main/java/feast/core/util/PackageUtil.java b/core/src/main/java/feast/core/util/PackageUtil.java index ef27332ac0f..20b2310644b 100644 --- a/core/src/main/java/feast/core/util/PackageUtil.java +++ b/core/src/main/java/feast/core/util/PackageUtil.java @@ -49,8 +49,9 @@ public class PackageUtil { * handled by default in Apache Beam. * * 

-   * @code
+   * 
    * URL url = new URL("jar:file:/tmp/springexample/target/spring-example-1.0-SNAPSHOT.jar!/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar!/");
+   * 
    * String resolvedPath = resolveSpringBootPackageClasspath(url);
    * // resolvedPath should point to "/tmp/springexample/target/spring-example-1.0-SNAPSHOT/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar"
    * // Note that spring-example-1.0-SNAPSHOT.jar is extracted in the process.
diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile
index c4cfe34b71b..91ef030dc90 100644
--- a/infra/docker/core/Dockerfile
+++ b/infra/docker/core/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects core,ingestion -Drevision=$REVISION \
+RUN mvn --also-make --projects core,ingestion \
   -DskipTests=true --batch-mode package
 #
 # Unpack the jar and copy the files into production Docker image
diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile
index 3517183d782..5605c8846de 100644
--- a/infra/docker/serving/Dockerfile
+++ b/infra/docker/serving/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects serving -Drevision=$REVISION \
+RUN mvn --also-make --projects serving \
   -DskipTests=true --batch-mode package
 
 # ============================================================
diff --git a/ingestion/pom.xml b/ingestion/pom.xml
index eb892335180..fce645c51b3 100644
--- a/ingestion/pom.xml
+++ b/ingestion/pom.xml
@@ -21,9 +21,9 @@
   4.0.0
 
   
-    feast
+    dev.feast
     feast-parent
-    ${revision}
+    0.3.6-SNAPSHOT
   
 
   Feast Ingestion
diff --git a/pom.xml b/pom.xml
index edf2a0244e9..882e24ab8ea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,9 +22,9 @@
     Feature Store for Machine Learning
     ${github.url}
 
-    feast
+    dev.feast
     feast-parent
-    ${revision}
+    0.3.6-SNAPSHOT
     pom
 
     
@@ -35,7 +35,6 @@
     
 
     
-        0.3.2-SNAPSHOT
         https://github.com/gojek/feast
 
         UTF-8
@@ -83,15 +82,10 @@
     
 
     
-        
         
-            feast-snapshot
-            file:///tmp/snapshot
+            ossrh
+            https://oss.sonatype.org/content/repositories/snapshots
         
-        
-            feast
-            file:///tmp/snapshot
-        
     
 
     
@@ -280,6 +274,43 @@
         
 
         
+            
+                org.sonatype.plugins
+                nexus-staging-maven-plugin
+                1.6.7
+                true
+                
+                    ossrh
+                    https://oss.sonatype.org/
+                    false
+                
+            
+            
+                org.apache.maven.plugins
+                maven-source-plugin
+                2.2.1
+                
+                    
+                        attach-sources
+                        
+                            jar-no-fork
+                        
+                    
+                
+            
+            
+                org.apache.maven.plugins
+                maven-javadoc-plugin
+                2.9.1
+                
+                    
+                        attach-javadocs
+                        
+                            jar
+                        
+                    
+                
+            
             
                 com.diffplug.spotless
                 spotless-maven-plugin
diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml
index 2c8b1d837a3..b3543446e2e 100644
--- a/sdk/java/pom.xml
+++ b/sdk/java/pom.xml
@@ -6,12 +6,12 @@
 
   Feast SDK for Java
   SDK for registering, storing, and retrieving features
-  feast-client
+  feast-sdk
 
   
-    feast
+    dev.feast
     feast-parent
-    ${revision}
+    0.3.6-SNAPSHOT
     ../..
   
 
diff --git a/serving/pom.xml b/serving/pom.xml
index ab9efaff26e..4a28eb739e8 100644
--- a/serving/pom.xml
+++ b/serving/pom.xml
@@ -21,9 +21,9 @@
   4.0.0
 
   
-    feast
+    dev.feast
     feast-parent
-    ${revision}
+    0.3.6-SNAPSHOT
   
 
   feast-serving

From e37292414a8b139aee95c7180a542c7778f5a65f Mon Sep 17 00:00:00 2001
From: David Heryanto 
Date: Tue, 7 Jan 2020 10:41:04 +0800
Subject: [PATCH 2/4] Java SDK release script (#406)

* Use back revision variable in pom.xml
So user or CI system can easily override revision from external sources such as Git tag name

* Add flatten maven plugin
This plugin is useful during deployment so the final pom is resolved without parent dependency, i.e. we do not necessarily need to upload parent library

* Increase versions for maven source,javadoc,spotless plugins
So it has newer features and more fixes

* Add gpg-plugin needed to sign releases

* Use oss configure for flatten plugin, add developers info in pom.xml (required for releasing library

* Add publish-java-sdk script

* Add more logs to publish-java-sdk.sh

* Add ProwJob publish-java-sdk

* Use GPG_KEY_IMPORT_DIR variable

* Update revision in pom.xml to 0.4.2-SNAPSHOT
---
 .gitignore                        |   6 ++
 .prow/config.yaml                 |  29 ++++++++
 .prow/scripts/publish-java-sdk.sh |  72 ++++++++++++++++++++
 core/pom.xml                      |   2 +-
 ingestion/pom.xml                 |   2 +-
 pom.xml                           | 109 +++++++++++++++++++++++++-----
 sdk/java/pom.xml                  |   2 +-
 serving/pom.xml                   |   2 +-
 8 files changed, 203 insertions(+), 21 deletions(-)
 create mode 100755 .prow/scripts/publish-java-sdk.sh

diff --git a/.gitignore b/.gitignore
index 8aaa0b69200..a8c5e3fe0ba 100644
--- a/.gitignore
+++ b/.gitignore
@@ -173,3 +173,9 @@ dmypy.json
 # Pyre type checker
 .pyre/
 .vscode
+
+# .flattened-pom.xml is generated by flatten-maven-plugin.
+# This pom should not be committed because it is only used during release / deployment.
+.flattened-pom.xml
+
+sdk/python/docs/html
diff --git a/.prow/config.yaml b/.prow/config.yaml
index 4b6e352a12f..39b81d76fd6 100644
--- a/.prow/config.yaml
+++ b/.prow/config.yaml
@@ -169,6 +169,34 @@ postsubmits:
     # https://github.com/semver/semver/issues/232
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
 
+  - name: publish-java-sdk
+    decorate: true
+    spec:
+      containers:
+      - image: maven:3.6-jdk-8
+        command:
+        - bash
+        - -c
+        - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
+        volumeMounts:
+        - name: gpg-keys
+          mountPath: /etc/gpg
+          readOnly: true
+        - name: maven-settings
+          mountPath: /root/.m2/settings.xml
+          subPath: settings.xml
+          readOnly: true
+      volumes:
+      - name: gpg-keys
+        secret:
+          secretName: gpg-keys
+      - name: maven-settings
+        secret:
+          secretName: maven-settings
+    branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
+    - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
+
   - name: publish-docker-images
     decorate: true
     spec:
@@ -278,4 +306,5 @@ postsubmits:
         secret:
           secretName: feast-service-account
     branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh
new file mode 100755
index 00000000000..17513d0eb0d
--- /dev/null
+++ b/.prow/scripts/publish-java-sdk.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -e 
+set -o pipefail
+
+GPG_KEY_IMPORT_DIR=/etc/gpg
+
+usage()
+{
+    echo "usage: publish-java-sdk.sh
+
+  --revision            Value for the revision e.g. '0.2.3'
+  --gpg-key-import-dir  Directory containing existing GPG keys to import.
+                        The directory should contain these 2 files:
+                        - public-key
+                        - private-key
+                        The default value is '/etc/gpg'
+  
+  This script assumes the GPG private key is protected by a passphrase.
+  The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml
+  file, credentials to upload releases to Sonatype must also be provided.
+
+  # Example settings: ~/.m2/settings.xml
+  
+    
+      
+        ossrh
+        SONATYPE_USER
+        SONATYPE_PASSWORD
+      
+    
+    
+      
+        ossrh
+        
+          GPG_PASSPHRASE
+        
+      
+    
+  
+"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --revision )             REVISION="$2";            shift;;
+      --gpg-key-import-dir )   GPG_KEY_IMPORT_DIR="$2";  shift;;
+      -h | --help )            usage;                    exit;; 
+      * )                      usage;                    exit 1
+  esac
+  shift
+done
+
+if [ -z $REVISION ]; then usage; exit 1; fi
+
+echo "============================================================"
+echo "Checking Maven and GPG versions"
+echo "============================================================"
+mvn --version 
+echo ""
+gpg --version
+
+echo "============================================================"
+echo "Importing GPG keys"
+echo "============================================================"
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key
+
+echo "============================================================"
+echo "Deploying Java SDK with revision: $REVISION"
+echo "============================================================"
+mvn --projects sdk/java -Drevision=$REVISION --batch-mode clean deploy
diff --git a/core/pom.xml b/core/pom.xml
index c0d889e2832..954c7c00185 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -23,7 +23,7 @@
     
         dev.feast
         feast-parent
-        0.3.6-SNAPSHOT
+        ${revision}
     
 
     Feast Core
diff --git a/ingestion/pom.xml b/ingestion/pom.xml
index fce645c51b3..4908b546985 100644
--- a/ingestion/pom.xml
+++ b/ingestion/pom.xml
@@ -23,7 +23,7 @@
   
     dev.feast
     feast-parent
-    0.3.6-SNAPSHOT
+    ${revision}
   
 
   Feast Ingestion
diff --git a/pom.xml b/pom.xml
index 882e24ab8ea..9cd37d5338c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,7 +24,7 @@
 
     dev.feast
     feast-parent
-    0.3.6-SNAPSHOT
+    ${revision}
     pom
 
     
@@ -35,6 +35,7 @@
     
 
     
+        0.3.6-SNAPSHOT
         https://github.com/gojek/feast
 
         UTF-8
@@ -47,7 +48,6 @@
         2.16.0
         1.91.0
         0.8.0
-
         1.9.10
         1.3
         2.3.0
@@ -58,9 +58,18 @@
 
     
         Gojek
-        https://www.gojek.io/
+        https://www.gojek.com
     
 
+    
+        
+            Feast Authors
+            ${github.url}
+            Gojek
+            https://www.gojek.com
+        
+    
+
     
         
             Apache License, Version 2.0
@@ -81,11 +90,16 @@
         ${github.url}/issues
     
 
+    
     
         
             ossrh
             https://oss.sonatype.org/content/repositories/snapshots
         
+        
+            ossrh
+            https://oss.sonatype.org/service/local/staging/deploy/maven2/
+        
     
 
     
@@ -274,21 +288,10 @@
         
 
         
-            
-                org.sonatype.plugins
-                nexus-staging-maven-plugin
-                1.6.7
-                true
-                
-                    ossrh
-                    https://oss.sonatype.org/
-                    false
-                
-            
             
                 org.apache.maven.plugins
                 maven-source-plugin
-                2.2.1
+                3.2.1
                 
                     
                         attach-sources
@@ -301,7 +304,7 @@
             
                 org.apache.maven.plugins
                 maven-javadoc-plugin
-                2.9.1
+                3.1.1
                 
                     
                         attach-javadocs
@@ -314,7 +317,7 @@
             
                 com.diffplug.spotless
                 spotless-maven-plugin
-                1.26.0
+                1.26.1
                 
                     
                         
@@ -429,6 +432,78 @@
                     true
                 
             
+            
+            
+                org.sonatype.plugins
+                nexus-staging-maven-plugin
+                1.6.8
+                true
+                
+                    ossrh
+                    https://oss.sonatype.org/
+                    
+                    true
+                
+            
+            
+            
+                org.codehaus.mojo
+                flatten-maven-plugin
+                1.1.0
+                
+                    oss
+                
+                
+                    
+                        flatten
+                        process-resources
+                        
+                            flatten
+                        
+                    
+                    
+                        flatten.clean
+                        clean
+                        
+                            clean
+                        
+                    
+                
+            
+            
+            
+                org.apache.maven.plugins
+                maven-gpg-plugin
+                1.6
+                
+                    
+                        sign-artifacts
+                        verify
+                        
+                            sign
+                        
+                        
+                        
+                            
+                                --pinentry-mode
+                                loopback
+                            
+                            
+                            
+                            ${gpg.passphrase}
+                        
+                    
+                
+            
         
 
         
diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml
index b3543446e2e..2970dae3ee2 100644
--- a/sdk/java/pom.xml
+++ b/sdk/java/pom.xml
@@ -11,7 +11,7 @@
   
     dev.feast
     feast-parent
-    0.3.6-SNAPSHOT
+    ${revision}
     ../..
   
 
diff --git a/serving/pom.xml b/serving/pom.xml
index 4a28eb739e8..dc3391df62f 100644
--- a/serving/pom.xml
+++ b/serving/pom.xml
@@ -23,7 +23,7 @@
   
     dev.feast
     feast-parent
-    0.3.6-SNAPSHOT
+    ${revision}
   
 
   feast-serving

From 1bcbd0169a2ce3d4ad20ee14e32c3f74e154dcde Mon Sep 17 00:00:00 2001
From: Ches Martin 
Date: Tue, 24 Dec 2019 16:06:40 +0700
Subject: [PATCH 3/4] Introduce datatypes/java module for proto generation

Rather than the Maven protobuf plugin running on the same symlinked
definitions in several Java modules, localize this process into one
module that the others depend on.

This provides a single module that can be depended on by third-party
extensions with the bare minimum of dependencies.

Also removes proto files that are no longer used.
---
 core/pom.xml                                  |  4 --
 core/src/main/proto/feast                     |  1 -
 core/src/main/proto/third_party               |  1 -
 datatypes/java/README.md                      | 43 +++++++++++
 datatypes/java/pom.xml                        | 72 +++++++++++++++++++
 {sdk => datatypes}/java/src/main/proto/feast  |  0
 datatypes/java/src/main/proto/third_party     |  1 +
 ingestion/pom.xml                             | 10 +--
 ingestion/src/main/proto/feast                |  1 -
 .../feast_ingestion/types/CoalesceAccum.proto | 35 ---------
 .../feast_ingestion/types/CoalesceKey.proto   | 25 -------
 ingestion/src/main/proto/third_party          |  1 -
 ingestion/src/test/proto/DriverArea.proto     | 10 ---
 ingestion/src/test/proto/Ping.proto           | 12 ----
 pom.xml                                       | 20 +-----
 sdk/java/pom.xml                              | 10 +--
 serving/pom.xml                               | 10 +--
 serving/src/main/proto/feast                  |  1 -
 serving/src/main/proto/third_party            |  1 -
 19 files changed, 135 insertions(+), 123 deletions(-)
 delete mode 120000 core/src/main/proto/feast
 delete mode 120000 core/src/main/proto/third_party
 create mode 100644 datatypes/java/README.md
 create mode 100644 datatypes/java/pom.xml
 rename {sdk => datatypes}/java/src/main/proto/feast (100%)
 create mode 120000 datatypes/java/src/main/proto/third_party
 delete mode 120000 ingestion/src/main/proto/feast
 delete mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto
 delete mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto
 delete mode 120000 ingestion/src/main/proto/third_party
 delete mode 100644 ingestion/src/test/proto/DriverArea.proto
 delete mode 100644 ingestion/src/test/proto/Ping.proto
 delete mode 120000 serving/src/main/proto/feast
 delete mode 120000 serving/src/main/proto/third_party

diff --git a/core/pom.xml b/core/pom.xml
index 954c7c00185..e1567ae8fe3 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -39,10 +39,6 @@
                     false
                 
             
-            
-                org.xolstice.maven.plugins
-                protobuf-maven-plugin
-            
         
     
 
diff --git a/core/src/main/proto/feast b/core/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/core/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/core/src/main/proto/third_party b/core/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/core/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/datatypes/java/README.md b/datatypes/java/README.md
new file mode 100644
index 00000000000..a062144ff3f
--- /dev/null
+++ b/datatypes/java/README.md
@@ -0,0 +1,43 @@
+Feast Data Types for Java
+=========================
+
+This module produces Java class files for Feast's data type and gRPC service
+definitions, from Protobuf IDL. These are used across Feast components for wire
+interchange, contracts, etc.
+
+End users of Feast will be best served by our Java SDK which adds higher-level
+conveniences, but the data types are published independently for custom needs,
+without any additional dependencies the SDK may add.
+
+Dependency Coordinates
+----------------------
+
+```xml
+
+  dev.feast
+  datatypes-java
+  0.3.6-SNAPSHOT
+
+```
+
+Using the `.proto` Definitions
+------------------------------
+
+The `.proto` definitions are packaged as resources within the Maven artifact,
+which may be useful to `include` them in dependent Protobuf definitions in a
+downstream project, or for other JVM languages to consume from their builds to
+generate more idiomatic bindings.
+
+Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle]
+either for `include` or to compile with a different `protoc` plugin than Java.
+
+[sbt-protoc] offers similar functionality for sbt/Scala.
+
+[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies
+[sbt-protoc]: https://github.com/thesamet/sbt-protoc
+
+Publishing
+----------
+
+TODO: this module should be published to Maven Central upon Feast releases—this
+needs to be set up in POM configuration and release automation.
diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml
new file mode 100644
index 00000000000..a6dfa8e345a
--- /dev/null
+++ b/datatypes/java/pom.xml
@@ -0,0 +1,72 @@
+
+
+
+    4.0.0
+
+    Feast Data Types for Java
+    
+        Data types and service contracts used throughout Feast components and
+        their interchanges. These are generated from Protocol Buffers and gRPC
+        definitions included in the package.
+    
+    datatypes-java
+
+    
+      dev.feast
+      feast-parent
+      ${revision}
+      ../..
+    
+
+    
+      
+        
+          org.xolstice.maven.plugins
+          protobuf-maven-plugin
+          
+            true
+            
+                com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
+            
+            grpc-java
+            
+                io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
+            
+          
+          
+            
+              
+                compile
+                compile-custom
+                test-compile
+              
+            
+          
+        
+      
+    
+
+    
+      
+        io.grpc
+        grpc-services
+      
+    
+
diff --git a/sdk/java/src/main/proto/feast b/datatypes/java/src/main/proto/feast
similarity index 100%
rename from sdk/java/src/main/proto/feast
rename to datatypes/java/src/main/proto/feast
diff --git a/datatypes/java/src/main/proto/third_party b/datatypes/java/src/main/proto/third_party
new file mode 120000
index 00000000000..f015f8477d1
--- /dev/null
+++ b/datatypes/java/src/main/proto/third_party
@@ -0,0 +1 @@
+../../../../../protos/third_party
\ No newline at end of file
diff --git a/ingestion/pom.xml b/ingestion/pom.xml
index 4908b546985..2e1dee65536 100644
--- a/ingestion/pom.xml
+++ b/ingestion/pom.xml
@@ -31,10 +31,6 @@
 
   
     
-      
-        org.xolstice.maven.plugins
-        protobuf-maven-plugin
-      
       
         org.apache.maven.plugins
         maven-shade-plugin
@@ -90,6 +86,12 @@
   
 
   
+    
+      dev.feast
+      datatypes-java
+      ${project.version}
+    
+
     
       org.glassfish
       javax.el
diff --git a/ingestion/src/main/proto/feast b/ingestion/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/ingestion/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto
deleted file mode 100644
index cb64dd715f6..00000000000
--- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2018 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-import "google/protobuf/timestamp.proto";
-import "feast/types/Field.proto";
-
-option java_package = "feast_ingestion.types";
-option java_outer_classname = "CoalesceAccumProto";
-
-// Accumlator for merging feature rows.
-message CoalesceAccum {
-  string entityKey = 1;
-  google.protobuf.Timestamp eventTimestamp = 3;
-  string entityName = 4;
-
-  map features = 6;
-  // map of features to their counter values when they were last added to accumulator
-  map featureMarks = 7;
-  int64 counter = 8;
-}
\ No newline at end of file
diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto
deleted file mode 100644
index 9730b49ec3b..00000000000
--- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2018 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto3";
-
-option java_package = "feast_ingestion.types";
-option java_outer_classname = "CoalesceKeyProto";
-
-message CoalesceKey {
-  string entityName = 1;
-  string entityKey = 2;
-}
\ No newline at end of file
diff --git a/ingestion/src/main/proto/third_party b/ingestion/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/ingestion/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/ingestion/src/test/proto/DriverArea.proto b/ingestion/src/test/proto/DriverArea.proto
deleted file mode 100644
index fee838b9e17..00000000000
--- a/ingestion/src/test/proto/DriverArea.proto
+++ /dev/null
@@ -1,10 +0,0 @@
-syntax = "proto3";
-
-package feast;
-
-option java_outer_classname = "DriverAreaProto";
-
-message DriverArea {
-  int32 driverId = 1;
-  int32 areaId = 2;
-}
\ No newline at end of file
diff --git a/ingestion/src/test/proto/Ping.proto b/ingestion/src/test/proto/Ping.proto
deleted file mode 100644
index b1069afa5bd..00000000000
--- a/ingestion/src/test/proto/Ping.proto
+++ /dev/null
@@ -1,12 +0,0 @@
-syntax = "proto3";
-
-package feast;
-import "google/protobuf/timestamp.proto";
-
-option java_outer_classname = "PingProto";
-
-message Ping {
-  double lat = 1;
-  double lng = 2;
-  google.protobuf.Timestamp timestamp = 3;
-}
diff --git a/pom.xml b/pom.xml
index 9cd37d5338c..600c42b4e5f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,6 +28,7 @@
     pom
 
     
+        datatypes/java
         ingestion
         core
         serving
@@ -542,25 +543,6 @@
                     org.xolstice.maven.plugins
                     protobuf-maven-plugin
                     0.6.1
-                    
-                        true
-                        
-                            com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
-                        
-                        grpc-java
-                        
-                            io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
-                        
-                    
-                    
-                        
-                            
-                                compile
-                                compile-custom
-                                test-compile
-                            
-                        
-                    
                 
             
         
diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml
index 2970dae3ee2..e8a82a485fc 100644
--- a/sdk/java/pom.xml
+++ b/sdk/java/pom.xml
@@ -21,6 +21,12 @@
   
 
   
+    
+      dev.feast
+      datatypes-java
+      ${project.version}
+    
+
     
     
       io.grpc
@@ -79,10 +85,6 @@
 
   
     
-      
-        org.xolstice.maven.plugins
-        protobuf-maven-plugin
-      
       
       
         org.apache.maven.plugins
diff --git a/serving/pom.xml b/serving/pom.xml
index dc3391df62f..c15881030e2 100644
--- a/serving/pom.xml
+++ b/serving/pom.xml
@@ -47,10 +47,6 @@
           false
         
       
-      
-        org.xolstice.maven.plugins
-        protobuf-maven-plugin
-      
       
         org.apache.maven.plugins
         maven-failsafe-plugin
@@ -74,6 +70,12 @@
   
 
   
+    
+      dev.feast
+      datatypes-java
+      ${project.version}
+    
+
     
     
       org.slf4j
diff --git a/serving/src/main/proto/feast b/serving/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/serving/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/serving/src/main/proto/third_party b/serving/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/serving/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file

From 0948ca0baa8a53be53ec96207c47899f6dda7481 Mon Sep 17 00:00:00 2001
From: Ches Martin 
Date: Wed, 8 Jan 2020 16:02:59 +0700
Subject: [PATCH 4/4] Publish datatypes/java along with sdk/java

---
 .prow/scripts/publish-java-sdk.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh
index 17513d0eb0d..91123c8d4ee 100755
--- a/.prow/scripts/publish-java-sdk.sh
+++ b/.prow/scripts/publish-java-sdk.sh
@@ -69,4 +69,4 @@ gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key
 echo "============================================================"
 echo "Deploying Java SDK with revision: $REVISION"
 echo "============================================================"
-mvn --projects sdk/java -Drevision=$REVISION --batch-mode clean deploy
+mvn --projects datatypes/java,sdk/java -Drevision=$REVISION --batch-mode clean deploy