[REEF-2025] A new module containing the new Java bridge

    This Jira introduces a new Java bridge for Drivers implemented in
    alternative languages. It provides the following artifacts (note: client
    driver refers to the application driver implemented in an alternative
    programming language):

	1. A generic framework for passing information between the Java driver and the client driver.
	2. A gRPC based implementation of the bridge that passes information via protocol buffers over gRPC.
	3. Protocol buffer definitions for all information that flows between the Java driver and the client driver.
	4. A Java implementation of the driver client that can be used for developing unit tests and serve as a template for implementing a driver client (say in C#).
	5. Test cases to fail based unit tests that cover the Java bridge and client.

Pull Request:
    Closes apache#1466

.gitignore

@@ -11,7 +11,6 @@ target
 generated
 build
 StyleCop.Cache
-dependency-reduced-pom.xml
 #
 # ----------------------------------------------------------------------
 # IDE settings

lang/common/proto/bridge/ClientProtocol.proto

@@ -25,6 +25,73 @@ option csharp_namespace = "Org.Apache.REEF.Bridge.Proto";
 
 package driverbridge;
 
+import "DriverCommonProtocol.proto";
+
+// Bridge Client RPC and messages
+service BridgeClient {
+
+  rpc RegisterREEFClient (REEFClientRegistration) returns (Void) {}
+
+  rpc JobMessageHandler (JobMessageEvent) returns (Void) {}
+
+  rpc JobSumittedHandler (JobSubmittedEvent) returns (Void) {}
+
+  rpc JobRunningHandler (JobRunningEvent) returns (Void) {}
+
+  rpc JobCompletedHandler (JobCompletedEvent) returns (Void) {}
+
+  rpc JobFailedHandler (JobFailedEvent) returns (Void) {}
+
+  rpc RuntimeErrorHandler (ExceptionInfo) returns (Void) {}
+
+  rpc WakeErrorHandler (ExceptionInfo) returns (Void) {}
+}
+
+message REEFClientRegistration {
+  string hostname = 1;
+  uint32 port = 2;
+}
+
+message JobMessageEvent {
+  string job_id = 1;
+  bytes message = 2;
+}
+
+message JobSubmittedEvent {
+  string job_id = 1;
+}
+
+message JobRunningEvent {
+  string job_id = 1;
+}
+
+message JobCompletedEvent {
+  string job_id = 1;
+}
+
+message JobFailedEvent {
+  string job_id = 1;
+  ExceptionInfo exception = 2;
+}
+
+// REEF Client RPC and messages
+service REEFClient {
+
+  rpc DriverControlHandler (DriverControlOp) returns (Void) {}
+}
+
+message DriverControlOp {
+  string job_id = 1;
+
+  bytes message = 2;
+
+  enum Operation {
+    CLOSE = 0;
+    MESSAGE = 1;
+  }
+  Operation operation = 3;
+}
+
 /*
  * Local runtime parameters.
  */

lang/common/proto/bridge/DriverClientProtocol.proto

@@ -121,6 +121,7 @@ message StopTimeInfo {
 // Information associated with an alarm that was set.
 message AlarmTriggerInfo {
   string alarm_id = 1;
+  int64 timestamp = 2;
 }
 
 // Evaluator descriptor information.
@@ -156,6 +157,9 @@ message EvaluatorInfo {
      string message = 1;
      repeated string failed_contexts = 2;
      string failed_task_id = 3;
+
+     // Exception specific to failed evaluator
+     ExceptionInfo exception = 5;
   }
   FailureInfo failure = 2;
 

lang/common/proto/bridge/DriverServiceProtocol.proto

@@ -82,6 +82,8 @@ message ResourceRequest {
   bool relax_locality = 7;
 
   string runtime_name = 8;
+
+  string node_label = 9;
 }
 
 // Request for an alarm to be set
@@ -128,6 +130,13 @@ message AllocatedEvaluatorRequest {
     string standard_err = 4;
 
     repeated string options = 5;
+
+    enum Type {
+      JVM = 0;
+      CLR = 1;
+      DOTNET = 2;
+    }
+    Type process_type = 10;
   }
   EvaluatorProcessRequest set_process = 8;
 }

lang/cs/App.config

@@ -38,6 +38,10 @@ under the License.
       <dependentAssembly>
         <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
         <bindingRedirect oldVersion="0.0.0.0-5.8.1.0" newVersion="5.8.1.0" />
+      </dependentAssembly>
+	  <dependentAssembly>
+        <assemblyIdentity name="System.Interactive.Async" publicKeyToken="94bc3704cddfc263" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-3.0.3000.0" newVersion="3.0.3000.0" />
       </dependentAssembly>
     </assemblyBinding>
   </runtime>

lang/cs/Org.Apache.REEF.Bridge.Core.Client/Common/Config/ClientConfiguration.cs

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Org.Apache.REEF.Bridge.Core.Client.Common.Config.Parameters;
+using Org.Apache.REEF.Common.Client;
+using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Util;
+
+namespace Org.Apache.REEF.Bridge.Core.Client.Common.Config
+{
+    public sealed class ClientConfiguration : ConfigurationModuleBuilder
+    {
+
+        public static readonly OptionalParameter<IObserver<IRunningJob>> OnRunningJob = new OptionalParameter<IObserver<IRunningJob>>();
+
+        public static readonly OptionalParameter<IObserver<ISubmittedJob>> OnSubmittedJob = new OptionalParameter<IObserver<ISubmittedJob>>();
+
+        public static readonly OptionalParameter<IObserver<ICompletedJob>> OnCompletedJob = new OptionalParameter<IObserver<ICompletedJob>>();
+
+        public static readonly OptionalParameter<IObserver<IFailedJob>> OnFailedJob = new OptionalParameter<IObserver<IFailedJob>>();
+
+        public static readonly OptionalParameter<IObserver<IJobMessage>> OnJobMessage = new OptionalParameter<IObserver<IJobMessage>>();
+
+        public static readonly OptionalParameter<IObserver<IFailedRuntime>> OnFailedRuntime = new OptionalParameter<IObserver<IFailedRuntime>>();
+
+        public static readonly OptionalParameter<IObserver<IWakeError>> OnWakeError = new OptionalParameter<IObserver<IWakeError>>();
+
+        public static ConfigurationModule ConfigurationModule => new DriverBridgeConfiguration()
+                    .BindNamedParameter(GenericType<RunningJobHandler>.Class, OnRunningJob)
+                    .BindNamedParameter(GenericType<SubmittedJobHandler>.Class, OnSubmittedJob)
+                    .BindNamedParameter(GenericType<CompletedJobHandler>.Class, OnCompletedJob)
+                    .BindNamedParameter(GenericType<FailedJobHandler>.Class, OnFailedJob)
+                    .BindNamedParameter(GenericType<JobMessageHandler>.Class, OnJobMessage)
+                    .BindNamedParameter(GenericType<FailedRuntimeHandler>.Class, OnFailedRuntime)
+                    .BindNamedParameter(GenericType<WakeErrorHandler>.Class, OnWakeError)
+                    .Build();
+
+    }
+}

lang/cs/Org.Apache.REEF.Bridge.Core.Client/Common/Config/Parameters/CompletedJobHandler.cs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Org.Apache.REEF.Bridge.Core.Client.Common.Default;
+using Org.Apache.REEF.Common.Client;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Bridge.Core.Client.Common.Config.Parameters
+{
+    [NamedParameter("completed job handler", defaultClass:typeof(DefaultCompletedJobHandler))]
+    public sealed class CompletedJobHandler : Name<IObserver<ICompletedJob>>
+    {
+    }
+}

lang/cs/Org.Apache.REEF.Bridge.Core.Client/Common/Config/Parameters/FailedJobHandler.cs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Org.Apache.REEF.Bridge.Core.Client.Common.Default;
+using Org.Apache.REEF.Common.Client;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Bridge.Core.Client.Common.Config.Parameters
+{
+    [NamedParameter("failed job handler", defaultClass:typeof(DefaultFailedJobHandler))]
+    public sealed class FailedJobHandler : Name<IObserver<IFailedJob>>
+    {
+    }
+}

lang/cs/Org.Apache.REEF.Bridge.Core.Client/Common/Config/Parameters/FailedRuntimeHandler.cs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Org.Apache.REEF.Bridge.Core.Client.Common.Default;
+using Org.Apache.REEF.Common.Client;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Bridge.Core.Client.Common.Config.Parameters
+{
+    [NamedParameter("failed runtime handler", defaultClass:typeof(DefaultFailedRuntimeHandler))]
+    public sealed class FailedRuntimeHandler : Name<IObserver<IFailedRuntime>>
+    {
+    }
+}

lang/cs/Org.Apache.REEF.Bridge.Core.Client/Common/Config/Parameters/JobMessageHandler.cs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Org.Apache.REEF.Bridge.Core.Client.Common.Default;
+using Org.Apache.REEF.Common.Client;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Bridge.Core.Client.Common.Config.Parameters
+{
+    [NamedParameter("job message handler", defaultClass:typeof(DefaultJobMessageHandler))]
+    public sealed class JobMessageHandler : Name<IObserver<IJobMessage>>
+    {
+    }
+}

lang/cs/Org.Apache.REEF.Bridge.Core.Client/Common/Config/Parameters/RunningJobHandler.cs

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using System;
+using Org.Apache.REEF.Bridge.Core.Client.Common.Default;
+using Org.Apache.REEF.Common.Client;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Bridge.Core.Client.Common.Config.Parameters
+{
+    [NamedParameter("running job handler", defaultClass:typeof(DefaultRunningJobHandler))]
+    public sealed class RunningJobHandler : Name<IObserver<IRunningJob>>
+    {
+    }
+}

lang/cs/Org.Apache.REEF.Bridge.Core.Client/Common/Config/Parameters/SubmittedJobHandler.cs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Org.Apache.REEF.Bridge.Core.Client.Common.Default;
+using Org.Apache.REEF.Common.Client;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Bridge.Core.Client.Common.Config.Parameters
+{
+    [NamedParameter("submitted job handler", defaultClass:typeof(DefaultSubmittedJobHandler))]
+    public sealed class SubmittedJobHandler : Name<IObserver<ISubmittedJob>>
+    {
+    }
+}