[REEF-2025] A new module containing the new Java bridge

    This Jira introduces a new Java bridge for Drivers implemented in
    alternative languages. It provides the following artifacts (note: client
    driver refers to the application driver implemented in an alternative
    programming language):

	1. A generic framework for passing information between the Java driver and the client driver.
	2. A gRPC based implementation of the bridge that passes information via protocol buffers over gRPC.
	3. Protocol buffer definitions for all information that flows between the Java driver and the client driver.
	4. A Java implementation of the driver client that can be used for developing unit tests and serve as a template for implementing a driver client (say in C#).
	5. Test cases to fail based unit tests that cover the Java bridge and client.

Pull Request:
    Closes apache#1466

.gitignore

@@ -11,6 +11,7 @@ target
 generated
 build
 StyleCop.Cache
+dependency-reduced-pom.xml
 #
 # ----------------------------------------------------------------------
 # IDE settings

lang/common/proto/bridge/ClientProtocol.proto

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+syntax = "proto3";
+
+option java_package = "org.apache.reef.bridge.proto";
+option java_outer_classname = "ClientProtocol";
+option csharp_namespace = "Org.Apache.REEF.Bridge.Proto";
+
+package driverbridge;
+
+message LocalRuntimeParameters {
+  uint32 max_number_of_evaluators = 1;
+  string runtime_root_folder = 2;
+  float jvm_heap_slack = 3;
+  repeated string rack_names = 4;
+}
+
+message YarnRuntimeParameters {
+  string queue = 1;
+  uint32 priority = 2;
+  bool unmanged_driver = 3;
+
+  // providers
+  string filesystem_url = 5;
+  string job_submission_directory_prefix = 6;
+}
+
+message AzureBatchRuntimeParameters {
+  // Azure Batch Account Information
+  string azure_batch_account_name = 1;
+
+  string azure_batch_account_key = 2;
+
+  string azure_batch_account_uri = 3;
+
+  string azure_batch_pool_id = 4;
+
+  // Azure Storage Account Information
+  string azure_storage_account_name = 10;
+
+  string azure_storage_account_key = 11;
+
+  string azure_storage_container_name = 12;
+}
+
+message MesosRuntimeParameters {
+
+}
+
+message DriverClientConfiguration {
+  string jobid = 1;
+
+  // driver machine resources
+  uint32 cpu_cores = 2;
+  uint32 memory_mb = 3;
+
+  string driver_job_submission_directory = 4;
+
+  // the runtime on which to launch
+  oneof runtime {
+    LocalRuntimeParameters local_runtime = 5;
+    YarnRuntimeParameters yarn_runtime = 6;
+    AzureBatchRuntimeParameters azbatch_runtime = 7;
+    MesosRuntimeParameters mesos_runtime = 8;
+  }
+
+  // The command to launch the driver client
+  string driver_client_launch_command = 10;
+
+  // Enable driver restart?
+  bool driver_restart_enable = 11;
+
+  // Driver restart evaluator recovery seconds (optional)
+  uint32 driver_restart_evaluator_recovery_seconds = 12;
+
+  // TCP port range
+  uint32 tcp_port_range_begin = 15;
+  uint32 tcp_port_range_count = 16;
+  uint32 tcp_port_range_try_count = 17;
+
+  // file dependencies
+  repeated string global_files = 20;
+  repeated string local_files = 21;
+  repeated string global_libraries = 22;
+  repeated string local_libraries = 23;
+
+  // enable http driver
+  bool enable_http_driver = 25;
+
+  // General information
+  enum OS {
+    WINDOWS = 0;
+    LINUX = 1;
+  }
+  OS operating_system = 30;
+}

lang/common/proto/bridge/DriverClientProtocol.proto

@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+syntax = "proto3";
+
+// option java_generic_services = true;
+option java_multiple_files = true;
+option java_package = "org.apache.reef.bridge.proto";
+option java_outer_classname = "DriverClientProtocol";
+option csharp_namespace = "Org.Apache.REEF.Bridge.Proto";
+
+package driverbridge;
+
+import "DriverCommonProtocol.proto";
+
+// The java driver service definition.
+service DriverClient {
+  // Inquire if idle
+  rpc IdlenessCheckHandler (Void) returns (IdleStatus) {}
+
+  // Request for resources
+  rpc StartHandler (StartTimeInfo) returns (Void) {}
+
+  rpc StopHandler (StopTimeInfo) returns (ExceptionInfo) {}
+
+  rpc AlarmTrigger (AlarmTriggerInfo) returns (Void) {}
+
+  // Evaluator handlers
+  rpc AllocatedEvaluatorHandler (EvaluatorInfo) returns (Void) {}
+
+  rpc CompletedEvaluatorHandler (EvaluatorInfo) returns (Void) {}
+
+  rpc FailedEvaluatorHandler (EvaluatorInfo) returns (Void) {}
+
+  // Context handlers
+  rpc ActiveContextHandler (ContextInfo) returns (Void) {}
+
+  rpc ClosedContextHandler (ContextInfo) returns (Void) {}
+
+  rpc FailedContextHandler (ContextInfo) returns (Void) {}
+
+  rpc ContextMessageHandler (ContextMessageInfo) returns (Void) {}
+
+  // Task handlers
+  rpc RunningTaskHandler (TaskInfo) returns (Void) {}
+
+  rpc FailedTaskHandler (TaskInfo) returns (Void) {}
+
+  rpc CompletedTaskHandler (TaskInfo) returns (Void) {}
+
+  rpc SuspendedTaskHandler (TaskInfo) returns (Void) {}
+
+  rpc TaskMessageHandler (TaskMessageInfo) returns (Void) {}
+
+  // Client Handlers
+  rpc ClientMessageHandler (ClientMessageInfo) returns (Void) {}
+
+  rpc ClientCloseHandler (Void) returns (Void) {}
+
+  rpc ClientCloseWithMessageHandler (ClientMessageInfo) returns (Void) {}
+
+  // Driver Restart Handlers
+  rpc DriverRestartHandler (DriverRestartInfo) returns (Void) {}
+
+  rpc DriverRestartActiveContextHandler (ContextInfo) returns (Void) {}
+
+  rpc DriverRestartRunningTaskHandler (TaskInfo) returns (Void) {}
+
+  rpc DriverRestartCompletedHandler (DriverRestartCompletedInfo) returns (Void) {}
+
+  rpc DriverRestartFailedEvaluatorHandler (EvaluatorInfo) returns (Void) {}
+}
+
+// Driver restart information
+message DriverRestartInfo {
+  uint32 resubmission_attempts = 1;
+
+  StartTimeInfo start_time = 2;
+
+  repeated string expected_evaluator_ids = 3;
+}
+
+// Driver restart completed information
+message DriverRestartCompletedInfo {
+  StopTimeInfo completion_time = 1;
+
+  bool is_timed_out = 2;
+}
+
+// IdleStatus response to idleness inquiry
+message IdleStatus {
+  bool is_idle = 1;
+  string reason = 2;
+}
+
+// The request message containing resource request.
+message StartTimeInfo {
+  int64 start_time = 1;
+}
+
+message StopTimeInfo {
+  int64 stop_time = 1;
+}
+
+// Information associated with an alarm that was set.
+message AlarmTriggerInfo {
+  string alarm_id = 1;
+}
+
+message EvaluatorDescriptorInfo {
+  // the amount of memory allocated
+  int32 memory = 1;
+
+  // the number of virtual cores allocated
+  int32 cores = 2;
+
+  // name of the runtime
+  string runtime_name = 3;
+}
+
+message EvaluatorInfo {
+  string evaluator_id = 1;
+
+  message FailureInfo {
+     string message = 1;
+     repeated string failedContexts = 2;
+     string failedTaskId = 3;
+  }
+  FailureInfo failure = 2;
+
+  EvaluatorDescriptorInfo descriptor_info = 3;
+}
+
+message ContextInfo {
+  string context_id = 1;
+
+  string evaluator_id = 2;
+
+  string parent_id = 3;
+
+  // Carry this with us for driver restart
+  EvaluatorDescriptorInfo evaluator_descriptor_info = 4;
+
+  // Optional exception information
+  ExceptionInfo exception = 5;
+}
+
+message ContextMessageInfo {
+  string context_id = 1;
+
+  bytes payload = 2;
+
+  int64 sequence_number = 3;
+
+  string message_source_id = 4;
+}
+
+message TaskInfo {
+  // Task identifier.
+  string task_id = 1;
+
+  // Task result.
+  bytes result = 2;
+
+  /* Carry entire context info since client may not have received it
+   * when submitting task against allocated evalautor.
+   */
+  ContextInfo context = 5;
+
+  // Possible exception encountered in task execution.
+  ExceptionInfo exception = 10;
+}
+
+message TaskMessageInfo {
+  string task_id = 1;
+
+  bytes payload = 2;
+
+  int64 sequence_number = 3;
+
+  string context_id = 4;
+
+  string message_source_id = 5;
+}
+
+message ClientMessageInfo {
+  bytes payload = 1;
+}

lang/common/proto/bridge/DriverCommonProtocol.proto

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+syntax = "proto3";
+
+// option java_generic_services = true;
+option java_multiple_files = true;
+option java_package = "org.apache.reef.bridge.proto";
+option csharp_namespace = "Org.Apache.REEF.Bridge.Proto";
+
+package driverbridge;
+
+// Void message type
+message Void {}
+
+message ExceptionInfo {
+    // no error present if true
+    bool no_error = 1;
+
+    // Exception name/type
+    string name = 2;
+
+    // Exception message
+    string message = 3;
+
+    // Stack trace
+    repeated string stack_trace = 4;
+
+    // Data associated with exception
+    bytes data = 5;
+}