From c277126fb9199ea428a1581aa86721190ac8574f Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Tue, 30 Mar 2021 16:48:47 +0200 Subject: [PATCH 01/51] minor fix to sandbox agent shutdown --- SandboxAgent/sandboxagent.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/SandboxAgent/sandboxagent.py b/SandboxAgent/sandboxagent.py index e792c82..5c8a88a 100644 --- a/SandboxAgent/sandboxagent.py +++ b/SandboxAgent/sandboxagent.py @@ -269,7 +269,6 @@ def shutdown(self, reason=None): # we can't do this here, because there may be other sandboxes running the same workflow #self._management_data_layer_client.put("workflow_status_" + self._workflowid, "undeployed") - self._management_data_layer_client.shutdown() self._logger.info("Shutting down fluent-bit...") time.sleep(2) # flush interval of fluent-bit @@ -284,14 +283,18 @@ def shutdown(self, reason=None): self._frontend_process.kill() _, _ = self._frontend_process.communicate() - self._logger.info("Shutdown complete.") if reason is not None: self._update_deployment_status(True, errmsg) - self._management_data_layer_client.shutdown() - os._exit(1) else: self._update_deployment_status(False, errmsg) - self._management_data_layer_client.shutdown() + + self._management_data_layer_client.shutdown() + + self._logger.info("Shutdown complete.") + + if reason is not None: + os._exit(1) + else: os._exit(0) def _stop_deployment(self, reason, errmsg): From 1ab16329b14e488502880fcdcb3626f18d4c514a Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Tue, 30 Mar 2021 17:02:41 +0200 Subject: [PATCH 02/51] minor fix to sandbox frontend to handle shutdown properly --- Sandbox/frontend/frontend.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Sandbox/frontend/frontend.go b/Sandbox/frontend/frontend.go index 0955d28..017f07e 100644 --- a/Sandbox/frontend/frontend.go +++ b/Sandbox/frontend/frontend.go @@ -151,6 +151,12 @@ func ConsumeResults(quit <-chan bool, done chan<- bool) { Block: 0, }).Result() rt_rcvdlq = time.Now().UnixNano() + if len(res) == 0 { + continue + } else if len(res[0].Messages) == 0 { + continue + } + lqcm := res[0].Messages[0].Values lqcm_id := res[0].Messages[0].ID From c05b4902eeb6fb724aede707864220f4cd706d61 Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Tue, 30 Mar 2021 17:49:35 +0200 Subject: [PATCH 03/51] fix connect loop in DataLayerClient and LocalQueueClient --- FunctionWorker/python/DataLayerClient.py | 5 ++++- FunctionWorker/python/LocalQueueClient.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/FunctionWorker/python/DataLayerClient.py b/FunctionWorker/python/DataLayerClient.py index 345694e..365ef7c 100644 --- a/FunctionWorker/python/DataLayerClient.py +++ b/FunctionWorker/python/DataLayerClient.py @@ -69,6 +69,8 @@ def __init__(self, locality=1, sid=None, wid=None, suid=None, is_wf_private=Fals #print("Creating datalayer client in keyspace=%s, tablename=%s, maptablename=%s, settablename=%s, countertablename=%s" % (self.keyspace,self.tablename, self.maptablename, self.settablename, self.countertablename)) self.locality = locality + self._is_running = True + self.connect() if init_tables: @@ -105,7 +107,7 @@ def _drop_keyspace(self): def connect(self): retry = 0.5 #s - while True: + while self._is_running: try: host, port = self.dladdress.split(':') self.socket = TSocket.TSocket(host, int(port)) @@ -607,6 +609,7 @@ def listKeys(self, start, count, tableName=None): return keys_response def shutdown(self): + self._is_running = False try: self.transport.close() except Thrift.TException as exc: diff --git a/FunctionWorker/python/LocalQueueClient.py b/FunctionWorker/python/LocalQueueClient.py index e9ba295..3fd00ca 100644 --- a/FunctionWorker/python/LocalQueueClient.py +++ b/FunctionWorker/python/LocalQueueClient.py @@ -27,10 +27,13 @@ class LocalQueueClient: ''' def __init__(self, connect="127.0.0.1:4999"): self._qaddress = connect + + self._is_running = True + self.connect() def connect(self): - while True: + while self._is_running: try: self._queue = redis.Redis.from_url("redis://" + self._qaddress, decode_responses=True) break @@ -88,6 +91,7 @@ def getMultipleMessages(self, topic, max_count, timeout): return msg_list def shutdown(self): + self._is_running = False self._queue.close() def addTopic(self, topic): From 1b9b1cd2de8bcfb2ea905c8eeca80e4be7c0e81e Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Fri, 2 Apr 2021 10:19:08 +0200 Subject: [PATCH 04/51] minor separate test cases for interaction latency --- tests/performance_interaction_latency/test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/performance_interaction_latency/test.py b/tests/performance_interaction_latency/test.py index a2a2a02..1d8b211 100644 --- a/tests/performance_interaction_latency/test.py +++ b/tests/performance_interaction_latency/test.py @@ -35,7 +35,7 @@ def setUpClass(self): self._test_tuple_list.append((json.dumps(inp0), json.dumps(res0))) #@unittest.skip("") - def test_function_interaction_latency(self): + def test_function_interaction_latency_off(self): test = MFNTest(test_name='function_interaction_latency_checkpoints_off', workflow_filename='wf_function_interaction_latency_checkpoints_off.json') test.exec_tests(self._test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) @@ -48,6 +48,8 @@ def test_function_interaction_latency(self): test.undeploy_workflow() test.cleanup() + #@unittest.skip("") + def test_function_interaction_latency_on(self): test = MFNTest(test_name='function_interaction_latency_checkpoints_on', workflow_filename='wf_function_interaction_latency_checkpoints_on.json') test.exec_tests(self._test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) @@ -61,7 +63,7 @@ def test_function_interaction_latency(self): test.cleanup() #@unittest.skip("") - def test_chain_response_latency(self): + def test_chain_response_latency_off(self): test = MFNTest(test_name='chain_checkpoints_off', workflow_filename='wf_chain_checkpoints_off.json') print("----------------") print("Checkpoints: False") @@ -73,6 +75,8 @@ def test_chain_response_latency(self): test.undeploy_workflow() test.cleanup() + #@unittest.skip("") + def test_chain_response_latency_on(self): test = MFNTest(test_name='chain_checkpoints', workflow_filename='wf_chain_checkpoints.json') print("----------------") print("Checkpoints: True") From fd49b403e3b3601f9ed57ca450af59debc3705da Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Fri, 2 Apr 2021 14:07:47 +0200 Subject: [PATCH 05/51] replace loopback with unix domain socket for local queue with redis --- FunctionWorker/python/LocalQueueClient.py | 4 ++-- ManagementService/management_init.py | 11 +---------- ManagementService/python/deployWorkflow.py | 8 ++++---- SandboxAgent/sandboxagent.py | 11 ++++++----- redis/Makefile | 7 ++++--- 5 files changed, 17 insertions(+), 24 deletions(-) diff --git a/FunctionWorker/python/LocalQueueClient.py b/FunctionWorker/python/LocalQueueClient.py index 3fd00ca..2b37c6b 100644 --- a/FunctionWorker/python/LocalQueueClient.py +++ b/FunctionWorker/python/LocalQueueClient.py @@ -25,7 +25,7 @@ class LocalQueueClient: as the publication manager in the queue service. ''' - def __init__(self, connect="127.0.0.1:4999"): + def __init__(self, connect="/opt/mfn/redis-server/redis.sock"): self._qaddress = connect self._is_running = True @@ -35,7 +35,7 @@ def __init__(self, connect="127.0.0.1:4999"): def connect(self): while self._is_running: try: - self._queue = redis.Redis.from_url("redis://" + self._qaddress, decode_responses=True) + self._queue = redis.Redis.from_url("unix://" + self._qaddress, decode_responses=True) break except Exception as exc: if retry < 60: diff --git a/ManagementService/management_init.py b/ManagementService/management_init.py index 09b99b6..76446f7 100755 --- a/ManagementService/management_init.py +++ b/ManagementService/management_init.py @@ -425,15 +425,6 @@ def printUsage(): endpoint_list = [url] DLCLIENT_MANAGEMENT.put("management_endpoints", json.dumps(endpoint_list)) else: - #os.environ["MFN_QUEUE"] = os.getenv("QUEUE_CONNECT",hostname+":4999") - #os.environ["MFN_DATALAYER"] = os.getenv("DATALAYER_CONNECT",hostname+":4998") - #os.environ["MFN_ELASTICSEARCH"] = os.getenv("ELASTICSEARCH_CONNECT",hostname+":9200") - #os.environ["HTTP_PROXY"] = os.getenv("HTTP_PROXY", "") - #os.environ["HTTPS_PROXY"] = os.getenv("HTTPS_PROXY", "") - #os.environ["http_proxy"] = os.getenv("http_proxy", "") - #os.environ["https_proxy"] = os.getenv("https_proxy", "") - #os.environ["no_proxy"] = os.getenv("no_proxy", "") - #os.environ["NO_PROXY"] = os.getenv("NO_PROXY", "") host_to_deploy=(hostname,socket.gethostbyname(hostname)) #Container in the bare metal case will be started by start_management.sh #deployWorkflow.start_docker_sandbox(host_to_deploy, email, workflow_info["sandboxId"], workflow_info["workflowId"], workflow_info["workflowName"]) @@ -461,7 +452,7 @@ def printUsage(): f.write("SANDBOXID="+ workflow_info["sandboxId"] + "\n") f.write("WORKFLOWID="+ workflow_info["workflowId"] + "\n") f.write("WORKFLOWNAME="+ workflow_info["workflowName"] + "\n") - f.write("MFN_QUEUE=" + "127.0.0.1:4999" + "\n") + f.write("MFN_QUEUE=/opt/mfn/redis-server/redis.sock\n") f.write("MFN_DATALAYER="+ os.getenv("DATALAYER_CONNECT",hostname+":4998") + "\n") f.write("MFN_ELASTICSEARCH="+ os.getenv("ELASTICSEARCH_CONNECT",hostname+":9200") + "\n") f.write("MFN_ENDPOINT_KEY=" + endpoint_key + "\n") diff --git a/ManagementService/python/deployWorkflow.py b/ManagementService/python/deployWorkflow.py index 68b2275..cfb3d06 100644 --- a/ManagementService/python/deployWorkflow.py +++ b/ManagementService/python/deployWorkflow.py @@ -167,7 +167,7 @@ def start_docker_sandbox(host_to_deploy, uid, sid, wid, wname, sandbox_image_nam env_vars = {} env_vars["MFN_HOSTNAME"] = host_to_deploy[0] env_vars["MFN_ELASTICSEARCH"] = os.getenv("MFN_ELASTICSEARCH") - env_vars["MFN_QUEUE"] = "127.0.0.1:"+os.getenv("MFN_QUEUE").split(':')[1] + env_vars["MFN_QUEUE"] = "/opt/mfn/redis-server/redis.sock" env_vars["MFN_DATALAYER"] = host_to_deploy[0]+":"+os.getenv("MFN_DATALAYER").split(':')[1] env_vars["USERID"] = uid env_vars["SANDBOXID"] = sid @@ -698,7 +698,7 @@ def addWorkflowToTrigger(email, workflow_name, workflow_state, workflow_details, tf_ip_port = global_trigger_info["frontend_ip_port"] if tf_ip_port not in tf_hosts: raise Exception("Frontend: " + tf_ip_port + " not available") - + url = "http://" + tf_ip_port + "/add_workflows" # send the request and wait for response @@ -712,7 +712,7 @@ def addWorkflowToTrigger(email, workflow_name, workflow_state, workflow_details, res_obj = res.json() except Exception as e: status_msg = "Error: trigger_id" + trigger_id + "," + str(e) - + if "status" in res_obj and res_obj["status"].lower() == "success": # if success then update the global trigger table to add a new workflow. print("[addTriggerForWorkflow] Success response from " + url) @@ -728,7 +728,7 @@ def addWorkflowToTrigger(email, workflow_name, workflow_state, workflow_details, except Exception as e: print("[addTriggerForWorkflow] exception: " + str(e)) # TODO: why remove this? - #if 'associatedTriggers' in workflow_details and trigger_name in workflow_details['associatedTriggers']: + #if 'associatedTriggers' in workflow_details and trigger_name in workflow_details['associatedTriggers']: # associatedTriggers = workflow_details['associatedTriggers'] # del associatedTriggers[trigger_name] # workflow_details['associatedTriggers'] = associatedTriggers diff --git a/SandboxAgent/sandboxagent.py b/SandboxAgent/sandboxagent.py index 5c8a88a..14f3e07 100644 --- a/SandboxAgent/sandboxagent.py +++ b/SandboxAgent/sandboxagent.py @@ -329,11 +329,12 @@ def run(self): ts_qs_launch = time.time() # 1. launch the QueueService here self._logger.info("Launching local queue with redis...") - cmdqs = "/opt/mfn/redis-server/./redis-server /opt/mfn/redis-server/redis_4999.conf" + #cmdqs = "/opt/mfn/redis-server/./redis-server /opt/mfn/redis-server/redis_4999.conf" + cmdqs = "/opt/mfn/redis-server/./redis-server /opt/mfn/redis-server/redis_sock.conf" command_args_map_qs = {} command_args_map_qs["command"] = cmdqs - command_args_map_qs["wait_until"] = "Ready to accept connections" - error, self._queue_service_process = process_utils.run_command(cmdqs, self._logger, wait_until="Ready to accept connections") + command_args_map_qs["wait_until"] = "The server is now ready to accept connections at /opt/mfn/redis-server/redis.sock" + error, self._queue_service_process = process_utils.run_command(cmdqs, self._logger, wait_until=command_args_map_qs["wait_until"]) if error is not None: has_error = True errmsg = "Could not start the sandbox local queue: " + str(error) @@ -497,7 +498,7 @@ def find_k8s_ep(fqdn): userid = sys.argv[3] sandboxid = sys.argv[4] workflowid = sys.argv[5] - queue = "127.0.0.1:4999" + queue = "/opt/mfn/redis-server/redis.sock" datalayer = hostname + ":4998" elasticsearch = sys.argv[6] endpoint_key = sys.argv[7] @@ -505,7 +506,7 @@ def find_k8s_ep(fqdn): else: logger.info("Getting parameters from environment variables...") hostname = os.getenv("MFN_HOSTNAME", os.getenv("HOSTNAME", socket.gethostname())) - queue = os.getenv("MFN_QUEUE", "127.0.0.1:4999") + queue = os.getenv("MFN_QUEUE", "/opt/mfn/redis-server/redis.sock") datalayer = os.getenv("MFN_DATALAYER", hostname+":4998") userid = os.getenv("USERID") sandboxid = os.getenv("SANDBOXID") diff --git a/redis/Makefile b/redis/Makefile index 7e731c1..3de56dd 100644 --- a/redis/Makefile +++ b/redis/Makefile @@ -26,9 +26,10 @@ redis-server: Dockerfile_redis_build $(call build_image,Dockerfile_redis_build,redis_local_queue) docker run -i --rm -u $$(id -u):$$(id -g) -v $(CURDIR):/temp -w /temp --ulimit nofile=262144:262144 redis_local_queue:$(VERSION) cp /build/redis-6.2.1/src/redis-server redis-server/. touch redis-server/redis_4999.conf - echo "port 4999" >> redis-server/redis_4999.conf - echo "appendonly no" >> redis-server/redis_4999.conf - echo "save \"\"" >> redis-server/redis_4999.conf + echo "appendonly no" >> redis-server/redis_sock.conf + echo "save \"\"" >> redis-server/redis_sock.conf + echo "unixsocket /opt/mfn/redis-server/redis.sock" >> redis-server/redis_sock.conf + echo "unixsocketperm 700" >> redis-server/redis_sock.conf clean: rm -rf redis-server From 372e77151d1bbf343ecafafacb611391684869d9 Mon Sep 17 00:00:00 2001 From: Manuel Stein Date: Tue, 6 Apr 2021 13:21:28 +0000 Subject: [PATCH 06/51] fix(management): python deployWorkflow workflownameforlabel contained slash (invalid for k8s label) Signed-off-by: Manuel Stein --- ManagementService/python/deployWorkflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ManagementService/python/deployWorkflow.py b/ManagementService/python/deployWorkflow.py index cfb3d06..3c7dd66 100644 --- a/ManagementService/python/deployWorkflow.py +++ b/ManagementService/python/deployWorkflow.py @@ -260,7 +260,7 @@ def create_k8s_deployment(email, workflow_info, runtime, management=False): raise Exception("Unable to load "+ksvc_file+". Ensure that the configmap has been setup properly", e) # Kubernetes labels cannot contain @ or _ and should start and end with alphanumeric characters (and not be greater than 63 chars) - workflowNameForLabel = workflow_info["workflowName"].replace('@', '-').replace('_', '-').lower() + workflowNameForLabel = workflow_info["workflowName"].replace('@', '-').replace('/', '-').replace('_', '-').lower() wfNameSanitized = 'w-' + workflowNameForLabel[:59] + '-w' emailForLabel = email.replace('@', '-').replace('_', '-').lower() From 03011ebc6dc127acf547406a565c4513567a0dab Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Thu, 8 Apr 2021 11:28:40 +0200 Subject: [PATCH 07/51] Simplify and unify function worker object creation (#115) * simplify object arguments in FunctionWorker by passing the worker parameters as dict * minor fix workflow error handling test * unify object creation (state_utils, publication_utils, session_utils, sapi) in FunctionWorker in parent process but set appropriate values after fork() --- FunctionWorker/python/FunctionWorker.py | 167 ++++++++---------- FunctionWorker/python/MicroFunctionsAPI.py | 34 ++-- FunctionWorker/python/PublicationUtils.py | 108 +++++------ FunctionWorker/python/SessionHelperThread.py | 4 +- FunctionWorker/python/SessionUtils.py | 54 +++--- FunctionWorker/python/StateUtils.py | 47 +++-- Sandbox/Dockerfile_java | 12 +- SandboxAgent/deployment.py | 67 +++---- redis/Makefile | 1 - .../dynamic_parallel_execution_groups/test.py | 12 +- .../wf_mapreduce.json | 1 + .../wf_mapreduce_blocking.json | 1 + tests/echo/test.py | 7 +- tests/performance2/test.py | 12 +- tests/performance_interaction_latency/test.py | 8 +- tests/workflow_error_handling/test.py | 4 +- 16 files changed, 278 insertions(+), 261 deletions(-) diff --git a/FunctionWorker/python/FunctionWorker.py b/FunctionWorker/python/FunctionWorker.py index b9e371c..dc6c08b 100644 --- a/FunctionWorker/python/FunctionWorker.py +++ b/FunctionWorker/python/FunctionWorker.py @@ -75,13 +75,33 @@ def __init__(self, args_dict): self._POLL_TIMEOUT = py3utils.ensure_long(10000) self._set_args(args_dict) + # instead of passing individual fields, pass a bigger object to improve readibility + self._worker_params = args_dict self._prefix = self._sandboxid + "-" + self._workflowid + "-" self._wf_local = {} self._setup_loggers() - self._state_utils = StateUtils(self._function_state_type, self._function_state_name, self._function_state_info, self._function_runtime, self._logger, self._workflowid, self._sandboxid, self._function_topic, self._datalayer, self._storage_userid, self._internal_endpoint) + # set up API objects once and let the CoW handle the accesses from forked processes + self._state_utils = StateUtils(self._worker_params, self._logger) + + self._publication_utils = PublicationUtils(self._worker_params, self._state_utils, self._logger) + + self._session_utils = None + # session utils if needed + if self._is_session_workflow: + self._session_utils = SessionUtils(self._worker_params, self._publication_utils, self._logger) + + # TODO: SAPI + # pass the SessionUtils object for API calls to send a message to other running functions? + # MicroFunctionsAPI object checks before sending a message (i.e., allow only if this is_session_workflow is True) + # Maybe allow only if the destination is a session function? Requires a list of session functions and passing them to the MicroFunctionsAPI and SessionUtils + # Nonetheless, currently, MicroFunctionsAPI and SessionUtils write warning messages to the workflow log to indicate such problems + # (e.g., when this is not a workflow session or session function, when the destination running function instance does not exist) + self._sapi = MicroFunctionsAPI(self._worker_params, self._publication_utils, self._session_utils, self._usertoken, self._logger) + # need this to retrieve and publish the in-memory, transient data (i.e., stored/deleted via is_queued = True) + self._publication_utils.set_sapi(self._sapi) # check the runtime if self._function_runtime == "java": @@ -126,42 +146,46 @@ def __init__(self, args_dict): def _set_args(self, args): self._userid = args["userid"] - self._storage_userid = args["storageuserid"] + self._storage_userid = args["storage_userid"] self._sandboxid = args["sandboxid"] self._workflowid = args["workflowid"] self._workflowname = args["workflowname"] - self._function_path = args["fpath"] - self._function_name = args["fname"] - self._function_folder = args["ffolder"] - self._function_state_type = args["functionstatetype"] - self._function_state_name = args["functionstatename"] - self._function_state_info = args["functionstateinfo"] - self._function_topic = args["ftopic"] + + self._function_topic = args["function_topic"] + self._function_path = args["function_path"] + self._function_name = args["function_name"] + self._function_runtime = args["function_runtime"] + self._function_folder = args["function_folder"] + + self._function_state_type = args["function_state_type"] + self._function_state_name = args["function_state_name"] + self._function_state_info = args["function_state_info"] + self._hostname = args["hostname"] self._queue = args["queue"] self._datalayer = args["datalayer"] - self._external_endpoint = args["externalendpoint"] - self._internal_endpoint = args["internalendpoint"] - self._management_endpoints = args["managementendpoints"] - self._wf_next = args["fnext"] - self._wf_pot_next = args["fpotnext"] - self._function_runtime = args["fruntime"] + self._external_endpoint = args["external_endpoint"] + self._internal_endpoint = args["internal_endpoint"] + self._management_endpoints = args["management_endpoints"] + + self._wf_next = args["wf_next"] + self._wf_pot_next = args["wf_pot_next"] # _XXX_: also includes the workflow end point (even though it is not an actual function) - self._wf_function_list = args["workflowfunctionlist"] - self._wf_exit = args["workflowexit"] + self._wf_function_list = args["wf_function_list"] + self._wf_exit = args["wf_exit"] self._is_session_workflow = False - if args["sessionworkflow"]: + if args["is_session_workflow"]: self._is_session_workflow = True self._is_session_function = False - if args["sessionfunction"]: + if args["is_session_function"]: self._is_session_function = True - self._session_function_parameters = args["sessionfunctionparameters"] + self._session_function_parameters = args["session_function_parameters"] self._usertoken = os.environ["USERTOKEN"] - self._should_checkpoint = args["shouldcheckpoint"] + self._should_checkpoint = args["should_checkpoint"] def _setup_loggers(self): global LOGGER_HOSTNAME @@ -247,41 +271,7 @@ def _fork_and_handle_message(self, key, encapsulated_value): error_type = "" timestamp_map["t_start_pubutils"] = time.time() * 1000.0 - # 0. Setup publication utils - if not has_error: - try: - publication_utils = PublicationUtils(self._sandboxid, self._workflowid, self._function_topic, self._function_runtime, self._wf_next, self._wf_pot_next, self._wf_local, self._wf_function_list, self._wf_exit, self._should_checkpoint, self._state_utils, self._logger, self._queue, self._datalayer) - except Exception as exc: - self._logger.exception("PublicationUtils exception: %s\n%s", str(instance_pid), str(exc)) - publication_utils = None - error_type = "PublicationUtils exception" - has_error = True - # _XXX_: move the following check at the end of execution - # there we have to have the output backups, so the initialization of data layer client - # happens anyway. - # if there was an error, we'll simply not publish the output to the next function - # and stop the workflow execution there - ''' - # check the workflow stop flag - # if some other function execution had an error and we had been - # simultaneously triggered, we don't need to continue execution - timestamp_map["t_start_backdatalayer"] = time.time() * 1000.0 - if not has_error: - try: - dlc_backup = publication_utils.get_backup_data_layer_client() - timestamp_map["t_start_backdatalayer_r"] = time.time() * 1000.0 - workflow_exec_stop = dlc_backup.get("workflow_execution_stop_" + key) - if workflow_exec_stop is not None and workflow_exec_stop != "": - self._logger.info("Not continuing because workflow execution has been stopped... %s", key) - publication_utils.shutdown_backup_data_layer_client() - os._exit(0) - except Exception as exc: - self._logger.exception("PublicationUtils data layer client exception: %s\n%s", str(instance_pid), str(exc)) - publication_utils = None - error_type = "PublicationUtils data layer client exception" - has_error = True - ''' # Start of pre-processing # 1. Decapsulate the input. @@ -291,7 +281,7 @@ def _fork_and_handle_message(self, key, encapsulated_value): timestamp_map["t_start_decapsulate"] = time.time() * 1000.0 if not has_error: try: - value, metadata = publication_utils.decapsulate_input(encapsulated_value) + value, metadata = self._publication_utils.decapsulate_input(encapsulated_value) if "state_counter" not in metadata: metadata["state_counter"] = 1 else: @@ -302,9 +292,9 @@ def _fork_and_handle_message(self, key, encapsulated_value): #self._logger.debug("[FunctionWorker] Enclosed metadata:" + str(type(metadata)) + ":" + str(metadata)) # pass the metadata to the publication_utils, so that we can use it for sending immediate triggers - publication_utils.set_metadata(metadata) + self._publication_utils.set_metadata(metadata) except Exception as exc: - self._logger.exception("User input decapsulation error: %s\n%s", str(instance_pid), str(exc)) + self._logger.exception("User input decapsulation error: %s\n%s", str(key), str(exc)) error_type = "User Input Decapsulation Error" has_error = True @@ -325,10 +315,10 @@ def _fork_and_handle_message(self, key, encapsulated_value): timestamp_map["t_start_decodeinput"] = time.time() * 1000.0 if not has_error: try: - raw_state_input = publication_utils.decode_input(value) + raw_state_input = self._publication_utils.decode_input(value) #self._logger.debug("[FunctionWorker] Decoded state input:" + str(type(raw_state_input)) + ":" + str(raw_state_input)) except Exception as exc: - self._logger.exception("State Input Decoding exception: %s\n%s", str(instance_pid), str(exc)) + self._logger.exception("State Input Decoding exception: %s\n%s", str(key), str(exc)) error_type = "State Input Decoding exception" has_error = True @@ -351,7 +341,7 @@ def _fork_and_handle_message(self, key, encapsulated_value): else: function_input = raw_state_input except Exception as exc: - self._logger.exception("InputPath processing exception: %s\n%s", str(instance_pid), str(exc)) + self._logger.exception("InputPath processing exception: %s\n%s", str(key), str(exc)) error_type = "InputPath processing exception" has_error = True @@ -359,7 +349,6 @@ def _fork_and_handle_message(self, key, encapsulated_value): timestamp_map["t_start_sessutils"] = time.time() * 1000.0 # 4. Setup session related stuff here if necessary - session_utils = None if not has_error: # set up session related stuff here, if this is a session workflow/function # do this after fork(), so that we don't bottleneck the parent @@ -368,7 +357,7 @@ def _fork_and_handle_message(self, key, encapsulated_value): # TODO: 3. other metadata (e.g., direct data pipe endpoints) # 4. health check mechanism (e.g., a thread in session_utils?) # 5. Telemetry can be handled by the function instance writing to the data layer, or sending out a message immediately - # (see MicroFunctionsAPI.send_to_running_function_in_session() with send_now = True) + # (see MicroFunctionsAPI.send_to_running_function_in_session() with send_now=True) if self._is_session_workflow: # set a given session id if it is present in the incoming event # for all messages coming to a session @@ -378,32 +367,22 @@ def _fork_and_handle_message(self, key, encapsulated_value): elif "session_id" in function_input and function_input["session_id"] != "" and function_input["session_id"] is not None: session_id = function_input["session_id"] - session_utils = SessionUtils(self._hostname, self._userid, self._sandboxid, self._workflowid, self._logger, self._function_state_name, self._function_topic, key, session_id, publication_utils, self._queue, self._datalayer, self._internal_endpoint) + self._session_utils.set_key(key) + # even if session_id is None, this will initialize it and set up the necessary objects + self._session_utils.set_session_id(session_id) if self._is_session_function: try: - session_utils.setup_session_function(self._session_function_parameters) + self._session_utils.setup_session_function(self._session_function_parameters) except Exception as exc: - self._logger.exception("Session function instantiation exception: %s\n%s", str(instance_pid), str(exc)) + self._logger.exception("Session function instantiation exception: %s\n%s", str(key), str(exc)) error_type = "sessionFunctionId error" has_error = True timestamp_map["t_start_sapi"] = time.time() * 1000.0 # 5. Setup the MicroFunctionsAPI object if not has_error: - try: - # pass the SessionUtils object for API calls to send a message to other running functions? - # MicroFunctionsAPI object checks before sending a message (i.e., allow only if this is_session_workflow is True) - # Maybe allow only if the destination is a session function? Requires a list of session functions and passing them to the MicroFunctionsAPI and SessionUtils - # Nonetheless, currently, MicroFunctionsAPI and SessionUtils write warning messages to the workflow log to indicate such problems - # (e.g., when this is not a workflow session or session function, when the destination running function instance does not exist) - sapi = MicroFunctionsAPI(self._storage_userid, self._sandboxid, self._workflowid, self._function_state_name, key, publication_utils, self._is_session_workflow, self._is_session_function, session_utils, self._logger, self._datalayer, self._external_endpoint, self._internal_endpoint, self._userid, self._usertoken, self._management_endpoints) - # need this to retrieve and publish the in-memory, transient data (i.e., stored/deleted via is_queued = True) - publication_utils.set_sapi(sapi) - except Exception as exc: - self._logger.exception("MicroFunctionsAPI exception: %s\n%s", str(instance_pid), str(exc)) - error_type = "MicroFunctionsAPI exception" - has_error = True + self._sapi.set_key(key) timestamp_map["t_start"] = time.time() * 1000.0 # todo add catch retry @@ -429,9 +408,9 @@ def _fork_and_handle_message(self, key, encapsulated_value): exec_arguments = {} exec_arguments["function"] = self.code.handle exec_arguments["function_input"] = function_input - function_output = self._state_utils.exec_function_catch_retry(self._function_runtime, exec_arguments, sapi) + function_output = self._state_utils.exec_function_catch_retry(self._function_runtime, exec_arguments, self._sapi) except Exception as exc: - self._logger.exception("User code exception: %s\n%s", str(instance_pid), str(exc)) + self._logger.exception("User code exception: %s\n%s", str(key), str(exc)) sys.stdout.flush() error_type = "User code exception: " + str(exc.__class__.__name__) has_error = True @@ -439,12 +418,11 @@ def _fork_and_handle_message(self, key, encapsulated_value): else: # Processing for Non 'Task' states try: - self._logger.debug("[FunctionWorker] Before evaluateNonTaskState, input: " + str(function_input) + str(metadata)) + #self._logger.debug("[FunctionWorker] Before evaluateNonTaskState, input: " + str(function_input) + str(metadata)) #TODO: catch-retry for non-task functions? - function_output, metadata_updated = self._state_utils.evaluateNonTaskState(function_input, key, metadata, sapi) - metadata = metadata_updated + function_output, metadata_updated = self._state_utils.evaluateNonTaskState(function_input, key, metadata, self._sapi) # update metadata in the publication utils - publication_utils.set_metadata(metadata) + self._publication_utils.set_metadata(metadata_updated) #self._logger.debug("[FunctionWorker] After evaluateNonTaskState, result: " + str(function_output) + str(function_input)) except Exception as exc: @@ -472,7 +450,7 @@ def _fork_and_handle_message(self, key, encapsulated_value): exec_arguments["function_input"] = java_input - function_output = self._state_utils.exec_function_catch_retry(self._function_runtime, exec_arguments, sapi) + function_output = self._state_utils.exec_function_catch_retry(self._function_runtime, exec_arguments, self._sapi) timestamp_map["t_end"] = timestamp_map["t_start_resultpath"] = time.time() * 1000.0 @@ -506,7 +484,7 @@ def _fork_and_handle_message(self, key, encapsulated_value): value_output = 'null' if not has_error: try: - value_output = publication_utils.encode_output(raw_state_output) + value_output = self._publication_utils.encode_output(raw_state_output) #self._logger.debug("[FunctionWorker] Encoded state output:" + str(type(value_output)) + ":" + value_output) except Exception as exc: self._logger.exception("State Output Encoding exception: %s\n%s", str(instance_pid), str(exc)) @@ -517,7 +495,7 @@ def _fork_and_handle_message(self, key, encapsulated_value): timestamp_map["t_start_branchterminal"] = time.time() * 1000.0 if not has_error: try: - self._state_utils.processBranchTerminalState(key, value_output, metadata, sapi) # not supposed to have a return value + self._state_utils.processBranchTerminalState(key, value_output, metadata, self._sapi) # not supposed to have a return value except Exception as exc: self._logger.exception("ProcessBranchTerminalState: %s\n%s", str(instance_pid), str(exc)) error_type = "ProcessBranchTerminalState exception" @@ -536,15 +514,15 @@ def _fork_and_handle_message(self, key, encapsulated_value): # need a way to sync the cleanup of the local queue client? # 1. shutdown the helper thread before publishing # 2. ensure in the helper thread no other heartbeat is published when it just exits the polling loop - if session_utils is not None and self._is_session_function: - session_utils.shutdown_helper_thread() + if self._session_utils is not None and self._is_session_function: + self._session_utils.shutdown_helper_thread() - if publication_utils is not None: - publication_utils.publish_output_direct(key, value_output, has_error, error_type, timestamp_map) + if self._publication_utils is not None: + self._publication_utils.publish_output_direct(key, value_output, has_error, error_type, timestamp_map) # remove session function metadata from the session metadata tables if this is a session function - if session_utils is not None and self._is_session_function: - session_utils.cleanup() + if self._session_utils is not None and self._is_session_function: + self._session_utils.cleanup() os._exit(0) @@ -587,6 +565,7 @@ def _process_update(self, value): self.shutdown() elif action == "update-local-functions": self._wf_local = update["localFunctions"] + self._publication_utils.set_workflow_local_functions(self._wf_local) except Exception as exc: self._logger.error("Could not parse update message: %s; ignored...", str(exc)) diff --git a/FunctionWorker/python/MicroFunctionsAPI.py b/FunctionWorker/python/MicroFunctionsAPI.py index 8dd49a0..65e5f60 100644 --- a/FunctionWorker/python/MicroFunctionsAPI.py +++ b/FunctionWorker/python/MicroFunctionsAPI.py @@ -62,7 +62,7 @@ class MicroFunctionsAPI: - session customization ''' - def __init__(self, uid, sid, wid, funcstatename, key, publication_utils, is_session_workflow, is_session_function, session_utils, logger, datalayer, external_endpoint, internal_endpoint, useremail, usertoken, management_endpoints): + def __init__(self, worker_params, publication_utils, session_utils, usertoken, logger): ''' Initialize data structures for MicroFunctionsAPI object created for a function instance. @@ -84,32 +84,31 @@ def __init__(self, uid, sid, wid, funcstatename, key, publication_utils, is_sess ''' self._logger = logger - self._datalayer = datalayer - self._external_endpoint = external_endpoint - self._internal_endpoint = internal_endpoint - self._management_endpoints = management_endpoints - self._useremail = useremail + self._datalayer = worker_params["datalayer"] + self._external_endpoint = worker_params["external_endpoint"] + self._internal_endpoint = worker_params["internal_endpoint"] + self._management_endpoints = worker_params["management_endpoints"] self._usertoken = usertoken - self._sid = sid - self._wid = wid + self._sid = worker_params["sandboxid"] + self._wid = worker_params["workflowid"] + self._suid = worker_params["storage_userid"] - self._data_layer_operator = DataLayerOperator( - uid, sid, wid, self._datalayer) + self._data_layer_operator = DataLayerOperator(self._suid, self._sid, self._wid, self._datalayer) # for sending immediate triggers to other functions self._publication_utils = publication_utils - self._is_session_workflow = is_session_workflow - self._is_session_function = is_session_function + self._is_session_workflow = worker_params["is_session_workflow"] + self._is_session_function = worker_params["is_session_function"] self._session_utils = session_utils - self._function_state_name = funcstatename + self._function_state_name = worker_params["function_state_name"] self._function_version = 1 # Currently hardcoded to 1 - self._instanceid = key + self._instanceid = None self._is_privileged = False - if sid == "Management" and wid == "Management": + if self._sid == "Management" and self._wid == "Management": self._is_privileged = True ''' @@ -161,6 +160,11 @@ def __init__(self, uid, sid, wid, funcstatename, key, publication_utils, is_sess #self._logger.debug("[MicroFunctionsAPI] init done.") + def set_key(self, key): + self._instanceid = key + # The AWS identifier of the invocation request. Return the KNIX message key instead + self.aws_request_id = self._instanceid + def get_context_object_properties(self): context_properties = {} context_properties["function_name"] = self.function_name diff --git a/FunctionWorker/python/PublicationUtils.py b/FunctionWorker/python/PublicationUtils.py index 291bae3..f3004a3 100644 --- a/FunctionWorker/python/PublicationUtils.py +++ b/FunctionWorker/python/PublicationUtils.py @@ -26,25 +26,25 @@ import py3utils class PublicationUtils(): - def __init__(self, sandboxid, workflowid, functopic, funcruntime, wfnext, wfpotnext, wflocal, wflist, wfexit, cpon, stateutils, logger, queue, datalayer): + def __init__(self, worker_params, state_utils, logger): self._logger = logger - self._function_topic = functopic - self._sandboxid = sandboxid - self._workflowid = workflowid + self._function_topic = worker_params["function_topic"] + self._workflowid = worker_params["workflowid"] + self._sandboxid = worker_params["sandboxid"] - self._function_runtime = funcruntime + self._function_runtime = worker_params["function_runtime"] self._prefix = self._sandboxid + "-" + self._workflowid + "-" - self._wf_next = wfnext - self._wf_pot_next = wfpotnext - self._wf_local = wflocal - self._wf_function_list = wflist - self._wf_exit = wfexit + self._wf_next = worker_params["wf_next"] + self._wf_pot_next = worker_params["wf_pot_next"] + self._wf_local = {} + self._wf_function_list = worker_params["wf_function_list"] + self._wf_exit = worker_params["wf_exit"] # whether we should store backups of triggers before publishing the output - self._should_checkpoint = cpon + self._should_checkpoint = worker_params["should_checkpoint"] # the topic to send out messages to remote functions # TODO: pub_topic_global becomes a new request to another sandbox? @@ -53,12 +53,14 @@ def __init__(self, sandboxid, workflowid, functopic, funcruntime, wfnext, wfpotn self._recovery_manager_topic = "RecoveryManager" - self._state_utils = stateutils + self._state_utils = state_utils self._metadata = None - self._queue = queue + self._queue = worker_params["queue"] + self._datalayer = worker_params["datalayer"] + self._local_queue_client = None - self._datalayer = datalayer + self._backup_data_layer_client = None self._sapi = None @@ -66,12 +68,14 @@ def __init__(self, sandboxid, workflowid, functopic, funcruntime, wfnext, wfpotn self._dynamic_workflow = [] - self._backup_data_layer_client = None self._execution_info_map_name = None self._next_backup_list = [] #self._logger.debug("[PublicationUtils] init done.") + def set_workflow_local_functions(self, wf_local): + self._wf_local = wf_local + # only to be called from the function worker def set_sapi(self, sapi): self._sapi = sapi @@ -97,13 +101,12 @@ def _shutdown_local_queue_client(self): if self._local_queue_client is not None: self._local_queue_client.shutdown() - def get_backup_data_layer_client(self): + def _get_backup_data_layer_client(self): if self._backup_data_layer_client is None: - # locality = -1 means that the writes happen to the local data layer first and then asynchronously to the global data layer - self._backup_data_layer_client = DataLayerClient(locality=-1, for_mfn=True, sid=self._sandboxid, connect=self._datalayer) + self._backup_data_layer_client = DataLayerClient(locality=0, for_mfn=True, sid=self._sandboxid, connect=self._datalayer) return self._backup_data_layer_client - def shutdown_backup_data_layer_client(self): + def _shutdown_backup_data_layer_client(self): if self._backup_data_layer_client is not None: self._backup_data_layer_client.shutdown() @@ -364,7 +367,7 @@ def _generate_trigger_metadata(self, topic_next): # get current state type. if map state add marker to execution Id state_type = self._state_utils.functionstatetype - self._logger.debug("self._state_utils.functionstatetype: " + str(state_type)) + #self._logger.debug("self._state_utils.functionstatetype: " + str(state_type)) if state_type == 'Map': next_function_execution_id = self._metadata["__function_execution_id"] + "_" + str(output_instance_id)+"-M" @@ -417,31 +420,25 @@ def _publish_output(self, key, trigger, lqcpub, timestamp_map=None): timestamp_map['t_pub_localqueue'] = time.time() * 1000.0 self._send_local_queue_message(lqcpub, topic_next, key, output["value"]) else: + # TODO: need to also ensure that a message to a non-local topic gets properly handled + # for multi-host deployments for load redirection + # currently, we don't have such cases + self._send_local_queue_message(lqcpub, topic_next, key, output["value"]) + # check if 'next' is exit topic if next == self._wf_exit: + if timestamp_map is not None: + timestamp_map['t_pub_exittopic'] = time.time() * 1000.0 + timestamp_map['exitsize'] = len(output["value"]) + key = self._metadata["__execution_id"] - dlc = self.get_backup_data_layer_client() + dlc = self._get_backup_data_layer_client() # store the workflow's final result dlc.put("result_" + key, output["value"]) #self._logger.debug("[__mfn_backup] [exitresult] [%s] %s", "result_" + key, output["value"]) - # _XXX_: this is not handled properly by the frontend - # this was an async execution - # just send an empty message to the frontend to signal end of execution - #if "__async_execution" in self._metadata and self._metadata["__async_execution"]: - # output["value"] = "" - - if timestamp_map is not None: - timestamp_map['t_pub_exittopic'] = time.time() * 1000.0 - timestamp_map['exitsize'] = len(output["value"]) - - # TODO: need to also ensure that a message to a non-local topic gets properly handled - # for multi-host deployments for load redirection - # currently, we don't have such cases - self._send_local_queue_message(lqcpub, topic_next, key, output["value"]) - return (next_function_execution_id, output) # keep track of the execution instances with their updated keys @@ -509,11 +506,12 @@ def send_to_function_now(self, key, trigger, lqcpub=None): self._log_trigger_backups(input_backup_map, current_function_instance_id, store_next_backup_list=any_next) - for next_func_exec_id in starting_next: - next_func_topic = starting_next[next_func_exec_id] - self._send_message_to_recovery_manager(key, "start", next_func_topic, next_func_exec_id, False, "", lqcpub) + # TODO: recovery manager; not used at this time + #for next_func_exec_id in starting_next: + # next_func_topic = starting_next[next_func_exec_id] + # self._send_message_to_recovery_manager(key, "start", next_func_topic, next_func_exec_id, False, "", lqcpub) - self._send_message_to_recovery_manager(key, "running", self._function_topic, self._metadata["__function_execution_id"], False, "", lqcpub) + #self._send_message_to_recovery_manager(key, "running", self._function_topic, self._metadata["__function_execution_id"], False, "", lqcpub) # utilize the workflow to publish directly to the next function's topic # publish directly to the next function's topic, accumulate backups @@ -537,7 +535,7 @@ def publish_output_direct(self, key, value_output, has_error, error_type, timest if has_error: timestamp_map["t_start_dlcbackup"] = time.time() * 1000.0 - dlc = self.get_backup_data_layer_client() + dlc = self._get_backup_data_layer_client() # set data layer flag to stop further execution of function instances # that may have been triggered concurrently via a new message @@ -571,10 +569,6 @@ def publish_output_direct(self, key, value_output, has_error, error_type, timest timestamp_map["t_start_resultmap"] = time.time() * 1000.0 self._logger.info("[__mfn_backup] [%s] [%s] %s", self._execution_info_map_name, "result_" + current_function_instance_id, encapsulated_value_output) - timestamp_map["t_start_storeoutput"] = time.time() * 1000.0 - # store self._sapi.transient_output into the data layer - self._store_output_data() - # get the combined (next, value) tuple list for the output # use here the original output: # we'll update the metadata separately for each trigger and encapsulate the output with it @@ -589,16 +583,20 @@ def publish_output_direct(self, key, value_output, has_error, error_type, timest if len(converted_function_output) == 1 and converted_function_output[0]["next"] == self._wf_exit: check_error_flag = False + timestamp_map["t_start_storeoutput"] = time.time() * 1000.0 + # store self._sapi.transient_output into the data layer + self._store_output_data() + if check_error_flag: timestamp_map["t_start_dlcbackup_err"] = time.time() * 1000.0 - dlc = self.get_backup_data_layer_client() + dlc = self._get_backup_data_layer_client() # check the workflow stop flag # if some other function execution had an error and we had been # simultaneously triggered, we can finish but don't need to publish # to the next function in the workflow, so we can stop execution of the workflow timestamp_map["t_start_dlcbackup_err_flag"] = time.time() * 1000.0 - workflow_exec_stop = dlc.get("workflow_execution_stop_" + key, locality=0) - if workflow_exec_stop is not None and workflow_exec_stop != "": + workflow_exec_stop = dlc.get("workflow_execution_stop_" + key) + if workflow_exec_stop == "1": self._logger.info("Not continuing because workflow execution has been stopped... %s", key) continue_publish_flag = False @@ -638,13 +636,15 @@ def publish_output_direct(self, key, value_output, has_error, error_type, timest # backups for next of successfully completed function execution instances self._log_trigger_backups(input_backup_map, current_function_instance_id, store_next_backup_list=any_next) - for next_func_exec_id in starting_next: - next_func_topic = starting_next[next_func_exec_id] - self._send_message_to_recovery_manager(key, "start", next_func_topic, next_func_exec_id, False, "", lqcpub) + # TODO: recovery manager; not used at this time + #for next_func_exec_id in starting_next: + # next_func_topic = starting_next[next_func_exec_id] + # self._send_message_to_recovery_manager(key, "start", next_func_topic, next_func_exec_id, False, "", lqcpub) - if self._should_checkpoint: + # TODO: recovery manager; not used at this time + #if self._should_checkpoint: # regardless whether this function execution had an error or not, we are finished and need to let the recovery manager know - self._send_message_to_recovery_manager(key, "finish", self._function_topic, self._metadata["__function_execution_id"], has_error, error_type, lqcpub) + #self._send_message_to_recovery_manager(key, "finish", self._function_topic, self._metadata["__function_execution_id"], has_error, error_type, lqcpub) # log the timestamps timestamp_map["t_pub_end"] = timestamp_map["t_end_pub"] = timestamp_map["t_end_fork"] = time.time() * 1000.0 @@ -658,5 +658,5 @@ def publish_output_direct(self, key, value_output, has_error, error_type, timest # shut down the local queue client self._shutdown_local_queue_client() - self.shutdown_backup_data_layer_client() + self._shutdown_backup_data_layer_client() diff --git a/FunctionWorker/python/SessionHelperThread.py b/FunctionWorker/python/SessionHelperThread.py index 1902ab7..c910c6a 100644 --- a/FunctionWorker/python/SessionHelperThread.py +++ b/FunctionWorker/python/SessionHelperThread.py @@ -86,7 +86,7 @@ def __init__(self, helper_params, logger, pubutils, sessutils, queueservice, dat def _init_heartbeat_parameters(self, heartbeat_params): if "heartbeat_method" not in heartbeat_params: - self._logger.debug("No heartbeat method is specified; disabling heartbeat.") + #self._logger.debug("No heartbeat method is specified; disabling heartbeat.") return else: self._heartbeat_enabled = True @@ -214,7 +214,7 @@ def _process_message(self, lqm): except Exception as exc: is_json = False msg = value - self._logger.debug("[SessionHelperThread] non-JSON value: " + str(msg)) + #self._logger.debug("[SessionHelperThread] non-JSON value: " + str(msg)) # cannot be a special message; queue whatever it is # _XXX_: we are encoding/decoding the delivered message; should not actually execute this code diff --git a/FunctionWorker/python/SessionUtils.py b/FunctionWorker/python/SessionUtils.py index e2886db..731fe51 100644 --- a/FunctionWorker/python/SessionUtils.py +++ b/FunctionWorker/python/SessionUtils.py @@ -22,29 +22,25 @@ import json from DataLayerClient import DataLayerClient -from LocalQueueClient import LocalQueueClient from SessionHelperThread import SessionHelperThread class SessionUtils: - def __init__(self, hostname, uid, sid, wid, logger, funcstatename, functopic, key, session_id, publication_utils, queue, datalayer, internal_endpoint): - + def __init__(self, worker_params, publication_utils, logger): self._logger = logger - self._queue = queue - self._datalayer = datalayer + self._queue = worker_params["queue"] + self._datalayer = worker_params["datalayer"] - self._session_id = session_id self._session_function_id = None - self._hostname = hostname - self._userid = uid - self._sandboxid = sid - self._workflowid = wid - self._function_state_name = funcstatename - self._function_topic = functopic - self._internal_endpoint = internal_endpoint - self._key = key + self._hostname = worker_params["hostname"] + self._userid = worker_params["userid"] + self._sandboxid = worker_params["sandboxid"] + self._workflowid = worker_params["workflowid"] + self._function_state_name = worker_params["function_state_name"] + self._function_topic = worker_params["function_topic"] + self._internal_endpoint = worker_params["internal_endpoint"] self._publication_utils = publication_utils @@ -52,17 +48,16 @@ def __init__(self, hostname, uid, sid, wid, logger, funcstatename, functopic, ke self._helper_thread = None - self._global_data_layer_client = DataLayerClient(locality=1, sid=sid, for_mfn=True, connect=self._datalayer) + self._global_data_layer_client = None # only valid if this is a session function (i.e., session_function_id is not None) self._local_topic_communication = None self._session_function_parameters = None - if self._session_id is None: - self._generate_session_id() - - self._setup_metadata_tablenames() + # to be set later + self._key = None + self._session_id = None # _XXX_: the following does not have any effect and makes unnecessary calls # to the data layer @@ -74,6 +69,19 @@ def __init__(self, hostname, uid, sid, wid, logger, funcstatename, functopic, ke #self._logger.debug("[SessionUtils] init done.") + def set_key(self, key): + self._key = key + + def set_session_id(self, session_id=None): + if session_id is None: + self._generate_session_id() + else: + self._session_id = session_id + + self._setup_metadata_tablenames() + + self._global_data_layer_client = DataLayerClient(locality=1, sid=self._sandboxid, for_mfn=True, connect=self._datalayer) + ########################### # @@ -246,7 +254,7 @@ def _generate_session_id(self): # due to key being different for each request to the workflow plain_session_id_bytes = (self._userid + "_" + self._sandboxid + "_" + self._workflowid + "_" + self._key).encode() self._session_id = hashlib.sha256(plain_session_id_bytes).hexdigest() - self._logger.debug("[SessionUtils] Session id: " + self._session_id) + #self._logger.debug("[SessionUtils] Session id: " + self._session_id) def _generate_session_function_id(self): if self._session_function_id is None: @@ -257,7 +265,7 @@ def _generate_session_function_id(self): random.seed() plain_session_function_id_bytes = (self._function_state_name + "_" + self._key + "_" + str(random.uniform(0, 100000))).encode() self._session_function_id = hashlib.sha256(plain_session_function_id_bytes).hexdigest() - self._logger.debug("[SessionUtils] Session function id: " + self._session_function_id) + #self._logger.debug("[SessionUtils] Session function id: " + self._session_function_id) # these calls don't have an effect until an entry is added # and the entries still succeed even without calling to createSet or createMap @@ -378,7 +386,8 @@ def _setup_session_function_helper(self): self._helper_thread.start() def shutdown_helper_thread(self): - self._helper_thread.shutdown() + if self._helper_thread is not None: + self._helper_thread.shutdown() def cleanup(self): self._remove_metadata() @@ -483,4 +492,3 @@ def get_session_update_messages_with_local_queue(self, count=1, block=False): messages = self._helper_thread.get_messages(count=count, block=block) return messages return None - diff --git a/FunctionWorker/python/StateUtils.py b/FunctionWorker/python/StateUtils.py index 977f963..13134ca 100644 --- a/FunctionWorker/python/StateUtils.py +++ b/FunctionWorker/python/StateUtils.py @@ -46,7 +46,7 @@ class StateUtils: mapFunctionOutput = {} - def __init__(self, functionstatetype=defaultStateType, functionstatename='', functionstateinfo='{}', functionruntime="", logger=None, workflowid=None, sandboxid=None, functiontopic=None, datalayer=None, storage_userid=None, internal_endpoint=None): + def __init__(self, worker_params, logger=None): self.operators = ['And', 'BooleanEquals', 'Not', 'NumericEquals', 'NumericGreaterThan', 'NumericGreaterThanEquals',\ 'NumericLessThan', 'NumericLessThanEquals', 'Or', 'StringEquals', 'StringGreaterThan',\ 'StringGreaterThanEquals', 'StringLessThan', 'StringLessThanEquals', 'TimestampEquals', 'TimestampGreaterThan',\ @@ -67,21 +67,34 @@ def __init__(self, functionstatetype=defaultStateType, functionstatename='', fun self.result_path_dict = {} self.output_path_dict = {} self.parameters_dict = {} - self.functionstatetype = functionstatetype - self.functionstatename = functionstatename - self.functionstateinfo = functionstateinfo - self.functiontopic = functiontopic - self._datalayer = datalayer - self._storage_userid = storage_userid - self._internal_endpoint = internal_endpoint - self._function_runtime = functionruntime + + if "function_state_type" in worker_params: + self.functionstatetype = worker_params["function_state_type"] + else: + self.functionstatetype = defaultStateType + + if "function_state_name" in worker_params: + self.functionstatename = worker_params["function_state_name"] + else: + self.functionstatename = "" + + if "function_state_info" in worker_params: + self.functionstateinfo = worker_params["function_state_info"] + else: + self.functionstateinfo = "{}" + + self.functiontopic = worker_params["function_topic"] + self._datalayer = worker_params["datalayer"] + self._storage_userid = worker_params["storage_userid"] + self._internal_endpoint = worker_params["internal_endpoint"] + self._function_runtime = worker_params["function_runtime"] if self._function_runtime == "java": # if java, this is the address we'll send requests to be handled self._java_handler_address = "/tmp/java_handler_" + self.functionstatename + ".uds" self.parsedfunctionstateinfo = {} - self.workflowid = workflowid - self.sandboxid = sandboxid + self.workflowid = worker_params["workflowid"] + self.sandboxid = worker_params["sandboxid"] self.choiceNext = '' self.mapStateCounter = 0 @@ -666,16 +679,16 @@ def evaluateParallelState(self, function_input, key, metadata, sapi): if "WaitForNumBranches" in self.parsedfunctionstateinfo: klist = self.parsedfunctionstateinfo["WaitForNumBranches"] if not isinstance(klist, list): - self._logger.info("(StateUtils) WaitForNumBranches must be a sorted list with 1 or more integers") - raise Exception("(StateUtils) WaitForNumBranches must be a sorted list with 1 or more integers") + self._logger.info("[StateUtils] WaitForNumBranches must be a sorted list with 1 or more integers") + raise Exception("[StateUtils] WaitForNumBranches must be a sorted list with 1 or more integers") klist.sort() for k in klist: if not isinstance(k, int): - self._logger.info("(StateUtils) Values inside WaitForNumBranches must be integers") - raise Exception("(StateUtils) Values inside WaitForNumBranches must be integers") + self._logger.info("[StateUtils] Values inside WaitForNumBranches must be integers") + raise Exception("[StateUtils] Values inside WaitForNumBranches must be integers") if k > total_branch_count: - self._logger.info("(StateUtils) Values inside WaitForNumBranches list cannot be greater than the number of branches in the parallel state") - raise Exception("(StateUtils) Values inside WaitForNumBranches list cannot be greater than the number of branches in the parallel state") + self._logger.info("[StateUtils] Values inside WaitForNumBranches list cannot be greater than the number of branches in the parallel state") + raise Exception("[StateUtils] Values inside WaitForNumBranches list cannot be greater than the number of branches in the parallel state") else: klist.append(total_branch_count) diff --git a/Sandbox/Dockerfile_java b/Sandbox/Dockerfile_java index 83e45b9..b1d6246 100644 --- a/Sandbox/Dockerfile_java +++ b/Sandbox/Dockerfile_java @@ -47,6 +47,12 @@ RUN apt-get -y --no-install-recommends install maven RUN groupadd -o -g 1000 -r mfn && useradd -d /opt/mfn -u 1000 -m -r -g mfn mfn RUN mkdir /opt/mfn/logs +ADD build/JavaRequestHandler.tar.gz /opt/mfn/ + +RUN chmod +x /opt/mfn/JavaRequestHandler/setup_maven.sh +RUN /opt/mfn/JavaRequestHandler/./setup_maven.sh True +RUN mvn -Duser.home=/tmp -DskipTests -gs /opt/mfn/JavaRequestHandler/maven/sandbox-mvn-settings.xml -f /opt/mfn/JavaRequestHandler/maven/init-mvn.pom.xml dependency:resolve-plugins + RUN /usr/bin/python3 -m pip install redis ADD build/redis-server.tar.gz /opt/mfn/ ADD frontend/frontend /opt/mfn/frontend @@ -54,12 +60,6 @@ ADD build/SandboxAgent.tar.gz /opt/mfn/ ADD build/FunctionWorker.tar.gz /opt/mfn/ ADD build/LoggingService.tar.gz /opt/mfn/ -ADD build/JavaRequestHandler.tar.gz /opt/mfn/ - -RUN chmod +x /opt/mfn/JavaRequestHandler/setup_maven.sh -RUN /opt/mfn/JavaRequestHandler/./setup_maven.sh True -RUN mvn -Duser.home=/tmp -DskipTests -gs /opt/mfn/JavaRequestHandler/maven/sandbox-mvn-settings.xml -f /opt/mfn/JavaRequestHandler/maven/init-mvn.pom.xml dependency:resolve-plugins - RUN chown mfn:mfn -R /opt/mfn USER mfn WORKDIR /opt/mfn diff --git a/SandboxAgent/deployment.py b/SandboxAgent/deployment.py index 788bfa6..d028e91 100644 --- a/SandboxAgent/deployment.py +++ b/SandboxAgent/deployment.py @@ -233,8 +233,8 @@ def _wait_for_child_processes(self): def _start_python_function_worker(self, worker_params, env_var_list): error = None - function_name = worker_params["fname"] - state_name = worker_params["functionstatename"] + function_name = worker_params["function_name"] + state_name = worker_params["function_state_name"] custom_env = os.environ.copy() old_ld_library_path = "" if "LD_LIBRARY_PATH" in custom_env: @@ -295,13 +295,13 @@ def _start_function_worker(self, worker_params, runtime, env_var_list): # if jar, the contents have already been extracted as if it was a zip archive # start the java request handler if self._function_runtime == "java" # we wrote the parameters to json file at the state directory - self._logger.info("Launching JavaRequestHandler for state: %s", worker_params["functionstatename"]) + self._logger.info("Launching JavaRequestHandler for state: %s", worker_params["function_state_name"]) cmdjavahandler = "java -jar /opt/mfn/JavaRequestHandler/target/javaworker.jar " - cmdjavahandler += "/opt/mfn/workflow/states/" + worker_params["functionstatename"] + "/java_worker_params.json" + cmdjavahandler += "/opt/mfn/workflow/states/" + worker_params["function_state_name"] + "/java_worker_params.json" error, process = process_utils.run_command(cmdjavahandler, self._logger, wait_until="Waiting for requests on:") if error is not None: - error = "Could not launch JavaRequestHandler: " + worker_params["fname"] + " " + error + error = "Could not launch JavaRequestHandler: " + worker_params["function_name"] + " " + error self._logger.error(error) else: self._javarequesthandler_process_list.append(process) @@ -598,32 +598,39 @@ def _initialize_data_layer_storage(self): def _populate_worker_params(self, function_topic, wf_node, state): worker_params = {} worker_params["userid"] = self._userid - worker_params["storageuserid"] = self._storage_userid + worker_params["storage_userid"] = self._storage_userid worker_params["sandboxid"] = self._sandboxid worker_params["workflowid"] = self._workflowid worker_params["workflowname"] = self._workflowname - worker_params["ffolder"] = state["resource_dirpath"] - worker_params["fpath"] = state["resource_filepath"] - worker_params["fname"] = state["resource_filename"] - worker_params["fruntime"] = state["resource_runtime"] - worker_params["ftopic"] = function_topic + + worker_params["function_topic"] = function_topic + worker_params["function_path"] = state["resource_filepath"] + worker_params["function_name"] = state["resource_filename"] + worker_params["function_folder"] = state["resource_dirpath"] + worker_params["function_runtime"] = state["resource_runtime"] + + worker_params["function_state_type"] = wf_node.getGWFType() + worker_params["function_state_name"] = wf_node.getGWFStateName() + worker_params["function_state_info"] = wf_node.getGWFStateInfo() + worker_params["hostname"] = self._hostname worker_params["queue"] = self._queue worker_params["datalayer"] = self._datalayer - worker_params["externalendpoint"] = self._external_endpoint - worker_params["internalendpoint"] = self._internal_endpoint - worker_params["managementendpoints"] = self._management_endpoints - worker_params["fnext"] = wf_node.getNextMap() - worker_params["fpotnext"] = wf_node.getPotentialNextMap() - worker_params["functionstatetype"] = wf_node.getGWFType() - worker_params["functionstatename"] = wf_node.getGWFStateName() - worker_params["functionstateinfo"] = wf_node.getGWFStateInfo() - worker_params["workflowfunctionlist"] = self._workflow.getWorkflowFunctionMap() - worker_params["workflowexit"] = self._workflow.getWorkflowExitPoint() - worker_params["sessionworkflow"] = self._workflow.is_session_workflow() - worker_params["sessionfunction"] = wf_node.is_session_function() - worker_params["sessionfunctionparameters"] = wf_node.get_session_function_parameters() - worker_params["shouldcheckpoint"] = self._workflow.are_checkpoints_enabled() + worker_params["external_endpoint"] = self._external_endpoint + worker_params["internal_endpoint"] = self._internal_endpoint + worker_params["management_endpoints"] = self._management_endpoints + + worker_params["wf_next"] = wf_node.getNextMap() + worker_params["wf_pot_next"] = wf_node.getPotentialNextMap() + worker_params["wf_function_list"] = self._workflow.getWorkflowFunctionMap() + worker_params["wf_exit"] = self._workflow.getWorkflowExitPoint() + worker_params["wf_entry"] = self._workflow.getWorkflowEntryTopic() + + worker_params["is_session_workflow"] = self._workflow.is_session_workflow() + worker_params["is_session_function"] = wf_node.is_session_function() + worker_params["session_function_parameters"] = wf_node.get_session_function_parameters() + + worker_params["should_checkpoint"] = self._workflow.are_checkpoints_enabled() return worker_params @@ -922,13 +929,13 @@ def process_deployment_info(self): if state["resource_runtime"].find("java") != -1: java_worker_params = {} - java_worker_params["functionPath"] = worker_params["ffolder"] - java_worker_params["functionName"] = worker_params["fname"] - java_worker_params["serverSocketFilename"] = "/tmp/java_handler_" + worker_params["functionstatename"] + ".uds" + java_worker_params["functionPath"] = worker_params["function_folder"] + java_worker_params["functionName"] = worker_params["function_name"] + java_worker_params["serverSocketFilename"] = "/tmp/java_handler_" + worker_params["function_state_name"] + ".uds" if SINGLE_JVM_FOR_FUNCTIONS: any_java_function = True - single_jvm_worker_params[worker_params["functionstatename"]] = java_worker_params + single_jvm_worker_params[worker_params["function_state_name"]] = java_worker_params else: java_params_filename = state["dirpath"] + "java_worker_params.json" with open(java_params_filename, "w") as javaparamsf: @@ -938,7 +945,7 @@ def process_deployment_info(self): error = self._start_function_worker(worker_params, state["resource_runtime"], state["resource_env_var_list"]) if error is not None: - errmsg = "Problem launching function worker for: " + worker_params["fname"] + errmsg = "Problem launching function worker for: " + worker_params["function_name"] self._logger.error(errmsg) has_error = True return has_error, errmsg diff --git a/redis/Makefile b/redis/Makefile index 3de56dd..0ef7527 100644 --- a/redis/Makefile +++ b/redis/Makefile @@ -25,7 +25,6 @@ redis-server: Dockerfile_redis_build mkdir -p redis-server $(call build_image,Dockerfile_redis_build,redis_local_queue) docker run -i --rm -u $$(id -u):$$(id -g) -v $(CURDIR):/temp -w /temp --ulimit nofile=262144:262144 redis_local_queue:$(VERSION) cp /build/redis-6.2.1/src/redis-server redis-server/. - touch redis-server/redis_4999.conf echo "appendonly no" >> redis-server/redis_sock.conf echo "save \"\"" >> redis-server/redis_sock.conf echo "unixsocket /opt/mfn/redis-server/redis.sock" >> redis-server/redis_sock.conf diff --git a/tests/dynamic_parallel_execution_groups/test.py b/tests/dynamic_parallel_execution_groups/test.py index 03b824f..ef42d8e 100644 --- a/tests/dynamic_parallel_execution_groups/test.py +++ b/tests/dynamic_parallel_execution_groups/test.py @@ -64,8 +64,8 @@ def test_wordcount(self): print(job["type"]) print("Simple time total (ms): " + str(total_time_simple)) - print("MFN time total (get_session_update_messages(block=False) (ms): " + str(total_time)) - print("MFN time total (get_session_update_messages(block=True) (ms): " + str(total_time_blocking)) + print("MFN time total (get_session_update_messages(block=False)) (ms): " + str(total_time)) + print("MFN time total (get_session_update_messages(block=True)) (ms): " + str(total_time_blocking)) #@unittest.skip("") def test_wordcount_nested(self): @@ -115,8 +115,8 @@ def test_wordcount_nested(self): print(job["type"]) print("Simple time total (ms): " + str(total_time_simple)) - print("MFN time total (get_session_update_messages(block=False) (ms): " + str(total_time)) - print("MFN time total (get_session_update_messages(block=True) (ms): " + str(total_time_blocking)) + print("MFN time total (get_session_update_messages(block=False)) (ms): " + str(total_time)) + print("MFN time total (get_session_update_messages(block=True)) (ms): " + str(total_time_blocking)) #@unittest.skip("") def test_mergesort(self): @@ -166,8 +166,8 @@ def test_mergesort(self): print(job["type"]) print("Simple time total (ms): " + str(total_time_simple)) - print("MFN time total (get_session_update_messages(block=False) (ms): " + str(total_time)) - print("MFN time total (get_session_update_messages(block=True) (ms): " + str(total_time_blocking)) + print("MFN time total (get_session_update_messages(block=False)) (ms): " + str(total_time)) + print("MFN time total (get_session_update_messages(block=True)) (ms): " + str(total_time_blocking)) #################### # internal functions diff --git a/tests/dynamic_parallel_execution_groups/wf_mapreduce.json b/tests/dynamic_parallel_execution_groups/wf_mapreduce.json index 3866e90..a7c93dc 100644 --- a/tests/dynamic_parallel_execution_groups/wf_mapreduce.json +++ b/tests/dynamic_parallel_execution_groups/wf_mapreduce.json @@ -1,6 +1,7 @@ { "Comment": "MapReduce in KNIX", "StartAt": "driver", + "EnableCheckpoints": false, "AllowImmediateMessages": true, "States": { "driver": { diff --git a/tests/dynamic_parallel_execution_groups/wf_mapreduce_blocking.json b/tests/dynamic_parallel_execution_groups/wf_mapreduce_blocking.json index 25034d2..4f9eb13 100644 --- a/tests/dynamic_parallel_execution_groups/wf_mapreduce_blocking.json +++ b/tests/dynamic_parallel_execution_groups/wf_mapreduce_blocking.json @@ -1,6 +1,7 @@ { "Comment": "MapReduce in KNIX", "StartAt": "driver", + "EnableCheckpoints": false, "AllowImmediateMessages": true, "States": { "driver": { diff --git a/tests/echo/test.py b/tests/echo/test.py index cbd8f48..88e9f23 100644 --- a/tests/echo/test.py +++ b/tests/echo/test.py @@ -36,6 +36,9 @@ def test_echo_wfd(self): test_tuple_list.append((json.dumps(inp0), json.dumps(res0))) test = MFNTest(test_name='echo', workflow_filename='echo.json') - test.exec_tests(test_tuple_list, check_duration=True) + test.exec_tests(test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) - #test.plot_latency_breakdown(20) + #test.plot_latency_breakdown(COUNT) + + test.undeploy_workflow() + test.cleanup() \ No newline at end of file diff --git a/tests/performance2/test.py b/tests/performance2/test.py index a24eed0..e91eed6 100644 --- a/tests/performance2/test.py +++ b/tests/performance2/test.py @@ -39,12 +39,12 @@ def test_echo_0_bytes(self): test_tuple_list.append((json.dumps(inp0), json.dumps(res0))) - self._test.exec_tests(test_tuple_list, check_duration=True, should_undeploy=False) + self._test.exec_tests(test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) #test.plot_latency_breakdown(20) time.sleep(5) - self._test_checkpoints_off.exec_tests(test_tuple_list, check_duration=True, should_undeploy=False) + self._test_checkpoints_off.exec_tests(test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) #@unittest.skip("") def test_echo_4_bytes(self): @@ -56,12 +56,12 @@ def test_echo_4_bytes(self): test_tuple_list.append((json.dumps(inp0), json.dumps(res0))) - self._test.exec_tests(test_tuple_list, check_duration=True, should_undeploy=False) + self._test.exec_tests(test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) #test.plot_latency_breakdown(20) time.sleep(5) - self._test_checkpoints_off.exec_tests(test_tuple_list, check_duration=True, should_undeploy=False) + self._test_checkpoints_off.exec_tests(test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) #@unittest.skip("") def test_echo_1_MByte(self): @@ -73,12 +73,12 @@ def test_echo_1_MByte(self): test_tuple_list.append((json.dumps(inp0), json.dumps(res0))) - self._test.exec_tests(test_tuple_list, check_duration=True, should_undeploy=False) + self._test.exec_tests(test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) #test.plot_latency_breakdown(20) time.sleep(5) - self._test_checkpoints_off.exec_tests(test_tuple_list, check_duration=True, should_undeploy=False) + self._test_checkpoints_off.exec_tests(test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) @classmethod def tearDownClass(self): diff --git a/tests/performance_interaction_latency/test.py b/tests/performance_interaction_latency/test.py index 1d8b211..55ea353 100644 --- a/tests/performance_interaction_latency/test.py +++ b/tests/performance_interaction_latency/test.py @@ -35,7 +35,7 @@ def setUpClass(self): self._test_tuple_list.append((json.dumps(inp0), json.dumps(res0))) #@unittest.skip("") - def test_function_interaction_latency_off(self): + def test_function_interaction_latency_checkpoints_off(self): test = MFNTest(test_name='function_interaction_latency_checkpoints_off', workflow_filename='wf_function_interaction_latency_checkpoints_off.json') test.exec_tests(self._test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) @@ -49,7 +49,7 @@ def test_function_interaction_latency_off(self): test.cleanup() #@unittest.skip("") - def test_function_interaction_latency_on(self): + def test_function_interaction_latency_checkpoints_on(self): test = MFNTest(test_name='function_interaction_latency_checkpoints_on', workflow_filename='wf_function_interaction_latency_checkpoints_on.json') test.exec_tests(self._test_tuple_list, check_duration=True, print_report=False, should_undeploy=False) @@ -63,7 +63,7 @@ def test_function_interaction_latency_on(self): test.cleanup() #@unittest.skip("") - def test_chain_response_latency_off(self): + def test_chain_response_latency_checkpoints_off(self): test = MFNTest(test_name='chain_checkpoints_off', workflow_filename='wf_chain_checkpoints_off.json') print("----------------") print("Checkpoints: False") @@ -76,7 +76,7 @@ def test_chain_response_latency_off(self): test.cleanup() #@unittest.skip("") - def test_chain_response_latency_on(self): + def test_chain_response_latency_checkpoints_on(self): test = MFNTest(test_name='chain_checkpoints', workflow_filename='wf_chain_checkpoints.json') print("----------------") print("Checkpoints: True") diff --git a/tests/workflow_error_handling/test.py b/tests/workflow_error_handling/test.py index 57cc1f9..8ac6a4d 100644 --- a/tests/workflow_error_handling/test.py +++ b/tests/workflow_error_handling/test.py @@ -41,7 +41,7 @@ def test_error_frontend_propagation(self): test_tuple_list.append((json.dumps(event), json.dumps(expected_output))) - test = MFNTest(test_name="Workflow Error Propogation", workflow_filename="workflow_error_handling.json") + test = MFNTest(test_name="Workflow Error Propagation", workflow_filename="workflow_error_handling.json") test.exec_tests(test_tuple_list) #@unittest.skip("") @@ -91,6 +91,8 @@ def test_error_stop(self): print(log) if log: test.report(True, "workflow_stop_with_error", True, True) + else: + test.report(False, "workflow_stop_with_error", True, True) test.undeploy_workflow() test.cleanup() From f5a277dfbbf23cefb173813e791a3d5d8a669f2e Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Wed, 21 Apr 2021 17:22:48 +0200 Subject: [PATCH 08/51] minor fix to redis Makefile --- redis/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis/Makefile b/redis/Makefile index 0ef7527..3e52334 100644 --- a/redis/Makefile +++ b/redis/Makefile @@ -25,7 +25,7 @@ redis-server: Dockerfile_redis_build mkdir -p redis-server $(call build_image,Dockerfile_redis_build,redis_local_queue) docker run -i --rm -u $$(id -u):$$(id -g) -v $(CURDIR):/temp -w /temp --ulimit nofile=262144:262144 redis_local_queue:$(VERSION) cp /build/redis-6.2.1/src/redis-server redis-server/. - echo "appendonly no" >> redis-server/redis_sock.conf + echo "appendonly no" > redis-server/redis_sock.conf echo "save \"\"" >> redis-server/redis_sock.conf echo "unixsocket /opt/mfn/redis-server/redis.sock" >> redis-server/redis_sock.conf echo "unixsocketperm 700" >> redis-server/redis_sock.conf From fd35c90d95e0c575ee75f8af512b30f1ffeda1d3 Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Mon, 3 May 2021 21:39:37 +0200 Subject: [PATCH 09/51] create execution entry in sandbox frontend for async request to make their results queryable even after some time has passed --- FunctionWorker/python/PublicationUtils.py | 2 +- Sandbox/frontend/frontend.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/FunctionWorker/python/PublicationUtils.py b/FunctionWorker/python/PublicationUtils.py index f3004a3..887a964 100644 --- a/FunctionWorker/python/PublicationUtils.py +++ b/FunctionWorker/python/PublicationUtils.py @@ -103,7 +103,7 @@ def _shutdown_local_queue_client(self): def _get_backup_data_layer_client(self): if self._backup_data_layer_client is None: - self._backup_data_layer_client = DataLayerClient(locality=0, for_mfn=True, sid=self._sandboxid, connect=self._datalayer) + self._backup_data_layer_client = DataLayerClient(locality=-1, for_mfn=True, sid=self._sandboxid, connect=self._datalayer) return self._backup_data_layer_client def _shutdown_backup_data_layer_client(self): diff --git a/Sandbox/frontend/frontend.go b/Sandbox/frontend/frontend.go index 017f07e..da6c65f 100644 --- a/Sandbox/frontend/frontend.go +++ b/Sandbox/frontend/frontend.go @@ -440,9 +440,16 @@ func handler(w http.ResponseWriter, r *http.Request) { if err != nil { http.Error(w, "Error submitting event to system", http.StatusInternalServerError) } else { - // w.Header().Set("Content-Type", "application/json") - // w.WriteHeader(http.StatusAccepted) - w.Write([]byte(id)) + // w.Header().Set("Content-Type", "application/json") + // w.WriteHeader(http.StatusAccepted) + // Create entry and lock to wait on + m := sync.Mutex{} + c := sync.NewCond(&m) + e := Execution{c,nil,0} + ExecutionCond.L.Lock() + ExecutionResults[id] = &e + ExecutionCond.L.Unlock() + w.Write([]byte(id)) } } else { // Create entry and lock to wait on From 2d39e5640f6b196cdc83b30afe09618c89e057e3 Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Tue, 4 May 2021 14:35:46 +0200 Subject: [PATCH 10/51] distinguish between regular async calls and special async calls (e.g., post-parallel message, session message) in sandbox frontend --- Sandbox/frontend/frontend.go | 48 ++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/Sandbox/frontend/frontend.go b/Sandbox/frontend/frontend.go index da6c65f..03b9e91 100644 --- a/Sandbox/frontend/frontend.go +++ b/Sandbox/frontend/frontend.go @@ -246,13 +246,36 @@ func handler(w http.ResponseWriter, r *http.Request) { http.Error(w, "Can't fetch result", http.StatusInternalServerError) } else if res == nil { if async { - log.Printf("handler: Result not yet available of execution ID %s, redirecting", id) + /* + the client asked for the result in async mode. + if the result is not available yet, then just return the execution id again + and let the client handle the retry. + */ + log.Printf("handler: Result not yet available of execution ID %s.", id) + //log.Printf("handler: Result not yet available of execution ID %s, redirecting", id) // TODO: 300 location redirect - http.Redirect(w, r, r.URL.Path + "?executionId=" + id, http.StatusTemporaryRedirect) - // w.Header().Set("Content-Type", "application/json") + //http.Redirect(w, r, r.URL.Path + "?executionId=" + id, http.StatusTemporaryRedirect) + //w.Header().Set("Content-Type", "application/json") // w.Header().Set("Retry-After", "5") // http.Redirect(w, r, r.URL.Path + "?executionId=" + id, http.StatusFound) - // w.Write([]byte(id)) + + type ResultMessage struct { + ExecutionId string `json:"executionId"` + Result []byte `json:"result"` + } + var res ResultMessage + res.ExecutionId = id + res.Result = nil + var msgb []byte + msgb, err := json.Marshal(res) + if err != nil { + http.Error(w, "Couldn't marshal result to JSON", http.StatusInternalServerError) + log.Println("handler: Couldn't marshal result to JSON: ", err) + } else { + w.Header().Set("Content-Type", "application/json") + w.Write(msgb) + } + //w.Write([]byte(id)) } else { log.Printf("handler: Result not yet available of execution ID %s, waiting", id) ExecutionCond.L.Lock() @@ -305,12 +328,14 @@ func handler(w http.ResponseWriter, r *http.Request) { topic string msg MfnMessage id string + is_special_message bool ) actionhdr := r.Header.Get("x-mfn-action") actiondata := r.Header.Get("x-mfn-action-data") if (actionhdr != "" && actiondata != "") { // handle different cases here log.Printf("Got a special message: [%s] [%s]", actionhdr, actiondata) + is_special_message = true if (actionhdr == "session-update") { log.Printf("New session update message...") @@ -410,6 +435,7 @@ func handler(w http.ResponseWriter, r *http.Request) { log.Printf("Trigger-Event message topic: [%s], id: [%s]", topic, id) } } else { + is_special_message = false // the headers are not special, so this must be a regular message to trigger a workflow // CREATE NEW MfnMessage id, err = GenerateExecutionID() @@ -443,12 +469,14 @@ func handler(w http.ResponseWriter, r *http.Request) { // w.Header().Set("Content-Type", "application/json") // w.WriteHeader(http.StatusAccepted) // Create entry and lock to wait on - m := sync.Mutex{} - c := sync.NewCond(&m) - e := Execution{c,nil,0} - ExecutionCond.L.Lock() - ExecutionResults[id] = &e - ExecutionCond.L.Unlock() + if !is_special_message { + m := sync.Mutex{} + c := sync.NewCond(&m) + e := Execution{c,nil,0} + ExecutionCond.L.Lock() + ExecutionResults[id] = &e + ExecutionCond.L.Unlock() + } w.Write([]byte(id)) } } else { From 9e37e5738d3b7efe19c66f3f83229b00e2a99511 Mon Sep 17 00:00:00 2001 From: Paarijaat Aditya Date: Tue, 4 May 2021 18:39:16 +0200 Subject: [PATCH 11/51] Triggersfrontend, adding Cargo.lock to lock versions of dependencies --- TriggersFrontend/Cargo.lock | 2644 +++++++++++++++++++++++++++++++++++ 1 file changed, 2644 insertions(+) create mode 100644 TriggersFrontend/Cargo.lock diff --git a/TriggersFrontend/Cargo.lock b/TriggersFrontend/Cargo.lock new file mode 100644 index 0000000..acf093f --- /dev/null +++ b/TriggersFrontend/Cargo.lock @@ -0,0 +1,2644 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "TriggersFrontend" +version = "0.1.0" +dependencies = [ + "actix-web", + "bytes", + "chrono", + "env_logger", + "futures", + "json", + "lapin", + "log", + "rand", + "regex", + "reqwest", + "rumqttc", + "serde", + "serde_json", + "tokio", + "tokio-amqp", +] + +[[package]] +name = "actix-codec" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78d1833b3838dbe990df0f1f87baf640cf6146e898166afe401839d1b001e570" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-connect" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177837a10863f15ba8d3ae3ec12fac1099099529ed20083a27fdfe247381d0dc" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "derive_more", + "either", + "futures-util", + "http", + "log", + "trust-dns-proto", + "trust-dns-resolver", +] + +[[package]] +name = "actix-http" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05dd80ba8f27c4a34357c07e338c8f5c38f8520e6d626ca1727d8fecc41b0cab" +dependencies = [ + "actix-codec", + "actix-connect", + "actix-rt", + "actix-service", + "actix-threadpool", + "actix-utils", + "base64", + "bitflags", + "brotli2", + "bytes", + "cookie", + "copyless", + "derive_more", + "either", + "encoding_rs", + "flate2", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "h2", + "http", + "httparse", + "indexmap", + "itoa", + "language-tags", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project", + "rand", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha-1", + "slab", + "time 0.2.22", +] + +[[package]] +name = "actix-macros" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a60f9ba7c4e6df97f3aacb14bb5c0cd7d98a49dcbaed0d7f292912ad9a6a3ed2" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd1f7dbda1645bf7da33554db60891755f6c01c1b2169e2f4c492098d30c235" +dependencies = [ + "bytestring", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143fcc2912e0d1de2bcf4e2f720d2a60c28652ab4179685a1ee159e0fb3db227" +dependencies = [ + "actix-macros", + "actix-threadpool", + "copyless", + "futures-channel", + "futures-util", + "smallvec", + "tokio", +] + +[[package]] +name = "actix-server" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45407e6e672ca24784baa667c5d32ef109ccdd8d5e0b5ebb9ef8a67f4dfb708e" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "futures-channel", + "futures-util", + "log", + "mio 0.6.22", + "mio-uds", + "num_cpus", + "slab", + "socket2", +] + +[[package]] +name = "actix-service" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" +dependencies = [ + "futures-util", + "pin-project", +] + +[[package]] +name = "actix-testing" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47239ca38799ab74ee6a8a94d1ce857014b2ac36f242f70f3f75a66f691e791c" +dependencies = [ + "actix-macros", + "actix-rt", + "actix-server", + "actix-service", + "log", + "socket2", +] + +[[package]] +name = "actix-threadpool" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d209f04d002854b9afd3743032a27b066158817965bf5d036824d19ac2cc0e30" +dependencies = [ + "derive_more", + "futures-channel", + "lazy_static", + "log", + "num_cpus", + "parking_lot", + "threadpool", +] + +[[package]] +name = "actix-tls" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24789b7d7361cf5503a504ebe1c10806896f61e96eca9a7350e23001aca715fb" +dependencies = [ + "actix-codec", + "actix-service", + "actix-utils", + "futures-util", +] + +[[package]] +name = "actix-utils" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9022dec56632d1d7979e59af14f0597a28a830a9c1c7fec8b2327eb9f16b5a" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "bitflags", + "bytes", + "either", + "futures-channel", + "futures-sink", + "futures-util", + "log", + "pin-project", + "slab", +] + +[[package]] +name = "actix-web" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b12fe25e11cd9ed2ef2e428427eb6178a1b363f3f7f0dab8278572f11b2da1" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-testing", + "actix-threadpool", + "actix-tls", + "actix-utils", + "actix-web-codegen", + "awc", + "bytes", + "derive_more", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "fxhash", + "log", + "mime", + "pin-project", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "socket2", + "time 0.2.22", + "tinyvec 1.0.1", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "750ca8fb60bbdc79491991650ba5d2ae7cd75f3fc00ead51390cfe9efda0d4d8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "aho-corasick" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +dependencies = [ + "memchr", +] + +[[package]] +name = "amq-protocol" +version = "6.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede228647914ef8ac4ef4cfddbe3be4c23afea8d8c7f95451a2f53efe6ce8f2f" +dependencies = [ + "amq-protocol-tcp", + "amq-protocol-types", + "amq-protocol-uri", + "cookie-factory", + "nom", +] + +[[package]] +name = "amq-protocol-tcp" +version = "6.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4860b3d63ffb63b3ed440b78ccd549b009e0f72d995b9e9aec51e8e1af01e694" +dependencies = [ + "amq-protocol-uri", + "tcp-stream", + "tracing", +] + +[[package]] +name = "amq-protocol-types" +version = "6.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ebe159bbe5fff48b5702d639ba9b0a7b1b2f24c4404fe38db0cbcafd057c01" +dependencies = [ + "cookie-factory", + "nom", + "serde", + "serde_json", +] + +[[package]] +name = "amq-protocol-uri" +version = "6.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82403806f8fef7e9d8dc6dcb41839a47a8eab9419f9dca52ef6b19904e0047e" +dependencies = [ + "percent-encoding", + "url", +] + +[[package]] +name = "arc-swap" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "async-channel" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-task" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab27c1aa62945039e44edaeee1dc23c74cc0c303dd5fe0fb462a184f1c3a518" + +[[package]] +name = "async-trait" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "awc" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150e00c06683ab44c5f97d033950e5d87a7a042d06d77f5eecb443cbd23d0575" +dependencies = [ + "actix-codec", + "actix-http", + "actix-rt", + "actix-service", + "base64", + "bytes", + "derive_more", + "futures-core", + "log", + "mime", + "percent-encoding", + "rand", + "serde", + "serde_json", + "serde_urlencoded", +] + +[[package]] +name = "backtrace" +version = "0.3.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f813291114c186a042350e787af10c26534601062603d888be110f59f85ef8fa" +dependencies = [ + "addr2line", + "cfg-if 0.1.10", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitvec" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytestring" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7c05fa5172da78a62d9949d662d2ac89d4cc7355d7b49adee5163f1fb3f363" +dependencies = [ + "bytes", +] + +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + +[[package]] +name = "cc" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time 0.1.44", + "winapi 0.3.9", +] + +[[package]] +name = "cloudabi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +dependencies = [ + "bitflags", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "const_fn" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" + +[[package]] +name = "cookie" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" +dependencies = [ + "percent-encoding", + "time 0.2.22", + "version_check", +] + +[[package]] +name = "cookie-factory" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f21b581d2f0cb891554812435667bb9610d74feb1a4c6415bf09c28ff0381d" + +[[package]] +name = "copyless" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "flate2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee" +dependencies = [ + "cfg-if 0.1.10", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "funty" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba62103ce691c2fd80fbae2213dfdda9ce60804973ac6b6e97de818ea7f52c8" + +[[package]] +name = "futures" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d8e3078b7b2a8a671cb7a3d17b4760e4181ea243227776ba83fd043b4ca034e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a4d35f7401e948629c9c3d6638fb9bf94e0b2121e96c3b428cc4e631f3eb74" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674eaa0056896d5ada519900dbf97ead2e46a7b6621e8160d79e2f2e1e2784b" + +[[package]] +name = "futures-executor" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc709ca1da6f66143b8c9bec8e6260181869893714e9b5a490b169b0414144ab" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc94b64bb39543b4e432f1790b6bf18e3ee3b74653c5449f63310e9a74b123c" + +[[package]] +name = "futures-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f57ed14da4603b2554682e9f2ff3c65d7567b53188db96cb71538217fc64581b" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8764258ed64ebc5d9ed185cf86a95db5cac810269c5d20ececb32e0088abbd" + +[[package]] +name = "futures-task" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd26820a9f3637f1302da8bceba3ff33adbe53464b54ca24d4e2d4f1db30f94" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a894a0acddba51a2d49a6f4263b1e64b8c579ece8af50fa86503d52cd1eea34" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" + +[[package]] +name = "h2" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f9e0baeed60001cf565546b0d3dbe6a6ad23f2bd31644a133c641eccf6d53" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hyper" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-tls", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "js-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lapin" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a04173b1524419f51c00c2f0f0e7b52bb9cd2e5bd9ea030da8bee086bda7e7b2" +dependencies = [ + "amq-protocol", + "async-task", + "crossbeam-channel", + "futures-core", + "log", + "mio 0.7.6", + "parking_lot", + "pinky-swear", + "serde", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if 0.1.10", + "ryu", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" + +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + +[[package]] +name = "lock_api" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.1", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f33bc887064ef1fd66020c9adfc45bb9f33d75a42096c81e7c56c65b75dd1a8b" +dependencies = [ + "libc", + "log", + "miow 0.3.6", + "ntapi", + "winapi 0.3.9", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio 0.6.22", + "miow 0.3.6", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.22", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "mqtt4bytes" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c179518ace4914d2532e5f733891c032fcebf6641a03d8349fdf0cec7ee4f94" +dependencies = [ + "bytes", +] + +[[package]] +name = "native-tls" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nom" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88034cfd6b4a0d54dd14f4a507eceee36c0b70e5a02236c4e4df571102be17f0" +dependencies = [ + "bitvec", + "lexical-core", + "memchr", + "version_check", +] + +[[package]] +name = "ntapi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a31937dea023539c72ddae0e3571deadc1414b300483fa7aaec176168cfa9d2" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "num-integer" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" + +[[package]] +name = "once_cell" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +dependencies = [ + "bitflags", + "cfg-if 0.1.10", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "pem" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59698ea79df9bf77104aefd39cc3ec990cb9693fb59c3b0a70ddf2646fdffb4b" +dependencies = [ + "base64", + "once_cell", + "regex", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13fbdfd6bdee3dc9be46452f86af4a4072975899cf8592466668620bebfbcc17" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c82fb1329f632c3552cf352d14427d57a511b1cf41db93b3a7d77906a82dcc8e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e555d9e657502182ac97b539fb3dae8b79cda19e3e4f8ffb5e8de4f18df93c95" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pinky-swear" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bf8cda6f8e1500338634e4e3ce90ac59eb7929a1e088b6946c742be1cc44dc1" +dependencies = [ + "doc-comment", + "parking_lot", + "tracing", +] + +[[package]] +name = "pkg-config" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" + +[[package]] +name = "pollster" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9824e18e85003f0b5a38fa1932ae8be8c2aac9447c2f28ab6f9704dbe0a1ab58" + +[[package]] +name = "ppv-lite86" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" + +[[package]] +name = "proc-macro-hack" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_urlencoded", + "tokio", + "tokio-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11834e137f3b14e309437a8276714eed3a80d1ef894869e510f2c0c0b98b9f4a" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "ring" +version = "0.16.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "rumqttc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c95a88d738f7b09f0556080a9755753c825e286ede49a4b42fae453e8034a0" +dependencies = [ + "async-channel", + "bytes", + "log", + "mqtt4bytes", + "pollster", + "thiserror", + "tokio", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2610b7f643d18c87dff3b489950269617e6601a51f1f05aa5daefee36f64f0b" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a230ea9107ca2220eea9d46de97eddcb04cd00e92d13dda78e478dd33fa82bd4" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url", +] + +[[package]] +name = "sha-1" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770" +dependencies = [ + "block-buffer", + "cfg-if 0.1.10", + "cpuid-bool", + "digest", + "opaque-debug", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "signal-hook-registry" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e12110bc539e657a646068aaf5eb5b63af9d0c1f7b29c97113fad80e15f035" +dependencies = [ + "arc-swap", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" + +[[package]] +name = "socket2" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c29947abdee2a218277abeca306f25789c938e500ea5a9d4b12a5a504466902" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "standback" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e0831040d2cf2bdfd51b844be71885783d489898a192f254ae25d57cce725c" +dependencies = [ + "version_check", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "syn" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tap" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36474e732d1affd3a6ed582781b3683df3d0563714c59c39591e8ff707cf078e" + +[[package]] +name = "tcp-stream" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c180af0da0f0a75ca080465175892fcdaa750076f125cb953127721e676413" +dependencies = [ + "cfg-if 1.0.0", + "mio 0.7.6", + "native-tls", + "pem", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b7151c9065e80917fbf285d9a5d1432f60db41d170ccafc749a136b41a93af" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" + +[[package]] +name = "tinyvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b78a366903f506d2ad52ca8dc552102ffdd3e937ba8a227f024dc1d1eae28575" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio 0.6.22", + "mio-named-pipes", + "mio-uds", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-amqp" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2938fb5b638e6d8992c304e2426db2de4502f19084b91ec4562d9b45bf896fc4" +dependencies = [ + "lapin", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +dependencies = [ + "futures-core", + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +dependencies = [ + "cfg-if 0.1.10", + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "trust-dns-proto" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd7061ba6f4d4d9721afedffbfd403f20f39a4301fee1b70d6fcd09cca69f28" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand", + "smallvec", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f23cdfdc3d8300b3c50c9e84302d3bd6d860fb9529af84ace6cf9665f181b77" +dependencies = [ + "backtrace", + "cfg-if 0.1.10", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +dependencies = [ + "tinyvec 0.3.4", +] + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +dependencies = [ + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" +dependencies = [ + "cfg-if 0.1.10", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" +dependencies = [ + "cfg-if 0.1.10", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" + +[[package]] +name = "web-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" From ac7d35edb95806952f0b2ea68272cb95a304e78f Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Fri, 7 May 2021 10:32:29 +0200 Subject: [PATCH 12/51] move the java request handler to the end of the docker build --- Sandbox/Dockerfile_java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Sandbox/Dockerfile_java b/Sandbox/Dockerfile_java index b1d6246..d9b7009 100644 --- a/Sandbox/Dockerfile_java +++ b/Sandbox/Dockerfile_java @@ -47,12 +47,6 @@ RUN apt-get -y --no-install-recommends install maven RUN groupadd -o -g 1000 -r mfn && useradd -d /opt/mfn -u 1000 -m -r -g mfn mfn RUN mkdir /opt/mfn/logs -ADD build/JavaRequestHandler.tar.gz /opt/mfn/ - -RUN chmod +x /opt/mfn/JavaRequestHandler/setup_maven.sh -RUN /opt/mfn/JavaRequestHandler/./setup_maven.sh True -RUN mvn -Duser.home=/tmp -DskipTests -gs /opt/mfn/JavaRequestHandler/maven/sandbox-mvn-settings.xml -f /opt/mfn/JavaRequestHandler/maven/init-mvn.pom.xml dependency:resolve-plugins - RUN /usr/bin/python3 -m pip install redis ADD build/redis-server.tar.gz /opt/mfn/ ADD frontend/frontend /opt/mfn/frontend @@ -60,6 +54,13 @@ ADD build/SandboxAgent.tar.gz /opt/mfn/ ADD build/FunctionWorker.tar.gz /opt/mfn/ ADD build/LoggingService.tar.gz /opt/mfn/ +ADD build/JavaRequestHandler.tar.gz /opt/mfn/ + +RUN chmod +x /opt/mfn/JavaRequestHandler/setup_maven.sh +RUN /opt/mfn/JavaRequestHandler/./setup_maven.sh True +RUN mvn -Duser.home=/tmp -DskipTests -gs /opt/mfn/JavaRequestHandler/maven/sandbox-mvn-settings.xml -f /opt/mfn/JavaRequestHandler/maven/init-mvn.pom.xml dependency:resolve-plugins + + RUN chown mfn:mfn -R /opt/mfn USER mfn WORKDIR /opt/mfn From f347d3eb986f4a340340fe149e017a220060731e Mon Sep 17 00:00:00 2001 From: Paarijaat Aditya Date: Tue, 11 May 2021 13:51:04 +0200 Subject: [PATCH 13/51] allow specifying custom knative resource and annotation specifications via the modifyWorkflow action in management --- ManagementService/python/deployWorkflow.py | 26 +++++++ ManagementService/python/modifyWorkflow.py | 83 ++++++++++++++++++++-- 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/ManagementService/python/deployWorkflow.py b/ManagementService/python/deployWorkflow.py index 3c7dd66..00f80cc 100644 --- a/ManagementService/python/deployWorkflow.py +++ b/ManagementService/python/deployWorkflow.py @@ -320,6 +320,31 @@ def create_k8s_deployment(email, workflow_info, runtime, management=False): continue for container in kservice['spec']['template']['spec']['containers']: container['env'].append({'name': k, 'value': os.getenv(k)}) + + # get user provided requests, limits, and other knative autoscaling annotations + if not management: + if "workflowMetadata" in workflow_info and "knative" in workflow_info["workflowMetadata"]: + knative_groups = workflow_info["workflowMetadata"]["knative"] + for group_name in ["annotations", "container", "spec"]: + if group_name not in knative_groups: + continue + if group_name is "annotations": + if "annotations" not in kservice['spec']['template']['metadata']: + kservice['spec']['template']['metadata']['annotations'] = {} + group_to_update = kservice['spec']['template']['metadata']['annotations'] + elif group_name is "container": + group_to_update = kservice['spec']['template']['spec']['containers'][0] + else: + group_to_update = kservice['spec']['template']['spec'] + + assert(type(group_to_update) == type({})) + group = knative_groups[group_name] + for user_provided_value_key in group: + # overwrite or key a user provided key + group_to_update[user_provided_value_key] = group[user_provided_value_key] + print('Updated Knative service definition to: ' + str(kservice)) + + print('Checking if kservice exists') resp = requests.get( "https://"+os.getenv("KUBERNETES_SERVICE_HOST")+":"+os.getenv("KUBERNETES_SERVICE_PORT_HTTPS")+"/apis/serving.knative.dev/v1/namespaces/"+namespace+"/services/"+ksvcname, @@ -448,6 +473,7 @@ def handle(value, sapi): workflow_info["json_ref"] = "workflow_json_" + wfmeta["id"] workflow_info["workflowName"] = wfmeta["name"] workflow_info["usertoken"] = data["usertoken"] + workflow_info["workflowMetadata"] = wfmeta req = {} req["installer"] = "pip" workflow_info["sandbox_requirements"] = req diff --git a/ManagementService/python/modifyWorkflow.py b/ManagementService/python/modifyWorkflow.py index 9e4c4da..376c75b 100644 --- a/ManagementService/python/modifyWorkflow.py +++ b/ManagementService/python/modifyWorkflow.py @@ -15,6 +15,58 @@ import json import time +''' +# sample input +{ + "action": "modifyWorkflow", + "data": { + "user": { + "token": "...." + }, + "workflow": { + # to update workflow name, specify both id and name + # to update workflows resources/annotations specify id and/or name and the optional field below + + "id": "asdf", + "name": "qwerty", + "knative": { + "annotations": { # + "autoscaling.knative.dev/target": "10", + "autoscaling.knative.dev/panicWindowPercentage": "5.0", + "autoscaling.knative.dev/panicThresholdPercentage": "110.0", + "autoscaling.knative.dev/targetUtilizationPercentage": "70", + "autoscaling.knative.dev/targetBurstCapacity": "10", + "autoscaling.knative.dev/...": "..." + }, + "container": { # + "image": "asdfasdfdf", # + "resources": { # + "requests": { + "memory": "64Mi", + "cpu": "250m", + "tencent.com/vcuda-core": 4, + "tencent.com/vcuda-memory": 3, + "....": "..." + }, + "limits": { + "tencent.com/vcuda-core": 4, + "tencent.com/vcuda-memory": 3, + "memory": "128Mi", + "cpu": "500m", + "....": "..." + } + } + }, + "spec": { # ' + "containerConcurrency": 50 + } + } + } + } +} +''' + + def handle(value, sapi): assert isinstance(value, dict) data = value @@ -54,17 +106,29 @@ def handle(value, sapi): else: wf["ASL_type"] = "unknown" - sapi.put(email + "_workflow_" + wf["id"], json.dumps(wf), True, True) + sapi.put(email + "_workflow_" + wf["id"], json.dumps(wf), True) - sapi.put(email + "_list_workflows", json.dumps(workflows), True, True) + sapi.put(email + "_list_workflows", json.dumps(workflows), True) response_data["message"] = "Successfully modified workflow " + workflow["id"] + "." response_data["workflow"] = wf - success = True - + elif "knative" in workflow and ("id" in workflow or "name" in workflow): + id = None + if "id" not in workflow: + id = get_workflow_id(workflow["name"], email, sapi) else: - response_data["message"] = "Couldn't modify workflow; workflow metadata is not valid." + id = workflow["id"] + + if id is not None: + wf = sapi.get(email + "_workflow_" + id, True) + wf = json.loads(wf) + wf["knative"] = workflow["knative"] + sapi.put(email + "_workflow_" + id, json.dumps(wf), True) + success = True + response_data["message"] = "New knative resource and annotation specifications will be applied next time the workflow is deployed." + else: + response_data["message"] = "Couldn't modify workflow resources; Unable to find workflow id." else: response_data["message"] = "Couldn't modify workflow; malformed input." else: @@ -81,3 +145,12 @@ def handle(value, sapi): return response + +def get_workflow_id(workflow_name, email, context): + id = None + workflows = context.get(email + "_list_workflows", True) + if workflows is not None and workflows != "": + workflows = json.loads(workflows) + if workflow_name in workflows: + id = workflows[workflow_name] + return id \ No newline at end of file From d16e6e5650ac944353e1f1f69b8a445a1c5d727c Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Fri, 14 May 2021 10:15:48 +0200 Subject: [PATCH 14/51] update ansible readme; fixes #117 --- deploy/ansible/README.md | 74 +++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 5eb66be..93f2a78 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -1,5 +1,5 @@