From 83e2d4115362355e3e055c2ffc05da58ab4c4a57 Mon Sep 17 00:00:00 2001 From: Joe Batt Date: Mon, 25 Jul 2022 11:12:37 +0100 Subject: [PATCH] updated check for Task Manager and Workflow Executor start up Signed-off-by: Joe Batt --- docker-compose/MinIO/docker-compose.yml | 75 ------------------- docker-compose/MongoDB/docker-compose.yml | 14 ---- docker-compose/RabbitMQ/docker-compose.yml | 19 ----- .../docker-compose.yml | 0 .../TaskManager.IntegrationTests/Hooks.cs | 30 +++----- .../Support/TaskManagerStartup.cs | 4 +- .../Hooks.cs | 30 +++----- .../Support/WorkflowExecutorStartup.cs | 4 +- .../docker-compose.yml | 47 ------------ 9 files changed, 28 insertions(+), 195 deletions(-) delete mode 100644 docker-compose/MinIO/docker-compose.yml delete mode 100644 docker-compose/MongoDB/docker-compose.yml delete mode 100644 docker-compose/RabbitMQ/docker-compose.yml rename {tests/IntegrationTests/TaskManager.IntegrationTests => docker-compose}/docker-compose.yml (100%) delete mode 100644 tests/IntegrationTests/WorkflowExecutor.IntegrationTests/docker-compose.yml diff --git a/docker-compose/MinIO/docker-compose.yml b/docker-compose/MinIO/docker-compose.yml deleted file mode 100644 index a0bffbb72..000000000 --- a/docker-compose/MinIO/docker-compose.yml +++ /dev/null @@ -1,75 +0,0 @@ -version: '3.7' - -# Settings and configurations that are common for all containers -x-minio-common: &minio-common - image: quay.io/minio/minio:RELEASE.2022-04-12T06-55-35Z - command: server --console-address ":9001" http://minio{1...4}/data{1...2} - expose: - - "9000" - - "9001" - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - -# starts 4 docker containers running minio server instances. -# using nginx reverse proxy, load balancing, you can access -# it through port 9000. -services: - minio1: - <<: *minio-common - hostname: minio1 - volumes: - - data1-1:/data1 - - data1-2:/data2 - - minio2: - <<: *minio-common - hostname: minio2 - volumes: - - data2-1:/data1 - - data2-2:/data2 - - minio3: - <<: *minio-common - hostname: minio3 - volumes: - - data3-1:/data1 - - data3-2:/data2 - - minio4: - <<: *minio-common - hostname: minio4 - volumes: - - data4-1:/data1 - - data4-2:/data2 - - nginx: - image: nginx:1.19.2-alpine - hostname: nginx - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf:ro - ports: - - "9000:9000" - - "9001:9001" - depends_on: - - minio1 - - minio2 - - minio3 - - minio4 - -## By default this config uses default local driver, -## For custom volumes replace with volume driver configuration. -volumes: - data1-1: - data1-2: - data2-1: - data2-2: - data3-1: - data3-2: - data4-1: - data4-2: diff --git a/docker-compose/MongoDB/docker-compose.yml b/docker-compose/MongoDB/docker-compose.yml deleted file mode 100644 index 38d56b414..000000000 --- a/docker-compose/MongoDB/docker-compose.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: '3.7' -services: - mongodb_container: - image: mongo:latest - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: rootpassword - ports: - - 27017:27017 - volumes: - - mongodb_data_container:/data/db - -volumes: - mongodb_data_container: \ No newline at end of file diff --git a/docker-compose/RabbitMQ/docker-compose.yml b/docker-compose/RabbitMQ/docker-compose.yml deleted file mode 100644 index fe6ebfbc3..000000000 --- a/docker-compose/RabbitMQ/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3.9' -services: - rabbit1: - image: "rabbitmq:3.8.18-management" - hostname: "rabbit1" - environment: - RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQG" - RABBITMQ_DEFAULT_USER: "admin" - RABBITMQ_DEFAULT_PASS: "admin" - RABBITMQ_DEFAULT_VHOST: "monaideploy" - ports: - - "15672:15672" - - "5672:5672" - labels: - NAME: "rabbitmq1" - -volumes: - data01: - driver: local diff --git a/tests/IntegrationTests/TaskManager.IntegrationTests/docker-compose.yml b/docker-compose/docker-compose.yml similarity index 100% rename from tests/IntegrationTests/TaskManager.IntegrationTests/docker-compose.yml rename to docker-compose/docker-compose.yml diff --git a/tests/IntegrationTests/TaskManager.IntegrationTests/Hooks.cs b/tests/IntegrationTests/TaskManager.IntegrationTests/Hooks.cs index 1b3b5cedf..ceca2487b 100644 --- a/tests/IntegrationTests/TaskManager.IntegrationTests/Hooks.cs +++ b/tests/IntegrationTests/TaskManager.IntegrationTests/Hooks.cs @@ -7,6 +7,8 @@ using Microsoft.Extensions.Hosting; using Monai.Deploy.WorkflowManager.IntegrationTests.Support; using Monai.Deploy.WorkflowManager.TaskManager.IntegrationTests.Support; +using Polly; +using Polly.Retry; namespace Monai.Deploy.WorkflowManager.TaskManager.IntegrationTests { @@ -30,6 +32,7 @@ public Hooks(IObjectContainer objectContainer) private static RabbitConsumer? TaskUpdateConsumer { get; set; } public static RabbitConsumer? ClinicalReviewConsumer { get; private set; } private static MinioClientUtil? MinioClient { get; set; } + public static AsyncRetryPolicy RetryPolicy { get; private set; } private IObjectContainer ObjectContainer { get; set; } private static IHost? Host { get; set; } private static HttpClient? HttpClient { get; set; } @@ -65,6 +68,7 @@ public static void Init() Host = TaskManagerStartup.StartTaskManager(); HttpClient = new HttpClient(); MinioClient = new MinioClientUtil(); + RetryPolicy = Policy.Handle().WaitAndRetryAsync(retryCount: 20, sleepDurationProvider: _ => TimeSpan.FromMilliseconds(500)); } // @@ -75,30 +79,20 @@ public static void Init() [BeforeTestRun(Order = 1)] public static async Task CheckTaskManagerConsumersStarted() { - var response = await TaskManagerStartup.GetConsumers(HttpClient); - var content = response.Content.ReadAsStringAsync().Result; - - for (var i = 1; i <= 10; i++) + await RetryPolicy.ExecuteAsync(async () => { - if (string.IsNullOrEmpty(content) || content == "[]") + var response = await TaskManagerStartup.GetQueueStatus(HttpClient, TestExecutionConfig.RabbitConfig.VirtualHost, TestExecutionConfig.RabbitConfig.TaskDispatchQueue); + var content = response.Content.ReadAsStringAsync().Result; + + if (content.Contains("error")) { - Debug.Write($"Task Manager not started. Recheck times {i}"); - response = await TaskManagerStartup.GetConsumers(HttpClient); - content = response.Content.ReadAsStringAsync().Result; + throw new Exception("Task Manager not started!"); } else { - Debug.Write("Task Manager started. Integration tests will begin!"); - break; - } - - if (i == 10) - { - throw new Exception("Task Manager not started! Integration tests will not continue"); + Console.WriteLine("Task Manager started. Integration Tests will begin."); } - - Thread.Sleep(1000); - } + }); TaskDispatchPublisher = new RabbitPublisher(RabbitConnectionFactory.GetConnectionFactory(), TestExecutionConfig.RabbitConfig.Exchange, TestExecutionConfig.RabbitConfig.TaskDispatchQueue); TaskCallbackPublisher = new RabbitPublisher(RabbitConnectionFactory.GetConnectionFactory(), TestExecutionConfig.RabbitConfig.Exchange, TestExecutionConfig.RabbitConfig.TaskCallbackQueue); diff --git a/tests/IntegrationTests/TaskManager.IntegrationTests/Support/TaskManagerStartup.cs b/tests/IntegrationTests/TaskManager.IntegrationTests/Support/TaskManagerStartup.cs index e51204081..e7cc77b81 100644 --- a/tests/IntegrationTests/TaskManager.IntegrationTests/Support/TaskManagerStartup.cs +++ b/tests/IntegrationTests/TaskManager.IntegrationTests/Support/TaskManagerStartup.cs @@ -101,13 +101,13 @@ public static IHost StartTaskManager() return host; } - public static async Task GetConsumers(HttpClient httpClient) + public static async Task GetQueueStatus(HttpClient httpClient, string vhost, string queue) { var svcCredentials = Convert.ToBase64String(Encoding.ASCII.GetBytes(TestExecutionConfig.RabbitConfig.User + ":" + TestExecutionConfig.RabbitConfig.Password)); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", svcCredentials); - return await httpClient.GetAsync($"http://{TestExecutionConfig.RabbitConfig.Host}:{TestExecutionConfig.RabbitConfig.Port}/api/consumers"); + return await httpClient.GetAsync($"http://{TestExecutionConfig.RabbitConfig.Host}:{TestExecutionConfig.RabbitConfig.Port}/api/queues/{vhost}/{queue}"); } } } diff --git a/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Hooks.cs b/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Hooks.cs index 51d225967..71327ed4a 100644 --- a/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Hooks.cs +++ b/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Hooks.cs @@ -7,6 +7,8 @@ using Microsoft.Extensions.Hosting; using Monai.Deploy.WorkflowManager.IntegrationTests.POCO; using Monai.Deploy.WorkflowManager.IntegrationTests.Support; +using Polly; +using Polly.Retry; namespace Monai.Deploy.WorkflowManagerIntegrationTests { @@ -26,6 +28,7 @@ public Hooks(IObjectContainer objectContainer) } private static HttpClient? HttpClient { get; set; } + public static AsyncRetryPolicy RetryPolicy { get; private set; } private static RabbitPublisher? WorkflowPublisher { get; set; } private static RabbitConsumer? TaskDispatchConsumer { get; set; } private static RabbitPublisher? TaskUpdatePublisher { get; set; } @@ -75,6 +78,7 @@ public static void Init() MinioClient = new MinioClientUtil(); Host = WorkflowExecutorStartup.StartWorkflowExecutor(); HttpClient = new HttpClient(); + RetryPolicy = Policy.Handle().WaitAndRetryAsync(retryCount: 20, sleepDurationProvider: _ => TimeSpan.FromMilliseconds(500)); } /// @@ -85,30 +89,20 @@ public static void Init() [BeforeTestRun(Order = 1)] public static async Task CheckWorkflowConsumerStarted() { - var response = await WorkflowExecutorStartup.GetConsumers(HttpClient); - var content = response.Content.ReadAsStringAsync().Result; - - for (var i = 1; i <= 10; i++) + await RetryPolicy.ExecuteAsync(async () => { - if (string.IsNullOrEmpty(content) || content == "[]") + var response = await WorkflowExecutorStartup.GetQueueStatus(HttpClient, TestExecutionConfig.RabbitConfig.VirtualHost, TestExecutionConfig.RabbitConfig.TaskUpdateQueue); + var content = response.Content.ReadAsStringAsync().Result; + + if (content.Contains("error")) { - Debug.Write($"Workflow consumer not started. Recheck times {i}"); - response = await WorkflowExecutorStartup.GetConsumers(HttpClient); - content = response.Content.ReadAsStringAsync().Result; + throw new Exception("Workflow Executor not started!"); } else { - Debug.Write("Consumer started. Integration tests will begin!"); - break; + Console.WriteLine("Workfow Executor started. Integration Tests will begin."); } - - if (i == 10) - { - throw new Exception("Workflow Mangaer Consumer not started! Integration tests will not continue"); - } - - Thread.Sleep(1000); - } + }); WorkflowPublisher = new RabbitPublisher(RabbitConnectionFactory.GetConnectionFactory(), TestExecutionConfig.RabbitConfig.Exchange, TestExecutionConfig.RabbitConfig.WorkflowRequestQueue); TaskDispatchConsumer = new RabbitConsumer(RabbitConnectionFactory.GetConnectionFactory(), TestExecutionConfig.RabbitConfig.Exchange, TestExecutionConfig.RabbitConfig.TaskDispatchQueue); diff --git a/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Support/WorkflowExecutorStartup.cs b/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Support/WorkflowExecutorStartup.cs index 3d80b8a51..311d86265 100644 --- a/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Support/WorkflowExecutorStartup.cs +++ b/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/Support/WorkflowExecutorStartup.cs @@ -114,13 +114,13 @@ public static IHost StartWorkflowExecutor() return host; } - public static async Task GetConsumers(HttpClient httpClient) + public static async Task GetQueueStatus(HttpClient httpClient, string vhost, string queue) { var svcCredentials = Convert.ToBase64String(Encoding.ASCII.GetBytes(TestExecutionConfig.RabbitConfig.User + ":" + TestExecutionConfig.RabbitConfig.Password)); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", svcCredentials); - return await httpClient.GetAsync($"http://{TestExecutionConfig.RabbitConfig.Host}:{TestExecutionConfig.RabbitConfig.Port}/api/consumers"); + return await httpClient.GetAsync($"http://{TestExecutionConfig.RabbitConfig.Host}:{TestExecutionConfig.RabbitConfig.Port}/api/queues/{vhost}/{queue}"); } } } diff --git a/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/docker-compose.yml b/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/docker-compose.yml deleted file mode 100644 index ec3f62020..000000000 --- a/tests/IntegrationTests/WorkflowExecutor.IntegrationTests/docker-compose.yml +++ /dev/null @@ -1,47 +0,0 @@ -version: '3.9' - -services: - minio: - image: "minio/minio:latest" - command: server --console-address ":9001" /data - hostname: minio - volumes: - - ./.run/minio/data:/data - - ./.run/minio/config:/root/.minio - ports: - - "9000:9000" - - "9001:9001" - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 15s - timeout: 30s - retries: 3 - rabbit1: - image: "rabbitmq:3.8.18-management" - hostname: "rabbit1" - environment: - RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQG" - RABBITMQ_DEFAULT_USER: "admin" - RABBITMQ_DEFAULT_PASS: "admin" - RABBITMQ_DEFAULT_VHOST: "monaideploy" - ports: - - "15672:15672" - - "5672:5672" - labels: - NAME: "rabbitmq1" - - mongodb_container: - image: mongo:latest - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: rootpassword - ports: - - 27017:27017 - volumes: - - mongodb_data_container:/data/db - -volumes: - mongodb_data_container: