From 2b87c48e19ecb53a7b787b9a36930c28b0f32ad9 Mon Sep 17 00:00:00 2001 From: stas Date: Fri, 4 Dec 2020 10:33:42 -0800 Subject: [PATCH] Move result URL into service, and make it set on Job Created Closes #69, #70 --- cli/raft-tools/tools/ZAP/config.json | 2 +- cli/raft_sdk/raft_service.py | 24 ++----- .../JobVerb_Tests_DeleteJob.fs | 2 +- .../JobVerb_Tests_GetJobStatus.fs | 2 +- .../JobVerb_Tests_ListJobStatuses.fs | 2 +- .../JobVerb_Tests_RePostJob.fs | 2 +- src/APIService/ApiService/Controllers/Job.fs | 20 +++--- .../ApiService/Controllers/Webhook.fs | 3 +- src/APIService/ApiService/DTOs.fs | 1 + src/Agent/RESTlerAgent/AgentMain.fs | 5 ++ src/Contracts/JobEvents.fs | 1 + src/Contracts/StorageEntities.fs | 6 +- .../OrchestratorLogic/Orchestrator.fs | 63 ++++++++++++------- 13 files changed, 78 insertions(+), 55 deletions(-) diff --git a/cli/raft-tools/tools/ZAP/config.json b/cli/raft-tools/tools/ZAP/config.json index 687fd658..a460bcbb 100644 --- a/cli/raft-tools/tools/ZAP/config.json +++ b/cli/raft-tools/tools/ZAP/config.json @@ -3,7 +3,7 @@ "run" : { "command" : "bash", "arguments" : ["-c", - "sleep $RAFT_STARTUP_DELAY; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; python3 run.py" ] + "touch /.dockerenv; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; sleep $RAFT_STARTUP_DELAY; python3 run.py" ] }, "idle" : { "command" : "bash", diff --git a/cli/raft_sdk/raft_service.py b/cli/raft_sdk/raft_service.py index 6cc9112c..38e35444 100644 --- a/cli/raft_sdk/raft_service.py +++ b/cli/raft_sdk/raft_service.py @@ -72,25 +72,6 @@ def __init__(self, context=None): self.context['tenantId'], self.context.get('secret')) - def result_url(self, job_id): - ''' - Constructs Azure File Storage results URL - - Parameters: - job_id: job ID - - Returns: - URL that contains results of the job run - ''' - return( - "https://ms.portal.azure.com/#blade/Microsoft_Azure_FileStorage/" - "FileShareMenuBlade/overview/storageAccountId/" - f"%2Fsubscriptions%2F{self.definitions.subscription}" - f"%2FresourceGroups%2F{self.definitions.resource_group}" - f"%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2F" - f"{self.definitions.storage_account}/" - f"path/{job_id}/protocol/") - def job_status(self, job_id): ''' Gets job status @@ -286,6 +267,10 @@ def print_status(self, status): for s in status: if s['agentName'] == s['jobId']: print(f"{s['jobId']} {s['state']}") + if s.get('utcEventTime'): + print(f'UtcEventTime: {s["utcEventTime"]}') + if s.get('resultsUrl'): + print(f'Results: {s["resultsUrl"]}') if s.get('details'): print("Details:") for k in s['details']: @@ -321,6 +306,7 @@ def print_status(self, status): print("Details:") for k in s['details']: print(f"{k} : {s['details'][k]}") + print('======================') def poll(self, job_id, poll_interval=10): diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs index 5dcf4d39..54970028 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs @@ -33,7 +33,7 @@ type DeleteJobTests() = let fakeMessageSender = Fixtures.createFakeMessageSender Raft.Message.ServiceBus.Queue.delete let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(System.Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created") + let entity = JobStatusEntity(System.Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs index 8fffc5cd..1f83a477 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs @@ -33,7 +33,7 @@ type GetJobStatusTests() = member this.``GET /jobs/restler succeeds`` () = async { let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "29211868-8178-4e81-9b8d-d52025b4c2d4", jobStatusJson, "Created") + let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "29211868-8178-4e81-9b8d-d52025b4c2d4", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) let jobController = jobsController(Fixtures.createFakeTelemetryClient, Fixtures.createFakeLogger) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs index 08901830..5bed0b22 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs @@ -30,7 +30,7 @@ type ListJobStatusesTests() = member this.``LIST /jobs/restler succeeds`` () = async { let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created") + let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) let jobController = jobsController(Fixtures.createFakeTelemetryClient, Fixtures.createFakeLogger) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs index c4353988..fa22a3e5 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs @@ -20,7 +20,7 @@ type jobsRePOSTTests() = let jobId = System.Guid.NewGuid().ToString() let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(jobId, jobId, jobStatusJson, "Created") + let entity = JobStatusEntity(jobId, jobId, jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) Raft.Utilities.toolsSchemas <- Map.empty.Add("RESTler", None) diff --git a/src/APIService/ApiService/Controllers/Job.fs b/src/APIService/ApiService/Controllers/Job.fs index 08c9d00a..a1414ddd 100644 --- a/src/APIService/ApiService/Controllers/Job.fs +++ b/src/APIService/ApiService/Controllers/Job.fs @@ -355,6 +355,17 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger return Some (r, results) } + let convertToJobStatus(overallJobStatus: JobStatusEntity) (results: JobStatusEntity seq) = + results + |> Seq.map (fun jobStatusEntity -> (Raft.Message.RaftEvent.deserializeEvent jobStatusEntity.JobStatus): Message.RaftEvent.RaftJobEvent) + |> Seq.map (fun jobStatus -> + if jobStatus.Message.AgentName = jobStatus.Message.JobId then + { jobStatus.Message with ResultsUrl = overallJobStatus.ResultsUrl } + else + jobStatus.Message + ) + + [] /// /// Submit a job definition. @@ -596,12 +607,8 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger - let decodedMessages = - results - |> Seq.map (fun jobStatusEntity -> (Raft.Message.RaftEvent.deserializeEvent jobStatusEntity.JobStatus): Message.RaftEvent.RaftJobEvent) - |> Seq.map (fun jobStatus -> jobStatus.Message) - + | Some (r, results) -> + let decodedMessages = convertToJobStatus r results stopWatch.Stop() Central.Telemetry.TrackMetric (TelemetryValues.ApiRequest(method, float stopWatch.ElapsedMilliseconds), "milliseconds", this :> ControllerBase) return JsonResult(decodedMessages) @@ -641,7 +648,6 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger TableEntity diff --git a/src/APIService/ApiService/DTOs.fs b/src/APIService/ApiService/DTOs.fs index 0b98b20d..2877f18b 100644 --- a/src/APIService/ApiService/DTOs.fs +++ b/src/APIService/ApiService/DTOs.fs @@ -318,4 +318,5 @@ module DTOs = AgentName : string + ResultsUrl : string } diff --git a/src/Agent/RESTlerAgent/AgentMain.fs b/src/Agent/RESTlerAgent/AgentMain.fs index 4759cc1d..ebf89653 100644 --- a/src/Agent/RESTlerAgent/AgentMain.fs +++ b/src/Agent/RESTlerAgent/AgentMain.fs @@ -506,6 +506,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = None + ResultsUrl = None }: Raft.JobEvents.JobStatus) printfn "Got job configuration message: %A" restlerPayload @@ -569,6 +570,7 @@ let main argv = Metrics = summary UtcEventTime = System.DateTime.UtcNow Details = Some( details.Add("numberOfBugsFound", sprintf "%d" bugsListLen)) + ResultsUrl = None } : Raft.JobEvents.JobStatus) } @@ -841,6 +843,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = Some (Map.ofSeq replaySummaryDetails) + ResultsUrl = None } : Raft.JobEvents.JobStatus) return replaySummaryDetails @@ -924,6 +927,7 @@ let main argv = Metrics = summary UtcEventTime = System.DateTime.UtcNow Details = details + ResultsUrl = None } : Raft.JobEvents.JobStatus) let restlerTelemetry = Restler.Telemetry.getDataFromTestingSummary testingSummary @@ -958,6 +962,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = Some (Map.empty.Add("Error", ex.Message)) + ResultsUrl = None } : Raft.JobEvents.JobStatus) do! System.Console.Error.FlushAsync().ToAsync diff --git a/src/Contracts/JobEvents.fs b/src/Contracts/JobEvents.fs index b3d87030..08f5c29d 100644 --- a/src/Contracts/JobEvents.fs +++ b/src/Contracts/JobEvents.fs @@ -73,6 +73,7 @@ type JobStatus = Details: Map option Metadata : Map option AgentName: string + ResultsUrl : string option } static member EventType = Events.JobEventTypes.JobStatus.ToString() diff --git a/src/Contracts/StorageEntities.fs b/src/Contracts/StorageEntities.fs index 2d0f9054..c92a71d1 100644 --- a/src/Contracts/StorageEntities.fs +++ b/src/Contracts/StorageEntities.fs @@ -6,11 +6,13 @@ module Raft.StorageEntities open Microsoft.Azure.Cosmos.Table let JobStatusTableName = "JobStatus" -type JobStatusEntity(jobId, agentName, jobStatus, jobState) = +type JobStatusEntity(jobId, agentName, jobStatus, jobState, utcEventTime, resultsUrl) = inherit TableEntity(partitionKey=jobId, rowKey=agentName) - new() = JobStatusEntity(null, null, null, null) + new() = JobStatusEntity(null, null, null, null, System.DateTime.MinValue, null) member val JobStatus : string = jobStatus with get, set member val JobState : string = jobState with get, set + member val ResultsUrl : string = resultsUrl with get, set + member val UtcEventTime : System.DateTime = utcEventTime with get, set let JobTableName = "Job" diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index fa1a1593..e08d6b86 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -124,7 +124,7 @@ module ContainerInstances = let containerGroupName (jobId: string) = jobId - let createJobStatus (jobId: string) (state: JobState) (details: Map option) = + let createJobStatus (jobId: string) (state: JobState) (resultsUrl : string option) (details: Map option) = let message: JobStatus = { AgentName = jobId.ToString() @@ -135,12 +135,13 @@ module ContainerInstances = UtcEventTime = System.DateTime.UtcNow Details = details Metadata = None + ResultsUrl = resultsUrl } Raft.Message.RaftEvent.createJobEvent message - let postStatus (jobStatusSender: ServiceBus.Core.MessageSender) (jobId: string) (state: JobState) (details: Map option) = + let postStatus (jobStatusSender: ServiceBus.Core.MessageSender) (jobId: string) (state: JobState) (resultsUrl : string option) (details: Map option) = async { - let jobStatus = createJobStatus jobId state details + let jobStatus = createJobStatus jobId state resultsUrl details do! jobStatusSender.SendAsync( ServiceBus.Message ( RaftEvent.serializeToBytes jobStatus ) ).ToAsync @@ -333,6 +334,15 @@ module ContainerInstances = return failwithf "Failed to get configuration for unsupported tool: %A" task.ToolName } + let jobResultsUrl subscription resourceGroup storageAccountName containerGroupName rootFileShare = + "https://ms.portal.azure.com/#blade/Microsoft_Azure_FileStorage/" + + "FileShareMenuBlade/overview/storageAccountId/" + + "%2Fsubscriptions%2F" + subscription + + "%2FresourceGroups%2F" + resourceGroup + + "%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2F" + + (sprintf "%s/" storageAccountName) + + (sprintf "path/%s/protocol/" (Option.defaultValue containerGroupName rootFileShare)) + let createJobShareAndFolders (logger: ILogger) (containerGroupName: string) (sasUrl: string) (jobCreateRequest: CreateJobRequest) = async { let shareName, createSubDirectory, shareQuota = @@ -974,14 +984,14 @@ module ContainerInstances = | :? Microsoft.Rest.Azure.CloudException as ce -> // it looks like the error when container group is transitioning states is OK to ignore. Need to get more info on that. logError "Failed to deploy container group %s due to %A (status code : %A)" containerGroupName ex ce.Response.StatusCode - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) | _ -> logError "Failed to deploy container group %s due to %A" containerGroupName ex - do! postStatus JobState.Error (Some (Map.empty.Add ("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add ("Error", ex.Message))) | _ -> logError "Failed to deploy container group %s due to %A" containerGroupName ex - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) } match existingContainerGroupOpt with @@ -1001,7 +1011,7 @@ module ContainerInstances = if decodedMessage.MessagePostCount > 0 && isError then logInfo "Message for job %A will not be reposted initial container group creation did not succeed" decodedMessage.Message.JobId else - do! postStatus JobState.Creating None + do! postStatus JobState.Creating None None match! createContainerGroupInstance containerGroupName logger azure secrets agentConfig (dockerConfigs, toolsConfigs) decodedMessage.Message reportDeploymentError with | Result.Ok () -> //this is newly created container. Poll until it is fully running and then update job status @@ -1014,7 +1024,7 @@ module ContainerInstances = stopWatch.Stop() logError "Failed to create container group for job : %A due to %A (Time it took: %f total seconds)" decodedMessage.Message.JobId ex stopWatch.Elapsed.TotalSeconds Central.Telemetry.TrackError (TelemetryValues.Exception ex) - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) | Some existingContainerGroup -> match Option.ofObj existingContainerGroup.State with @@ -1024,7 +1034,9 @@ module ContainerInstances = stopWatch.Stop() logInfo "Time took to deploy job: %s total seconds %f. State: %s; Provisioning State : %s" containerGroupName stopWatch.Elapsed.TotalSeconds state existingContainerGroup.ProvisioningState - do! postStatus JobState.Created None + + let resultsUrl = jobResultsUrl azure.SubscriptionId agentConfig.ResourceGroup agentConfig.StorageAccount containerGroupName decodedMessage.Message.JobDefinition.RootFileShare + do! postStatus JobState.Created (Some resultsUrl) None if decodedMessage.Message.IsIdlingRun then do! runDebugContainers existingContainerGroup logger agentConfig dockerConfigs toolsConfigs decodedMessage.Message @@ -1104,7 +1116,7 @@ module ContainerInstances = } - let setRow (agentConfig : AgentConfig) (jobId: string, agentName : string) (message: string, state: Raft.JobEvents.JobState) (etag: string) = + let setRow (agentConfig : AgentConfig) (jobId: string, agentName : string) (message: string, state: Raft.JobEvents.JobState, utcEventTime : DateTime, resultsUrl : string) (etag: string) = async { let! table = getJobStatusTable agentConfig.StorageTableConnectionString let entity = JobStatusEntity( @@ -1112,6 +1124,8 @@ module ContainerInstances = agentName, message, state |> Microsoft.FSharpLu.Json.Compact.serialize, + utcEventTime, + resultsUrl, ETag = etag) let insertOp = TableOperation.InsertOrReplace(entity) @@ -1147,25 +1161,29 @@ module ContainerInstances = match! JobStatus.getRow agentConfig (jobId, agentName) with | Result.Error() -> logInfo "[STATUS] Failed to retrieve job status table row for %s:%s" jobId agentName | Result.Ok(r) -> - let currentStatusWithHigherPrecedence, etag = + let currentStatusWithHigherPrecedence, utcEventTime, resultsUrl, etag = match r with | Some row -> + let resultsUrl = Option.defaultValue row.ResultsUrl decodedMessage.Message.ResultsUrl let state = JobStatus.getState row - // if current row job state is one of the next states down the line, then ignore current message altogether. - // Since current message is "late" - if state ??> decodedMessage.Message.State then - Some (JobStatus.getEvent row), row.ETag + if state = decodedMessage.Message.State && decodedMessage.Message.UtcEventTime > row.UtcEventTime then + None, decodedMessage.Message.UtcEventTime, resultsUrl, row.ETag + else if (state ??> decodedMessage.Message.State) then + // if current row job state is one of the next states down the line, then ignore current message altogether. + // Since current message is "late" + Some (JobStatus.getEvent row), row.UtcEventTime, resultsUrl, row.ETag else - None, row.ETag + None, decodedMessage.Message.UtcEventTime, resultsUrl, row.ETag | None -> - None, null + let resultsUrl = Option.defaultValue null decodedMessage.Message.ResultsUrl + None, decodedMessage.Message.UtcEventTime, resultsUrl, null match currentStatusWithHigherPrecedence with | Some currentRowMessage -> logInfo "Dropping new status message since current status has higher precedence : %A and new message state is : %A [current status: %A new message status: %s ]" currentRowMessage.Message.State decodedMessage.Message.State currentRowMessage message | None -> - let! updated = JobStatus.setRow agentConfig (jobId, agentName) (message, decodedMessage.Message.State) etag + let! updated = JobStatus.setRow agentConfig (jobId, agentName) (message, decodedMessage.Message.State, utcEventTime, resultsUrl) etag if not updated then //Table record got updated by someone else, figure out what to do next. match decodedMessage.Message.State with @@ -1298,7 +1316,10 @@ module ContainerInstances = let webhookDefinition = Microsoft.FSharpLu.Json.Compact.deserialize(jobEntity.Webhook) match webhookDefinition with | Some webhook -> - return Some webhook.Metadata + if webhook.Metadata.IsEmpty then + return None + else + return Some webhook.Metadata | None -> return None else @@ -1518,7 +1539,7 @@ module ContainerInstances = | JobState.Completing -> return false | s when JobState.Completing ??> s -> - do! postStatus communicationClients.JobEventsSender cg.Name JobState.Completing None + do! postStatus communicationClients.JobEventsSender cg.Name JobState.Completing None None let testTargets = cg.Containers |> Seq.map(fun (KeyValue(_, c)) -> c) @@ -1680,7 +1701,7 @@ module ContainerInstances = return details } - do! postStatus communicationClients.JobEventsSender g.Name state (Some detailsWithUsage) + do! postStatus communicationClients.JobEventsSender g.Name state None (Some detailsWithUsage) for v in instancesExitedWithError do let! failedContainerLogs = g.GetLogContentAsync(v.Name).ToAsync