From 361268f17f3c7af1ab2953b3a838cd9644fe0eb8 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 30 Nov 2020 14:09:59 -0800 Subject: [PATCH 01/18] Use the public restler image to build the agent. (#59) --- src/Agent/Dockerfile | 2 +- src/Agent/RESTlerAgent/RESTlerDriver.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Agent/Dockerfile b/src/Agent/Dockerfile index f1008058..726010d4 100644 --- a/src/Agent/Dockerfile +++ b/src/Agent/Dockerfile @@ -1,3 +1,3 @@ -FROM acrrestler.azurecr.io/restler:restlerv7.0 +FROM mcr.microsoft.com/restlerfuzzer/restler:v7.1.0 COPY RestlerAgent /raft/agent COPY RaftResultAnalyzer /raft/result-analyzer diff --git a/src/Agent/RESTlerAgent/RESTlerDriver.fs b/src/Agent/RESTlerAgent/RESTlerDriver.fs index 5c368a0d..d86fe1e4 100644 --- a/src/Agent/RESTlerAgent/RESTlerDriver.fs +++ b/src/Agent/RESTlerAgent/RESTlerDriver.fs @@ -7,7 +7,7 @@ open System open Microsoft.FSharpLu module RESTler = - let version = "7.0.0" + let version = "7.1.0" module private RESTlerInternal = let inline (++) (path1: string) (path2 : string) = IO.Path.Join(path1, path2) From 20509eabc1b4684f2d0758bec4f9798a7562944f Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 30 Nov 2020 15:18:33 -0800 Subject: [PATCH 02/18] Update version for new release (#60) Fields removed from restler v7.1.0 Update doc --- README.md | 4 ++-- ado/variables/version-variables.yml | 4 ++-- docs/how-it-works.md | 5 ----- src/Agent/RESTlerAgent/AgentMain.fs | 3 --- src/Agent/RESTlerAgent/RESTlerTypes.fs | 3 --- 5 files changed, 4 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ae3d19bd..d0b967a2 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ These can be configured and used in the system via configuration files and requi This project is designed to run on [Azure](https://azure.microsoft.com). To deploy the service download the CLI release and run `python raft.py service deploy`. See -the [documentation](docs/deploying/deploying.md) for more details. +the [documentation](docs/how-to-deploy.md) for more details. Once deployed, read about [how to submit a job](docs/how-to-submit-a-job.md) and use the [samples](docs/samples.md) to try out the service and fuzzers! @@ -24,7 +24,7 @@ use the [samples](docs/samples.md) to try out the service and fuzzers! ### Documentation * [Table of Contents](docs/index.md) -* [Overview](docs/how-it-works/overview.md) +* [Overview](docs/how-it-works) * [FAQ](docs/faq.md) ### Swagger Documentation diff --git a/ado/variables/version-variables.yml b/ado/variables/version-variables.yml index 403c96fc..f49722f8 100644 --- a/ado/variables/version-variables.yml +++ b/ado/variables/version-variables.yml @@ -2,9 +2,9 @@ variables: - name: version.major value: 1 - name: version.minor - value: 0 + value: 1 - name: imageTag - value: 'v1.0.0' + value: 'v1.1.0' - name: imageTagLatest value: 'v1.latest' - name: devRevision diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 086bda1b..d2084feb 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -52,11 +52,6 @@ Here's a breakdown of the estimated Azure spend for running RAFT: - Container Instances - about $0.045 cents per running hour. (A container instance is created for each submitted job and runs for the specified duration in the job definition.) -In summary, a RAFT instance that ran a series of daily tests against ten REST -APIs would cost less than $100 a month. - -NOTE: `defaults.json` provides a flag to disable deployment of Application Insights during the initial service deployment. This will reduce Azure costs without affecting service functionality. -
## How a job executes diff --git a/src/Agent/RESTlerAgent/AgentMain.fs b/src/Agent/RESTlerAgent/AgentMain.fs index 5636a45b..4759cc1d 100644 --- a/src/Agent/RESTlerAgent/AgentMain.fs +++ b/src/Agent/RESTlerAgent/AgentMain.fs @@ -910,10 +910,7 @@ let main argv = //.Add("numFullyValid", sprintf "%d" status.num_fully_valid) //.Add("numSequenceFailures", sprintf "%d" status.num_sequence_failures) //.Add("numInvalidByFailedResourceCreations", sprintf "%d" status.num_invalid_by_failed_resource_creations) - //.Add("throughput", sprintf "%f" status.throughput) //.Add("totalObjectCreations", sprintf "%d" status.total_object_creations) - //.Add("totalUniqueTestCases", sprintf "%f" status.total_unique_test_cases) - //.Add("totalSequences", sprintf "%d" status.total_sequences) ) printfn "Sending final event: %A with summary: %A and details %A" state summary details diff --git a/src/Agent/RESTlerAgent/RESTlerTypes.fs b/src/Agent/RESTlerAgent/RESTlerTypes.fs index b89992ae..903167a6 100644 --- a/src/Agent/RESTlerAgent/RESTlerTypes.fs +++ b/src/Agent/RESTlerAgent/RESTlerTypes.fs @@ -310,10 +310,7 @@ module Logs = num_fully_valid: int num_sequence_failures: int num_invalid_by_failed_resource_creations: int - throughput: float total_object_creations: int - total_unique_test_cases: float - total_sequences: int total_requests_sent : IDictionary bug_buckets : IDictionary } From ca4b4d45c93f8f10a0432e9835adb36f3e1353e0 Mon Sep 17 00:00:00 2001 From: Marc Date: Tue, 1 Dec 2020 15:24:40 -0800 Subject: [PATCH 03/18] Update variables for production build (#61) --- ado/production-build.yml | 9 +++++---- ado/variables/version-variables.yml | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ado/production-build.yml b/ado/production-build.yml index 7bb51182..500c17bd 100644 --- a/ado/production-build.yml +++ b/ado/production-build.yml @@ -5,10 +5,11 @@ trigger: - 'main' variables: - versionNumber: 1.0.0 - imageTag: 'v1.0.0' - imageTagWithBuildDate: $(imageTag)-$(Build.BuildNumber) - imageTagLatest: 'v1.latest' + - template: 'variables/version-variables.yml' + - name: versionNumber + value: $(version.major).$(version.minor).$(version.revision) + - name: imageTagWithBuildDate + value: $(imageTag)-$(Build.BuildNumber) stages: - template: stages/build/build.yml \ No newline at end of file diff --git a/ado/variables/version-variables.yml b/ado/variables/version-variables.yml index f49722f8..8ac24f2a 100644 --- a/ado/variables/version-variables.yml +++ b/ado/variables/version-variables.yml @@ -3,6 +3,8 @@ variables: value: 1 - name: version.minor value: 1 + - name: version.revision + value: 0 - name: imageTag value: 'v1.1.0' - name: imageTagLatest From 9afd1d7620312d823203c08091a9c7a4e7c939d2 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 2 Dec 2020 11:03:20 -0800 Subject: [PATCH 04/18] Fix regression when common file share is used (#63) Co-authored-by: stas --- src/Agent/RESTlerAgent/RESTlerDriver.fs | 46 ++++++++++--------- .../OrchestratorLogic/Orchestrator.fs | 9 +++- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/Agent/RESTlerAgent/RESTlerDriver.fs b/src/Agent/RESTlerAgent/RESTlerDriver.fs index d86fe1e4..b7b2f3b0 100644 --- a/src/Agent/RESTlerAgent/RESTlerDriver.fs +++ b/src/Agent/RESTlerAgent/RESTlerDriver.fs @@ -467,30 +467,32 @@ let pollForBugFound workingDirectory (token: Threading.CancellationToken) (runSt let restlerExperimentLogs = experiment.FullName ++ "logs" if IO.Directory.Exists restlerExperimentLogs then - match! getListOfBugs workingDirectory runStartTime with - | None -> () - | Some bugFiles -> - let bugsFoundPosted = restlerExperimentLogs ++ "raft-bugsfound.posted.txt" - let! postedBugs = - async { - if IO.File.Exists bugsFoundPosted then - let! bugsPosted = IO.File.ReadAllLinesAsync(bugsFoundPosted) |> Async.AwaitTask - return Set.ofArray bugsPosted - else - return ignoreBugHashes - } - let! updatedBugsPosted = - bugFiles - |> Seq.map (fun (KeyValue(bugHash, bugFile)) -> + try + match! getListOfBugs workingDirectory runStartTime with + | None -> () + | Some bugFiles -> + let bugsFoundPosted = restlerExperimentLogs ++ "raft-bugsfound.posted.txt" + let! postedBugs = async { - if not <| postedBugs.Contains bugHash then - printfn "Posting bug found %s with hash %s" bugFile.file_path bugHash - do! onBugFound (Map.empty.Add("Experiment", experiment.Name).Add("BugBucket", bugFile.file_path).Add("BugHash", bugHash)) - return bugHash + if IO.File.Exists bugsFoundPosted then + let! bugsPosted = IO.File.ReadAllLinesAsync(bugsFoundPosted) |> Async.AwaitTask + return Set.ofArray bugsPosted + else + return ignoreBugHashes } - ) |> Async.Sequential - do! IO.File.WriteAllLinesAsync(bugsFoundPosted, updatedBugsPosted) |> Async.AwaitTask - + let! updatedBugsPosted = + bugFiles + |> Seq.map (fun (KeyValue(bugHash, bugFile)) -> + async { + if not <| postedBugs.Contains bugHash then + printfn "Posting bug found %s with hash %s" bugFile.file_path bugHash + do! onBugFound (Map.empty.Add("Experiment", experiment.Name).Add("BugBucket", bugFile.file_path).Add("BugHash", bugHash)) + return bugHash + } + ) |> Async.Sequential + do! IO.File.WriteAllLinesAsync(bugsFoundPosted, updatedBugsPosted) |> Async.AwaitTask + with + | :? System.ArgumentNullException as ex -> printfn "Got excpetion while polling for bug found: %A" ex return! poll() } poll() diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index efa3967c..fa1a1593 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -505,6 +505,11 @@ module ContainerInstances = ) r, isIdling + let getTaskWorkDirectoryPath (containerGroupName : string) (rootFileShare: string option) (workDirectory : string) (taskOutputFolder : string) = + match rootFileShare with + | None -> sprintf "%s/%s" workDirectory taskOutputFolder + | Some _ -> sprintf "%s/%s/%s" workDirectory containerGroupName taskOutputFolder + let getContainerGroupInstanceConfiguration (containerGroupName: string) (logger:ILogger) @@ -539,7 +544,7 @@ module ContainerInstances = return { RunDirectory = Some runDirectory - WorkDirectory = Some(sprintf "%s/%s" workDirectory task.OutputFolder) + WorkDirectory = Some(getTaskWorkDirectoryPath containerGroupName jobCreateRequest.JobDefinition.RootFileShare workDirectory task.OutputFolder) ContainerName = (sprintf "%d-%s" i task.OutputFolder).ToLowerInvariant() ToolConfiguration = toolConfig } @@ -803,7 +808,7 @@ module ContainerInstances = RunDirectory = None WorkDirectory = match target.OutputFolder with - | Some x -> Some(sprintf "%s/%s" workDirectory x) + | Some x -> Some(getTaskWorkDirectoryPath containerGroupName jobCreateRequest.JobDefinition.RootFileShare workDirectory x) | None -> None ToolConfiguration = { From 8ae0664a09da239b4110ea90ad6763e567ec64fd Mon Sep 17 00:00:00 2001 From: Marc Date: Wed, 2 Dec 2020 11:30:50 -0800 Subject: [PATCH 05/18] Bump version for new release. (#64) --- ado/variables/version-variables.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ado/variables/version-variables.yml b/ado/variables/version-variables.yml index 8ac24f2a..d99610a1 100644 --- a/ado/variables/version-variables.yml +++ b/ado/variables/version-variables.yml @@ -4,9 +4,9 @@ variables: - name: version.minor value: 1 - name: version.revision - value: 0 + value: 1 - name: imageTag - value: 'v1.1.0' + value: 'v1.1.1' - name: imageTagLatest value: 'v1.latest' - name: devRevision From 520169030e90cf5bb807f48d6b19a2193289b553 Mon Sep 17 00:00:00 2001 From: Marc Date: Wed, 2 Dec 2020 11:38:31 -0800 Subject: [PATCH 06/18] Remove timeout from script, retain timeout in job definition. (#65) --- Scripts/Tests/bvt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Scripts/Tests/bvt.py b/Scripts/Tests/bvt.py index 365f280f..5b732080 100644 --- a/Scripts/Tests/bvt.py +++ b/Scripts/Tests/bvt.py @@ -86,7 +86,6 @@ def bvt(cli, definitions, bvt_host): fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'no-authentication', 'sample.restler.fuzz.json')) subs['{compile.jobId}'] = compile_job['jobId'] fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs) - fuzz_config.config['duration'] = '00:20:00' fuzz_job = cli.new_job(fuzz_config) cli.poll(fuzz_job['jobId'], 10) From 09d7231e9e7d1058dbd2ea9f580d97a32c1a59e0 Mon Sep 17 00:00:00 2001 From: Stas Date: Wed, 2 Dec 2020 13:04:20 -0800 Subject: [PATCH 07/18] Handle case when bughashes file deserialization returns NULL (#66) Co-authored-by: stas --- src/Agent/RESTlerAgent/RESTlerDriver.fs | 49 ++++++++++++------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/Agent/RESTlerAgent/RESTlerDriver.fs b/src/Agent/RESTlerAgent/RESTlerDriver.fs index b7b2f3b0..ea1a291e 100644 --- a/src/Agent/RESTlerAgent/RESTlerDriver.fs +++ b/src/Agent/RESTlerAgent/RESTlerDriver.fs @@ -433,7 +433,10 @@ let getListOfBugsFromBugBuckets bugBuckets = let path = bugBuckets ++ "bug_buckets.json" if IO.File.Exists path then let bugHashes: RESTlerTypes.Logs.BugHashes = Json.Compact.Strict.deserializeFile path - return Some bugHashes + if isNull (box bugHashes) then + return None + else + return Some bugHashes else return Some Map.empty else @@ -467,32 +470,28 @@ let pollForBugFound workingDirectory (token: Threading.CancellationToken) (runSt let restlerExperimentLogs = experiment.FullName ++ "logs" if IO.Directory.Exists restlerExperimentLogs then - try - match! getListOfBugs workingDirectory runStartTime with - | None -> () - | Some bugFiles -> - let bugsFoundPosted = restlerExperimentLogs ++ "raft-bugsfound.posted.txt" - let! postedBugs = + match! getListOfBugs workingDirectory runStartTime with + | None -> () + | Some bugFiles -> + let bugsFoundPosted = restlerExperimentLogs ++ "raft-bugsfound.posted.txt" + let! postedBugs = + async { + if IO.File.Exists bugsFoundPosted then + let! bugsPosted = IO.File.ReadAllLinesAsync(bugsFoundPosted) |> Async.AwaitTask + return Set.ofArray bugsPosted + else + return ignoreBugHashes + } + let! updatedBugsPosted = + bugFiles + |> Seq.map (fun (KeyValue(bugHash, bugFile)) -> async { - if IO.File.Exists bugsFoundPosted then - let! bugsPosted = IO.File.ReadAllLinesAsync(bugsFoundPosted) |> Async.AwaitTask - return Set.ofArray bugsPosted - else - return ignoreBugHashes + if not <| postedBugs.Contains bugHash then + do! onBugFound (Map.empty.Add("Experiment", experiment.Name).Add("BugBucket", bugFile.file_path).Add("BugHash", bugHash)) + return bugHash } - let! updatedBugsPosted = - bugFiles - |> Seq.map (fun (KeyValue(bugHash, bugFile)) -> - async { - if not <| postedBugs.Contains bugHash then - printfn "Posting bug found %s with hash %s" bugFile.file_path bugHash - do! onBugFound (Map.empty.Add("Experiment", experiment.Name).Add("BugBucket", bugFile.file_path).Add("BugHash", bugHash)) - return bugHash - } - ) |> Async.Sequential - do! IO.File.WriteAllLinesAsync(bugsFoundPosted, updatedBugsPosted) |> Async.AwaitTask - with - | :? System.ArgumentNullException as ex -> printfn "Got excpetion while polling for bug found: %A" ex + ) |> Async.Sequential + do! IO.File.WriteAllLinesAsync(bugsFoundPosted, updatedBugsPosted) |> Async.AwaitTask return! poll() } poll() From 281bdf3a5a33b5e92760a86a9441baec1fb7412d Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 3 Dec 2020 14:34:04 -0800 Subject: [PATCH 08/18] fix zap requirements (#71) Co-authored-by: stas --- cli/raft-tools/tools/ZAP/config.json | 5 ++++- cli/raft-tools/tools/ZAP/requirements.txt | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cli/raft-tools/tools/ZAP/config.json b/cli/raft-tools/tools/ZAP/config.json index 7eba08bc..687fd658 100644 --- a/cli/raft-tools/tools/ZAP/config.json +++ b/cli/raft-tools/tools/ZAP/config.json @@ -3,11 +3,14 @@ "run" : { "command" : "bash", "arguments" : ["-c", - "sleep $RAFT_STARTUP_DELAY; touch /.dockerenv; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; python3 run.py" ] + "sleep $RAFT_STARTUP_DELAY; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; python3 run.py" ] }, "idle" : { "command" : "bash", "arguments" : ["-c", "echo DebugMode; while true; do sleep 100000; done;"] + }, + "environmentVariables" :{ + "IS_CONTAINERIZED" : "true" } } diff --git a/cli/raft-tools/tools/ZAP/requirements.txt b/cli/raft-tools/tools/ZAP/requirements.txt index 372a51c7..23509fab 100644 --- a/cli/raft-tools/tools/ZAP/requirements.txt +++ b/cli/raft-tools/tools/ZAP/requirements.txt @@ -1,2 +1,2 @@ -applicationinsights -azure-servicebus \ No newline at end of file +applicationinsights~=0.11.9 +azure-servicebus~=0.50.3 \ No newline at end of file From 990c159d301ffd2c10bc802e50df275873bafa5b Mon Sep 17 00:00:00 2001 From: Stas Date: Thu, 3 Dec 2020 14:42:32 -0800 Subject: [PATCH 09/18] Bump version to v1.2.0 for new release (#72) Co-authored-by: stas --- ado/variables/version-variables.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ado/variables/version-variables.yml b/ado/variables/version-variables.yml index d99610a1..088b4609 100644 --- a/ado/variables/version-variables.yml +++ b/ado/variables/version-variables.yml @@ -2,11 +2,11 @@ variables: - name: version.major value: 1 - name: version.minor - value: 1 + value: 2 - name: version.revision - value: 1 + value: 0 - name: imageTag - value: 'v1.1.1' + value: 'v1.2.0' - name: imageTagLatest value: 'v1.latest' - name: devRevision From b0b7762346611c59f9341030e21028e1ec4cc296 Mon Sep 17 00:00:00 2001 From: Marc Date: Fri, 4 Dec 2020 10:33:14 -0800 Subject: [PATCH 10/18] Documentation updates (#73) --- README.md | 4 +- cli/raft.py | 3 +- docs/how-to-deploy.md | 95 +++++++++++++++++++----------- docs/images/cloud-shell-icon.jpg | Bin 0 -> 6129 bytes docs/raft-restler-relationship.md | 21 ++++--- 5 files changed, 77 insertions(+), 46 deletions(-) create mode 100644 docs/images/cloud-shell-icon.jpg diff --git a/README.md b/README.md index d0b967a2..1d70a94d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,9 @@ As a platform, RAFT is designed to host any API fuzzers that are packaged into a These can be configured and used in the system via configuration files and require no code changes to integrate. ### Getting Started -This project is designed to run on [Azure](https://azure.microsoft.com). +This project is designed to run on [Azure](https://azure.microsoft.com). See https://azure.com/free to create a free +subscription and receive $200 in credits. You can run this service (and much more!) +free for 30 days! To deploy the service download the CLI release and run `python raft.py service deploy`. See the [documentation](docs/how-to-deploy.md) for more details. diff --git a/cli/raft.py b/cli/raft.py index 628e5ba2..da1bccba 100644 --- a/cli/raft.py +++ b/cli/raft.py @@ -33,8 +33,9 @@ - no dashes region - Region to deploy RAFT (e.g. westus2) + See the documentation on container instance region availability at https://docs.microsoft.com/en-us/azure/container-instances/container-instances-region-availability - for to pick the optimal region for your deployment. + to pick the optimal region for your deployment. All jobs will be deployed by default in the same region as your service deployment diff --git a/docs/how-to-deploy.md b/docs/how-to-deploy.md index 059224ac..31f1dc74 100644 --- a/docs/how-to-deploy.md +++ b/docs/how-to-deploy.md @@ -2,32 +2,54 @@ The following guide should get you up and running with an instance of RAFT. -
+There are two main ways you can approach setting up RAFT. +- Download all requirements to your workstation and then use the RAFT CLI in a command window. +- Use the Azure Portal Cloud Shell. This requires no changes to your workstation. -### The first option is to setup all the dependencies on your workstation and use the RAFT CLI from there. The second option is to use the Azure Portal Shell. When using the portal's shell, you will only need to upload the CLI package as all required dependencies are already installed. +
-## Step 1: Enable the RAFT Command Line Interface (CLI) +## Step 1: Install the RAFT Command Line Interface Let's start out by getting the RAFT command line interface (CLI from now on) up and running. It functions just the same on Windows and Linux clients. -These two steps are required if you've decided to run the CLI from your workstation: -- First, you'll need to [install Python](https://www.python.org/downloads/) if -you don't have it installed already; RAFT requires at least **version 3.6**. +### If you are using your workstation -- Next, you'll need to [install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +- [Install Python](https://www.python.org/downloads/) if +you don't have it installed already; RAFT requires at least **version 3.6**. +- [Install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) if you haven't already; RAFT requires at least **version 2.12**. -If you've decided to use the Azure Portal Shell, keep in mind that the path to Python is `/opt/az/bin/python3` +### If you are using the Cloud Shell + +If you are going to use the Cloud Shell it is assumed that you have already acquired an Azure subscription +from https://azure.com/free or you have an existing subscription. + +Access the [Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview) from your +browser by clicking on the Cloud Shell icon.
+![](images/cloud-shell-icon.jpg) -- Now download the RAFT CLI, either just the binaries or the source tree if you intend to build them from source: +Or access it directly from your browser at https://shell.azure.com. +When using the shell for the first time it will create a storage account. This is normal and is needed to +persist data from one session to another. - - Get the RAFT CLI from [releases](https://github.com/microsoft/rest-api-fuzz-testing/releases) - - Clone the repo at https://github.com/microsoft/raft +In the cloud shell the path to Python version 3.6 `/opt/az/bin/python3` -- At this point, you're able to run a the one-time prep script using Python's -[pip package installer](https://pypi.org/project/pip/) as follows from the root -of the RAFT CLI folder: +### Common install instructions + +You will need the RAFT CLI files. You can do this in a number of ways: +- Download the RAFT CLI from a specific release
+ For example:
+ `wget https://github.com/microsoft/rest-api-fuzz-testing/releases/download/v1.2/cli.zip` +
+ Then run unzip `unzip cli.zip` +- Clone the repo +- Copy the sources + + +Once you have the python CLI files, you will need to install a few dependencies using Python's +[pip package installer](https://pypi.org/project/pip/) from the root +of the RAFT CLI folder. ```javascript $ pip install -r .\requirements.txt @@ -43,7 +65,7 @@ C:\Users\[user]\AppData\Local\Programs\Python\Python39\Scripts> pip.exe install
-- At this point, the RAFT CLI should be functional: +The RAFT CLI is now functional. ```javascript D:\REPO\raft\cli>py raft.py --help @@ -68,18 +90,18 @@ optional arguments: ## Step 2: Azure Subscription Prep -First, you will need an Azure subscription to host the RAFT service. If you +You will need an Azure subscription to host the RAFT service. If you don't already have access to an Azure subscription, please follow [these instructions](https://docs.microsoft.com/en-us/dynamics-nav/how-to--sign-up-for-a-microsoft-azure-subscription) -to sign up for one. +to sign up for one, or sign up for a free subscription at https://azure.com/free. -Second, you must be an [owner](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles) -on the target subscription to deploy RAFT, though once it's deployed you only need +You must be an [owner](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles) +on the subscription to deploy RAFT, though once it's deployed you only need [contributor](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles) rights to use it. -Last, RAFT uses [container instances](https://azure.microsoft.com/en-us/services/container-instances/) -to host running jobs; by default, Azure subscriptions are limited to 100. If your -target subscription is already using container instances, or you anticipate running +RAFT uses [container instances](https://azure.microsoft.com/en-us/services/container-instances/) +to host running jobs; by default, Azure subscriptions are limited to 100 container instances. If your +subscription is already using container instances, or you anticipate running more than 100 simultaneous jobs, you should reach out to Azure support and request they increase your [quota](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits) of this object type. @@ -96,14 +118,15 @@ Note that only four of these are required. | `subscription` | Yes | The subscription ID (GUID) of the subscription to which to deploy an instance of the RAFT service | | `deploymentName` | Yes | The name of your deployment; we will use this as the base name for all objects we create to instantiate the RAFT service | | `region` | Yes | The [region identifier](https://azure.microsoft.com/en-us/global-infrastructure/geographies/) that is closest to your location, or that's necessary for any compliance purposes | -| `metricsOptIn`* | Yes | Whether you want the service to send us anonymized usage data; we use this to improve the service and respond to errors and other problems proactively (Note: to change you choice, just update the field and redeploy) | -| `isDevelop` | No | Is this deployment for developing the RAFT service? Setting this value to true will generate yaml variables for use in your build pipelines | -| `useAppInsights` | No | deploy AppInsights and use it to write all service logs | -| `registry`** | No | registry which stores service images. Default: mcr.microsoft.com | +| `metricsOptIn`* | Yes | Whether you want the service to send us anonymized usage data; we use this to improve the service and respond to errors and other problems proactively (Note: to change you choice, just update the field and redeploy) Default: true| +| `isDevelop` | No | Is this deployment for developing the RAFT service? Default: false | +| `isPrivateRegistry` | No | When developing for the RAFT service, indicates a private registry is used to find images. Default: false | +| `useAppInsights` | Yes | Deploy AppInsights and use it to write all service logs. Default: true | +| `registry` | Yes | Registry which stores service images. Default: mcr.microsoft.com | ### *Telemetry *By default, we collect anonymous usage data from your RAFT instance, which helps -us understand how users use RAFT and the problems they experience, which in turn +us understand how users use RAFT and the problems they experience, which in turn, helps us improve the quality of the offering over time. Specifically, We do **not** collect any data about the targets and results of tools you might run. The data fields we collect are defined in the `src/Contracts/Telemetry.fs` source file. To opt-out of @@ -146,12 +169,11 @@ deploymentName - RAFT deployment name - no dashes region - Region to deploy RAFT (e.g. westus2) - See https://azure.microsoft.com/en-us/global-infrastructure/regions/ - for a list of regions - -isDevelop - Is this deployment for developing the RAFT service? - Setting this value to true will generate yaml variables for use in your - build pipelines. + See the documentation on container instance region availability at + https://docs.microsoft.com/en-us/azure/container-instances/container-instances-region-availability + to pick the optimal region for your deployment. + All jobs will be deployed by default in the same + region as your service deployment metricsOptIn - allow Microsoft collect anonymized metrics from the deployment. @@ -159,6 +181,7 @@ useAppInsights - deploy AppInsights and use it to write all service logs registry - registry which stores service images. Default: mcr.microsoft.com + ------------------------- To apply any changes made to the defaults.json file, please run 'raft.py service deploy' @@ -167,7 +190,7 @@ please run 'raft.py service deploy'
-Using a text editor of your choice, please update the `defaults.json` file with +Using a text editor of your choice, update the `defaults.json` file with the values you determined in Step 3, and then re-run: ```javascript @@ -196,6 +219,6 @@ get familiar with the output. Two tools are deployed by default: [RESTler](https://github.com/microsoft/restler) and [ZAP](https://www.zaproxy.org/). -You can see their configuration under the `cli/raft-tools/tools` folder. +You can see how they are configured by looking at the configuration files under the `cli/raft-tools/tools` folder. -See an explanation of the `config.json` file in [How a job executes](how-it-works/how-a-job-executes.md). \ No newline at end of file +See an explanation of the `config.json` file in [How to onboard a tool](how-to-onboard-a-tool.md). \ No newline at end of file diff --git a/docs/images/cloud-shell-icon.jpg b/docs/images/cloud-shell-icon.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3df73d416c107d20753e01e8b61c948f2e0d03f4 GIT binary patch literal 6129 zcmeHKXIK+kyB(TT!5~F!Q3R=yqkt$V5s@xUsvvSi5g`In1q9}ZM}a6rq$w>*RfjQd+ucBN1kM_cdfnG-t#W_6FdxTG1N8C z1=!fwfZvf10QUph00%q!#*JK@$j!yW#l^|V#mCLPiAR7>K!Bf*pMSHU@RrSjC_#Sy zEn-_xTSY`gMFoV!w~2{t6BZE_*#KeVK*n%#@p5tTifrcJEb>3T;MV|(2T);in~$%b|Mh^tpj&_5z7rl1iGL6q7oU*$@KI8F zMrKxa&eLbP#U-U>FUwz5RMyouG&VKAZE5Z5{z&fW{q(tSWOQtN;_Kwp^bB=rd1dw6 z8ja4_(8UID{D-W+l>MR$h3I1E)aO&*bx zF;9x#@a|JSvncB1-N`2=r7|K--H`TO*?&)1*#An|U&8)VS05n2!Gk8rgL<)bX+A@aB1ff^>z50kn(E*3k_UCG{a|lRFzNhDA*r_qsooiS zo=1@x(Tj=dP-bz);jD+4FJNHY$eqeD>UNJ9p@ytRp%e^k_hc!WMuID5&#{YK*rsP~ zP<~f#RL0Hju3Dd6F?E4?ZA%Q5F3YjQlk@gfaT(5S=Kfg2mSsgzrR~vyX;y`DhSk&3 z2Az0AmFad$T3jk+fBF0qPe@Frfy9utLDSebobqFZ+BY{D62H(zH_+3duZrh^Zn6Z5{_po_WISL;Qe^$-qtWf>=LaMVa( zG*#W_S+3>Ql8Kk8@6j|#SnQ_MjGwWdb?W_4_dZx~wwbjtw&em0yiNRt=9OQta{boS=E|6gG$(BMx4|RS zfY`~=2S#z;YE3f!#9Vp+k|EO8(Ttacn z4qczH9YSxW-(XYKYZHB*%=A{ekB3?fq)odQh#_`w(W8Tx<{S=?^#%+*TBH4&Bob+rmK+X`zB$aCw~kA5$At*KXH-B zGJ%2gn_e*RhXJGl)`3(F2+!3FvB}WL>as9kDVESwcb#?CfiSH?Sn%2~;+=x;l0VBoa* z@jRni97c#HWLV@WFzqv=zO(n_nC%Bek6NySkp?R61Y=OR*M*Y|lnN3Y8x6W`jW5m5cvch*(PIa<}qp?$7FmP4v{v&^7r z!FBAm?HJnWQM^K)eMiX^hH?Ib&*z%G#FNER-FM=Pb>9_~b;iX@UD`g9$!u9IREQvr z6VI1|D-+5K$21TZH(`ex!I|Suh78Ow7LTsiBk}FamO{qA%=D_L2G> zc=Cru(I1MTzngjyQWd^HsDR13+2M!2#y%EZEM>2Yem;i=4KDg9mg7`Y7Ngz^ljOUe zdy7w4HR(@I`2=}nOEkx~wJ5BFh%sQ`5xvLU+@j%`kL+}n|4d~~DQVo-kvI4PHJ90F zCzg>D?TnuAr!>zq$tL7dW&y@5+%wtr2EVqv zv-akhJn^`FM`?gB(v@;bwW1_D0QXHiO;?iTiD1s>_nv zr8g6*e9aE~-(JKD(_+o2{xvu9-Aa;^nJvd!F1_aX0C2ON0DvRNEqce%Gymf_9$#!M z-9~=3K0Pp?ly;eZ8!9kLGclRZ9#T7%wWjKPnL(F`GuH{>UCud+^lXO374A2zKORV6 z^q%Y{m1;K3St;%OqV2i8h;!GI1<{A>yjFWqDZNvPbB$D&)3l<+=+?SHds)@@hd$U2 zx(@228btkd>-Ok6$&25-N%lVIm^LK1S}5pRX?pi+wN_y6Ss1wLO0|8nKl#G0o-Yn> zuC>3JybFbpPe`;cc^@9BT9cgEXN4Hzq;6b)Va2G zi}T>qPq7DsTOV0C3K56pzFv#p9&PA4D0VNYOMX_z-9Ge&zvg84GT$Qy{hsXfVRBBO zj%uLBafcNH-B_tT_Kfp-1I9jJy+AYGN93}=B<}LeapB+_tR719pEADnwJ8s zhLYU_3QMz1=l6Pw2h=yLs?J%o) z;7u|fA1#_K-}e>SAeM zO0>(a$}{lFwyk${7oge?=pU={yYBx^S!}Z;D>L5w-lZ2yL92xD$~pCgTiLYGcB2{v z$F5AG#fzk_QJxDGRrdlD<|SXfE7^2lJL@xFy~KoR$wMC|<~nY6+J%(R?-uYGlqp*) zf0oJ->WYRD^p@m?b;U+vokKsn!)^L@rtl%PK^( zF#Bv@W83&8g^Q9n7l7`Q>~%J3TYX zJG#{4nt9n(i5sg)iPui^PmjhcbTpVn88l{54wua(R0hq{=P!P!?~OlVptg{?vM`v$ z@TA+V9W|6qui@z6TLsyty{6Pn{^;qRVt|psx-Mz`w!N))9S(O1Pf1)?kbPjVFJ3!d z+F$95m{g!nP+?em%vcn+Znx*ioi)o&oyPp2tG!{Jlmq)+?(Yb+o!{E1_~3}{$l4!l zdv1Llqs%4ZX4f==3vojxjB_h5#J&~QlrqWlleoUVZ$0%Y=!ikdp3+pL7jEcV&B3eO z<={y^N!>MObi7M{LcjD2|GqurrB|zT)$iSE3RtdxL9=PJsSL^Q@VXeWp!oIvh4Jzv z)daB%pB|2%^lCB=x-`VSQX|$h$<#LORk`BACs;T-Yt`*kOt^OWisV^|g^4|)X0N!t zG;S0=qAfR=(eelay>6tI%~k#St+$Z<>kVMDyc2; z(k6n+7!!F64cCFhPPL8Lxdn-xeM6Nn5EcLf9_QMgA)U^6NYVS|`1Po&%ga>~%oFX( zy`thQjq6aZ3}lXFrjz7yB1WF;ytf)0$U*vGV!mZPRqa@XoPwh<(igXEBQ5r(3Btf* z52WP^-3tR_rC|N}01fD7<>Wdo90sC$dVfG8(EKUumZq$9a81W`HF_J;>vb$3?VhB| zYADhQ5)~-O5Gf}Z_!$G~!8QjXNq5sMacvY0Y1lh1&JQu366vR4U;&RE=_jq4W0xXf zU_d0~;vMYdGIN7#IGL6K1sB$$SuuBi0nlTqP9jlx2NIPdK-#=JSYHm#q=OSkny*B7 zgnPgM8NqIqfdOChA9@SO9Pj5F+XFVf1QZ3G8hmXKmr%SE6x;}%7AuNA>A%g zLYBDRsn>JlA_QV9K~_YFBOW{1${azabEi!d47hm;%tAsvrB%Dsti^45y!$fF=?P0x zPF6_in@V(;keAj&^8OAQ38cr~{)Y0(um|OkKq?a(^>&#R9G7R_&-{Qi%7n-X7_fG2 zS+kjTv<^PA9_EMPpov@btPoHG0eNO+OVtIEZKpgBhjgXys&e*Ox}$h4=k{^+-mtg zPBZo5XFQa$WSP&SWOFFSGPyWo6LtI?IAX}sLz2N?Xd+-;JdJ>~m1~Vy+J3}OqY6VX za6SNp!q5<#1q|HRG==ucAg1}Ch+Pat!$3lcpWa5lyJF0!3<>(jb0+wq$>Kl&9WkM34%DG{i& z4OVrLtFZ7NPwLCGCpR95bs4;d4G%WBKY9P1@^;j1+3x4b-#$9$KefE)8>KriAyw+& z_hGqGVczivU!#BYCKJ1whnVPlq9M_@B5$@;i=)VQBrIX+`$lvBW+;*@h~INmF|ugd z>z3`{M&uBJ^l7SPB)_N_1!o4r-fzUk9+NxU1L`a(!YnOHY7v|{lCsNou91mZ?v9sH8PNvwc~0d Zb_HcUl}<|QnEE{(FAMyO9}f8QKLGPha^(O3 literal 0 HcmV?d00001 diff --git a/docs/raft-restler-relationship.md b/docs/raft-restler-relationship.md index f9c1dfa0..8e2bbdd6 100644 --- a/docs/raft-restler-relationship.md +++ b/docs/raft-restler-relationship.md @@ -6,23 +6,28 @@ In order to provide a consistent platform for supporting multiple test tools the RAFT supports all of the same configuration values that RESTler supports. RESTler uses a mix of camel case and underscore parameters that it accepts as part of it's run configurations. On the other hand RAFT only uses camel case parameters. For example RESTler uses parameter `restler_custom_payload` where the same parameter is called `restlerCustomPayload` in RAFT job definition. -RESTler documentation: +You can find the RESTler documentation at https://github.com/microsoft/restler-fuzzer/tree/main/docs/user-guide -When using RESTler documentation for configuring RESTler tasks you can use RAFT swagger definition for paramater name conversion. +When using RESTler documentation for configuring RESTler tasks use the RAFT swagger definition for paramater name conversion. -*https://-raft-apiservice.azurewebsites.net/swagger/index.html* +The RAFT swagger definition can be found using this URL : *https://\-raft-apiservice.azurewebsites.net/swagger/index.html* -RESTler requires that IP and port number are specified in order to run a test. However, RAFT will do a DNS lookup on the host parameter, specified in the job definition file, on your behalf and fill in the IP parameter for you. RAFT also defaults the port number to 443 when using SSL and 80 when not using a secure connection. If for some reason you find that you still need to specify the IP and port number then manually provided values in TargetEndpoint configuration will override any lookup or default values. +RAFT will do a DNS lookup on the host parameter, specified in the job definition file, +on your behalf and fill in the IP parameter for you. RAFT also defaults the port number +to 443 when using SSL and 80 when not using a secure connection. If for some reason you +find that you still need to specify the IP and port number then manually provided values +in TargetEndpoint configuration will override any lookup or default values. ## RESTler mode of opearation -First RESTler compiles Swagger specifications into RESTler grammar. The output of compile step can be consumed by any of the following steps: Compile, Test, Fuzz. +RESTler needs to compile the Swagger specifications into RESTler grammar as a first step. The output of the compile step is then consumed by any of the following steps: Test, TestFuzzLean, Fuzz. To enable passing of data from one step to the next RAFT allows any file share in the storage account to be mounted by any task. -This way RAFT jobs can be executed in a "pipeline" manner by passing the output of the Compile job as input Test, Fuzz or Compile jobs. +This way RAFT jobs can be executed in a "pipeline" manner by passing the output of the Compile job as input to the Test, TestFuzzLean, or Fuzz jobs. -Compile job produces a job ID and a file share named the same as the job ID. This makes it possible for you to mount the output of one job as input to another. -The diagram below demonstrates how to pass output from the Compile step as input to the Test step of a RESTler task. +A compile job produces a job ID and a file share is created and named using the job ID. +This makes it possible for you to take the output written to a file share of one job and mount it to use as input to another. +The diagram below illustrates this behavior in the job definition files. ![RESTler ](images/restler-configs-flow.png) From 857aa0f51cd411ead236a1b96023edd1c37cfe1b Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 7 Dec 2020 08:26:18 -0800 Subject: [PATCH 11/18] Move result URL into service, and make it set on Job Created (#75) Closes #69, #70 Co-authored-by: stas --- cli/raft-tools/tools/ZAP/config.json | 2 +- cli/raft_sdk/raft_service.py | 24 ++----- .../JobVerb_Tests_DeleteJob.fs | 2 +- .../JobVerb_Tests_GetJobStatus.fs | 2 +- .../JobVerb_Tests_ListJobStatuses.fs | 2 +- .../JobVerb_Tests_RePostJob.fs | 2 +- src/APIService/ApiService/Controllers/Job.fs | 20 +++--- .../ApiService/Controllers/Webhook.fs | 3 +- src/APIService/ApiService/DTOs.fs | 1 + src/Agent/RESTlerAgent/AgentMain.fs | 5 ++ src/Contracts/JobEvents.fs | 1 + src/Contracts/StorageEntities.fs | 6 +- .../OrchestratorLogic/Orchestrator.fs | 63 ++++++++++++------- 13 files changed, 78 insertions(+), 55 deletions(-) diff --git a/cli/raft-tools/tools/ZAP/config.json b/cli/raft-tools/tools/ZAP/config.json index 687fd658..a460bcbb 100644 --- a/cli/raft-tools/tools/ZAP/config.json +++ b/cli/raft-tools/tools/ZAP/config.json @@ -3,7 +3,7 @@ "run" : { "command" : "bash", "arguments" : ["-c", - "sleep $RAFT_STARTUP_DELAY; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; python3 run.py" ] + "touch /.dockerenv; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; sleep $RAFT_STARTUP_DELAY; python3 run.py" ] }, "idle" : { "command" : "bash", diff --git a/cli/raft_sdk/raft_service.py b/cli/raft_sdk/raft_service.py index 6cc9112c..38e35444 100644 --- a/cli/raft_sdk/raft_service.py +++ b/cli/raft_sdk/raft_service.py @@ -72,25 +72,6 @@ def __init__(self, context=None): self.context['tenantId'], self.context.get('secret')) - def result_url(self, job_id): - ''' - Constructs Azure File Storage results URL - - Parameters: - job_id: job ID - - Returns: - URL that contains results of the job run - ''' - return( - "https://ms.portal.azure.com/#blade/Microsoft_Azure_FileStorage/" - "FileShareMenuBlade/overview/storageAccountId/" - f"%2Fsubscriptions%2F{self.definitions.subscription}" - f"%2FresourceGroups%2F{self.definitions.resource_group}" - f"%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2F" - f"{self.definitions.storage_account}/" - f"path/{job_id}/protocol/") - def job_status(self, job_id): ''' Gets job status @@ -286,6 +267,10 @@ def print_status(self, status): for s in status: if s['agentName'] == s['jobId']: print(f"{s['jobId']} {s['state']}") + if s.get('utcEventTime'): + print(f'UtcEventTime: {s["utcEventTime"]}') + if s.get('resultsUrl'): + print(f'Results: {s["resultsUrl"]}') if s.get('details'): print("Details:") for k in s['details']: @@ -321,6 +306,7 @@ def print_status(self, status): print("Details:") for k in s['details']: print(f"{k} : {s['details'][k]}") + print('======================') def poll(self, job_id, poll_interval=10): diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs index 5dcf4d39..54970028 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs @@ -33,7 +33,7 @@ type DeleteJobTests() = let fakeMessageSender = Fixtures.createFakeMessageSender Raft.Message.ServiceBus.Queue.delete let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(System.Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created") + let entity = JobStatusEntity(System.Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs index 8fffc5cd..1f83a477 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs @@ -33,7 +33,7 @@ type GetJobStatusTests() = member this.``GET /jobs/restler succeeds`` () = async { let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "29211868-8178-4e81-9b8d-d52025b4c2d4", jobStatusJson, "Created") + let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "29211868-8178-4e81-9b8d-d52025b4c2d4", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) let jobController = jobsController(Fixtures.createFakeTelemetryClient, Fixtures.createFakeLogger) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs index 08901830..5bed0b22 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs @@ -30,7 +30,7 @@ type ListJobStatusesTests() = member this.``LIST /jobs/restler succeeds`` () = async { let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created") + let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) let jobController = jobsController(Fixtures.createFakeTelemetryClient, Fixtures.createFakeLogger) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs index c4353988..fa22a3e5 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs @@ -20,7 +20,7 @@ type jobsRePOSTTests() = let jobId = System.Guid.NewGuid().ToString() let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(jobId, jobId, jobStatusJson, "Created") + let entity = JobStatusEntity(jobId, jobId, jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) Raft.Utilities.toolsSchemas <- Map.empty.Add("RESTler", None) diff --git a/src/APIService/ApiService/Controllers/Job.fs b/src/APIService/ApiService/Controllers/Job.fs index 08c9d00a..a1414ddd 100644 --- a/src/APIService/ApiService/Controllers/Job.fs +++ b/src/APIService/ApiService/Controllers/Job.fs @@ -355,6 +355,17 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger return Some (r, results) } + let convertToJobStatus(overallJobStatus: JobStatusEntity) (results: JobStatusEntity seq) = + results + |> Seq.map (fun jobStatusEntity -> (Raft.Message.RaftEvent.deserializeEvent jobStatusEntity.JobStatus): Message.RaftEvent.RaftJobEvent) + |> Seq.map (fun jobStatus -> + if jobStatus.Message.AgentName = jobStatus.Message.JobId then + { jobStatus.Message with ResultsUrl = overallJobStatus.ResultsUrl } + else + jobStatus.Message + ) + + [] /// /// Submit a job definition. @@ -596,12 +607,8 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger - let decodedMessages = - results - |> Seq.map (fun jobStatusEntity -> (Raft.Message.RaftEvent.deserializeEvent jobStatusEntity.JobStatus): Message.RaftEvent.RaftJobEvent) - |> Seq.map (fun jobStatus -> jobStatus.Message) - + | Some (r, results) -> + let decodedMessages = convertToJobStatus r results stopWatch.Stop() Central.Telemetry.TrackMetric (TelemetryValues.ApiRequest(method, float stopWatch.ElapsedMilliseconds), "milliseconds", this :> ControllerBase) return JsonResult(decodedMessages) @@ -641,7 +648,6 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger TableEntity diff --git a/src/APIService/ApiService/DTOs.fs b/src/APIService/ApiService/DTOs.fs index 0b98b20d..2877f18b 100644 --- a/src/APIService/ApiService/DTOs.fs +++ b/src/APIService/ApiService/DTOs.fs @@ -318,4 +318,5 @@ module DTOs = AgentName : string + ResultsUrl : string } diff --git a/src/Agent/RESTlerAgent/AgentMain.fs b/src/Agent/RESTlerAgent/AgentMain.fs index 4759cc1d..ebf89653 100644 --- a/src/Agent/RESTlerAgent/AgentMain.fs +++ b/src/Agent/RESTlerAgent/AgentMain.fs @@ -506,6 +506,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = None + ResultsUrl = None }: Raft.JobEvents.JobStatus) printfn "Got job configuration message: %A" restlerPayload @@ -569,6 +570,7 @@ let main argv = Metrics = summary UtcEventTime = System.DateTime.UtcNow Details = Some( details.Add("numberOfBugsFound", sprintf "%d" bugsListLen)) + ResultsUrl = None } : Raft.JobEvents.JobStatus) } @@ -841,6 +843,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = Some (Map.ofSeq replaySummaryDetails) + ResultsUrl = None } : Raft.JobEvents.JobStatus) return replaySummaryDetails @@ -924,6 +927,7 @@ let main argv = Metrics = summary UtcEventTime = System.DateTime.UtcNow Details = details + ResultsUrl = None } : Raft.JobEvents.JobStatus) let restlerTelemetry = Restler.Telemetry.getDataFromTestingSummary testingSummary @@ -958,6 +962,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = Some (Map.empty.Add("Error", ex.Message)) + ResultsUrl = None } : Raft.JobEvents.JobStatus) do! System.Console.Error.FlushAsync().ToAsync diff --git a/src/Contracts/JobEvents.fs b/src/Contracts/JobEvents.fs index b3d87030..08f5c29d 100644 --- a/src/Contracts/JobEvents.fs +++ b/src/Contracts/JobEvents.fs @@ -73,6 +73,7 @@ type JobStatus = Details: Map option Metadata : Map option AgentName: string + ResultsUrl : string option } static member EventType = Events.JobEventTypes.JobStatus.ToString() diff --git a/src/Contracts/StorageEntities.fs b/src/Contracts/StorageEntities.fs index 2d0f9054..c92a71d1 100644 --- a/src/Contracts/StorageEntities.fs +++ b/src/Contracts/StorageEntities.fs @@ -6,11 +6,13 @@ module Raft.StorageEntities open Microsoft.Azure.Cosmos.Table let JobStatusTableName = "JobStatus" -type JobStatusEntity(jobId, agentName, jobStatus, jobState) = +type JobStatusEntity(jobId, agentName, jobStatus, jobState, utcEventTime, resultsUrl) = inherit TableEntity(partitionKey=jobId, rowKey=agentName) - new() = JobStatusEntity(null, null, null, null) + new() = JobStatusEntity(null, null, null, null, System.DateTime.MinValue, null) member val JobStatus : string = jobStatus with get, set member val JobState : string = jobState with get, set + member val ResultsUrl : string = resultsUrl with get, set + member val UtcEventTime : System.DateTime = utcEventTime with get, set let JobTableName = "Job" diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index fa1a1593..e08d6b86 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -124,7 +124,7 @@ module ContainerInstances = let containerGroupName (jobId: string) = jobId - let createJobStatus (jobId: string) (state: JobState) (details: Map option) = + let createJobStatus (jobId: string) (state: JobState) (resultsUrl : string option) (details: Map option) = let message: JobStatus = { AgentName = jobId.ToString() @@ -135,12 +135,13 @@ module ContainerInstances = UtcEventTime = System.DateTime.UtcNow Details = details Metadata = None + ResultsUrl = resultsUrl } Raft.Message.RaftEvent.createJobEvent message - let postStatus (jobStatusSender: ServiceBus.Core.MessageSender) (jobId: string) (state: JobState) (details: Map option) = + let postStatus (jobStatusSender: ServiceBus.Core.MessageSender) (jobId: string) (state: JobState) (resultsUrl : string option) (details: Map option) = async { - let jobStatus = createJobStatus jobId state details + let jobStatus = createJobStatus jobId state resultsUrl details do! jobStatusSender.SendAsync( ServiceBus.Message ( RaftEvent.serializeToBytes jobStatus ) ).ToAsync @@ -333,6 +334,15 @@ module ContainerInstances = return failwithf "Failed to get configuration for unsupported tool: %A" task.ToolName } + let jobResultsUrl subscription resourceGroup storageAccountName containerGroupName rootFileShare = + "https://ms.portal.azure.com/#blade/Microsoft_Azure_FileStorage/" + + "FileShareMenuBlade/overview/storageAccountId/" + + "%2Fsubscriptions%2F" + subscription + + "%2FresourceGroups%2F" + resourceGroup + + "%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2F" + + (sprintf "%s/" storageAccountName) + + (sprintf "path/%s/protocol/" (Option.defaultValue containerGroupName rootFileShare)) + let createJobShareAndFolders (logger: ILogger) (containerGroupName: string) (sasUrl: string) (jobCreateRequest: CreateJobRequest) = async { let shareName, createSubDirectory, shareQuota = @@ -974,14 +984,14 @@ module ContainerInstances = | :? Microsoft.Rest.Azure.CloudException as ce -> // it looks like the error when container group is transitioning states is OK to ignore. Need to get more info on that. logError "Failed to deploy container group %s due to %A (status code : %A)" containerGroupName ex ce.Response.StatusCode - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) | _ -> logError "Failed to deploy container group %s due to %A" containerGroupName ex - do! postStatus JobState.Error (Some (Map.empty.Add ("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add ("Error", ex.Message))) | _ -> logError "Failed to deploy container group %s due to %A" containerGroupName ex - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) } match existingContainerGroupOpt with @@ -1001,7 +1011,7 @@ module ContainerInstances = if decodedMessage.MessagePostCount > 0 && isError then logInfo "Message for job %A will not be reposted initial container group creation did not succeed" decodedMessage.Message.JobId else - do! postStatus JobState.Creating None + do! postStatus JobState.Creating None None match! createContainerGroupInstance containerGroupName logger azure secrets agentConfig (dockerConfigs, toolsConfigs) decodedMessage.Message reportDeploymentError with | Result.Ok () -> //this is newly created container. Poll until it is fully running and then update job status @@ -1014,7 +1024,7 @@ module ContainerInstances = stopWatch.Stop() logError "Failed to create container group for job : %A due to %A (Time it took: %f total seconds)" decodedMessage.Message.JobId ex stopWatch.Elapsed.TotalSeconds Central.Telemetry.TrackError (TelemetryValues.Exception ex) - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) | Some existingContainerGroup -> match Option.ofObj existingContainerGroup.State with @@ -1024,7 +1034,9 @@ module ContainerInstances = stopWatch.Stop() logInfo "Time took to deploy job: %s total seconds %f. State: %s; Provisioning State : %s" containerGroupName stopWatch.Elapsed.TotalSeconds state existingContainerGroup.ProvisioningState - do! postStatus JobState.Created None + + let resultsUrl = jobResultsUrl azure.SubscriptionId agentConfig.ResourceGroup agentConfig.StorageAccount containerGroupName decodedMessage.Message.JobDefinition.RootFileShare + do! postStatus JobState.Created (Some resultsUrl) None if decodedMessage.Message.IsIdlingRun then do! runDebugContainers existingContainerGroup logger agentConfig dockerConfigs toolsConfigs decodedMessage.Message @@ -1104,7 +1116,7 @@ module ContainerInstances = } - let setRow (agentConfig : AgentConfig) (jobId: string, agentName : string) (message: string, state: Raft.JobEvents.JobState) (etag: string) = + let setRow (agentConfig : AgentConfig) (jobId: string, agentName : string) (message: string, state: Raft.JobEvents.JobState, utcEventTime : DateTime, resultsUrl : string) (etag: string) = async { let! table = getJobStatusTable agentConfig.StorageTableConnectionString let entity = JobStatusEntity( @@ -1112,6 +1124,8 @@ module ContainerInstances = agentName, message, state |> Microsoft.FSharpLu.Json.Compact.serialize, + utcEventTime, + resultsUrl, ETag = etag) let insertOp = TableOperation.InsertOrReplace(entity) @@ -1147,25 +1161,29 @@ module ContainerInstances = match! JobStatus.getRow agentConfig (jobId, agentName) with | Result.Error() -> logInfo "[STATUS] Failed to retrieve job status table row for %s:%s" jobId agentName | Result.Ok(r) -> - let currentStatusWithHigherPrecedence, etag = + let currentStatusWithHigherPrecedence, utcEventTime, resultsUrl, etag = match r with | Some row -> + let resultsUrl = Option.defaultValue row.ResultsUrl decodedMessage.Message.ResultsUrl let state = JobStatus.getState row - // if current row job state is one of the next states down the line, then ignore current message altogether. - // Since current message is "late" - if state ??> decodedMessage.Message.State then - Some (JobStatus.getEvent row), row.ETag + if state = decodedMessage.Message.State && decodedMessage.Message.UtcEventTime > row.UtcEventTime then + None, decodedMessage.Message.UtcEventTime, resultsUrl, row.ETag + else if (state ??> decodedMessage.Message.State) then + // if current row job state is one of the next states down the line, then ignore current message altogether. + // Since current message is "late" + Some (JobStatus.getEvent row), row.UtcEventTime, resultsUrl, row.ETag else - None, row.ETag + None, decodedMessage.Message.UtcEventTime, resultsUrl, row.ETag | None -> - None, null + let resultsUrl = Option.defaultValue null decodedMessage.Message.ResultsUrl + None, decodedMessage.Message.UtcEventTime, resultsUrl, null match currentStatusWithHigherPrecedence with | Some currentRowMessage -> logInfo "Dropping new status message since current status has higher precedence : %A and new message state is : %A [current status: %A new message status: %s ]" currentRowMessage.Message.State decodedMessage.Message.State currentRowMessage message | None -> - let! updated = JobStatus.setRow agentConfig (jobId, agentName) (message, decodedMessage.Message.State) etag + let! updated = JobStatus.setRow agentConfig (jobId, agentName) (message, decodedMessage.Message.State, utcEventTime, resultsUrl) etag if not updated then //Table record got updated by someone else, figure out what to do next. match decodedMessage.Message.State with @@ -1298,7 +1316,10 @@ module ContainerInstances = let webhookDefinition = Microsoft.FSharpLu.Json.Compact.deserialize(jobEntity.Webhook) match webhookDefinition with | Some webhook -> - return Some webhook.Metadata + if webhook.Metadata.IsEmpty then + return None + else + return Some webhook.Metadata | None -> return None else @@ -1518,7 +1539,7 @@ module ContainerInstances = | JobState.Completing -> return false | s when JobState.Completing ??> s -> - do! postStatus communicationClients.JobEventsSender cg.Name JobState.Completing None + do! postStatus communicationClients.JobEventsSender cg.Name JobState.Completing None None let testTargets = cg.Containers |> Seq.map(fun (KeyValue(_, c)) -> c) @@ -1680,7 +1701,7 @@ module ContainerInstances = return details } - do! postStatus communicationClients.JobEventsSender g.Name state (Some detailsWithUsage) + do! postStatus communicationClients.JobEventsSender g.Name state None (Some detailsWithUsage) for v in instancesExitedWithError do let! failedContainerLogs = g.GetLogContentAsync(v.Name).ToAsync From 7d7adc939f07425df64f50789f6df5f7567014e7 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 7 Dec 2020 08:40:09 -0800 Subject: [PATCH 12/18] Get the correct assembly for the version number. #74 (#77) --- src/Contracts/Telemetry.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Contracts/Telemetry.fs b/src/Contracts/Telemetry.fs index b999c98a..abfbb2f2 100644 --- a/src/Contracts/Telemetry.fs +++ b/src/Contracts/Telemetry.fs @@ -64,7 +64,7 @@ module Central = type TelemetryImpl(telemetryConfig: (TelemetryClient * string) option) = let convertTagsToProperties tags = tags |> dict - let version = Assembly.GetEntryAssembly().GetName().Version + let version = Assembly.GetCallingAssembly().GetName().Version member _.TelemetryConfig = telemetryConfig From 2b2cb591f2820a8ee45a6eb004f238ce6d835ead Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 7 Dec 2020 14:12:20 -0800 Subject: [PATCH 13/18] Switch BVT runs to petstore container (rather than permanent live web service) (#78) Co-authored-by: stas --- .../{bvt-petstore.py => bvt-petstore3.py} | 39 ++++----- ado/stages/test/steps/bvt.yml | 2 +- .../swagger-petstore/restler.compile.json | 5 +- .../swagger-petstore/restler.fuzz.json | 7 +- .../swagger-petstore/restler.test.json | 7 +- .../self-contained/swagger-petstore/run.py | 4 - .../{restler.compile.json => compile.json} | 13 +-- .../{restler.zap.fuzz.json => fuzz.json} | 46 ++++++----- .../self-contained/swagger-petstore3/run.py | 79 +++++++++++-------- .../swagger-petstore3/test-fuzz-lean.json | 59 ++++++++++++++ .../{restler.test.json => test.json} | 17 +++- src/Agent/RESTlerAgent/RESTlerDriver.fs | 23 +++--- src/Contracts/Job.fs | 2 +- .../OrchestratorLogic/Orchestrator.fs | 9 ++- 14 files changed, 192 insertions(+), 120 deletions(-) rename Scripts/Tests/{bvt-petstore.py => bvt-petstore3.py} (86%) rename cli/samples/restler/self-contained/swagger-petstore3/{restler.compile.json => compile.json} (87%) rename cli/samples/restler/self-contained/swagger-petstore3/{restler.zap.fuzz.json => fuzz.json} (54%) create mode 100644 cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json rename cli/samples/restler/self-contained/swagger-petstore3/{restler.test.json => test.json} (69%) diff --git a/Scripts/Tests/bvt-petstore.py b/Scripts/Tests/bvt-petstore3.py similarity index 86% rename from Scripts/Tests/bvt-petstore.py rename to Scripts/Tests/bvt-petstore3.py index f24a7a4b..b5e94fd0 100644 --- a/Scripts/Tests/bvt-petstore.py +++ b/Scripts/Tests/bvt-petstore3.py @@ -33,30 +33,30 @@ def webhook_triggers_results(job_id, test_url): def time_span(t_start, t_end): return time.strftime("%H:%M:%S", time.gmtime(t_end - t_start)) -def bvt(cli, definitions): +def bvt(cli, definitions, subs): print('Getting available wehook events') webhook_events = cli.list_available_webhooks_events() try: test_url = webhooks_test_url(definitions.subscription, definitions.resource_group, definitions.test_infra) for event in webhook_events: print(f'Setting webhook for {event}') - compile_webhook = cli.set_webhooks_subscription('petstore-compile', event, test_url) - fuzz_webhook = cli.set_webhooks_subscription('petstore-fuzz', event, test_url) + compile_webhook = cli.set_webhooks_subscription('petstore3-compile', event, test_url) + fuzz_webhook = cli.set_webhooks_subscription('petstore3-fuzz', event, test_url) - added_compile = cli.list_webhooks('petstore-compile', event) + added_compile = cli.list_webhooks('petstore3-compile', event) if len(added_compile) == 0: - raise Exception('Expected petstore-compile webhooks not to be empty after creation') + raise Exception('Expected petstore3-compile webhooks not to be empty after creation') - added_fuzz = cli.list_webhooks('petstore-fuzz', event) + added_fuzz = cli.list_webhooks('petstore3-fuzz', event) if len(added_fuzz) == 0: - raise Exception('Expected petstore-fuzz webhooks not to be empty after creation') + raise Exception('Expected petstore3-fuzz webhooks not to be empty after creation') t_pre_compile = time.time() print('Compile') - compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.compile.json')) + compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'compile.json')) - compile_config = raft.RaftJobConfig(file_path=compile_config_path) + compile_config = raft.RaftJobConfig(file_path=compile_config_path, substitutions=subs) compile_job = cli.new_job(compile_config) cli.poll(compile_job['jobId'], 10) @@ -81,11 +81,9 @@ def bvt(cli, definitions): f' {after_compile_pre_fuzz}') print('Fuzz') - fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.fuzz.json')) - subs = {} + fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'fuzz.json')) subs['{compile.jobId}'] = compile_job['jobId'] fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs) - fuzz_config.config['duration'] = '00:20:00' fuzz_job = cli.new_job(fuzz_config) cli.poll(fuzz_job['jobId'], 10) @@ -104,8 +102,8 @@ def bvt(cli, definitions): raise Exception('Expected job to be in completed state when retrieved job list.' f'{after_fuzz}') - if m != 2: - raise Exception('Expected 2 after compile job step' + if m != 3: + raise Exception('Expected 3 after compile job step' f' for job {fuzz_job["jobId"]}' f' got {m}' f' {after_fuzz}') @@ -136,7 +134,7 @@ def bvt(cli, definitions): print('Validating that bugs posted events matches total bugs found in job status') total_bugs_found = 0 for r in job_status_events: - if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId']: + if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId'] and r['Data']['Tool'] == 'RESTler': total_bugs_found += r['Data']['Metrics']['TotalBugBucketsCount'] print(f'Total bugs found: {total_bugs_found}') @@ -161,12 +159,11 @@ def bvt(cli, definitions): if len(deleted_fuzz) > 0: raise Exception('Expected petstore-fuzz webhooks to be empty after deletion, instead got %s', deleted_compile) - - if __name__ == "__main__": formatter = argparse.ArgumentDefaultsHelpFormatter parser = argparse.ArgumentParser(description='bvt', formatter_class=formatter) raft.add_defaults_and_secret_args(parser) + parser.add_argument('--build', required=True) args = parser.parse_args() if args.defaults_context_json: @@ -179,4 +176,10 @@ def bvt(cli, definitions): definitions = RaftDefinitions(defaults) defaults['secret'] = args.secret cli = RaftCLI(defaults) - bvt(cli, definitions) \ No newline at end of file + subs = { + "{build-url}" : os.environ.get('SYSTEM_COLLECTIONURI'), + "{build-id}" : os.environ.get('BUILD_BUILDID'), + "{ci-run}" : args.build.replace('.', '-') + } + print(f"SUBS: {subs}") + bvt(cli, definitions, subs) \ No newline at end of file diff --git a/ado/stages/test/steps/bvt.yml b/ado/stages/test/steps/bvt.yml index b51c15f2..7e962d9b 100644 --- a/ado/stages/test/steps/bvt.yml +++ b/ado/stages/test/steps/bvt.yml @@ -13,4 +13,4 @@ steps: azureSubscription: $(raft-subscription) scriptType: 'pscore' scriptLocation: 'inlineScript' - inlineScript: "python Scripts/Tests/bvt.py --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret) --host $(bvt-host)" + inlineScript: "python Scripts/Tests/bvt-petstore3.py --build $(Build.BuildNumber) --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret)" diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json index a2877fc4..da08370b 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json @@ -3,10 +3,7 @@ "URL" : "http://localhost:8080/api/swagger.json" }, "webhook": { - "name": "petstore-compile", - "metadata": { - "action" : "compile" - } + "name": "petstore-compile" }, "testTargets" : { "targets" : [ diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json index d1637606..a9dd63c1 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json @@ -1,12 +1,7 @@ { "host": "localhost", "webhook": { - "name": "petstore-fuzz", - "metadata": { - "app" : "petstore", - "swagger_version" : "v2", - "action" : "fuzz" - } + "name": "petstore-fuzz" }, "readonlyFileShareMounts": [ diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json index ec84ef8a..88f127f1 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json @@ -2,12 +2,7 @@ "host": "localhost", "webhook": { - "name": "petstore", - "metadata": { - "app" : "petstore", - "swagger_version" : "v2", - "action" : "test" - } + "name": "petstore-test" }, "readonlyFileShareMounts": [ { diff --git a/cli/samples/restler/self-contained/swagger-petstore/run.py b/cli/samples/restler/self-contained/swagger-petstore/run.py index cc378e62..f0f13fe5 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/run.py +++ b/cli/samples/restler/self-contained/swagger-petstore/run.py @@ -14,8 +14,6 @@ def run(compile, test, fuzz, replay): cli = RaftCLI() # Create compilation job configuration compile_job_config = RaftJobConfig(file_path=compile) - # add webhook metadata that will be included in every triggered webhook by Compile job - compile_job_config.add_metadata({"branch":"wizbangFeature"}) print('Compile') # submit a new job with the Compile config and get new job ID compile_job = cli.new_job(compile_job_config) @@ -37,8 +35,6 @@ def run(compile, test, fuzz, replay): # create a new job config with Fuzz configuration JSON fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs) print('Fuzz') - # add webhook metadata that will included in every triggered webhook by Fuzz job - fuzz_job_config.add_metadata({"branch":"wizbangFeature"}) # create new fuzz job configuration fuzz_job = cli.new_job(fuzz_job_config) diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore3/compile.json similarity index 87% rename from cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json rename to cli/samples/restler/self-contained/swagger-petstore3/compile.json index 1eb801d7..71ec7692 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/compile.json @@ -1,18 +1,11 @@ -{ +{ + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-compile-", "swaggerLocation": { "URL" : "http://localhost:8082/api/v3/openapi.json" }, - "resources": { - "Cores": 4, - "MemoryGBs": 8 - }, - "testTargets" : { - "resources" : { - "Cores" : 2, - "MemoryGBs" : 4 - }, "targets" : [ { "Container" : "swaggerapi/petstore3", diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json similarity index 54% rename from cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json rename to cli/samples/restler/self-contained/swagger-petstore3/fuzz.json index 4e1a2ee5..4c6678b5 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json @@ -1,4 +1,6 @@ { + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-fuzz-lean-", "host": "localhost", "resources": { @@ -6,10 +8,17 @@ "MemoryGBs": 8 }, + "webhook":{ + "name" : "petstore3-fuzz", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + "readonlyFileShareMounts": [ { - "FileShareName": "{compile.jobId}", - "MountPath": "/job-compile" + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" } ], @@ -26,21 +35,16 @@ "Run" : { "Command" : "/bin/sh", "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] - }, - "Shell" : "/bin/sh", - "OutputFolder" : "restler_petstore3" + } }, { "Container" : "swaggerapi/petstore3", - "Port" : 8080, - "ExpectedDurationUntilReady" : "00:00:30", - "PostRun" : { - "Command" : "/bin/sh", - "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], - "ExpectedRunDuration" : "00:00:10" - }, - "Shell" : "/bin/sh", - "OutputFolder" : "zap_petstore3" + "Port" : 8082, + "ExpectedDurationUntilReady" : "00:02:00", + "Run" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8082 /swagger-petstore/server.war"] + } } ] }, @@ -48,7 +52,7 @@ { "toolName": "RESTler", "outputFolder": "fuzz", - "duration": "00:30:00", + "duration": "00:10:00", "toolConfiguration": { "task": "Fuzz", "runConfiguration": { @@ -56,16 +60,16 @@ "Port" : 8081 }, "useSsl" : false, - "inputFolderPath": "/job-compile/compile" + "inputFolderPath": "/ci-run/{compile.jobId}/compile" } } - }, + }, { - "toolName" : "ZAP", - "outputFolder" : "zap", + "toolName": "ZAP", + "outputFolder": "zap", "swaggerLocation": { - "URL" : "http://localhost:8080/api/v3/openapi.json" + "URL" : "http://localhost:8082/api/v3/openapi.json" } } ] -} \ No newline at end of file +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/run.py b/cli/samples/restler/self-contained/swagger-petstore3/run.py index 5d46e0e7..13af9b15 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/run.py +++ b/cli/samples/restler/self-contained/swagger-petstore3/run.py @@ -4,47 +4,60 @@ import pathlib import sys import os +import json cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..')) -from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError +sys.path.append(os.path.join(cur_dir, '..')) +from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError, RaftDefinitions -def run(compile, test, fuzz): - # instantiate RAFT CLI - cli = RaftCLI() +def run(cli, config, subs): # Create compilation job configuration - compile_job_config = RaftJobConfig(file_path=compile) - print('Compile') + job_config = RaftJobConfig(file_path=config, substitutions=subs) + print(f'Running {config}') # submit a new job with the Compile config and get new job ID - compile_job = cli.new_job(compile_job_config) + job = cli.new_job(job_config) # wait for a job with ID from compile_job to finish the run - cli.poll(compile_job['jobId']) - - # use compile job as input for fuzz job - subs = {} - subs['{compile.jobId}'] = compile_job['jobId'] - - test_job_config = RaftJobConfig(file_path=test, substitutions=subs) - print('Test') - # create new fuzz job configuration - test_job = cli.new_job(test_job_config) - # wait for job ID from fuzz_job to finish the run - cli.poll(test_job['jobId']) - - # create a new job config with Fuzz configuration JSON - fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs) - print('Fuzz') - # create new fuzz job configuration - fuzz_job = cli.new_job(fuzz_job_config) - - # wait for job ID from fuzz_job to finish the run - cli.poll(fuzz_job['jobId']) + cli.poll(job['jobId']) + return job['jobId'] if __name__ == "__main__": try: - run(os.path.join(cur_dir, "restler.compile.json"), - os.path.join(cur_dir, "restler.test.json"), - os.path.join(cur_dir, "restler.zap.fuzz.json")) + defaults = None + + if sys.argv[1] == '--build': + build_id = sys.argv[2].replace(".", "-") + print(f"BUILD ID : {build_id}") + + with open(os.path.join(cur_dir, '..', 'defaults.json'), 'r') as defaults_json: + defaults = json.load(defaults_json) + if sys.argv[3] == '--secret': + defaults['secret'] = sys.argv[4] + + # instantiate RAFT CLI + cli = RaftCLI(defaults) + defs = RaftDefinitions(defaults) + + compile_job_id = None + subs = { + "{ci-run}" : f"{build_id}", + "{build-url}" : os.environ['SYSTEM_COLLECTIONURI'], + "{build-id}" : os.environ['BUILD_BUILDID'], + "{raft-subscription}": defs.subscription, + "{raft-resource-group}" : defs.resource_group, + "{raft-storage-account}" : defs.storage_account + } + for arg in sys.argv[1:]: + if arg == 'compile': + compile_job_id = run(cli, os.path.join(cur_dir, 'compile.json'), subs) + subs['{compile.jobId}'] = compile_job_id + + if arg == 'test': + run(cli, os.path.join(cur_dir, "test.json"), subs), + + if arg == 'test-fuzz-lean': + run(cli, os.path.join(cur_dir, "test-fuzz-lean.json"), subs), + except RaftJobError as ex: - print(f'ERROR: {ex.message}') \ No newline at end of file + print(f'ERROR: {ex.message}') + sys.exit(1) \ No newline at end of file diff --git a/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json b/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json new file mode 100644 index 00000000..80a59def --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json @@ -0,0 +1,59 @@ +{ + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-fuzz-lean-", + "host": "localhost", + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + }, + + "webhook":{ + "name" : "petstore3-test-fuzz-lean", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + + "readonlyFileShareMounts": [ + { + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" + } + ], + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Port" : 8081, + "ExpectedDurationUntilReady" : "00:02:00", + "Run" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] + } + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "test-fuzz-lean", + "duration": "01:00:00", + "toolConfiguration": { + "task": "TestFuzzLean", + "runConfiguration": { + "targetEndpointConfiguration" : { + "Port" : 8081 + }, + "useSsl" : false, + "inputFolderPath": "/ci-run/{compile.jobId}/compile" + } + } + } + ] +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore3/test.json similarity index 69% rename from cli/samples/restler/self-contained/swagger-petstore3/restler.test.json rename to cli/samples/restler/self-contained/swagger-petstore3/test.json index f1e1ff6b..ddda2f2a 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/restler.test.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/test.json @@ -1,4 +1,6 @@ { + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-", "host": "localhost", "resources": { @@ -6,10 +8,17 @@ "MemoryGBs": 8 }, + "webhook":{ + "name" : "petstore3-test", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + "readonlyFileShareMounts": [ { - "FileShareName": "{compile.jobId}", - "MountPath": "/job-compile" + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" } ], @@ -29,7 +38,7 @@ "ExpectedRunDuration" : "00:00:10" }, "Shell" : "/bin/sh", - "OutputFolder" : "zap_petstore3" + "OutputFolder" : "petstore3" } ] }, @@ -44,7 +53,7 @@ "Port" : 8080 }, "useSsl" : false, - "inputFolderPath": "/job-compile/compile" + "inputFolderPath": "/ci-run/{compile.jobId}/compile" } } } diff --git a/src/Agent/RESTlerAgent/RESTlerDriver.fs b/src/Agent/RESTlerAgent/RESTlerDriver.fs index ea1a291e..1f3b0d5b 100644 --- a/src/Agent/RESTlerAgent/RESTlerDriver.fs +++ b/src/Agent/RESTlerAgent/RESTlerDriver.fs @@ -151,15 +151,20 @@ module private RESTlerInternal = if Seq.isEmpty experiments then None else - let startedExperiments = - experiments - |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime) - |> Seq.sortBy ( fun e -> e.CreationTimeUtc ) - - if (Seq.length startedExperiments > 1) then - printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." - (Seq.length startedExperiments) startedExperiments runStartTime - startedExperiments |> Seq.tryHead + try + let startedExperiments = + experiments + |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime) + |> Seq.sortBy ( fun e -> e.CreationTimeUtc ) + + if (Seq.length startedExperiments > 1) then + printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." + (Seq.length startedExperiments) startedExperiments runStartTime + startedExperiments |> Seq.tryHead + with + | :? System.IO.IOException as ioex -> + printfn "Getting experiment folder interrupted due to : %s" ioex.Message + None else None diff --git a/src/Contracts/Job.fs b/src/Contracts/Job.fs index 9ef3d736..bb2b3c39 100644 --- a/src/Contracts/Job.fs +++ b/src/Contracts/Job.fs @@ -87,7 +87,7 @@ type FileShareMount = type Webhook = { Name : string - Metadata : Map + Metadata : Map option } diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index e08d6b86..c284facf 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -1316,10 +1316,13 @@ module ContainerInstances = let webhookDefinition = Microsoft.FSharpLu.Json.Compact.deserialize(jobEntity.Webhook) match webhookDefinition with | Some webhook -> - if webhook.Metadata.IsEmpty then + match webhook.Metadata with + | None -> + return None + | Some m when m.IsEmpty -> return None - else - return Some webhook.Metadata + | Some m -> + return Some m | None -> return None else From 097f494835b4a2a6b228f436d21968f88c83eb4e Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 8 Dec 2020 08:38:59 -0800 Subject: [PATCH 14/18] add results when firing webhooks (#79) Co-authored-by: stas --- .../ApiService/Controllers/Webhook.fs | 1 + src/Agent/RESTlerAgent/AgentMain.fs | 1 + src/Contracts/JobEvents.fs | 1 + .../OrchestratorLogic/Orchestrator.fs | 20 +++++++++++++++++-- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/APIService/ApiService/Controllers/Webhook.fs b/src/APIService/ApiService/Controllers/Webhook.fs index dc25ff2e..a9d485c7 100644 --- a/src/APIService/ApiService/Controllers/Webhook.fs +++ b/src/APIService/ApiService/Controllers/Webhook.fs @@ -400,6 +400,7 @@ type webhooksController(configuration : IConfiguration, telemetryClient : Teleme JobId = jobId AgentName = "1" Metadata = None + ResultsUrl = Some "https://azure-storage/results" BugDetails = Some( Map.empty diff --git a/src/Agent/RESTlerAgent/AgentMain.fs b/src/Agent/RESTlerAgent/AgentMain.fs index ebf89653..2fc9ebfc 100644 --- a/src/Agent/RESTlerAgent/AgentMain.fs +++ b/src/Agent/RESTlerAgent/AgentMain.fs @@ -587,6 +587,7 @@ let main argv = AgentName = agentName Metadata = None BugDetails = Some bugDetails + ResultsUrl = None } : Raft.JobEvents.BugFound) } diff --git a/src/Contracts/JobEvents.fs b/src/Contracts/JobEvents.fs index 08f5c29d..7e9a9cf4 100644 --- a/src/Contracts/JobEvents.fs +++ b/src/Contracts/JobEvents.fs @@ -85,6 +85,7 @@ type BugFound = AgentName : string Metadata : Map option BugDetails : Map option + ResultsUrl : string option } static member EventType = Events.JobEventTypes.BugFound.ToString() \ No newline at end of file diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index c284facf..cb7a2659 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -1330,21 +1330,37 @@ module ContainerInstances = return None } + let getResultsUrl jobId = + async { + match! JobStatus.getRow agentConfig (jobId, jobId) with + | Result.Error() -> return None + | Result.Ok(r) -> + match r with + | None -> return None + | Some row -> return (if String.IsNullOrWhiteSpace row.ResultsUrl then None else Some row.ResultsUrl) + } + let eventType = RaftEvent.getEventType message if eventType = JobStatus.EventType then let jobStatus : RaftEvent.RaftJobEvent = RaftEvent.deserializeEvent message let! metadata = getMetadata jobStatus.Message.JobId + let! resultsUrl = getResultsUrl jobStatus.Message.JobId let updatedJobStatus = { jobStatus with Message = { jobStatus.Message with - Metadata = metadata } + Metadata = metadata + ResultsUrl = resultsUrl + } } do! processMessage updatedJobStatus.Message.JobId updatedJobStatus else if eventType = BugFound.EventType then let bugFound : RaftEvent.RaftJobEvent = RaftEvent.deserializeEvent message let! metadata = getMetadata bugFound.Message.JobId + let! resultsUrl = getResultsUrl bugFound.Message.JobId let updatedBugFound = { bugFound with Message = { bugFound.Message with - Metadata = metadata } + Metadata = metadata + ResultsUrl = resultsUrl + } } do! processMessage updatedBugFound.Message.JobId updatedBugFound From b8b48437c6599580a97af5ef8daa765e439388c9 Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 8 Dec 2020 11:44:15 -0800 Subject: [PATCH 15/18] v1.3.0 (#80) Co-authored-by: stas --- ado/variables/version-variables.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ado/variables/version-variables.yml b/ado/variables/version-variables.yml index 088b4609..4e8fdc8f 100644 --- a/ado/variables/version-variables.yml +++ b/ado/variables/version-variables.yml @@ -2,11 +2,11 @@ variables: - name: version.major value: 1 - name: version.minor - value: 2 + value: 3 - name: version.revision value: 0 - name: imageTag - value: 'v1.2.0' + value: 'v1.3.0' - name: imageTagLatest value: 'v1.latest' - name: devRevision From 613e5ecce95b15e7b6eabaf9643ee5f295e1a906 Mon Sep 17 00:00:00 2001 From: Stas Date: Mon, 30 Nov 2020 11:13:54 -0800 Subject: [PATCH 16/18] Support for listening on multiple ports in test targets (#58) Co-authored-by: stas --- Scripts/CustomTools/tools/RESTler/config.json | 1 + .../swagger-petstore/restler.compile.json | 2 +- .../swagger-petstore/restler.fuzz.json | 2 +- .../restler.replay-all-fuzz-bugs.json | 2 +- .../swagger-petstore/restler.test.json | 2 +- .../swagger-petstore3/compile.json | 2 +- .../swagger-petstore3/fuzz.json | 4 ++-- .../swagger-petstore3/test.json | 2 +- src/APIService/ApiService/Controllers/Job.fs | 1 - src/APIService/ApiService/DTOs.fs | 2 +- src/Contracts/Job.fs | 11 ++++++++- .../OrchestratorLogic/Orchestrator.fs | 24 +++++-------------- 12 files changed, 26 insertions(+), 29 deletions(-) diff --git a/Scripts/CustomTools/tools/RESTler/config.json b/Scripts/CustomTools/tools/RESTler/config.json index 45e9026b..82312ca0 100644 --- a/Scripts/CustomTools/tools/RESTler/config.json +++ b/Scripts/CustomTools/tools/RESTler/config.json @@ -10,6 +10,7 @@ "command" : "/bin/sh", "arguments" : ["-c", "echo DebugMode; while true; do sleep 100000; done;"] }, + "shell" : "/bin/sh", "environmentVariables" : { "RESTLER_TELEMETRY_OPTOUT" : "0" } diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json index da08370b..321d794b 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json @@ -9,7 +9,7 @@ "targets" : [ { "Container" : "swaggerapi/petstore", - "Port" : 8080, + "Ports" : [8080], "ExpectedDurationUntilReady" : "00:02:00", "Shell" : "/bin/sh", "OutputFolder" : "petstore", diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json index a9dd63c1..6f09f566 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json @@ -15,7 +15,7 @@ "targets" : [ { "Container" : "swaggerapi/petstore", - "Port" : 8080, + "Ports" : [8080], "ExpectedDurationUntilReady" : "00:02:00", "Shell" : "/bin/sh", "OutputFolder" : "petstore", diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json b/cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json index 05c466eb..3c6ba5f3 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json @@ -9,7 +9,7 @@ "targets" : [ { "Container" : "swaggerapi/petstore", - "Port" : 8080, + "Ports" : [8080], "ExpectedDurationUntilReady" : "00:02:00", "Shell" : "/bin/sh", "OutputFolder" : "petstore", diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json index 88f127f1..4c17b21c 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json @@ -15,7 +15,7 @@ "targets" : [ { "Container" : "swaggerapi/petstore", - "Port" : 8080, + "Ports" : [8080], "ExpectedDurationUntilReady" : "00:00:30", "Shell" : "/bin/sh", "OutputFolder" : "petstore", diff --git a/cli/samples/restler/self-contained/swagger-petstore3/compile.json b/cli/samples/restler/self-contained/swagger-petstore3/compile.json index 71ec7692..8c5b5c69 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/compile.json @@ -9,7 +9,7 @@ "targets" : [ { "Container" : "swaggerapi/petstore3", - "Port" : 8082, + "Ports" : [8082], "ExpectedDurationUntilReady" : "00:00:30", "Run" : { "Command" : "java", diff --git a/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json index 4c6678b5..39530a50 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json @@ -30,7 +30,7 @@ "targets" : [ { "Container" : "swaggerapi/petstore3", - "Port" : 8081, + "Ports" : [8081], "ExpectedDurationUntilReady" : "00:02:00", "Run" : { "Command" : "/bin/sh", @@ -39,7 +39,7 @@ }, { "Container" : "swaggerapi/petstore3", - "Port" : 8082, + "Ports" : [8082], "ExpectedDurationUntilReady" : "00:02:00", "Run" : { "Command" : "/bin/sh", diff --git a/cli/samples/restler/self-contained/swagger-petstore3/test.json b/cli/samples/restler/self-contained/swagger-petstore3/test.json index ddda2f2a..447dc4b5 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/test.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/test.json @@ -30,7 +30,7 @@ "targets" : [ { "Container" : "swaggerapi/petstore3", - "Port" : 8080, + "Ports" : [8080], "ExpectedDurationUntilReady" : "00:00:30", "PostRun" : { "Command" : "/bin/sh", diff --git a/src/APIService/ApiService/Controllers/Job.fs b/src/APIService/ApiService/Controllers/Job.fs index a1414ddd..c414f401 100644 --- a/src/APIService/ApiService/Controllers/Job.fs +++ b/src/APIService/ApiService/Controllers/Job.fs @@ -214,7 +214,6 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger Array.iter(fun auth -> let enabled = diff --git a/src/APIService/ApiService/DTOs.fs b/src/APIService/ApiService/DTOs.fs index 2877f18b..352904c5 100644 --- a/src/APIService/ApiService/DTOs.fs +++ b/src/APIService/ApiService/DTOs.fs @@ -148,7 +148,7 @@ module DTOs = { [] Container : string - Port : int + Ports : int array ExpectedDurationUntilReady: TimeSpan IsIdling : Nullable diff --git a/src/Contracts/Job.fs b/src/Contracts/Job.fs index bb2b3c39..67c5b7d9 100644 --- a/src/Contracts/Job.fs +++ b/src/Contracts/Job.fs @@ -16,10 +16,19 @@ type SwaggerLocation = | FilePath of string +type GpuConfig = + { + Sku : string //available SKUs v100, k80, P100 + Cores : int + } + + type Resources = { Cores : int MemoryGBs : int + + GPU : GpuConfig option } type Command = @@ -34,7 +43,7 @@ type TestTargetDefinition = { Container : string ExpectedDurationUntilReady: System.TimeSpan - Port : int + Ports : int array option IsIdling : bool option Run : Command option diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index cb7a2659..5c1d162b 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -161,7 +161,7 @@ module ContainerInstances = { Tool: string Container: string - Port : int option + Ports : int array option IsIdling : bool @@ -182,18 +182,6 @@ module ContainerInstances = ToolConfiguration: ToolConfiguration } - type GpuConfig = - { - Sku : string //available SKUs v100, k80, P100 - Cores : int - } - type ContainerConfig = - { - CPUs : int option - MemorySizeInGB : float option - GPUs : GpuConfig option - } - type ToolConfig = { Container : string @@ -305,7 +293,7 @@ module ContainerInstances = return runDirectory, { Tool = task.ToolName - Port = None + Ports = None Container = container Secrets = task.KeyVaultSecrets @@ -466,11 +454,11 @@ module ContainerInstances = b.WithImage(toolContainerRun.ToolConfiguration.Container) let b2 = - match toolContainerRun.ToolConfiguration.Port with + match toolContainerRun.ToolConfiguration.Ports with | None -> b1.WithoutPorts().WithCpuCoreCount(cpu) | Some p -> - (b1.WithInternalTcpPort p).WithCpuCoreCount(cpu) + (b1.WithInternalTcpPorts p).WithCpuCoreCount(cpu) b2 .WithMemorySizeInGB(ram) .WithVolumeMountSetting(workVolume, workDirectory) @@ -814,7 +802,7 @@ module ContainerInstances = t.Targets |> Array.mapi (fun i target -> { - ContainerName = sprintf "%s-%d" TestTarget target.Port + ContainerName = sprintf "%s-%d" TestTarget i RunDirectory = None WorkDirectory = match target.OutputFolder with @@ -824,7 +812,7 @@ module ContainerInstances = ToolConfiguration = { Tool = target.Container Container = target.Container - Port = Some target.Port + Ports = target.Ports IsIdling = match target.IsIdling with None -> false | Some v -> v From bf0949da0d87960adde3fc671c7a2f05234d967f Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 1 Dec 2020 16:32:47 -0800 Subject: [PATCH 17/18] Support for a list of swagger specifications (#62) Co-authored-by: stas --- cli/raft-tools/tools/ZAP/config.json | 2 +- cli/raft-tools/tools/ZAP/requirements.txt | 2 +- cli/raft-tools/tools/ZAP/run.py | 21 +++++- cli/raft-tools/tools/ZAP/scan.py | 69 +++++++++--------- .../compile.json | 8 +-- .../compile.yaml | 8 +-- .../fuzz.json | 8 +-- .../sample.restler.compile.json | 4 +- .../sample.restler.compile.json | 4 +- .../raft.restler.compile.json | 6 +- .../raft.restler.compile.yaml | 4 +- .../raft.restler.compile.json | 4 +- .../raft.restler.compile.json | 6 +- .../swagger-petstore/restler.compile.json | 4 +- .../swagger-petstore3/compile.json | 6 +- .../swagger-petstore3/fuzz.json | 14 +++- .../swagger-petstore3/restler.compile.json | 44 ++++++++++++ .../swagger-petstore3/restler.test.json | 52 ++++++++++++++ .../swagger-petstore3/restler.zap.fuzz.json | 71 +++++++++++++++++++ .../swagger-petstore3/test-fuzz-lean.json | 59 --------------- .../zap/no-authentication/sample.zap.json | 4 +- .../raft.zap.json | 4 +- .../zap/running-against-raft/raft.zap.json | 4 +- cli/samples/zap/swagger-petstore3/zap.json | 7 +- docs/how-to-submit-a-job.md | 10 +-- docs/sdk/swagger.md | 8 +-- .../APIServiceTests/test-restler-compile.json | 4 +- .../APIServiceTests/test-restler-fuzz.json | 4 +- src/APIService/ApiService/Controllers/Job.fs | 13 ++-- src/APIService/ApiService/DTOs.fs | 8 +-- src/Agent/RESTlerAgent/AgentMain.fs | 47 +++++++----- src/Contracts/Job.fs | 4 +- 32 files changed, 336 insertions(+), 177 deletions(-) create mode 100644 cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json create mode 100644 cli/samples/restler/self-contained/swagger-petstore3/restler.test.json create mode 100644 cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json delete mode 100644 cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json diff --git a/cli/raft-tools/tools/ZAP/config.json b/cli/raft-tools/tools/ZAP/config.json index a460bcbb..687fd658 100644 --- a/cli/raft-tools/tools/ZAP/config.json +++ b/cli/raft-tools/tools/ZAP/config.json @@ -3,7 +3,7 @@ "run" : { "command" : "bash", "arguments" : ["-c", - "touch /.dockerenv; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; sleep $RAFT_STARTUP_DELAY; python3 run.py" ] + "sleep $RAFT_STARTUP_DELAY; cd $RAFT_TOOL_RUN_DIRECTORY; ln -s $RAFT_WORK_DIRECTORY /zap/wrk; python3 run.py install; python3 run.py" ] }, "idle" : { "command" : "bash", diff --git a/cli/raft-tools/tools/ZAP/requirements.txt b/cli/raft-tools/tools/ZAP/requirements.txt index 23509fab..cd6ce089 100644 --- a/cli/raft-tools/tools/ZAP/requirements.txt +++ b/cli/raft-tools/tools/ZAP/requirements.txt @@ -1,2 +1,2 @@ applicationinsights~=0.11.9 -azure-servicebus~=0.50.3 \ No newline at end of file +azure-servicebus~=7.0.0 \ No newline at end of file diff --git a/cli/raft-tools/tools/ZAP/run.py b/cli/raft-tools/tools/ZAP/run.py index 8e2b6bd0..0e878ac4 100644 --- a/cli/raft-tools/tools/ZAP/run.py +++ b/cli/raft-tools/tools/ZAP/run.py @@ -38,11 +38,28 @@ def auth_token(init): print(f'Unhandled authentication configuration {auth_config}') return None +def get_swagger(target): + if target and target.get("url"): + return target["url"] + elif target.get("filePath"): + return target["filePath"] + if __name__ == "__main__": if len(sys.argv) == 2 and sys.argv[1] == "install": subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", os.path.join(run_directory, "requirements.txt")]) auth_token(True) else: token = auth_token(False) - import scan - scan.run(token) \ No newline at end of file + + work_directory = os.environ['RAFT_WORK_DIRECTORY'] + with open(os.path.join(work_directory, 'task-config.json'), 'r') as task_config: + config = json.load(task_config) + i = 0 + n_targets = len(config.get("swaggerLocations")) + for t in config.get("swaggerLocations"): + print(f'Starting zap for target {t}') + if token: + subprocess.check_call([sys.executable, "scan.py", f"{i}", f"{n_targets}", get_swagger(t), token]) + else: + subprocess.check_call([sys.executable, "scan.py", f"{i}", f"{n_targets}", get_swagger(t)]) + i = i + 1 diff --git a/cli/raft-tools/tools/ZAP/scan.py b/cli/raft-tools/tools/ZAP/scan.py index 642d340e..ecdf7ca9 100644 --- a/cli/raft-tools/tools/ZAP/scan.py +++ b/cli/raft-tools/tools/ZAP/scan.py @@ -6,9 +6,10 @@ import io import logging from logging import StreamHandler +import importlib from applicationinsights import TelemetryClient -from azure.servicebus import TopicClient, Message +from azure.servicebus import ServiceBusClient, ServiceBusMessage from contextlib import redirect_stdout class RaftUtils(): @@ -18,8 +19,10 @@ def __init__(self): self.config = json.load(task_config) connection_str = os.environ['RAFT_SB_OUT_SAS'] - self.topic_client = TopicClient.from_connection_string(connection_str) + self.sb_client = ServiceBusClient.from_connection_string(connection_str) + self.topic_client = self.sb_client.get_topic_sender(self.sb_client._entity_name) + self.telemetry_client = TelemetryClient(instrumentation_key=os.environ['RAFT_APP_INSIGHTS_KEY']) self.job_id = os.environ['RAFT_JOB_ID'] @@ -43,8 +46,8 @@ def report_status(self, state, details): 'state' : state } } - msg = Message(str.encode(json.dumps(m))) - self.topic_client.send(msg) + msg = ServiceBusMessage(str.encode(json.dumps(m))) + self.topic_client.send_messages([msg]) def report_status_created(self, details=None): self.report_status('Created', details) @@ -67,24 +70,16 @@ def log_exception(self): def flush(self): self.telemetry_client.flush() - def get_swagger_target(self): - swagger = self.config.get("swaggerLocation") - if swagger and swagger.get("url"): - return swagger["url"] - elif swagger.get("filePath"): - return swagger["filePath"] - - zap_dir = '/zap' sys.path.append(zap_dir) -zap = __import__("zap-api-scan") utils = RaftUtils() class StatusReporter(StreamHandler): - def __init__(self): + def __init__(self, details): StreamHandler.__init__(self) self.last_txt = None + self.details = details def emit(self, record): txt = self.format(record) @@ -93,10 +88,12 @@ def emit(self, record): progress='Active Scan progress %:' i = txt.find(progress) if i != -1: - utils.report_status_running({"Scan progress" : txt[i :]}) + self.details["Scan progress"] = txt[i :] + utils.report_status_running(self.details) + +zap = __import__("zap-api-scan") -def run_zap(token): - target = utils.get_swagger_target() +def run_zap(target_index, targets_total, target, token): if token: utils.log_trace('Authentication token is set') auth = ('-config replacer.full_list(0).description=auth1' @@ -120,29 +117,33 @@ def run_zap(token): try: utils.log_trace("Starting ZAP") - utils.report_status_running() - - status_reporter = StatusReporter() + details = {"targetIndex": target_index, "numberOfTargets" : targets_total, "target": target} + utils.report_status_running(details) + status_reporter = StatusReporter(details) logger = logging.getLogger() logger.addHandler(status_reporter) - zap.main(['-t', target, '-f', 'openapi', '-J', 'report.json', '-r', 'report.html', '-w', 'report.md', '-x', 'report.xml', '-d'] + zap_auth_config) + zap.main(['-t', target, '-f', 'openapi', '-J', f'{target_index}-report.json', '-r', f'{target_index}-report.html', '-w', f'{target_index}-report.md', '-x', f'{target_index}-report.xml', '-d'] + zap_auth_config) + details["Scan progress"] = "Active scan progress %: 100" + utils.report_status_running(details) except SystemExit as e: r = e.code utils.log_trace(f"ZAP exited with exit code: {r}") - shutil.copy('/zap/zap.out', '/zap/wrk/zap.out') + shutil.copy('/zap/zap.out', f'/zap/wrk/{target_index}-zap.out') - utils.report_status_completed() if r <= 2: - return 0 - else: - return r - -def run(token): + r = 0 + + if target_index + 1 == targets_total: + utils.report_status_completed() + utils.sb_client.close() + return r + +def run(target_index, targets_total, target, token): try: utils.report_status_created() - return run_zap(token) + return run_zap(target_index, targets_total, target, token) except Exception as ex: utils.log_exception() utils.report_status_error({"Error" : f"{ex}"}) @@ -152,8 +153,12 @@ def run(token): if __name__ == "__main__": - if len(sys.argv) == 2: - run(sys.argv[1]) + target_index = int(sys.argv[1]) + targets_total = int(sys.argv[2]) + target = sys.argv[3] + + if len(sys.argv) == 5: + run(target_index, targets_total, target, sys.argv[4]) else: - run(None) + run(target_index, targets_total, target, None) diff --git a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.json b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.json index cca1b55d..ea916895 100644 --- a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.json +++ b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "tasks": [ { "toolName": "RESTler", @@ -20,9 +20,9 @@ "toolConfiguration": { "task": "Compile" }, - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{sample.host}/swagger/v1/swagger.json" - } + }] } ] } diff --git a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.yaml b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.yaml index 2aba031e..f1c29c79 100644 --- a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.yaml +++ b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.yaml @@ -1,5 +1,5 @@ -swaggerLocation: - URL: "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" +swaggerLocations: +- URL: "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" tasks: - toolName: "RESTler" @@ -11,8 +11,8 @@ tasks: - toolName: "RESTler" outputFolder: "sample-compile" - swaggerLocation: - URL: "https://{sample.host}/swagger/v1/swagger.json" + swaggerLocations: + - URL: "https://{sample.host}/swagger/v1/swagger.json" toolConfiguration: task : Compile compileConfiguration: diff --git a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/fuzz.json b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/fuzz.json index ea5ef25d..246b44be 100644 --- a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/fuzz.json +++ b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/fuzz.json @@ -32,9 +32,9 @@ "toolName": "ZAP", "keyVaultSecrets": [ "RaftServicePrincipal" ], "outputFolder": "{defaults.deploymentName}-ZAP-out", - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "authenticationMethod": { "MSAL": "RaftServicePrincipal" @@ -43,9 +43,9 @@ { "host": "{sample.host}", "toolName": "ZAP", - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{sample.host}/swagger/v1/swagger.json" - }, + }], "outputFolder": "sample-zap-out" }, { diff --git a/cli/samples/restler/no-authentication-common-file-share/sample.restler.compile.json b/cli/samples/restler/no-authentication-common-file-share/sample.restler.compile.json index 7141e83f..b365472c 100644 --- a/cli/samples/restler/no-authentication-common-file-share/sample.restler.compile.json +++ b/cli/samples/restler/no-authentication-common-file-share/sample.restler.compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{sample.host}/swagger/v1/swagger.json" - }, + }], "host": "{sample.host}", "namePrefix": "sample-compile-", "rootFileshare": "sample", diff --git a/cli/samples/restler/no-authentication/sample.restler.compile.json b/cli/samples/restler/no-authentication/sample.restler.compile.json index 640b7d51..e68f7f02 100644 --- a/cli/samples/restler/no-authentication/sample.restler.compile.json +++ b/cli/samples/restler/no-authentication/sample.restler.compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{sample.host}/swagger/v1/swagger.json" - }, + }], "namePrefix": "sample-compile-", "webhook": { "name": "sample-compile", diff --git a/cli/samples/restler/running-60-tasks/raft.restler.compile.json b/cli/samples/restler/running-60-tasks/raft.restler.compile.json index 57575e48..2fc6a6ad 100644 --- a/cli/samples/restler/running-60-tasks/raft.restler.compile.json +++ b/cli/samples/restler/running-60-tasks/raft.restler.compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{host}/swagger/v1/swagger.json" - }, + }], "resources": { "Cores": 2, "MemoryGBs": 3 @@ -19,7 +19,7 @@ "customDictionary": { "customPayload": { "duration": [ "00:10:00" ], - "swaggerLocation": [ "{\"URL\": \"https://some.service.azurewebsites.net/swagger.json\"}" ], + "swaggerLocations": [ "[{\"URL\": \"https://some.service.azurewebsites.net/swagger.json\"}]" ], "authenticationMethod": [ "{\"CommandLine\": \"abc\" }" ], "targetUrl": [ "{replace-with-valid-webhook-URL}" ], "event": [ "JobStatus" ], diff --git a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.compile.yaml b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.compile.yaml index 510408c4..7340deed 100644 --- a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.compile.yaml +++ b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.compile.yaml @@ -1,5 +1,5 @@ -swaggerLocation: - Url: https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json +swaggerLocations: +- Url: https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json rootFileShare: raft tasks: - toolName: RESTler diff --git a/cli/samples/restler/running-against-raft-common-file-share/raft.restler.compile.json b/cli/samples/restler/running-against-raft-common-file-share/raft.restler.compile.json index 1306eb80..d6403f22 100644 --- a/cli/samples/restler/running-against-raft-common-file-share/raft.restler.compile.json +++ b/cli/samples/restler/running-against-raft-common-file-share/raft.restler.compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "Url": "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "rootFileShare": "raft", "tasks": [ { diff --git a/cli/samples/restler/running-against-raft/raft.restler.compile.json b/cli/samples/restler/running-against-raft/raft.restler.compile.json index 908245f5..7afc66ed 100644 --- a/cli/samples/restler/running-against-raft/raft.restler.compile.json +++ b/cli/samples/restler/running-against-raft/raft.restler.compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "tasks": [ { "toolName": "RESTler", @@ -17,7 +17,7 @@ "memoryGBs" : ["1"], "toolName" : ["ZAP"], "duration": [ "00:10:00" ], - "swaggerLocation": [ "{\"URL\": \"https://some.service.azurewebsites.net/swagger.json\"}", "{\"FilePath\": \"/swagger-folder/swagger.json\"}" ], + "swaggerLocations": [ "[{\"URL\": \"https://some.service.azurewebsites.net/swagger.json\"}", "{\"FilePath\": \"/swagger-folder/swagger.json\"}]" ], "authenticationMethod": [ "{\"CommandLine\": \"abc\" }" ], "targetUrl": [ "{replace-with-valid-webhook-URL}" ], "event": [ "JobStatus" ], diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json index 321d794b..fffb6c8f 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL" : "http://localhost:8080/api/swagger.json" - }, + }], "webhook": { "name": "petstore-compile" }, diff --git a/cli/samples/restler/self-contained/swagger-petstore3/compile.json b/cli/samples/restler/self-contained/swagger-petstore3/compile.json index 8c5b5c69..cd149901 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/compile.json @@ -3,9 +3,13 @@ "namePrefix" : "petstore3-compile-", "swaggerLocation": { "URL" : "http://localhost:8082/api/v3/openapi.json" - }, + }], "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, "targets" : [ { "Container" : "swaggerapi/petstore3", diff --git a/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json index 39530a50..42945a81 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json @@ -36,9 +36,19 @@ "Command" : "/bin/sh", "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] } + }, + "Shell" : "/bin/sh", + "OutputFolder" : "restler_petstore3" }, { "Container" : "swaggerapi/petstore3", + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + }, + "Shell" : "/bin/sh", + "OutputFolder" : "zap_petstore3" "Ports" : [8082], "ExpectedDurationUntilReady" : "00:02:00", "Run" : { @@ -67,9 +77,9 @@ { "toolName": "ZAP", "outputFolder": "zap", - "swaggerLocation": { + "swaggerLocations": [{ "URL" : "http://localhost:8082/api/v3/openapi.json" - } + }] } ] } diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json new file mode 100644 index 00000000..efb42ad2 --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json @@ -0,0 +1,44 @@ +{ + "swaggerLocations": [{ + "URL" : "http://localhost:8082/api/v3/openapi.json" + }], + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + }, + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Ports" : [8082], + "ExpectedDurationUntilReady" : "00:00:30", + "Run" : { + "Command" : "java", + "Arguments" : ["-jar", "/swagger-petstore/jetty-runner.jar", "--log", "/var/log/yyyy_mm_dd-requests.log", "--port", "8082", "/swagger-petstore/server.war"] + }, + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + }, + "Shell" : "/bin/sh", + "OutputFolder" : "petstore3" + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "compile", + "toolConfiguration": { + "task": "Compile" + } + } + ] +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore3/restler.test.json new file mode 100644 index 00000000..b957dc3c --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/restler.test.json @@ -0,0 +1,52 @@ +{ + "host": "localhost", + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + }, + + "readonlyFileShareMounts": [ + { + "FileShareName": "{compile.jobId}", + "MountPath": "/job-compile" + } + ], + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Ports" : [8080], + "ExpectedDurationUntilReady" : "00:00:30", + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + }, + "Shell" : "/bin/sh", + "OutputFolder" : "zap_petstore3" + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "test", + "toolConfiguration": { + "task": "test", + "runConfiguration": { + "targetEndpointConfiguration" : { + "Port" : 8080 + }, + "useSsl" : false, + "inputFolderPath": "/job-compile/compile" + } + } + } + ] +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json new file mode 100644 index 00000000..e6212b01 --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json @@ -0,0 +1,71 @@ +{ + "host": "localhost", + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + }, + + "readonlyFileShareMounts": [ + { + "FileShareName": "{compile.jobId}", + "MountPath": "/job-compile" + } + ], + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Ports" : [8081], + "ExpectedDurationUntilReady" : "00:02:00", + "Run" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] + }, + "Shell" : "/bin/sh", + "OutputFolder" : "restler_petstore3" + }, + { + "Container" : "swaggerapi/petstore3", + "Ports" : [8080], + "ExpectedDurationUntilReady" : "00:00:30", + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + }, + "Shell" : "/bin/sh", + "OutputFolder" : "zap_petstore3" + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "fuzz", + "duration": "00:30:00", + "toolConfiguration": { + "task": "Fuzz", + "runConfiguration": { + "targetEndpointConfiguration" : { + "Port" : 8081 + }, + "useSsl" : false, + "inputFolderPath": "/job-compile/compile" + } + } + }, + { + "toolName" : "ZAP", + "outputFolder" : "zap", + "swaggerLocations": [{ + "URL" : "http://localhost:8080/api/v3/openapi.json" + }] + } + ] +} \ No newline at end of file diff --git a/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json b/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json deleted file mode 100644 index 80a59def..00000000 --- a/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "rootFileShare" : "{ci-run}", - "namePrefix" : "petstore3-test-fuzz-lean-", - "host": "localhost", - - "resources": { - "Cores": 4, - "MemoryGBs": 8 - }, - - "webhook":{ - "name" : "petstore3-test-fuzz-lean", - "metadata" : { - "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" - } - }, - - "readonlyFileShareMounts": [ - { - "FileShareName": "{ci-run}", - "MountPath": "/ci-run" - } - ], - - "testTargets" : { - "resources" : { - "Cores" : 2, - "MemoryGBs" : 4 - }, - "targets" : [ - { - "Container" : "swaggerapi/petstore3", - "Port" : 8081, - "ExpectedDurationUntilReady" : "00:02:00", - "Run" : { - "Command" : "/bin/sh", - "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] - } - } - ] - }, - "tasks": [ - { - "toolName": "RESTler", - "outputFolder": "test-fuzz-lean", - "duration": "01:00:00", - "toolConfiguration": { - "task": "TestFuzzLean", - "runConfiguration": { - "targetEndpointConfiguration" : { - "Port" : 8081 - }, - "useSsl" : false, - "inputFolderPath": "/ci-run/{compile.jobId}/compile" - } - } - } - ] -} diff --git a/cli/samples/zap/no-authentication/sample.zap.json b/cli/samples/zap/no-authentication/sample.zap.json index cfc6381e..e7b71844 100644 --- a/cli/samples/zap/no-authentication/sample.zap.json +++ b/cli/samples/zap/no-authentication/sample.zap.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{sample.host}/swagger/v1/swagger.json" - }, + }], "host": "{sample.host}", "tasks": [ { diff --git a/cli/samples/zap/running-against-raft-common-file-share/raft.zap.json b/cli/samples/zap/running-against-raft-common-file-share/raft.zap.json index 19a1ca5f..308c4a79 100644 --- a/cli/samples/zap/running-against-raft-common-file-share/raft.zap.json +++ b/cli/samples/zap/running-against-raft-common-file-share/raft.zap.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "Url": "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "host": "{defaults.deploymentName}-raft-apiservice.azurewebsites.net", "rootFileshare": "raft", "tasks": [ diff --git a/cli/samples/zap/running-against-raft/raft.zap.json b/cli/samples/zap/running-against-raft/raft.zap.json index ce1a71cb..723611e3 100644 --- a/cli/samples/zap/running-against-raft/raft.zap.json +++ b/cli/samples/zap/running-against-raft/raft.zap.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "host": "{defaults.deploymentName}-raft-apiservice.azurewebsites.net", "tasks": [ { diff --git a/cli/samples/zap/swagger-petstore3/zap.json b/cli/samples/zap/swagger-petstore3/zap.json index 6fcc335e..80109d4b 100644 --- a/cli/samples/zap/swagger-petstore3/zap.json +++ b/cli/samples/zap/swagger-petstore3/zap.json @@ -22,9 +22,12 @@ { "toolName" : "ZAP", "outputFolder" : "zap", - "swaggerLocation": { + "swaggerLocations": [{ "URL" : "http://localhost:8080/api/v3/openapi.json" - } + }, + { + "URL" : "http://localhost:8080/api/v3/openapi.json" + }] } ] } diff --git a/docs/how-to-submit-a-job.md b/docs/how-to-submit-a-job.md index 405edbf1..ffac45af 100644 --- a/docs/how-to-submit-a-job.md +++ b/docs/how-to-submit-a-job.md @@ -75,7 +75,7 @@ Now that you've settled on the options you'll set and their respective values, i to create the job definition JSON file you'll use to submit the job and put RAFT to work. All Job Definition JSON files must define at least 1 entry in the `tasks` array. -The `swaggerLocation`, `host`, and `duration` fields can be defined globally +The `swaggerLocations`, `host`, and `duration` fields can be defined globally or locally, depending on whether you want these values to apply to all tasks or to individual tasks. @@ -84,9 +84,9 @@ swagger location and host settings, along with two tasks: ```json { - "swaggerLocation": { + "swaggerLocations": [{ "URL" : "https://{sample.host}/swagger/v1/swagger.json" - }, + }], "host": "{sample.host}", "tasks": [ { @@ -107,12 +107,12 @@ occur: - A container group would be created and be named using the JOBID -- A container associated with "MyFirstTool" would be created, taking the `swaggerLocation` and `host` +- A container associated with "MyFirstTool" would be created, taking the `swaggerLocations` and `host` parameters from the global position, and `toolName` and `outputFolder` parameters from the task definition. Since there is no `duration` field, it will run until it completes. -- A container named "MySecondTool" would be created, taking the `swaggerLocation` and `host` +- A container named "MySecondTool" would be created, taking the `swaggerLocations` and `host` parameters from the global position, and `toolName`, `duration`, and `outputFolder` parameters from the task definition. It will run until it completes, or until 2 hours ten minutes have passed, whichever occurs first. diff --git a/docs/sdk/swagger.md b/docs/sdk/swagger.md index 0dfcca7f..109c659c 100644 --- a/docs/sdk/swagger.md +++ b/docs/sdk/swagger.md @@ -681,7 +681,7 @@ If the event name is not a supported value.
  • Resources
  • RunConfiguration
  • RunSummary
    • -
  • SwaggerLocation
    • +
  • SwaggerLocations
  • TargetEndpointConfiguration
  • WebHook
  • Webhook
    • @@ -801,7 +801,7 @@ and multiple errors need to be returned.

    JobDefinition Up

    RAFT job run definition
    -
    swaggerLocation (optional)
    SwaggerLocation
    +
    swaggerLocations (optional)
    SwaggerLocations
    namePrefix (optional)
    String String used as a prefix added to service generated job ID. Prefix can contain only lowercase letters, numbers, and hyphens, and must begin with a letter or a number. Prefix cannot contain two consecutive hyphens. @@ -856,7 +856,7 @@ For RESTler jobs - time limit is only useful for Fuzz task format: date-spantoolName
    String Tool defined by folder name located in cli/raft-tools/tools/{ToolName}
    outputFolder
    String Output folder name to store agent generated output Must not contain: /*?:|&lt;&gt;"
    -
    swaggerLocation (optional)
    SwaggerLocation
    +
    swaggerLocations (optional)
    SwaggerLocations
    host (optional)
    String Override the Host for each request.
    isIdling (optional)
    Boolean If true - do not run the task. Idle container to allow user to connect to it.
    duration (optional)
    String Duration of the task; if not set, then job level duration is used. @@ -918,7 +918,7 @@ For Replay task: path to RESTler Fuzz or Test run that contains bug buckets to r
    -

    SwaggerLocation Up

    +

    SwaggerLocations Up

    url (optional)
    String
    diff --git a/src/APIService/ApiService/APIServiceTests/test-restler-compile.json b/src/APIService/ApiService/APIServiceTests/test-restler-compile.json index 4d7ca82d..75276403 100644 --- a/src/APIService/ApiService/APIServiceTests/test-restler-compile.json +++ b/src/APIService/ApiService/APIServiceTests/test-restler-compile.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL": "https://a-test-service.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "NamePrefix": "test-compile-", "host": "a-test-service.azurewebsites.net", diff --git a/src/APIService/ApiService/APIServiceTests/test-restler-fuzz.json b/src/APIService/ApiService/APIServiceTests/test-restler-fuzz.json index ef6466f5..6794b4c0 100644 --- a/src/APIService/ApiService/APIServiceTests/test-restler-fuzz.json +++ b/src/APIService/ApiService/APIServiceTests/test-restler-fuzz.json @@ -1,7 +1,7 @@ { - "swaggerLocation": { + "swaggerLocations": [{ "URL" : "https://a-test-service.azurewebsites.net/swagger/v1/swagger.json" - }, + }], "duration": "00:10:00", "host": "a-test-service.azurewebsites.net", "tasks": [ diff --git a/src/APIService/ApiService/Controllers/Job.fs b/src/APIService/ApiService/Controllers/Job.fs index c414f401..ea3b8668 100644 --- a/src/APIService/ApiService/Controllers/Job.fs +++ b/src/APIService/ApiService/Controllers/Job.fs @@ -58,7 +58,7 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger Array.map (fun t -> t.OutputFolder) |> Array.sort - let validateSwaggerLocation (swaggerLocation:DTOs.SwaggerLocation) = + let validateSwaggerLocations (swaggerLocation:DTOs.SwaggerLocation) = // Validate the swagger URL. if not <| isNull (box swaggerLocation) then if not <| String.IsNullOrWhiteSpace(swaggerLocation.URL) then @@ -85,7 +85,8 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger Array.iter validateSwaggerLocations let requestPayload = match isNull (box requestPayload.Resources), isNull(box requestPayload.TestTargets) with @@ -178,8 +179,8 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger Array.map (fun t -> - if isNull (box t.SwaggerLocation) then - { t with SwaggerLocation = requestPayload.SwaggerLocation } + if isNull (box t.SwaggerLocations) then + { t with SwaggerLocations = requestPayload.SwaggerLocations } else t ) @@ -280,8 +281,8 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger Array.iter validateSwaggerLocations ) if not <| isNull requestPayload.ReadOnlyFileShareMounts then diff --git a/src/APIService/ApiService/DTOs.fs b/src/APIService/ApiService/DTOs.fs index 352904c5..e94632d0 100644 --- a/src/APIService/ApiService/DTOs.fs +++ b/src/APIService/ApiService/DTOs.fs @@ -60,9 +60,9 @@ module DTOs = OutputFolder : string /// - /// Override swagger specification location + /// Override swagger specification locations /// - SwaggerLocation : SwaggerLocation + SwaggerLocations : SwaggerLocation array /// /// Override the Host for each request. @@ -194,9 +194,9 @@ module DTOs = type JobDefinition = { /// - /// Swagger specification location for the job run + /// Swagger specifications location for the job run /// - SwaggerLocation : SwaggerLocation + SwaggerLocations : SwaggerLocation array /// /// String used as a prefix added to service generated job ID. diff --git a/src/Agent/RESTlerAgent/AgentMain.fs b/src/Agent/RESTlerAgent/AgentMain.fs index 2fc9ebfc..913889b4 100644 --- a/src/Agent/RESTlerAgent/AgentMain.fs +++ b/src/Agent/RESTlerAgent/AgentMain.fs @@ -220,13 +220,13 @@ let createRESTlerEngineParameters } type GrammarType = - | Swagger of string + | Swagger of string list | Json of string let createRESTlerCompilerConfiguration (workDirectory: string) (grammar: GrammarType) (customDictionary: string) (compileConfig: CompileConfiguration ) : Raft.RESTlerTypes.Compiler.Config = { - swaggerSpecFilePath = match grammar with Swagger path -> Some [path] | Json _ -> None + swaggerSpecFilePath = match grammar with Swagger paths -> Some paths | Json _ -> None // If specified, use this as the input and generate the python grammar. // This is path to JSON grammar. And we might need to accept as the step to FUZZ task (but do not need it for COMPILE task) @@ -517,24 +517,35 @@ let main argv = let compileSwaggerGrammar(compilerConfiguration) = async { - let! swagger = - async { - match task.SwaggerLocation with - | Some (Raft.Job.SwaggerLocation.URL url) -> - let! swagger = downloadFile workDirectory "swagger.json" url - printfn "Downloaded swagger spec to :%s" swagger - return swagger - | Some(Raft.Job.FilePath path) -> - return path - | None -> return failwith "Cannot perform compilation step, since Swagger Grammar location is not set" - } + match task.SwaggerLocations with + | None -> return failwith "Cannot perform compilation step, since Swagger Grammar location is not set" + | Some swaggerLocations -> + let! swagger = + swaggerLocations + |> Array.map (fun swaggerLocation -> + async { + let! swagger = + async { + match swaggerLocation with + | Raft.Job.SwaggerLocation.URL url -> + let! swagger = downloadFile workDirectory "swagger.json" url + printfn "Downloaded swagger spec to :%s" swagger + return swagger + | Raft.Job.FilePath path -> + return path + } - match validateJsonFile swagger with - | Result.Ok () -> () - | Result.Error (err) -> failwithf "File %s is not a valid JSON file due to %s" swagger err + match validateJsonFile swagger with + | Result.Ok () -> () + | Result.Error (err) -> failwithf "File %s is not a valid JSON file due to %s" swagger err - let compilerConfig = createRESTlerCompilerConfiguration workDirectory (Swagger swagger) customDictionaryPath compilerConfiguration - do! Raft.RESTlerDriver.compile restlerPath workDirectory compilerConfig + return swagger + } + ) + |> Async.Sequential + + let compilerConfig = createRESTlerCompilerConfiguration workDirectory (Swagger (swagger |> Array.toList)) customDictionaryPath compilerConfiguration + do! Raft.RESTlerDriver.compile restlerPath workDirectory compilerConfig } let compileJsonGrammar (jsonGrammarPath: string) (compilerConfiguration) = diff --git a/src/Contracts/Job.fs b/src/Contracts/Job.fs index 67c5b7d9..0e3a930a 100644 --- a/src/Contracts/Job.fs +++ b/src/Contracts/Job.fs @@ -71,7 +71,7 @@ type RaftTask = OutputFolder : string /// overwrite where to get swagger definition from - SwaggerLocation : SwaggerLocation option + SwaggerLocations : (SwaggerLocation array) option /// The string to use in overriding the Host for each request Host : string option @@ -103,7 +103,7 @@ type Webhook = type JobDefinition = { /// where to get swagger definition from - SwaggerLocation : SwaggerLocation option + SwaggerLocations : (SwaggerLocation array) option /// prefix for jobId NamePrefix : string option From da4fcfa7c2f76094eaabd328ae5f1dded147e3ca Mon Sep 17 00:00:00 2001 From: Stas Date: Tue, 1 Dec 2020 16:32:47 -0800 Subject: [PATCH 18/18] Support for a list of swagger specifications (#62) Co-authored-by: stas --- .../restler/self-contained/swagger-petstore3/compile.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/samples/restler/self-contained/swagger-petstore3/compile.json b/cli/samples/restler/self-contained/swagger-petstore3/compile.json index cd149901..ce824f3e 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/compile.json @@ -5,6 +5,10 @@ "URL" : "http://localhost:8082/api/v3/openapi.json" }], + "resources": { + "Cores": 4, + "MemoryGBs": 8 + "testTargets" : { "resources" : { "Cores" : 2,