diff --git a/Scripts/Tests/bvt-petstore.py b/Scripts/Tests/bvt-petstore3.py similarity index 86% rename from Scripts/Tests/bvt-petstore.py rename to Scripts/Tests/bvt-petstore3.py index f24a7a4b..b5e94fd0 100644 --- a/Scripts/Tests/bvt-petstore.py +++ b/Scripts/Tests/bvt-petstore3.py @@ -33,30 +33,30 @@ def webhook_triggers_results(job_id, test_url): def time_span(t_start, t_end): return time.strftime("%H:%M:%S", time.gmtime(t_end - t_start)) -def bvt(cli, definitions): +def bvt(cli, definitions, subs): print('Getting available wehook events') webhook_events = cli.list_available_webhooks_events() try: test_url = webhooks_test_url(definitions.subscription, definitions.resource_group, definitions.test_infra) for event in webhook_events: print(f'Setting webhook for {event}') - compile_webhook = cli.set_webhooks_subscription('petstore-compile', event, test_url) - fuzz_webhook = cli.set_webhooks_subscription('petstore-fuzz', event, test_url) + compile_webhook = cli.set_webhooks_subscription('petstore3-compile', event, test_url) + fuzz_webhook = cli.set_webhooks_subscription('petstore3-fuzz', event, test_url) - added_compile = cli.list_webhooks('petstore-compile', event) + added_compile = cli.list_webhooks('petstore3-compile', event) if len(added_compile) == 0: - raise Exception('Expected petstore-compile webhooks not to be empty after creation') + raise Exception('Expected petstore3-compile webhooks not to be empty after creation') - added_fuzz = cli.list_webhooks('petstore-fuzz', event) + added_fuzz = cli.list_webhooks('petstore3-fuzz', event) if len(added_fuzz) == 0: - raise Exception('Expected petstore-fuzz webhooks not to be empty after creation') + raise Exception('Expected petstore3-fuzz webhooks not to be empty after creation') t_pre_compile = time.time() print('Compile') - compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.compile.json')) + compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'compile.json')) - compile_config = raft.RaftJobConfig(file_path=compile_config_path) + compile_config = raft.RaftJobConfig(file_path=compile_config_path, substitutions=subs) compile_job = cli.new_job(compile_config) cli.poll(compile_job['jobId'], 10) @@ -81,11 +81,9 @@ def bvt(cli, definitions): f' {after_compile_pre_fuzz}') print('Fuzz') - fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.fuzz.json')) - subs = {} + fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'fuzz.json')) subs['{compile.jobId}'] = compile_job['jobId'] fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs) - fuzz_config.config['duration'] = '00:20:00' fuzz_job = cli.new_job(fuzz_config) cli.poll(fuzz_job['jobId'], 10) @@ -104,8 +102,8 @@ def bvt(cli, definitions): raise Exception('Expected job to be in completed state when retrieved job list.' f'{after_fuzz}') - if m != 2: - raise Exception('Expected 2 after compile job step' + if m != 3: + raise Exception('Expected 3 after compile job step' f' for job {fuzz_job["jobId"]}' f' got {m}' f' {after_fuzz}') @@ -136,7 +134,7 @@ def bvt(cli, definitions): print('Validating that bugs posted events matches total bugs found in job status') total_bugs_found = 0 for r in job_status_events: - if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId']: + if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId'] and r['Data']['Tool'] == 'RESTler': total_bugs_found += r['Data']['Metrics']['TotalBugBucketsCount'] print(f'Total bugs found: {total_bugs_found}') @@ -161,12 +159,11 @@ def bvt(cli, definitions): if len(deleted_fuzz) > 0: raise Exception('Expected petstore-fuzz webhooks to be empty after deletion, instead got %s', deleted_compile) - - if __name__ == "__main__": formatter = argparse.ArgumentDefaultsHelpFormatter parser = argparse.ArgumentParser(description='bvt', formatter_class=formatter) raft.add_defaults_and_secret_args(parser) + parser.add_argument('--build', required=True) args = parser.parse_args() if args.defaults_context_json: @@ -179,4 +176,10 @@ def bvt(cli, definitions): definitions = RaftDefinitions(defaults) defaults['secret'] = args.secret cli = RaftCLI(defaults) - bvt(cli, definitions) \ No newline at end of file + subs = { + "{build-url}" : os.environ.get('SYSTEM_COLLECTIONURI'), + "{build-id}" : os.environ.get('BUILD_BUILDID'), + "{ci-run}" : args.build.replace('.', '-') + } + print(f"SUBS: {subs}") + bvt(cli, definitions, subs) \ No newline at end of file diff --git a/ado/stages/test/steps/bvt.yml b/ado/stages/test/steps/bvt.yml index b51c15f2..7e962d9b 100644 --- a/ado/stages/test/steps/bvt.yml +++ b/ado/stages/test/steps/bvt.yml @@ -13,4 +13,4 @@ steps: azureSubscription: $(raft-subscription) scriptType: 'pscore' scriptLocation: 'inlineScript' - inlineScript: "python Scripts/Tests/bvt.py --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret) --host $(bvt-host)" + inlineScript: "python Scripts/Tests/bvt-petstore3.py --build $(Build.BuildNumber) --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret)" diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json index a2877fc4..da08370b 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json @@ -3,10 +3,7 @@ "URL" : "http://localhost:8080/api/swagger.json" }, "webhook": { - "name": "petstore-compile", - "metadata": { - "action" : "compile" - } + "name": "petstore-compile" }, "testTargets" : { "targets" : [ diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json index d1637606..a9dd63c1 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json @@ -1,12 +1,7 @@ { "host": "localhost", "webhook": { - "name": "petstore-fuzz", - "metadata": { - "app" : "petstore", - "swagger_version" : "v2", - "action" : "fuzz" - } + "name": "petstore-fuzz" }, "readonlyFileShareMounts": [ diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json index ec84ef8a..88f127f1 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json @@ -2,12 +2,7 @@ "host": "localhost", "webhook": { - "name": "petstore", - "metadata": { - "app" : "petstore", - "swagger_version" : "v2", - "action" : "test" - } + "name": "petstore-test" }, "readonlyFileShareMounts": [ { diff --git a/cli/samples/restler/self-contained/swagger-petstore/run.py b/cli/samples/restler/self-contained/swagger-petstore/run.py index cc378e62..f0f13fe5 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/run.py +++ b/cli/samples/restler/self-contained/swagger-petstore/run.py @@ -14,8 +14,6 @@ def run(compile, test, fuzz, replay): cli = RaftCLI() # Create compilation job configuration compile_job_config = RaftJobConfig(file_path=compile) - # add webhook metadata that will be included in every triggered webhook by Compile job - compile_job_config.add_metadata({"branch":"wizbangFeature"}) print('Compile') # submit a new job with the Compile config and get new job ID compile_job = cli.new_job(compile_job_config) @@ -37,8 +35,6 @@ def run(compile, test, fuzz, replay): # create a new job config with Fuzz configuration JSON fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs) print('Fuzz') - # add webhook metadata that will included in every triggered webhook by Fuzz job - fuzz_job_config.add_metadata({"branch":"wizbangFeature"}) # create new fuzz job configuration fuzz_job = cli.new_job(fuzz_job_config) diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore3/compile.json similarity index 87% rename from cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json rename to cli/samples/restler/self-contained/swagger-petstore3/compile.json index 1eb801d7..71ec7692 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/restler.compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/compile.json @@ -1,18 +1,11 @@ -{ +{ + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-compile-", "swaggerLocation": { "URL" : "http://localhost:8082/api/v3/openapi.json" }, - "resources": { - "Cores": 4, - "MemoryGBs": 8 - }, - "testTargets" : { - "resources" : { - "Cores" : 2, - "MemoryGBs" : 4 - }, "targets" : [ { "Container" : "swaggerapi/petstore3", diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json similarity index 54% rename from cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json rename to cli/samples/restler/self-contained/swagger-petstore3/fuzz.json index 4e1a2ee5..4c6678b5 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/restler.zap.fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json @@ -1,4 +1,6 @@ { + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-fuzz-lean-", "host": "localhost", "resources": { @@ -6,10 +8,17 @@ "MemoryGBs": 8 }, + "webhook":{ + "name" : "petstore3-fuzz", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + "readonlyFileShareMounts": [ { - "FileShareName": "{compile.jobId}", - "MountPath": "/job-compile" + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" } ], @@ -26,21 +35,16 @@ "Run" : { "Command" : "/bin/sh", "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] - }, - "Shell" : "/bin/sh", - "OutputFolder" : "restler_petstore3" + } }, { "Container" : "swaggerapi/petstore3", - "Port" : 8080, - "ExpectedDurationUntilReady" : "00:00:30", - "PostRun" : { - "Command" : "/bin/sh", - "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], - "ExpectedRunDuration" : "00:00:10" - }, - "Shell" : "/bin/sh", - "OutputFolder" : "zap_petstore3" + "Port" : 8082, + "ExpectedDurationUntilReady" : "00:02:00", + "Run" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8082 /swagger-petstore/server.war"] + } } ] }, @@ -48,7 +52,7 @@ { "toolName": "RESTler", "outputFolder": "fuzz", - "duration": "00:30:00", + "duration": "00:10:00", "toolConfiguration": { "task": "Fuzz", "runConfiguration": { @@ -56,16 +60,16 @@ "Port" : 8081 }, "useSsl" : false, - "inputFolderPath": "/job-compile/compile" + "inputFolderPath": "/ci-run/{compile.jobId}/compile" } } - }, + }, { - "toolName" : "ZAP", - "outputFolder" : "zap", + "toolName": "ZAP", + "outputFolder": "zap", "swaggerLocation": { - "URL" : "http://localhost:8080/api/v3/openapi.json" + "URL" : "http://localhost:8082/api/v3/openapi.json" } } ] -} \ No newline at end of file +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/run.py b/cli/samples/restler/self-contained/swagger-petstore3/run.py index 5d46e0e7..13af9b15 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/run.py +++ b/cli/samples/restler/self-contained/swagger-petstore3/run.py @@ -4,47 +4,60 @@ import pathlib import sys import os +import json cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..')) -from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError +sys.path.append(os.path.join(cur_dir, '..')) +from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError, RaftDefinitions -def run(compile, test, fuzz): - # instantiate RAFT CLI - cli = RaftCLI() +def run(cli, config, subs): # Create compilation job configuration - compile_job_config = RaftJobConfig(file_path=compile) - print('Compile') + job_config = RaftJobConfig(file_path=config, substitutions=subs) + print(f'Running {config}') # submit a new job with the Compile config and get new job ID - compile_job = cli.new_job(compile_job_config) + job = cli.new_job(job_config) # wait for a job with ID from compile_job to finish the run - cli.poll(compile_job['jobId']) - - # use compile job as input for fuzz job - subs = {} - subs['{compile.jobId}'] = compile_job['jobId'] - - test_job_config = RaftJobConfig(file_path=test, substitutions=subs) - print('Test') - # create new fuzz job configuration - test_job = cli.new_job(test_job_config) - # wait for job ID from fuzz_job to finish the run - cli.poll(test_job['jobId']) - - # create a new job config with Fuzz configuration JSON - fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs) - print('Fuzz') - # create new fuzz job configuration - fuzz_job = cli.new_job(fuzz_job_config) - - # wait for job ID from fuzz_job to finish the run - cli.poll(fuzz_job['jobId']) + cli.poll(job['jobId']) + return job['jobId'] if __name__ == "__main__": try: - run(os.path.join(cur_dir, "restler.compile.json"), - os.path.join(cur_dir, "restler.test.json"), - os.path.join(cur_dir, "restler.zap.fuzz.json")) + defaults = None + + if sys.argv[1] == '--build': + build_id = sys.argv[2].replace(".", "-") + print(f"BUILD ID : {build_id}") + + with open(os.path.join(cur_dir, '..', 'defaults.json'), 'r') as defaults_json: + defaults = json.load(defaults_json) + if sys.argv[3] == '--secret': + defaults['secret'] = sys.argv[4] + + # instantiate RAFT CLI + cli = RaftCLI(defaults) + defs = RaftDefinitions(defaults) + + compile_job_id = None + subs = { + "{ci-run}" : f"{build_id}", + "{build-url}" : os.environ['SYSTEM_COLLECTIONURI'], + "{build-id}" : os.environ['BUILD_BUILDID'], + "{raft-subscription}": defs.subscription, + "{raft-resource-group}" : defs.resource_group, + "{raft-storage-account}" : defs.storage_account + } + for arg in sys.argv[1:]: + if arg == 'compile': + compile_job_id = run(cli, os.path.join(cur_dir, 'compile.json'), subs) + subs['{compile.jobId}'] = compile_job_id + + if arg == 'test': + run(cli, os.path.join(cur_dir, "test.json"), subs), + + if arg == 'test-fuzz-lean': + run(cli, os.path.join(cur_dir, "test-fuzz-lean.json"), subs), + except RaftJobError as ex: - print(f'ERROR: {ex.message}') \ No newline at end of file + print(f'ERROR: {ex.message}') + sys.exit(1) \ No newline at end of file diff --git a/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json b/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json new file mode 100644 index 00000000..80a59def --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/test-fuzz-lean.json @@ -0,0 +1,59 @@ +{ + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-fuzz-lean-", + "host": "localhost", + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + }, + + "webhook":{ + "name" : "petstore3-test-fuzz-lean", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + + "readonlyFileShareMounts": [ + { + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" + } + ], + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Port" : 8081, + "ExpectedDurationUntilReady" : "00:02:00", + "Run" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] + } + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "test-fuzz-lean", + "duration": "01:00:00", + "toolConfiguration": { + "task": "TestFuzzLean", + "runConfiguration": { + "targetEndpointConfiguration" : { + "Port" : 8081 + }, + "useSsl" : false, + "inputFolderPath": "/ci-run/{compile.jobId}/compile" + } + } + } + ] +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore3/test.json similarity index 69% rename from cli/samples/restler/self-contained/swagger-petstore3/restler.test.json rename to cli/samples/restler/self-contained/swagger-petstore3/test.json index f1e1ff6b..ddda2f2a 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/restler.test.json +++ b/cli/samples/restler/self-contained/swagger-petstore3/test.json @@ -1,4 +1,6 @@ { + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-", "host": "localhost", "resources": { @@ -6,10 +8,17 @@ "MemoryGBs": 8 }, + "webhook":{ + "name" : "petstore3-test", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + "readonlyFileShareMounts": [ { - "FileShareName": "{compile.jobId}", - "MountPath": "/job-compile" + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" } ], @@ -29,7 +38,7 @@ "ExpectedRunDuration" : "00:00:10" }, "Shell" : "/bin/sh", - "OutputFolder" : "zap_petstore3" + "OutputFolder" : "petstore3" } ] }, @@ -44,7 +53,7 @@ "Port" : 8080 }, "useSsl" : false, - "inputFolderPath": "/job-compile/compile" + "inputFolderPath": "/ci-run/{compile.jobId}/compile" } } } diff --git a/src/Agent/RESTlerAgent/RESTlerDriver.fs b/src/Agent/RESTlerAgent/RESTlerDriver.fs index ea1a291e..1f3b0d5b 100644 --- a/src/Agent/RESTlerAgent/RESTlerDriver.fs +++ b/src/Agent/RESTlerAgent/RESTlerDriver.fs @@ -151,15 +151,20 @@ module private RESTlerInternal = if Seq.isEmpty experiments then None else - let startedExperiments = - experiments - |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime) - |> Seq.sortBy ( fun e -> e.CreationTimeUtc ) - - if (Seq.length startedExperiments > 1) then - printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." - (Seq.length startedExperiments) startedExperiments runStartTime - startedExperiments |> Seq.tryHead + try + let startedExperiments = + experiments + |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime) + |> Seq.sortBy ( fun e -> e.CreationTimeUtc ) + + if (Seq.length startedExperiments > 1) then + printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." + (Seq.length startedExperiments) startedExperiments runStartTime + startedExperiments |> Seq.tryHead + with + | :? System.IO.IOException as ioex -> + printfn "Getting experiment folder interrupted due to : %s" ioex.Message + None else None diff --git a/src/Contracts/Job.fs b/src/Contracts/Job.fs index 9ef3d736..bb2b3c39 100644 --- a/src/Contracts/Job.fs +++ b/src/Contracts/Job.fs @@ -87,7 +87,7 @@ type FileShareMount = type Webhook = { Name : string - Metadata : Map + Metadata : Map option } diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index e08d6b86..c284facf 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -1316,10 +1316,13 @@ module ContainerInstances = let webhookDefinition = Microsoft.FSharpLu.Json.Compact.deserialize(jobEntity.Webhook) match webhookDefinition with | Some webhook -> - if webhook.Metadata.IsEmpty then + match webhook.Metadata with + | None -> + return None + | Some m when m.IsEmpty -> return None - else - return Some webhook.Metadata + | Some m -> + return Some m | None -> return None else