diff --git a/README.md b/README.md index ae3d19bd..1d70a94d 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,12 @@ As a platform, RAFT is designed to host any API fuzzers that are packaged into a These can be configured and used in the system via configuration files and require no code changes to integrate. ### Getting Started -This project is designed to run on [Azure](https://azure.microsoft.com). +This project is designed to run on [Azure](https://azure.microsoft.com). See https://azure.com/free to create a free +subscription and receive $200 in credits. You can run this service (and much more!) +free for 30 days! To deploy the service download the CLI release and run `python raft.py service deploy`. See -the [documentation](docs/deploying/deploying.md) for more details. +the [documentation](docs/how-to-deploy.md) for more details. Once deployed, read about [how to submit a job](docs/how-to-submit-a-job.md) and use the [samples](docs/samples.md) to try out the service and fuzzers! @@ -24,7 +26,7 @@ use the [samples](docs/samples.md) to try out the service and fuzzers! ### Documentation * [Table of Contents](docs/index.md) -* [Overview](docs/how-it-works/overview.md) +* [Overview](docs/how-it-works) * [FAQ](docs/faq.md) ### Swagger Documentation diff --git a/Scripts/Tests/bvt-petstore.py b/Scripts/Tests/bvt-petstore3.py similarity index 86% rename from Scripts/Tests/bvt-petstore.py rename to Scripts/Tests/bvt-petstore3.py index f24a7a4b..b5e94fd0 100644 --- a/Scripts/Tests/bvt-petstore.py +++ b/Scripts/Tests/bvt-petstore3.py @@ -33,30 +33,30 @@ def webhook_triggers_results(job_id, test_url): def time_span(t_start, t_end): return time.strftime("%H:%M:%S", time.gmtime(t_end - t_start)) -def bvt(cli, definitions): +def bvt(cli, definitions, subs): print('Getting available wehook events') webhook_events = cli.list_available_webhooks_events() try: test_url = webhooks_test_url(definitions.subscription, definitions.resource_group, definitions.test_infra) for event in webhook_events: print(f'Setting webhook for {event}') - compile_webhook = cli.set_webhooks_subscription('petstore-compile', event, test_url) - fuzz_webhook = cli.set_webhooks_subscription('petstore-fuzz', event, test_url) + compile_webhook = cli.set_webhooks_subscription('petstore3-compile', event, test_url) + fuzz_webhook = cli.set_webhooks_subscription('petstore3-fuzz', event, test_url) - added_compile = cli.list_webhooks('petstore-compile', event) + added_compile = cli.list_webhooks('petstore3-compile', event) if len(added_compile) == 0: - raise Exception('Expected petstore-compile webhooks not to be empty after creation') + raise Exception('Expected petstore3-compile webhooks not to be empty after creation') - added_fuzz = cli.list_webhooks('petstore-fuzz', event) + added_fuzz = cli.list_webhooks('petstore3-fuzz', event) if len(added_fuzz) == 0: - raise Exception('Expected petstore-fuzz webhooks not to be empty after creation') + raise Exception('Expected petstore3-fuzz webhooks not to be empty after creation') t_pre_compile = time.time() print('Compile') - compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.compile.json')) + compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'compile.json')) - compile_config = raft.RaftJobConfig(file_path=compile_config_path) + compile_config = raft.RaftJobConfig(file_path=compile_config_path, substitutions=subs) compile_job = cli.new_job(compile_config) cli.poll(compile_job['jobId'], 10) @@ -81,11 +81,9 @@ def bvt(cli, definitions): f' {after_compile_pre_fuzz}') print('Fuzz') - fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.fuzz.json')) - subs = {} + fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'fuzz.json')) subs['{compile.jobId}'] = compile_job['jobId'] fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs) - fuzz_config.config['duration'] = '00:20:00' fuzz_job = cli.new_job(fuzz_config) cli.poll(fuzz_job['jobId'], 10) @@ -104,8 +102,8 @@ def bvt(cli, definitions): raise Exception('Expected job to be in completed state when retrieved job list.' f'{after_fuzz}') - if m != 2: - raise Exception('Expected 2 after compile job step' + if m != 3: + raise Exception('Expected 3 after compile job step' f' for job {fuzz_job["jobId"]}' f' got {m}' f' {after_fuzz}') @@ -136,7 +134,7 @@ def bvt(cli, definitions): print('Validating that bugs posted events matches total bugs found in job status') total_bugs_found = 0 for r in job_status_events: - if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId']: + if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId'] and r['Data']['Tool'] == 'RESTler': total_bugs_found += r['Data']['Metrics']['TotalBugBucketsCount'] print(f'Total bugs found: {total_bugs_found}') @@ -161,12 +159,11 @@ def bvt(cli, definitions): if len(deleted_fuzz) > 0: raise Exception('Expected petstore-fuzz webhooks to be empty after deletion, instead got %s', deleted_compile) - - if __name__ == "__main__": formatter = argparse.ArgumentDefaultsHelpFormatter parser = argparse.ArgumentParser(description='bvt', formatter_class=formatter) raft.add_defaults_and_secret_args(parser) + parser.add_argument('--build', required=True) args = parser.parse_args() if args.defaults_context_json: @@ -179,4 +176,10 @@ def bvt(cli, definitions): definitions = RaftDefinitions(defaults) defaults['secret'] = args.secret cli = RaftCLI(defaults) - bvt(cli, definitions) \ No newline at end of file + subs = { + "{build-url}" : os.environ.get('SYSTEM_COLLECTIONURI'), + "{build-id}" : os.environ.get('BUILD_BUILDID'), + "{ci-run}" : args.build.replace('.', '-') + } + print(f"SUBS: {subs}") + bvt(cli, definitions, subs) \ No newline at end of file diff --git a/Scripts/Tests/bvt.py b/Scripts/Tests/bvt.py index 365f280f..5b732080 100644 --- a/Scripts/Tests/bvt.py +++ b/Scripts/Tests/bvt.py @@ -86,7 +86,6 @@ def bvt(cli, definitions, bvt_host): fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'no-authentication', 'sample.restler.fuzz.json')) subs['{compile.jobId}'] = compile_job['jobId'] fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs) - fuzz_config.config['duration'] = '00:20:00' fuzz_job = cli.new_job(fuzz_config) cli.poll(fuzz_job['jobId'], 10) diff --git a/ado/production-build.yml b/ado/production-build.yml index 7bb51182..500c17bd 100644 --- a/ado/production-build.yml +++ b/ado/production-build.yml @@ -5,10 +5,11 @@ trigger: - 'main' variables: - versionNumber: 1.0.0 - imageTag: 'v1.0.0' - imageTagWithBuildDate: $(imageTag)-$(Build.BuildNumber) - imageTagLatest: 'v1.latest' + - template: 'variables/version-variables.yml' + - name: versionNumber + value: $(version.major).$(version.minor).$(version.revision) + - name: imageTagWithBuildDate + value: $(imageTag)-$(Build.BuildNumber) stages: - template: stages/build/build.yml \ No newline at end of file diff --git a/ado/stages/test/steps/bvt.yml b/ado/stages/test/steps/bvt.yml index b51c15f2..7e962d9b 100644 --- a/ado/stages/test/steps/bvt.yml +++ b/ado/stages/test/steps/bvt.yml @@ -13,4 +13,4 @@ steps: azureSubscription: $(raft-subscription) scriptType: 'pscore' scriptLocation: 'inlineScript' - inlineScript: "python Scripts/Tests/bvt.py --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret) --host $(bvt-host)" + inlineScript: "python Scripts/Tests/bvt-petstore3.py --build $(Build.BuildNumber) --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret)" diff --git a/ado/variables/version-variables.yml b/ado/variables/version-variables.yml index 403c96fc..4e8fdc8f 100644 --- a/ado/variables/version-variables.yml +++ b/ado/variables/version-variables.yml @@ -2,9 +2,11 @@ variables: - name: version.major value: 1 - name: version.minor + value: 3 + - name: version.revision value: 0 - name: imageTag - value: 'v1.0.0' + value: 'v1.3.0' - name: imageTagLatest value: 'v1.latest' - name: devRevision diff --git a/cli/raft.py b/cli/raft.py index 628e5ba2..da1bccba 100644 --- a/cli/raft.py +++ b/cli/raft.py @@ -33,8 +33,9 @@ - no dashes region - Region to deploy RAFT (e.g. westus2) + See the documentation on container instance region availability at https://docs.microsoft.com/en-us/azure/container-instances/container-instances-region-availability - for to pick the optimal region for your deployment. + to pick the optimal region for your deployment. All jobs will be deployed by default in the same region as your service deployment diff --git a/cli/raft_sdk/raft_service.py b/cli/raft_sdk/raft_service.py index 6cc9112c..38e35444 100644 --- a/cli/raft_sdk/raft_service.py +++ b/cli/raft_sdk/raft_service.py @@ -72,25 +72,6 @@ def __init__(self, context=None): self.context['tenantId'], self.context.get('secret')) - def result_url(self, job_id): - ''' - Constructs Azure File Storage results URL - - Parameters: - job_id: job ID - - Returns: - URL that contains results of the job run - ''' - return( - "https://ms.portal.azure.com/#blade/Microsoft_Azure_FileStorage/" - "FileShareMenuBlade/overview/storageAccountId/" - f"%2Fsubscriptions%2F{self.definitions.subscription}" - f"%2FresourceGroups%2F{self.definitions.resource_group}" - f"%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2F" - f"{self.definitions.storage_account}/" - f"path/{job_id}/protocol/") - def job_status(self, job_id): ''' Gets job status @@ -286,6 +267,10 @@ def print_status(self, status): for s in status: if s['agentName'] == s['jobId']: print(f"{s['jobId']} {s['state']}") + if s.get('utcEventTime'): + print(f'UtcEventTime: {s["utcEventTime"]}') + if s.get('resultsUrl'): + print(f'Results: {s["resultsUrl"]}') if s.get('details'): print("Details:") for k in s['details']: @@ -321,6 +306,7 @@ def print_status(self, status): print("Details:") for k in s['details']: print(f"{k} : {s['details'][k]}") + print('======================') def poll(self, job_id, poll_interval=10): diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json index 15879125..fffb6c8f 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.compile.json @@ -3,10 +3,7 @@ "URL" : "http://localhost:8080/api/swagger.json" }], "webhook": { - "name": "petstore-compile", - "metadata": { - "action" : "compile" - } + "name": "petstore-compile" }, "testTargets" : { "targets" : [ diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json index cec6d2cb..6f09f566 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json @@ -1,12 +1,7 @@ { "host": "localhost", "webhook": { - "name": "petstore-fuzz", - "metadata": { - "app" : "petstore", - "swagger_version" : "v2", - "action" : "fuzz" - } + "name": "petstore-fuzz" }, "readonlyFileShareMounts": [ diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json index 6e242a3c..4c17b21c 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json @@ -2,12 +2,7 @@ "host": "localhost", "webhook": { - "name": "petstore", - "metadata": { - "app" : "petstore", - "swagger_version" : "v2", - "action" : "test" - } + "name": "petstore-test" }, "readonlyFileShareMounts": [ { diff --git a/cli/samples/restler/self-contained/swagger-petstore/run.py b/cli/samples/restler/self-contained/swagger-petstore/run.py index cc378e62..f0f13fe5 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/run.py +++ b/cli/samples/restler/self-contained/swagger-petstore/run.py @@ -14,8 +14,6 @@ def run(compile, test, fuzz, replay): cli = RaftCLI() # Create compilation job configuration compile_job_config = RaftJobConfig(file_path=compile) - # add webhook metadata that will be included in every triggered webhook by Compile job - compile_job_config.add_metadata({"branch":"wizbangFeature"}) print('Compile') # submit a new job with the Compile config and get new job ID compile_job = cli.new_job(compile_job_config) @@ -37,8 +35,6 @@ def run(compile, test, fuzz, replay): # create a new job config with Fuzz configuration JSON fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs) print('Fuzz') - # add webhook metadata that will included in every triggered webhook by Fuzz job - fuzz_job_config.add_metadata({"branch":"wizbangFeature"}) # create new fuzz job configuration fuzz_job = cli.new_job(fuzz_job_config) diff --git a/cli/samples/restler/self-contained/swagger-petstore3/compile.json b/cli/samples/restler/self-contained/swagger-petstore3/compile.json new file mode 100644 index 00000000..ce824f3e --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/compile.json @@ -0,0 +1,45 @@ +{ + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-compile-", + "swaggerLocation": { + "URL" : "http://localhost:8082/api/v3/openapi.json" + }], + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Ports" : [8082], + "ExpectedDurationUntilReady" : "00:00:30", + "Run" : { + "Command" : "java", + "Arguments" : ["-jar", "/swagger-petstore/jetty-runner.jar", "--log", "/var/log/yyyy_mm_dd-requests.log", "--port", "8082", "/swagger-petstore/server.war"] + }, + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + }, + "Shell" : "/bin/sh", + "OutputFolder" : "petstore3" + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "compile", + "toolConfiguration": { + "task": "Compile" + } + } + ] +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json new file mode 100644 index 00000000..42945a81 --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/fuzz.json @@ -0,0 +1,85 @@ +{ + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-fuzz-lean-", + "host": "localhost", + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + }, + + "webhook":{ + "name" : "petstore3-fuzz", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + + "readonlyFileShareMounts": [ + { + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" + } + ], + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Ports" : [8081], + "ExpectedDurationUntilReady" : "00:02:00", + "Run" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8081 /swagger-petstore/server.war"] + } + }, + "Shell" : "/bin/sh", + "OutputFolder" : "restler_petstore3" + }, + { + "Container" : "swaggerapi/petstore3", + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + }, + "Shell" : "/bin/sh", + "OutputFolder" : "zap_petstore3" + "Ports" : [8082], + "ExpectedDurationUntilReady" : "00:02:00", + "Run" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "java -jar /swagger-petstore/jetty-runner.jar --log $RAFT_WORK_DIRECTORY/yyyy_mm_dd-requests.log --port 8082 /swagger-petstore/server.war"] + } + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "fuzz", + "duration": "00:10:00", + "toolConfiguration": { + "task": "Fuzz", + "runConfiguration": { + "targetEndpointConfiguration" : { + "Port" : 8081 + }, + "useSsl" : false, + "inputFolderPath": "/ci-run/{compile.jobId}/compile" + } + } + }, + { + "toolName": "ZAP", + "outputFolder": "zap", + "swaggerLocations": [{ + "URL" : "http://localhost:8082/api/v3/openapi.json" + }] + } + ] +} diff --git a/cli/samples/restler/self-contained/swagger-petstore3/run.py b/cli/samples/restler/self-contained/swagger-petstore3/run.py index 5d46e0e7..13af9b15 100644 --- a/cli/samples/restler/self-contained/swagger-petstore3/run.py +++ b/cli/samples/restler/self-contained/swagger-petstore3/run.py @@ -4,47 +4,60 @@ import pathlib import sys import os +import json cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..')) -from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError +sys.path.append(os.path.join(cur_dir, '..')) +from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError, RaftDefinitions -def run(compile, test, fuzz): - # instantiate RAFT CLI - cli = RaftCLI() +def run(cli, config, subs): # Create compilation job configuration - compile_job_config = RaftJobConfig(file_path=compile) - print('Compile') + job_config = RaftJobConfig(file_path=config, substitutions=subs) + print(f'Running {config}') # submit a new job with the Compile config and get new job ID - compile_job = cli.new_job(compile_job_config) + job = cli.new_job(job_config) # wait for a job with ID from compile_job to finish the run - cli.poll(compile_job['jobId']) - - # use compile job as input for fuzz job - subs = {} - subs['{compile.jobId}'] = compile_job['jobId'] - - test_job_config = RaftJobConfig(file_path=test, substitutions=subs) - print('Test') - # create new fuzz job configuration - test_job = cli.new_job(test_job_config) - # wait for job ID from fuzz_job to finish the run - cli.poll(test_job['jobId']) - - # create a new job config with Fuzz configuration JSON - fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs) - print('Fuzz') - # create new fuzz job configuration - fuzz_job = cli.new_job(fuzz_job_config) - - # wait for job ID from fuzz_job to finish the run - cli.poll(fuzz_job['jobId']) + cli.poll(job['jobId']) + return job['jobId'] if __name__ == "__main__": try: - run(os.path.join(cur_dir, "restler.compile.json"), - os.path.join(cur_dir, "restler.test.json"), - os.path.join(cur_dir, "restler.zap.fuzz.json")) + defaults = None + + if sys.argv[1] == '--build': + build_id = sys.argv[2].replace(".", "-") + print(f"BUILD ID : {build_id}") + + with open(os.path.join(cur_dir, '..', 'defaults.json'), 'r') as defaults_json: + defaults = json.load(defaults_json) + if sys.argv[3] == '--secret': + defaults['secret'] = sys.argv[4] + + # instantiate RAFT CLI + cli = RaftCLI(defaults) + defs = RaftDefinitions(defaults) + + compile_job_id = None + subs = { + "{ci-run}" : f"{build_id}", + "{build-url}" : os.environ['SYSTEM_COLLECTIONURI'], + "{build-id}" : os.environ['BUILD_BUILDID'], + "{raft-subscription}": defs.subscription, + "{raft-resource-group}" : defs.resource_group, + "{raft-storage-account}" : defs.storage_account + } + for arg in sys.argv[1:]: + if arg == 'compile': + compile_job_id = run(cli, os.path.join(cur_dir, 'compile.json'), subs) + subs['{compile.jobId}'] = compile_job_id + + if arg == 'test': + run(cli, os.path.join(cur_dir, "test.json"), subs), + + if arg == 'test-fuzz-lean': + run(cli, os.path.join(cur_dir, "test-fuzz-lean.json"), subs), + except RaftJobError as ex: - print(f'ERROR: {ex.message}') \ No newline at end of file + print(f'ERROR: {ex.message}') + sys.exit(1) \ No newline at end of file diff --git a/cli/samples/restler/self-contained/swagger-petstore3/test.json b/cli/samples/restler/self-contained/swagger-petstore3/test.json new file mode 100644 index 00000000..447dc4b5 --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore3/test.json @@ -0,0 +1,61 @@ +{ + "rootFileShare" : "{ci-run}", + "namePrefix" : "petstore3-test-", + "host": "localhost", + + "resources": { + "Cores": 4, + "MemoryGBs": 8 + }, + + "webhook":{ + "name" : "petstore3-test", + "metadata" : { + "buildUrl" : "{build-url}/Raft/_build/results?buildId={build-id}&view=results" + } + }, + + "readonlyFileShareMounts": [ + { + "FileShareName": "{ci-run}", + "MountPath": "/ci-run" + } + ], + + "testTargets" : { + "resources" : { + "Cores" : 2, + "MemoryGBs" : 4 + }, + "targets" : [ + { + "Container" : "swaggerapi/petstore3", + "Ports" : [8080], + "ExpectedDurationUntilReady" : "00:00:30", + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + }, + "Shell" : "/bin/sh", + "OutputFolder" : "petstore3" + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "test", + "toolConfiguration": { + "task": "test", + "runConfiguration": { + "targetEndpointConfiguration" : { + "Port" : 8080 + }, + "useSsl" : false, + "inputFolderPath": "/ci-run/{compile.jobId}/compile" + } + } + } + ] +} diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 086bda1b..d2084feb 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -52,11 +52,6 @@ Here's a breakdown of the estimated Azure spend for running RAFT: - Container Instances - about $0.045 cents per running hour. (A container instance is created for each submitted job and runs for the specified duration in the job definition.) -In summary, a RAFT instance that ran a series of daily tests against ten REST -APIs would cost less than $100 a month. - -NOTE: `defaults.json` provides a flag to disable deployment of Application Insights during the initial service deployment. This will reduce Azure costs without affecting service functionality. -
## How a job executes diff --git a/docs/how-to-deploy.md b/docs/how-to-deploy.md index 059224ac..31f1dc74 100644 --- a/docs/how-to-deploy.md +++ b/docs/how-to-deploy.md @@ -2,32 +2,54 @@ The following guide should get you up and running with an instance of RAFT. -
+There are two main ways you can approach setting up RAFT. +- Download all requirements to your workstation and then use the RAFT CLI in a command window. +- Use the Azure Portal Cloud Shell. This requires no changes to your workstation. -### The first option is to setup all the dependencies on your workstation and use the RAFT CLI from there. The second option is to use the Azure Portal Shell. When using the portal's shell, you will only need to upload the CLI package as all required dependencies are already installed. +
-## Step 1: Enable the RAFT Command Line Interface (CLI) +## Step 1: Install the RAFT Command Line Interface Let's start out by getting the RAFT command line interface (CLI from now on) up and running. It functions just the same on Windows and Linux clients. -These two steps are required if you've decided to run the CLI from your workstation: -- First, you'll need to [install Python](https://www.python.org/downloads/) if -you don't have it installed already; RAFT requires at least **version 3.6**. +### If you are using your workstation -- Next, you'll need to [install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +- [Install Python](https://www.python.org/downloads/) if +you don't have it installed already; RAFT requires at least **version 3.6**. +- [Install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) if you haven't already; RAFT requires at least **version 2.12**. -If you've decided to use the Azure Portal Shell, keep in mind that the path to Python is `/opt/az/bin/python3` +### If you are using the Cloud Shell + +If you are going to use the Cloud Shell it is assumed that you have already acquired an Azure subscription +from https://azure.com/free or you have an existing subscription. + +Access the [Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview) from your +browser by clicking on the Cloud Shell icon.
+![](images/cloud-shell-icon.jpg) -- Now download the RAFT CLI, either just the binaries or the source tree if you intend to build them from source: +Or access it directly from your browser at https://shell.azure.com. +When using the shell for the first time it will create a storage account. This is normal and is needed to +persist data from one session to another. - - Get the RAFT CLI from [releases](https://github.com/microsoft/rest-api-fuzz-testing/releases) - - Clone the repo at https://github.com/microsoft/raft +In the cloud shell the path to Python version 3.6 `/opt/az/bin/python3` -- At this point, you're able to run a the one-time prep script using Python's -[pip package installer](https://pypi.org/project/pip/) as follows from the root -of the RAFT CLI folder: +### Common install instructions + +You will need the RAFT CLI files. You can do this in a number of ways: +- Download the RAFT CLI from a specific release
+ For example:
+ `wget https://github.com/microsoft/rest-api-fuzz-testing/releases/download/v1.2/cli.zip` +
+ Then run unzip `unzip cli.zip` +- Clone the repo +- Copy the sources + + +Once you have the python CLI files, you will need to install a few dependencies using Python's +[pip package installer](https://pypi.org/project/pip/) from the root +of the RAFT CLI folder. ```javascript $ pip install -r .\requirements.txt @@ -43,7 +65,7 @@ C:\Users\[user]\AppData\Local\Programs\Python\Python39\Scripts> pip.exe install
-- At this point, the RAFT CLI should be functional: +The RAFT CLI is now functional. ```javascript D:\REPO\raft\cli>py raft.py --help @@ -68,18 +90,18 @@ optional arguments: ## Step 2: Azure Subscription Prep -First, you will need an Azure subscription to host the RAFT service. If you +You will need an Azure subscription to host the RAFT service. If you don't already have access to an Azure subscription, please follow [these instructions](https://docs.microsoft.com/en-us/dynamics-nav/how-to--sign-up-for-a-microsoft-azure-subscription) -to sign up for one. +to sign up for one, or sign up for a free subscription at https://azure.com/free. -Second, you must be an [owner](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles) -on the target subscription to deploy RAFT, though once it's deployed you only need +You must be an [owner](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles) +on the subscription to deploy RAFT, though once it's deployed you only need [contributor](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles) rights to use it. -Last, RAFT uses [container instances](https://azure.microsoft.com/en-us/services/container-instances/) -to host running jobs; by default, Azure subscriptions are limited to 100. If your -target subscription is already using container instances, or you anticipate running +RAFT uses [container instances](https://azure.microsoft.com/en-us/services/container-instances/) +to host running jobs; by default, Azure subscriptions are limited to 100 container instances. If your +subscription is already using container instances, or you anticipate running more than 100 simultaneous jobs, you should reach out to Azure support and request they increase your [quota](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits) of this object type. @@ -96,14 +118,15 @@ Note that only four of these are required. | `subscription` | Yes | The subscription ID (GUID) of the subscription to which to deploy an instance of the RAFT service | | `deploymentName` | Yes | The name of your deployment; we will use this as the base name for all objects we create to instantiate the RAFT service | | `region` | Yes | The [region identifier](https://azure.microsoft.com/en-us/global-infrastructure/geographies/) that is closest to your location, or that's necessary for any compliance purposes | -| `metricsOptIn`* | Yes | Whether you want the service to send us anonymized usage data; we use this to improve the service and respond to errors and other problems proactively (Note: to change you choice, just update the field and redeploy) | -| `isDevelop` | No | Is this deployment for developing the RAFT service? Setting this value to true will generate yaml variables for use in your build pipelines | -| `useAppInsights` | No | deploy AppInsights and use it to write all service logs | -| `registry`** | No | registry which stores service images. Default: mcr.microsoft.com | +| `metricsOptIn`* | Yes | Whether you want the service to send us anonymized usage data; we use this to improve the service and respond to errors and other problems proactively (Note: to change you choice, just update the field and redeploy) Default: true| +| `isDevelop` | No | Is this deployment for developing the RAFT service? Default: false | +| `isPrivateRegistry` | No | When developing for the RAFT service, indicates a private registry is used to find images. Default: false | +| `useAppInsights` | Yes | Deploy AppInsights and use it to write all service logs. Default: true | +| `registry` | Yes | Registry which stores service images. Default: mcr.microsoft.com | ### *Telemetry *By default, we collect anonymous usage data from your RAFT instance, which helps -us understand how users use RAFT and the problems they experience, which in turn +us understand how users use RAFT and the problems they experience, which in turn, helps us improve the quality of the offering over time. Specifically, We do **not** collect any data about the targets and results of tools you might run. The data fields we collect are defined in the `src/Contracts/Telemetry.fs` source file. To opt-out of @@ -146,12 +169,11 @@ deploymentName - RAFT deployment name - no dashes region - Region to deploy RAFT (e.g. westus2) - See https://azure.microsoft.com/en-us/global-infrastructure/regions/ - for a list of regions - -isDevelop - Is this deployment for developing the RAFT service? - Setting this value to true will generate yaml variables for use in your - build pipelines. + See the documentation on container instance region availability at + https://docs.microsoft.com/en-us/azure/container-instances/container-instances-region-availability + to pick the optimal region for your deployment. + All jobs will be deployed by default in the same + region as your service deployment metricsOptIn - allow Microsoft collect anonymized metrics from the deployment. @@ -159,6 +181,7 @@ useAppInsights - deploy AppInsights and use it to write all service logs registry - registry which stores service images. Default: mcr.microsoft.com + ------------------------- To apply any changes made to the defaults.json file, please run 'raft.py service deploy' @@ -167,7 +190,7 @@ please run 'raft.py service deploy'
-Using a text editor of your choice, please update the `defaults.json` file with +Using a text editor of your choice, update the `defaults.json` file with the values you determined in Step 3, and then re-run: ```javascript @@ -196,6 +219,6 @@ get familiar with the output. Two tools are deployed by default: [RESTler](https://github.com/microsoft/restler) and [ZAP](https://www.zaproxy.org/). -You can see their configuration under the `cli/raft-tools/tools` folder. +You can see how they are configured by looking at the configuration files under the `cli/raft-tools/tools` folder. -See an explanation of the `config.json` file in [How a job executes](how-it-works/how-a-job-executes.md). \ No newline at end of file +See an explanation of the `config.json` file in [How to onboard a tool](how-to-onboard-a-tool.md). \ No newline at end of file diff --git a/docs/images/cloud-shell-icon.jpg b/docs/images/cloud-shell-icon.jpg new file mode 100644 index 00000000..3df73d41 Binary files /dev/null and b/docs/images/cloud-shell-icon.jpg differ diff --git a/docs/raft-restler-relationship.md b/docs/raft-restler-relationship.md index f9c1dfa0..8e2bbdd6 100644 --- a/docs/raft-restler-relationship.md +++ b/docs/raft-restler-relationship.md @@ -6,23 +6,28 @@ In order to provide a consistent platform for supporting multiple test tools the RAFT supports all of the same configuration values that RESTler supports. RESTler uses a mix of camel case and underscore parameters that it accepts as part of it's run configurations. On the other hand RAFT only uses camel case parameters. For example RESTler uses parameter `restler_custom_payload` where the same parameter is called `restlerCustomPayload` in RAFT job definition. -RESTler documentation: +You can find the RESTler documentation at https://github.com/microsoft/restler-fuzzer/tree/main/docs/user-guide -When using RESTler documentation for configuring RESTler tasks you can use RAFT swagger definition for paramater name conversion. +When using RESTler documentation for configuring RESTler tasks use the RAFT swagger definition for paramater name conversion. -*https://-raft-apiservice.azurewebsites.net/swagger/index.html* +The RAFT swagger definition can be found using this URL : *https://\-raft-apiservice.azurewebsites.net/swagger/index.html* -RESTler requires that IP and port number are specified in order to run a test. However, RAFT will do a DNS lookup on the host parameter, specified in the job definition file, on your behalf and fill in the IP parameter for you. RAFT also defaults the port number to 443 when using SSL and 80 when not using a secure connection. If for some reason you find that you still need to specify the IP and port number then manually provided values in TargetEndpoint configuration will override any lookup or default values. +RAFT will do a DNS lookup on the host parameter, specified in the job definition file, +on your behalf and fill in the IP parameter for you. RAFT also defaults the port number +to 443 when using SSL and 80 when not using a secure connection. If for some reason you +find that you still need to specify the IP and port number then manually provided values +in TargetEndpoint configuration will override any lookup or default values. ## RESTler mode of opearation -First RESTler compiles Swagger specifications into RESTler grammar. The output of compile step can be consumed by any of the following steps: Compile, Test, Fuzz. +RESTler needs to compile the Swagger specifications into RESTler grammar as a first step. The output of the compile step is then consumed by any of the following steps: Test, TestFuzzLean, Fuzz. To enable passing of data from one step to the next RAFT allows any file share in the storage account to be mounted by any task. -This way RAFT jobs can be executed in a "pipeline" manner by passing the output of the Compile job as input Test, Fuzz or Compile jobs. +This way RAFT jobs can be executed in a "pipeline" manner by passing the output of the Compile job as input to the Test, TestFuzzLean, or Fuzz jobs. -Compile job produces a job ID and a file share named the same as the job ID. This makes it possible for you to mount the output of one job as input to another. -The diagram below demonstrates how to pass output from the Compile step as input to the Test step of a RESTler task. +A compile job produces a job ID and a file share is created and named using the job ID. +This makes it possible for you to take the output written to a file share of one job and mount it to use as input to another. +The diagram below illustrates this behavior in the job definition files. ![RESTler ](images/restler-configs-flow.png) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs index 5dcf4d39..54970028 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_DeleteJob.fs @@ -33,7 +33,7 @@ type DeleteJobTests() = let fakeMessageSender = Fixtures.createFakeMessageSender Raft.Message.ServiceBus.Queue.delete let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(System.Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created") + let entity = JobStatusEntity(System.Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs index 8fffc5cd..1f83a477 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_GetJobStatus.fs @@ -33,7 +33,7 @@ type GetJobStatusTests() = member this.``GET /jobs/restler succeeds`` () = async { let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "29211868-8178-4e81-9b8d-d52025b4c2d4", jobStatusJson, "Created") + let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "29211868-8178-4e81-9b8d-d52025b4c2d4", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) let jobController = jobsController(Fixtures.createFakeTelemetryClient, Fixtures.createFakeLogger) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs index 08901830..5bed0b22 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_ListJobStatuses.fs @@ -30,7 +30,7 @@ type ListJobStatusesTests() = member this.``LIST /jobs/restler succeeds`` () = async { let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created") + let entity = JobStatusEntity(Guid.Parse("29211868-8178-4e81-9b8d-d52025b4c2d4").ToString(), "testAgent", jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) let jobController = jobsController(Fixtures.createFakeTelemetryClient, Fixtures.createFakeLogger) diff --git a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs index c4353988..fa22a3e5 100644 --- a/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs +++ b/src/APIService/ApiService/APIServiceTests/JobVerb_Tests_RePostJob.fs @@ -20,7 +20,7 @@ type jobsRePOSTTests() = let jobId = System.Guid.NewGuid().ToString() let jobStatusJson = File.ReadAllText("test-job-status.json") - let entity = JobStatusEntity(jobId, jobId, jobStatusJson, "Created") + let entity = JobStatusEntity(jobId, jobId, jobStatusJson, "Created", System.DateTime.UtcNow, "http://some-url") Raft.Utilities.raftStorage <- Fixtures.createFakeRaftStorage (Some entity) Raft.Utilities.toolsSchemas <- Map.empty.Add("RESTler", None) diff --git a/src/APIService/ApiService/Controllers/Job.fs b/src/APIService/ApiService/Controllers/Job.fs index 7550576e..ea3b8668 100644 --- a/src/APIService/ApiService/Controllers/Job.fs +++ b/src/APIService/ApiService/Controllers/Job.fs @@ -355,6 +355,17 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger return Some (r, results) } + let convertToJobStatus(overallJobStatus: JobStatusEntity) (results: JobStatusEntity seq) = + results + |> Seq.map (fun jobStatusEntity -> (Raft.Message.RaftEvent.deserializeEvent jobStatusEntity.JobStatus): Message.RaftEvent.RaftJobEvent) + |> Seq.map (fun jobStatus -> + if jobStatus.Message.AgentName = jobStatus.Message.JobId then + { jobStatus.Message with ResultsUrl = overallJobStatus.ResultsUrl } + else + jobStatus.Message + ) + + [] /// /// Submit a job definition. @@ -596,12 +607,8 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger - let decodedMessages = - results - |> Seq.map (fun jobStatusEntity -> (Raft.Message.RaftEvent.deserializeEvent jobStatusEntity.JobStatus): Message.RaftEvent.RaftJobEvent) - |> Seq.map (fun jobStatus -> jobStatus.Message) - + | Some (r, results) -> + let decodedMessages = convertToJobStatus r results stopWatch.Stop() Central.Telemetry.TrackMetric (TelemetryValues.ApiRequest(method, float stopWatch.ElapsedMilliseconds), "milliseconds", this :> ControllerBase) return JsonResult(decodedMessages) @@ -641,7 +648,6 @@ type jobsController(telemetryClient : TelemetryClient, logger : ILogger TableEntity @@ -399,6 +400,7 @@ type webhooksController(configuration : IConfiguration, telemetryClient : Teleme JobId = jobId AgentName = "1" Metadata = None + ResultsUrl = Some "https://azure-storage/results" BugDetails = Some( Map.empty diff --git a/src/APIService/ApiService/DTOs.fs b/src/APIService/ApiService/DTOs.fs index 8c4d8a84..e94632d0 100644 --- a/src/APIService/ApiService/DTOs.fs +++ b/src/APIService/ApiService/DTOs.fs @@ -318,4 +318,5 @@ module DTOs = AgentName : string + ResultsUrl : string } diff --git a/src/Agent/Dockerfile b/src/Agent/Dockerfile index f1008058..726010d4 100644 --- a/src/Agent/Dockerfile +++ b/src/Agent/Dockerfile @@ -1,3 +1,3 @@ -FROM acrrestler.azurecr.io/restler:restlerv7.0 +FROM mcr.microsoft.com/restlerfuzzer/restler:v7.1.0 COPY RestlerAgent /raft/agent COPY RaftResultAnalyzer /raft/result-analyzer diff --git a/src/Agent/RESTlerAgent/AgentMain.fs b/src/Agent/RESTlerAgent/AgentMain.fs index 12ec0413..913889b4 100644 --- a/src/Agent/RESTlerAgent/AgentMain.fs +++ b/src/Agent/RESTlerAgent/AgentMain.fs @@ -506,6 +506,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = None + ResultsUrl = None }: Raft.JobEvents.JobStatus) printfn "Got job configuration message: %A" restlerPayload @@ -580,6 +581,7 @@ let main argv = Metrics = summary UtcEventTime = System.DateTime.UtcNow Details = Some( details.Add("numberOfBugsFound", sprintf "%d" bugsListLen)) + ResultsUrl = None } : Raft.JobEvents.JobStatus) } @@ -596,6 +598,7 @@ let main argv = AgentName = agentName Metadata = None BugDetails = Some bugDetails + ResultsUrl = None } : Raft.JobEvents.BugFound) } @@ -852,6 +855,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = Some (Map.ofSeq replaySummaryDetails) + ResultsUrl = None } : Raft.JobEvents.JobStatus) return replaySummaryDetails @@ -921,10 +925,7 @@ let main argv = //.Add("numFullyValid", sprintf "%d" status.num_fully_valid) //.Add("numSequenceFailures", sprintf "%d" status.num_sequence_failures) //.Add("numInvalidByFailedResourceCreations", sprintf "%d" status.num_invalid_by_failed_resource_creations) - //.Add("throughput", sprintf "%f" status.throughput) //.Add("totalObjectCreations", sprintf "%d" status.total_object_creations) - //.Add("totalUniqueTestCases", sprintf "%f" status.total_unique_test_cases) - //.Add("totalSequences", sprintf "%d" status.total_sequences) ) printfn "Sending final event: %A with summary: %A and details %A" state summary details @@ -938,6 +939,7 @@ let main argv = Metrics = summary UtcEventTime = System.DateTime.UtcNow Details = details + ResultsUrl = None } : Raft.JobEvents.JobStatus) let restlerTelemetry = Restler.Telemetry.getDataFromTestingSummary testingSummary @@ -972,6 +974,7 @@ let main argv = Metrics = None UtcEventTime = System.DateTime.UtcNow Details = Some (Map.empty.Add("Error", ex.Message)) + ResultsUrl = None } : Raft.JobEvents.JobStatus) do! System.Console.Error.FlushAsync().ToAsync diff --git a/src/Agent/RESTlerAgent/RESTlerDriver.fs b/src/Agent/RESTlerAgent/RESTlerDriver.fs index 5c368a0d..1f3b0d5b 100644 --- a/src/Agent/RESTlerAgent/RESTlerDriver.fs +++ b/src/Agent/RESTlerAgent/RESTlerDriver.fs @@ -7,7 +7,7 @@ open System open Microsoft.FSharpLu module RESTler = - let version = "7.0.0" + let version = "7.1.0" module private RESTlerInternal = let inline (++) (path1: string) (path2 : string) = IO.Path.Join(path1, path2) @@ -151,15 +151,20 @@ module private RESTlerInternal = if Seq.isEmpty experiments then None else - let startedExperiments = - experiments - |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime) - |> Seq.sortBy ( fun e -> e.CreationTimeUtc ) - - if (Seq.length startedExperiments > 1) then - printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." - (Seq.length startedExperiments) startedExperiments runStartTime - startedExperiments |> Seq.tryHead + try + let startedExperiments = + experiments + |> Seq.filter ( fun e -> e.CreationTimeUtc >= runStartTime) + |> Seq.sortBy ( fun e -> e.CreationTimeUtc ) + + if (Seq.length startedExperiments > 1) then + printfn "There are : %d [%A] that have been create past %A. Using one closest to start time of this run." + (Seq.length startedExperiments) startedExperiments runStartTime + startedExperiments |> Seq.tryHead + with + | :? System.IO.IOException as ioex -> + printfn "Getting experiment folder interrupted due to : %s" ioex.Message + None else None @@ -433,7 +438,10 @@ let getListOfBugsFromBugBuckets bugBuckets = let path = bugBuckets ++ "bug_buckets.json" if IO.File.Exists path then let bugHashes: RESTlerTypes.Logs.BugHashes = Json.Compact.Strict.deserializeFile path - return Some bugHashes + if isNull (box bugHashes) then + return None + else + return Some bugHashes else return Some Map.empty else @@ -476,7 +484,7 @@ let pollForBugFound workingDirectory (token: Threading.CancellationToken) (runSt if IO.File.Exists bugsFoundPosted then let! bugsPosted = IO.File.ReadAllLinesAsync(bugsFoundPosted) |> Async.AwaitTask return Set.ofArray bugsPosted - else + else return ignoreBugHashes } let! updatedBugsPosted = @@ -484,13 +492,11 @@ let pollForBugFound workingDirectory (token: Threading.CancellationToken) (runSt |> Seq.map (fun (KeyValue(bugHash, bugFile)) -> async { if not <| postedBugs.Contains bugHash then - printfn "Posting bug found %s with hash %s" bugFile.file_path bugHash do! onBugFound (Map.empty.Add("Experiment", experiment.Name).Add("BugBucket", bugFile.file_path).Add("BugHash", bugHash)) return bugHash } ) |> Async.Sequential do! IO.File.WriteAllLinesAsync(bugsFoundPosted, updatedBugsPosted) |> Async.AwaitTask - return! poll() } poll() diff --git a/src/Agent/RESTlerAgent/RESTlerTypes.fs b/src/Agent/RESTlerAgent/RESTlerTypes.fs index b89992ae..903167a6 100644 --- a/src/Agent/RESTlerAgent/RESTlerTypes.fs +++ b/src/Agent/RESTlerAgent/RESTlerTypes.fs @@ -310,10 +310,7 @@ module Logs = num_fully_valid: int num_sequence_failures: int num_invalid_by_failed_resource_creations: int - throughput: float total_object_creations: int - total_unique_test_cases: float - total_sequences: int total_requests_sent : IDictionary bug_buckets : IDictionary } diff --git a/src/Contracts/Job.fs b/src/Contracts/Job.fs index e5d9b643..0e3a930a 100644 --- a/src/Contracts/Job.fs +++ b/src/Contracts/Job.fs @@ -96,7 +96,7 @@ type FileShareMount = type Webhook = { Name : string - Metadata : Map + Metadata : Map option } diff --git a/src/Contracts/JobEvents.fs b/src/Contracts/JobEvents.fs index b3d87030..7e9a9cf4 100644 --- a/src/Contracts/JobEvents.fs +++ b/src/Contracts/JobEvents.fs @@ -73,6 +73,7 @@ type JobStatus = Details: Map option Metadata : Map option AgentName: string + ResultsUrl : string option } static member EventType = Events.JobEventTypes.JobStatus.ToString() @@ -84,6 +85,7 @@ type BugFound = AgentName : string Metadata : Map option BugDetails : Map option + ResultsUrl : string option } static member EventType = Events.JobEventTypes.BugFound.ToString() \ No newline at end of file diff --git a/src/Contracts/StorageEntities.fs b/src/Contracts/StorageEntities.fs index 2d0f9054..c92a71d1 100644 --- a/src/Contracts/StorageEntities.fs +++ b/src/Contracts/StorageEntities.fs @@ -6,11 +6,13 @@ module Raft.StorageEntities open Microsoft.Azure.Cosmos.Table let JobStatusTableName = "JobStatus" -type JobStatusEntity(jobId, agentName, jobStatus, jobState) = +type JobStatusEntity(jobId, agentName, jobStatus, jobState, utcEventTime, resultsUrl) = inherit TableEntity(partitionKey=jobId, rowKey=agentName) - new() = JobStatusEntity(null, null, null, null) + new() = JobStatusEntity(null, null, null, null, System.DateTime.MinValue, null) member val JobStatus : string = jobStatus with get, set member val JobState : string = jobState with get, set + member val ResultsUrl : string = resultsUrl with get, set + member val UtcEventTime : System.DateTime = utcEventTime with get, set let JobTableName = "Job" diff --git a/src/Contracts/Telemetry.fs b/src/Contracts/Telemetry.fs index b999c98a..abfbb2f2 100644 --- a/src/Contracts/Telemetry.fs +++ b/src/Contracts/Telemetry.fs @@ -64,7 +64,7 @@ module Central = type TelemetryImpl(telemetryConfig: (TelemetryClient * string) option) = let convertTagsToProperties tags = tags |> dict - let version = Assembly.GetEntryAssembly().GetName().Version + let version = Assembly.GetCallingAssembly().GetName().Version member _.TelemetryConfig = telemetryConfig diff --git a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs index 7d5747a5..5c1d162b 100644 --- a/src/Orchestrator/OrchestratorLogic/Orchestrator.fs +++ b/src/Orchestrator/OrchestratorLogic/Orchestrator.fs @@ -124,7 +124,7 @@ module ContainerInstances = let containerGroupName (jobId: string) = jobId - let createJobStatus (jobId: string) (state: JobState) (details: Map option) = + let createJobStatus (jobId: string) (state: JobState) (resultsUrl : string option) (details: Map option) = let message: JobStatus = { AgentName = jobId.ToString() @@ -135,12 +135,13 @@ module ContainerInstances = UtcEventTime = System.DateTime.UtcNow Details = details Metadata = None + ResultsUrl = resultsUrl } Raft.Message.RaftEvent.createJobEvent message - let postStatus (jobStatusSender: ServiceBus.Core.MessageSender) (jobId: string) (state: JobState) (details: Map option) = + let postStatus (jobStatusSender: ServiceBus.Core.MessageSender) (jobId: string) (state: JobState) (resultsUrl : string option) (details: Map option) = async { - let jobStatus = createJobStatus jobId state details + let jobStatus = createJobStatus jobId state resultsUrl details do! jobStatusSender.SendAsync( ServiceBus.Message ( RaftEvent.serializeToBytes jobStatus ) ).ToAsync @@ -321,6 +322,15 @@ module ContainerInstances = return failwithf "Failed to get configuration for unsupported tool: %A" task.ToolName } + let jobResultsUrl subscription resourceGroup storageAccountName containerGroupName rootFileShare = + "https://ms.portal.azure.com/#blade/Microsoft_Azure_FileStorage/" + + "FileShareMenuBlade/overview/storageAccountId/" + + "%2Fsubscriptions%2F" + subscription + + "%2FresourceGroups%2F" + resourceGroup + + "%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2F" + + (sprintf "%s/" storageAccountName) + + (sprintf "path/%s/protocol/" (Option.defaultValue containerGroupName rootFileShare)) + let createJobShareAndFolders (logger: ILogger) (containerGroupName: string) (sasUrl: string) (jobCreateRequest: CreateJobRequest) = async { let shareName, createSubDirectory, shareQuota = @@ -493,6 +503,11 @@ module ContainerInstances = ) r, isIdling + let getTaskWorkDirectoryPath (containerGroupName : string) (rootFileShare: string option) (workDirectory : string) (taskOutputFolder : string) = + match rootFileShare with + | None -> sprintf "%s/%s" workDirectory taskOutputFolder + | Some _ -> sprintf "%s/%s/%s" workDirectory containerGroupName taskOutputFolder + let getContainerGroupInstanceConfiguration (containerGroupName: string) (logger:ILogger) @@ -527,7 +542,7 @@ module ContainerInstances = return { RunDirectory = Some runDirectory - WorkDirectory = Some(sprintf "%s/%s" workDirectory task.OutputFolder) + WorkDirectory = Some(getTaskWorkDirectoryPath containerGroupName jobCreateRequest.JobDefinition.RootFileShare workDirectory task.OutputFolder) ContainerName = (sprintf "%d-%s" i task.OutputFolder).ToLowerInvariant() ToolConfiguration = toolConfig } @@ -791,7 +806,7 @@ module ContainerInstances = RunDirectory = None WorkDirectory = match target.OutputFolder with - | Some x -> Some(sprintf "%s/%s" workDirectory x) + | Some x -> Some(getTaskWorkDirectoryPath containerGroupName jobCreateRequest.JobDefinition.RootFileShare workDirectory x) | None -> None ToolConfiguration = { @@ -957,14 +972,14 @@ module ContainerInstances = | :? Microsoft.Rest.Azure.CloudException as ce -> // it looks like the error when container group is transitioning states is OK to ignore. Need to get more info on that. logError "Failed to deploy container group %s due to %A (status code : %A)" containerGroupName ex ce.Response.StatusCode - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) | _ -> logError "Failed to deploy container group %s due to %A" containerGroupName ex - do! postStatus JobState.Error (Some (Map.empty.Add ("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add ("Error", ex.Message))) | _ -> logError "Failed to deploy container group %s due to %A" containerGroupName ex - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) } match existingContainerGroupOpt with @@ -984,7 +999,7 @@ module ContainerInstances = if decodedMessage.MessagePostCount > 0 && isError then logInfo "Message for job %A will not be reposted initial container group creation did not succeed" decodedMessage.Message.JobId else - do! postStatus JobState.Creating None + do! postStatus JobState.Creating None None match! createContainerGroupInstance containerGroupName logger azure secrets agentConfig (dockerConfigs, toolsConfigs) decodedMessage.Message reportDeploymentError with | Result.Ok () -> //this is newly created container. Poll until it is fully running and then update job status @@ -997,7 +1012,7 @@ module ContainerInstances = stopWatch.Stop() logError "Failed to create container group for job : %A due to %A (Time it took: %f total seconds)" decodedMessage.Message.JobId ex stopWatch.Elapsed.TotalSeconds Central.Telemetry.TrackError (TelemetryValues.Exception ex) - do! postStatus JobState.Error (Some (Map.empty.Add("Error", ex.Message))) + do! postStatus JobState.Error None (Some (Map.empty.Add("Error", ex.Message))) | Some existingContainerGroup -> match Option.ofObj existingContainerGroup.State with @@ -1007,7 +1022,9 @@ module ContainerInstances = stopWatch.Stop() logInfo "Time took to deploy job: %s total seconds %f. State: %s; Provisioning State : %s" containerGroupName stopWatch.Elapsed.TotalSeconds state existingContainerGroup.ProvisioningState - do! postStatus JobState.Created None + + let resultsUrl = jobResultsUrl azure.SubscriptionId agentConfig.ResourceGroup agentConfig.StorageAccount containerGroupName decodedMessage.Message.JobDefinition.RootFileShare + do! postStatus JobState.Created (Some resultsUrl) None if decodedMessage.Message.IsIdlingRun then do! runDebugContainers existingContainerGroup logger agentConfig dockerConfigs toolsConfigs decodedMessage.Message @@ -1087,7 +1104,7 @@ module ContainerInstances = } - let setRow (agentConfig : AgentConfig) (jobId: string, agentName : string) (message: string, state: Raft.JobEvents.JobState) (etag: string) = + let setRow (agentConfig : AgentConfig) (jobId: string, agentName : string) (message: string, state: Raft.JobEvents.JobState, utcEventTime : DateTime, resultsUrl : string) (etag: string) = async { let! table = getJobStatusTable agentConfig.StorageTableConnectionString let entity = JobStatusEntity( @@ -1095,6 +1112,8 @@ module ContainerInstances = agentName, message, state |> Microsoft.FSharpLu.Json.Compact.serialize, + utcEventTime, + resultsUrl, ETag = etag) let insertOp = TableOperation.InsertOrReplace(entity) @@ -1130,25 +1149,29 @@ module ContainerInstances = match! JobStatus.getRow agentConfig (jobId, agentName) with | Result.Error() -> logInfo "[STATUS] Failed to retrieve job status table row for %s:%s" jobId agentName | Result.Ok(r) -> - let currentStatusWithHigherPrecedence, etag = + let currentStatusWithHigherPrecedence, utcEventTime, resultsUrl, etag = match r with | Some row -> + let resultsUrl = Option.defaultValue row.ResultsUrl decodedMessage.Message.ResultsUrl let state = JobStatus.getState row - // if current row job state is one of the next states down the line, then ignore current message altogether. - // Since current message is "late" - if state ??> decodedMessage.Message.State then - Some (JobStatus.getEvent row), row.ETag + if state = decodedMessage.Message.State && decodedMessage.Message.UtcEventTime > row.UtcEventTime then + None, decodedMessage.Message.UtcEventTime, resultsUrl, row.ETag + else if (state ??> decodedMessage.Message.State) then + // if current row job state is one of the next states down the line, then ignore current message altogether. + // Since current message is "late" + Some (JobStatus.getEvent row), row.UtcEventTime, resultsUrl, row.ETag else - None, row.ETag + None, decodedMessage.Message.UtcEventTime, resultsUrl, row.ETag | None -> - None, null + let resultsUrl = Option.defaultValue null decodedMessage.Message.ResultsUrl + None, decodedMessage.Message.UtcEventTime, resultsUrl, null match currentStatusWithHigherPrecedence with | Some currentRowMessage -> logInfo "Dropping new status message since current status has higher precedence : %A and new message state is : %A [current status: %A new message status: %s ]" currentRowMessage.Message.State decodedMessage.Message.State currentRowMessage message | None -> - let! updated = JobStatus.setRow agentConfig (jobId, agentName) (message, decodedMessage.Message.State) etag + let! updated = JobStatus.setRow agentConfig (jobId, agentName) (message, decodedMessage.Message.State, utcEventTime, resultsUrl) etag if not updated then //Table record got updated by someone else, figure out what to do next. match decodedMessage.Message.State with @@ -1281,7 +1304,13 @@ module ContainerInstances = let webhookDefinition = Microsoft.FSharpLu.Json.Compact.deserialize(jobEntity.Webhook) match webhookDefinition with | Some webhook -> - return Some webhook.Metadata + match webhook.Metadata with + | None -> + return None + | Some m when m.IsEmpty -> + return None + | Some m -> + return Some m | None -> return None else @@ -1289,21 +1318,37 @@ module ContainerInstances = return None } + let getResultsUrl jobId = + async { + match! JobStatus.getRow agentConfig (jobId, jobId) with + | Result.Error() -> return None + | Result.Ok(r) -> + match r with + | None -> return None + | Some row -> return (if String.IsNullOrWhiteSpace row.ResultsUrl then None else Some row.ResultsUrl) + } + let eventType = RaftEvent.getEventType message if eventType = JobStatus.EventType then let jobStatus : RaftEvent.RaftJobEvent = RaftEvent.deserializeEvent message let! metadata = getMetadata jobStatus.Message.JobId + let! resultsUrl = getResultsUrl jobStatus.Message.JobId let updatedJobStatus = { jobStatus with Message = { jobStatus.Message with - Metadata = metadata } + Metadata = metadata + ResultsUrl = resultsUrl + } } do! processMessage updatedJobStatus.Message.JobId updatedJobStatus else if eventType = BugFound.EventType then let bugFound : RaftEvent.RaftJobEvent = RaftEvent.deserializeEvent message let! metadata = getMetadata bugFound.Message.JobId + let! resultsUrl = getResultsUrl bugFound.Message.JobId let updatedBugFound = { bugFound with Message = { bugFound.Message with - Metadata = metadata } + Metadata = metadata + ResultsUrl = resultsUrl + } } do! processMessage updatedBugFound.Message.JobId updatedBugFound @@ -1501,7 +1546,7 @@ module ContainerInstances = | JobState.Completing -> return false | s when JobState.Completing ??> s -> - do! postStatus communicationClients.JobEventsSender cg.Name JobState.Completing None + do! postStatus communicationClients.JobEventsSender cg.Name JobState.Completing None None let testTargets = cg.Containers |> Seq.map(fun (KeyValue(_, c)) -> c) @@ -1663,7 +1708,7 @@ module ContainerInstances = return details } - do! postStatus communicationClients.JobEventsSender g.Name state (Some detailsWithUsage) + do! postStatus communicationClients.JobEventsSender g.Name state None (Some detailsWithUsage) for v in instancesExitedWithError do let! failedContainerLogs = g.GetLogContentAsync(v.Name).ToAsync