Stas schema v2

README.md

@@ -13,18 +13,20 @@ As a platform, RAFT is designed to host any API fuzzers that are packaged into a
 These can be configured and used in the system via configuration files and require no code changes to integrate.
 
 ### Getting Started
-This project is designed to run on [Azure](https://azure.microsoft.com). 
+This project is designed to run on [Azure](https://azure.microsoft.com). See https://azure.com/free to create a free
+subscription and receive $200 in credits. You can run this service (and much more!)
+free for 30 days!
 
 To deploy the service download the CLI release and run `python raft.py service deploy`. See
-the [documentation](docs/deploying/deploying.md) for more details.
+the [documentation](docs/how-to-deploy.md) for more details.
 
 Once deployed, read about [how to submit a job](docs/how-to-submit-a-job.md) and
 use the [samples](docs/samples.md) to try out the service and fuzzers!
 
 ### Documentation
 
 * [Table of Contents](docs/index.md)
-* [Overview](docs/how-it-works/overview.md)
+* [Overview](docs/how-it-works)
 * [FAQ](docs/faq.md)
 
 ### Swagger Documentation

Scripts/Tests/bvt-petstore.py → Scripts/Tests/bvt-petstore3.py

@@ -33,30 +33,30 @@ def webhook_triggers_results(job_id, test_url):
 def time_span(t_start, t_end):
     return time.strftime("%H:%M:%S", time.gmtime(t_end - t_start))
 
-def bvt(cli, definitions):
+def bvt(cli, definitions, subs):
     print('Getting available wehook events')
     webhook_events = cli.list_available_webhooks_events()
     try:
         test_url = webhooks_test_url(definitions.subscription, definitions.resource_group, definitions.test_infra)
         for event in webhook_events:
             print(f'Setting webhook for {event}')
-            compile_webhook = cli.set_webhooks_subscription('petstore-compile', event, test_url)
-            fuzz_webhook = cli.set_webhooks_subscription('petstore-fuzz', event, test_url)
+            compile_webhook = cli.set_webhooks_subscription('petstore3-compile', event, test_url)
+            fuzz_webhook = cli.set_webhooks_subscription('petstore3-fuzz', event, test_url)
 
-        added_compile = cli.list_webhooks('petstore-compile', event)
+        added_compile = cli.list_webhooks('petstore3-compile', event)
         if len(added_compile) == 0:
-            raise Exception('Expected petstore-compile webhooks not to be empty after creation')
+            raise Exception('Expected petstore3-compile webhooks not to be empty after creation')
 
-        added_fuzz = cli.list_webhooks('petstore-fuzz', event)
+        added_fuzz = cli.list_webhooks('petstore3-fuzz', event)
         if len(added_fuzz) == 0:
-            raise Exception('Expected petstore-fuzz webhooks not to be empty after creation')
+            raise Exception('Expected petstore3-fuzz webhooks not to be empty after creation')
 
         t_pre_compile = time.time()
 
         print('Compile')
-        compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.compile.json'))
+        compile_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'compile.json'))
 
-        compile_config = raft.RaftJobConfig(file_path=compile_config_path)
+        compile_config = raft.RaftJobConfig(file_path=compile_config_path, substitutions=subs)
         compile_job = cli.new_job(compile_config)
         cli.poll(compile_job['jobId'], 10)
 
@@ -81,11 +81,9 @@ def bvt(cli, definitions):
                             f' {after_compile_pre_fuzz}')
 
         print('Fuzz')
-        fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore', 'restler.fuzz.json'))
-        subs = {}
+        fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'self-contained', 'swagger-petstore3', 'fuzz.json'))
         subs['{compile.jobId}'] = compile_job['jobId']
         fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs)
-        fuzz_config.config['duration'] = '00:20:00'
         fuzz_job = cli.new_job(fuzz_config)
         cli.poll(fuzz_job['jobId'], 10)
 
@@ -104,8 +102,8 @@ def bvt(cli, definitions):
                     raise Exception('Expected job to be in completed state when retrieved job list.'
                                     f'{after_fuzz}')
 
-        if m != 2:
-            raise Exception('Expected 2 after compile job step'
+        if m != 3:
+            raise Exception('Expected 3 after compile job step'
                             f' for job {fuzz_job["jobId"]}'
                             f' got {m}'
                             f' {after_fuzz}')
@@ -136,7 +134,7 @@ def bvt(cli, definitions):
         print('Validating that bugs posted events matches total bugs found in job status')
         total_bugs_found = 0
         for r in job_status_events:
-            if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId']:
+            if r['Data']['State'] == 'Completed' and r['Data']['AgentName'] != r['Data']['JobId'] and r['Data']['Tool'] == 'RESTler':
                 total_bugs_found += r['Data']['Metrics']['TotalBugBucketsCount']
 
         print(f'Total bugs found: {total_bugs_found}')
@@ -161,12 +159,11 @@ def bvt(cli, definitions):
         if len(deleted_fuzz) > 0:
             raise Exception('Expected petstore-fuzz webhooks to be empty after deletion, instead got %s', deleted_compile)
 
-
-
 if __name__ == "__main__":
     formatter = argparse.ArgumentDefaultsHelpFormatter
     parser = argparse.ArgumentParser(description='bvt', formatter_class=formatter)
     raft.add_defaults_and_secret_args(parser)
+    parser.add_argument('--build', required=True)
     args = parser.parse_args()
 
     if args.defaults_context_json:
@@ -179,4 +176,10 @@ def bvt(cli, definitions):
     definitions = RaftDefinitions(defaults)
     defaults['secret'] = args.secret
     cli = RaftCLI(defaults)
-    bvt(cli, definitions)
+    subs = {
+        "{build-url}" : os.environ.get('SYSTEM_COLLECTIONURI'),
+        "{build-id}" : os.environ.get('BUILD_BUILDID'),
+        "{ci-run}" : args.build.replace('.', '-')
+    }
+    print(f"SUBS: {subs}")
+    bvt(cli, definitions, subs)

Scripts/Tests/bvt.py

@@ -86,7 +86,6 @@ def bvt(cli, definitions, bvt_host):
         fuzz_config_path = os.path.abspath(os.path.join(cli_path, 'samples', 'restler', 'no-authentication', 'sample.restler.fuzz.json'))
         subs['{compile.jobId}'] = compile_job['jobId']
         fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path, substitutions=subs)
-        fuzz_config.config['duration'] = '00:20:00'
         fuzz_job = cli.new_job(fuzz_config) 
         cli.poll(fuzz_job['jobId'], 10)
 

ado/production-build.yml

@@ -5,10 +5,11 @@ trigger:
       - 'main'
 
 variables:
-  versionNumber: 1.0.0
-  imageTag: 'v1.0.0'
-  imageTagWithBuildDate: $(imageTag)-$(Build.BuildNumber)
-  imageTagLatest: 'v1.latest'
+  - template: 'variables/version-variables.yml'
+  - name: versionNumber
+    value: $(version.major).$(version.minor).$(version.revision)
+  - name: imageTagWithBuildDate
+    value: $(imageTag)-$(Build.BuildNumber)
 
 stages:
   - template: stages/build/build.yml

ado/stages/test/steps/bvt.yml

@@ -13,4 +13,4 @@ steps:
       azureSubscription: $(raft-subscription)
       scriptType: 'pscore'
       scriptLocation: 'inlineScript'
-      inlineScript: "python Scripts/Tests/bvt.py --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret) --host $(bvt-host)"
+      inlineScript: "python Scripts/Tests/bvt-petstore3.py --build $(Build.BuildNumber) --defaults-context-json '$(raft-defaults)' --secret $(bvt-secret)"

ado/variables/version-variables.yml

@@ -2,9 +2,11 @@ variables:
   - name: version.major
     value: 1
   - name: version.minor
+    value: 3
+  - name: version.revision
     value: 0
   - name: imageTag
-    value: 'v1.0.0'
+    value: 'v1.3.0'
   - name: imageTagLatest
     value: 'v1.latest'
   - name: devRevision

cli/raft.py

@@ -33,8 +33,9 @@
         - no dashes
 
 region - Region to deploy RAFT (e.g. westus2)
+    See the documentation on container instance region availability at
     https://docs.microsoft.com/en-us/azure/container-instances/container-instances-region-availability
-    for to pick the optimal region for your deployment.
+    to pick the optimal region for your deployment.
     All jobs will be deployed by default in the same
     region as your service deployment
 

cli/raft_sdk/raft_service.py

@@ -72,25 +72,6 @@ def __init__(self, context=None):
                             self.context['tenantId'],
                             self.context.get('secret'))
 
-    def result_url(self, job_id):
-        '''
-            Constructs Azure File Storage results URL
-
-            Parameters:
-                job_id: job ID
-
-            Returns:
-                URL that contains results of the job run
-        '''
-        return(
-            "https://ms.portal.azure.com/#blade/Microsoft_Azure_FileStorage/"
-            "FileShareMenuBlade/overview/storageAccountId/"
-            f"%2Fsubscriptions%2F{self.definitions.subscription}"
-            f"%2FresourceGroups%2F{self.definitions.resource_group}"
-            f"%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2F"
-            f"{self.definitions.storage_account}/"
-            f"path/{job_id}/protocol/")
-
     def job_status(self, job_id):
         '''
             Gets job status
@@ -286,6 +267,10 @@ def print_status(self, status):
         for s in status:
             if s['agentName'] == s['jobId']:
                 print(f"{s['jobId']} {s['state']}")
+                if s.get('utcEventTime'):
+                    print(f'UtcEventTime: {s["utcEventTime"]}')
+                if s.get('resultsUrl'):
+                    print(f'Results: {s["resultsUrl"]}')
                 if s.get('details'):
                     print("Details:")
                     for k in s['details']:
@@ -321,6 +306,7 @@ def print_status(self, status):
                     print("Details:")
                     for k in s['details']:
                         print(f"{k} : {s['details'][k]}")
+
                 print('======================')
 
     def poll(self, job_id, poll_interval=10):

cli/samples/restler/self-contained/swagger-petstore/restler.compile.json

@@ -3,10 +3,7 @@
     "URL" : "http://localhost:8080/api/swagger.json"
   }],
   "webhook": {
-    "name": "petstore-compile",
-    "metadata": {
-	    "action" : "compile"
-    }
+    "name": "petstore-compile"
   },
   "testTargets" : {
     "targets" : [

cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json

@@ -1,12 +1,7 @@
 {
   "host": "localhost",
   "webhook": {
-    "name": "petstore-fuzz",
-    "metadata": {
-      "app" : "petstore",
-      "swagger_version" : "v2",
-      "action" : "fuzz"	
-    }
+    "name": "petstore-fuzz"
   },
 
   "readonlyFileShareMounts": [

cli/samples/restler/self-contained/swagger-petstore/restler.test.json

@@ -2,12 +2,7 @@
   "host": "localhost",
 
   "webhook": {
-    "name": "petstore",
-    "metadata": {
-      "app" : "petstore",
-      "swagger_version" : "v2",
-      "action" : "test"	
-    }
+    "name": "petstore-test"
   },
   "readonlyFileShareMounts": [
     {

cli/samples/restler/self-contained/swagger-petstore/run.py

@@ -14,8 +14,6 @@ def run(compile, test, fuzz, replay):
     cli = RaftCLI()
     # Create compilation job configuration
     compile_job_config = RaftJobConfig(file_path=compile)
-    # add webhook metadata that will be included in every triggered webhook by Compile job
-    compile_job_config.add_metadata({"branch":"wizbangFeature"})
     print('Compile')
     # submit a new job with the Compile config and get new job ID
     compile_job = cli.new_job(compile_job_config)
@@ -37,8 +35,6 @@ def run(compile, test, fuzz, replay):
     # create a new job config with Fuzz configuration JSON
     fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs)
     print('Fuzz')
-    # add webhook metadata that will included in every triggered webhook by Fuzz job
-    fuzz_job_config.add_metadata({"branch":"wizbangFeature"})
     # create new fuzz job configuration
     fuzz_job = cli.new_job(fuzz_job_config)
 

cli/samples/restler/self-contained/swagger-petstore3/compile.json

@@ -0,0 +1,45 @@
+{ 
+  "rootFileShare" : "{ci-run}",
+  "namePrefix" : "petstore3-compile-",
+  "swaggerLocation": {
+    "URL" : "http://localhost:8082/api/v3/openapi.json"
+  }],
+
+  "resources": {
+    "Cores": 4,
+    "MemoryGBs": 8
+
+  "testTargets" : {
+    "resources" : {
+      "Cores" : 2,
+      "MemoryGBs" : 4
+    },
+    "targets" : [
+      {
+        "Container" : "swaggerapi/petstore3",
+        "Ports" : [8082],
+        "ExpectedDurationUntilReady" : "00:00:30",
+        "Run" : {
+          "Command" : "java",
+          "Arguments" : ["-jar", "/swagger-petstore/jetty-runner.jar", "--log", "/var/log/yyyy_mm_dd-requests.log", "--port", "8082", "/swagger-petstore/server.war"]
+        },
+        "PostRun" : {
+          "Command" : "/bin/sh", 
+          "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"],
+          "ExpectedRunDuration" : "00:00:10"
+        },
+        "Shell" : "/bin/sh",
+        "OutputFolder" : "petstore3"
+        }
+    ]
+  },
+  "tasks": [
+    {
+      "toolName": "RESTler",
+      "outputFolder": "compile",
+      "toolConfiguration": {
+        "task": "Compile"
+      }
+    }
+  ]
+}