diff --git a/cli/raft_sdk/raft_service.py b/cli/raft_sdk/raft_service.py index bd6f40b7..6cc9112c 100644 --- a/cli/raft_sdk/raft_service.py +++ b/cli/raft_sdk/raft_service.py @@ -2,9 +2,11 @@ # Licensed under the MIT License. import json +import yaml import os import sys import time +from pathlib import Path import tabulate from .raft_common import RaftApiException, RestApiClient, RaftDefinitions @@ -12,7 +14,6 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) dos2unix_file_types = [".sh", ".bash"] - class RaftJobConfig(): def __init__(self, *, @@ -24,7 +25,15 @@ def __init__(self, c = config_file.read() for src in substitutions: c = c.replace(src, substitutions[src]) - config = json.loads(c) + + ext = Path(file_path).suffix + if ext == '.json': + config = json.loads(c) + elif ext == '.yml' or ext == '.yaml': + config= yaml.load(c, Loader= yaml.FullLoader) + else: + raise Exception('Unsupported config file type') + self.config = config elif json: self.config = json_config diff --git a/cli/requirements.txt b/cli/requirements.txt index 2d2cf83e..a02774e9 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1,3 +1,4 @@ msal~=1.4.3 requests~=2.24.0 tabulate~=0.8.7 +pyyaml~=5.3.1 diff --git a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/combined.py b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/combined.py index 8256dbfb..d62edf9d 100644 --- a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/combined.py +++ b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/combined.py @@ -16,7 +16,6 @@ def run(compile, fuzz, sample_host): subs = { '{sample.host}' : sample_host, '{defaults.deploymentName}' : cli.definitions.deployment - } compile_job_config = raft.RaftJobConfig(file_path=compile, substitutions=subs) print('Compile') @@ -35,6 +34,6 @@ def run(compile, fuzz, sample_host): substitute {sample.host} in compile.json and fuzz.json config files') else: host = sys.argv[1] - run(os.path.join(cur_dir, "compile.json"), + run(os.path.join(cur_dir, "compile.yaml"), os.path.join(cur_dir, "fuzz.json"), host) \ No newline at end of file diff --git a/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.yaml b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.yaml new file mode 100644 index 00000000..2aba031e --- /dev/null +++ b/cli/samples/multiple-tools/running-against-raft-using-restler-and-zap/compile.yaml @@ -0,0 +1,19 @@ +swaggerLocation: + URL: "https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json" + +tasks: +- toolName: "RESTler" + outputFolder: "{defaults.deploymentName}-compile" + toolConfiguration: + task : Compile + compileConfiguration: + useRefreshableToken: true + +- toolName: "RESTler" + outputFolder: "sample-compile" + swaggerLocation: + URL: "https://{sample.host}/swagger/v1/swagger.json" + toolConfiguration: + task : Compile + compileConfiguration: + useRefreshableToken: true diff --git a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft-restler-replay.py b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft-restler-replay.py new file mode 100644 index 00000000..40a8574c --- /dev/null +++ b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft-restler-replay.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import pathlib +import sys +import os +import json +import urllib.parse + +cur_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(cur_dir, '..', '..', '..')) +from raft_sdk.raft_service import RaftCLI, RaftJobConfig + +def run(replay, fuzz_job_id, replay_job_id=None): + cli = RaftCLI() + substitutions = { + '{defaults.deploymentName}': cli.definitions.deployment, + '{jobRunId}' : fuzz_job_id + } + replay_job_config = RaftJobConfig(file_path=replay, substitutions=substitutions) + + print('Replay') + isIdle = False + for task in replay_job_config.config['tasks']: + isIdle = isIdle or task['isIdling'] + + if isIdle and replay_job_id: + cli.update_job(replay_job_id, replay_job_config) + print(f'Idle Job: {replay_job_id}') + else: + # create new fuzz job configuration + replay_job_id = cli.new_job(replay_job_config) + if isIdle: + print(f'New Idle Job: {replay_job_id}') + else: + print(f'New Job: {replay_job_id}') + + if not isIdle: + # wait for job ID from fuzz_job to finish the run + cli.poll(replay_job_id['jobId']) + + +if __name__ == "__main__": + run(replay = "raft.restler.replay.json", + #job ID that produced bugs and those bugs going to be replayed + fuzz_job_id = "d29c7a2a-1815-4edb-91c1-56dd4faea0ce", + replay_job_id=None) + \ No newline at end of file diff --git a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft-restler.py b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft-restler.py new file mode 100644 index 00000000..365a2639 --- /dev/null +++ b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft-restler.py @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import pathlib +import sys +import os +import json +import urllib.parse + +cur_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(cur_dir, '..', '..', '..')) +from raft_sdk.raft_service import RaftCLI, RaftJobConfig + +def run(compile, test, fuzz): + # instantiate RAFT CLI + cli = RaftCLI() + substitutions = { + '{defaults.deploymentName}': cli.definitions.deployment + } + compile_job_config = RaftJobConfig(file_path=compile, substitutions=substitutions) + print('Compile') + # create a new job with the Compile config and get new job ID + # in compile_job + compile_job = cli.new_job(compile_job_config) + # wait for a job with ID from compile_job to finish the run + cli.poll(compile_job['jobId']) + substitutions['{compile.jobId}'] = compile_job['jobId'] + + print('Test') + test_job_config = RaftJobConfig(file_path=test, substitutions=substitutions) + test_job = cli.new_job(test_job_config) + cli.poll(test_job['jobId']) + + print('Fuzz') + fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=substitutions) + fuzz_job = cli.new_job(fuzz_job_config) + cli.poll(fuzz_job['jobId']) + +if __name__ == "__main__": + run(os.path.join(cur_dir, "raft.restler.compile.yaml"), + os.path.join(cur_dir, "raft.restler.test.yaml"), + os.path.join(cur_dir, "raft.restler.fuzz.yaml")) \ No newline at end of file diff --git a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.compile.yaml b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.compile.yaml new file mode 100644 index 00000000..510408c4 --- /dev/null +++ b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.compile.yaml @@ -0,0 +1,10 @@ +swaggerLocation: + Url: https://{defaults.deploymentName}-raft-apiservice.azurewebsites.net/swagger/v1/swagger.json +rootFileShare: raft +tasks: +- toolName: RESTler + outputFolder: compile + toolConfiguration: + task: compile + compileConfiguration: + useRefreshableToken: true diff --git a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.fuzz.yaml b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.fuzz.yaml new file mode 100644 index 00000000..01ba6671 --- /dev/null +++ b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.fuzz.yaml @@ -0,0 +1,38 @@ +duration: 00:10:00 +host: {defaults.deploymentName}-raft-apiservice.azurewebsites.net +rootFileShare: raft +readOnlyFileShareMounts: +- FileShareName: raft + MountPath: /job-compile +tasks: +- toolName: RESTler + outputFolder: fuzz-1 + keyVaultSecrets: + - RaftServicePrincipal + authenticationMethod: + MSAL: RaftServicePrincipal + toolConfiguration: + task: Fuzz + runConfiguration: + inputFolderPath: /job-compile/{compile.jobId}/compile + +- toolName: RESTler + outputFolder: RESTler-fuzz-random-walk-2 + keyVaultSecrets: + - RaftServicePrincipal + authenticationMethod: + MSAL: RaftServicePrincipal + toolConfiguration: + task: FuzzRandomWalk + runConfiguration: + inputFolderPath: /job-compile/{compile.jobId}/compile +- toolName: RESTler + outputFolder: RESTler-fuzz-bfscheap-3 + keyVaultSecrets: + - RaftServicePrincipal + authenticationMethod: + MSAL: RaftServicePrincipal + toolConfiguration: + task: FuzzBfsCheap + runConfiguration: + inputFolderPath: /job-compile/{compile.jobId}/compile \ No newline at end of file diff --git a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.replay.yaml b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.replay.yaml new file mode 100644 index 00000000..cedeb5ba --- /dev/null +++ b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.replay.yaml @@ -0,0 +1,16 @@ +host: {defaults.deploymentName}-raft-apiservice.azurewebsites.net +rootFileShare: raft +readOnlyFileShareMounts: +- FileShareName: raft + MountPath: /job-run +tasks: +- toolName: RESTler + outputFolder: RESTLer-replay + keyVaultSecrets: + - RaftServicePrincipal + authenticationMethod: + MSAL: RaftServicePrincipal + toolConfiguration: + task: Replay + runConfiguration: + inputFolderPath: /job-run/{jobRunId}/fuzz-1 \ No newline at end of file diff --git a/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.test.yaml b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.test.yaml new file mode 100644 index 00000000..4b777d5f --- /dev/null +++ b/cli/samples/restler/running-against-raft-common-file-share-yaml/raft.restler.test.yaml @@ -0,0 +1,27 @@ +host: {defaults.deploymentName}-raft-apiservice.azurewebsites.net +rootFileShare: raft +readOnlyFileShareMounts: + - FileShareName: raft + MountPath: "/job-compile" +tasks: +- toolName: RESTler + outputFolder: RESTler-test + keyVaultSecrets: + - RaftServicePrincipal + authenticationMethod: + MSAL: RaftServicePrincipal + toolConfiguration: + task: Test + runConfiguration: + inputFolderPath: /job-compile/{compile.jobId}/compile +- toolName: RESTler + outputFolder: RESTler-test-fuzz-lean + keyVaultSecrets: + - RaftServicePrincipal + authenticationMethod: + MSAL: RaftServicePrincipal + toolConfiguration: + task: TestFuzzLean + runConfiguration: + inputFolderPath: /job-compile/{compile.jobId}/compile + authenticationTokenRefreshIntervalSeconds: 300 \ No newline at end of file diff --git a/src/APIService/ApiService/Program.fs b/src/APIService/ApiService/Program.fs index 03f601a7..758fe22b 100644 --- a/src/APIService/ApiService/Program.fs +++ b/src/APIService/ApiService/Program.fs @@ -127,7 +127,12 @@ module main = .UseExceptionHandler(fun options -> options.Run(RequestDelegate ErrorHandling.handleExceptions)) .UseMvc() .UseSwagger() - .UseSwaggerUI(fun config -> config.SwaggerEndpoint("/swagger/v1/swagger.json", "RAFT API")) + .UseSwaggerUI(fun config -> + config.ConfigObject.Urls <- [ + Swashbuckle.AspNetCore.SwaggerUI.UrlDescriptor(Name = "RAFT API Json", Url = "/swagger/v1/swagger.json") + Swashbuckle.AspNetCore.SwaggerUI.UrlDescriptor(Name = "RAFT API Yaml", Url = "/swagger/v1/swagger.yaml") + ] + ) |> ignore if Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") <> "Development" then