diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json index ae4d5cec..d1637606 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.fuzz.json @@ -35,7 +35,7 @@ "tasks": [ { "toolName": "RESTler", - "outputFolder": "fuzz", + "outputFolder": "{outputFolder}", "duration": "00:10:00", "toolConfiguration": { "task": "Fuzz", @@ -44,7 +44,11 @@ "Port" : 8080 }, "useSsl" : false, - "inputFolderPath": "/job-compile/compile" + "inputFolderPath": "/job-compile/compile", + "IgnoreBugHashes" : [ + "PayloadBodyChecker_500_3d6e2d1e897967f3bc14cd8f08252a98b25c4d48", + "PayloadBodyChecker_500_6a4f4fb484d57121a271a4c65948ba8149e92dee" + ] } } } diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json b/cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json new file mode 100644 index 00000000..05c466eb --- /dev/null +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.replay-all-fuzz-bugs.json @@ -0,0 +1,44 @@ +{ + "readOnlyFileShareMounts": [ + { + "FileShareName": "{jobId}", + "MountPath": "/job-run" + } + ], + "testTargets" : { + "targets" : [ + { + "Container" : "swaggerapi/petstore", + "Port" : 8080, + "ExpectedDurationUntilReady" : "00:02:00", + "Shell" : "/bin/sh", + "OutputFolder" : "petstore", + "PostRun" : { + "Command" : "/bin/sh", + "Arguments" : ["-c", "cp /var/log/*-requests.log $RAFT_WORK_DIRECTORY"], + "ExpectedRunDuration" : "00:00:10" + } + } + ] + }, + "tasks": [ + { + "toolName": "RESTler", + "outputFolder": "RESTler-replay-{experiment}-all-bug-buckets", + "toolConfiguration": { + "task": "Replay", + "runConfiguration": { + "targetEndpointConfiguration" : { + "Port" : 8080 + }, + "useSsl" : false, + "inputFolderPath": "/job-run/{outputFolder}/RestlerResults/{experiment}/bug_buckets", + "ignoreBugHashes" : [ + "PayloadBodyChecker_500_acb1c0dcc79e949c24bec51f0ea3f841c61c38a3", + "PayloadBodyChecker_500_b8ebb5eaf46cdad18ac1d8e9ed67e2cdc729db3d" + ] + } + } + } + ] +} diff --git a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json index 97a6840a..ec84ef8a 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/restler.test.json +++ b/cli/samples/restler/self-contained/swagger-petstore/restler.test.json @@ -36,7 +36,7 @@ "tasks": [ { "toolName": "RESTler", - "outputFolder": "test-fuzz-lean", + "outputFolder": "test", "toolConfiguration": { "task": "Test", "runConfiguration": { @@ -44,7 +44,11 @@ "targetEndpointConfiguration" : { "Port" : 8080 }, - "inputFolderPath": "/job-compile/compile" + "inputFolderPath": "/job-compile/compile", + "ignoreBugHashes" : [ + "PayloadBodyChecker_500_7c2ae3b8c78eabad55a8075b487322dbbe146e46", + "PayloadBodyChecker_500_6349a559db300ca67b3caa316142c02a6bea0fbe" + ] } } } diff --git a/cli/samples/restler/self-contained/swagger-petstore/run.py b/cli/samples/restler/self-contained/swagger-petstore/run.py index 2ae579ed..cc378e62 100644 --- a/cli/samples/restler/self-contained/swagger-petstore/run.py +++ b/cli/samples/restler/self-contained/swagger-petstore/run.py @@ -9,10 +9,9 @@ sys.path.append(os.path.join(cur_dir, '..', '..', '..', '..')) from raft_sdk.raft_service import RaftCLI, RaftJobConfig, RaftJobError -def run(compile, test, fuzz): +def run(compile, test, fuzz, replay): # instantiate RAFT CLI cli = RaftCLI() - # Create compilation job configuration compile_job_config = RaftJobConfig(file_path=compile) # add webhook metadata that will be included in every triggered webhook by Compile job @@ -23,8 +22,8 @@ def run(compile, test, fuzz): # wait for a job with ID from compile_job to finish the run cli.poll(compile_job['jobId']) - # use compile job as input for fuzz job subs = {} + # use compile job as input for fuzz job subs['{compile.jobId}'] = compile_job['jobId'] test_job_config = RaftJobConfig(file_path=test, substitutions=subs) @@ -34,6 +33,7 @@ def run(compile, test, fuzz): # wait for job ID from fuzz_job to finish the run cli.poll(test_job['jobId']) + subs['{outputFolder}'] = 'fuzz' # create a new job config with Fuzz configuration JSON fuzz_job_config = RaftJobConfig(file_path=fuzz, substitutions=subs) print('Fuzz') @@ -45,10 +45,28 @@ def run(compile, test, fuzz): # wait for job ID from fuzz_job to finish the run cli.poll(fuzz_job['jobId']) + status = cli.job_status(fuzz_job['jobId']) + experiment = None + for s in status: + if (s['agentName'] != s['jobId']): + if (int(s['details']['numberOfBugsFound']) > 0): + experiment = s['details']['experiment'] + else: + print("Did not find any bugs") + + subs['{experiment}'] = experiment + subs['{jobId}'] = fuzz_job['jobId'] + + replay_job_config = RaftJobConfig(file_path=replay, substitutions=subs) + replay_job = cli.new_job(replay_job_config) + cli.poll(replay_job['jobId']) + + if __name__ == "__main__": try: run(os.path.join(cur_dir, "restler.compile.json"), os.path.join(cur_dir, "restler.test.json"), - os.path.join(cur_dir, "restler.fuzz.json")) + os.path.join(cur_dir, "restler.fuzz.json"), + os.path.join(cur_dir, "restler.replay-all-fuzz-bugs.json")) except RaftJobError as ex: print(f'ERROR: {ex.message}') \ No newline at end of file diff --git a/src/Agent/RESTlerAgent/AgentMain.fs b/src/Agent/RESTlerAgent/AgentMain.fs index 139d73b6..5636a45b 100644 --- a/src/Agent/RESTlerAgent/AgentMain.fs +++ b/src/Agent/RESTlerAgent/AgentMain.fs @@ -547,10 +547,17 @@ let main argv = do! Raft.RESTlerDriver.compile restlerPath workDirectory compilerConfig } - let report state (summary: Raft.JobEvents.RunSummary option) = + let report state (experiment : string option, summary:Raft.JobEvents.RunSummary option) = async { printfn "Reporting summary [%A]: %A" state summary let! bugsList = Raft.RESTlerDriver.getListOfBugs workDirectory globalRunStartTime + let bugsListLen = match bugsList with None -> 0 | Some xs -> Seq.length xs + + let details = + match experiment with + | Some e -> Map.empty.Add("Experiment", e) + | None -> Map.empty + do! jobEventSender.SendRaftJobEvent jobId ({ AgentName = agentName @@ -561,7 +568,7 @@ let main argv = Metrics = summary UtcEventTime = System.DateTime.UtcNow - Details = Some <| Map.empty.Add("numberOfBugsFound", sprintf "%d" (Seq.length bugsList)) + Details = Some( details.Add("numberOfBugsFound", sprintf "%d" bugsListLen)) } : Raft.JobEvents.JobStatus) } @@ -617,7 +624,13 @@ let main argv = let targetIp, targetPort = getIpAndPort jobConfiguration.TargetEndpointConfiguration let engineParameters = createRESTlerEngineParameters (targetIp, targetPort) grammarPy dictJson task checkerOptions jobConfiguration printfn "Starting RESTler test task" - do! Raft.RESTlerDriver.test testType restlerPath workDirectory engineParameters onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval) + + let ignoreBugHashes = + match jobConfiguration.IgnoreBugHashes with + | None -> Set.empty + | Some hs -> Set.ofArray hs + + do! Raft.RESTlerDriver.test testType restlerPath workDirectory engineParameters ignoreBugHashes onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval) } let fuzz (fuzzType:string) checkerOptions (jobConfiguration: RunConfiguration) = @@ -626,7 +639,13 @@ let main argv = let targetIp, targetPort = getIpAndPort jobConfiguration.TargetEndpointConfiguration let engineParameters = createRESTlerEngineParameters (targetIp, targetPort) grammarPy dictJson task checkerOptions jobConfiguration printfn "Starting RESTler fuzz task" - do! Raft.RESTlerDriver.fuzz fuzzType restlerPath workDirectory engineParameters onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval) + + let ignoreBugHashes = + match jobConfiguration.IgnoreBugHashes with + | None -> Set.empty + | Some hs -> Set.ofArray hs + + do! Raft.RESTlerDriver.fuzz fuzzType restlerPath workDirectory engineParameters ignoreBugHashes onBugFound (report Raft.JobEvents.JobState.Running) (globalRunStartTime, resultAnalyzerReportInterval) } let replay replayLogFile (jobConfiguration: RunConfiguration) = @@ -658,7 +677,7 @@ let main argv = // Start RESTler process with following flags // compile, test, fuzz // --telemetryRootDirPath - let! state, details, exitCode, summary = + let! state, details, exitCode, (experiment, summary) = async { try match restlerPayload.Task with @@ -697,7 +716,7 @@ let main argv = | None -> return failwithf "Job-run configuration is not set for Test task for job payload: %A" restlerPayload | Some jobConfiguration -> copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath]) - do! report Raft.JobEvents.Running None + do! report Raft.JobEvents.Running (None, None) do! test "directed-smoke-test" [] jobConfiguration let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime return Raft.JobEvents.Completed, None, 0, summary @@ -708,7 +727,7 @@ let main argv = | None -> return failwithf "Job-run configuration is not set for Test task for job payload: %A" restlerPayload | Some jobConfiguration -> copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath]) - do! report Raft.JobEvents.Running None + do! report Raft.JobEvents.Running (None, None) let fuzzLeanCheckers = [ ("--enable_checkers", "*") @@ -724,7 +743,7 @@ let main argv = | None -> return failwithf "Job-run configuration is not set for Compile task for job payload: %A" restlerPayload | Some jobConfiguration -> copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath]) - do! report Raft.JobEvents.Running None + do! report Raft.JobEvents.Running (None, None) let allCheckers = ["--enable_checkers", "*"] do! fuzz "bfs" allCheckers jobConfiguration let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime @@ -736,7 +755,7 @@ let main argv = | None -> return failwithf "Job-run configuration is not set for Test task for job payload: %A" restlerPayload | Some jobConfiguration -> copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath]) - do! report Raft.JobEvents.Running None + do! report Raft.JobEvents.Running (None, None) let allCheckers = ["--enable_checkers", "*"] do! fuzz "bfs-cheap" allCheckers jobConfiguration let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime @@ -748,7 +767,7 @@ let main argv = | None -> return failwithf "Job-run configuration is not set for Fuzz task for job payload: %A" restlerPayload | Some jobConfiguration -> copyDir jobConfiguration.InputFolderPath workDirectory (set [taskConfigurationPath]) - do! report Raft.JobEvents.Running None + do! report Raft.JobEvents.Running (None, None) let allCheckers = ["--enable_checkers", "*"] do! fuzz "random-walk" allCheckers jobConfiguration let! summary = Raft.RESTlerDriver.processRunSummary workDirectory globalRunStartTime @@ -759,23 +778,59 @@ let main argv = | None -> return failwithf "Job-run configuration is not set Replay task for job payload: %A" restlerPayload | Some replayRunConfiguration -> let replaySourcePath = replayRunConfiguration.InputFolderPath - do! report Raft.JobEvents.Running None + do! report Raft.JobEvents.Running (None, None) let replayAndReport (bugs: IO.FileInfo seq) = async { + let! existingBugBuckets = Raft.RESTlerDriver.getListOfBugsFromBugBuckets replayRunConfiguration.InputFolderPath + + let remapBugHashes (bugHashes : Raft.RESTlerTypes.Logs.BugHashes) = + bugHashes + |> Seq.map(fun (KeyValue(k, v)) -> v.file_path, k) + |> Map.ofSeq + + let bugs, fileNameToBugHashMap = + match replayRunConfiguration.IgnoreBugHashes, existingBugBuckets with + | None, None | Some _, None -> bugs, Map.empty + | None, Some existingBugs -> + bugs, (remapBugHashes existingBugs) + + | Some ignoreHashes, Some existingBugs -> + let filesToIgnore = + ignoreHashes + |> Array.map(fun h -> + match Map.tryFind h existingBugs with + | None -> None + | Some b -> Some b.file_path + ) + |> Array.filter Option.isSome + |> Array.map Option.get + |> Set.ofArray + + let filteredBugs = + bugs + |> Seq.filter (fun b -> + not (filesToIgnore.Contains b.Name) + ) + filteredBugs, (remapBugHashes existingBugs) + let replaySummaryDetails = ResizeArray() for bug in bugs do printfn "Running replay on %s" bug.FullName - let! replaySummary = replay bug.FullName replayRunConfiguration + let! (experiment, replaySummary) = replay bug.FullName replayRunConfiguration let summary = match replaySummary with | None -> sprintf "%s : No results" bug.Name | Some s -> - sprintf "%s: %A" bug.Name (s.ResponseCodeCounts |> Map.toList) + sprintf "%s/%s: %A" (Option.defaultValue "ExperimentNotSet" experiment) bug.Name (s.ResponseCodeCounts |> Map.toList) - replaySummaryDetails.Add(bug.Name, summary) + let bugKey = + match Map.tryFind bug.Name fileNameToBugHashMap with + | None -> bug.Name + | Some h -> h + replaySummaryDetails.Add(bugKey, summary) do! jobEventSender.SendRaftJobEvent jobId ({ AgentName = agentName @@ -823,7 +878,7 @@ let main argv = return Some details | None -> return! replayAll() } - return Raft.JobEvents.Completed, (details |> Option.map Map.ofSeq), 0, None + return Raft.JobEvents.Completed, (details |> Option.map Map.ofSeq), 0, (None, None) with | ex -> printfn "%A" ex @@ -832,9 +887,16 @@ let main argv = } let! bugsList = Raft.RESTlerDriver.getListOfBugs workDirectory globalRunStartTime + let bugsListLen = match bugsList with None -> 0 | Some xs -> Seq.length xs let testingSummary = Raft.RESTlerDriver.loadTestRunSummary workDirectory globalRunStartTime + let details = + let d = Option.defaultValue Map.empty details + match experiment with + | Some e -> Some (d.Add("Experiment", e)) + | None -> Some d + let details = match testingSummary with | None -> details @@ -842,7 +904,7 @@ let main argv = let d = Option.defaultValue Map.empty details Some( d .Add("finalSpecCoverage", status.final_spec_coverage) - .Add("numberOfBugsFound", sprintf "%d" (Seq.length bugsList)) + .Add("numberOfBugsFound", sprintf "%d" bugsListLen) //.Add("renderedRequests", status.rendered_requests) //.Add("renderedRequestsValidStatus", status.rendered_requests_valid_status) //.Add("numFullyValid", sprintf "%d" status.num_fully_valid) diff --git a/src/Agent/RESTlerAgent/RESTlerAgentConfigTypes.fs b/src/Agent/RESTlerAgent/RESTlerAgentConfigTypes.fs index 7cf2689a..e5acd6c7 100644 --- a/src/Agent/RESTlerAgent/RESTlerAgentConfigTypes.fs +++ b/src/Agent/RESTlerAgent/RESTlerAgentConfigTypes.fs @@ -102,8 +102,11 @@ type RunConfiguration = /// Path regex for filtering tested endpoints PathRegex : string option - } + // In context of Replay - do not replay bugs specified in the list + // In context of Test or Fuzz - do not post onBugFound events if they are in the list + IgnoreBugHashes : string array option + } type AgentConfiguration = diff --git a/src/Agent/RESTlerAgent/RESTlerDriver.fs b/src/Agent/RESTlerAgent/RESTlerDriver.fs index 38d57b33..5c368a0d 100644 --- a/src/Agent/RESTlerAgent/RESTlerDriver.fs +++ b/src/Agent/RESTlerAgent/RESTlerDriver.fs @@ -200,16 +200,15 @@ module private RESTlerInternal = printfn "Result Analyzer did not produce exit code" let summaryPath = restlerExperimentLogs ++ "raft-analyzer-summary.json" - if IO.File.Exists summaryPath then - return Some summaryPath + return Some (experimentFolder.Name, summaryPath) else eprintfn "RESTLER Summary file was not found: %s" summaryPath return None else return None } - return Some summaryPath + return summaryPath | None -> return None } @@ -372,27 +371,23 @@ let compile restlerRootDirectory workingDirectory config = RESTlerInternal.compile restlerRootDirectory workingDirectory config -type ReportRunSummary = Raft.JobEvents.RunSummary option -> Async +type ReportRunSummary = (string option) * (Raft.JobEvents.RunSummary option) -> Async let inline (++) (path1: string) (path2 : string) = IO.Path.Join(path1, path2) let processRunSummary workingDirectory runStartTime = async { match! RESTlerInternal.runResultsAnalyzer workingDirectory runStartTime with | None -> - return None - | Some runSummaryPath -> + return None, None + | Some (experiment, runSummaryPath) -> let summary = - match runSummaryPath with - | None -> + match Json.Compact.tryDeserializeFile runSummaryPath with + | Choice1Of2 (runSummary: Raft.JobEvents.RunSummary) -> + runSummary + | Choice2Of2 err -> + eprintfn "Failed to process run summary: %s" err Raft.JobEvents.RunSummary.Empty - | Some path -> - match Json.Compact.tryDeserializeFile path with - | Choice1Of2 (runSummary: Raft.JobEvents.RunSummary) -> - runSummary - | Choice2Of2 err -> - eprintfn "Failed to process run summary: %s" err - Raft.JobEvents.RunSummary.Empty - return Some summary + return Some experiment, Some summary } let resultAnalyzer workingDirectory (token: Threading.CancellationToken) (report: ReportRunSummary) (runStartTime: DateTime, reportInterval: TimeSpan option) = @@ -418,7 +413,7 @@ let resultAnalyzer workingDirectory (token: Threading.CancellationToken) (report analyze() let isBugFile (file: IO.FileInfo) = - file.Name <> "bug_buckets.txt" + file.Name <> "bug_buckets.txt" && file.Name <> "bug_buckets.json" let loadTestRunSummary workingDirectory runStartTime = match RESTlerInternal.getRunExperimentFolder workingDirectory runStartTime with @@ -432,27 +427,32 @@ let loadTestRunSummary workingDirectory runStartTime = | None -> None +let getListOfBugsFromBugBuckets bugBuckets = + async { + if IO.Directory.Exists bugBuckets then + let path = bugBuckets ++ "bug_buckets.json" + if IO.File.Exists path then + let bugHashes: RESTlerTypes.Logs.BugHashes = Json.Compact.Strict.deserializeFile path + return Some bugHashes + else + return Some Map.empty + else + return None + } let getListOfBugs workingDirectory (runStartTime: DateTime) = async { match RESTlerInternal.getRunExperimentFolder workingDirectory runStartTime with - | None -> return Seq.empty - + | None -> + return None | Some experiment -> - let bugBuckets = IO.DirectoryInfo(experiment.FullName ++ "bug_buckets") - if bugBuckets.Exists then - let bugFiles = - bugBuckets.EnumerateFiles("*.txt") - |> Seq.filter isBugFile - |> Seq.map (fun f -> f.Name) - return bugFiles - else - return Seq.empty + return! getListOfBugsFromBugBuckets (experiment.FullName ++ "bug_buckets") } + let bugFoundPollInterval = TimeSpan.FromSeconds (10.0) type OnBugFound = Map -> Async -let pollForBugFound workingDirectory (token: Threading.CancellationToken) (runStartTime: DateTime) (onBugFound : OnBugFound) = +let pollForBugFound workingDirectory (token: Threading.CancellationToken) (runStartTime: DateTime) (ignoreBugHashes: string Set) (onBugFound : OnBugFound) = let rec poll() = async { if token.IsCancellationRequested then @@ -467,29 +467,29 @@ let pollForBugFound workingDirectory (token: Threading.CancellationToken) (runSt let restlerExperimentLogs = experiment.FullName ++ "logs" if IO.Directory.Exists restlerExperimentLogs then - let bugsFoundPosted = restlerExperimentLogs ++ "raft-bugsfound.posted.txt" - let! postedBugs = - async { - if IO.File.Exists bugsFoundPosted then - let! bugsPosted = IO.File.ReadAllLinesAsync(bugsFoundPosted) |> Async.AwaitTask - return Set.ofArray bugsPosted - else - return Set.empty - } - - let! bugFiles = getListOfBugs workingDirectory runStartTime - let! _ = - bugFiles - |> Seq.map (fun bugFile -> + match! getListOfBugs workingDirectory runStartTime with + | None -> () + | Some bugFiles -> + let bugsFoundPosted = restlerExperimentLogs ++ "raft-bugsfound.posted.txt" + let! postedBugs = async { - if not <| postedBugs.Contains bugFile then - printfn "Posting bug found %s" bugFile - do! onBugFound (Map.empty.Add("Experiment", experiment.Name).Add("BugBucket", bugFile)) - return () + if IO.File.Exists bugsFoundPosted then + let! bugsPosted = IO.File.ReadAllLinesAsync(bugsFoundPosted) |> Async.AwaitTask + return Set.ofArray bugsPosted + else + return ignoreBugHashes } - ) |> Async.Sequential - - do! IO.File.WriteAllLinesAsync(bugsFoundPosted, bugFiles) |> Async.AwaitTask + let! updatedBugsPosted = + bugFiles + |> Seq.map (fun (KeyValue(bugHash, bugFile)) -> + async { + if not <| postedBugs.Contains bugHash then + printfn "Posting bug found %s with hash %s" bugFile.file_path bugHash + do! onBugFound (Map.empty.Add("Experiment", experiment.Name).Add("BugBucket", bugFile.file_path).Add("BugHash", bugHash)) + return bugHash + } + ) |> Async.Sequential + do! IO.File.WriteAllLinesAsync(bugsFoundPosted, updatedBugsPosted) |> Async.AwaitTask return! poll() } @@ -506,6 +506,7 @@ let replay restlerRootDirectory workingDirectory replayLogFile (parameters: Raft let test (testType: string) restlerRootDirectory workingDirectory (parameters: Raft.RESTlerTypes.Engine.EngineParameters) + (ignoreBugHashes: string Set) (onBugFound: OnBugFound) (report: ReportRunSummary)(runStartTime: DateTime, reportInterval: TimeSpan option) = async { @@ -515,11 +516,8 @@ let test (testType: string) do! RESTlerInternal.test testType restlerRootDirectory workingDirectory parameters token.Cancel() } - resultAnalyzer workingDirectory token.Token report (runStartTime, reportInterval) - - pollForBugFound workingDirectory token.Token runStartTime onBugFound - + pollForBugFound workingDirectory token.Token runStartTime ignoreBugHashes onBugFound ] return () } @@ -528,6 +526,7 @@ let fuzz (fuzzType: string) restlerRootDirectory workingDirectory (parameters: Raft.RESTlerTypes.Engine.EngineParameters) + (ignoreBugHashes: string Set) (onBugFound : OnBugFound) (report: ReportRunSummary) (runStartTime: DateTime, reportInterval: TimeSpan option) = @@ -538,10 +537,8 @@ let fuzz (fuzzType: string) do! RESTlerInternal.fuzz fuzzType restlerRootDirectory workingDirectory parameters token.Cancel() } - resultAnalyzer workingDirectory token.Token report (runStartTime, reportInterval) - - pollForBugFound workingDirectory token.Token runStartTime onBugFound + pollForBugFound workingDirectory token.Token runStartTime ignoreBugHashes onBugFound ] return () } diff --git a/src/Agent/RESTlerAgent/RESTlerTypes.fs b/src/Agent/RESTlerAgent/RESTlerTypes.fs index a5b15e07..b89992ae 100644 --- a/src/Agent/RESTlerAgent/RESTlerTypes.fs +++ b/src/Agent/RESTlerAgent/RESTlerTypes.fs @@ -317,3 +317,10 @@ module Logs = total_requests_sent : IDictionary bug_buckets : IDictionary } + + type BugPath = + { + file_path : string + } + type BugHashes = Map + diff --git a/src/Agent/RESTlerAgent/Telemetry.fs b/src/Agent/RESTlerAgent/Telemetry.fs index 4a114d7f..23726a19 100644 --- a/src/Agent/RESTlerAgent/Telemetry.fs +++ b/src/Agent/RESTlerAgent/Telemetry.fs @@ -34,7 +34,7 @@ type TelemetryClient(machineId: string, instrumentationKey: string) = bugBucketCounts) = client.TrackEvent("restler finished", dict ([ - "machineId", sprintf "%A" machineId + "machineId", sprintf "%s" machineId "version", version "task", task "executionId", sprintf "%A" executionId @@ -44,7 +44,7 @@ type TelemetryClient(machineId: string, instrumentationKey: string) = member __.ResultsAnalyzerFinished(version, task, executionId, status) = client.TrackEvent("results analyzer finished", dict ([ - "machineId", sprintf "%A" machineId + "machineId", sprintf "%s" machineId "version", version "task", task "executionId", sprintf "%A" executionId