Merge branch 'main' into Release-8.5.0

.github/workflows/ci.yml

@@ -132,7 +132,7 @@ jobs:
           python-version: 3.7
       - uses: actions/download-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-onefuzztypes
           path: artifacts
       - name: Build
         shell: bash
@@ -157,7 +157,7 @@ jobs:
           cp dist/onefuzz.exe ${GITHUB_WORKSPACE}/artifacts/windows-cli/
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-cli
           path: artifacts
       - name: lint
         shell: bash
@@ -220,7 +220,7 @@ jobs:
       - run: src/ci/onefuzztypes.sh
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-onefuzztypes
           path: artifacts
   proxy:
     runs-on: ubuntu-20.04
@@ -239,7 +239,7 @@ jobs:
       - run: src/ci/proxy.sh
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-proxy
           path: artifacts
   service:
     runs-on: ubuntu-22.04
@@ -314,19 +314,31 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        id: cache-build
+        with:
+          key: afl|${{runner.os}}-${{runner.arch}}|${{ hashFiles('src/ci/afl.sh') }}
+          path: artifacts
       - run: src/ci/afl.sh
+        if: steps.cache-build.outputs.cache-hit != 'true'
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-afl
           path: artifacts
   aflpp:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        id: cache-build
+        with:
+          key: aflpp|${{runner.os}}-${{runner.arch}}|${{ hashFiles('src/ci/aflpp.sh') }}
+          path: artifacts
       - run: src/ci/aflpp.sh
+        if: steps.cache-build.outputs.cache-hit != 'true'
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-aflpp
           path: artifacts
   bicep-check:
     name: Check Bicep files
@@ -352,7 +364,7 @@ jobs:
         shell: bash
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-dotnet-fuzzing-tools-linux
           path: artifacts
   dotnet-fuzzing-tools-windows:
     runs-on: windows-2022
@@ -367,7 +379,7 @@ jobs:
         shell: pwsh
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-dotnet-fuzzing-tools-windows
           path: artifacts
   radamsa-linux:
     runs-on: ubuntu-20.04
@@ -378,13 +390,13 @@ jobs:
         with:
           # key on the shell script only: this script fixes the
           # version to a particular commit, so if it changes we need to rebuild
-          key: radamsa-linux-${{ hashFiles('src/ci/radamsa-linux.sh') }}
+          key: radamsa|${{runner.os}}-${{runner.arch}}|${{ hashFiles('src/ci/radamsa-linux.sh') }}
           path: artifacts
       - run: src/ci/radamsa-linux.sh
         if: steps.cache-radamsa-build-linux.outputs.cache-hit != 'true'
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-radamsa-linux
           path: artifacts
   radamsa-win64:
     runs-on: windows-2019
@@ -395,24 +407,25 @@ jobs:
         with:
           # key on the shell script only: this script fixes the
           # version to a particular commit, so if it changes we need to rebuild
-          key: radamsa-windows-${{ hashFiles('src/ci/radamsa-windows.sh') }}
+          key: radamsa|${{runner.os}}-${{runner.arch}}|${{ hashFiles('src/ci/radamsa-windows.sh') }}
           path: artifacts
       - run: c:\msys64\usr\bin\bash src/ci/radamsa-windows.sh
         if: steps.cache-radamsa-build-windows.outputs.cache-hit != 'true'
       - uses: actions/upload-artifact@v3
         with:
-          name: build-artifacts
+          name: artifact-radamsa-windows
           path: artifacts
   package:
     needs:
       - agent
       - azcopy
       - cli
-      - onefuzztypes
       - proxy
       - service
       - afl
       - aflpp
+      - dotnet-fuzzing-tools-linux
+      - dotnet-fuzzing-tools-windows
       - radamsa-linux
       - radamsa-win64
     runs-on: ubuntu-20.04
@@ -422,6 +435,38 @@ jobs:
         with:
           name: build-artifacts
           path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-cli
+          path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-proxy
+          path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-radamsa-linux
+          path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-radamsa-windows
+          path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-afl
+          path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-aflpp
+          path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-dotnet-fuzzing-tools-linux
+          path: artifacts
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact-dotnet-fuzzing-tools-windows
+          path: artifacts
       - uses: actions/setup-python@v4
         with:
           python-version: 3.7
@@ -436,6 +481,7 @@ jobs:
           isort --profile black . --check
           black . --check
           flake8 *.py
+          rm -r .mypy_cache
       - name: Package Onefuzz
         run: |
           set -ex
@@ -540,7 +586,8 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: integration-test-artifacts
+          # NB: this name is used by check-pr.py
+          name: artifact-integration-tests-linux
           path: src/integration-tests/artifacts
   build-integration-tests-windows:
     runs-on: windows-2019
@@ -623,52 +670,39 @@ jobs:
         shell: powershell
       - uses: actions/upload-artifact@v3
         with:
-          name: integration-test-artifacts
+          # NB: this name is used by check-pr.py
+          name: artifact-integration-tests-windows
           path: src/integration-tests/artifacts
   integration-tests-linux:
     runs-on: ubuntu-20.04
     needs:
       - build-integration-tests-linux
       - dotnet-fuzzing-tools-linux
-      # even though this job doesn't use the artifacts for these other jobs,
-      # we must include them or we get spurious failures when the download-artifact
-      # step tries to download the named artifact which includes files from
-      # all of these jobs
-      - agent
-      - azcopy
-      - cli
-      - onefuzztypes
-      - proxy
-      - service
-      - afl
-      - aflpp
-      - radamsa-linux
-      - radamsa-win64
     steps:
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
-          name: build-artifacts
-          path: build-artifacts
+          name: artifact-dotnet-fuzzing-tools-linux
+          path: dotnet-fuzzing-tools-linux
       - uses: actions/download-artifact@v3
         with:
-          name: integration-test-artifacts
-          path: integration-test-artifacts
+          name: artifact-integration-tests-linux
+          path: integration-tests-linux
       - name: test
         shell: bash
         run: |
           set -ex -o pipefail
 
           # Must be absolute paths.
-          export GOODBAD_DOTNET="${GITHUB_WORKSPACE}/integration-test-artifacts/GoodBadDotnet"
+          export GOODBAD_DOTNET="${GITHUB_WORKSPACE}/integration-tests-linux/GoodBadDotnet"
 
-          export LIBFUZZER_DOTNET="${GITHUB_WORKSPACE}/build-artifacts/third-party/dotnet-fuzzing-linux/libfuzzer-dotnet/libfuzzer-dotnet"
+          export LIBFUZZER_DOTNET="${GITHUB_WORKSPACE}/dotnet-fuzzing-tools-linux/third-party/dotnet-fuzzing-linux/libfuzzer-dotnet/libfuzzer-dotnet"
           chmod +x $LIBFUZZER_DOTNET
 
-          export LIBFUZZER_DOTNET_LOADER="${GITHUB_WORKSPACE}/build-artifacts/third-party/dotnet-fuzzing-linux/LibFuzzerDotnetLoader/LibFuzzerDotnetLoader"
+          export LIBFUZZER_DOTNET_LOADER="${GITHUB_WORKSPACE}/dotnet-fuzzing-tools-linux/third-party/dotnet-fuzzing-linux/LibFuzzerDotnetLoader/LibFuzzerDotnetLoader"
           chmod +x $LIBFUZZER_DOTNET_LOADER
 
-          export SHARPFUZZ="${GITHUB_WORKSPACE}/build-artifacts/third-party/dotnet-fuzzing-linux/sharpfuzz/SharpFuzz.CommandLine"
+          export SHARPFUZZ="${GITHUB_WORKSPACE}/dotnet-fuzzing-tools-linux/third-party/dotnet-fuzzing-linux/sharpfuzz/SharpFuzz.CommandLine"
           chmod +x $SHARPFUZZ
 
           ./src/ci/test-libfuzzer-dotnet.sh

src/ApiService/ApiService/ApiService.csproj

@@ -11,6 +11,7 @@
 
     <PackageReference Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.21.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="1.0.0-preview4" />
+    <PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.24" />
 
     <PackageReference Include="Semver" Version="2.1.0" />
     <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.3.0" />

src/ApiService/ApiService/Functions/Jobs.cs

@@ -46,14 +46,16 @@ public class Jobs {
         var job = new Job(
             JobId: Guid.NewGuid(),
             State: JobState.Init,
-            Config: cfg) {
-            UserInfo = userInfo.UserInfo,
-        };
+            Config: cfg,
+            UserInfo: new(
+                ObjectId: userInfo.UserInfo.ObjectId,
+                ApplicationId: userInfo.UserInfo.ApplicationId));
 
         // create the job logs container
         var metadata = new Dictionary<string, string>{
             { "container_type", "logs" }, // TODO: use ContainerType.Logs enum somehow; needs snake case name
         };
+
         var containerName = Container.Parse($"logs-{job.JobId}");
         var containerSas = await _context.Containers.CreateContainer(containerName, StorageType.Corpus, metadata);
         if (containerSas is null) {
@@ -79,9 +81,9 @@ public class Jobs {
                 ),
                 "job");
         }
-        await _context.Events.SendEvent(new EventJobCreated(job.JobId, job.Config, job.UserInfo));
 
-        return await RequestHandling.Ok(req, JobResponse.ForJob(job));
+        await _context.Events.SendEvent(new EventJobCreated(job.JobId, job.Config, job.UserInfo));
+        return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo: null));
     }
 
     private async Task<HttpResponseData> Delete(HttpRequestData req) {
@@ -111,7 +113,7 @@ public class Jobs {
             }
         }
 
-        return await RequestHandling.Ok(req, JobResponse.ForJob(job));
+        return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo: null));
     }
 
     private async Task<HttpResponseData> Get(HttpRequestData req) {
@@ -135,11 +137,10 @@ public class Jobs {
             // TODO: search.WithTasks is not checked in Python code?
 
             var taskInfo = await _context.TaskOperations.SearchStates(jobId).Select(TaskToJobTaskInfo).ToListAsync();
-            job = job with { TaskInfo = taskInfo };
-            return await RequestHandling.Ok(req, JobResponse.ForJob(job));
+            return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo));
         }
 
         var jobs = await _context.JobOperations.SearchState(states: search.State ?? Enumerable.Empty<JobState>()).ToListAsync();
-        return await RequestHandling.Ok(req, jobs.Select(j => JobResponse.ForJob(j)));
+        return await RequestHandling.Ok(req, jobs.Select(j => JobResponse.ForJob(j, taskInfo: null)));
     }
 }

src/ApiService/ApiService/Functions/Node.cs

@@ -59,18 +59,18 @@ public class Node {
                 _context.NodeMessageOperations.GetMessages(machineId).ToListAsync().AsTask());
 
             var commands = messages.Select(m => m.Message).ToList();
-            return await RequestHandling.Ok(req, NodeToNodeSearchResult(node with { Tasks = tasks, Messages = commands }));
+            return await RequestHandling.Ok(req, NodeToNodeSearchResult(node, tasks, commands));
         }
 
         var nodes = await _context.NodeOperations.SearchStates(
             states: search.State,
             poolName: search.PoolName,
             scalesetId: search.ScalesetId).ToListAsync();
 
-        return await RequestHandling.Ok(req, nodes.Select(NodeToNodeSearchResult));
+        return await RequestHandling.Ok(req, nodes.Select(x => NodeToNodeSearchResult(x, null, null)));
     }
 
-    private static NodeSearchResult NodeToNodeSearchResult(Service.Node node) {
+    private static NodeSearchResult NodeToNodeSearchResult(Service.Node node, List<NodeTasks>? tasks, List<NodeCommand>? messages) {
         return new NodeSearchResult(
             PoolId: node.PoolId,
             PoolName: node.PoolName,
@@ -82,7 +82,9 @@ public class Node {
             ScalesetId: node.ScalesetId,
             ReimageRequested: node.ReimageRequested,
             DeleteRequested: node.DeleteRequested,
-            DebugKeepNode: node.DebugKeepNode);
+            DebugKeepNode: node.DebugKeepNode,
+            Tasks: tasks,
+            Messages: messages);
     }
 
     private async Async.Task<HttpResponseData> Patch(HttpRequestData req) {

src/ApiService/ApiService/Functions/TimerRetention.cs

@@ -71,45 +71,6 @@ public class TimerRetention {
             }
         }
 
-        await foreach (var job in _jobOps.QueryAsync(Query.And(timeFilter, Query.EqualEnum("state", JobState.Enabled)))) {
-            if (job.UserInfo is not null && job.UserInfo.Upn is not null) {
-                _log.LogInformation("removing PII from job {JobId}", job.JobId);
-                var userInfo = job.UserInfo with { Upn = null };
-                var updatedJob = job with { UserInfo = userInfo };
-                var r = await _jobOps.Replace(updatedJob);
-                if (!r.IsOk) {
-                    _log.AddHttpStatus(r.ErrorV);
-                    _log.LogError("Failed to save job {JobId}", updatedJob.JobId);
-                }
-            }
-        }
-
-        await foreach (var task in _taskOps.QueryAsync(Query.And(timeFilter, Query.EqualEnum("state", TaskState.Stopped)))) {
-            if (task.UserInfo is not null && task.UserInfo.Upn is not null) {
-                _log.LogInformation("removing PII from task {TaskId}", task.TaskId);
-                var userInfo = task.UserInfo with { Upn = null };
-                var updatedTask = task with { UserInfo = userInfo };
-                var r = await _taskOps.Replace(updatedTask);
-                if (!r.IsOk) {
-                    _log.AddHttpStatus(r.ErrorV);
-                    _log.LogError("Failed to save task {TaskId}", updatedTask.TaskId);
-                }
-            }
-        }
-
-        await foreach (var repro in _reproOps.QueryAsync(timeFilter)) {
-            if (repro.UserInfo is not null && repro.UserInfo.Upn is not null) {
-                _log.LogInformation("removing PII from repro: {VmId}", repro.VmId);
-                var userInfo = repro.UserInfo with { Upn = null };
-                var updatedRepro = repro with { UserInfo = userInfo };
-                var r = await _reproOps.Replace(updatedRepro);
-                if (!r.IsOk) {
-                    _log.AddHttpStatus(r.ErrorV);
-                    _log.LogError("Failed to save repro {VmId}", updatedRepro.VmId);
-                }
-            }
-        }
-
         //delete Task queues for tasks that do not exist in the table (manually deleted from the table)
         //delete Pool queues for pools that were deleted before https://github.com/microsoft/onefuzz/issues/2430 got fixed
         await foreach (var q in _queue.ListQueues(StorageType.Corpus)) {