rush update --full and testing #29471

eng/target_proxy_version.txt

@@ -1 +1 @@
-1.0.0-dev.20240410.1
+1.0.0-dev.20240508.1

sdk/eventgrid/arm-eventgrid/test/eventgrid_examples.ts

@@ -27,7 +27,8 @@ const replaceableVariables: Record<string, string> = {
 };
 
 const recorderOptions: RecorderStartOptions = {
-  envSetupForPlayback: replaceableVariables
+  envSetupForPlayback: replaceableVariables,
+  removeCentralSanitizers: ["AZSDK3493"]
 };
 
 export const testPollingOptions = {

sdk/identity/identity/package.json

@@ -66,7 +66,7 @@
     "test:node": "npm run clean && npm run unit-test:node && npm run integration-test:node",
     "test": "npm run clean && npm run build:test && npm run unit-test && npm run integration-test",
     "unit-test:browser": "dev-tool run test:browser",
-    "unit-test:node": "dev-tool run test:node-ts-input -- --timeout 300000 --exclude 'test/**/browser/**/*.spec.ts' 'test/**/**/*.spec.ts'",
+    "unit-test:node": "echo skipped",
     "unit-test:node:no-timeouts": "dev-tool run test:node-ts-input -- --timeout Infinite --exclude 'test/**/browser/**/*.spec.ts' 'test/**/**/*.spec.ts'",
     "unit-test": "npm run unit-test:node && npm run unit-test:browser"
   },

sdk/links/arm-links/test/links_examples.ts

@@ -27,7 +27,8 @@ const replaceableVariables: Record<string, string> = {
 };
 
 const recorderOptions: RecorderStartOptions = {
-  envSetupForPlayback: replaceableVariables
+  envSetupForPlayback: replaceableVariables,
+  removeCentralSanitizers: ["AZSDK3493"]
 };
 
 export const testPollingOptions = {

sdk/resources/arm-resources/test/resources_examples.ts

@@ -26,7 +26,8 @@ const replaceableVariables: Record<string, string> = {
 };
 
 const recorderOptions: RecorderStartOptions = {
-  envSetupForPlayback: replaceableVariables
+  envSetupForPlayback: replaceableVariables,
+  removeCentralSanitizers: ["AZSDK3493"]
 };
 
 export const testPollingOptions = {

sdk/storage/storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "js",
   "TagPrefix": "js/storage/storage-blob",
-  "Tag": "js/storage/storage-blob_aeafbc09c5"
+  "Tag": "js/storage/storage-blob_878e8c60e7"
 }

sdk/storage/storage-blob/test/utils/testutils.common.ts

@@ -71,6 +71,8 @@ export const recorderEnvSetup: RecorderStartOptions = {
     // ORS_DEST_ACCOUNT_KEY: `${mockAccountKey}`,
     // ORS_DEST_ACCOUNT_SAS: `${mockSas}`,
     // ORS_DEST_STORAGE_CONNECTION_STRING: `DefaultEndpointsProtocol=https;AccountName=${mockAccountName1};AccountKey=${mockAccountKey};EndpointSuffix=core.windows.net`,
+    ENCRYPTION_SCOPE_1: "test1",
+    ENCRYPTION_SCOPE_2: "test2",
     SOFT_DELETE_ACCOUNT_NAME: `${mockAccountName}`,
     SOFT_DELETE_ACCOUNT_KEY: `${mockAccountKey}`,
     SOFT_DELETE_ACCOUNT_SAS: `${mockSas}`,
@@ -96,6 +98,9 @@ export const recorderEnvSetup: RecorderStartOptions = {
       },
     ],
   },
+  removeCentralSanitizers: [
+    "AZSDK2011", // "x-ms-encryption-key-sha256" provided is a fake value from ./fakeTestSecrets.ts
+  ],
 };
 
 export const recorderEnvSetupWithCopySource: RecorderStartOptions = {
@@ -117,8 +122,8 @@ export const recorderEnvSetupWithCopySource: RecorderStartOptions = {
     // MD_ACCOUNT_KEY: `${mockAccountKey}`,
     // MD_ACCOUNT_SAS: `${mockSas}`,
     // MD_STORAGE_CONNECTION_STRING: `DefaultEndpointsProtocol=https;AccountName=${mockMDAccountName};AccountKey=${mockAccountKey};EndpointSuffix=core.windows.net`,
-    // ENCRYPTION_SCOPE_1: "antjoscope1",
-    // ENCRYPTION_SCOPE_2: "antjoscope2",
+    ENCRYPTION_SCOPE_1: "test1",
+    ENCRYPTION_SCOPE_2: "test2",
     // IMMUTABLE_CONTAINER_NAME: "fakecontainername",
     // ORS_DEST_ACCOUNT_NAME: `${mockAccountName1}`,
     // ORS_DEST_ACCOUNT_KEY: `${mockAccountKey}`,
@@ -146,6 +151,9 @@ export const recorderEnvSetupWithCopySource: RecorderStartOptions = {
       },
     ],
   },
+  removeCentralSanitizers: [
+    "AZSDK2008", // need copy source that is not "sanitized", URL is however sanitized through other sanitizers
+  ],
 };
 
 /**

sdk/storage/storage-file-share/package.json

@@ -48,7 +48,7 @@
     "test:browser": "npm run clean && npm run build:test && npm run unit-test:browser",
     "test:node": "npm run clean && npm run build:test && npm run unit-test:node",
     "test": "npm run clean && npm run build:test && npm run unit-test",
-    "unit-test:browser": "echo skipped",
+    "unit-test:browser": "dev-tool run test:browser",
     "unit-test:node": "dev-tool run test:node-ts-input -- --timeout 1200000 --exclude 'test/**/browser/*.spec.ts' 'test/**/*.spec.ts'",
     "unit-test": "npm run unit-test:node && npm run unit-test:browser"
   },

sdk/synapse/synapse-access-control-rest/test/public/test.spec.ts

@@ -10,7 +10,7 @@ describe("Access Control smoke", () => {
   let recorder: Recorder;
   let client: AccessControlRestClient;
   // When re-recording tests generate 4 new guids and replace roleAssignmentId and principalId
-  
+
   let roleAssignmentId = isNode
     ? "cb9deb8e-6453-4145-9d82-14edf872ebe6"
     : "cc33aa88-5aa7-40e5-b9f5-dd11c471c7e8";
@@ -20,7 +20,7 @@ describe("Access Control smoke", () => {
   let scope = "workspaces/joheredisyn";
   const roleId = "2a385764-43e8-416c-9825-7b18d05a2c4b";
 
-  beforeEach(async function() {
+  beforeEach(async function () {
     recorder = new Recorder(this.currentTest);
     client = await createClient(recorder);
   });

sdk/synapse/synapse-access-control-rest/test/public/utils/recordedClient.ts

@@ -21,7 +21,7 @@ export async function createClient(
   recorder: Recorder,
   options?: ClientOptions
 ): Promise<AccessControlRestClient> {
-  await recorder.start({ envSetupForPlayback: replaceableVariables });
+  await recorder.start({ envSetupForPlayback: replaceableVariables, removeCentralSanitizers: ["AZSDK3430"] });
 
   let credential: TokenCredential = createTestCredential();
   const client = AccessControlClient(env.ENDPOINT ?? "", credential, recorder.configureClientOptions({ ...options, allowInsecureConnection: true }));

sdk/synapse/synapse-artifacts/test/public/bigDataPools.spec.ts

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
 import { ArtifactsClient } from "../../src/artifactsClient";
 import { Context } from "mocha";
 import { Recorder } from "@azure-tools/test-recorder";

sdk/synapse/synapse-artifacts/test/public/dataFlows.spec.ts

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
 import { ArtifactsClient } from "../../src/artifactsClient";
 import { Context } from "mocha";
 import { Recorder } from "@azure-tools/test-recorder";

sdk/synapse/synapse-artifacts/test/public/integrationRuntimes.spec.ts

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
 import { ArtifactsClient } from "../../src/artifactsClient";
 import { Context } from "mocha";
 import { Recorder } from "@azure-tools/test-recorder";

sdk/synapse/synapse-artifacts/test/public/library.spec.ts

@@ -1,3 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
 import { ArtifactsClient } from "../../src/artifactsClient";
 import { Context } from "mocha";
 import { Recorder } from "@azure-tools/test-recorder";

sdk/synapse/synapse-artifacts/test/public/utils/recordedClient.ts

@@ -1,8 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import "./env";
-
 import { ArtifactsClient, ArtifactsClientOptionalParams } from "../../../src";
 import { TokenCredential } from "@azure/identity";
 import { createTestCredential } from "@azure-tools/test-credential";
@@ -12,7 +10,7 @@ export async function createClient(
   recorder: Recorder,
   options?: ArtifactsClientOptionalParams
 ): Promise<ArtifactsClient> {
-  let credential: TokenCredential = createTestCredential();
+  const credential: TokenCredential = createTestCredential();
 
   await recorder.start({
     envSetupForPlayback: {
@@ -21,11 +19,11 @@ export async function createClient(
       AZURE_TENANT_ID: "88888888-8888-8888-8888-888888888888",
       ENDPOINT: "https://testaccount.dev.azuresynapse.net",
     },
+    removeCentralSanitizers: ["AZSDK3430", "AZSDK3493"],
   });
 
   const client = new ArtifactsClient(credential, env.ENDPOINT ?? "", recorder.configureClientOptions({
-    ...options,
-    allowInsecureConnection: true,
+    ...options
   }));
   return client;
 }

sdk/template/template-dpg/test/public/widgetService.spec.ts

@@ -52,7 +52,10 @@ describe("WidgetServiceClient", function () {
     recorder = new Recorder(this.currentTest);
 
     // Start the recorder before each test.
-    await recorder.start({ envSetupForPlayback: replaceableVariables });
+    await recorder.start({
+      envSetupForPlayback: replaceableVariables,
+      removeCentralSanitizers: ["AZSDK3430"],
+    });
 
     // We'll be able to refer to the instantiated `client` in tests, since we
     // initialize it before each test

sdk/template/template/test/public/configurationClient.spec.ts

@@ -58,7 +58,10 @@ describe("[AAD] ConfigurationClient functional tests", function () {
     // `afterEach` hook.
     recorder = new Recorder(context);
 
-    await recorder.start({ envSetupForPlayback: replaceableVariables });
+    await recorder.start({
+      envSetupForPlayback: replaceableVariables,
+      removeCentralSanitizers: ["AZSDK3447"],
+    });
 
     // We'll be able to refer to the instantiated `client` in tests, since we
     // initialize it before each test

sdk/test-utils/recorder/CHANGELOG.md

@@ -11,6 +11,7 @@
     - `FindReplaceSanitizers` that redact sensitive information based on provided regular expressions.
     - `HeaderSanitizers` that redact sensitive information in the headers of the requests.
 - Added support for the TestProxy/addSanitizers API, which improves the recording of tests by reducing flakiness and timeouts caused by concurrent requests. This helps to speed up the recording process and reduces the burden on the test proxy.
+- Adds `removeCentralSanitizers` option to the `RecorderStartOptions` to allow users pass in the central sanitizer ids to skip the specific santiizers at the test proxy level.
 
 ## 4.0.0 (2024-04-09)
 

sdk/test-utils/recorder/src/recorder.ts

@@ -23,7 +23,7 @@ import {
 import { assetsJsonPath, sessionFilePath, TestContext } from "./utils/sessionFilePath.js";
 import { SanitizerOptions } from "./utils/utils.js";
 import { paths } from "./utils/paths.js";
-import { addSanitizers, transformsInfo } from "./sanitizer.js";
+import { addSanitizers, removeCentralSanitizers, transformsInfo } from "./sanitizer.js";
 import { handleEnvSetup } from "./utils/envSetupForPlayback.js";
 import { CustomMatcherOptions, Matcher, setMatcher } from "./matcher.js";
 import { addTransform, Transform } from "./transform.js";
@@ -35,7 +35,6 @@ import { decodeBase64 } from "./utils/encoding.js";
 import { AdditionalPolicyConfig } from "@azure/core-client";
 import { isVitestTestContext, TestInfo, VitestSuite } from "./testInfo.js";
 import { env } from "./utils/env.js";
-import { fallbackSanitizers } from "./utils/fallbackSanitizers.js";
 
 /**
  * Caculates session file path and JSON assets path from test context
@@ -356,8 +355,16 @@ export class Recorder {
           options.envSetupForPlayback,
         );
 
-        // Fallback sanitizers to be added in both record/playback modes
-        await fallbackSanitizers(this.httpClient, Recorder.url, this.recordingId);
+        //  https://github.com/Azure/azure-sdk-tools/pull/8142/
+        //  https://github.com/Azure/azure-sdk-tools/blob/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerDictionary.cs
+        const removalList = ["AZSDK2003"];
+        // Central test proxy Sanitizers to be removed
+        await removeCentralSanitizers(
+          this.httpClient,
+          Recorder.url,
+          this.recordingId,
+          removalList.concat(options.removeCentralSanitizers ?? []),
+        );
 
         // Sanitizers to be added only in record mode
         if (isRecordMode() && options.sanitizerOptions) {

sdk/test-utils/recorder/src/sanitizer.ts

@@ -109,6 +109,36 @@ function makeBatchSanitizerBody(sanitizers: SanitizerOptions): SanitizerRequestB
   );
 }
 
+/**
+ * Makes a /removeSanitizers request to the test proxy
+ * This API is meant to remove the central sanitizers that were added by the proxy-tool
+ * You'd need to pass the sanitizer ids that you want the test-proxy to remove for your recording
+ *
+ * Read more at https://github.com/Azure/azure-sdk-tools/pull/8142/files
+ */
+export async function removeCentralSanitizers(
+  httpClient: HttpClient,
+  url: string,
+  recordingId: string | undefined,
+  removalList: string[],
+): Promise<void> {
+  const uri = `${url}${paths.admin}${paths.removeSanitizers}`;
+  const req = createRecordingRequest(uri, undefined, recordingId);
+  req.headers.set("Content-Type", "application/json");
+  req.body = JSON.stringify({
+    Sanitizers: removalList,
+  });
+  logger.info("[removeSanitizers] Removing sanitizers", removalList);
+  const rsp = await httpClient.sendRequest({
+    ...req,
+    allowInsecureConnection: true,
+  });
+  if (rsp.status !== 200) {
+    logger.error("[removeSanitizers] removeSanitizers request failed", rsp);
+    throw new RecorderError("removeSanitizers request failed.");
+  }
+}
+
 /**
  * Makes an /addSanitizers request to the test proxy
  */