feat: added healing for sqs xml

.structignore

@@ -1 +1,2 @@
 test/util/cache-clearer-disabled.spec.js
+test/modules/response-healing.spec.js

package.json

@@ -86,6 +86,7 @@
     "object-scan": "19.0.5",
     "smart-fs": "4.0.1",
     "timekeeper": "2.3.1",
-    "tmp": "0.2.1"
+    "tmp": "0.2.1",
+    "xml2js": "0.6.2"
   }
 }

src/modules/request-recorder.js

@@ -11,7 +11,7 @@ import nockCommon from 'nock/lib/common.js';
 import compareUrls from '../util/compare-urls.js';
 import nockListener from './request-recorder/nock-listener.js';
 import nockMock from './request-recorder/nock-mock.js';
-import healSqsSendMessageBatch from './request-recorder/heal-sqs-send-message-batch.js';
+import healSqs from './request-recorder/heal-sqs.js';
 import applyModifiers from './request-recorder/apply-modifiers.js';
 import requestInjector from './request-recorder/request-injector.js';
 
@@ -223,7 +223,7 @@ export default (opts) => {
 
               if (anyFlagPresent(['magic', 'response'])) {
                 const responseBody = tryParseJson([
-                  healSqsSendMessageBatch
+                  healSqs
                 ].reduce(
                   (respBody, fn) => fn(requestBodyString, respBody, scope, req),
                   interceptor.body

src/modules/request-recorder/heal-sqs.js

@@ -0,0 +1,30 @@
+import crypto from 'crypto';
+import { tryParseJson } from './util.js';
+import migration from './heal-sqs/migration.js';
+
+export default (requestBody, responseBody, scope, req) => {
+  if (scope?.basePath !== 'https://sqs.us-west-2.amazonaws.com:443') {
+    return responseBody;
+  }
+
+  const header = req?.options?.headers?.['x-amz-target'];
+
+  if (typeof responseBody === 'string' && responseBody.startsWith('<?xml')) {
+    return migration({ responseBody, header });
+  }
+
+  const requestJson = tryParseJson(requestBody);
+  const responseJson = tryParseJson(responseBody);
+
+  if (header === 'AmazonSQS.SendMessageBatch') {
+    return {
+      Successful: requestJson.Entries.map(({ Id, MessageBody }, idx) => ({
+        Id,
+        MessageId: responseJson?.Successful?.[idx]?.MessageId || crypto.randomUUID(),
+        MD5OfMessageBody: crypto.createHash('md5').update(MessageBody).digest('hex')
+      }))
+    };
+  }
+
+  return responseBody;
+};

src/modules/request-recorder/heal-sqs/migration.js

@@ -0,0 +1,103 @@
+// This code logic is used to migrate legacy AWS SQS xml to json
+import xml2js from 'xml2js';
+import objectScan from 'object-scan';
+
+const tryParseXML = (body) => {
+  let parsed = body;
+  try {
+    xml2js.parseString(body, (err, result) => {
+      parsed = JSON.parse(JSON.stringify(result));
+    });
+  } catch (e) { /* ignored */
+  }
+  return parsed;
+};
+
+export default ({ responseBody, header }) => {
+  const responseXml = tryParseXML(responseBody);
+  if (header === 'AmazonSQS.ListQueueTags') {
+    const scanner = objectScan(['ListQueueTagsResponse.ListQueueTagsResult[0].Tag[*]'], {
+      rtn: ({ value }) => [value.Key, value.Value[0]],
+      afterFn: ({ result }) => Object.fromEntries(result)
+    });
+    const Tags = scanner(responseXml);
+    return { Tags };
+  }
+  if (header === 'AmazonSQS.GetQueueAttributes') {
+    const scanner = objectScan(['GetQueueAttributesResponse.GetQueueAttributesResult[0].Attribute[*]'], {
+      rtn: ({ value }) => [value.Name[0], value.Value[0]],
+      afterFn: ({ result }) => Object.fromEntries(result)
+    });
+    const Attributes = scanner(responseXml);
+    return { Attributes };
+  }
+  if (header === 'AmazonSQS.GetQueueUrl') {
+    if (responseXml?.ErrorResponse?.Error?.[0]?.Code?.[0] === 'AWS.SimpleQueueService.NonExistentQueue') {
+      return {
+        __type: 'com.amazonaws.sqs#QueueDoesNotExist',
+        message: 'The specified queue does not exist.'
+      };
+    }
+    const QueueUrl = responseXml?.GetQueueUrlResponse?.GetQueueUrlResult?.[0]?.QueueUrl?.[0];
+    return { QueueUrl };
+  }
+  if (header === 'AmazonSQS.CreateQueue') {
+    if (responseXml?.ErrorResponse?.Error?.[0]?.Code?.[0] === 'QueueAlreadyExists') {
+      return {
+        __type: 'com.amazonaws.sqs#QueueNameExists',
+        message: 'The specified queue name does exist.'
+      };
+    }
+    const QueueUrl = responseXml?.CreateQueueResponse?.CreateQueueResult?.[0]?.QueueUrl?.[0];
+    return { QueueUrl };
+  }
+  if (header === 'AmazonSQS.ListQueues') {
+    const scannerQueueUrls = objectScan(
+      ['ListQueuesResponse.ListQueuesResult[0].QueueUrl[*]'],
+      { rtn: 'value', reverse: false }
+    );
+    const scannerNextToken = objectScan(
+      ['ListQueuesResponse.ListQueuesResult[0].NextToken[0]'],
+      { rtn: 'value', reverse: false, abort: true }
+    );
+    return {
+      QueueUrls: scannerQueueUrls(responseXml),
+      NextToken: scannerNextToken(responseXml)
+    };
+  }
+  if (header === 'AmazonSQS.TagQueue') {
+    return {};
+  }
+  if (header === 'AmazonSQS.SetQueueAttributes') {
+    return {};
+  }
+  if (header === 'AmazonSQS.SendMessageBatch') {
+    const scannerSuccessful = objectScan(
+      ['SendMessageBatchResponse.SendMessageBatchResult[0].SendMessageBatchResultEntry[*]'],
+      {
+        rtn: ({ value }) => ({
+          Id: value.Id[0],
+          MessageId: value.MessageId[0],
+          MD5OfMessageBody: value.MD5OfMessageBody[0]
+        }),
+        reverse: false
+      }
+    );
+    const scannerFailed = objectScan(
+      ['SendMessageBatchResponse.SendMessageBatchResult[0].BatchResultErrorEntry[*]'],
+      {
+        rtn: ({ value }) => ({
+          Id: value.Id[0],
+          SenderFault: value.SenderFault[0],
+          Code: value.Code[0]
+        }),
+        reverse: false
+      }
+    );
+    return {
+      Successful: scannerSuccessful(responseXml),
+      Failed: scannerFailed(responseXml)
+    };
+  }
+  return responseBody;
+};

test/modules/request-recorder.spec.js

@@ -3,26 +3,10 @@ import https from 'https';
 import fs from 'smart-fs';
 import get from 'lodash.get';
 import axios from 'axios';
-import { logger } from 'lambda-monitor-logger';
 import { expect } from 'chai';
-import awsSdkWrap from 'aws-sdk-wrap';
-import {
-  SQSClient,
-  SendMessageBatchCommand
-} from '@aws-sdk/client-sqs';
 import { describe } from '../../src/index.js';
 import { NockRecord, spawnServer } from '../server.js';
 
-const aws = awsSdkWrap({
-  logger,
-  services: {
-    SQS: SQSClient,
-    'SQS:CMD': {
-      SendMessageBatchCommand
-    }
-  }
-});
-
 describe('Testing RequestRecorder', { useTmpDir: true, timestamp: 0 }, () => {
   const cassetteFile = 'file1.json';
   let tmpDir;
@@ -383,20 +367,6 @@ describe('Testing RequestRecorder', { useTmpDir: true, timestamp: 0 }, () => {
       await runner('prune,record', { qs: [1], raises: true });
     });
 
-    describe('Testing magic healing', { cryptoSeed: 'd28095c6-19f4-4dc2-a7cc-f7640c032967' }, () => {
-      it('Testing heal SQS response', async ({ fixture }) => {
-        fs.smartWrite(path.join(tmpDir, cassetteFile), fixture('sqs-cassette-bad'));
-        const r = await nockRecord(() => aws.sqs.sendMessageBatch({
-          messages: [{ k: 1 }, { k: 2 }],
-          queueUrl: process.env.QUEUE_URL
-        }), { heal: 'magic' });
-        const expected = fixture('sqs-cassette-expected');
-        expected[0].reqheaders['user-agent'] = r.expectedCassette[0].reqheaders['user-agent'];
-        expect(r.expectedCassette[0].reqheaders['user-agent'].startsWith('aws-sdk-js/3.')).to.equal(true);
-        expect(r.expectedCassette).to.deep.equal(expected);
-      });
-    });
-
     it('Testing record (with headers)', async () => {
       await runner('record', {
         qs: [1, 2, 3],

test/modules/response-healing.spec.js

@@ -0,0 +1,57 @@
+import path from 'path';
+import fs from 'smart-fs';
+import { logger } from 'lambda-monitor-logger';
+import { expect } from 'chai';
+import awsSdkWrap from 'aws-sdk-wrap';
+import { SendMessageBatchCommand, SQSClient } from '@aws-sdk/client-sqs';
+import { describe } from '../../src/index.js';
+import { NockRecord } from '../server.js';
+
+const aws = awsSdkWrap({
+  logger,
+  services: {
+    SQS: SQSClient,
+    'SQS:CMD': {
+      SendMessageBatchCommand
+    }
+  }
+});
+
+describe('Testing Response Healing', {
+  useTmpDir: true,
+  timestamp: 0,
+  cryptoSeed: 'd28095c6-19f4-4dc2-a7cc-f7640c032967'
+}, () => {
+  const cassetteFile = 'file1.json';
+  let tmpDir;
+  let nockRecord;
+
+  beforeEach(async ({ dir }) => {
+    tmpDir = dir;
+    nockRecord = NockRecord(tmpDir, cassetteFile);
+  });
+
+  it('Testing heal bad SQS response', async ({ fixture }) => {
+    fs.smartWrite(path.join(tmpDir, cassetteFile), fixture('sqs-cassette-bad'));
+    const r = await nockRecord(() => aws.sqs.sendMessageBatch({
+      messages: [{ k: 1 }, { k: 2 }],
+      queueUrl: process.env.QUEUE_URL
+    }), { heal: 'magic' });
+    const expected = fixture('sqs-cassette-bad-expected');
+    expected[0].reqheaders['user-agent'] = r.expectedCassette[0].reqheaders['user-agent'];
+    expect(r.expectedCassette[0].reqheaders['user-agent'].startsWith('aws-sdk-js/3.')).to.equal(true);
+    expect(r.expectedCassette).to.deep.equal(expected);
+  });
+
+  it('Testing heal xml SQS response', async ({ fixture }) => {
+    fs.smartWrite(path.join(tmpDir, cassetteFile), fixture('sqs-cassette-xml'));
+    const r = await nockRecord(() => aws.sqs.sendMessageBatch({
+      messages: [{ k: 1 }, { k: 2 }],
+      queueUrl: process.env.QUEUE_URL
+    }), { heal: 'magic' });
+    const expected = fixture('sqs-cassette-xml-expected');
+    expected[0].reqheaders['user-agent'] = r.expectedCassette[0].reqheaders['user-agent'];
+    expect(r.expectedCassette[0].reqheaders['user-agent'].startsWith('aws-sdk-js/3.')).to.equal(true);
+    expect(r.expectedCassette).to.deep.equal(expected);
+  });
+});

test/modules/response-healing.spec.js.env.yml

@@ -0,0 +1,4 @@
+AWS_REGION: "us-west-2"
+AWS_ACCESS_KEY_ID: "XXXXXXXXXXXXXXXXXXXX"
+AWS_SECRET_ACCESS_KEY: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+QUEUE_URL: "https://sqs.us-west-2.amazonaws.com/123456789101/service-name-data-local-SomeQueue"

test/modules/response-healing.spec.js__fixtures/sqs-cassette-xml-expected.json

@@ -0,0 +1,49 @@
+[
+  {
+    "scope": "https://sqs.us-west-2.amazonaws.com:443",
+    "method": "POST",
+    "path": "/",
+    "body": {
+      "Entries": [
+        {
+          "Id": "4f494d6b422d1e5fbbe6b6d82584ce7f70c1530f",
+          "MessageBody": "{\"k\":1}"
+        },
+        {
+          "Id": "743966ceb52ce2e8c7a2424f4d652e3b7936f1ee",
+          "MessageBody": "{\"k\":2}"
+        }
+      ],
+      "QueueUrl": "https://sqs.us-west-2.amazonaws.com/123456789101/service-name-data-local-SomeQueue"
+    },
+    "status": 200,
+    "response": {
+      "Successful": [
+        {
+          "Id": "4f494d6b422d1e5fbbe6b6d82584ce7f70c1530f",
+          "MessageId": "c0004b0c-ea00-498a-a5a2-362703c0b621",
+          "MD5OfMessageBody": "e59138f35192e320088b83d28388d6bb"
+        },
+        {
+          "Id": "743966ceb52ce2e8c7a2424f4d652e3b7936f1ee",
+          "MessageId": "c0004b0c-ea00-498a-a5a2-362703c0b621",
+          "MD5OfMessageBody": "0dc3f48bc3e78aed8f69f72be8a11087"
+        }
+      ],
+      "Failed": []
+    },
+    "reqheaders": {
+      "amz-sdk-invocation-id": "cae6ff7b-efc4-4695-b1ad-cf7b80d80573",
+      "amz-sdk-request": "attempt=1; max=3",
+      "authorization": "AWS4-HMAC-SHA256 Credential=XXXXXXXXXXXXXXXXXXXX/19700101/us-west-2/sqs/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target;x-amz-user-agent, Signature=04d81ab3a1f18a8d0d9e1bf44af30633d6f0cda97c5db36fed60e229d63ccaf6",
+      "content-length": "261",
+      "content-type": "application/x-amz-json-1.0",
+      "host": "sqs.us-west-2.amazonaws.com",
+      "user-agent": "aws-sdk-js/3.458.0 ua/2.0 os/linux#5.4.0-166-generic lang/js md/nodejs#18.18.2 api/sqs#3.458.0",
+      "x-amz-content-sha256": "42d9205526e005eece0404131937f30da55578dcb4a17bdd14751d2706474acf",
+      "x-amz-date": "19700101T000000Z",
+      "x-amz-target": "AmazonSQS.SendMessageBatch",
+      "x-amz-user-agent": "aws-sdk-js/3.458.0"
+    }
+  }
+]

test/modules/response-healing.spec.js__fixtures/sqs-cassette-xml.json

@@ -0,0 +1,22 @@
+[
+  {
+    "scope": "https://sqs.us-west-2.amazonaws.com:443",
+    "method": "POST",
+    "path": "/",
+    "body": "QueueUrl=https%3A%2F%2Fsqs.us-west-2.amazonaws.com%2F123456789101%2Fservice-name-data-local-SomeQueue&SendMessageBatchRequestEntry.1.Id=4f494d6b422d1e5fbbe6b6d82584ce7f70c1530f&SendMessageBatchRequestEntry.1.MessageBody=%7B%22k%22%3A1%7D&SendMessageBatchRequestEntry.2.Id=743966ceb52ce2e8c7a2424f4d652e3b7936f1ee&SendMessageBatchRequestEntry.2.MessageBody=%7B%22k%22%3A2%7D&Action=SendMessageBatch&Version=2012-11-05",
+    "status": 200,
+    "response": "<?xml version=\"1.0\"?><SendMessageBatchResponse xmlns=\"http://queue.amazonaws.com/doc/2012-11-05/\"><SendMessageBatchResult><SendMessageBatchResultEntry><Id>4f494d6b422d1e5fbbe6b6d82584ce7f70c1530f</Id><MessageId>c0004b0c-ea00-498a-a5a2-362703c0b621</MessageId><MD5OfMessageBody>e59138f35192e320088b83d28388d6bb</MD5OfMessageBody></SendMessageBatchResultEntry><SendMessageBatchResultEntry><Id>743966ceb52ce2e8c7a2424f4d652e3b7936f1ee</Id><MessageId>c0004b0c-ea00-498a-a5a2-362703c0b621</MessageId><MD5OfMessageBody>0dc3f48bc3e78aed8f69f72be8a11087</MD5OfMessageBody></SendMessageBatchResultEntry></SendMessageBatchResult><ResponseMetadata><RequestId>584b2818-1c3b-5bfd-97e0-7d388e494f1e</RequestId></ResponseMetadata></SendMessageBatchResponse>",
+    "reqheaders": {
+      "amz-sdk-invocation-id": "e9725510-7b42-4901-b553-969b5a03e7d4",
+      "amz-sdk-request": "attempt=1; max=3",
+      "authorization": "AWS4-HMAC-SHA256 Credential=XXXXXXXXXXXXXXXXXXXX/19700101/us-west-2/sqs/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-user-agent, Signature=46051a63bda908cbd07034a1a035c6fc53cacb9b617264fbf9fd1b6f27a62f5b",
+      "content-length": "416",
+      "content-type": "application/x-www-form-urlencoded",
+      "host": "sqs.us-west-2.amazonaws.com",
+      "user-agent": "aws-sdk-js/3.370.0 ua/2.0 os/linux#5.4.0-153-generic lang/js md/nodejs#16.20.1 api/sqs#3.370.0",
+      "x-amz-content-sha256": "5f6cdab08798db82adcf77e1bdef480fb8fe70e558deeeb78045a5bfcf49cc67",
+      "x-amz-date": "19700101T000000Z",
+      "x-amz-user-agent": "aws-sdk-js/3.385.0"
+    }
+  }
+]

yarn.lock

@@ -4780,7 +4780,7 @@ safe-regex-test@^1.0.0:
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
 
-sax@^1.2.4:
+sax@>=0.6.0, sax@^1.2.4:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0"
   integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==
@@ -5506,6 +5506,19 @@ xml-js@1.6.11:
   dependencies:
     sax "^1.2.4"
 
+xml2js@0.6.2:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
+  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~11.0.0"
+
+xmlbuilder@~11.0.0:
+  version "11.0.1"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
+  integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
+
 xtend@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"