Added support for handling Remote URL refs in the bundle method

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+-   Added support for remote $ref resolution in bundle() API.
+
 ## [v4.15.0] - 2023-06-27
 
 ### Added
@@ -41,7 +45,7 @@
 -   Fixed an issue where definition validation was not considering multiple white space characters.
 -   Fixed issue [#708](https://github.com/postmanlabs/openapi-to-postman/issues/708) where if string is defined for required field, conversion was failing.
 -   Fixed issue where for certain path segments, collection generation failed.
--   Fixed TypeError occurring while checking typeof bodyContent in getXmlVersionContent. 
+-   Fixed TypeError occurring while checking typeof bodyContent in getXmlVersionContent.
 
 ## [v4.12.0] - 2023-05-04
 
@@ -295,7 +299,7 @@ Newer releases follow the [Keep a Changelog](https://keepachangelog.com/en/1.0.0
 -   Added support for internal $ref resolution in validation flows.
 -   Fixed issue where parameter resolution was "schema" when "example" was specified.
 -   Add supported formats for schema resolution (deref).
--   Fix for [#7643](https://github.com/postmanlabs/postman-app-support/issues/7643), [#7914](https://github.com/postmanlabs/postman-app-support/issues/7914), [#9004](https://github.com/postmanlabs/postman-app-support/issues/9004) - Added support for Auth params in response/example. 
+-   Fix for [#7643](https://github.com/postmanlabs/postman-app-support/issues/7643), [#7914](https://github.com/postmanlabs/postman-app-support/issues/7914), [#9004](https://github.com/postmanlabs/postman-app-support/issues/9004) - Added support for Auth params in response/example.
 -   Bumped up multiple dependecies and dev-dependencies versions to keep them up-to-date.
 -   Updated code coverage tool from deprecated istanbul to nyc.
 
@@ -382,7 +386,7 @@ Newer releases follow the [Keep a Changelog](https://keepachangelog.com/en/1.0.0
 #### v1.1.13 (April 21, 2020)
 
 -   Added support for detailed validation body mismatches with option detailedBlobValidation.
--   Fix for [#8098](https://github.com/postmanlabs/postman-app-support/issues/8098) - Unable to validate schema with type array. 
+-   Fix for [#8098](https://github.com/postmanlabs/postman-app-support/issues/8098) - Unable to validate schema with type array.
 -   Fixed URIError for invalid URI in transaction.
 -   Fix for [#152](https://github.com/postmanlabs/openapi-to-postman/issues/152) - Path references not resolved due to improver handling of special characters.
 -   Fix for [#160](https://github.com/postmanlabs/openapi-to-postman/issues/160) - Added handling for variables in local servers not a part of a URL segment. All path servers to be added as collection variables.

lib/bundle.js

@@ -1,10 +1,14 @@
 const _ = require('lodash'),
   {
     isExtRef,
+    isExtURLRef,
+    stringIsAValidUrl,
+    isExtRemoteRef,
     getKeyInComponents,
     getJsonPointerRelationToRoot,
     removeLocalReferenceFromPath,
     localPointer,
+    httpSeparator,
     jsonPointerLevelSeparator,
     isLocalRef,
     jsonPointerDecodeAndReplace,
@@ -83,14 +87,21 @@ function calculatePath(parentFileName, referencePath) {
    * @returns {object} - Detect root files result object
    */
 function findNodeFromPath(referencePath, allData) {
-  const partialComponents = referencePath.split(localPointer);
-  let isPartial = partialComponents.length > 1,
-    node = allData.find((node) => {
-      if (isPartial) {
-        referencePath = partialComponents[0];
-      }
-      return comparePaths(node.fileName, referencePath);
-    });
+  const isReferenceRemoteURL = stringIsAValidUrl(referencePath),
+    partialComponents = referencePath.split(localPointer),
+    isPartial = partialComponents.length > 1;
+
+  let node = allData.find((node) => {
+    if (isPartial && !isReferenceRemoteURL) {
+      referencePath = partialComponents[0];
+    }
+
+    if (isReferenceRemoteURL) {
+      return _.startsWith(node.path, referencePath);
+    }
+
+    return comparePaths(node.fileName, referencePath);
+  });
 
   return node;
 }
@@ -290,13 +301,86 @@ function handleLocalCollisions(trace, initialMainKeys) {
    * @param {string} commonPathFromData - The common path in the file's paths
    * @param {Array} allData -  array of { path, content} objects
    * @param {object} globalReferences - The accumulated global references from all nodes
+   * @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
    * @returns {object} - The references in current node and the new content from the node
    */
-function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, version, rootMainKeys,
-  commonPathFromData, allData, globalReferences) {
+async function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, version, rootMainKeys,
+  commonPathFromData, allData, globalReferences, remoteRefResolver) {
   let referencesInNode = [],
     nodeReferenceDirectory = {},
-    mainKeys = {};
+    mainKeys = {},
+    remoteRefContentMap = new Map(),
+    remoteRefSet = new Set(),
+    remoteRefResolutionPromises = [];
+
+  remoteRefResolver && traverseUtility(currentNode).forEach(function (property) {
+    if (property) {
+      let hasReferenceTypeKey;
+
+      hasReferenceTypeKey = Object.keys(property)
+        .find(
+          (key) => {
+            const isExternal = isExtURLRef(property, key),
+              isReferenciable = isExternal;
+
+            return isReferenciable;
+          }
+        );
+
+      if (hasReferenceTypeKey) {
+        const tempRef = calculatePath(parentFilename, property.$ref),
+          isRefEncountered = remoteRefSet.has(tempRef);
+
+        if (isRefEncountered) {
+          return;
+        }
+
+        remoteRefResolutionPromises.push(
+          new Promise(async (resolveInner) => {
+
+            /**
+             * Converts contents received from remoteRefResolver into stringified JSON
+             * @param {string | object} content - contents from remoteRefResolver
+             * @returns {string} Stringified JSON contents
+             */
+            function convertToJSONString (content) {
+              if (typeof content === 'object') {
+                return JSON.stringify(content);
+              }
+
+              const parsedFile = parseFile(content);
+
+              return JSON.stringify(parsedFile.oasObject);
+            }
+
+            try {
+              let contentFromRemote = await remoteRefResolver(property.$ref),
+                nodeTemp = {
+                  fileName: tempRef,
+                  path: tempRef,
+                  content: convertToJSONString(contentFromRemote),
+                  href: property.$ref
+                };
+
+              remoteRefContentMap.set(tempRef, contentFromRemote);
+
+              allData.push(nodeTemp);
+            }
+            catch (err) {
+              // swallow the err
+            }
+            finally {
+              resolveInner();
+            }
+          })
+        );
+
+        remoteRefSet.add(tempRef);
+      }
+    }
+  });
+
+  await Promise.all(remoteRefResolutionPromises);
 
   traverseUtility(currentNode).forEach(function (property) {
     if (property) {
@@ -371,6 +455,94 @@ function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, ve
           referencesInNode.push({ path: pathSolver(property), keyInComponents: nodeTrace, newValue: this.node });
         }
       }
+
+      const hasRemoteReferenceTypeKey = Object.keys(property)
+          .find(
+            (key) => {
+              const isExternal = isExtURLRef(property, key),
+
+                // Only process URL refs if remoteRefResolver is provided and a valid function
+                isReferenciable = isExternal && _.isFunction(remoteRefResolver);
+
+              return isReferenciable;
+            }
+          ),
+        handleRemoteURLReference = () => {
+          const tempRef = calculatePath(parentFilename, property.$ref);
+
+          if (remoteRefContentMap.get(tempRef) === undefined) {
+            return;
+          }
+
+          let nodeTrace = handleLocalCollisions(
+              getTraceFromParentKeyInComponents(this, tempRef, mainKeys, version, commonPathFromData),
+              rootMainKeys
+            ),
+            componentKey = nodeTrace[nodeTrace.length - 1],
+            referenceInDocument = getJsonPointerRelationToRoot(
+              tempRef,
+              nodeTrace,
+              version
+            ),
+            traceToParent = [...this.parents.map((item) => {
+              return item.key;
+            }).filter((item) => {
+              return item !== undefined;
+            }), this.key],
+            newValue = Object.assign({}, this.node),
+            [, local] = tempRef.split(localPointer),
+            nodeFromData,
+            refHasContent = false,
+            parseResult,
+            newRefInDoc,
+            inline,
+            contentFromRemote = remoteRefContentMap.get(tempRef),
+            nodeTemp = {
+              fileName: tempRef,
+              path: tempRef,
+              content: contentFromRemote
+            };
+
+          nodeFromData = nodeTemp;
+
+          if (nodeFromData && nodeFromData.content) {
+            parseResult = parseFile(JSON.stringify(nodeFromData.content));
+            if (parseResult.result) {
+              newValue.$ref = referenceInDocument;
+              refHasContent = true;
+              nodeFromData.parsed = parseResult;
+            }
+          }
+          this.update({ $ref: tempRef });
+
+          if (nodeTrace.length === 0) {
+            inline = true;
+          }
+
+          if (_.isNil(globalReferences[tempRef])) {
+            nodeReferenceDirectory[tempRef] = {
+              local,
+              keyInComponents: nodeTrace,
+              node: newValue,
+              reference: inline ? newRefInDoc : referenceInDocument,
+              traceToParent,
+              parentNodeKey: parentFilename,
+              mainKeyInTrace: nodeTrace[nodeTrace.length - 1],
+              refHasContent,
+              inline
+            };
+          }
+
+          mainKeys[componentKey] = tempRef;
+
+          if (!added(property.$ref, referencesInNode)) {
+            referencesInNode.push({ path: pathSolver(property), keyInComponents: nodeTrace, newValue: this.node });
+          }
+        };
+
+      if (hasRemoteReferenceTypeKey) {
+        handleRemoteURLReference();
+      }
     }
   });
 
@@ -386,10 +558,11 @@ function getReferences (currentNode, isOutOfRoot, pathSolver, parentFilename, ve
    * @param {object} rootMainKeys - A dictionary with the component keys in local components object and its mainKeys
    * @param {string} commonPathFromData - The common path in the file's paths
    * @param {object} globalReferences - The accumulated global refernces from all nodes
+   * @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
    * @returns {object} - Detect root files result object
    */
-function getNodeContentAndReferences (currentNode, allData, specRoot, version, rootMainKeys,
-  commonPathFromData, globalReferences) {
+async function getNodeContentAndReferences (currentNode, allData, specRoot, version, rootMainKeys,
+  commonPathFromData, globalReferences, remoteRefResolver) {
   let graphAdj = [],
     missingNodes = [],
     nodeContent,
@@ -406,7 +579,7 @@ function getNodeContentAndReferences (currentNode, allData, specRoot, version, r
     nodeContent = parseResult.oasObject;
   }
 
-  const { referencesInNode, nodeReferenceDirectory } = getReferences(
+  const { referencesInNode, nodeReferenceDirectory } = await getReferences(
     nodeContent,
     currentNode.fileName !== specRoot.fileName,
     removeLocalReferenceFromPath,
@@ -415,7 +588,8 @@ function getNodeContentAndReferences (currentNode, allData, specRoot, version, r
     rootMainKeys,
     commonPathFromData,
     allData,
-    globalReferences
+    globalReferences,
+    remoteRefResolver
   );
 
   referencesInNode.forEach((reference) => {
@@ -516,9 +690,11 @@ function handleCircularReference(traverseContext, documentContext) {
  * @param {function} refTypeResolver - The resolver function to test if node has a reference
  * @param {object} components - The global components object
  * @param {string} version - The current version
+   * @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
  * @returns {object} The components object related to the file
  */
-function generateComponentsObject (documentContext, rootContent, refTypeResolver, components, version) {
+function generateComponentsObject(documentContext, rootContent,
+  refTypeResolver, components, version, remoteRefResolver) {
   let notInLine = Object.entries(documentContext.globalReferences).filter(([, value]) => {
       return value.keyInComponents.length !== 0;
     }),
@@ -555,13 +731,37 @@ function generateComponentsObject (documentContext, rootContent, refTypeResolver
             isMissingNode = documentContext.missing.find((missingNode) => {
               return missingNode.path === nodeRef;
             });
+
           if (isMissingNode) {
             refData.nodeContent = refData.node;
             refData.local = false;
           }
           else if (!refData) {
             return;
           }
+          else if (!isExtRef(property, '$ref') && isExtURLRef(property, '$ref')) {
+            let splitPathByHttp = property.$ref.split(httpSeparator),
+              prefix = splitPathByHttp
+                .slice(0, splitPathByHttp.length - 1).join(httpSeparator) +
+                httpSeparator + splitPathByHttp[splitPathByHttp.length - 1]
+                .split(localPointer)[0],
+              separatedPaths = [prefix, splitPathByHttp[splitPathByHttp.length - 1].split(localPointer)[1]];
+
+            nodeRef = separatedPaths[0];
+            local = separatedPaths[1];
+
+            refData.nodeContent = documentContext.nodeContents[nodeRef];
+
+            const isReferenceRemoteURL = stringIsAValidUrl(nodeRef);
+
+            if (isReferenceRemoteURL && _.isFunction(remoteRefResolver)) {
+              Object.keys(documentContext.nodeContents).forEach((key) => {
+                if (_.startsWith(key, nodeRef) && !key.split(nodeRef)[1].includes(httpSeparator)) {
+                  refData.nodeContent = documentContext.nodeContents[key];
+                }
+              });
+            }
+          }
           else {
             refData.nodeContent = documentContext.nodeContents[nodeRef];
           }
@@ -697,9 +897,10 @@ module.exports = {
    * @param {Array} allData -  array of { path, content} objects
    * @param {Array} origin - process origin (BROWSER or node)
    * @param {string} version - The version we are using
+   * @param {function} remoteRefResolver - The function that would be called to fetch remote ref contents
    * @returns {object} - Detect root files result object
    */
-  getBundleContentAndComponents: function (specRoot, allData, origin, version) {
+  getBundleContentAndComponents: async function (specRoot, allData, origin, version, remoteRefResolver) {
     if (origin === BROWSER) {
       path = pathBrowserify;
     }
@@ -716,15 +917,16 @@ module.exports = {
     commonPathFromData = Utils.findCommonSubpath(allData.map((fileData) => {
       return fileData.fileName;
     }));
-    rootContextData = algorithm.traverseAndBundle(specRoot, (currentNode, globalReferences) => {
+    rootContextData = await algorithm.traverseAndBundle(specRoot, (currentNode, globalReferences) => {
       return getNodeContentAndReferences(
         currentNode,
         allData,
         specRoot,
         version,
         initialMainKeys,
         commonPathFromData,
-        globalReferences
+        globalReferences,
+        remoteRefResolver
       );
     });
     components = generateComponentsWrapper(
@@ -735,10 +937,12 @@ module.exports = {
     finalElements = generateComponentsObject(
       rootContextData,
       rootContextData.nodeContents[specRoot.fileName],
-      isExtRef,
+      isExtRemoteRef,
       components,
-      version
+      version,
+      remoteRefResolver
     );
+
     return {
       fileContent: finalElements.resRoot,
       components: finalElements.newComponents,