Add some NPM methods to support checking if a version was unpublished…

…, or if a package is a security holding pacakge. (#349) * Add PackageVersionPulled to BaseProjectManager. It is only implemented for NPM as of now where it checks to see if the package version was at one point published but since has been pulled/unpublished. * Add PackageSecurityHolding to check if the NPM package has only one version, and if that version is a security holding package meaning NPM pulled it and marked it as a package being held on the registry for security purposes. * Added unit tests.
microsoft · Sep 6, 2022 · 1a247cc · 1a247cc
1 parent eb2e002
commit 1a247cc
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 1 deletion.
diff --git a/src/Shared/PackageManagers/BaseProjectManager.cs b/src/Shared/PackageManagers/BaseProjectManager.cs
@@ -362,6 +362,17 @@ public virtual async Task<bool> PackageVersionExistsAsync(PackageURL purl, bool
 
             return (await EnumerateVersionsAsync(purl, useCache)).Contains(purl.Version);
         }
+
+        /// <summary>
+        /// Check if the package version was pulled from the repository.
+        /// </summary>
+        /// <param name="purl">The PackageURL to check.</param>
+        /// <param name="useCache">If the cache should be checked for the existence of this package.</param>
+        /// <returns>True if the package was pulled from the repository. False otherwise.</returns>
+        public virtual Task<bool> PackageVersionPulled(PackageURL purl, bool useCache = true)
+        {
+            throw new NotImplementedException("BaseProjectManager does not implement PackageVersionPulled.");
+        }
 
         /// <summary>
         /// Static overload for getting the latest version.

diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs
@@ -31,6 +31,8 @@ public class NPMProjectManager : TypedManager<IManagerPackageVersionMetadata, NP
         public static string ENV_NPM_API_ENDPOINT { get; set; } = "https://registry.npmjs.org";
         public static string ENV_NPM_ENDPOINT { get; set; } = "https://www.npmjs.com";
 
+        private static readonly string NPM_SECURITY_HOLDING_VERSION = "0.0.1-security";
+
         public NPMProjectManager(
             string directory,
             IManagerPackageActions<IManagerPackageVersionMetadata>? actions = null,
@@ -467,14 +469,44 @@ public override List<Version> GetVersions(JsonDocument? contentJSON)
             return allVersions;
         }
 
+        public override async Task<bool> PackageVersionPulled(PackageURL purl, bool useCache = true)
+        {
+            string? content = await GetMetadataAsync(purl, useCache);
+            if (string.IsNullOrEmpty(content)) { return false; }
+
+            JsonDocument contentJSON = JsonDocument.Parse(content);
+            JsonElement root = contentJSON.RootElement;
+            if (root.TryGetProperty("time", out JsonElement time))
+            {
+                if (time.TryGetProperty("unpublished", out JsonElement unpublished))
+                {
+                    List<string>? versions = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(unpublished, "versions"));
+                    return versions?.Contains(purl.Version) ?? false;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Check to see if the package only has one version, and if that version is a NPM security holding package.
+        /// </summary>
+        /// <param name="purl">The <see cref="PackageURL"/> to check.</param>
+        /// <param name="useCache">If the cache should be checked for the existence of this package.</param>
+        /// <returns>True if this package is a NPM security holding package. False otherwise.</returns>
+        public async Task<bool> PackageSecurityHolding(PackageURL purl, bool useCache = true)
+        {
+            List<string> versions = (await this.EnumerateVersionsAsync(purl, useCache)).ToList();
+
+            return versions.Count == 1 && versions[0].Equals(NPM_SECURITY_HOLDING_VERSION);
+        }
+
         /// <summary>
         /// Searches the package manager metadata to figure out the source code repository
         /// </summary>
         /// <param name="purl">the package for which we need to find the source code repository</param>
         /// <returns>
         /// A dictionary, mapping each possible repo source entry to its probability/empty dictionary
         /// </returns>
-
         protected override async Task<Dictionary<PackageURL, double>> SearchRepoUrlsInPackageMetadata(PackageURL purl,
             string metadata)
         {

diff --git a/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs
@@ -26,6 +26,8 @@ public class NPMProjectManagerTests
         private readonly IDictionary<string, string> _packages = new Dictionary<string, string>()
         {
             { "https://registry.npmjs.org/lodash", Resources.lodash_json },
+            { "https://registry.npmjs.org/lodash.js", Resources.lodashjs_json },
+            { "https://registry.npmjs.org/%40somosme/webflowutils", Resources.unpublishedpackage_json },
             { "https://registry.npmjs.org/%40angular/core", Resources.angular_core_json },
             { "https://registry.npmjs.org/ds-modal", Resources.ds_modal_json },
             { "https://registry.npmjs.org/monorepolint", Resources.monorepolint_json },
@@ -94,6 +96,7 @@ public async Task EnumerateVersionsSucceeds(string purlString, int count, string
         [DataRow("pkg:npm/monorepolint@0.4.0")]
         [DataRow("pkg:npm/example@0.0.0")]
         [DataRow("pkg:npm/rly-cli@0.0.2")]
+        [DataRow("pkg:npm/lodash.js@0.0.1-security")]
         public async Task PackageVersionExistsAsyncSucceeds(string purlString)
         {
             PackageURL purl = new(purlString);
@@ -111,6 +114,27 @@ public async Task PackageVersionDoesntExistsAsyncSucceeds(string purlString)
             Assert.IsFalse(await _projectManager.PackageVersionExistsAsync(purl, useCache: false));
         }
 
+        [DataTestMethod]
+        [DataRow("pkg:npm/%40somosme/webflowutils@1.0.0")]
+        [DataRow("pkg:npm/%40somosme/webflowutils@1.2.3", false)]
+        public async Task PackageVersionPulledAsync(string purlString, bool expectedPulled = true)
+        {
+            PackageURL purl = new(purlString);
+
+            Assert.AreEqual(expectedPulled, await _projectManager.PackageVersionPulled(purl, useCache: false));
+        }
+
+        [DataTestMethod]
+        [DataRow("pkg:npm/lodash.js")]
+        [DataRow("pkg:npm/lodash.js@1.0.0")]
+        [DataRow("pkg:npm/lodash", false)]
+        public async Task PackageSecurityHoldingAsync(string purlString, bool expectedToHaveSecurityHolding = true)
+        {
+            PackageURL purl = new(purlString);
+
+            Assert.AreEqual(expectedToHaveSecurityHolding, await _projectManager.PackageSecurityHolding(purl, useCache: false));
+        }
+
         [DataTestMethod]
         [DataRow("pkg:npm/lodash@4.17.15", "2019-07-19T02:28:46.584Z")]
         [DataRow("pkg:npm/%40angular/core@13.2.5", "2022-03-02T18:25:31.169Z")]

diff --git a/src/oss-tests/Properties/Resources.Designer.cs b/src/oss-tests/Properties/Resources.Designer.cs
diff --git a/src/oss-tests/Properties/Resources.resx b/src/oss-tests/Properties/Resources.resx
@@ -27,6 +27,9 @@
     <data name="lodash.json" type="System.Resources.ResXFileRef, System.Windows.Forms">
         <value>..\TestData\NPM\lodash.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
     </data>
+    <data name="lodashjs.json" type="System.Resources.ResXFileRef, System.Windows.Forms">
+        <value>..\TestData\NPM\lodashjs.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+    </data>
     <data name="monorepolint.json" type="System.Resources.ResXFileRef, System.Windows.Forms">
         <value>..\TestData\NPM\monorepolint.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
     </data>
@@ -36,6 +39,9 @@
     <data name="minimum_json.json" type="System.Resources.ResXFileRef, System.Windows.Forms">
         <value>..\TestData\NPM\minimum_json.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
     </data>
+    <data name="unpublishedpackage.json" type="System.Resources.ResXFileRef, System.Windows.Forms">
+        <value>..\TestData\NPM\unpublishedpackage.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+    </data>
     <data name="razorengine.json" type="System.Resources.ResXFileRef, System.Windows.Forms">
         <value>..\TestData\NuGet\razorengine\index.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
     </data>

diff --git a/src/oss-tests/TestData/NPM/lodashjs.json b/src/oss-tests/TestData/NPM/lodashjs.json
@@ -0,0 +1 @@
+{"_id":"lodash.js","_rev":"7-b033eedf4d1b14d686a8c537585bb116","time":{"4.17.16":"2019-11-25T16:45:29.666Z","created":"2019-11-25T18:50:27.244Z","0.0.1-security":"2019-11-25T18:50:27.445Z","modified":"2022-05-08T08:11:30.890Z"},"name":"lodash.js","dist-tags":{"latest":"0.0.1-security"},"versions":{"0.0.1-security":{"name":"lodash.js","version":"0.0.1-security","description":"security holding package","repository":{"type":"git","url":"git+https://github.com/npm/security-holder.git"},"gitHead":"ac50be87aafecba67fcacca3b32bf36a1f8f7a71","bugs":{"url":"https://github.com/npm/security-holder/issues"},"homepage":"https://github.com/npm/security-holder#readme","_id":"lodash.js@0.0.1-security","_nodeVersion":"11.14.0","_npmVersion":"6.13.0","dist":{"integrity":"sha512-YHdFXeKSXAHuY5pusUO4sltubTuLrTqhatFgdKuiSisSuQ/UdrQNlCtgB+CDnzgsNp/Y3Mdfa3K4rInyEDHugg==","shasum":"9f4bddb7df0b7e36b101c2054549300d38b6f518","tarball":"https://registry.npmjs.org/lodash.js/-/lodash.js-0.0.1-security.tgz","fileCount":2,"unpackedSize":469,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJd3CJzCRA9TVsSAnZWagAAN7kP/inmZqIyY4YHkc+HuG/4\n4zZTOiweO+duew6VZbIT7/fvPydBFuPuTBjmBC9D/JdDG10dgFxmxOf9RMDI\nwW4B1XQn8bc/6BNkaWXDFkJXlOsu60X08wvntRPRr+FNRjwyMvfyzrg3ad44\nnalTEr2PQpNyP+sDhamEgUXjIqGXM7wNCrcI0kf8gB0aqR/WH8fKYKFtVhHs\nF4kL0sUwiYzhwdImgc1ZmYEV1d9Gcnam+OiLImL7HCljG3SXmWNlpKOuRmUp\n3G2bz6mz9bs+QsmPDFXh4rg44ErplTuRXd4rUGbLFSyHqz7k8oBdmzzUQmu4\nKcvQmTC9UPrq989s8QEnnmlx48OcH99Ig6cMtIj7ZkoscpH/KSH4Bvc65CtQ\nFzqsSLv3jlU/TgRjTxNwMOurjNE4FbQOKb9ICC1OYVGv3cIG31rrzcwGzrX6\ndhsinq47C7AdKYcmTOUqhCSQwqonbrDUO/nKtIpQ1t3shxx3KKUef4DOmYcN\nu36+h8mNR8De6fqLxBNRhazV5YG4Sfab+CQ+r7KvEusrT//lJpnUw2rqGHW3\naqSypAfwjvodnr0Mwh6pNhB3eUAO9on9t/MfB39NVh5kpa3mW+pT4Q558Mnh\ncsGKmTygH+fWmzA1ty/VGRUSHqrMwOAdEV81xo0UXIennkKwfB/WBu7/EjCm\nuqJ+\r\n=yUZ9\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHsnZDHbTNbN1PrubcwvLrhEqTjvHi0ym/5AxK5Fpr7cAiEAnzTEI7y2HFivNLF+5L9OBGlf0/5mmDwOrFO0hkf0JqY="}]},"maintainers":[{"name":"andreeleuterio","email":"andre@npmjs.com"}],"_npmUser":{"name":"andreeleuterio","email":"andre@npmjs.com"},"directories":{},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/lodash.js_0.0.1-security_1574707827244_0.7707519668716685"},"_hasShrinkwrap":false}},"maintainers":[{"email":"npm@npmjs.com","name":"npm"},{"email":"andre@npmjs.com","name":"andreeleuterio"}],"description":"security holding package","homepage":"https://github.com/npm/security-holder#readme","repository":{"type":"git","url":"git+https://github.com/npm/security-holder.git"},"bugs":{"url":"https://github.com/npm/security-holder/issues"},"readme":"# Security holding package\n\nThis package name is not currently in use, but was formerly occupied\nby another package. To avoid malicious use, npm is hanging on to the\npackage name, but loosely, and we'll probably give it to you if you\nwant it.\n\nYou may adopt this package by contacting support@npmjs.com and\nrequesting the name.\n","readmeFilename":"README.md"}
diff --git a/src/oss-tests/TestData/NPM/unpublishedpackage.json b/src/oss-tests/TestData/NPM/unpublishedpackage.json
@@ -0,0 +1 @@
+{"_id":"@somosme/webflowutils","_rev":"3-9be941baa0509bbc96a1349f8464fd6d","name":"@somosme/webflowutils","time":{"created":"2022-08-10T19:49:56.913Z","1.0.0":"2022-08-10T19:49:57.198Z","modified":"2022-08-10T21:31:52.293Z","unpublished":{"time":"2022-08-10T21:31:32.856Z","versions":["1.0.0"]}},"maintainers":[{"email":"antonio@somos.me","name":"antoniomb"}]}