feat(conan): add revisions support

lib/modules/datasource/conan/index.spec.ts

@@ -7,6 +7,10 @@ import { defaultRegistryUrl } from './common';
 import { ConanDatasource } from '.';
 
 const pocoJson = Fixtures.get('poco.json');
+const pocoRevisions = Fixtures.getJson('poco_revisions.json');
+const malformedPocoRevisions = Fixtures.getJson(
+  'poco_revisions_malformed.json'
+);
 const pocoYamlGitHubContent = Fixtures.get('poco.yaml');
 const malformedJson = Fixtures.get('malformed.json');
 const fakeJson = Fixtures.get('fake.json');
@@ -117,7 +121,94 @@ describe('modules/datasource/conan/index', () => {
       });
     });
 
-    it('uses github isntead of conan center', async () => {
+    it('processes real versioned data real data with revisions', async () => {
+      httpMock
+        .scope(nonDefaultRegistryUrl)
+        .get('/v2/conans/search?q=poco')
+        .reply(200, pocoJson);
+      for (const version of ['1.8.1', '1.9.3', '1.9.4', '1.10.0', '1.10.1']) {
+        httpMock
+          .scope(nonDefaultRegistryUrl)
+          .get(`/v2/conans/poco/${version}/_/_/revisions`)
+          .reply(200, pocoRevisions[version]);
+      }
+      config.depName = 'poco';
+      expect(
+        await getPkgReleases({
+          ...config,
+          packageName: 'poco/1.2@_/_#4fc13d60fd91ba44fefe808ad719a5af',
+        })
+      ).toEqual({
+        registryUrl: 'https://not.conan.io',
+        releases: [
+          {
+            version: '1.8.1',
+            newDigest: '3a9b47caee2e2c1d3fb7d97788339aa8',
+          },
+          {
+            version: '1.9.3',
+            newDigest: 'dc80dcf02da270cc5f629c00df151a56',
+          },
+          {
+            version: '1.9.4',
+            newDigest: '736fdb96cd0add0cf85ca420e0bac453',
+          },
+          {
+            version: '1.10.0',
+            newDigest: '7c3fbb12b69d7e7fd7134ba1efb04d5e',
+          },
+          {
+            version: '1.10.1',
+            newDigest: '32815db5b18046e4f8879af7bb743422',
+          },
+        ],
+      });
+    });
+
+    it('processes maformed revisions', async () => {
+      httpMock
+        .scope(nonDefaultRegistryUrl)
+        .get('/v2/conans/search?q=poco')
+        .reply(200, pocoJson);
+      for (const version of ['1.8.1', '1.9.3', '1.9.4', '1.10.0', '1.10.1']) {
+        httpMock
+          .scope(nonDefaultRegistryUrl)
+          .get(`/v2/conans/poco/${version}/_/_/revisions`)
+          .reply(200, malformedPocoRevisions[version]);
+      }
+      config.depName = 'poco';
+      expect(
+        await getPkgReleases({
+          ...config,
+          packageName: 'poco/1.2@_/_#4fc13d60fd91ba44fefe808ad719a5af',
+        })
+      ).toEqual({
+        registryUrl: 'https://not.conan.io',
+        releases: [
+          {
+            version: '1.8.1',
+            newDigest: '3a9b47caee2e2c1d3fb7d97788339aa8',
+          },
+          {
+            version: '1.9.3',
+            newDigest: 'dc80dcf02da270cc5f629c00df151a56',
+          },
+          {
+            version: '1.9.4',
+            newDigest: '736fdb96cd0add0cf85ca420e0bac453',
+          },
+          {
+            version: '1.10.0',
+            newDigest: '7c3fbb12b69d7e7fd7134ba1efb04d5e',
+          },
+          {
+            version: '1.10.1',
+          },
+        ],
+      });
+    });
+
+    it('uses github instead of conan center', async () => {
       httpMock
         .scope('https://api.github.com')
         .get(

lib/modules/datasource/conan/index.ts

@@ -7,7 +7,16 @@ import { ensureTrailingSlash, joinUrlParts } from '../../../util/url';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
 import { conanDatasourceRegex, datasource, defaultRegistryUrl } from './common';
-import type { ConanJSON, ConanYAML } from './types';
+import type { ConanJSON, ConanRevisionsJSON, ConanYAML } from './types';
+
+function getRevision(packageName: string): string | undefined {
+  const splitted = packageName.split('#');
+  if (splitted.length <= 1) {
+    return undefined;
+  } else {
+    return splitted[1];
+  }
+}
 
 export class ConanDatasource extends Datasource {
   static readonly id = datasource;
@@ -50,6 +59,21 @@ export class ConanDatasource extends Datasource {
     };
   }
 
+  async getNewDigest(
+    url: string,
+    packageName: string
+  ): Promise<string | undefined> {
+    const revisionLookUp = joinUrlParts(
+      url,
+      `v2/conans/${packageName}/revisions`
+    );
+    const revisionRep = await this.http.getJson<ConanRevisionsJSON>(
+      revisionLookUp
+    );
+    const revisions = revisionRep?.body.revisions;
+    return revisions ? revisions[0].revision : undefined;
+  }
+
   @cache({
     namespace: `datasource-${datasource}`,
     key: ({ registryUrl, packageName }: GetReleasesConfig) =>
@@ -62,10 +86,12 @@ export class ConanDatasource extends Datasource {
     packageName,
   }: GetReleasesConfig): Promise<ReleaseResult | null> {
     const depName = packageName.split('/')[0];
-    const userAndChannel = '@' + packageName.split('@')[1];
+    const userAndChannel = '@' + packageName.split('@')[1].split('#')[0];
+    const revision = getRevision(packageName);
     if (
       is.string(registryUrl) &&
-      ensureTrailingSlash(registryUrl) === defaultRegistryUrl
+      ensureTrailingSlash(registryUrl) === defaultRegistryUrl &&
+      is.undefined(revision)
     ) {
       return this.getConanCenterReleases(depName, userAndChannel);
     }
@@ -87,8 +113,17 @@ export class ConanDatasource extends Datasource {
             if (fromMatch?.groups?.version && fromMatch?.groups?.userChannel) {
               const version = fromMatch.groups.version;
               if (fromMatch.groups.userChannel === userAndChannel) {
+                let newDigest: string | undefined = undefined;
+                if (revision) {
+                  const currentPackageName = `${depName}/${version}${userAndChannel.replace(
+                    '@',
+                    '/'
+                  )}`;
+                  newDigest = await this.getNewDigest(url, currentPackageName);
+                }
                 const result: Release = {
                   version,
+                  newDigest,
                 };
                 dep.releases.push(result);
               }

lib/modules/datasource/conan/types.ts

@@ -2,6 +2,14 @@ export interface ConanJSON {
   results?: Record<string, string>;
 }
 
+export interface ConanRevisionJSON {
+  revision?: string;
+}
+
+export interface ConanRevisionsJSON {
+  revisions?: Record<string, ConanRevisionJSON>;
+}
+
 export interface ConanYAML {
   versions?: Record<string, unknown>;
 }

lib/modules/manager/conan/__fixtures__/conanfile.py

@@ -12,6 +12,10 @@ class Pkg(ConanFile):
    requires = (("req_c/1.0@user/stable", "private"), )
    requires = ("req_f/1.0@user/stable", ("req_h/3.0@other/beta", "override"))
    requires = "req_g/[>1.0 <1.8]@user/stable"
+#  requires = "commentedout/[>1.0 <1.8]@user/stable"
+   #  requires = "commentedout2/[>1.0 <1.8]@user/stable"
+   requires = (("req_l/1.0@user/stable#bc592346b33fd19c1fbffce25d1e4236", "private"), )
+
 
 
 def requirements(self):

lib/modules/manager/conan/__fixtures__/conanfile.txt

@@ -2,6 +2,7 @@
 poco/1.9.4
 zlib/[~1.2.3, loose=False]  
 fake/8.62.134@test/dev
+cairo/1.17.2@_/_#aff2d03608351db075ec1348a3afc9ff
 
 [build_requires]
 7zip/[>1.1 <2.1, include_prerelease=True]
@@ -13,6 +14,7 @@ cmake/[>1.1 || 0.8]
 cryptopp/[1.2.7 || >=1.2.9 <2.0.0]@test/local
 #commentedout/1.2
 # commentedout/3.4
+meson/0.63.0@_/_#bc592346b33fd19c1fbffce25d1e4236
 
 [generators]
 xcode

lib/modules/manager/conan/extract.spec.ts

@@ -35,6 +35,16 @@ describe('modules/manager/conan/extract', () => {
           packageName: 'fake/8.62.134@test/dev',
           replaceString: 'fake/8.62.134@test/dev',
         },
+        {
+          autoReplaceStringTemplate:
+            '{{depName}}/{{newValue}}@_/_{{#if newDigest}}#{{newDigest}}{{/if}}',
+          currentDigest: 'aff2d03608351db075ec1348a3afc9ff',
+          currentValue: '1.17.2',
+          depName: 'cairo',
+          depType: 'requires',
+          packageName: 'cairo/1.17.2@_/_#aff2d03608351db075ec1348a3afc9ff',
+          replaceString: 'cairo/1.17.2@_/_#aff2d03608351db075ec1348a3afc9ff',
+        },
         {
           currentValue: '[>1.1 <2.1, include_prerelease=True]',
           depName: '7zip',
@@ -86,6 +96,16 @@ describe('modules/manager/conan/extract', () => {
           packageName: 'cryptopp/[1.2.7 || >=1.2.9 <2.0.0]@test/local',
           replaceString: 'cryptopp/[1.2.7 || >=1.2.9 <2.0.0]@test/local',
         },
+        {
+          autoReplaceStringTemplate:
+            '{{depName}}/{{newValue}}@_/_{{#if newDigest}}#{{newDigest}}{{/if}}',
+          currentDigest: 'bc592346b33fd19c1fbffce25d1e4236',
+          currentValue: '0.63.0',
+          depName: 'meson',
+          depType: 'build_requires',
+          packageName: 'meson/0.63.0@_/_#bc592346b33fd19c1fbffce25d1e4236',
+          replaceString: 'meson/0.63.0@_/_#bc592346b33fd19c1fbffce25d1e4236',
+        },
       ]);
     });
 
@@ -181,6 +201,17 @@ describe('modules/manager/conan/extract', () => {
           packageName: 'req_g/[>1.0 <1.8]@user/stable',
           replaceString: 'req_g/[>1.0 <1.8]@user/stable',
         },
+        {
+          autoReplaceStringTemplate:
+            '{{depName}}/{{newValue}}@user/stable{{#if newDigest}}#{{newDigest}}{{/if}}',
+          currentDigest: 'bc592346b33fd19c1fbffce25d1e4236',
+          currentValue: '1.0',
+          depName: 'req_l',
+          depType: 'requires',
+          packageName: 'req_l/1.0@user/stable#bc592346b33fd19c1fbffce25d1e4236',
+          replaceString:
+            'req_l/1.0@user/stable#bc592346b33fd19c1fbffce25d1e4236',
+        },
         {
           currentValue: '1.2',
           depName: 'req_i',

lib/modules/manager/conan/extract.ts

@@ -3,7 +3,7 @@ import { regEx } from '../../../util/regex';
 import type { PackageDependency, PackageFile } from '../types';
 
 const regex = regEx(
-  `(?<name>[-_a-z0-9]+)/(?<version>[^@\n{*"']+)(?<userChannel>@[-_a-zA-Z0-9]+/[^\n.{*"' ]+)?`
+  `(?<name>[-_a-z0-9]+)/(?<version>[^@\n{*"']+)(?<userChannel>@[-_a-zA-Z0-9]+/[^#\n.{*"' ]+)?#?(?<revision>[-_a-f0-9]+[^\n{*"'])?`
 );
 
 function setDepType(content: string, originalType: string): string {
@@ -18,6 +18,10 @@ function setDepType(content: string, originalType: string): string {
   return depType;
 }
 
+function isComment(line: string): boolean {
+  return line.trim().startsWith('#');
+}
+
 export function extractPackageFile(content: string): PackageFile | null {
   // only process sections where requirements are defined
   const sections = content.split(/def |\n\[/).filter(
@@ -32,13 +36,11 @@ export function extractPackageFile(content: string): PackageFile | null {
     let depType = setDepType(section, 'requires');
     const rawLines = section.split('\n').filter(is.nonEmptyString);
 
-    for (const rawline of rawLines) {
-      // don't process after a comment
-      const sanitizedLine = rawline.split('#')[0].split('//')[0];
-      if (sanitizedLine) {
-        depType = setDepType(sanitizedLine, depType);
+    for (const rawLine of rawLines) {
+      if (!isComment(rawLine)) {
+        depType = setDepType(rawLine, depType);
         // extract all dependencies from each line
-        const lines = sanitizedLine.split(/["'],/);
+        const lines = rawLine.split(/["'],/);
         for (const line of lines) {
           const matches = regex.exec(line.trim());
           if (matches?.groups) {
@@ -64,6 +66,13 @@ export function extractPackageFile(content: string): PackageFile | null {
               replaceString,
               depType,
             };
+            if (matches.groups.revision) {
+              dep.currentDigest = matches.groups.revision;
+              dep.autoReplaceStringTemplate = `{{depName}}/{{newValue}}${userAndChannel}{{#if newDigest}}#{{newDigest}}{{/if}}`;
+              dep.replaceString = `${replaceString}#${dep.currentDigest}`;
+              dep.packageName = `${packageName}#${dep.currentDigest}`;
+            }
+
             deps.push(dep);
           }
         }