feat(conan): add revisions support

lib/modules/datasource/conan/__fixtures__/poco_revisions.json

@@ -0,0 +1,51 @@
+{
+  "1.10.1": {
+      "reference": "poco/1.10.1@_/_",
+      "revisions": [
+          {
+              "revision": "32815db5b18046e4f8879af7bb743422",
+              "time": "2022-07-18T09:38:37.372+0000"
+          }
+      ]
+  },
+  "1.10.0": {
+    "reference": "poco/1.10.0@_/_",
+    "revisions": [
+        {
+            "revision": "7c3fbb12b69d7e7fd7134ba1efb04d5e",
+            "time": "2022-07-18T09:38:37.768+0000"
+        }
+      ]
+  },
+  "1.9.4": {
+    "reference": "poco/1.9.4@_/_",
+    "revisions": [
+        {
+            "revision": "736fdb96cd0add0cf85ca420e0bac453",
+            "time": "2022-07-18T09:38:38.396+0000"
+        }
+    ]
+  },
+  "1.9.3": {
+      "reference": "poco/1.9.3@_/_",
+      "revisions": [
+          {
+              "revision": "dc80dcf02da270cc5f629c00df151a56",
+              "time": "2022-07-18T09:38:38.975+0000"
+          },
+          {
+              "revision": "92b4e6cf83f6be0b9589ba61cc4a5b4e",
+              "time": "2022-07-11T16:37:30.926+0000"
+          }
+      ]
+  },
+  "1.8.1": {
+      "reference": "poco/1.8.1@_/_",
+      "revisions": [
+          {
+              "revision": "3a9b47caee2e2c1d3fb7d97788339aa8",
+              "time": "2022-07-18T09:38:38.815+0000"
+          }
+      ]
+    }
+}

lib/modules/datasource/conan/__fixtures__/poco_revisions_malformed.json

@@ -0,0 +1,45 @@
+{
+  "1.10.1": {
+      "reference": "poco/1.10.1@_/_"
+  },
+  "1.10.0": {
+    "reference": "poco/1.10.0@_/_",
+    "revisions": [
+        {
+            "revision": "7c3fbb12b69d7e7fd7134ba1efb04d5e",
+            "time": "2022-07-18T09:38:37.768+0000"
+        }
+      ]
+  },
+  "1.9.4": {
+    "reference": "poco/1.9.4@_/_",
+    "revisions": [
+        {
+            "revision": "736fdb96cd0add0cf85ca420e0bac453",
+            "time": "2022-07-18T09:38:38.396+0000"
+        }
+    ]
+  },
+  "1.9.3": {
+      "reference": "poco/1.9.3@_/_",
+      "revisions": [
+          {
+              "revision": "dc80dcf02da270cc5f629c00df151a56",
+              "time": "2022-07-18T09:38:38.975+0000"
+          },
+          {
+              "revision": "92b4e6cf83f6be0b9589ba61cc4a5b4e",
+              "time": "2022-07-11T16:37:30.926+0000"
+          }
+      ]
+  },
+  "1.8.1": {
+      "reference": "poco/1.8.1@_/_",
+      "revisions": [
+          {
+              "revision": "3a9b47caee2e2c1d3fb7d97788339aa8",
+              "time": "2022-07-18T09:38:38.815+0000"
+          }
+      ]
+    }
+}

lib/modules/datasource/conan/common.ts

@@ -8,3 +8,12 @@ export const conanDatasourceRegex = regEx(
   /(?<name>[a-z\-_0-9]+)\/(?<version>[^@/\n]+)(?<userChannel>@\S+\/\S+)/,
   'gim'
 );
+
+export function getRevision(packageName: string): string | null {
+  const splitted = packageName.split('#');
+  if (splitted.length <= 1) {
+    return null;
+  } else {
+    return splitted[1];
+  }
+}

lib/modules/datasource/conan/index.spec.ts

@@ -1,12 +1,16 @@
-import { getPkgReleases } from '..';
+import { getDigest, getPkgReleases } from '..';
 import { Fixtures } from '../../../../test/fixtures';
 import * as httpMock from '../../../../test/http-mock';
 import * as conan from '../../versioning/conan';
-import type { GetPkgReleasesConfig } from '../types';
+import type { GetDigestInputConfig, GetPkgReleasesConfig } from '../types';
 import { defaultRegistryUrl } from './common';
 import { ConanDatasource } from '.';
 
 const pocoJson = Fixtures.get('poco.json');
+const pocoRevisions = Fixtures.getJson('poco_revisions.json');
+const malformedPocoRevisions = Fixtures.getJson(
+  'poco_revisions_malformed.json'
+);
 const pocoYamlGitHubContent = Fixtures.get('poco.yaml');
 const malformedJson = Fixtures.get('malformed.json');
 const fakeJson = Fixtures.get('fake.json');
@@ -21,11 +25,36 @@ const config: GetPkgReleasesConfig = {
   registryUrls: [nonDefaultRegistryUrl],
 };
 
+const digestConfig: GetDigestInputConfig = {
+  depName: 'fake',
+  datasource,
+  registryUrls: [nonDefaultRegistryUrl],
+};
+
 describe('modules/datasource/conan/index', () => {
   beforeEach(() => {
     config.registryUrls = [nonDefaultRegistryUrl];
   });
 
+  describe('getDigest', () => {
+    it('handles package without digest', async () => {
+      digestConfig.packageName = 'fakepackage/1.2@_/_';
+      expect(await getDigest(digestConfig)).toBeNull();
+    });
+
+    it('handles digest', async () => {
+      const version = '1.8.1';
+      httpMock
+        .scope(nonDefaultRegistryUrl)
+        .get(`/v2/conans/poco/${version}/_/_/revisions`)
+        .reply(200, pocoRevisions['1.8.1']);
+      digestConfig.packageName = `poco/${version}@_/_#4fc13d60fd91ba44fefe808ad719a5af`;
+      expect(await getDigest(digestConfig)).toBe(
+        '3a9b47caee2e2c1d3fb7d97788339aa8'
+      );
+    });
+  });
+
   describe('getReleases', () => {
     it('handles bad return', async () => {
       httpMock
@@ -117,7 +146,94 @@ describe('modules/datasource/conan/index', () => {
       });
     });
 
-    it('uses github isntead of conan center', async () => {
+    it('processes real versioned data real data with revisions', async () => {
+      httpMock
+        .scope(nonDefaultRegistryUrl)
+        .get('/v2/conans/search?q=poco')
+        .reply(200, pocoJson);
+      for (const version of ['1.8.1', '1.9.3', '1.9.4', '1.10.0', '1.10.1']) {
+        httpMock
+          .scope(nonDefaultRegistryUrl)
+          .get(`/v2/conans/poco/${version}/_/_/revisions`)
+          .reply(200, pocoRevisions[version]);
+      }
+      config.depName = 'poco';
+      expect(
+        await getPkgReleases({
+          ...config,
+          packageName: 'poco/1.2@_/_#4fc13d60fd91ba44fefe808ad719a5af',
+        })
+      ).toEqual({
+        registryUrl: 'https://not.conan.io',
+        releases: [
+          {
+            version: '1.8.1',
+            newDigest: '3a9b47caee2e2c1d3fb7d97788339aa8',
+          },
+          {
+            version: '1.9.3',
+            newDigest: 'dc80dcf02da270cc5f629c00df151a56',
+          },
+          {
+            version: '1.9.4',
+            newDigest: '736fdb96cd0add0cf85ca420e0bac453',
+          },
+          {
+            version: '1.10.0',
+            newDigest: '7c3fbb12b69d7e7fd7134ba1efb04d5e',
+          },
+          {
+            version: '1.10.1',
+            newDigest: '32815db5b18046e4f8879af7bb743422',
+          },
+        ],
+      });
+    });
+
+    it('processes maformed revisions', async () => {
+      httpMock
+        .scope(nonDefaultRegistryUrl)
+        .get('/v2/conans/search?q=poco')
+        .reply(200, pocoJson);
+      for (const version of ['1.8.1', '1.9.3', '1.9.4', '1.10.0', '1.10.1']) {
+        httpMock
+          .scope(nonDefaultRegistryUrl)
+          .get(`/v2/conans/poco/${version}/_/_/revisions`)
+          .reply(200, malformedPocoRevisions[version]);
+      }
+      config.depName = 'poco';
+      expect(
+        await getPkgReleases({
+          ...config,
+          packageName: 'poco/1.2@_/_#4fc13d60fd91ba44fefe808ad719a5af',
+        })
+      ).toEqual({
+        registryUrl: 'https://not.conan.io',
+        releases: [
+          {
+            version: '1.8.1',
+            newDigest: '3a9b47caee2e2c1d3fb7d97788339aa8',
+          },
+          {
+            version: '1.9.3',
+            newDigest: 'dc80dcf02da270cc5f629c00df151a56',
+          },
+          {
+            version: '1.9.4',
+            newDigest: '736fdb96cd0add0cf85ca420e0bac453',
+          },
+          {
+            version: '1.10.0',
+            newDigest: '7c3fbb12b69d7e7fd7134ba1efb04d5e',
+          },
+          {
+            version: '1.10.1',
+          },
+        ],
+      });
+    });
+
+    it('uses github instead of conan center', async () => {
       httpMock
         .scope('https://api.github.com')
         .get(

lib/modules/datasource/conan/index.ts

@@ -6,8 +6,13 @@ import { GithubHttp } from '../../../util/http/github';
 import { ensureTrailingSlash, joinUrlParts } from '../../../util/url';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
-import { conanDatasourceRegex, datasource, defaultRegistryUrl } from './common';
-import type { ConanJSON, ConanYAML } from './types';
+import {
+  conanDatasourceRegex,
+  datasource,
+  defaultRegistryUrl,
+  getRevision,
+} from './common';
+import type { ConanJSON, ConanRevisionsJSON, ConanYAML } from './types';
 
 export class ConanDatasource extends Datasource {
   static readonly id = datasource;
@@ -50,6 +55,44 @@ export class ConanDatasource extends Datasource {
     };
   }
 
+  async getNewDigest(url: string, packageName: string): Promise<string | null> {
+    const revisionLookUp = joinUrlParts(
+      url,
+      `v2/conans/${packageName}/revisions`
+    );
+    const revisionRep = await this.http.getJson<ConanRevisionsJSON>(
+      revisionLookUp
+    );
+    const revisions = revisionRep?.body.revisions;
+    return revisions ? revisions[0].revision : null;
+  }
+
+  @cache({
+    namespace: `datasource-${datasource}-revisions`,
+    key: ({ registryUrl, packageName }: GetReleasesConfig) =>
+      // TODO: types (#7154)
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+      `${registryUrl}:${packageName}`,
+  })
+  override async getDigest(
+    { registryUrl, packageName }: GetReleasesConfig,
+    newValue?: string
+  ): Promise<string | null> {
+    const revision = getRevision(packageName);
+    if (!revision || is.undefined(registryUrl)) {
+      return null;
+    } else {
+      const url = ensureTrailingSlash(registryUrl);
+      const depName = packageName.split('/')[0];
+      const userAndChannel = '/' + packageName.split('@')[1].split('#')[0];
+      const packageNameWithoutRevision = newValue
+        ? `${depName}/${newValue}${userAndChannel}`
+        : packageName.split('#')[0].replace('@', '/');
+      const digest = await this.getNewDigest(url, packageNameWithoutRevision);
+      return digest ? digest : null;
+    }
+  }
+
   @cache({
     namespace: `datasource-${datasource}`,
     key: ({ registryUrl, packageName }: GetReleasesConfig) =>
@@ -62,7 +105,7 @@ export class ConanDatasource extends Datasource {
     packageName,
   }: GetReleasesConfig): Promise<ReleaseResult | null> {
     const depName = packageName.split('/')[0];
-    const userAndChannel = '@' + packageName.split('@')[1];
+    const userAndChannel = '@' + packageName.split('@')[1].split('#')[0];
     if (
       is.string(registryUrl) &&
       ensureTrailingSlash(registryUrl) === defaultRegistryUrl

lib/modules/datasource/conan/types.ts

@@ -2,6 +2,15 @@ export interface ConanJSON {
   results?: Record<string, string>;
 }
 
+export interface ConanRevisionJSON {
+  revision: string;
+  time: string
+}
+
+export interface ConanRevisionsJSON {
+  revisions?: Record<string, ConanRevisionJSON>;
+}
+
 export interface ConanYAML {
   versions?: Record<string, unknown>;
 }

lib/modules/manager/conan/__fixtures__/conanfile.py

@@ -12,6 +12,10 @@ class Pkg(ConanFile):
    requires = (("req_c/1.0@user/stable", "private"), )
    requires = ("req_f/1.0@user/stable", ("req_h/3.0@other/beta", "override"))
    requires = "req_g/[>1.0 <1.8]@user/stable"
+#  requires = "commentedout/[>1.0 <1.8]@user/stable"
+   #  requires = "commentedout2/[>1.0 <1.8]@user/stable"
+   requires = (("req_l/1.0@user/stable#bc592346b33fd19c1fbffce25d1e4236", "private"), )
+
 
 
 def requirements(self):

lib/modules/manager/conan/__fixtures__/conanfile.txt

@@ -2,6 +2,7 @@
 poco/1.9.4
 zlib/[~1.2.3, loose=False]  
 fake/8.62.134@test/dev
+cairo/1.17.2@_/_#aff2d03608351db075ec1348a3afc9ff
 
 [build_requires]
 7zip/[>1.1 <2.1, include_prerelease=True]
@@ -13,6 +14,7 @@ cmake/[>1.1 || 0.8]
 cryptopp/[1.2.7 || >=1.2.9 <2.0.0]@test/local
 #commentedout/1.2
 # commentedout/3.4
+meson/0.63.0@_/_#bc592346b33fd19c1fbffce25d1e4236
 
 [generators]
 xcode

lib/modules/manager/conan/common.ts

@@ -0,0 +1,3 @@
+export function isComment(line: string): boolean {
+  return line.trim().startsWith('#');
+}