From 965b3ca8f9699c21222a93f010995b066032a5ce Mon Sep 17 00:00:00 2001
From: Trim21 <i@trim21.me>
Date: Sun, 7 Jun 2020 17:35:07 +0800
Subject: [PATCH] feat(pypi): changelog lookup in project_urls (#6421)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
 .../pypi/__snapshots__/index.spec.ts.snap     | 17 ++++++-----
 lib/datasource/pypi/index.spec.ts             | 27 +++++++++--------
 lib/datasource/pypi/index.ts                  | 30 +++++++++++++++----
 3 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
index 7dc46ba59b9467..f29de15db3bc55 100644
--- a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
@@ -1,5 +1,14 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`datasource/pypi getReleases find url from project_urls 1`] = `
+Object {
+  "changelogUrl": "https://github.com/Flexget/wiki/blob/master/ChangeLog.md",
+  "homepage": "https://flexget.com",
+  "releases": Array [],
+  "sourceUrl": "https://github.com/Flexget/Flexget",
+}
+`;
+
 exports[`datasource/pypi getReleases process data from +simple endpoint 1`] = `
 Object {
   "releases": Array [
@@ -206,14 +215,6 @@ Object {
 }
 `;
 
-exports[`datasource/pypi getReleases return sourceUrl in project_urls 1`] = `
-Object {
-  "homepage": "https://flexget.com",
-  "releases": Array [],
-  "sourceUrl": "https://github.com/Flexget/Flexget",
-}
-`;
-
 exports[`datasource/pypi getReleases returns non-github home_page 1`] = `
 Object {
   "homepage": "https://microsoft.com",
diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts
index 4e13bea7273a65..7268ca4b9a7870 100644
--- a/lib/datasource/pypi/index.spec.ts
+++ b/lib/datasource/pypi/index.spec.ts
@@ -122,25 +122,28 @@ describe('datasource/pypi', () => {
         })
       ).toMatchSnapshot();
     });
-    it('return sourceUrl in project_urls', async () => {
+    it('find url from project_urls', async () => {
+      const info = {
+        name: 'flexget',
+        home_page: 'https://flexget.com',
+        project_urls: {
+          Forum: 'https://discuss.flexget.com',
+          Homepage: 'https://flexget.com',
+          changelog: 'https://github.com/Flexget/wiki/blob/master/ChangeLog.md',
+          'Issue Tracker': 'https://github.com/Flexget/Flexget/issues',
+          Repository: 'https://github.com/Flexget/Flexget',
+        },
+      };
       got.mockReturnValueOnce({
         body: {
-          info: {
-            name: 'flexget',
-            home_page: 'https://flexget.com',
-            project_urls: {
-              Forum: 'https://discuss.flexget.com',
-              Homepage: 'https://flexget.com',
-              'Issue Tracker': 'https://github.com/Flexget/Flexget/issues',
-              Repository: 'https://github.com/Flexget/Flexget',
-            },
-          },
+          info,
         },
       });
       const result = await pypi.getReleases({
         lookupName: 'flexget',
       });
-      expect(result.sourceUrl).toBe('https://github.com/Flexget/Flexget');
+      expect(result.sourceUrl).toBe(info.project_urls.Repository);
+      expect(result.changelogUrl).toBe(info.project_urls.changelog);
       expect(result).toMatchSnapshot();
     });
     it('returns null if mismatched name', async () => {
diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts
index edc0b7f7ab17f8..311a98d32e6394 100644
--- a/lib/datasource/pypi/index.ts
+++ b/lib/datasource/pypi/index.ts
@@ -1,5 +1,6 @@
 import url from 'url';
 import is from '@sindresorhus/is';
+import changelogFilenameRegex from 'changelog-filename-regex';
 import { parse } from 'node-html-parser';
 import { logger } from '../../logger';
 import { Http } from '../../util/http';
@@ -83,17 +84,34 @@ async function getDependency(
       }
     }
 
-    if (dep.info?.project_urls && !dependency.sourceUrl) {
+    if (dep.info?.project_urls) {
       for (const [name, projectUrl] of Object.entries(dep.info.project_urls)) {
         const lower = name.toLowerCase();
+
         if (
-          lower.startsWith('repo') ||
-          lower === 'code' ||
-          lower === 'source' ||
-          github_repo_pattern.exec(projectUrl)
+          !dependency.sourceUrl &&
+          (lower.startsWith('repo') ||
+            lower === 'code' ||
+            lower === 'source' ||
+            github_repo_pattern.exec(projectUrl))
         ) {
           dependency.sourceUrl = projectUrl;
-          break;
+        }
+
+        if (
+          !dependency.changelogUrl &&
+          ([
+            'changelog',
+            'change log',
+            'changes',
+            'release notes',
+            'news',
+            "what's new",
+          ].includes(lower) ||
+            changelogFilenameRegex.exec(lower))
+        ) {
+          // from https://github.com/pypa/warehouse/blob/418c7511dc367fb410c71be139545d0134ccb0df/warehouse/templates/packaging/detail.html#L24
+          dependency.changelogUrl = projectUrl;
         }
       }
     }