fix(pypi): Treat yanked versions as deprecated (#6925)

lib/datasource/pypi/__fixtures__/azure-cli-monitor.json

@@ -754,7 +754,8 @@
         "python_version": "source",
         "size": 32541,
         "upload_time": "2018-05-07T17:59:37",
-        "url": "https://files.pythonhosted.org/packages/3a/d6/ec2f81b036f1aefc24bb6bf6336051335c519a9976bd6529636e4c20121c/azure-cli-monitor-0.1.6.tar.gz"
+        "url": "https://files.pythonhosted.org/packages/3a/d6/ec2f81b036f1aefc24bb6bf6336051335c519a9976bd6529636e4c20121c/azure-cli-monitor-0.1.6.tar.gz",
+        "yanked": true
       }
     ],
     "0.1.7": [

lib/datasource/pypi/__fixtures__/versions-html.html

@@ -18,6 +18,6 @@
     <a href="https://files.pythonhosted.org/packages/c8/4b/b23dbcf4c5711f26e2222bb2e300915c9c8d35e643b0af00c2d8f36c9490/dj-database-url-0.4.2.tar.gz#sha256=a6832d8445ee9d788c5baa48aef8130bf61fdc442f7d9a548424d25cd85c9f08">dj-database-url-0.4.2.tar.gz</a><br/>
     <a href="https://files.pythonhosted.org/packages/91/84/50cbfabb91593cff18a37046986f7c2eb69224a694a52ae614711dfa11c6/dj_database_url-0.4.2-py2.py3-none-any.whl#sha256=e16d94c382ea0564c48038fa7fe8d9c890ef1ab1a8ec4cb48e732c124b9482fd">dj_database_url-0.4.2-py2.py3-none-any.whl</a><br/>
     <a href="https://files.pythonhosted.org/packages/01/c4/98fbf678e810029be8078419f7bba626aafa2e81bc38748757db954c477c/dj-database-url-0.5.0.tar.gz#sha256=4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163">dj-database-url-0.5.0.tar.gz</a><br/>
-    <a href="https://files.pythonhosted.org/packages/d4/a6/4b8578c1848690d0c307c7c0596af2077536c9ef2a04d42b00fabaa7e49d/dj_database_url-0.5.0-py2.py3-none-any.whl#sha256=851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9">dj_database_url-0.5.0-py2.py3-none-any.whl</a><br/>
+    <a href="https://files.pythonhosted.org/packages/d4/a6/4b8578c1848690d0c307c7c0596af2077536c9ef2a04d42b00fabaa7e49d/dj_database_url-0.5.0-py2.py3-none-any.whl#sha256=851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9" data-yanked="">dj_database_url-0.5.0-py2.py3-none-any.whl</a><br/>
   </body>
-</html>
+</html>

lib/datasource/pypi/__snapshots__/index.spec.ts.snap

@@ -34,6 +34,7 @@ Object {
       "version": "0.4.2",
     },
     Object {
+      "isDeprecated": true,
       "version": "0.5.0",
     },
   ],
@@ -113,6 +114,7 @@ Object {
       "version": "0.4.2",
     },
     Object {
+      "isDeprecated": true,
       "version": "0.5.0",
     },
   ],
@@ -167,6 +169,7 @@ Object {
       "version": "0.4.2",
     },
     Object {
+      "isDeprecated": true,
       "version": "0.5.0",
     },
   ],
@@ -295,6 +298,7 @@ Object {
       "version": "0.1.5",
     },
     Object {
+      "isDeprecated": true,
       "releaseTimestamp": "2018-05-07T17:59:09",
       "version": "0.1.6",
     },

lib/datasource/pypi/index.ts

@@ -6,7 +6,7 @@ import { Http } from '../../util/http';
 import { ensureTrailingSlash } from '../../util/url';
 import { matches } from '../../versioning/pep440';
 import * as pep440 from '../../versioning/pep440';
-import { GetReleasesConfig, ReleaseResult } from '../common';
+import { GetReleasesConfig, Release, ReleaseResult } from '../common';
 
 export const id = 'pypi';
 export const defaultRegistryUrls = [
@@ -19,7 +19,11 @@ const http = new Http(id);
 
 type Releases = Record<
   string,
-  { requires_python?: boolean; upload_time?: string }[]
+  {
+    requires_python?: boolean;
+    upload_time?: string;
+    yanked?: boolean;
+  }[]
 >;
 type PypiJSON = {
   info: {
@@ -120,10 +124,19 @@ async function getDependency(
   dependency.releases = [];
   if (dep.releases) {
     const versions = compatibleVersions(dep.releases, compatibility);
-    dependency.releases = versions.map((version) => ({
+    dependency.releases = versions.map((version) => {
-      version,
+      const releases = dep.releases[version] || [];
-      releaseTimestamp: (dep.releases[version][0] || {}).upload_time,
+      const { upload_time: releaseTimestamp } = releases[0] || {};
-    }));
+      const isDeprecated = releases.some(({ yanked }) => yanked);
+      const result: Release = {
+        version,
+        releaseTimestamp,
+      };
+      if (isDeprecated) {
+        result.isDeprecated = isDeprecated;
+      }
+      return result;
+    });
   }
   return dependency;
 }
@@ -169,19 +182,19 @@ async function getSimpleDependency(
   }
   const root: HTMLElement = parse(dep.replace(/<\/?pre>/, '')) as any;
   const links = root.querySelectorAll('a');
-  const versions = new Set<string>();
+  const releases = new Map<string, Release>();
   for (const link of Array.from(links)) {
-    const result = extractVersionFromLinkText(link.text, packageName);
+    const version = extractVersionFromLinkText(link.text, packageName);
-    if (result) {
+    if (version) {
-      versions.add(result);
+      const release = releases.get(version) || { version };
+      const isDeprecated = link.hasAttribute('data-yanked');
+      if (isDeprecated) {
+        release.isDeprecated = isDeprecated;
+      }
+      releases.set(version, release);
     }
   }
-  dependency.releases = [];
+  dependency.releases = [...releases.values()];
-  if (versions && versions.size > 0) {
-    dependency.releases = [...versions].map((version) => ({
-      version,
-    }));
-  }
   return dependency;
 }