Guard lock analysis against Pip-cached artifacts. (#2103)

Previously Pex could hit cases where an artifact expected to have been
just downloaded was found in the Pip cache instead and fail to hash the
artifact as a result. Delay hashing in this case to the post-analysis
phase.

Fixes #2098

pex/resolve/locker.py

@@ -344,13 +344,16 @@ def analyze(self, line):
             build_result = self._artifact_build_observer.build_result(line)
             if build_result:
                 artifact_url = build_result.url
+                source_fingerprint = None  # type: Optional[Fingerprint]
+                verified = False
                 if isinstance(artifact_url.scheme, VCSScheme):
                     source_fingerprint, archive_path = fingerprint_downloaded_vcs_archive(
                         download_dir=self._download_dir,
                         project_name=str(build_result.pin.project_name),
                         version=str(build_result.pin.version),
                         vcs=artifact_url.scheme.vcs,
                     )
+                    verified = True
                     selected_path = os.path.basename(archive_path)
                     artifact_url = ArtifactURL.parse(
                         self._vcs_url_manager.normalize_url(artifact_url.raw_url)
@@ -359,14 +362,15 @@ def analyze(self, line):
                 elif isinstance(artifact_url.scheme, ArchiveScheme.Value):
                     selected_path = os.path.basename(artifact_url.path)
                     source_archive_path = os.path.join(self._download_dir, selected_path)
-                    if not os.path.isfile(source_archive_path):
-                        raise AnalyzeError(
-                            "Failed to lock {artifact}. Could not obtain its content for "
-                            "analysis.".format(artifact=artifact_url)
-                        )
-                    digest = Sha256()
-                    hashing.file_hash(source_archive_path, digest)
-                    source_fingerprint = Fingerprint.from_digest(digest)
+                    # If Pip resolves the artifact from its own cache, we will not find it in the
+                    # download dir for this run; so guard against that. In this case the existing
+                    # machinery that finalizes a locks missing fingerprints will download the
+                    # artifact and hash it.
+                    if os.path.isfile(source_archive_path):
+                        digest = Sha256()
+                        hashing.file_hash(source_archive_path, digest)
+                        source_fingerprint = Fingerprint.from_digest(digest)
+                        verified = True
                     self._selected_path_to_pin[selected_path] = build_result.pin
                 elif "file" == artifact_url.scheme:
                     digest = Sha256()
@@ -386,6 +390,7 @@ def analyze(self, line):
                         self._local_projects.add(artifact_url.path)
                         self._saved.add(build_result.pin)
                     source_fingerprint = Fingerprint.from_digest(digest)
+                    verified = True
                 else:
                     raise AnalyzeError(
                         "Unexpected scheme {scheme!r} for artifact at {url}".format(
@@ -401,7 +406,7 @@ def analyze(self, line):
                         requirement=build_result.requirement,
                         pin=build_result.pin,
                         artifact=PartialArtifact(
-                            url=artifact_url, fingerprint=source_fingerprint, verified=True
+                            url=artifact_url, fingerprint=source_fingerprint, verified=verified
                         ),
                         additional_artifacts=tuple(additional_artifacts.values()),
                     )

tests/integration/cli/commands/test_issue_2098.py

@@ -0,0 +1,111 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import json
+import os.path
+from textwrap import dedent
+
+from pex.cli.testing import run_pex3
+from pex.common import safe_open, touch
+from pex.compatibility import commonpath
+from pex.interpreter import PythonInterpreter
+from pex.testing import run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def test_missing_download_lock_analysis_handling(
+    tmpdir,  # type: Any
+    py310,  # type: PythonInterpreter
+):
+    # type: (...) -> None
+
+    my_feast = os.path.join(str(tmpdir), "intermediary")
+    touch(os.path.join(my_feast, "README.rst"))
+    with safe_open(os.path.join(my_feast, "pyproject.toml"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                [build-system]
+                requires = ["setuptools", "wheel"]
+                build-backend = "setuptools.build_meta"
+
+                [project]
+                name = "my_feast"
+                version = "0.0.1"
+                authors = [
+                    {name = "John Sirois", email = "john.sirois@gmail.com"},
+                ]
+                description = "Simulates the more complex and expensive 'feast' in the issue OP."
+                readme = "README.rst"
+                requires-python = ">=3.7"
+                license = {text = "BSD-3-Clause"}
+                dependencies = [
+                    "SQLAlchemy[mypy]>1,<2",
+                ]
+                """
+            )
+        )
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    lock = os.path.join(str(tmpdir), "lock.json")
+    run_pex3(
+        "lock",
+        "create",
+        "--pex-root",
+        pex_root,
+        "--python-path",
+        py310.binary,
+        "--interpreter-constraint",
+        "==3.10.*",
+        "--style",
+        "universal",
+        "--resolver-version",
+        "pip-2020-resolver",
+        "--target-system",
+        "linux",
+        "--target-system",
+        "mac",
+        my_feast,
+        "sqlalchemy==1.3.24",
+        "--indent",
+        "2",
+        "-o",
+        lock,
+    ).assert_success()
+
+    result = run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--lock",
+            lock,
+            "sqlalchemy",
+            "--",
+            "-c",
+            dedent(
+                """\
+                import json
+                import sys
+
+                import sqlalchemy
+
+
+                json.dump(
+                    {"version": sqlalchemy.__version__, "file": sqlalchemy.__file__},
+                    sys.stdout,
+                )
+                """
+            ),
+        ],
+        python=py310.binary,
+    )
+    result.assert_success()
+
+    data = json.loads(result.output)
+    assert "1.3.24" == data["version"]
+    assert pex_root == commonpath([pex_root, data["file"]])