rust_uprev: fetch Rust sources from upstream if not present on local mirror

Previously, fetching sources from upstream, verifying their integrity,
and uploading them to the local mirror was a manual process. With this
change, this happens automatically. The integrity of the sources is
verified using GPG, which checks that the sources have the expected
checksum, that the checksum information was signed by the signing
key used by the Rust project, and that the key and signature are
current (that is, the key has not been revoked and the key and
signature have not expired). The expected key is hardcoded in the
script so that if we ever encounter sources signed by a different
key, the verification will fail and we will have to manually
verify that the new key is legitimate.

BUG=b:271016462
TEST=unittest, use new script to create rust 1.71.1 uprev

Change-Id: I9b2129ed82ca7de9f9aadfd275683f49cb72561a
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/4767736
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Bob Haarman <inglorion@chromium.org>
Commit-Queue: Bob Haarman <inglorion@chromium.org>

2 files changed, 361 insertions, 1 deletions

diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py
index 6d6ca5a4..ae425f93 100755
--- a/rust_tools/rust_uprev.py
+++ b/rust_tools/rust_uprev.py
@@ -47,6 +47,7 @@ from typing import (
     TypeVar,
     Union,
 )
+import urllib.request
 
 from llvm_tools import chroot
 from llvm_tools import git
@@ -75,10 +76,34 @@ class RunStepFn(Protocol):
 
 
 EQUERY = "equery"
-GSUTIL = "gsutil.py"
+GPG = "gpg"
+GSUTIL = "gsutil"
 MIRROR_PATH = "gs://chromeos-localmirror/distfiles"
 EBUILD_PREFIX = Path("/mnt/host/source/src/third_party/chromiumos-overlay")
+# Keyserver to use with GPG. Not all keyservers have Rust's signing key;
+# this must be set to a keyserver that does.
+GPG_KEYSERVER = "keyserver.ubuntu.com"
 RUST_PATH = Path(EBUILD_PREFIX, "dev-lang", "rust")
+# This is the signing key used by upstream Rust as of 2023-08-09.
+# If the project switches to a different key, this will have to be updated.
+# We require the key to be updated manually so that we have an opportunity
+# to verify that the key change is legitimate.
+RUST_SIGNING_KEY = "85AB96E6FA1BE5FE"
+RUST_SRC_BASE_URI = "https://static.rust-lang.org/dist/"
+
+
+class SignatureVerificationError(Exception):
+    """Error that indicates verification of a downloaded file failed.
+
+    Attributes:
+        message: explanation of why the verification failed.
+        path: the path to the file whose integrity was being verified.
+    """
+
+    def __init__(self, message: str, path: Path):
+        super(SignatureVerificationError, self).__init__()
+        self.message = message
+        self.path = path
 
 
 def get_command_output(command: Command, *args, **kwargs) -> str:
@@ -523,11 +548,115 @@ def fetch_rust_distfiles(version: RustVersion) -> None:
     fetch_distfile_from_mirror(compute_rustc_src_name(version))
 
 
+def fetch_rust_src_from_upstream(uri: str, local_path: Path) -> None:
+    """Fetches Rust sources from upstream.
+
+    This downloads the source distribution and the .asc file
+    containing the signatures. It then verifies that the sources
+    have the expected signature and have been signed by
+    the expected key.
+    """
+    subprocess.run(
+        [GPG, "--keyserver", GPG_KEYSERVER, "--recv-keys", RUST_SIGNING_KEY],
+        check=True,
+    )
+    subprocess.run(
+        [GPG, "--keyserver", GPG_KEYSERVER, "--refresh-keys", RUST_SIGNING_KEY],
+        check=True,
+    )
+    asc_uri = uri + ".asc"
+    local_asc_path = Path(local_path.parent, local_path.name + ".asc")
+    logging.info("Fetching %s", uri)
+    urllib.request.urlretrieve(uri, local_path)
+    logging.info("%s fetched", uri)
+
+    # Raise SignatureVerificationError if we cannot get the signature.
+    try:
+        logging.info("Fetching %s", asc_uri)
+        urllib.request.urlretrieve(asc_uri, local_asc_path)
+        logging.info("%s fetched", asc_uri)
+    except Exception as e:
+        raise SignatureVerificationError(
+            f"error fetching signature file {asc_uri}",
+            local_path,
+        ) from e
+
+    # Raise SignatureVerificationError if verifying the signature
+    # failed.
+    try:
+        output = get_command_output(
+            [GPG, "--verify", "--status-fd", "1", local_asc_path]
+        )
+    except subprocess.CalledProcessError as e:
+        raise SignatureVerificationError(
+            f"error verifying signature. GPG output:\n{e.stdout}",
+            local_path,
+        ) from e
+
+    # Raise SignatureVerificationError if the file was not signed
+    # with the expected key.
+    if f"GOODSIG {RUST_SIGNING_KEY}" not in output:
+        message = f"GOODSIG {RUST_SIGNING_KEY} not found in output"
+        if f"REVKEYSIG {RUST_SIGNING_KEY}" in output:
+            message = "signing key has been revoked"
+        elif f"EXPKEYSIG {RUST_SIGNING_KEY}" in output:
+            message = "signing key has expired"
+        elif f"EXPSIG {RUST_SIGNING_KEY}" in output:
+            message = "signature has expired"
+        raise SignatureVerificationError(
+            f"{message}. GPG output:\n{output}",
+            local_path,
+        )
+
+
 def get_distdir() -> str:
     """Returns portage's distdir."""
     return get_command_output(["portageq", "distdir"])
 
 
+def mirror_has_file(name: str) -> bool:
+    """Checks if the mirror has the named file."""
+    mirror_file = MIRROR_PATH + "/" + name
+    cmd: Command = [GSUTIL, "ls", mirror_file]
+    proc = subprocess.run(
+        cmd,
+        check=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        encoding="utf-8",
+    )
+    if "URLs matched no objects" in proc.stdout:
+        return False
+    elif proc.returncode == 0:
+        return True
+
+    raise Exception(
+        "Unexpected result from gsutil ls:"
+        f" rc {proc.returncode} output:\n{proc.stdout}"
+    )
+
+
+def mirror_rust_source(version: RustVersion) -> None:
+    """Ensures source code for a Rust version is on the local mirror.
+
+    If the source code is not found on the mirror, it is fetched
+    from upstream, its integrity is verified, and it is uploaded
+    to the mirror.
+    """
+    filename = compute_rustc_src_name(version)
+    if mirror_has_file(filename):
+        logging.info("%s is present on the mirror", filename)
+        return
+    uri = f"{RUST_SRC_BASE_URI}{filename}"
+    local_path = Path(get_distdir()) / filename
+    mirror_path = f"{MIRROR_PATH}/{filename}"
+    fetch_rust_src_from_upstream(uri, local_path)
+    subprocess.run(
+        [GSUTIL, "cp", "-a", "public-read", local_path, mirror_path],
+        check=True,
+    )
+
+
 def update_manifest(ebuild_file: PathOrStr) -> None:
     """Updates the MANIFEST for the ebuild at the given path."""
     ebuild = Path(ebuild_file)
@@ -659,6 +788,19 @@ def create_rust_uprev(
         return
     template_version, template_ebuild, old_bootstrap_version = prepared
 
+    run_step(
+        "mirror bootstrap sources",
+        lambda: mirror_rust_source(
+            template_version,
+        ),
+    )
+    run_step(
+        "mirror rust sources",
+        lambda: mirror_rust_source(
+            rust_version,
+        ),
+    )
+
     # The fetch steps will fail (on purpose) if the files they check for
     # are not available on the mirror. To make them pass, fetch the
     # required files yourself, verify their checksums, then upload them
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py
index 271557e3..7abe9d11 100755
--- a/rust_tools/rust_uprev_test.py
+++ b/rust_tools/rust_uprev_test.py
@@ -95,6 +95,151 @@ class FetchDistfileTest(unittest.TestCase):
                 )
 
 
+class FetchRustSrcFromUpstreamTest(unittest.TestCase):
+    """Tests for rust_uprev.fetch_rust_src_from_upstream."""
+
+    def setUp(self) -> None:
+        self._mock_get_distdir = start_mock(
+            self,
+            "rust_uprev.get_distdir",
+            return_value="/fake/distfiles",
+        )
+
+        self._mock_gpg = start_mock(
+            self,
+            "subprocess.run",
+            side_effect=self.fake_gpg,
+        )
+
+        self._mock_urlretrieve = start_mock(
+            self,
+            "urllib.request.urlretrieve",
+            side_effect=self.fake_urlretrieve,
+        )
+
+        self._mock_rust_signing_key = start_mock(
+            self,
+            "rust_uprev.RUST_SIGNING_KEY",
+            "1234567",
+        )
+
+    @staticmethod
+    def fake_urlretrieve(src: str, dest: Path) -> None:
+        pass
+
+    @staticmethod
+    def fake_gpg(cmd, **_kwargs):
+        val = mock.Mock()
+        val.returncode = 0
+        val.stdout = ""
+        if "--verify" in cmd:
+            val.stdout = "GOODSIG 1234567"
+        return val
+
+    def test_success(self):
+        with mock.patch("rust_uprev.GPG", "gnupg"):
+            rust_uprev.fetch_rust_src_from_upstream(
+                "fakehttps://rustc-1.60.3-src.tar.gz",
+                Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"),
+            )
+            self._mock_urlretrieve.has_calls(
+                [
+                    (
+                        "fakehttps://rustc-1.60.3-src.tar.gz",
+                        Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"),
+                    ),
+                    (
+                        "fakehttps://rustc-1.60.3-src.tar.gz.asc",
+                        Path("/fake/distfiles/rustc-1.60.3-src.tar.gz.asc"),
+                    ),
+                ]
+            )
+            self._mock_gpg.has_calls(
+                [
+                    (["gnupg", "--refresh-keys", "1234567"], {"check": True}),
+                ]
+            )
+
+    def test_no_signature_file(self):
+        def _urlretrieve(src, dest):
+            if src.endswith(".asc"):
+                raise Exception("404 not found")
+            return self.fake_urlretrieve(src, dest)
+
+        self._mock_urlretrieve.side_effect = _urlretrieve
+
+        with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx:
+            rust_uprev.fetch_rust_src_from_upstream(
+                "fakehttps://rustc-1.60.3-src.tar.gz",
+                Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"),
+            )
+        self.assertIn("error fetching signature file", ctx.exception.message)
+
+    def test_key_expired(self):
+        def _gpg_verify(cmd, *args, **kwargs):
+            val = self.fake_gpg(cmd, *args, **kwargs)
+            if "--verify" in cmd:
+                val.stdout = "EXPKEYSIG 1234567"
+            return val
+
+        self._mock_gpg.side_effect = _gpg_verify
+
+        with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx:
+            rust_uprev.fetch_rust_src_from_upstream(
+                "fakehttps://rustc-1.60.3-src.tar.gz",
+                Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"),
+            )
+        self.assertIn("key has expired", ctx.exception.message)
+
+    def test_key_revoked(self):
+        def _gpg_verify(cmd, *args, **kwargs):
+            val = self.fake_gpg(cmd, *args, **kwargs)
+            if "--verify" in cmd:
+                val.stdout = "REVKEYSIG 1234567"
+            return val
+
+        self._mock_gpg.side_effect = _gpg_verify
+
+        with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx:
+            rust_uprev.fetch_rust_src_from_upstream(
+                "fakehttps://rustc-1.60.3-src.tar.gz",
+                Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"),
+            )
+        self.assertIn("key has been revoked", ctx.exception.message)
+
+    def test_signature_expired(self):
+        def _gpg_verify(cmd, *args, **kwargs):
+            val = self.fake_gpg(cmd, *args, **kwargs)
+            if "--verify" in cmd:
+                val.stdout = "EXPSIG 1234567"
+            return val
+
+        self._mock_gpg.side_effect = _gpg_verify
+
+        with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx:
+            rust_uprev.fetch_rust_src_from_upstream(
+                "fakehttps://rustc-1.60.3-src.tar.gz",
+                Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"),
+            )
+        self.assertIn("signature has expired", ctx.exception.message)
+
+    def test_wrong_key(self):
+        def _gpg_verify(cmd, *args, **kwargs):
+            val = self.fake_gpg(cmd, *args, **kwargs)
+            if "--verify" in cmd:
+                val.stdout = "GOODSIG 0000000"
+            return val
+
+        self._mock_gpg.side_effect = _gpg_verify
+
+        with self.assertRaises(rust_uprev.SignatureVerificationError) as ctx:
+            rust_uprev.fetch_rust_src_from_upstream(
+                "fakehttps://rustc-1.60.3-src.tar.gz",
+                Path("/fake/distfiles/rustc-1.60.3-src.tar.gz"),
+            )
+        self.assertIn("1234567 not found", ctx.exception.message)
+
+
 class FindEbuildPathTest(unittest.TestCase):
     """Tests for rust_uprev.find_ebuild_path()"""
 
@@ -159,6 +304,79 @@ class FindEbuildPathTest(unittest.TestCase):
             self.assertEqual(result, ebuild)
 
 
+class MirrorHasFileTest(unittest.TestCase):
+    """Tests for rust_uprev.mirror_has_file."""
+
+    @mock.patch.object(subprocess, "run")
+    def test_no(self, mock_run):
+        mock_run.return_value = mock.Mock(
+            returncode=1,
+            stdout="CommandException: One or more URLs matched no objects.",
+        )
+        self.assertFalse(rust_uprev.mirror_has_file("rustc-1.69.0-src.tar.gz"))
+
+    @mock.patch.object(subprocess, "run")
+    def test_yes(self, mock_run):
+        mock_run.return_value = mock.Mock(
+            returncode=0,
+            # pylint: disable=line-too-long
+            stdout="gs://chromeos-localmirror/distfiles/rustc-1.69.0-src.tar.gz",
+        )
+        self.assertTrue(rust_uprev.mirror_has_file("rustc-1.69.0-src.tar.gz"))
+
+
+class MirrorRustSourceTest(unittest.TestCase):
+    """Tests for rust_uprev.mirror_rust_source."""
+
+    def setUp(self) -> None:
+        start_mock(self, "rust_uprev.GSUTIL", "gsutil")
+        start_mock(self, "rust_uprev.MIRROR_PATH", "fakegs://fakemirror/")
+        start_mock(
+            self, "rust_uprev.get_distdir", return_value="/fake/distfiles"
+        )
+        self.mock_mirror_has_file = start_mock(
+            self,
+            "rust_uprev.mirror_has_file",
+        )
+        self.mock_fetch_rust_src_from_upstream = start_mock(
+            self,
+            "rust_uprev.fetch_rust_src_from_upstream",
+        )
+        self.mock_subprocess_run = start_mock(
+            self,
+            "subprocess.run",
+        )
+
+    def test_already_present(self):
+        self.mock_mirror_has_file.return_value = True
+        rust_uprev.mirror_rust_source(
+            rust_uprev.RustVersion.parse("1.67.3"),
+        )
+        self.mock_fetch_rust_src_from_upstream.assert_not_called()
+        self.mock_subprocess_run.assert_not_called()
+
+    def test_fetch_and_upload(self):
+        self.mock_mirror_has_file.return_value = False
+        rust_uprev.mirror_rust_source(
+            rust_uprev.RustVersion.parse("1.67.3"),
+        )
+        self.mock_fetch_rust_src_from_upstream.called_once()
+        self.mock_subprocess_run.has_calls(
+            [
+                (
+                    [
+                        "gsutil",
+                        "cp",
+                        "-a",
+                        "public-read",
+                        "/fake/distdir/rustc-1.67.3-src.tar.gz",
+                        "fakegs://fakemirror/rustc-1.67.3-src.tar.gz",
+                    ]
+                ),
+            ]
+        )
+
+
 class RemoveEbuildVersionTest(unittest.TestCase):
     """Tests for rust_uprev.remove_ebuild_version()"""