add landoscript

.taskcluster.yml

@@ -81,6 +81,7 @@ tasks:
               - bouncerscript
               - githubscript
               # - iscript (iscript has special release via ronin_puppet repo)
+              - landoscript
               - pushapkscript
               - pushflatpakscript
               - pushmsixscript

landoscript/MANIFEST.in

@@ -0,0 +1,9 @@
+include setup.py
+include landoscript/data/*
+
+recursive-include src *
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+recursive-exclude requirements *
+recursive-exclude tests *

landoscript/docker.d/init_worker.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -o errexit -o pipefail
+
+test_var_set() {
+  local varname=$1
+
+  if [[ -z "${!varname}" ]]; then
+    echo "error: ${varname} is not set"
+    exit 1
+  fi
+}
+
+# TODO: real URLs
+if [ "$ENV" == "prod" ]; then
+  export LANDO_API="https://lando.prod"
+else
+  export LANDO_API="https://lando.dev"
+fi
+
+test_var_set "LANDO_TOKEN"

landoscript/docker.d/worker.yml

@@ -0,0 +1,5 @@
+work_dir: { "$eval": "WORK_DIR" }
+artifact_dir: { "$eval": "ARTIFACTS_DIR" }
+verbose: { "$eval": "VERBOSE == 'true'" }
+lando_api: { "$eval": "LANDO_API" }
+lando_token: { "$eval": "LANDO_TOKEN" }

landoscript/pyproject.toml

@@ -0,0 +1,2 @@
+[tool.coverage.run]
+branch = true

landoscript/requirements/base.in

@@ -0,0 +1,8 @@
+aiohttp
+async-timeout
+gql
+mozilla-version
+moz.l10n
+scriptworker
+tomli
+yarl

landoscript/requirements/local.in

@@ -0,0 +1,3 @@
+-r test.in
+
+tox

landoscript/requirements/local.txt

@@ -0,0 +1,44 @@
+# SHA1:f88f7cbe68a488d5a461e166640111dd30a94bcb
+#
+# This file is autogenerated by pip-compile-multi
+# To update, run:
+#
+#    pip-compile-multi
+#
+-r test.txt
+cachetools==5.5.2 \
+    --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \
+    --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a
+    # via tox
+colorama==0.4.6 \
+    --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
+    --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
+    # via tox
+distlib==0.3.9 \
+    --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \
+    --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403
+    # via virtualenv
+filelock==3.18.0 \
+    --hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 \
+    --hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de
+    # via
+    #   tox
+    #   virtualenv
+platformdirs==4.3.7 \
+    --hash=sha256:a03875334331946f13c549dbd8f4bac7a13a50a895a0eb1e8c6a8ace80d40a94 \
+    --hash=sha256:eb437d586b6a0986388f0d6f74aa0cde27b48d0e3d66843640bfb6bdcdb6e351
+    # via
+    #   tox
+    #   virtualenv
+pyproject-api==1.9.0 \
+    --hash=sha256:326df9d68dea22d9d98b5243c46e3ca3161b07a1b9b18e213d1e24fd0e605766 \
+    --hash=sha256:7e8a9854b2dfb49454fae421cb86af43efbb2b2454e5646ffb7623540321ae6e
+    # via tox
+tox==4.25.0 \
+    --hash=sha256:4dfdc7ba2cc6fdc6688dde1b21e7b46ff6c41795fb54586c91a3533317b5255c \
+    --hash=sha256:dd67f030317b80722cf52b246ff42aafd3ed27ddf331c415612d084304cf5e52
+    # via -r requirements/local.in
+virtualenv==20.30.0 \
+    --hash=sha256:800863162bcaa5450a6e4d721049730e7f2dae07720e0902b0e4040bd6f9ada8 \
+    --hash=sha256:e34302959180fca3af42d1800df014b35019490b119eba981af27f2fa486e5d6
+    # via tox

landoscript/requirements/test.in

@@ -0,0 +1,7 @@
+-r base.in
+
+coverage
+pytest
+pytest-aioresponses
+pytest-asyncio
+pytest-cov

landoscript/requirements/test.txt

@@ -0,0 +1,113 @@
+# SHA1:d3a60c74bb67b74d68d13ca57aed356aceaeaa38
+#
+# This file is autogenerated by pip-compile-multi
+# To update, run:
+#
+#    pip-compile-multi
+#
+-r base.txt
+aioresponses==0.7.8 \
+    --hash=sha256:b73bd4400d978855e55004b23a3a84cb0f018183bcf066a85ad392800b5b9a94 \
+    --hash=sha256:b861cdfe5dc58f3b8afac7b0a6973d5d7b2cb608dd0f6253d16b8ee8eaf6df11
+    # via pytest-aioresponses
+coverage[toml]==7.8.0 \
+    --hash=sha256:042e7841a26498fff7a37d6fda770d17519982f5b7d8bf5278d140b67b61095f \
+    --hash=sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3 \
+    --hash=sha256:0915742f4c82208ebf47a2b154a5334155ed9ef9fe6190674b8a46c2fb89cb05 \
+    --hash=sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25 \
+    --hash=sha256:2931f66991175369859b5fd58529cd4b73582461877ecfd859b6549869287ffe \
+    --hash=sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257 \
+    --hash=sha256:3043ba1c88b2139126fc72cb48574b90e2e0546d4c78b5299317f61b7f718b78 \
+    --hash=sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada \
+    --hash=sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64 \
+    --hash=sha256:4dfd9a93db9e78666d178d4f08a5408aa3f2474ad4d0e0378ed5f2ef71640cb6 \
+    --hash=sha256:52a523153c568d2c0ef8826f6cc23031dc86cffb8c6aeab92c4ff776e7951b28 \
+    --hash=sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067 \
+    --hash=sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733 \
+    --hash=sha256:5a26c0c795c3e0b63ec7da6efded5f0bc856d7c0b24b2ac84b4d1d7bc578d676 \
+    --hash=sha256:5a570cd9bd20b85d1a0d7b009aaf6c110b52b5755c17be6962f8ccd65d1dbd23 \
+    --hash=sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008 \
+    --hash=sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd \
+    --hash=sha256:5c8a5c139aae4c35cbd7cadca1df02ea8cf28a911534fc1b0456acb0b14234f3 \
+    --hash=sha256:6b8af63b9afa1031c0ef05b217faa598f3069148eeee6bb24b79da9012423b82 \
+    --hash=sha256:769773614e676f9d8e8a0980dd7740f09a6ea386d0f383db6821df07d0f08545 \
+    --hash=sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00 \
+    --hash=sha256:77af0f6447a582fdc7de5e06fa3757a3ef87769fbb0fdbdeba78c23049140a47 \
+    --hash=sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501 \
+    --hash=sha256:821f7bcbaa84318287115d54becb1915eece6918136c6f91045bb84e2f88739d \
+    --hash=sha256:89b1f4af0d4afe495cd4787a68e00f30f1d15939f550e869de90a86efa7e0814 \
+    --hash=sha256:8a1d96e780bdb2d0cbb297325711701f7c0b6f89199a57f2049e90064c29f6bd \
+    --hash=sha256:8a40fcf208e021eb14b0fac6bdb045c0e0cab53105f93ba0d03fd934c956143a \
+    --hash=sha256:8f99eb72bf27cbb167b636eb1726f590c00e1ad375002230607a844d9e9a2318 \
+    --hash=sha256:90e7fbc6216ecaffa5a880cdc9c77b7418c1dcb166166b78dbc630d07f278cc3 \
+    --hash=sha256:94ec0be97723ae72d63d3aa41961a0b9a6f5a53ff599813c324548d18e3b9e8c \
+    --hash=sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42 \
+    --hash=sha256:96121edfa4c2dfdda409877ea8608dd01de816a4dc4a0523356067b305e4e17a \
+    --hash=sha256:a1f406a8e0995d654b2ad87c62caf6befa767885301f3b8f6f73e6f3c31ec3a6 \
+    --hash=sha256:a321c61477ff8ee705b8a5fed370b5710c56b3a52d17b983d9215861e37b642a \
+    --hash=sha256:a5761c70c017c1b0d21b0815a920ffb94a670c8d5d409d9b38857874c21f70d7 \
+    --hash=sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487 \
+    --hash=sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4 \
+    --hash=sha256:b44674870709017e4b4036e3d0d6c17f06a0e6d4436422e0ad29b882c40697d2 \
+    --hash=sha256:b571bf5341ba8c6bc02e0baeaf3b061ab993bf372d982ae509807e7f112554e9 \
+    --hash=sha256:b8194fb8e50d556d5849753de991d390c5a1edeeba50f68e3a9253fbd8bf8ccd \
+    --hash=sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73 \
+    --hash=sha256:bbb5cc845a0292e0c520656d19d7ce40e18d0e19b22cb3e0409135a575bf79fc \
+    --hash=sha256:be945402e03de47ba1872cd5236395e0f4ad635526185a930735f66710e1bd3f \
+    --hash=sha256:bf13d564d310c156d1c8e53877baf2993fb3073b2fc9f69790ca6a732eb4bfea \
+    --hash=sha256:cf60dd2696b457b710dd40bf17ad269d5f5457b96442f7f85722bdb16fa6c899 \
+    --hash=sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a \
+    --hash=sha256:d39fc4817fd67b3915256af5dda75fd4ee10621a3d484524487e33416c6f3543 \
+    --hash=sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1 \
+    --hash=sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7 \
+    --hash=sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d \
+    --hash=sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502 \
+    --hash=sha256:e5d2b9be5b0693cf21eb4ce0ec8d211efb43966f6657807f6859aab3814f946b \
+    --hash=sha256:e5ff52d790c7e1628241ffbcaeb33e07d14b007b6eb00a19320c7b8a7024c040 \
+    --hash=sha256:e75a2ad7b647fd8046d58c3132d7eaf31b12d8a53c0e4b21fa9c4d23d6ee6d3c \
+    --hash=sha256:e7ac22a0bb2c7c49f441f7a6d46c9c80d96e56f5a8bc6972529ed43c8b694e27 \
+    --hash=sha256:ed2144b8a78f9d94d9515963ed273d620e07846acd5d4b0a642d4849e8d91a0c \
+    --hash=sha256:f017a61399f13aa6d1039f75cd467be388d157cd81f1a119b9d9a68ba6f2830d \
+    --hash=sha256:f1d8a2a57b47142b10374902777e798784abf400a004b14f1b0b9eaf1e528ba4 \
+    --hash=sha256:f2d32f95922927186c6dbc8bc60df0d186b6edb828d299ab10898ef3f40052fe \
+    --hash=sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323 \
+    --hash=sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883 \
+    --hash=sha256:f9983d01d7705b2d1f7a95e10bbe4091fabc03a46881a256c2787637b087003f \
+    --hash=sha256:fa260de59dfb143af06dcf30c2be0b200bed2a73737a8a59248fcb9fa601ef0f
+    # via
+    #   -r requirements/test.in
+    #   pytest-cov
+iniconfig==2.1.0 \
+    --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \
+    --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760
+    # via pytest
+packaging==24.2 \
+    --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \
+    --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f
+    # via
+    #   aioresponses
+    #   pytest
+pluggy==1.5.0 \
+    --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \
+    --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669
+    # via pytest
+pytest==8.3.5 \
+    --hash=sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820 \
+    --hash=sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845
+    # via
+    #   -r requirements/test.in
+    #   pytest-aioresponses
+    #   pytest-asyncio
+    #   pytest-cov
+pytest-aioresponses==0.3.0 \
+    --hash=sha256:5677b32dfa1a36908b347524b5867aab35ac1c5ce1d4970244d6f66009bca7b6 \
+    --hash=sha256:60f3124ff05a0210a5f369dd95e4cf66090774ba76b322f7178858ce4e6c1647
+    # via -r requirements/test.in
+pytest-asyncio==0.26.0 \
+    --hash=sha256:7b51ed894f4fbea1340262bdae5135797ebbe21d8638978e35d31c6d19f72fb0 \
+    --hash=sha256:c4df2a697648241ff39e7f0e4a73050b03f123f760673956cf0d72a4990e312f
+    # via -r requirements/test.in
+pytest-cov==6.1.0 \
+    --hash=sha256:cd7e1d54981d5185ef2b8d64b50172ce97e6f357e6df5cb103e828c7f993e201 \
+    --hash=sha256:ec55e828c66755e5b74a21bd7cc03c303a9f928389c0563e50ba454a6dbe71db
+    # via -r requirements/test.in

landoscript/setup.py

@@ -0,0 +1,17 @@
+# noqa: D100
+from setuptools import find_packages, setup
+
+setup(
+    name="landoscript",
+    # never changes
+    version="1.0",
+    description="Landoscript scriptworker",
+    author="Mozilla Release Engineering",
+    author_email="release+python@mozilla.com",
+    url="https://github.com/mozilla-releng/scriptworker-scripts",
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    entry_points={"console_scripts": ["landoscript = landoscript.script:main"]},
+    python_requires=">=3.11",
+    license="MPL2",
+)

landoscript/src/landoscript/actions/android_l10n_import.py

@@ -0,0 +1,114 @@
+import logging
+import os.path
+from copy import deepcopy
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Self
+
+import tomli
+
+from landoscript.errors import LandoscriptError
+from landoscript.lando import LandoAction, create_commit_action
+from landoscript.util.diffs import diff_contents
+from landoscript.util.l10n import L10nFile, getL10nFilesFromToml
+from landoscript.util.log import log_file_contents
+from scriptworker_client.github import extract_github_repo_owner_and_name
+from scriptworker_client.github_client import GithubClient
+
+log = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class TomlInfo:
+    toml_path: str
+    dest_path: str
+
+
+@dataclass
+class AndroidL10nImportInfo:
+    from_repo_url: str
+    toml_info: list[TomlInfo]
+
+    @classmethod
+    def from_payload_data(cls, payload_data) -> Self:
+        # copy to avoid modifying the original
+        kwargs = deepcopy(payload_data)
+        kwargs["toml_info"] = [TomlInfo(**ti) for ti in payload_data["toml_info"]]
+        return cls(**kwargs)
+
+
+async def run(
+    github_client: GithubClient, github_config: dict[str, str], public_artifact_dir: str, android_l10n_import_info: AndroidL10nImportInfo, to_branch: str
+) -> LandoAction:
+    log.info("Preparing to import android l10n changesets.")
+
+    l10n_repo_url = android_l10n_import_info.from_repo_url
+    l10n_owner, l10n_repo = extract_github_repo_owner_and_name(l10n_repo_url)
+
+    async with GithubClient(github_config, l10n_owner, l10n_repo) as l10n_github_client:
+        toml_files = [info.toml_path for info in android_l10n_import_info.toml_info]
+        # we always take the tip of the default branch when importing new strings
+        toml_contents = await l10n_github_client.get_files(toml_files)
+        l10n_files: list[L10nFile] = []
+
+        for info in android_l10n_import_info.toml_info:
+            toml_file = info.toml_path
+            log.info(f"processing toml file: {toml_file}")
+
+            if toml_contents[toml_file] is None:
+                raise LandoscriptError(f"toml_file '{toml_file}' is not present in repository")
+
+            contents = tomli.loads(str(toml_contents[toml_file]))
+            src_file_prefix = Path(toml_file).parent
+            dst_file_prefix = Path(info.dest_path)
+            if "**" in contents["paths"][0]["reference"]:
+                # localized file paths contain globs; we need that directory
+                # structure to determine the files we need to fetch
+                force_paths = await l10n_github_client.get_file_listing(str(src_file_prefix))
+            else:
+                force_paths = []
+
+            for src_name in getL10nFilesFromToml(toml_file, contents, force_paths):
+                dst_name = dst_file_prefix / src_name.relative_to(src_file_prefix)
+                l10n_files.append(L10nFile(src_name=str(src_name), dst_name=str(dst_name)))
+
+        # fetch l10n_files from android-l10n
+        src_files = [f.src_name for f in l10n_files]
+        log.info(f"fetching updated files from l10n repository: {src_files}")
+        new_files = await l10n_github_client.get_files(src_files)
+
+        # fetch l10n_files from gecko repo
+        dst_files = [f.dst_name for f in l10n_files]
+        log.info(f"fetching original files from l10n repository: {dst_files}")
+        orig_files = await github_client.get_files(dst_files, branch=to_branch)
+
+        diff = ""
+        for l10n_file in l10n_files:
+            if l10n_file.dst_name not in orig_files:
+                log.warning(f"WEIRD: {l10n_file.dst_name} not in dst_files, continuing anyways...")
+                continue
+
+            if l10n_file.src_name not in new_files:
+                log.warning(f"WEIRD: {l10n_file.src_name} not in src_files, continuing anyways...")
+                continue
+
+            orig_file = orig_files[l10n_file.dst_name]
+            new_file = new_files[l10n_file.src_name]
+            if orig_file == new_file:
+                log.warning(f"old and new contents of {new_file} are the same, skipping bump...")
+                continue
+
+            diff += diff_contents(orig_file, new_file, l10n_file.dst_name)
+
+        if not diff:
+            return {}
+
+        with open(os.path.join(public_artifact_dir, "android-import.diff"), "w+") as f:
+            f.write(diff)
+
+        log.info("adding android l10n import! diff contents are:")
+        log_file_contents(diff)
+
+        # We always ignore closed trees for android l10n imports.
+        commitmsg = f"Import translations from {l10n_repo_url} CLOSED TREE"
+        return create_commit_action(commitmsg, diff)