Andrew/update last known good chrome

CHANGELOG.txt

@@ -1,3 +1,9 @@
+- Update default chrome from 135.0.7011.0/1418433 to 144.0.7527.0/1544685
+- Fix: New chrome takes longer/doesn't populate targets right away, so add a
+  retry loop to populate targets
+- Alter `get_chrome` verbose to print whole JSON
+- Change chrome download path to use XDG cache dir
+- Don't download chrome if we already have that version: add force argument
 - Remove unused system inspection code
 - Add a set of helper functions to await for tab loading and send javascript
 v1.2.1

ROADMAP.md

@@ -1,5 +1,8 @@
 # Roadmap
 
+- [ ] Fix up browser deps error (eliminate in-package analysis)
+- [ ] Switch to process group and kill that to catch all chromium children
+- [ ] Add helpers for running javascript
 - [ ] Working on better diagnostic information
   - [ ] Explain to user when their system has security restrictions
 - [ ] Eliminate synchronous API: it's unused, hard to maintain, and nearly
@@ -15,4 +18,3 @@
 - [ ] Do documentation
 - [ ] Browser Open/Close Status/PipeCheck status should happen at broker level
   - [ ] Broker should probably be opening browser and running watchdog...
-- [ ] Add a connect only for websockets

pyproject.toml

@@ -30,6 +30,7 @@ maintainers = [
 ]
 dependencies = [
   "logistro>=2.0.1",
+  "platformdirs>=4.3.6",
   "simplejson>=3.19.3",
 ]
 
@@ -124,10 +125,12 @@ sequence = ["test_proc", "test_fn"]
 help = "Run all tests quickly"
 
 [tool.poe.tasks.debug-test]
+env = { CHOREO_ENABLE_DEBUG="1" }
 sequence = ["debug-test_proc", "debug-test_fn"]
 help = "Run test by test, slowly, quitting after first error"
 
 [tool.poe.tasks.filter-test]
+env = { CHOREO_ENABLE_DEBUG="1" }
 cmd = "pytest --log-level=1 -W error -vvvx -rA --capture=no --show-capture=no"
 help = "Run any/all tests one by one with basic settings: can include filename and -k filters"
 

src/choreographer/browser_async.py

@@ -30,6 +30,9 @@
     from .browsers._interface_type import BrowserImplInterface
     from .channels._interface_type import ChannelInterface
 
+_N = MAX_POPULATE_LOOPS = 20
+
+
 _logger = logistro.getLogger(__name__)
 
 # Since I added locks to pipes, do we need locks here?
@@ -162,9 +165,15 @@ def run() -> subprocess.Popen[bytes] | subprocess.Popen[str]:  # depends on args
             self._channel.open()  # should this and below be in a broker run
             _logger.debug("Running read loop")
             self._broker.run_read_loop()
-            _logger.debug("Populating Targets")
-            await asyncio.sleep(0)  # let watchdog start
-            await self.populate_targets()
+            await asyncio.sleep(0)  # let watchdog start before populate
+            counter = 0
+            while not self.get_tab():
+                _logger.debug("Populating Targets")
+                await self.populate_targets()
+                await asyncio.sleep(0.1)
+                counter += 1
+                if counter == MAX_POPULATE_LOOPS:
+                    break
         except (BrowserClosedError, BrowserFailedError, asyncio.CancelledError) as e:
             raise BrowserFailedError(
                 "The browser seemed to close immediately after starting.",
@@ -345,7 +354,6 @@ async def populate_targets(self) -> None:
             raise RuntimeError("Could not get targets") from Exception(
                 response["error"],
             )
-
         for json_response in response["result"]["targetInfos"]:
             if (
                 json_response["type"] == "page"

src/choreographer/cli/_cli_utils.py

@@ -23,7 +23,7 @@
 supported_platform_strings = ["linux64", "win32", "win64", "mac-x64", "mac-arm64"]
 
 
-def get_google_supported_platform_string() -> tuple[str, str, str, str]:
+def get_google_supported_platform_string() -> str | None:
     arch_size_detected = "64" if sys.maxsize > 2**32 else "32"
     arch_detected = "arm" if platform.processor() == "arm" else "x"
 
@@ -39,16 +39,17 @@ def get_google_supported_platform_string() -> tuple[str, str, str, str]:
     if chrome_platform_detected in supported_platform_strings:
         platform_string = chrome_platform_detected
 
-    return platform_string, arch_size_detected, platform.processor(), platform.system()
+    return platform_string
 
 
 def get_chrome_download_path() -> Path | None:
-    _chrome_platform_detected, _, _, _ = get_google_supported_platform_string()
+    _chrome_platform_detected = get_google_supported_platform_string()
 
     if not _chrome_platform_detected:
         return None
 
-    _default_exe_path = Path()
+    _default_exe_path = default_download_path
+    _default_exe_path.mkdir(parents=True, exist_ok=True)
 
     if platform.system().startswith("Linux"):
         _default_exe_path = (
@@ -85,47 +86,49 @@ def _extract_member(self, member, targetpath, pwd):  # type: ignore [no-untyped-
         return path
 
 
-def get_chrome_sync(  # noqa: PLR0912, C901
+def get_chrome_sync(  # noqa: C901, PLR0912
     arch: str | None = None,
     i: int | None = None,
     path: str | Path = default_download_path,
     *,
     verbose: bool = False,
+    force: bool = False,
 ) -> Path | str:
     """Download chrome synchronously: see `get_chrome()`."""
-    if not arch:
-        arch, _, _, _ = get_google_supported_platform_string()
+    if isinstance(path, str):
+        path = Path(path)
 
+    arch = arch or get_google_supported_platform_string()
     if not arch:
         raise RuntimeError(
             "You must specify an arch, one of: "
             f"{', '.join(supported_platform_strings)}. "
             f"Detected {arch} is not supported.",
         )
 
-    if isinstance(path, str):
-        path = Path(path)
     if i:
         _logger.info("Loading chrome from list")
+        raw_json = urllib.request.urlopen(  # noqa: S310 audit url for schemes
+            _chrome_for_testing_url,
+        ).read()
         browser_list = json.loads(
-            urllib.request.urlopen(  # noqa: S310 audit url for schemes
-                _chrome_for_testing_url,
-            ).read(),
+            raw_json,
         )
         version_obj = browser_list["versions"][i]
+        raw_json = json.dumps(version_obj)
     else:
         _logger.info("Using last known good version of chrome")
-        with (
+        raw_json = (
             Path(__file__).resolve().parent.parent
             / "resources"
             / "last_known_good_chrome.json"
-        ).open() as f:
-            version_obj = json.load(f)
-    if verbose:
-        print(version_obj["version"])  # noqa: T201 allow print in cli
-        print(version_obj["revision"])  # noqa: T201 allow print in cli
+        ).read_text()
+        version_obj = json.loads(
+            raw_json,
+        )
+    version_string = f"{version_obj['version']}\n{version_obj['revision']}"
     chromium_sources = version_obj["downloads"]["chrome"]
-    url = ""
+
     for src in chromium_sources:
         if src["platform"] == arch:
             url = src["url"]
@@ -137,19 +140,16 @@ def get_chrome_sync(  # noqa: PLR0912, C901
             f"{arch} is not supported.",
         )
 
-    if not path.exists():
-        path.mkdir(parents=True)
-    filename = path / "chrome.zip"
-    with urllib.request.urlopen(url) as response, filename.open("wb") as out_file:  # noqa: S310 audit url
-        shutil.copyfileobj(response, out_file)
-    with _ZipFilePermissions(filename, "r") as zip_ref:
-        zip_ref.extractall(path)
-    filename.unlink()
+    if verbose:
+        print(raw_json)  # noqa: T201 allow print in cli
+    version_tag = path / "version_tag.txt"
+
+    path.mkdir(parents=True, exist_ok=True)
 
     if arch.startswith("linux"):
-        exe_name = path / f"chrome-{arch}" / "chrome"
+        exe_path = path / f"chrome-{arch}" / "chrome"
     elif arch.startswith("mac"):
-        exe_name = (
+        exe_path = (
             path
             / f"chrome-{arch}"
             / "Google Chrome for Testing.app"
@@ -158,10 +158,37 @@ def get_chrome_sync(  # noqa: PLR0912, C901
             / "Google Chrome for Testing"
         )
     elif arch.startswith("win"):
-        exe_name = path / f"chrome-{arch}" / "chrome.exe"
+        exe_path = path / f"chrome-{arch}" / "chrome.exe"
     else:
         raise RuntimeError("Couldn't calculate exe_name, unsupported architecture.")
-    return exe_name
+
+    if (
+        exe_path.exists()
+        and version_tag.is_file()
+        and version_tag.read_text() == version_string
+        and not force
+    ):
+        return exe_path
+    else:
+        if exe_path.exists():  # delete it
+            if exe_path.is_dir():
+                shutil.rmtree(exe_path)
+            else:
+                exe_path.unlink()
+            # It really should always be a dir but in testing we fake it
+        if version_tag.exists():  # delete it
+            version_tag.unlink()
+
+    # Download
+    zip_path = path / "chrome.zip"
+    with urllib.request.urlopen(url) as response, zip_path.open("wb") as out_file:  # noqa: S310 audit url
+        shutil.copyfileobj(response, out_file)
+    with _ZipFilePermissions(zip_path, "r") as zip_ref:
+        zip_ref.extractall(path)
+    zip_path.unlink()
+    version_tag.write_text(version_string)
+
+    return exe_path
 
 
 async def get_chrome(
@@ -170,6 +197,7 @@ async def get_chrome(
     path: str | Path = default_download_path,
     *,
     verbose: bool = False,
+    force: bool = False,
 ) -> Path | str:
     """
     Download google chrome from google-chrome-for-testing server.
@@ -180,10 +208,18 @@ async def get_chrome(
            still in the testing directory.
         path: where to download it too (the folder).
         verbose: print out version found
+        force: download chrome again even if already present at that version
 
     """
     loop = asyncio.get_running_loop()
-    fn = partial(get_chrome_sync, arch=arch, i=i, path=path, verbose=verbose)
+    fn = partial(
+        get_chrome_sync,
+        arch=arch,
+        i=i,
+        path=path,
+        verbose=verbose,
+        force=force,
+    )
     return await loop.run_in_executor(
         executor=None,
         func=fn,
@@ -231,12 +267,21 @@ def get_chrome_cli() -> None:
         action="store_true",
         help="Display found version number if using -i (to stdout)",
     )
+    parser.add_argument(
+        "-f",
+        "--force",
+        dest="force",
+        action="store_true",
+        default=False,
+        help="Force download even if already present.",
+    )
     parser.set_defaults(path=default_download_path)
     parser.set_defaults(arch=None)
     parser.set_defaults(verbose=False)
     parsed = parser.parse_args()
     i = parsed.i
     arch = parsed.arch
     path = Path(parsed.path)
+    force = parsed.force
     verbose = parsed.verbose
-    print(get_chrome_sync(arch=arch, i=i, path=path, verbose=verbose))  # noqa: T201 allow print in cli
+    print(get_chrome_sync(arch=arch, i=i, path=path, verbose=verbose, force=force))  # noqa: T201 allow print in cli

src/choreographer/cli/defaults.py

@@ -2,5 +2,12 @@
 
 from pathlib import Path
 
-default_download_path = Path(__file__).resolve().parent / "browser_exe"
+from platformdirs import PlatformDirs
+
+default_download_path = (
+    Path(
+        PlatformDirs("choreographer", "plotly").user_data_dir,
+    )
+    / "deps"
+)
 """The path where we download chrome if no path argument is supplied."""

src/choreographer/resources/last_known_good_chrome.json

@@ -1,72 +1 @@
-{
-    "version": "135.0.7011.0",
-    "revision": "1418433",
-    "downloads": {
-        "chrome": [
-            {
-                "platform": "linux64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/linux64/chrome-linux64.zip"
-            },
-            {
-                "platform": "mac-arm64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/mac-arm64/chrome-mac-arm64.zip"
-            },
-            {
-                "platform": "mac-x64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/mac-x64/chrome-mac-x64.zip"
-            },
-            {
-                "platform": "win32",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/win32/chrome-win32.zip"
-            },
-            {
-                "platform": "win64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/win64/chrome-win64.zip"
-            }
-        ],
-        "chromedriver": [
-            {
-                "platform": "linux64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/linux64/chromedriver-linux64.zip"
-            },
-            {
-                "platform": "mac-arm64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/mac-arm64/chromedriver-mac-arm64.zip"
-            },
-            {
-                "platform": "mac-x64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/mac-x64/chromedriver-mac-x64.zip"
-            },
-            {
-                "platform": "win32",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/win32/chromedriver-win32.zip"
-            },
-            {
-                "platform": "win64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/win64/chromedriver-win64.zip"
-            }
-        ],
-        "chrome-headless-shell": [
-            {
-                "platform": "linux64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/linux64/chrome-headless-shell-linux64.zip"
-            },
-            {
-                "platform": "mac-arm64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/mac-arm64/chrome-headless-shell-mac-arm64.zip"
-            },
-            {
-                "platform": "mac-x64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/mac-x64/chrome-headless-shell-mac-x64.zip"
-            },
-            {
-                "platform": "win32",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/win32/chrome-headless-shell-win32.zip"
-            },
-            {
-                "platform": "win64",
-                "url": "https://storage.googleapis.com/chrome-for-testing-public/135.0.7011.0/win64/chrome-headless-shell-win64.zip"
-            }
-        ]
-    }
-}
+{"version": "144.0.7527.0", "revision": "1544685", "downloads": {"chrome": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chrome-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chrome-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chrome-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chrome-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chrome-win64.zip"}], "chromedriver": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chromedriver-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chromedriver-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chromedriver-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chromedriver-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chromedriver-win64.zip"}], "chrome-headless-shell": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chrome-headless-shell-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chrome-headless-shell-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chrome-headless-shell-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chrome-headless-shell-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chrome-headless-shell-win64.zip"}]}}