diff --git a/Makefile b/Makefile index 8871d9b..fb0966f 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ clean: .PHONY: init init: pip install --upgrade uv - uv pip install --upgrade pip tox pre-commit + uv pip install --upgrade pip tox tox-uv pre-commit uv pip install --editable . uv pip install -r requirements/main.txt -r requirements/dev.txt rm -rf .tox @@ -25,7 +25,7 @@ update: update-deps init .PHONY: update-deps update-deps: pip install --upgrade uv - uv pip install pre-commit + uv pip install pre-commit tox-uv pre-commit autoupdate uv pip compile --upgrade --generate-hashes \ --output-file requirements/main.txt requirements/main.in diff --git a/pyproject.toml b/pyproject.toml index 3c3b843..56fe56b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ dynamic = ["version"] name = "Association of Universities for Research in Astronomy, Inc. (AURA)" email = "sqre-admin@lists.lsst.org" +[project.scripts] +runlab = "lsst.rsp.startup.cli:main" + [project.urls] Homepage = "https://rsp.lsst.io/" Source = "https://github.com/lsst-sqre/lsst-rsp" @@ -92,6 +95,7 @@ exclude_lines = [ [tool.mypy] disallow_untyped_defs = true disallow_incomplete_defs = true +exclude = "tests/support/files" ignore_missing_imports = true local_partial_types = true no_implicit_reexport = true @@ -123,6 +127,9 @@ python_files = ["tests/*.py", "tests/*/*.py"] [tool.ruff] line-length = 79 target-version = "py311" +exclude = [ + "tests/support/files/*" +] [tool.ruff.lint] ignore = [ diff --git a/requirements/dev.txt b/requirements/dev.txt index 7e8c2a3..fe9de22 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -305,9 +305,9 @@ types-deprecated==1.2.9.20240311 \ types-requests==2.31.0.20240403 \ --hash=sha256:06abf6a68f5c4f2a62f6bb006672dfb26ed50ccbfddb281e1ee6f09a65707d5d \ --hash=sha256:e1e0cd0b655334f39d9f872b68a1310f0e343647688bf2cee932ec4c2b04de59 -typing-extensions==4.10.0 \ - --hash=sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475 \ - --hash=sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb +typing-extensions==4.11.0 \ + --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \ + --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a # via mypy urllib3==2.2.1 \ --hash=sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d \ diff --git a/requirements/main.in b/requirements/main.in index 42ce57d..90d3e3a 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -9,3 +9,4 @@ Deprecated IPython pyvo structlog +symbolicmode diff --git a/requirements/main.txt b/requirements/main.txt index 73c219c..89243c8 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -205,9 +205,9 @@ packaging==24.0 \ --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 # via astropy -parso==0.8.3 \ - --hash=sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0 \ - --hash=sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75 +parso==0.8.4 \ + --hash=sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18 \ + --hash=sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d # via jedi pexpect==4.9.0 \ --hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \ @@ -225,18 +225,18 @@ pure-eval==0.2.2 \ --hash=sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350 \ --hash=sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3 # via stack-data -pyerfa==2.0.1.1 \ - --hash=sha256:08b5abb90b34e819c1fca69047a76c0d344cb0c8fe4f7c8773f032d8afd623b4 \ - --hash=sha256:0e95cf3d11f76f473bf011980e9ea367ca7e68ca675d8b32499814fb6e387d4c \ - --hash=sha256:1c0c1efa701cab986aa58d03c58f77e47ea1898bff2684377d29580a055f836a \ - --hash=sha256:1ce322ac30673c2aeb0ee22ced4938c1e9e26db0cbe175912a213aaff42383df \ - --hash=sha256:1db85db72ab352da6ffc790e41209d8f41feb5b175d682cf1f0e3e60e9e5cdf8 \ - --hash=sha256:30649047b7a8ce19f43e4d18a26b8a44405a6bb406df16c687330a3b937723b2 \ - --hash=sha256:34ee545780246fb0d1d3f7e46a6daa152be06a26b2d27fbfe309cab9ab488ea7 \ - --hash=sha256:67dfc00dcdea87a9b3c0bb4596fb0cfb54ee9c1c75fdcf19411d1029a18f6eec \ - --hash=sha256:94df7566ce5a5abb14e2dd1fe879204390639e9a76383ec27f10598eb24be760 \ - --hash=sha256:c50b7cdb005632931b7b56a679cf25361ed6b3aa7c21e491e65cc89cb337e66a \ - --hash=sha256:dbac74ef8d3d3b0f22ef0ad3bbbdb30b2a9e10570b1fa5a98be34c7be36c9a6b +pyerfa==2.0.1.3 \ + --hash=sha256:053ed25fdb7deb9d3d7cebecbb3d3dfbeea37c8c0011cc0616293e03d2c308eb \ + --hash=sha256:359327c88f1e5dea3974b284dabef141824ac54753c5cab6b3f23acd9d52071b \ + --hash=sha256:58c3a971a9fba8663b49dcc54c3419e837837140d81cc6be9f1c21fc56322f7b \ + --hash=sha256:60c0a73db5a42927fbafd12c623699c2c1b1233b6e1be1963970a5ad47e463c4 \ + --hash=sha256:779caac3737da68f4db43b0dec026ac479719e02d25b8c4e7b0756abadbcd416 \ + --hash=sha256:b0f621f26b5f31b3fb6bb113fb48a428e56eb00c7d729a242672dc4f886c8d18 \ + --hash=sha256:b7a85ac9d807ea71550e831e873916ed3a44300fe6e20e0b3ca0f2784c0b2757 \ + --hash=sha256:ba5eb932341beaf222726de8dce2b1645c97b48c321efb2af8a535a7eb90ebfa \ + --hash=sha256:ef6c5d2206f134bd95329a0c17d46c449c9b68e9828e97e9bc43b29cd8789f5d \ + --hash=sha256:f4472d2a2622e47d220a9436c953a487d8c051157f7b44b1f71964de17ee443b \ + --hash=sha256:fc554151de564b567e391b7c9c3b545efac63674ab1954382d38f886254c01fb # via astropy pygments==2.17.2 \ --hash=sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c \ @@ -313,16 +313,15 @@ stack-data==0.6.3 \ structlog==24.1.0 \ --hash=sha256:3f6efe7d25fab6e86f277713c218044669906537bb717c1807a09d46bca0714d \ --hash=sha256:41a09886e4d55df25bdcb9b5c9674bccfab723ff43e0a86a1b7b236be8e57b16 +symbolicmode==2.0.1 \ + --hash=sha256:1f88547b48eb7551c19b459b771b8f4de27f0109dedf68c7f0aea4caeadf85d1 \ + --hash=sha256:9f3e4bfa7ad0709fd3b21cfb0a6e6ef9d842eca29460e4796b5e65e72d49ba55 traitlets==5.14.2 \ --hash=sha256:8cdd83c040dab7d1dee822678e5f5d100b514f7b72b01615b26fc5718916fdf9 \ --hash=sha256:fcdf85684a772ddeba87db2f398ce00b40ff550d1528c03c14dbf6a02003cd80 # via # ipython # matplotlib-inline -typing-extensions==4.10.0 \ - --hash=sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475 \ - --hash=sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb - # via ipython urllib3==2.2.1 \ --hash=sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d \ --hash=sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19 diff --git a/src/lsst/rsp/startup/cli.py b/src/lsst/rsp/startup/cli.py new file mode 100644 index 0000000..94a8def --- /dev/null +++ b/src/lsst/rsp/startup/cli.py @@ -0,0 +1,11 @@ +"""Launcher for the Lab Runner.""" + +from .services.labrunner import LabRunner + + +def main() -> None: + """Make a LabRunner and call its single public method. All settings are + in the environment. + """ + lr = LabRunner() + lr.go() diff --git a/src/lsst/rsp/startup/constants.py b/src/lsst/rsp/startup/constants.py index d636755..641c7cc 100644 --- a/src/lsst/rsp/startup/constants.py +++ b/src/lsst/rsp/startup/constants.py @@ -4,19 +4,19 @@ __all__ = [ "app_name", + "etc", "logging_checksums", "max_number_outputs", "noninteractive_config", - "profile_path", "top_dir", ] app_name = "nublado" +etc = Path("/etc") logging_checksums = [ "2997fe99eb12846a1b724f0b82b9e5e6acbd1d4c29ceb9c9ae8f1ef5503892ec" ] max_number_outputs = 10000 -profile_path = Path("/etc/profile.d/local05-path.sh") top_dir = Path("/opt/lsst/software") noninteractive_config = Path( top_dir / "jupyterlab" / "noninteractive" / "command" / "command.json" diff --git a/src/lsst/rsp/startup/services/labrunner.py b/src/lsst/rsp/startup/services/labrunner.py index 80a87b9..b706a5f 100644 --- a/src/lsst/rsp/startup/services/labrunner.py +++ b/src/lsst/rsp/startup/services/labrunner.py @@ -7,22 +7,21 @@ import json import os import sys -import tempfile from pathlib import Path -from shlex import join from time import sleep from typing import Any from urllib.parse import urlparse import structlog +import symbolicmode from ... import get_access_token, get_digest from ..constants import ( app_name, + etc, logging_checksums, max_number_outputs, noninteractive_config, - profile_path, top_dir, ) from ..models.noninteractive import NonInteractiveExecution @@ -60,38 +59,13 @@ def go(self) -> None: # There's an argument for leaving it there forever, in case the # user has messed up their Python environment so bad that they # can't even start a Python interpreter. - # - # self._reset_user_env() # noqa: ERA001 - # - # Check to see whether LOADRSPSTACK is set and force it if it is - # not. We need this to source the correct file in the next step. - # - # As above, in order to bootstrap ourselves to a running state, we - # need a shell shim, so we need to figure this out in the shim too. - # - # self._ensure_loadrspstack() # noqa: ERA001 - # - # Check to see whether we are running within the stack, and do a - # complicated re-exec dance if we are not. Modular so we can rip - # this out when we are no longer using the stack Python. - # - # Since the stack container only provides Python3 within the DM - # stack...we have to do this in the shell launcher too. - # - # Eventually all of this should move into this module...although - # by the time we have an external Python 3, we will no longer need - # this part. - # - # self._ensure_environment() # noqa: ERA001 - - # Set up the (complicated) environment for the JupyterLab process self._configure_env() # Modify files. If $HOME is not mounted and writeable, things will # go wrong here. self._modify_files() - # Set up git parameters and git-lfs + # Check out notebooks, set up git parameters and git-lfs self._setup_git() # Clear EUPS cache @@ -101,133 +75,6 @@ def go(self) -> None: # things that change between those two, and launch the Lab self._launch() - def _reset_user_env(self) -> None: - # This is how the reset eventually should be done. However, - # since we still have to start from a shell script...we will - # reset the environment in the shell script too. - if not str_bool(os.environ.get("RESET_USER_ENV", "")): - self._logger.debug("User environment reset not requested") - return - self._logger.debug("User environment reset requested") - now = datetime.datetime.now(datetime.UTC).isoformat() - reloc = self._home / f".user_env.{now}" - reloc.mkdir() # Fail it it already exists--that would be weird - moved = False - # Top-level (relative to $HOME) dirs - for item in ("cache", "conda", "local", "jupyter"): - dir_base = Path(f".{item}") - dir_full = self._home / dir_base - if dir_full.is_dir(): - dir_full.rename(reloc / dir_base) - moved = True - # Files, not necessarily at top level - u_setups = self._home / "notebooks" / ".user_setups" - if u_setups.is_file(): - (reloc / "notebooks").mkdir() - u_setups.rename(reloc / "notebooks" / "user_setups") - moved = True - if moved: - self._logger.debug(f"Relocated files to {reloc!s}") - self._logger.debug("Restarting with cleaned-up filespace") - # We're about to re-exec: we don't want to keep looping. - del self._env["RESET_USER_ENV"] - self._ensure_loadrspstack() - # We cheat a bit here. Sourcing the stack setup twice is - # (I believe) harmless, and this is a fast way to re-exec - # thus ensuring a cleaner Python environment - if "LSST_CONDA_ENV_NAME" in self._env: - del self._env["LSST_CONDA_ENV_NAME"] - # This will re-exec so we get a new Python process - self._ensure_environment() - else: - self._logger.debug("No user files needed relocation") - # Nothing was actually moved, so throw away the directory. - reloc.rmdir() - - def _ensure_loadrspstack(self) -> None: - # This is not currently useful. At the moment we need to use the - # python3 inside the stack. We would like to break the Python we use - # to run JupyterLab itself apart from the stack environment. - # - # However, although the approach in this method would work *if* - # there were already an os-supplied python3 in the source container, - # (which is to say the DM stack container that the RSP is built from) - # there isn't. The only Python 3 in there is the one inside the - # stack. - # - # So, in short, we're going to have to launch the lab runner from a - # shell script that sources the stack anyway. - self._logger.debug("Ensuring that LOADRSPSTACK is set") - if "LOADRSPSTACK" in self._env: - self._logger.debug( - f"LOADRSPSTACK was set to '{self._env['LOADRSPSTACK']}'" - ) - return - rspstack = top_dir / "rspstack" / "loadrspstack.bash" - if not rspstack.is_file(): - rspstack = top_dir / "stack" / "loadLSST.bash" - self._env["LOADRSPSTACK"] = str(rspstack) - self._logger.debug(f"Newly set LOADRSPSTACK to {rspstack!s}") - - def _ensure_environment(self) -> None: - """If we are not running from within the stack environment, - restart from within it. - """ - # Currently the JupyterLab machinery relies on the stack - # Python to run. While this is expected to change, it has not - # yet, so... we test for an environment variable that, unless - # the user is extraordinarily perverse, will only be set in - # the stack environment. - # - # It's ``LSST_CONDA_ENV_NAME``. - # - # If we don't find it, we create an executable shell file that - # sets up the stack environment and then reruns the current - # command with its arguments. We know we have ``/bin/bash`` - # in the container, so we use that as the shell and use - # ``loadLSST.bash`` to set up the environment. We also add - # some paths set up in the profile. Then we use exec() in the - # shell script to reinvoke the Python process exactly as it - # was initially called. - # - # Finally we os.execl() that file, replacing this process with - # that one, which will bring us right back here, but with the - # stack initialized. We do leave the file sitting around, but - # since we're creating it as a temporary file, that's OK: it's - # a few dozen bytes, and it will go away when the container - # does. - - if os.environ.get("LSST_CONDA_ENV_NAME"): - self._logger.debug( - "LSST_CONDA_ENV_NAME is set: stack Python assumed" - ) - # All is well. - return - self._logger.debug( - "LSST_CONDA_ENV_NAME not set; must re-exec with stack Python" - ) - tf = tempfile.NamedTemporaryFile(mode="w", delete=False) - tfp = Path(tf.name) - tfp.write_text( - "#!/bin/bash\n" - f". {self._env['LOADRSPSTACK']}\n" - f". {profile_path!s}\n" - f"exec {join(sys.argv)}\n" - ) - # Make it executable - tfp.chmod(0o700) - # Ensure it's flushed to disk - os.sync() - # Run it - self._logger.debug(f"About to re-exec: running {tfp!s}") - os.execl(tf.name, tfp.name) - - # - # After all of that, if we get down here, we are running with our - # necessary stack environment set up and the homedir environment - # relocated if we asked for that. - # - # # Next up, a big block of setting up our subprocess environment. # @@ -436,8 +283,6 @@ def _modify_files(self) -> None: self._copy_dircolors() # Copy contents of /etc/skel self._copy_etc_skel() - # Refresh standard notebooks - self._refresh_notebooks() def _copy_butler_credentials(self) -> None: if ( @@ -533,7 +378,7 @@ def _copy_dircolors(self) -> None: self._logger.debug("Copying dircolors if needed") if not (self._home / ".dir_colors").exists(): self._logger.debug("Copying dircolors") - dc = Path("/etc/dircolors.ansi-universal") + dc = etc / "dircolors.ansi-universal" dc_txt = dc.read_text() (self._home / ".dir_colors").write_text(dc_txt) else: @@ -541,8 +386,7 @@ def _copy_dircolors(self) -> None: def _copy_etc_skel(self) -> None: self._logger.debug("Copying files from /etc/skel if they don't exist") - es_str = "/etc/skel" - es = Path(es_str) + es = etc / "skel" # alas, Path.walk() requires Python 3.12, which isn't in the # stack containers yet. contents = os.walk(es) @@ -554,10 +398,10 @@ def _copy_etc_skel(self) -> None: dirs = [Path(x) for x in entry[1]] files = [Path(x) for x in entry[2]] # Determine what the destination directory should be - if entry[0] == es_str: + if entry[0] == str(es): current_dir = self._home else: - current_dir = self._home / entry[0][(len(es_str) + 1) :] + current_dir = self._home / entry[0][(len(str(es)) + 1) :] # For each directory in the tree at this level: # if we don't already have one in our directory, make it. for d_item in dirs: @@ -574,11 +418,22 @@ def _copy_etc_skel(self) -> None: src_contents = src.read_bytes() (current_dir / f_item).write_bytes(src_contents) + def _setup_git(self) -> None: + # Refresh standard notebooks + self._refresh_notebooks() + # Set up email and name + self._set_git_email_and_name() + # Set up Git LFS + self._setup_gitlfs() + def _refresh_notebooks(self) -> None: # Find the notebook specs. I think we can ditch our fallbacks now. self._logger.debug("Refreshing notebooks") urls = self._env.get("AUTO_REPO_SPECS", "") url_l = urls.split(",") + if not url_l: + self._logger.debug("No repos listed in 'AUTO_REPO_SPECS'") + return # Specs should include the branch too. default_branch = self._env.get("AUTO_REPO_BRANCH", "prod") now = datetime.datetime.now(datetime.UTC).isoformat() @@ -589,6 +444,7 @@ def _refresh_notebooks(self) -> None: repo, branch = url.split("@", maxsplit=1) except ValueError: branch = default_branch + repo = url repo_path = urlparse(repo).path repo_name = Path(repo_path).name if repo_name.endswith(".git"): @@ -603,6 +459,7 @@ def _refresh_notebooks(self) -> None: # If not, probably a lot else has already gone wrong. can_write = perms & 0o222 if can_write: + self._logger.debug(f"'{dirname!s}' is writeable; moving") newname = Path(f"{dirname!s}.{now}") reloc_msg += f"* '{dirname!s}' -> '{newname!s}'\n" # We're also going to assume the user DOES have write @@ -613,48 +470,15 @@ def _refresh_notebooks(self) -> None: # If the repository exists and is not writeable, and has # the same last commit as the remote, then we don't # need to update it. - cwd = Path.cwd() - # - # Git wants you to be in the working tree - os.chdir(dirname) - rx = run( - "git", - "rev-parse", - "HEAD", - timeout=timeout, - logger=self._logger, - ) - local_sha = rx.stdout if rx else "" - rx = run( - "git", - "config", - "--get", - "remote.origin.url", - timeout=timeout, - logger=self._logger, - ) - remote = rx.stdout if rx else "" - rx = run( - "git", - "config", - "--get", - "ls-remote", - remote, - timeout=timeout, - logger=self._logger, - ) - ls_remote = rx.stdout if rx else "" - lsr_lines = ls_remote.split(ls_remote) - remote_sha = "" - for line in lsr_lines: - if line.endswith(f"refs/heads/{branch}"): - remote_sha = line.split()[0] - break - # Get outta there - os.chdir(cwd) - if local_sha and remote_sha != local_sha: + if self._compare_local_and_remote( + dirname, branch, timeout + ): + self._logger.debug(f"'{dirname!s}' is r/o and current") continue # Up-to-date; we don't need to do anything - self._recursive_make_writeable_and_remove(dirname) + # It's writeable or stale; re-clone. + self._logger.debug(f"Need to remove '{dirname!s}'") + symbolicmode.chmod(dirname, "u+w") + self._recursive_remove(dirname) # If the directory existed, it's gone now. self._logger.debug(f"Cloning {repo}@{branch}") run( @@ -669,7 +493,7 @@ def _refresh_notebooks(self) -> None: timeout=timeout, logger=self._logger, ) - self._recursive_make_readonly(dirname) + symbolicmode.chmod(dirname, "a-w", recurse=True) if reloc_msg: hdr = ( "# Directory relocation\n\n" @@ -684,41 +508,67 @@ def _refresh_notebooks(self) -> None: self._logger.debug("Refreshed notebooks") - def _recursive_make_writeable_and_remove(self, tgt: Path) -> None: + def _compare_local_and_remote( + self, path: Path, branch: str, timeout: int + ) -> bool: + # Returns True if git repo checked out to path has the same + # commit hash as the remote. + # + # Git wants you to be in the working tree + with contextlib.chdir(path): + rx = run( + "git", + "rev-parse", + "HEAD", + timeout=timeout, + logger=self._logger, + ) + local_sha = rx.stdout.strip() if rx else "" + rx = run( + "git", + "config", + "--get", + "remote.origin.url", + timeout=timeout, + logger=self._logger, + ) + remote = rx.stdout.strip() if rx else "" + rx = run( + "git", + "ls-remote", + remote, + timeout=timeout, + logger=self._logger, + ) + ls_remote = rx.stdout.strip() if rx else "" + lsr_lines = ls_remote.split("\n") + remote_sha = "" + for line in lsr_lines: + line.strip() + if line.endswith(f"\trefs/heads/{branch}"): + remote_sha = line.split()[0] + break + self._logger.debug(f"local /remote SHA: {local_sha}/{remote_sha}") + return local_sha == remote_sha + + def _recursive_remove(self, tgt: Path) -> None: # You can't rmdir() a directory with contents, so... if not tgt.is_dir(): self._logger.warning(f"Removal of non-directory {tgt!s} requested") return - self._logger.debug("Removing directory {str(tgt)}") + self._logger.debug(f"Removing directory {tgt!s}") contents = tgt.glob("*") for item in contents: if item.is_dir(): - self._recursive_make_writeable_and_remove(item) + self._recursive_remove(item) else: - item.chmod(0o200) item.unlink() - tgt.chmod(0o200) + self._logger.debug(f"Removed item {item!s}") + # All contents are gone; remove current directory tgt.rmdir() - self._logger.debug("Removed directory {str(tgt)}") + self._logger.debug(f"Removed directory {tgt!s}") - def _recursive_make_readonly(self, tgt: Path) -> None: - if not tgt.is_dir(): - self._logger.warning( - f"Recursive ro-chmod requested on non-dir {tgt!s}" - ) - return - self._logger.debug(f"Recursive ro-chmod requested for {tgt!s}") - contents = tgt.glob("*") - for item in contents: - if item.is_dir(): - self._recursive_make_readonly(item) - perms = item.stat().st_mode - nowrite = perms & 0o777555 - item.chmod(nowrite) - tgt.chmod(0o777555) - self._logger.debug("Recursive ro-chmod finished for {str(tgt)}") - - def _setup_git(self) -> None: + def _set_git_email_and_name(self) -> None: self._logger.debug("Setting up git") ge = self._env.get("GITHUB_EMAIL", "") gn = self._env.get("GITHUB_NAME", "") @@ -744,27 +594,34 @@ def _setup_git(self) -> None: gn, logger=self._logger, ) + + def _setup_gitlfs(self) -> None: # Check for git-lfs + self._logger.debug("Installing Git LFS if needed") + if not self._check_for_git_lfs(): + run("git", "lfs", "install", logger=self._logger) + self._logger.debug("Git LFS installed") + + def _check_for_git_lfs(self) -> bool: gitconfig = self._home / ".gitconfig" if gitconfig.is_file(): gc = gitconfig.read_text().splitlines() for line in gc: + line.strip() if line == '[filter "lfs"]': - # Already installed - return - self._logger.debug("Installing Git LFS") - run("git", "lfs", "install", logger=self._logger) + return True + return False def _launch(self) -> None: if str_bool(self._env.get("NONINTERACTIVE", "")): self._start_noninteractive() # We exec a lab; control never returns here self._modify_interactive_settings() - self._manage_access_token() self._start() def _modify_interactive_settings(self) -> None: self._logger.debug("Modifying interactive settings if needed") + self._manage_access_token() self._increase_log_limit() def _increase_log_limit(self) -> None: @@ -808,6 +665,7 @@ def _manage_access_token(self) -> None: with contextlib.suppress(NotImplementedError): tokfile.chmod(0o600, follow_symlinks=False) return + self._logger.debug("Did not find container token file") token = get_access_token() if token: tokfile.touch(mode=0o600) diff --git a/src/lsst/rsp/startup/storage/process.py b/src/lsst/rsp/startup/storage/process.py index 88c6f30..e4af577 100644 --- a/src/lsst/rsp/startup/storage/process.py +++ b/src/lsst/rsp/startup/storage/process.py @@ -43,7 +43,7 @@ def run( configure_logging() logger = structlog.get_logger(app_name) argstr = join(args) - logger.info(f"Running command '{argstr}'") + logger.debug(f"Running command '{argstr}'") try: proc = ProcessResult.from_proc( subprocess.run( diff --git a/tests/conftest.py b/tests/conftest.py index da97bd5..1b151c2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from collections.abc import Iterator from pathlib import Path +from shutil import copytree from tempfile import TemporaryDirectory from unittest.mock import patch @@ -10,11 +11,36 @@ @pytest.fixture def rsp_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + # For each of these, we want to cover both the "from ..constants import" + # and the "import lsst.rsp.constants" case. with patch( - "lsst.rsp.startup.constants.top_dir", - (Path(__file__).parent / "support" / "stack_top" / "files"), + "lsst.rsp.startup.services.labrunner.top_dir", + (Path(__file__).parent / "support" / "files" / "stack_top"), ): - with TemporaryDirectory() as homedir: - monkeypatch.setenv("HOME", homedir) - monkeypatch.setenv("USER", "hambone") - yield + with patch( + "lsst.rsp.startup.constants.top_dir", + (Path(__file__).parent / "support" / "files" / "stack_top"), + ): + with patch( + "lsst.rsp.startup.services.labrunner.etc", + (Path(__file__).parent / "support" / "files" / "etc"), + ): + with patch( + "lsst.rsp.startup.constants.etc", + (Path(__file__).parent / "support" / "files" / "etc"), + ): + template = ( + Path(__file__).parent / "support" / "files" / "homedir" + ) + # Set up a user with the files we're going to want to + # manipulate in the test suite. + with TemporaryDirectory() as homedir: + monkeypatch.setenv("HOME", homedir) + monkeypatch.setenv("USER", "hambone") + copytree( + template, + homedir, + dirs_exist_ok=True, + symlinks=True, + ) + yield diff --git a/tests/startup_test.py b/tests/startup_test.py index 7aa5e5f..02e127d 100644 --- a/tests/startup_test.py +++ b/tests/startup_test.py @@ -1,11 +1,17 @@ """Tests for startup object.""" +import configparser +import json import os +from collections.abc import Iterable from pathlib import Path import pytest +import symbolicmode +import lsst.rsp from lsst.rsp.startup.services.labrunner import LabRunner +from lsst.rsp.startup.storage.process import run from lsst.rsp.startup.util import str_bool @@ -147,11 +153,214 @@ def test_set_butler_credential_vars( def test_copy_butler_credentials( monkeypatch: pytest.MonkeyPatch, rsp_env: None ) -> None: - monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", "/etc/secret/aws.creds") - monkeypatch.setenv("PGPASSFILE", "/etc/secret/pgpass") + td = lsst.rsp.startup.constants.top_dir + secret_dir = td / "jupyterlab" / "secrets" + monkeypatch.setenv( + "AWS_SHARED_CREDENTIALS_FILE", str(secret_dir / "aws-credentials.ini") + ) + monkeypatch.setenv( + "PGPASSFILE", str(secret_dir / "postgres-credentials.txt") + ) lr = LabRunner() + pg = lr._home / ".lsst" / "postgres-credentials.txt" + lines = pg.read_text().splitlines() + aws = lr._home / ".lsst" / "aws-credentials.ini" + for line in lines: + if line.startswith("127.0.0.1:5432:db01:postgres:"): + assert line.rsplit(":", maxsplit=1)[1] == "gets_overwritten" + if line.startswith("127.0.0.1:5532:db02:postgres:"): + assert line.rsplit(":", maxsplit=1)[1] == "should_stay" + cp = configparser.ConfigParser() + cp.read(str(aws)) + assert set(cp.sections()) == {"default", "tertiary"} + assert cp["default"]["aws_secret_access_key"] == "gets_overwritten" + assert cp["tertiary"]["aws_secret_access_key"] == "key03" lr._set_butler_credential_variables() lr._copy_butler_credentials() + lines = pg.read_text().splitlines() + aws = lr._home / ".lsst" / "aws-credentials.ini" + for line in lines: + if line.startswith("127.0.0.1:5432:db01:postgres:"): + assert line.rsplit(":", maxsplit=1)[1] == "s33kr1t" + if line.startswith("127.0.0.1:5532:db02:postgres:"): + assert line.rsplit(":", maxsplit=1)[1] == "should_stay" + cp = configparser.ConfigParser() + cp.read(str(aws)) + assert set(cp.sections()) == {"default", "secondary", "tertiary"} + assert cp["default"]["aws_secret_access_key"] == "key01" + assert cp["secondary"]["aws_secret_access_key"] == "key02" + assert cp["tertiary"]["aws_secret_access_key"] == "key03" + + +def test_copy_logging_profile( + monkeypatch: pytest.MonkeyPatch, rsp_env: None +) -> None: + lr = LabRunner() + pfile = ( + lr._home / ".ipython" / "profile_default" / "startup" / "20-logging.py" + ) + td = lsst.rsp.startup.constants.top_dir + assert not pfile.exists() + pfile.parent.mkdir(parents=True) + lr._copy_logging_profile() + assert pfile.exists() + h_contents = pfile.read_text() + sfile = td / "jupyterlab" / "20-logging.py" + assert sfile.exists() + s_contents = sfile.read_text() + assert s_contents == h_contents + h_contents += "\n# Locally modified\n" + pfile.write_text(h_contents) + lr._copy_logging_profile() + new_contents = pfile.read_text() + assert new_contents == h_contents + assert new_contents != s_contents + + +def test_copy_dircolors( + monkeypatch: pytest.MonkeyPatch, rsp_env: None +) -> None: + lr = LabRunner() + assert not (lr._home / ".dir_colors").exists() + lr._copy_dircolors() + assert (lr._home / ".dir_colors").exists() + + +def test_copy_etc_skel(monkeypatch: pytest.MonkeyPatch, rsp_env: None) -> None: + lr = LabRunner() + assert not (lr._home / ".gitconfig").exists() + assert not (lr._home / ".pythonrc").exists() + etc = lsst.rsp.startup.constants.etc + prc = (etc / "skel" / ".pythonrc").read_text() + prc += "\n# Local mods\n" + (lr._home / ".pythonrc").write_text(prc) + lr._copy_etc_skel() + assert (lr._home / ".gitconfig").exists() + sgc = (etc / "skel" / ".gitconfig").read_text() + lgc = (lr._home / ".gitconfig").read_text() + assert sgc == lgc + src = (etc / "skel" / ".pythonrc").read_text() + lrc = (lr._home / ".pythonrc").read_text() + assert src != lrc + assert (lr._home / "notebooks" / ".user_setups").exists() + + +# +# Git +# + + +def test_refresh_notebooks( + monkeypatch: pytest.MonkeyPatch, rsp_env: None +) -> None: + source_repo = Path(__file__).parent / "support" / "repo" + monkeypatch.setenv("AUTO_REPO_SPECS", f"file://{source_repo!s}@main") + lr = LabRunner() + repo = lr._home / "notebooks" / "repo" + assert not repo.exists() + lr._refresh_notebooks() + paths = (repo, repo / "README.md") + assert _is_readonly(paths) + lr._refresh_notebooks() + assert _is_readonly(paths) + for p in paths: + symbolicmode.chmod(p, "u+w") + assert not _is_readonly(paths) + lr._refresh_notebooks() + assert _is_readonly(paths) + + +def _is_readonly(paths: Iterable[Path]) -> bool: + for p in paths: + assert p.exists() + mode = p.stat().st_mode + mask = 0o222 + if mode & mask != 0: + return False + return True + + +def test_set_git_params( + monkeypatch: pytest.MonkeyPatch, rsp_env: None +) -> None: + email = "hambone@opera.borphee.quendor" + name = "Hambone" + monkeypatch.setenv("GITHUB_EMAIL", email) + monkeypatch.setenv("GITHUB_NAME", name) + gc = run("git", "config", "user.email") + assert gc is not None + assert gc.stdout.strip() != email + gc = run("git", "config", "user.name") + assert gc is not None + assert gc.stdout.strip() != name + lr = LabRunner() + lr._set_git_email_and_name() + gc = run("git", "config", "user.email") + assert gc is not None + assert gc.stdout.strip() == email + gc = run("git", "config", "user.name") + assert gc is not None + assert gc.stdout.strip() == name + + +def test_setup_gitlfs(monkeypatch: pytest.MonkeyPatch, rsp_env: None) -> None: + lr = LabRunner() + assert lr._check_for_git_lfs() is False + lr._setup_gitlfs() + assert lr._check_for_git_lfs() is True + + +# +# Interactive-mode-only tests +# + + +def test_increase_log_limit( + monkeypatch: pytest.MonkeyPatch, rsp_env: None +) -> None: + lr = LabRunner() + settings = ( + lr._home + / ".jupyter" + / "lab" + / "user-settings" + / "@jupyterlab" + / "notebook-extension" + / "tracker.jupyterlab.settings" + ) + assert not settings.exists() + lr._increase_log_limit() + assert settings.exists() + with settings.open() as f: + obj = json.load(f) + assert obj["maxNumberOutputs"] >= 10000 + + +def test_manage_access_token( + monkeypatch: pytest.MonkeyPatch, rsp_env: None +) -> None: + monkeypatch.setenv("DEBUG", "1") + token = "token-of-esteem" + monkeypatch.setenv("ACCESS_TOKEN", token) + td = lsst.rsp.startup.constants.top_dir + ctr_file = td / "jupyterlab" / "secrets" / "token" + assert not ctr_file.exists() + lr = LabRunner() + tfile = lr._home / ".access_token" + assert not tfile.exists() + lr._manage_access_token() + assert tfile.exists() + assert tfile.read_text() == token + tfile.unlink() + ctr_file.write_text(token) + assert ctr_file.exists() + assert not tfile.exists() + lr = LabRunner() + lr._manage_access_token() + assert tfile.exists() + assert tfile.read_text() == token + ctr_file.unlink() + assert not ctr_file.exists() # diff --git a/tests/support/files/README.md b/tests/support/files/README.md new file mode 100644 index 0000000..13a8236 --- /dev/null +++ b/tests/support/files/README.md @@ -0,0 +1,4 @@ +# Mock Filesystem + +This is a directory that mimics enough of what would be found at +/opt/lsst/software to run our unit tests. diff --git a/tests/support/files/etc/dircolors.ansi-universal b/tests/support/files/etc/dircolors.ansi-universal new file mode 100644 index 0000000..d236850 --- /dev/null +++ b/tests/support/files/etc/dircolors.ansi-universal @@ -0,0 +1,484 @@ +# Exact Solarized color theme for the color GNU ls utility. +# Designed for dircolors (GNU coreutils) 5.97 +# +# This simple theme was simultaneously designed for these terminal color schemes: +# - Solarized dark (best) +# - Solarized light (best) +# - default dark +# - default light +# +# How the colors were selected: +# - Terminal emulators often have an option typically enabled by default that makes +# bold a different color. It is important to leave this option enabled so that +# you can access the entire 16-color Solarized palette, and not just 8 colors. +# - We favor universality over a greater number of colors. So we limit the number +# of colors so that this theme will work out of the box in all terminals, +# Solarized or not, dark or light. +# - We choose to have the following category of files: +# NORMAL & FILE, DIR, LINK, EXEC and +# editable text including source, unimportant text, binary docs & multimedia source +# files, viewable multimedia, archived/compressed, and unimportant non-text +# - For uniqueness, we stay away from the Solarized foreground colors are -- either +# base00 (brightyellow) or base0 (brightblue). However, they can be used if +# you know what the bg/fg colors of your terminal are, in order to optimize the display. +# - 3 different options are provided: universal, solarized dark, and solarized light. +# The only difference between the universal scheme and one that's optimized for +# dark/light is the color of "unimportant" files, which should blend more with the +# background +# - We note that blue is the hardest color to see on dark bg and yellow is the hardest +# color to see on light bg (with blue being particularly bad). So we choose yellow +# for multimedia files which are usually accessed in a GUI folder browser anyway. +# And blue is kept for custom use of this scheme's user. +# - See table below to see the assignments. + + +# Installation instructions: +# This file goes in the /etc directory, and must be world readable. +# You can copy this file to .dir_colors in your $HOME directory to override +# the system defaults. + +# COLOR needs one of these arguments: 'tty' colorizes output to ttys, but not +# pipes. 'all' adds color characters to all output. 'none' shuts colorization +# off. +COLOR tty + +# Below, there should be one TERM entry for each termtype that is colorizable +TERM alacritty +TERM ansi +TERM color_xterm +TERM color-xterm +TERM con132x25 +TERM con132x30 +TERM con132x43 +TERM con132x60 +TERM con80x25 +TERM con80x28 +TERM con80x30 +TERM con80x43 +TERM con80x50 +TERM con80x60 +TERM cons25 +TERM console +TERM cygwin +TERM dtterm +TERM dvtm +TERM dvtm-256color +TERM Eterm +TERM eterm-color +TERM fbterm +TERM gnome +TERM gnome-256color +TERM jfbterm +TERM konsole +TERM konsole-256color +TERM kterm +TERM linux +TERM linux-c +TERM mach-color +TERM mlterm +TERM nxterm +TERM putty +TERM putty-256color +TERM rxvt +TERM rxvt-256color +TERM rxvt-cygwin +TERM rxvt-cygwin-native +TERM rxvt-unicode +TERM rxvt-unicode256 +TERM rxvt-unicode-256color +TERM screen +TERM screen-16color +TERM screen-16color-bce +TERM screen-16color-s +TERM screen-16color-bce-s +TERM screen-256color +TERM screen-256color-bce +TERM screen-256color-s +TERM screen-256color-bce-s +TERM screen-256color-italic +TERM screen-bce +TERM screen-w +TERM screen.xterm-256color +TERM screen.linux +TERM screen.xterm-new +TERM st +TERM st-meta +TERM st-256color +TERM st-meta-256color +TERM tmux +TERM tmux-256color +TERM vt100 +TERM xterm +TERM xterm-new +TERM xterm-16color +TERM xterm-256color +TERM xterm-256color-italic +TERM xterm-88color +TERM xterm-color +TERM xterm-debian +TERM xterm-kitty +TERM xterm-termite + +# EIGHTBIT, followed by '1' for on, '0' for off. (8-bit output) +EIGHTBIT 1 + +############################################################################# +# Below are the color init strings for the basic file types. A color init +# string consists of one or more of the following numeric codes: +# +# Attribute codes: +# 00=none 01=bold 04=underscore 05=blink 07=reverse 08=concealed +# Text color codes: +# 30=black 31=red 32=green 33=yellow 34=blue 35=magenta 36=cyan 37=white +# Background color codes: +# 40=black 41=red 42=green 43=yellow 44=blue 45=magenta 46=cyan 47=white +# +# NOTES: +# - See http://www.oreilly.com/catalog/wdnut/excerpt/color_names.html +# - Color combinations +# ANSI Color code Solarized Notes Universal SolDark SolLight +# ~~~~~~~~~~~~~~~ ~~~~~~~~~ ~~~~~ ~~~~~~~~~ ~~~~~~~ ~~~~~~~~ +# 00 none NORMAL, FILE +# 30 black base02 +# 01;30 bright black base03 bg of SolDark +# 31 red red docs & mm src +# 01;31 bright red orange EXEC +# 32 green green editable text +# 01;32 bright green base01 unimportant text +# 33 yellow yellow unclear in light bg multimedia +# 01;33 bright yellow base00 fg of SolLight unimportant non-text +# 34 blue blue unclear in dark bg user customized +# 01;34 bright blue base0 fg in SolDark unimportant text +# 35 magenta magenta LINK +# 01;35 bright magenta violet archive/compressed +# 36 cyan cyan DIR +# 01;36 bright cyan base1 unimportant non-text +# 37 white base2 +# 01;37 bright white base3 bg in SolLight +# 05;37;41 unclear in Putty dark + + +### By file type + +# global default +NORMAL 00 +# normal file +FILE 00 +# directory +DIR 36 +# symbolic link +LINK 35 + +# pipe, socket, block device, character device (blue bg) +FIFO 30;44 +SOCK 35;44 +DOOR 35;44 # Solaris 2.5 and later +BLK 33;44 +CHR 37;44 + + +############################################################################# +### By file attributes + +# Orphaned symlinks (blinking white on red) +# Blink may or may not work (works on iTerm dark or light, and Putty dark) +ORPHAN 05;37;41 +# ... and the files that orphaned symlinks point to (blinking white on red) +MISSING 05;37;41 + +# files with execute permission +EXEC 01;31 # Unix +.cmd 01;31 # Win +.exe 01;31 # Win +.com 01;31 # Win +.bat 01;31 # Win +.reg 01;31 # Win +.app 01;31 # OSX + +############################################################################# +### By extension + +# List any file extensions like '.gz' or '.tar' that you would like ls +# to colorize below. Put the extension, a space, and the color init string. +# (and any comments you want to add after a '#') + +### Text formats + +# Text that we can edit with a regular editor +.txt 32 +.org 32 +.md 32 +.mkd 32 +.bib 32 + +# Source text +.h 32 +.hpp 32 +.c 32 +.C 32 +.cc 32 +.cpp 32 +.cxx 32 +.objc 32 +.cl 32 +.sh 32 +.bash 32 +.csh 32 +.zsh 32 +.el 32 +.vim 32 +.java 32 +.pl 32 +.pm 32 +.py 32 +.rb 32 +.hs 32 +.php 32 +.htm 32 +.html 32 +.shtml 32 +.erb 32 +.haml 32 +.xml 32 +.rdf 32 +.css 32 +.sass 32 +.scss 32 +.less 32 +.js 32 +.coffee 32 +.man 32 +.0 32 +.1 32 +.2 32 +.3 32 +.4 32 +.5 32 +.6 32 +.7 32 +.8 32 +.9 32 +.l 32 +.n 32 +.p 32 +.pod 32 +.tex 32 +.go 32 +.sql 32 +.csv 32 +.sv 32 +.svh 32 +.v 32 +.vh 32 +.vhd 32 + +### Multimedia formats + +# Image +.bmp 33 +.cgm 33 +.dl 33 +.dvi 33 +.emf 33 +.eps 33 +.gif 33 +.jpeg 33 +.jpg 33 +.JPG 33 +.mng 33 +.pbm 33 +.pcx 33 +.pdf 33 +.pgm 33 +.png 33 +.PNG 33 +.ppm 33 +.pps 33 +.ppsx 33 +.ps 33 +.svg 33 +.svgz 33 +.tga 33 +.tif 33 +.tiff 33 +.xbm 33 +.xcf 33 +.xpm 33 +.xwd 33 +.xwd 33 +.yuv 33 +.NEF 33 # Nikon RAW format +.nef 33 + +# Audio +.aac 33 +.au 33 +.flac 33 +.m4a 33 +.mid 33 +.midi 33 +.mka 33 +.mp3 33 +.mpa 33 +.mpeg 33 +.mpg 33 +.ogg 33 +.opus 33 +.ra 33 +.wav 33 + +# Video +.anx 33 +.asf 33 +.avi 33 +.axv 33 +.flc 33 +.fli 33 +.flv 33 +.gl 33 +.m2v 33 +.m4v 33 +.mkv 33 +.mov 33 +.MOV 33 +.mp4 33 +.mp4v 33 +.mpeg 33 +.mpg 33 +.nuv 33 +.ogm 33 +.ogv 33 +.ogx 33 +.qt 33 +.rm 33 +.rmvb 33 +.swf 33 +.vob 33 +.webm 33 +.wmv 33 + +### Misc + +# Binary document formats and multimedia source +.doc 31 +.docx 31 +.rtf 31 +.odt 31 +.dot 31 +.dotx 31 +.ott 31 +.xls 31 +.xlsx 31 +.ods 31 +.ots 31 +.ppt 31 +.pptx 31 +.odp 31 +.otp 31 +.fla 31 +.psd 31 + +# Archives, compressed +.7z 1;35 +.apk 1;35 +.arj 1;35 +.bin 1;35 +.bz 1;35 +.bz2 1;35 +.cab 1;35 # Win +.deb 1;35 +.dmg 1;35 # OSX +.gem 1;35 +.gz 1;35 +.iso 1;35 +.jar 1;35 +.msi 1;35 # Win +.rar 1;35 +.rpm 1;35 +.tar 1;35 +.tbz 1;35 +.tbz2 1;35 +.tgz 1;35 +.tx 1;35 +.war 1;35 +.xpi 1;35 +.xz 1;35 +.z 1;35 +.Z 1;35 +.zip 1;35 +.zst 1;35 + +# For testing +.ANSI-30-black 30 +.ANSI-01;30-brblack 01;30 +.ANSI-31-red 31 +.ANSI-01;31-brred 01;31 +.ANSI-32-green 32 +.ANSI-01;32-brgreen 01;32 +.ANSI-33-yellow 33 +.ANSI-01;33-bryellow 01;33 +.ANSI-34-blue 34 +.ANSI-01;34-brblue 01;34 +.ANSI-35-magenta 35 +.ANSI-01;35-brmagenta 01;35 +.ANSI-36-cyan 36 +.ANSI-01;36-brcyan 01;36 +.ANSI-37-white 37 +.ANSI-01;37-brwhite 01;37 + +############################################################################# +# Your customizations + +# Unimportant text files +# For universal scheme, use brightgreen 01;32 +# For optimal on light bg (but too prominent on dark bg), use white 01;34 +.log 01;32 +*~ 01;32 +*# 01;32 +#.log 01;34 +#*~ 01;34 +#*# 01;34 + +# Unimportant non-text files +# For universal scheme, use brightcyan 01;36 +# For optimal on dark bg (but too prominent on light bg), change to 01;33 +.bak 01;36 +.BAK 01;36 +.old 01;36 +.OLD 01;36 +.org_archive 01;36 +.off 01;36 +.OFF 01;36 +.dist 01;36 +.DIST 01;36 +.orig 01;36 +.ORIG 01;36 +.swp 01;36 +.swo 01;36 +*.v 01;36 +#.bak 01;33 +#.BAK 01;33 +#.old 01;33 +#.OLD 01;33 +#.org_archive 01;33 +#.off 01;33 +#.OFF 01;33 +#.dist 01;33 +#.DIST 01;33 +#.orig 01;33 +#.ORIG 01;33 +#.swp 01;33 +#.swo 01;33 +#*.v 01;33 + +# The brightmagenta (Solarized: purple) color is free for you to use for your +# custom file type +.gpg 34 +.gpg 34 +.pgp 34 +.asc 34 +.3des 34 +.aes 34 +.enc 34 +.sqlite 34 +.db 34 diff --git a/tests/support/files/etc/skel/.gitconfig b/tests/support/files/etc/skel/.gitconfig new file mode 100644 index 0000000..60406ed --- /dev/null +++ b/tests/support/files/etc/skel/.gitconfig @@ -0,0 +1,8 @@ +[push] + default = simple +# Cache anonymous access to LSST Git LFS S3 servers +[credential "https://git-lfs.lsst.cloud"] + helper = store +# Use cached GitHub token if we have it +[credential] + helper = store diff --git a/tests/support/files/etc/skel/.pythonrc b/tests/support/files/etc/skel/.pythonrc new file mode 100644 index 0000000..c4afa80 --- /dev/null +++ b/tests/support/files/etc/skel/.pythonrc @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import os +import readline +import rlcompleter +import atexit + +history_file = os.path.join(os.environ['HOME'], '.python_history') +try: + readline.read_history_file(history_file) +except IOError: + pass +readline.parse_and_bind("tab: complete") +readline.set_history_length(1000) +atexit.register(readline.write_history_file, history_file) diff --git a/tests/support/files/etc/skel/notebooks/.user_setups b/tests/support/files/etc/skel/notebooks/.user_setups new file mode 100644 index 0000000..1e3e2e5 --- /dev/null +++ b/tests/support/files/etc/skel/notebooks/.user_setups @@ -0,0 +1,6 @@ +#!/bin/sh +# This file is expected to be found in ${HOME}/notebooks/.user_setups +# It is a shell fragment that will be sourced during kernel startup +# when the LSST kernel is started in a JupyterLab environment. It runs +# in the user context and can contain arbitrary shell code. Exported changes +# in its environment will persist into the JupyterLab Python environment. diff --git a/tests/support/files/homedir/.lsst/aws-credentials.ini b/tests/support/files/homedir/.lsst/aws-credentials.ini new file mode 100644 index 0000000..528a5a0 --- /dev/null +++ b/tests/support/files/homedir/.lsst/aws-credentials.ini @@ -0,0 +1,6 @@ +[default] +aws_access_key_id=access01 +aws_secret_access_key=gets_overwritten +[tertiary] +aws_access_key_id=access03 +aws_secret_access_key=key03 diff --git a/tests/support/files/homedir/.lsst/postgres-credentials.txt b/tests/support/files/homedir/.lsst/postgres-credentials.txt new file mode 100644 index 0000000..56e3b97 --- /dev/null +++ b/tests/support/files/homedir/.lsst/postgres-credentials.txt @@ -0,0 +1,2 @@ +127.0.0.1:5432:db01:postgres:gets_overwritten +127.0.0.1:5432:db02:postgres:should_stay diff --git a/tests/support/files/stack_top/jupyterlab/20-logging.py b/tests/support/files/stack_top/jupyterlab/20-logging.py new file mode 100644 index 0000000..adcdf62 --- /dev/null +++ b/tests/support/files/stack_top/jupyterlab/20-logging.py @@ -0,0 +1,53 @@ +"""Set up custom logging for RSP. + +At startup, if ${HOME}/.ipython/profile_default/startup/20-logging.py does +not exist, this file will be copied to it from /opt/lsst/software/jupyterlab. + +It will also be copied if that file exists but is an earlier standard version +(as determined via sha256sum). + +If you don't like what it does, create an empty file (or one that does what +you want) at ${HOME}/.ipython/profile_default/startup/20-logging.py and it +will not be recopied. +""" + +import logging +import os +import sys + +customlogger = False + +try: + from lsst.rsp import IPythonHandler, forward_lsst_log + + customlogger = True +except ImportError: + pass # Probably a container that doesn't have our new code + +# If the whole container is in debug mode, enable debug logging by default. +# Otherwise, use the default level (which is warning); however, lsst logs +# should be at info level. +debug = os.getenv("DEBUG") +handlers = [] +if customlogger: + # Forward anything at INFO or above, unless debug is set, in which case, + # forward DEBUG and above. + if debug: + forward_lsst_log("DEBUG") + else: + forward_lsst_log("INFO") + handlers = [IPythonHandler()] +else: + # Set up WARNING and above as stderr, below that to stdout. This is + # intended to make GKE error reporting more consistent and to correspond + # to the usual Unix distinction between error and non-error output. + warnhandler = logging.StreamHandler(stream=sys.stderr) + warnhandler.setLevel(logging.WARNING) + handlers = [warnhandler] + if debug: + lowhandler = logging.StreamHandler(stream=sys.stdout) + lowhandler.setLevel(logging.DEBUG) + handlers.append(lowhandler) +logging.basicConfig(force=True, handlers=handlers) +# Now set up INFO for lsst logs everywhere +logging.getLogger("lsst").setLevel(logging.INFO) diff --git a/tests/support/files/stack_top/jupyterlab/secrets/aws-credentials.ini b/tests/support/files/stack_top/jupyterlab/secrets/aws-credentials.ini new file mode 100644 index 0000000..ed34dc7 --- /dev/null +++ b/tests/support/files/stack_top/jupyterlab/secrets/aws-credentials.ini @@ -0,0 +1,6 @@ +[default] +aws_access_key_id=access01 +aws_secret_access_key=key01 +[secondary] +aws_access_key_id=access02 +aws_secret_access_key=key02 diff --git a/tests/support/files/stack_top/jupyterlab/secrets/postgres-credentials.txt b/tests/support/files/stack_top/jupyterlab/secrets/postgres-credentials.txt new file mode 100644 index 0000000..1bb5548 --- /dev/null +++ b/tests/support/files/stack_top/jupyterlab/secrets/postgres-credentials.txt @@ -0,0 +1 @@ +127.0.0.1:5432:db01:postgres:s33kr1t diff --git a/tests/support/repo/HEAD b/tests/support/repo/HEAD new file mode 100644 index 0000000..b870d82 --- /dev/null +++ b/tests/support/repo/HEAD @@ -0,0 +1 @@ +ref: refs/heads/main diff --git a/tests/support/repo/config b/tests/support/repo/config new file mode 100644 index 0000000..e6da231 --- /dev/null +++ b/tests/support/repo/config @@ -0,0 +1,6 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true + ignorecase = true + precomposeunicode = true diff --git a/tests/support/repo/description b/tests/support/repo/description new file mode 100644 index 0000000..498b267 --- /dev/null +++ b/tests/support/repo/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/tests/support/repo/hooks/applypatch-msg.sample b/tests/support/repo/hooks/applypatch-msg.sample new file mode 100755 index 0000000..a5d7b84 --- /dev/null +++ b/tests/support/repo/hooks/applypatch-msg.sample @@ -0,0 +1,15 @@ +#!/bin/sh +# +# An example hook script to check the commit log message taken by +# applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. The hook is +# allowed to edit the commit message file. +# +# To enable this hook, rename this file to "applypatch-msg". + +. git-sh-setup +commitmsg="$(git rev-parse --git-path hooks/commit-msg)" +test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} +: diff --git a/tests/support/repo/hooks/commit-msg.sample b/tests/support/repo/hooks/commit-msg.sample new file mode 100755 index 0000000..b58d118 --- /dev/null +++ b/tests/support/repo/hooks/commit-msg.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to check the commit log message. +# Called by "git commit" with one argument, the name of the file +# that has the commit message. The hook should exit with non-zero +# status after issuing an appropriate message if it wants to stop the +# commit. The hook is allowed to edit the commit message file. +# +# To enable this hook, rename this file to "commit-msg". + +# Uncomment the below to add a Signed-off-by line to the message. +# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +# hook is more suited to it. +# +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" + +# This example catches duplicate Signed-off-by lines. + +test "" = "$(grep '^Signed-off-by: ' "$1" | + sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { + echo >&2 Duplicate Signed-off-by lines. + exit 1 +} diff --git a/tests/support/repo/hooks/fsmonitor-watchman.sample b/tests/support/repo/hooks/fsmonitor-watchman.sample new file mode 100755 index 0000000..23e856f --- /dev/null +++ b/tests/support/repo/hooks/fsmonitor-watchman.sample @@ -0,0 +1,174 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 2) and last update token +# formatted as a string and outputs to stdout a new update token and +# all files that have been modified since the update token. Paths must +# be relative to the root of the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $last_update_token) = @ARGV; + +# Uncomment for debugging +# print STDERR "$0 $version $last_update_token\n"; + +# Check the hook interface version +if ($version ne 2) { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree = get_working_dir(); + +my $retry = 1; + +my $json_pkg; +eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; +} or do { + require JSON::PP; + $json_pkg = "JSON::PP"; +}; + +launch_watchman(); + +sub launch_watchman { + my $o = watchman_query(); + if (is_work_tree_watched($o)) { + output_result($o->{clock}, @{$o->{files}}); + } +} + +sub output_result { + my ($clockid, @files) = @_; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # binmode $fh, ":utf8"; + # print $fh "$clockid\n@files\n"; + # close $fh; + + binmode STDOUT, ":utf8"; + print $clockid; + print "\0"; + local $, = "\0"; + print @files; +} + +sub watchman_clock { + my $response = qx/watchman clock "$git_work_tree"/; + die "Failed to get clock id on '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + return $json_pkg->new->utf8->decode($response); +} + +sub watchman_query { + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $last_update_token but not from the .git folder. + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. Then we're using the "expression" term to + # further constrain the results. + my $last_update_line = ""; + if (substr($last_update_token, 0, 1) eq "c") { + $last_update_token = "\"$last_update_token\""; + $last_update_line = qq[\n"since": $last_update_token,]; + } + my $query = <<" END"; + ["query", "$git_work_tree", {$last_update_line + "fields": ["name"], + "expression": ["not", ["dirname", ".git"]] + }] + END + + # Uncomment for debugging the watchman query + # open (my $fh, ">", ".git/watchman-query.json"); + # print $fh $query; + # close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; }; + + # Uncomment for debugging the watch response + # open ($fh, ">", ".git/watchman-response.json"); + # print $fh $response; + # close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + return $json_pkg->new->utf8->decode($response); +} + +sub is_work_tree_watched { + my ($output) = @_; + my $error = $output->{error}; + if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) { + $retry--; + my $response = qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + $output = $json_pkg->new->utf8->decode($response); + $error = $output->{error}; + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # close $fh; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + my $o = watchman_clock(); + $error = $output->{error}; + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + output_result($o->{clock}, ("/")); + $last_update_token = $o->{clock}; + + eval { launch_watchman() }; + return 0; + } + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + return 1; +} + +sub get_working_dir { + my $working_dir; + if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $working_dir = Win32::GetCwd(); + $working_dir =~ tr/\\/\//; + } else { + require Cwd; + $working_dir = Cwd::cwd(); + } + + return $working_dir; +} diff --git a/tests/support/repo/hooks/post-update.sample b/tests/support/repo/hooks/post-update.sample new file mode 100755 index 0000000..ec17ec1 --- /dev/null +++ b/tests/support/repo/hooks/post-update.sample @@ -0,0 +1,8 @@ +#!/bin/sh +# +# An example hook script to prepare a packed repository for use over +# dumb transports. +# +# To enable this hook, rename this file to "post-update". + +exec git update-server-info diff --git a/tests/support/repo/hooks/pre-applypatch.sample b/tests/support/repo/hooks/pre-applypatch.sample new file mode 100755 index 0000000..4142082 --- /dev/null +++ b/tests/support/repo/hooks/pre-applypatch.sample @@ -0,0 +1,14 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed +# by applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-applypatch". + +. git-sh-setup +precommit="$(git rev-parse --git-path hooks/pre-commit)" +test -x "$precommit" && exec "$precommit" ${1+"$@"} +: diff --git a/tests/support/repo/hooks/pre-commit.sample b/tests/support/repo/hooks/pre-commit.sample new file mode 100755 index 0000000..e144712 --- /dev/null +++ b/tests/support/repo/hooks/pre-commit.sample @@ -0,0 +1,49 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git commit" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message if +# it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-commit". + +if git rev-parse --verify HEAD >/dev/null 2>&1 +then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=$(git hash-object -t tree /dev/null) +fi + +# If you want to allow non-ASCII filenames set this variable to true. +allownonascii=$(git config --type=bool hooks.allownonascii) + +# Redirect output to stderr. +exec 1>&2 + +# Cross platform projects tend to avoid non-ASCII filenames; prevent +# them from being added to the repository. We exploit the fact that the +# printable range starts at the space character and ends with tilde. +if [ "$allownonascii" != "true" ] && + # Note that the use of brackets around a tr range is ok here, (it's + # even required, for portability to Solaris 10's /usr/bin/tr), since + # the square bracket bytes happen to fall in the designated range. + test $(git diff --cached --name-only --diff-filter=A -z $against | + LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +then + cat <<\EOF +Error: Attempt to add a non-ASCII file name. + +This can cause problems if you want to work with people on other platforms. + +To be portable it is advisable to rename the file. + +If you know what you are doing you can disable this check using: + + git config hooks.allownonascii true +EOF + exit 1 +fi + +# If there are whitespace errors, print the offending file names and fail. +exec git diff-index --check --cached $against -- diff --git a/tests/support/repo/hooks/pre-merge-commit.sample b/tests/support/repo/hooks/pre-merge-commit.sample new file mode 100755 index 0000000..399eab1 --- /dev/null +++ b/tests/support/repo/hooks/pre-merge-commit.sample @@ -0,0 +1,13 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git merge" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message to +# stderr if it wants to stop the merge commit. +# +# To enable this hook, rename this file to "pre-merge-commit". + +. git-sh-setup +test -x "$GIT_DIR/hooks/pre-commit" && + exec "$GIT_DIR/hooks/pre-commit" +: diff --git a/tests/support/repo/hooks/pre-push.sample b/tests/support/repo/hooks/pre-push.sample new file mode 100755 index 0000000..4ce688d --- /dev/null +++ b/tests/support/repo/hooks/pre-push.sample @@ -0,0 +1,53 @@ +#!/bin/sh + +# An example hook script to verify what is about to be pushed. Called by "git +# push" after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# +# +# This sample shows how to prevent push of commits where the log message starts +# with "WIP" (work in progress). + +remote="$1" +url="$2" + +zero=$(git hash-object --stdin &2 "Found WIP commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 diff --git a/tests/support/repo/hooks/pre-rebase.sample b/tests/support/repo/hooks/pre-rebase.sample new file mode 100755 index 0000000..6cbef5c --- /dev/null +++ b/tests/support/repo/hooks/pre-rebase.sample @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006, 2008 Junio C Hamano +# +# The "pre-rebase" hook is run just before "git rebase" starts doing +# its job, and can prevent the command from running by exiting with +# non-zero status. +# +# The hook is called with the following parameters: +# +# $1 -- the upstream the series was forked from. +# $2 -- the branch being rebased (or empty when rebasing the current branch). +# +# This sample shows how to prevent topic branches that are already +# merged to 'next' branch from getting rebased, because allowing it +# would result in rebasing already published history. + +publish=next +basebranch="$1" +if test "$#" = 2 +then + topic="refs/heads/$2" +else + topic=`git symbolic-ref HEAD` || + exit 0 ;# we do not interrupt rebasing detached HEAD +fi + +case "$topic" in +refs/heads/??/*) + ;; +*) + exit 0 ;# we do not interrupt others. + ;; +esac + +# Now we are dealing with a topic branch being rebased +# on top of master. Is it OK to rebase it? + +# Does the topic really exist? +git show-ref -q "$topic" || { + echo >&2 "No such branch $topic" + exit 1 +} + +# Is topic fully merged to master? +not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +if test -z "$not_in_master" +then + echo >&2 "$topic is fully merged to master; better remove it." + exit 1 ;# we could allow it, but there is no point. +fi + +# Is topic ever merged to next? If so you should not be rebasing it. +only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +only_next_2=`git rev-list ^master ${publish} | sort` +if test "$only_next_1" = "$only_next_2" +then + not_in_topic=`git rev-list "^$topic" master` + if test -z "$not_in_topic" + then + echo >&2 "$topic is already up to date with master" + exit 1 ;# we could allow it, but there is no point. + else + exit 0 + fi +else + not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` + /usr/bin/perl -e ' + my $topic = $ARGV[0]; + my $msg = "* $topic has commits already merged to public branch:\n"; + my (%not_in_next) = map { + /^([0-9a-f]+) /; + ($1 => 1); + } split(/\n/, $ARGV[1]); + for my $elem (map { + /^([0-9a-f]+) (.*)$/; + [$1 => $2]; + } split(/\n/, $ARGV[2])) { + if (!exists $not_in_next{$elem->[0]}) { + if ($msg) { + print STDERR $msg; + undef $msg; + } + print STDERR " $elem->[1]\n"; + } + } + ' "$topic" "$not_in_next" "$not_in_master" + exit 1 +fi + +<<\DOC_END + +This sample hook safeguards topic branches that have been +published from being rewound. + +The workflow assumed here is: + + * Once a topic branch forks from "master", "master" is never + merged into it again (either directly or indirectly). + + * Once a topic branch is fully cooked and merged into "master", + it is deleted. If you need to build on top of it to correct + earlier mistakes, a new topic branch is created by forking at + the tip of the "master". This is not strictly necessary, but + it makes it easier to keep your history simple. + + * Whenever you need to test or publish your changes to topic + branches, merge them into "next" branch. + +The script, being an example, hardcodes the publish branch name +to be "next", but it is trivial to make it configurable via +$GIT_DIR/config mechanism. + +With this workflow, you would want to know: + +(1) ... if a topic branch has ever been merged to "next". Young + topic branches can have stupid mistakes you would rather + clean up before publishing, and things that have not been + merged into other branches can be easily rebased without + affecting other people. But once it is published, you would + not want to rewind it. + +(2) ... if a topic branch has been fully merged to "master". + Then you can delete it. More importantly, you should not + build on top of it -- other people may already want to + change things related to the topic as patches against your + "master", so if you need further changes, it is better to + fork the topic (perhaps with the same name) afresh from the + tip of "master". + +Let's look at this example: + + o---o---o---o---o---o---o---o---o---o "next" + / / / / + / a---a---b A / / + / / / / + / / c---c---c---c B / + / / / \ / + / / / b---b C \ / + / / / / \ / + ---o---o---o---o---o---o---o---o---o---o---o "master" + + +A, B and C are topic branches. + + * A has one fix since it was merged up to "next". + + * B has finished. It has been fully merged up to "master" and "next", + and is ready to be deleted. + + * C has not merged to "next" at all. + +We would want to allow C to be rebased, refuse A, and encourage +B to be deleted. + +To compute (1): + + git rev-list ^master ^topic next + git rev-list ^master next + + if these match, topic has not merged in next at all. + +To compute (2): + + git rev-list master..topic + + if this is empty, it is fully merged to "master". + +DOC_END diff --git a/tests/support/repo/hooks/pre-receive.sample b/tests/support/repo/hooks/pre-receive.sample new file mode 100755 index 0000000..a1fd29e --- /dev/null +++ b/tests/support/repo/hooks/pre-receive.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to make use of push options. +# The example simply echoes all push options that start with 'echoback=' +# and rejects all pushes when the "reject" push option is used. +# +# To enable this hook, rename this file to "pre-receive". + +if test -n "$GIT_PUSH_OPTION_COUNT" +then + i=0 + while test "$i" -lt "$GIT_PUSH_OPTION_COUNT" + do + eval "value=\$GIT_PUSH_OPTION_$i" + case "$value" in + echoback=*) + echo "echo from the pre-receive-hook: ${value#*=}" >&2 + ;; + reject) + exit 1 + esac + i=$((i + 1)) + done +fi diff --git a/tests/support/repo/hooks/prepare-commit-msg.sample b/tests/support/repo/hooks/prepare-commit-msg.sample new file mode 100755 index 0000000..10fa14c --- /dev/null +++ b/tests/support/repo/hooks/prepare-commit-msg.sample @@ -0,0 +1,42 @@ +#!/bin/sh +# +# An example hook script to prepare the commit log message. +# Called by "git commit" with the name of the file that has the +# commit message, followed by the description of the commit +# message's source. The hook's purpose is to edit the commit +# message file. If the hook fails with a non-zero status, +# the commit is aborted. +# +# To enable this hook, rename this file to "prepare-commit-msg". + +# This hook includes three examples. The first one removes the +# "# Please enter the commit message..." help message. +# +# The second includes the output of "git diff --name-status -r" +# into the message, just before the "git status" output. It is +# commented because it doesn't cope with --amend or with squashed +# commits. +# +# The third example adds a Signed-off-by line to the message, that can +# still be edited. This is rarely a good idea. + +COMMIT_MSG_FILE=$1 +COMMIT_SOURCE=$2 +SHA1=$3 + +/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE" + +# case "$COMMIT_SOURCE,$SHA1" in +# ,|template,) +# /usr/bin/perl -i.bak -pe ' +# print "\n" . `git diff --cached --name-status -r` +# if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;; +# *) ;; +# esac + +# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE" +# if test -z "$COMMIT_SOURCE" +# then +# /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE" +# fi diff --git a/tests/support/repo/hooks/push-to-checkout.sample b/tests/support/repo/hooks/push-to-checkout.sample new file mode 100755 index 0000000..af5a0c0 --- /dev/null +++ b/tests/support/repo/hooks/push-to-checkout.sample @@ -0,0 +1,78 @@ +#!/bin/sh + +# An example hook script to update a checked-out tree on a git push. +# +# This hook is invoked by git-receive-pack(1) when it reacts to git +# push and updates reference(s) in its repository, and when the push +# tries to update the branch that is currently checked out and the +# receive.denyCurrentBranch configuration variable is set to +# updateInstead. +# +# By default, such a push is refused if the working tree and the index +# of the remote repository has any difference from the currently +# checked out commit; when both the working tree and the index match +# the current commit, they are updated to match the newly pushed tip +# of the branch. This hook is to be used to override the default +# behaviour; however the code below reimplements the default behaviour +# as a starting point for convenient modification. +# +# The hook receives the commit with which the tip of the current +# branch is going to be updated: +commit=$1 + +# It can exit with a non-zero status to refuse the push (when it does +# so, it must not modify the index or the working tree). +die () { + echo >&2 "$*" + exit 1 +} + +# Or it can make any necessary changes to the working tree and to the +# index to bring them to the desired state when the tip of the current +# branch is updated to the new commit, and exit with a zero status. +# +# For example, the hook can simply run git read-tree -u -m HEAD "$1" +# in order to emulate git fetch that is run in the reverse direction +# with git push, as the two-tree form of git read-tree -u -m is +# essentially the same as git switch or git checkout that switches +# branches while keeping the local changes in the working tree that do +# not interfere with the difference between the branches. + +# The below is a more-or-less exact translation to shell of the C code +# for the default behaviour for git's push-to-checkout hook defined in +# the push_to_deploy() function in builtin/receive-pack.c. +# +# Note that the hook will be executed from the repository directory, +# not from the working tree, so if you want to perform operations on +# the working tree, you will have to adapt your code accordingly, e.g. +# by adding "cd .." or using relative paths. + +if ! git update-index -q --ignore-submodules --refresh +then + die "Up-to-date check failed" +fi + +if ! git diff-files --quiet --ignore-submodules -- +then + die "Working directory has unstaged changes" +fi + +# This is a rough translation of: +# +# head_has_history() ? "HEAD" : EMPTY_TREE_SHA1_HEX +if git cat-file -e HEAD 2>/dev/null +then + head=HEAD +else + head=$(git hash-object -t tree --stdin &2 + echo " (if you want, you could supply GIT_DIR then run" >&2 + echo " $0 )" >&2 + exit 1 +fi + +if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# --- Config +allowunannotated=$(git config --type=bool hooks.allowunannotated) +allowdeletebranch=$(git config --type=bool hooks.allowdeletebranch) +denycreatebranch=$(git config --type=bool hooks.denycreatebranch) +allowdeletetag=$(git config --type=bool hooks.allowdeletetag) +allowmodifytag=$(git config --type=bool hooks.allowmodifytag) + +# check for no description +projectdesc=$(sed -e '1q' "$GIT_DIR/description") +case "$projectdesc" in +"Unnamed repository"* | "") + echo "*** Project description file hasn't been set" >&2 + exit 1 + ;; +esac + +# --- Check types +# if $newrev is 0000...0000, it's a commit to delete a ref. +zero=$(git hash-object --stdin &2 + echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 + exit 1 + fi + ;; + refs/tags/*,delete) + # delete tag + if [ "$allowdeletetag" != "true" ]; then + echo "*** Deleting a tag is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/tags/*,tag) + # annotated tag + if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 + then + echo "*** Tag '$refname' already exists." >&2 + echo "*** Modifying a tag is not allowed in this repository." >&2 + exit 1 + fi + ;; + refs/heads/*,commit) + # branch + if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then + echo "*** Creating a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/heads/*,delete) + # delete branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/remotes/*,commit) + # tracking branch + ;; + refs/remotes/*,delete) + # delete tracking branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a tracking branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + *) + # Anything else (is there anything else?) + echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 + exit 1 + ;; +esac + +# --- Finished +exit 0 diff --git a/tests/support/repo/info/exclude b/tests/support/repo/info/exclude new file mode 100644 index 0000000..a5196d1 --- /dev/null +++ b/tests/support/repo/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/tests/support/repo/objects/04/211dbf95a496f36ddcf82e8ee90cf3af39afde b/tests/support/repo/objects/04/211dbf95a496f36ddcf82e8ee90cf3af39afde new file mode 100644 index 0000000..dd5402a Binary files /dev/null and b/tests/support/repo/objects/04/211dbf95a496f36ddcf82e8ee90cf3af39afde differ diff --git a/tests/support/repo/objects/45/3ebed7650f18134fed4a035b5b36c00584294c b/tests/support/repo/objects/45/3ebed7650f18134fed4a035b5b36c00584294c new file mode 100644 index 0000000..58c5e3f Binary files /dev/null and b/tests/support/repo/objects/45/3ebed7650f18134fed4a035b5b36c00584294c differ diff --git a/tests/support/repo/objects/83/562ffd60bc1e4d242f04dce58987870bb879a2 b/tests/support/repo/objects/83/562ffd60bc1e4d242f04dce58987870bb879a2 new file mode 100644 index 0000000..242f6e9 --- /dev/null +++ b/tests/support/repo/objects/83/562ffd60bc1e4d242f04dce58987870bb879a2 @@ -0,0 +1,3 @@ +xM +0F]s%mѥ{/0&3hHk略,EN[c>ƜK(EL +_l.}Jpm<ZLƺAGo`'U6յ)8F= \ No newline at end of file diff --git a/tests/support/repo/refs/heads/main b/tests/support/repo/refs/heads/main new file mode 100644 index 0000000..95dc919 --- /dev/null +++ b/tests/support/repo/refs/heads/main @@ -0,0 +1 @@ +83562ffd60bc1e4d242f04dce58987870bb879a2