Skip to content

Commit

Permalink
Add colorized output to log.info and log.step_info for bracket prefixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@zacswolf
zacswolf committed Dec 5, 2020
1 parent 0277a88 commit f771867
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 25 deletions.
10 changes: 6 additions & 4 deletions src/popper/log.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,17 @@ class PopperFormatter(logging.Formatter):

log_format = {
"DEBUG": f"{BOLD_CYAN}%(levelname)s: %(msg)s {RESET}",
"STEP_INFO": "%(msg)s",
"INFO": "%(msg)s",
"STEP_INFO": f"{BOLD_CYAN}%(pretag)s{RESET} %(msg)s",
"INFO": f"{BOLD_CYAN}%(pretag)s{RESET} %(msg)s",
"WARNING": f"{BOLD_YELLOW}%(levelname)s: %(msg)s{RESET}",
"ERROR": f"{BOLD_RED}%(levelname)s: %(msg)s{RESET}",
"CRITICAL": f"{BOLD_RED}%(levelname)s: %(msg)s{RESET}",
}

log_format_no_colors = {
"DEBUG": "%(levelname)s: %(msg)s ",
"STEP_INFO": "%(msg)s",
"INFO": "%(msg)s",
"STEP_INFO": "%(pretag)s %(msg)s",
"INFO": "%(pretag)s %(msg)s",
"WARNING": "%(levelname)s: %(msg)s",
"ERROR": "%(levelname)s: %(msg)s",
"CRITICAL": "%(levelname)s: %(msg)s",
Expand Down Expand Up @@ -183,6 +183,8 @@ def filter(self, record):
bool : True/False according to values of pass levels and level number
of the record.
"""
if not hasattr(record, 'pretag'):
record.pretag = ""
if self.reject:
return record.levelno not in self.passlevels
else:
Expand Down
5 changes: 2 additions & 3 deletions src/popper/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,12 @@ def _clone_repos(self, wf):
continue

if not infoed:
log.info("[popper] Cloning step repositories")
log.info("Cloning step repositories", extra={'pretag':"[popper]"})
infoed = True

if f"{user}/{repo}" in cloned:
continue

log.info(f"[popper] - {url}/{user}/{repo}@{version}")
log.info(f"- {url}/{user}/{repo}@{version}", extra={'pretag':"[popper]"})
scm.clone(url, user, repo, repo_dir, version)
cloned.add(f"{user}/{repo}")

Expand Down
28 changes: 14 additions & 14 deletions src/popper/runner_host.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def run(self, step):
raise AttributeError("Expecting 'runs' attribute in step.")
cmd = step.runs + tuple(step.args)

log.info(f"[{step.id}] {cmd}")
log.info(f"{cmd}", extra={'pretag':f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down Expand Up @@ -146,7 +146,7 @@ def run(self, step):
if not container and not self._config.reuse:
container = self._create_container(cid, step)

log.info(f"[{step.id}] docker start")
log.info("docker start", extra={'pretag':f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down Expand Up @@ -177,7 +177,7 @@ def _create_container(self, cid, step):
build, _, img, tag, build_ctx_path = self._get_build_info(step)

if build:
log.info(f"[{step.id}] docker build {img}:{tag} {build_ctx_path}")
log.info(f"docker build {img}:{tag} {build_ctx_path}", extra={'pretag':f"[{step.id}]"})
if not self._config.dry_run:
streamer = self._d.api.build(
decode=True, path=build_ctx_path, tag=f"{img}:{tag}", rm=True,
Expand All @@ -191,7 +191,7 @@ def _create_container(self, cid, step):
log.step_info(line.strip())

elif not self._config.skip_pull and not step.skip_pull:
log.info(f"[{step.id}] docker pull {img}:{tag}")
log.info(f"docker pull {img}:{tag}", extra={'pretag':f"[{step.id}]"})
if not self._config.dry_run:
self._d.images.pull(repository=f"{img}:{tag}")

Expand All @@ -209,13 +209,13 @@ def _create_container(self, cid, step):

log.debug(f"Container args: {container_args}")

msg = f"[{step.id}] docker create name={cid}"
msg = f"docker create name={cid}"
msg += f' image={container_args["image"]}'
if container_args["entrypoint"]:
msg += f' entrypoint={container_args["entrypoint"]}'
if container_args["command"]:
msg += f' command={container_args["command"]}'
log.info(msg)
log.info(msg, extra={'pretag':f"[{step.id}]"})

container = self._d.containers.create(**container_args)

Expand Down Expand Up @@ -272,7 +272,7 @@ def run(self, step):
if not container and not self._config.reuse:
container = self._create_container(cid, step)

log.info(f"[{step.id}] podman start")
log.info("podman start", extra={'pretag':f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down Expand Up @@ -309,7 +309,7 @@ def _create_container(self, cid, step):
build, _, img, tag, build_ctx_path = self._get_build_info(step)

if build:
log.info(f"[{step.id}] podman build {img}:{tag} {build_ctx_path}")
log.info(f"podman build {img}:{tag} {build_ctx_path}", extra={'pretag':f"[{step.id}]"})
if not self._config.dry_run:
cmd = [
"podman",
Expand All @@ -322,7 +322,7 @@ def _create_container(self, cid, step):
]
HostRunner._exec_cmd(cmd)
elif not self._config.skip_pull and not step.skip_pull:
log.info(f"[{step.id}] podman pull {img}:{tag}")
log.info(f"podman pull {img}:{tag}", extra={'pretag':f"[{step.id}]"})
if not self._config.dry_run:
cmd = ["podman", "pull", f"{img}:{tag}"]
HostRunner._exec_cmd(cmd, logging=False)
Expand Down Expand Up @@ -517,11 +517,11 @@ def _create_container(self, step, cid):
build_ctx_path = None

if build:
log.info(f"[{step.id}] singularity build {cid} {build_ctx_path}")
log.info(f"singularity build {cid} {build_ctx_path}", extra={'pretag':f"[{step.id}]"})
if not self._config.dry_run:
self._build_from_recipe(build_ctx_path, self._singularity_cache, cid)
elif not self._config.skip_pull and not step.skip_pull:
log.info(f"[{step.id}] singularity pull {cid} {image}")
log.info(f"singularity pull {cid} {image}", extra={'pretag':f"[{step.id}]"})
if not self._config.dry_run:
self._s.pull(image=image, name=cid, pull_folder=self._singularity_cache)

Expand All @@ -537,15 +537,15 @@ def _singularity_start(self, step, cid):
ecode = None

if runs:
info = f"[{step.id}] singularity exec {cid} {runs}"
info = f"singularity exec {cid} {runs}"
commands = runs
start_fn = self._s.execute
else:
info = f"[{step.id}] singularity run {cid} {args}"
info = f"singularity run {cid} {args}"
commands = args
start_fn = self._s.run

log.info(info)
log.info(info, extra={'pretag':f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down
8 changes: 4 additions & 4 deletions src/popper/runner_slurm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,24 +167,24 @@ def run(self, step):

if build:
recipefile = self._get_recipe_file(build_ctx_path, cid)
log.info(f"[{step.id}] srun singularity build {self._container}")
log.info(f"srun singularity build {self._container}", extra={'pretag':f"[{step.id}]"})
self._exec_srun(
["singularity", "build", "--fakeroot", self._container, recipefile,],
step,
cwd=os.path.dirname(recipefile),
)
else:
log.info(f"[{step.id}] srun singularity pull {self._container}")
log.info(f"srun singularity pull {self._container}", extra={'pretag':f"[{step.id}]"})
self._exec_srun(["singularity", "pull", self._container, img], step)

cmd = self._create_cmd(step, cid)
self._spawned_containers.add(cid)

if self._config.resman_opts.get(step.id, {}).get("mpi", True):
log.info(f'[{step.id}] sbatch {" ".join(cmd)}')
log.info(f"sbatch {" ".join(cmd)}", extra={'pretag':f"[{step.id}]"})
ecode = self._exec_mpi(cmd, step)
else:
log.info(f'[{step.id}] srun {" ".join(cmd)}')
log.info(f"srun {" ".join(cmd)}", extra={'pretag':f"[{step.id}]"})
ecode = self._exec_srun(cmd, step, logging=True)

self._spawned_containers.remove(cid)
Expand Down

0 comments on commit f771867

Please sign in to comment.