Skip to content

Commit

Permalink
Fix formatting and Add support for colored msg_prefix
Browse files Browse the repository at this point in the history
  • Loading branch information
@zacswolf
zacswolf committed Dec 12, 2020
1 parent f771867 commit 1106e0a
Show file tree
Hide file tree
Showing 5 changed files with 38 additions and 22 deletions.
11 changes: 8 additions & 3 deletions src/popper/log.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ class PopperFormatter(logging.Formatter):
def __init__(self, colors=True):
super(PopperFormatter, self).__init__(fmt="%(levelname)s: %(msg)s")
self.log_fmt = self.log_format if colors else self.log_format_no_colors
self.colors = colors

def format(self, record):
"""
Expand All @@ -72,7 +73,11 @@ def format(self, record):
self._fmt = fmt
else:
self._style._fmt = fmt
result = f"{msg_prefix}{logging.Formatter.format(self, record)}"

if self.colors:
result = f"{self.BOLD_CYAN}{msg_prefix}{self.RESET}{logging.Formatter.format(self, record)}"
else:
result = f"{msg_prefix}{logging.Formatter.format(self, record)}"
return result


Expand Down Expand Up @@ -183,8 +188,8 @@ def filter(self, record):
bool : True/False according to values of pass levels and level number
of the record.
"""
if not hasattr(record, 'pretag'):
record.pretag = ""
if not hasattr(record, "pretag"):
record.pretag = ""
if self.reject:
return record.levelno not in self.passlevels
else:
Expand Down
4 changes: 2 additions & 2 deletions src/popper/runner.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,12 +126,12 @@ def _clone_repos(self, wf):
continue

if not infoed:
log.info("Cloning step repositories", extra={'pretag':"[popper]"})
log.info("Cloning step repositories", extra={"pretag": "[popper]"})
infoed = True

if f"{user}/{repo}" in cloned:
continue
log.info(f"- {url}/{user}/{repo}@{version}", extra={'pretag':"[popper]"})
log.info(f"- {url}/{user}/{repo}@{version}", extra={"pretag": "[popper]"})
scm.clone(url, user, repo, repo_dir, version)
cloned.add(f"{user}/{repo}")

Expand Down
33 changes: 22 additions & 11 deletions src/popper/runner_host.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def run(self, step):
raise AttributeError("Expecting 'runs' attribute in step.")
cmd = step.runs + tuple(step.args)

log.info(f"{cmd}", extra={'pretag':f"[{step.id}]"})
log.info(f"{cmd}", extra={"pretag": f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down Expand Up @@ -146,7 +146,7 @@ def run(self, step):
if not container and not self._config.reuse:
container = self._create_container(cid, step)

log.info("docker start", extra={'pretag':f"[{step.id}]"})
log.info("docker start", extra={"pretag": f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down Expand Up @@ -177,7 +177,10 @@ def _create_container(self, cid, step):
build, _, img, tag, build_ctx_path = self._get_build_info(step)

if build:
log.info(f"docker build {img}:{tag} {build_ctx_path}", extra={'pretag':f"[{step.id}]"})
log.info(
f"docker build {img}:{tag} {build_ctx_path}",
extra={"pretag": f"[{step.id}]"},
)
if not self._config.dry_run:
streamer = self._d.api.build(
decode=True, path=build_ctx_path, tag=f"{img}:{tag}", rm=True,
Expand All @@ -191,7 +194,7 @@ def _create_container(self, cid, step):
log.step_info(line.strip())

elif not self._config.skip_pull and not step.skip_pull:
log.info(f"docker pull {img}:{tag}", extra={'pretag':f"[{step.id}]"})
log.info(f"docker pull {img}:{tag}", extra={"pretag": f"[{step.id}]"})
if not self._config.dry_run:
self._d.images.pull(repository=f"{img}:{tag}")

Expand All @@ -215,7 +218,7 @@ def _create_container(self, cid, step):
msg += f' entrypoint={container_args["entrypoint"]}'
if container_args["command"]:
msg += f' command={container_args["command"]}'
log.info(msg, extra={'pretag':f"[{step.id}]"})
log.info(msg, extra={"pretag": f"[{step.id}]"})

container = self._d.containers.create(**container_args)

Expand Down Expand Up @@ -272,7 +275,7 @@ def run(self, step):
if not container and not self._config.reuse:
container = self._create_container(cid, step)

log.info("podman start", extra={'pretag':f"[{step.id}]"})
log.info("podman start", extra={"pretag": f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down Expand Up @@ -309,7 +312,10 @@ def _create_container(self, cid, step):
build, _, img, tag, build_ctx_path = self._get_build_info(step)

if build:
log.info(f"podman build {img}:{tag} {build_ctx_path}", extra={'pretag':f"[{step.id}]"})
log.info(
f"podman build {img}:{tag} {build_ctx_path}",
extra={"pretag": f"[{step.id}]"},
)
if not self._config.dry_run:
cmd = [
"podman",
Expand All @@ -322,7 +328,7 @@ def _create_container(self, cid, step):
]
HostRunner._exec_cmd(cmd)
elif not self._config.skip_pull and not step.skip_pull:
log.info(f"podman pull {img}:{tag}", extra={'pretag':f"[{step.id}]"})
log.info(f"podman pull {img}:{tag}", extra={"pretag": f"[{step.id}]"})
if not self._config.dry_run:
cmd = ["podman", "pull", f"{img}:{tag}"]
HostRunner._exec_cmd(cmd, logging=False)
Expand Down Expand Up @@ -517,11 +523,16 @@ def _create_container(self, step, cid):
build_ctx_path = None

if build:
log.info(f"singularity build {cid} {build_ctx_path}", extra={'pretag':f"[{step.id}]"})
log.info(
f"singularity build {cid} {build_ctx_path}",
extra={"pretag": f"[{step.id}]"},
)
if not self._config.dry_run:
self._build_from_recipe(build_ctx_path, self._singularity_cache, cid)
elif not self._config.skip_pull and not step.skip_pull:
log.info(f"singularity pull {cid} {image}", extra={'pretag':f"[{step.id}]"})
log.info(
f"singularity pull {cid} {image}", extra={"pretag": f"[{step.id}]"}
)
if not self._config.dry_run:
self._s.pull(image=image, name=cid, pull_folder=self._singularity_cache)

Expand All @@ -545,7 +556,7 @@ def _singularity_start(self, step, cid):
commands = args
start_fn = self._s.run

log.info(info, extra={'pretag':f"[{step.id}]"})
log.info(info, extra={"pretag": f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down
4 changes: 2 additions & 2 deletions src/popper/runner_kubernetes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ def run(self, step):

image = f"{img}:{tag}"

m = f"[{step.id}] kubernetes run {self._namespace}.{self._pod_name}"
log.info(m)
m = f"kubernetes run {self._namespace}.{self._pod_name}"
log.info(m, extra={"pretag": f"[{step.id}]"})

if self._config.dry_run:
return 0
Expand Down
8 changes: 4 additions & 4 deletions src/popper/runner_slurm.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,24 +167,24 @@ def run(self, step):

if build:
recipefile = self._get_recipe_file(build_ctx_path, cid)
log.info(f"srun singularity build {self._container}", extra={'pretag':f"[{step.id}]"})
log.info(f"srun singularity build {self._container}", extra={"pretag": f"[{step.id}]"})
self._exec_srun(
["singularity", "build", "--fakeroot", self._container, recipefile,],
step,
cwd=os.path.dirname(recipefile),
)
else:
log.info(f"srun singularity pull {self._container}", extra={'pretag':f"[{step.id}]"})
log.info(f"srun singularity pull {self._container}", extra={"pretag": f"[{step.id}]"})
self._exec_srun(["singularity", "pull", self._container, img], step)

cmd = self._create_cmd(step, cid)
self._spawned_containers.add(cid)

if self._config.resman_opts.get(step.id, {}).get("mpi", True):
log.info(f"sbatch {" ".join(cmd)}", extra={'pretag':f"[{step.id}]"})
log.info(f"sbatch {" ".join(cmd)}", extra={"pretag": f"[{step.id}]"})
ecode = self._exec_mpi(cmd, step)
else:
log.info(f"srun {" ".join(cmd)}", extra={'pretag':f"[{step.id}]"})
log.info(f"srun {" ".join(cmd)}", extra={"pretag": f"[{step.id}]"})
ecode = self._exec_srun(cmd, step, logging=True)

self._spawned_containers.remove(cid)
Expand Down

0 comments on commit 1106e0a

Please sign in to comment.