Skip to content

Commit

Permalink
Improve the output of the inmanta compile and inmanta export comm…
Browse files Browse the repository at this point in the history 
…ands, by using the logger name `compiler`, `exporter` or `<name-inmanta-module>` for log records produced by respectively the compiler, the exporter or an Inmanta module. (Issue #6489, PR #6630)

# Description

This PR improves the output of the `inmanta compile` and `inmanta export` commands, by using the logger name `compiler`, `exporter` or `<name-inmanta-module>` for log records produced by respectively the compiler, the exporter or an Inmanta module.

closes #6489

# Self Check:

- [x] Attached issue to pull request
- [x] Changelog entry
- [x] Type annotations are present
- [x] Code is clear and sufficiently documented
- [x] No (preventable) type errors (check using make mypy or make mypy-diff)
- [x] Sufficient test cases (reproduces the bug/tests the requested feature)
- [x] Correct, in line with design
- [ ] ~~End user documentation is included or an issue is created for end-user documentation~~
- [ ] ~~If this PR fixes a race condition in the test suite, also push the fix to the relevant stable branche(s) (see [test-fixes](https://internal.inmanta.com/development/core/tasks/build-master.html#test-fixes) for more info)~~
  • Loading branch information
@arnaudsjs @Hugo-Inmanta
arnaudsjs authored and Hugo-Inmanta committed Nov 6, 2023
1 parent 4bb396c commit ee8fdf5
Show file tree
Hide file tree
Showing 4 changed files with 405 additions and 132 deletions.
8 changes: 8 additions & 0 deletions changelogs/unreleased/6489-change-logger-name.yml
View file
@@ -0,0 +1,8 @@
---
description: Improve the output of the `inmanta compile` and `inmanta export` commands, by using the logger name `compiler`, `exporter` or `<name-inmanta-module>` for log records produced by respectively the compiler, the exporter or an Inmanta module.
issue-nr: 6489
issue-repo: inmanta-core
change-type: minor
destination-branches: [master, iso6]
sections:
minor-improvement: "{{description}}"
244 changes: 128 additions & 116 deletions src/inmanta/app.py
View file
Expand Up @@ -67,7 +67,7 @@
from inmanta.config import Config, Option
from inmanta.const import EXIT_START_FAILED
from inmanta.export import cfg_env
from inmanta.logging import InmantaLoggerConfig, _is_on_tty
from inmanta.logging import InmantaLoggerConfig, LoggerMode, _is_on_tty
from inmanta.server.bootloader import InmantaBootloader
from inmanta.util import get_compiler_version
from inmanta.warnings import WarningsManager
Expand Down Expand Up @@ -328,63 +328,65 @@ def compiler_config(parser: argparse.ArgumentParser, parent_parsers: abc.Sequenc
"compile", help_msg="Compile the project to a configuration model", parser_config=compiler_config, require_project=True
)
def compile_project(options: argparse.Namespace) -> None:
if options.environment is not None:
Config.set("config", "environment", options.environment)
inmanta_logger_config = InmantaLoggerConfig.get_current_instance()
with inmanta_logger_config.run_in_logger_mode(LoggerMode.COMPILER):
if options.environment is not None:
Config.set("config", "environment", options.environment)

if options.server is not None:
Config.set("compiler_rest_transport", "host", options.server)
if options.server is not None:
Config.set("compiler_rest_transport", "host", options.server)

if options.port is not None:
Config.set("compiler_rest_transport", "port", options.port)
if options.port is not None:
Config.set("compiler_rest_transport", "port", options.port)

if options.user is not None:
Config.set("compiler_rest_transport", "username", options.user)
if options.user is not None:
Config.set("compiler_rest_transport", "username", options.user)

if options.password is not None:
Config.set("compiler_rest_transport", "password", options.password)
if options.password is not None:
Config.set("compiler_rest_transport", "password", options.password)

if options.ssl:
Config.set("compiler_rest_transport", "ssl", "true")
if options.ssl:
Config.set("compiler_rest_transport", "ssl", "true")

if options.ca_cert is not None:
Config.set("compiler_rest_transport", "ssl-ca-cert-file", options.ca_cert)
if options.ca_cert is not None:
Config.set("compiler_rest_transport", "ssl-ca-cert-file", options.ca_cert)

if options.export_compile_data is True:
Config.set("compiler", "export_compile_data", "true")
if options.export_compile_data is True:
Config.set("compiler", "export_compile_data", "true")

if options.export_compile_data_file is not None:
Config.set("compiler", "export_compile_data_file", options.export_compile_data_file)
if options.export_compile_data_file is not None:
Config.set("compiler", "export_compile_data_file", options.export_compile_data_file)

if options.feature_compiler_cache is False:
Config.set("compiler", "cache", "false")
if options.feature_compiler_cache is False:
Config.set("compiler", "cache", "false")

if options.datatrace is True:
Config.set("compiler", "datatrace_enable", "true")
if options.datatrace is True:
Config.set("compiler", "datatrace_enable", "true")

if options.dataflow_graphic is True:
Config.set("compiler", "dataflow_graphic_enable", "true")
if options.dataflow_graphic is True:
Config.set("compiler", "dataflow_graphic_enable", "true")

strict_deps_check = moduletool.get_strict_deps_check(
no_strict_deps_check=options.no_strict_deps_check, strict_deps_check=options.strict_deps_check
)
module.Project.get(options.main_file, strict_deps_check=strict_deps_check)
strict_deps_check = moduletool.get_strict_deps_check(
no_strict_deps_check=options.no_strict_deps_check, strict_deps_check=options.strict_deps_check
)
module.Project.get(options.main_file, strict_deps_check=strict_deps_check)

summary_reporter = CompileSummaryReporter()
if options.profile:
import cProfile
import pstats
summary_reporter = CompileSummaryReporter()
if options.profile:
import cProfile
import pstats

with summary_reporter.compiler_exception.capture():
cProfile.runctx("do_compile()", globals(), {}, "run.profile")
p = pstats.Stats("run.profile")
p.strip_dirs().sort_stats("time").print_stats(20)
else:
t1 = time.time()
with summary_reporter.compiler_exception.capture():
do_compile()
LOGGER.debug("Compile time: %0.03f seconds", time.time() - t1)
with summary_reporter.compiler_exception.capture():
cProfile.runctx("do_compile()", globals(), {}, "run.profile")
p = pstats.Stats("run.profile")
p.strip_dirs().sort_stats("time").print_stats(20)
else:
t1 = time.time()
with summary_reporter.compiler_exception.capture():
do_compile()
LOGGER.debug("Compile time: %0.03f seconds", time.time() - t1)

summary_reporter.print_summary_and_exit(show_stack_traces=options.errors)
summary_reporter.print_summary_and_exit(show_stack_traces=options.errors)


@command("list-commands", help_msg="Print out an overview of all commands", add_verbose_flag=False)
Expand Down Expand Up @@ -545,98 +547,101 @@ def export_parser_config(parser: argparse.ArgumentParser, parent_parsers: abc.Se

@command("export", help_msg="Export the configuration", parser_config=export_parser_config, require_project=True)
def export(options: argparse.Namespace) -> None:
if not options.partial_compile and options.delete_resource_set:
raise CLIException(
"The --delete-resource-set option should always be used together with the --partial option", exitcode=1
)
if options.environment is not None:
Config.set("config", "environment", options.environment)
inmanta_logger_config = InmantaLoggerConfig.get_current_instance()
with inmanta_logger_config.run_in_logger_mode(LoggerMode.COMPILER):
if not options.partial_compile and options.delete_resource_set:
raise CLIException(
"The --delete-resource-set option should always be used together with the --partial option", exitcode=1
)
if options.environment is not None:
Config.set("config", "environment", options.environment)

if options.server is not None:
Config.set("compiler_rest_transport", "host", options.server)
if options.server is not None:
Config.set("compiler_rest_transport", "host", options.server)

if options.port is not None:
Config.set("compiler_rest_transport", "port", options.port)
if options.port is not None:
Config.set("compiler_rest_transport", "port", options.port)

if options.token is not None:
Config.set("compiler_rest_transport", "token", options.token)
if options.token is not None:
Config.set("compiler_rest_transport", "token", options.token)

if options.ssl is not None:
Config.set("compiler_rest_transport", "ssl", f"{options.ssl}".lower())
if options.ssl is not None:
Config.set("compiler_rest_transport", "ssl", f"{options.ssl}".lower())

if options.ca_cert is not None:
Config.set("compiler_rest_transport", "ssl-ca-cert-file", options.ca_cert)
if options.ca_cert is not None:
Config.set("compiler_rest_transport", "ssl-ca-cert-file", options.ca_cert)

if options.export_compile_data is True:
Config.set("compiler", "export_compile_data", "true")
if options.export_compile_data is True:
Config.set("compiler", "export_compile_data", "true")

if options.export_compile_data_file is not None:
Config.set("compiler", "export_compile_data_file", options.export_compile_data_file)
if options.export_compile_data_file is not None:
Config.set("compiler", "export_compile_data_file", options.export_compile_data_file)

if options.feature_compiler_cache is False:
Config.set("compiler", "cache", "false")
if options.feature_compiler_cache is False:
Config.set("compiler", "cache", "false")

# try to parse the metadata as json. If a normal string, create json for it.
if options.metadata is not None and len(options.metadata) > 0:
try:
metadata = json.loads(options.metadata)
except json.decoder.JSONDecodeError:
metadata = {"message": options.metadata}
else:
metadata = {"message": "Manual compile on the CLI by user"}
# try to parse the metadata as json. If a normal string, create json for it.
if options.metadata is not None and len(options.metadata) > 0:
try:
metadata = json.loads(options.metadata)
except json.decoder.JSONDecodeError:
metadata = {"message": options.metadata}
else:
metadata = {"message": "Manual compile on the CLI by user"}

if "cli-user" not in metadata and "USERNAME" in os.environ:
metadata["cli-user"] = os.environ["USERNAME"]
if "cli-user" not in metadata and "USERNAME" in os.environ:
metadata["cli-user"] = os.environ["USERNAME"]

if "hostname" not in metadata:
metadata["hostname"] = socket.gethostname()
if "hostname" not in metadata:
metadata["hostname"] = socket.gethostname()

if "type" not in metadata:
metadata["type"] = "manual"
if "type" not in metadata:
metadata["type"] = "manual"

strict_deps_check = moduletool.get_strict_deps_check(
no_strict_deps_check=options.no_strict_deps_check, strict_deps_check=options.strict_deps_check
)
module.Project.get(options.main_file, strict_deps_check=strict_deps_check)
strict_deps_check = moduletool.get_strict_deps_check(
no_strict_deps_check=options.no_strict_deps_check, strict_deps_check=options.strict_deps_check
)
module.Project.get(options.main_file, strict_deps_check=strict_deps_check)

from inmanta.export import Exporter # noqa: H307
from inmanta.export import Exporter # noqa: H307

summary_reporter = CompileSummaryReporter()
summary_reporter = CompileSummaryReporter()

types: Optional[Dict[str, inmanta_type.Type]]
scopes: Optional[Namespace]
types: Optional[Dict[str, inmanta_type.Type]]
scopes: Optional[Namespace]

with summary_reporter.compiler_exception.capture():
try:
(types, scopes) = do_compile()
except Exception:
types, scopes = (None, None)
raise

# Even if the compile failed we might have collected additional data such as unknowns. So
# continue the export

export = Exporter(options)
with summary_reporter.exporter_exception.capture():
results = export.run(
types,
scopes,
metadata=metadata,
model_export=options.model_export,
export_plugin=options.export_plugin,
partial_compile=options.partial_compile,
resource_sets_to_remove=options.delete_resource_set,
)
with summary_reporter.compiler_exception.capture():
try:
(types, scopes) = do_compile()
except Exception:
types, scopes = (None, None)
raise

with inmanta_logger_config.run_in_logger_mode(LoggerMode.EXPORTER):
# Even if the compile failed we might have collected additional data such as unknowns. So
# continue the export

export = Exporter(options)
with summary_reporter.exporter_exception.capture():
results = export.run(
types,
scopes,
metadata=metadata,
model_export=options.model_export,
export_plugin=options.export_plugin,
partial_compile=options.partial_compile,
resource_sets_to_remove=options.delete_resource_set,
)

if not summary_reporter.is_failure() and options.deploy:
version = results[0]
conn = protocol.SyncClient("compiler")
LOGGER.info("Triggering deploy for version %d" % version)
tid = cfg_env.get()
agent_trigger_method = const.AgentTriggerMethod.get_agent_trigger_method(options.full_deploy)
conn.release_version(tid, version, True, agent_trigger_method)
if not summary_reporter.is_failure() and options.deploy:
version = results[0]
conn = protocol.SyncClient("compiler")
LOGGER.info("Triggering deploy for version %d" % version)
tid = cfg_env.get()
agent_trigger_method = const.AgentTriggerMethod.get_agent_trigger_method(options.full_deploy)
conn.release_version(tid, version, True, agent_trigger_method)

summary_reporter.print_summary_and_exit(show_stack_traces=options.errors)
summary_reporter.print_summary_and_exit(show_stack_traces=options.errors)


class Color(enum.Enum):
Expand Down Expand Up @@ -823,6 +828,13 @@ def cmd_parser() -> argparse.ArgumentParser:
default=False,
required=False,
)
parser.add_argument(
"--keep-logger-names",
dest="keep_logger_names",
help="Display the log messages using the name of the logger that created the log messages.",
action="store_true",
default=False,
)

verbosity_parser = argparse.ArgumentParser(add_help=False)
verbosity_parser.add_argument(
Expand Down

0 comments on commit ee8fdf5

Please sign in to comment.