From bd9ae81a5895f72ff81ec6414eef778ad39c5a64 Mon Sep 17 00:00:00 2001 From: Leonardo Parente <23251360+leoparente@users.noreply.github.com> Date: Tue, 8 Jul 2025 12:11:11 -0300 Subject: [PATCH 1/5] feat: add diode-load-dyrun helper --- README.md | 14 +++++- netboxlabs/diode/sdk/dryrun_ingest.py | 68 +++++++++++++++++++++++++++ pyproject.toml | 3 +- 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 netboxlabs/diode/sdk/dryrun_ingest.py diff --git a/README.md b/README.md index 690040c..cb64445 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,8 @@ with DiodeDryRunClient(app_name="my_app", output_dir="/tmp") as client: ``` The produced file can later be ingested by a real Diode instance using -`load_dryrun_entities` with a standard `DiodeClient`: +`load_dryrun_entities` with a standard `DiodeClient` or via the bundled +`diode-load-dryrun` helper: ```python from netboxlabs.diode.sdk import DiodeClient, load_dryrun_entities @@ -104,6 +105,17 @@ with DiodeClient( client.ingest(entities=entities) ``` +Alternatively, the same file can be ingested using the `diode-load-dryrun` +command shipped with the SDK: + +```bash +diode-load-dryrun \ + --target grpc://localhost:8080/diode \ + --app-name my-test-app \ + --app-version 0.0.1 \ + my_app_92722156890707.json +``` + ## Supported entities (object types) * ASN diff --git a/netboxlabs/diode/sdk/dryrun_ingest.py b/netboxlabs/diode/sdk/dryrun_ingest.py new file mode 100644 index 0000000..3a83ff2 --- /dev/null +++ b/netboxlabs/diode/sdk/dryrun_ingest.py @@ -0,0 +1,68 @@ +"""CLI helper to ingest dry-run JSON messages into Diode.""" + +import argparse + +from netboxlabs.diode.sdk import DiodeClient, load_dryrun_entities + + +def main() -> None: + """Ingest JSON files generated by ``DiodeDryRunClient``.""" + parser = argparse.ArgumentParser( + description="Load dry-run JSON messages and ingest them into Diode" + ) + parser.add_argument( + "-t", + "--target", + required=True, + help="gRPC target of the Diode server, e.g. grpc://localhost:8080/diode", + ) + parser.add_argument( + "-a", + "--app-name", + required=True, + help="Application name used when producing the dry-run messages", + ) + parser.add_argument( + "-v", + "--app-version", + required=True, + help="Application version used when producing the dry-run messages", + ) + parser.add_argument( + "-c", + "--client-id", + help="OAuth2 client ID. Defaults to the DIODE_CLIENT_ID environment variable if not provided", + ) + parser.add_argument( + "-k", + "--client-secret", + help="OAuth2 client secret. Defaults to the DIODE_CLIENT_SECRET environment variable if not provided", + ) + parser.add_argument( + "files", + nargs="+", + metavar="FILE", + help="Dry-run JSON files to ingest", + ) + + args = parser.parse_args() + + with DiodeClient( + target=args.target, + app_name=args.app_name, + app_version=args.app_version, + client_id=args.client_id, + client_secret=args.client_secret, + ) as client: + for file_path in args.files: + entities = list(load_dryrun_entities(file_path)) + if entities: + response = client.ingest(entities=entities) + if response.errors: + print(f"Errors while ingesting {file_path}: {response.errors}") + else: + print(f"Ingested {len(entities)} entities from {file_path}") + + +if __name__ == "__main__": # pragma: no cover - entry point + main() diff --git a/pyproject.toml b/pyproject.toml index 249b6ed..7095766 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,8 @@ omit = [ [project.urls] # Optional "Homepage" = "https://netboxlabs.com/" -[project.scripts] # Optional +[project.scripts] +diode-load-dryrun = "netboxlabs.diode.sdk.dryrun_ingest:main" [tool.setuptools] packages = [ From aeee8fbd90b5c2df4eb83c15254657c0ccd01f7d Mon Sep 17 00:00:00 2001 From: Leonardo Parente <23251360+leoparente@users.noreply.github.com> Date: Tue, 8 Jul 2025 13:52:42 -0300 Subject: [PATCH 2/5] Update netboxlabs/diode/sdk/dryrun_ingest.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- netboxlabs/diode/sdk/dryrun_ingest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/netboxlabs/diode/sdk/dryrun_ingest.py b/netboxlabs/diode/sdk/dryrun_ingest.py index 3a83ff2..b35719e 100644 --- a/netboxlabs/diode/sdk/dryrun_ingest.py +++ b/netboxlabs/diode/sdk/dryrun_ingest.py @@ -54,14 +54,18 @@ def main() -> None: client_id=args.client_id, client_secret=args.client_secret, ) as client: + has_errors = False for file_path in args.files: entities = list(load_dryrun_entities(file_path)) if entities: response = client.ingest(entities=entities) if response.errors: print(f"Errors while ingesting {file_path}: {response.errors}") + has_errors = True else: print(f"Ingested {len(entities)} entities from {file_path}") + if has_errors: + sys.exit(1) if __name__ == "__main__": # pragma: no cover - entry point From cf9e0fefc014fa5ccff8087b581228b67923bdac Mon Sep 17 00:00:00 2001 From: Leonardo Parente <23251360+leoparente@users.noreply.github.com> Date: Tue, 8 Jul 2025 13:53:34 -0300 Subject: [PATCH 3/5] add import --- netboxlabs/diode/sdk/dryrun_ingest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netboxlabs/diode/sdk/dryrun_ingest.py b/netboxlabs/diode/sdk/dryrun_ingest.py index b35719e..97415ec 100644 --- a/netboxlabs/diode/sdk/dryrun_ingest.py +++ b/netboxlabs/diode/sdk/dryrun_ingest.py @@ -1,6 +1,7 @@ """CLI helper to ingest dry-run JSON messages into Diode.""" import argparse +import sys from netboxlabs.diode.sdk import DiodeClient, load_dryrun_entities From a32aa63612be276ea94f7de69bd27afaa0662155 Mon Sep 17 00:00:00 2001 From: Leonardo Parente <23251360+leoparente@users.noreply.github.com> Date: Tue, 8 Jul 2025 15:09:36 -0300 Subject: [PATCH 4/5] rename load to replay --- README.md | 6 +++--- netboxlabs/diode/sdk/{dryrun_ingest.py => dryrun_replay.py} | 0 pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename netboxlabs/diode/sdk/{dryrun_ingest.py => dryrun_replay.py} (100%) diff --git a/README.md b/README.md index cb64445..4d476dc 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ with DiodeDryRunClient(app_name="my_app", output_dir="/tmp") as client: The produced file can later be ingested by a real Diode instance using `load_dryrun_entities` with a standard `DiodeClient` or via the bundled -`diode-load-dryrun` helper: +`diode-replay-dryrun` helper: ```python from netboxlabs.diode.sdk import DiodeClient, load_dryrun_entities @@ -105,11 +105,11 @@ with DiodeClient( client.ingest(entities=entities) ``` -Alternatively, the same file can be ingested using the `diode-load-dryrun` +Alternatively, the same file can be ingested using the `diode-replay-dryrun` command shipped with the SDK: ```bash -diode-load-dryrun \ +diode-replay-dryrun \ --target grpc://localhost:8080/diode \ --app-name my-test-app \ --app-version 0.0.1 \ diff --git a/netboxlabs/diode/sdk/dryrun_ingest.py b/netboxlabs/diode/sdk/dryrun_replay.py similarity index 100% rename from netboxlabs/diode/sdk/dryrun_ingest.py rename to netboxlabs/diode/sdk/dryrun_replay.py diff --git a/pyproject.toml b/pyproject.toml index 7095766..d8a9771 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ omit = [ "Homepage" = "https://netboxlabs.com/" [project.scripts] -diode-load-dryrun = "netboxlabs.diode.sdk.dryrun_ingest:main" +diode-replay-dryrun = "netboxlabs.diode.sdk.dryrun_replay:main" [tool.setuptools] packages = [ From eddbf5a6ebff1d33c83bdafa234cdf60ee7a16c5 Mon Sep 17 00:00:00 2001 From: Leonardo Parente <23251360+leoparente@users.noreply.github.com> Date: Wed, 9 Jul 2025 09:36:50 -0300 Subject: [PATCH 5/5] move replay to scripts --- netboxlabs/diode/scripts/__init__.py | 3 +++ netboxlabs/diode/{sdk => scripts}/dryrun_replay.py | 8 +++++--- pyproject.toml | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 netboxlabs/diode/scripts/__init__.py rename netboxlabs/diode/{sdk => scripts}/dryrun_replay.py (90%) diff --git a/netboxlabs/diode/scripts/__init__.py b/netboxlabs/diode/scripts/__init__.py new file mode 100644 index 0000000..4d931d1 --- /dev/null +++ b/netboxlabs/diode/scripts/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# Copyright 2025 NetBox Labs Inc +"""NetBox Labs, Diode - Scripts.""" \ No newline at end of file diff --git a/netboxlabs/diode/sdk/dryrun_replay.py b/netboxlabs/diode/scripts/dryrun_replay.py similarity index 90% rename from netboxlabs/diode/sdk/dryrun_replay.py rename to netboxlabs/diode/scripts/dryrun_replay.py index 97415ec..962c8a9 100644 --- a/netboxlabs/diode/sdk/dryrun_replay.py +++ b/netboxlabs/diode/scripts/dryrun_replay.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python +# Copyright 2025 NetBox Labs Inc """CLI helper to ingest dry-run JSON messages into Diode.""" import argparse @@ -21,13 +23,13 @@ def main() -> None: "-a", "--app-name", required=True, - help="Application name used when producing the dry-run messages", + help="Application name used when ingesting the dry-run messages", ) parser.add_argument( "-v", "--app-version", required=True, - help="Application version used when producing the dry-run messages", + help="Application version used when ingesting the dry-run messages", ) parser.add_argument( "-c", @@ -69,5 +71,5 @@ def main() -> None: sys.exit(1) -if __name__ == "__main__": # pragma: no cover - entry point +if __name__ == "__main__": main() diff --git a/pyproject.toml b/pyproject.toml index d8a9771..a6ac5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ test = ["coverage", "pytest", "pytest-cov==6.0.0"] [tool.coverage.run] omit = [ + "*/netboxlabs/diode/scripts/*", "*/netboxlabs/diode/sdk/ingester.py", "*/netboxlabs/diode/sdk/diode/*", "*/netboxlabs/diode/sdk/validate/*", @@ -46,10 +47,11 @@ omit = [ "Homepage" = "https://netboxlabs.com/" [project.scripts] -diode-replay-dryrun = "netboxlabs.diode.sdk.dryrun_replay:main" +diode-replay-dryrun = "netboxlabs.diode.scripts.dryrun_replay:main" [tool.setuptools] packages = [ + "netboxlabs.diode.scripts", "netboxlabs.diode.sdk", "netboxlabs.diode.sdk.diode", "netboxlabs.diode.sdk.diode.v1",