Skip to content

Generate SBOMs for providers #35667

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pierrejeambrun merged 3 commits into from
Nov 15, 2023
Merged

Generate SBOMs for providers #35667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e908752
Working POC
pierrejeambrun Nov 13, 2023
94b5048
Small adjustements and cleaning
pierrejeambrun Nov 15, 2023
507655c
resolve merge conflicts
pierrejeambrun Nov 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 133 additions & 45 deletions dev/breeze/src/airflow_breeze/commands/sbom_commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from __future__ import annotations

import json
import os
import sys
from pathlib import Path

Expand All @@ -29,10 +30,14 @@
PROVIDER_DEPENDENCIES,
)
from airflow_breeze.utils.cdxgen import (
PROVIDER_REQUIREMENTS_DIR_PATH,
SbomApplicationJob,
SbomCoreJob,
SbomProviderJob,
build_all_airflow_versions_base_image,
get_cdxgen_port_mapping,
get_requirements_for_provider,
list_providers_from_providers_requirements,
)
from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.click_utils import BreezeGroup
Expand All @@ -57,7 +62,7 @@
check_async_run_results,
run_with_pool,
)
from airflow_breeze.utils.path_utils import AIRFLOW_TMP_DIR_PATH, PROVIDER_METADATA_JSON_FILE_PATH
from airflow_breeze.utils.path_utils import FILES_SBOM_DIR, PROVIDER_METADATA_JSON_FILE_PATH
from airflow_breeze.utils.shared_options import get_dry_run


Expand All @@ -71,10 +76,11 @@ def sbom():


SBOM_INDEX_TEMPLATE = """
{% set project_name = " " + provider_id + " " if provider_id else " " -%}
<html>
<head><title>CycloneDX SBOMs for Apache Airflow {{ version }}</title></head>
<head><title>CycloneDX SBOMs for Apache Airflow{{project_name}}{{ version }}</title></head>
<body>
<h1>CycloneDX SBOMs for Apache Airflow {{ version }}</h1>
<h1>CycloneDX SBOMs for Apache Airflow{{project_name}}{{ version }}</h1>
<ul>
{% for sbom_file in sbom_files %}
<li><a href="{{ sbom_file.name }}">{{ sbom_file.name }}</a></li>
Expand Down Expand Up @@ -119,6 +125,14 @@ def sbom():
@option_verbose
@option_dry_run
@option_answer
@click.option(
"--package-filter",
help="List of packages to consider. You can use `apache-airflow` for core "
"or `apache-airflow-providers` to consider all the providers.",
type=BetterChoice(["apache-airflow-providers", "apache-airflow"]),
required=False,
default="apache-airflow",
)
def update_sbom_information(
airflow_site_directory: Path,
airflow_version: str | None,
Expand All @@ -130,6 +144,7 @@ def update_sbom_information(
include_success_outputs: bool,
skip_cleanup: bool,
force: bool,
package_filter: tuple[str, ...],
):
import jinja2
from jinja2 import StrictUndefined
Expand All @@ -148,55 +163,114 @@ def update_sbom_information(
python_versions = ALL_HISTORICAL_PYTHON_VERSIONS
else:
python_versions = [python]
application_root_path = AIRFLOW_TMP_DIR_PATH
application_root_path = FILES_SBOM_DIR
start_cdxgen_server(application_root_path, run_in_parallel, parallelism)

jobs_to_run: list[SbomApplicationJob] = []

apache_airflow_directory = airflow_site_directory / "docs-archive" / "apache-airflow"
airflow_site_archive_directory = airflow_site_directory / "docs-archive"

for airflow_v in airflow_versions:
airflow_version_dir = apache_airflow_directory / airflow_v
if not airflow_version_dir.exists():
get_console().print(f"[warning]The {airflow_version_dir} does not exist. Skipping")
continue
destination_dir = airflow_version_dir / "sbom"
if destination_dir.exists():
def _dir_exists_warn_and_should_skip(dir: Path, force: bool) -> bool:
if dir.exists():
if not force:
get_console().print(f"[warning]The {destination_dir} already exists. Skipping")
continue
get_console().print(f"[warning]The {dir} already exists. Skipping")
return True
else:
get_console().print(f"[warning]The {destination_dir} already exists. Forcing update")
get_console().print(f"[warning]The {dir} already exists. Forcing update")
return False
return False

destination_dir.mkdir(parents=True, exist_ok=True)
if package_filter == "apache-airflow":
# Create core jobs
apache_airflow_documentation_directory = airflow_site_archive_directory / "apache-airflow"

get_console().print(f"[info]Attempting to update sbom for {airflow_v}.")
get_console().print(f"[success]The {destination_dir} exists. Proceeding.")
for python_version in python_versions:
target_sbom_file_name = f"apache-airflow-sbom-{airflow_v}-python{python_version}.json"
target_sbom_path = destination_dir / target_sbom_file_name
if target_sbom_path.exists():
if not force:
get_console().print(f"[warning]The {target_sbom_path} already exists. Skipping")
for airflow_v in airflow_versions:
airflow_version_dir = apache_airflow_documentation_directory / airflow_v
if not airflow_version_dir.exists():
get_console().print(f"[warning]The {airflow_version_dir} does not exist. Skipping")
continue
destination_dir = airflow_version_dir / "sbom"

if _dir_exists_warn_and_should_skip(destination_dir, force):
continue

destination_dir.mkdir(parents=True, exist_ok=True)

get_console().print(f"[info]Attempting to update sbom for {airflow_v}.")
for python_version in python_versions:
target_sbom_file_name = f"apache-airflow-sbom-{airflow_v}-python{python_version}.json"
target_sbom_path = destination_dir / target_sbom_file_name

if _dir_exists_warn_and_should_skip(target_sbom_path, force):
continue
else:
get_console().print(f"[warning]The {target_sbom_path} already exists. Forcing update")
jobs_to_run.append(
SbomApplicationJob(
airflow_version=airflow_v,
python_version=python_version,
application_root_path=application_root_path,
include_provider_dependencies=include_provider_dependencies,
target_path=target_sbom_path,

jobs_to_run.append(
SbomCoreJob(
airflow_version=airflow_v,
python_version=python_version,
application_root_path=application_root_path,
include_provider_dependencies=include_provider_dependencies,
target_path=target_sbom_path,
)
)
elif package_filter == "apache-airflow-providers":
# Create providers jobs
user_confirm(
"You are about to update sbom information for providers, did you refresh the "
"providers requirements with the command `breeze sbom generate-providers-requirements`?",
quit_allowed=False,
default_answer=Answer.YES,
)
for (
node_name,
provider_id,
provider_version,
provider_version_documentation_directory,
) in list_providers_from_providers_requirements(airflow_site_archive_directory):
destination_dir = provider_version_documentation_directory / "sbom"

if _dir_exists_warn_and_should_skip(destination_dir, force):
continue

destination_dir.mkdir(parents=True, exist_ok=True)

get_console().print(
f"[info]Attempting to update sbom for {provider_id} version {provider_version}."
)

python_versions = set(
dir_name.replace("python", "")
for dir_name in os.listdir(PROVIDER_REQUIREMENTS_DIR_PATH / node_name)
)

for python_version in python_versions:
target_sbom_file_name = (
f"apache-airflow-sbom-{provider_id}-{provider_version}-python{python_version}.json"
)
target_sbom_path = destination_dir / target_sbom_file_name

if _dir_exists_warn_and_should_skip(target_sbom_path, force):
continue

jobs_to_run.append(
SbomProviderJob(
provider_id=provider_id,
provider_version=provider_version,
python_version=python_version,
target_path=target_sbom_path,
folder_name=node_name,
)
)

if len(jobs_to_run) == 0:
get_console().print("[info]Nothing to do, there is no job to process")
return

if run_in_parallel:
parallelism = min(parallelism, len(jobs_to_run))
get_console().print(f"[info]Running {len(jobs_to_run)} jobs in parallel")
with ci_group(f"Generating SBoMs for {airflow_versions}:{python_versions}"):
all_params = [
f"Generate SBoMs for {job.airflow_version}:{job.python_version}" for job in jobs_to_run
]
with ci_group(f"Generating SBOMs for {jobs_to_run}"):
all_params = [f"Generate SBOMs for {job.get_job_name()}" for job in jobs_to_run]
with run_with_pool(
parallelism=parallelism,
all_params=all_params,
Expand All @@ -217,7 +291,7 @@ def update_sbom_information(
]
check_async_run_results(
results=results,
success="All SBoMs were generated successfully",
success="All SBOMs were generated successfully",
outputs=outputs,
include_success_outputs=include_success_outputs,
skip_cleanup=skip_cleanup,
Expand All @@ -226,20 +300,36 @@ def update_sbom_information(
for job in jobs_to_run:
produce_sbom_for_application_via_cdxgen_server(job, output=None)

for airflow_v in airflow_versions:
airflow_version_dir = apache_airflow_directory / airflow_v
destination_dir = airflow_version_dir / "sbom"
html_template = SBOM_INDEX_TEMPLATE

def _generate_index(destination_dir: Path, provider_id: str | None, version: str) -> None:
destination_index_path = destination_dir / "index.html"
get_console().print(f"[info]Generating index for {destination_dir}")
sbom_files = sorted(destination_dir.glob("apache-airflow-sbom-*"))
html_template = SBOM_INDEX_TEMPLATE
if not get_dry_run():
destination_index_path.write_text(
jinja2.Template(html_template, autoescape=True, undefined=StrictUndefined).render(
version=airflow_v, sbom_files=sbom_files
provider_id=provider_id,
version=version,
sbom_files=sbom_files,
)
)

if package_filter == "apache-airflow":
for airflow_v in airflow_versions:
airflow_version_dir = apache_airflow_documentation_directory / airflow_v
destination_dir = airflow_version_dir / "sbom"
_generate_index(destination_dir, None, airflow_v)
elif package_filter == "apache-airflow-providers":
for (
node_name,
provider_id,
provider_version,
provider_version_documentation_directory,
) in list_providers_from_providers_requirements(airflow_site_archive_directory):
destination_dir = provider_version_documentation_directory / "sbom"
_generate_index(destination_dir, provider_id, provider_version)


@sbom.command(name="build-all-airflow-images", help="Generate images with airflow versions pre-installed")
@option_historical_python_version
Expand Down Expand Up @@ -283,7 +373,6 @@ def build_all_airflow_images(
build_all_airflow_versions_base_image,
kwds={
"python_version": python_version,
"confirm": False,
"output": outputs[index],
},
)
Expand All @@ -300,7 +389,6 @@ def build_all_airflow_images(
for python_version in python_versions:
build_all_airflow_versions_base_image(
python_version=python_version,
confirm=False,
output=None,
)

Expand Down
1 change: 1 addition & 0 deletions dev/breeze/src/airflow_breeze/commands/sbom_commands_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"--airflow-version",
"--python",
"--include-provider-dependencies",
"--package-filter",
"--force",
],
},
Expand Down
Loading