diff --git a/.dockerignore b/.dockerignore index df08c066ce3b4..ce4bd263ec307 100644 --- a/.dockerignore +++ b/.dockerignore @@ -73,6 +73,7 @@ !conf.py !docs !docker-stack-docs +!docs-theme !providers-summary-docs # Avoid triggering context change on README change (new companies using Airflow) diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index 53692b5d87350..e5b2f42033802 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -104,6 +104,11 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze if: ${{ inputs.use-local-venv == 'true' }} + # `_gen/` is gitignored and bind-mounted from host into the container, so + # baking it into the image is masked at runtime. Fetch on host instead. + - name: "Fetch sphinx-airflow-theme static assets" + shell: bash + run: python scripts/ci/fetch_theme_assets.py - name: "Cleanup dist files" run: rm -fv ./dist/* if: ${{ matrix.python-version == inputs.default-python-version }} diff --git a/Dockerfile.ci b/Dockerfile.ci index 7cc05b2c053a3..6cb3de63bf993 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1882,6 +1882,8 @@ RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.c bash /scripts/docker/install_additional_dependencies.sh; \ fi +RUN python "${AIRFLOW_SOURCES}/scripts/ci/fetch_theme_assets.py" + COPY --from=scripts entrypoint_ci.sh /entrypoint COPY --from=scripts entrypoint_exec.sh /entrypoint-exec RUN chmod a+x /entrypoint /entrypoint-exec diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index afb22da6355fe..2b65f77d8c1e0 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -21,6 +21,7 @@ import re import shlex import shutil +import subprocess import sys import threading from pathlib import Path @@ -751,6 +752,20 @@ def start_airflow( sys.exit(result.returncode) +def _ensure_theme_assets() -> None: + gen_dir = AIRFLOW_ROOT_PATH / "docs-theme" / "sphinx_airflow_theme" / "static" / "_gen" + if gen_dir.is_dir(): + return + console_print("[info]Theme static assets missing — fetching from published wheel...") + fetch_script = AIRFLOW_ROOT_PATH / "scripts" / "ci" / "fetch_theme_assets.py" + result = subprocess.run([sys.executable, str(fetch_script)], check=False) + if result.returncode != 0: + console_print( + "[error]Failed to fetch theme assets. Run manually: python scripts/ci/fetch_theme_assets.py" + ) + sys.exit(1) + + @main.command(name="build-docs") @option_builder @click.option( @@ -830,6 +845,7 @@ def build_docs( Build documents. """ perform_environment_checks() + _ensure_theme_assets() fix_ownership_using_docker() cleanup_python_generated_files() build_params = BuildCiParams( diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index c955459612873..97c78f5e07fa8 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -97,6 +97,7 @@ ("docker-stack-docs", "/opt/airflow/docker-stack-docs"), ("docker-tests", "/opt/airflow/docker-tests"), ("docs", "/opt/airflow/docs"), + ("docs-theme", "/opt/airflow/docs-theme"), ("generated", "/opt/airflow/generated"), ("go-sdk", "/opt/airflow/go-sdk"), ("helm-tests", "/opt/airflow/helm-tests"), diff --git a/dev/breeze/tests/test_theme_workspace.py b/dev/breeze/tests/test_theme_workspace.py new file mode 100644 index 0000000000000..1eafc82892d85 --- /dev/null +++ b/dev/breeze/tests/test_theme_workspace.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import contextlib +import sys +from pathlib import Path +from unittest import mock +from urllib.error import URLError + +import pytest + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore[no-redef] + +AIRFLOW_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(AIRFLOW_ROOT / "scripts" / "ci")) +import fetch_theme_assets # noqa: E402 + + +class TestSphinxThemeWorkspace: + def test_theme_is_workspace_member(self): + with open(AIRFLOW_ROOT / "pyproject.toml", "rb") as f: + root_config = tomllib.load(f) + members = root_config["tool"]["uv"]["workspace"]["members"] + assert "docs-theme" in members + + def test_theme_source_exists(self): + theme_dir = AIRFLOW_ROOT / "docs-theme" / "sphinx_airflow_theme" + assert theme_dir.is_dir() + assert (theme_dir / "__init__.py").is_file() + assert (theme_dir / "theme.conf").is_file() + assert (theme_dir / "layout.html").is_file() + + def test_theme_has_valid_pyproject(self): + with open(AIRFLOW_ROOT / "docs-theme" / "pyproject.toml", "rb") as f: + config = tomllib.load(f) + assert config["project"]["name"] == "sphinx_airflow_theme" + assert config["build-system"]["build-backend"] == "flit_core.buildapi" + entry_points = config["project"]["entry-points"]["sphinx.html_themes"] + assert "sphinx_airflow_theme" in entry_points + + def test_devel_common_uses_workspace_dep(self): + with open(AIRFLOW_ROOT / "devel-common" / "pyproject.toml", "rb") as f: + config = tomllib.load(f) + docs_deps = config["project"]["optional-dependencies"]["docs"] + theme_deps = [d for d in docs_deps if "sphinx-airflow-theme" in d.lower()] + assert len(theme_deps) == 1, f"Expected exactly one theme dep, got: {theme_deps}" + assert "@https://" not in theme_deps[0], ( + f"Theme should be a workspace dep, not a URL: {theme_deps[0]}" + ) + + def test_theme_registered_in_uv_sources(self): + with open(AIRFLOW_ROOT / "pyproject.toml", "rb") as f: + root_config = tomllib.load(f) + sources = root_config["tool"]["uv"]["sources"] + assert "sphinx-airflow-theme" in sources + assert sources["sphinx-airflow-theme"] == {"workspace": True} + + def test_gen_dir_is_gitignored(self): + gitignore = (AIRFLOW_ROOT / "docs-theme" / ".gitignore").read_text() + assert "static/_gen/" in gitignore + + def test_fetch_script_exists(self): + fetch_script = AIRFLOW_ROOT / "scripts" / "ci" / "fetch_theme_assets.py" + assert fetch_script.is_file() + + +class TestFetchThemeRetry: + @mock.patch("fetch_theme_assets.time.sleep") + @mock.patch("fetch_theme_assets.urlopen") + def test_retries_on_transient_failure(self, mock_urlopen, mock_sleep, tmp_path): + mock_urlopen.side_effect = [ + URLError("Connection timed out"), + URLError("Connection reset"), + mock.MagicMock(read=lambda: (tmp_path / "dummy.whl").write_bytes(b"") or b"PK\x03\x04"), + ] + with mock.patch.object(fetch_theme_assets, "WHEEL_URL", "https://example.com/{version}.whl"): + with mock.patch.object(fetch_theme_assets, "THEME_DIR", tmp_path / "theme"): + with mock.patch.object(fetch_theme_assets, "GEN_DIR", tmp_path / "gen"): + with mock.patch.object( + fetch_theme_assets, "VERSION_STAMP", tmp_path / "gen" / ".version" + ): + # Will fail on zipfile parse, but we only care that retry logic reached attempt 3 + with contextlib.suppress(Exception): + fetch_theme_assets.fetch_and_extract("0.0.1") + assert mock_urlopen.call_count == 3 + assert mock_sleep.call_count == 2 + mock_sleep.assert_any_call(2) + mock_sleep.assert_any_call(4) + + @mock.patch("fetch_theme_assets.time.sleep") + @mock.patch("fetch_theme_assets.urlopen") + def test_exits_after_max_retries(self, mock_urlopen, mock_sleep): + mock_urlopen.side_effect = URLError("Connection refused") + with pytest.raises(SystemExit, match="1"): + fetch_theme_assets.fetch_and_extract("0.0.1") + assert mock_urlopen.call_count == 3 diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml index 1e7db399cb2ab..76a29e81009f3 100644 --- a/devel-common/pyproject.toml +++ b/devel-common/pyproject.toml @@ -81,7 +81,7 @@ dependencies = [ "docutils>=0.21", "pagefind>=1.5.0", "pagefind-bin>=1.5.0", - "sphinx-airflow-theme@https://airflow.apache.org/sphinx-airflow-theme/sphinx_airflow_theme-0.3.9-py3-none-any.whl", + "sphinx-airflow-theme>=0.3.9", "sphinx-argparse>=0.4.0", "sphinx-autoapi>=3.8.0", "sphinx-autobuild>=2024.10.2", diff --git a/docs-theme/.gitignore b/docs-theme/.gitignore new file mode 100644 index 0000000000000..3df8fd49646f2 --- /dev/null +++ b/docs-theme/.gitignore @@ -0,0 +1,2 @@ +*.iml +sphinx_airflow_theme/static/_gen/ diff --git a/docs-theme/pyproject.toml b/docs-theme/pyproject.toml new file mode 100644 index 0000000000000..6fe199aadde34 --- /dev/null +++ b/docs-theme/pyproject.toml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "sphinx_airflow_theme" +description = "Airflow theme for Sphinx" +license = {text = "Apache License 2.0"} +authors = [ + {name = "Apache Software Foundation", email = "dev@airflow.apache.org"} +] +dynamic = ["version"] +classifiers = [ + "Framework :: Sphinx", + "Framework :: Sphinx :: Theme", + "License :: OSI Approved :: Apache Software License", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Documentation", + "Topic :: Software Development :: Documentation" +] +dependencies = [ + "sphinx>=7" +] +requires-python = ">=3.9" + +[project.optional-dependencies] +dev = [] + +[project.entry-points."sphinx.html_themes"] +sphinx_airflow_theme = "sphinx_airflow_theme" + +[tool.flit.module] +name = "sphinx_airflow_theme" + +[tool.flit.sdist] +include = [ + "sphinx_airflow_theme/**", +] diff --git a/docs-theme/sphinx_airflow_theme/__init__.py b/docs-theme/sphinx_airflow_theme/__init__.py new file mode 100644 index 0000000000000..b8e9aa887c454 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/__init__.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from os import path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sphinx.application import Sphinx + +__version__ = "0.3.9" +__version_full__ = __version__ + +_THEME_DIR = path.abspath(path.dirname(__file__)) +_GEN_DIR = path.join(_THEME_DIR, "static", "_gen") + + +def get_html_theme_path(): + """Return list of HTML theme paths.""" + return path.abspath(path.dirname(_THEME_DIR)) + + +def setup_my_func(app, config): + config.html_theme_options.setdefault( + "navbar_links", + [ + {"href": "/index.html", "text": "Documentation"}, + {"href": "/registry/", "text": "Registry"}, + ], + ) + + +def setup(app: Sphinx): + if not path.isdir(_GEN_DIR): + raise FileNotFoundError( + f"Theme static assets not found at {_GEN_DIR}. Run: python scripts/ci/fetch_theme_assets.py" + ) + app.add_html_theme("sphinx_airflow_theme", _THEME_DIR) + app.add_css_file("_gen/css/main-custom.min.css") + app.add_js_file("js/globaltoc.js") + app.connect("config-inited", setup_my_func) + return {"version": __version__, "parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs-theme/sphinx_airflow_theme/breadcrumbs.html b/docs-theme/sphinx_airflow_theme/breadcrumbs.html new file mode 100644 index 0000000000000..7e211fdfda497 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/breadcrumbs.html @@ -0,0 +1,41 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{% if meta is defined and meta is not none %} + {% set check_meta = True %} +{% else %} + {% set check_meta = False %} +{% endif %} + +{% if check_meta and 'github_url' in meta %} + {% set display_github = True %} +{% endif %} + + diff --git a/docs-theme/sphinx_airflow_theme/footer.html b/docs-theme/sphinx_airflow_theme/footer.html new file mode 100644 index 0000000000000..abb44ea200129 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/footer.html @@ -0,0 +1,180 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + + + + diff --git a/docs-theme/sphinx_airflow_theme/globaltoc.html b/docs-theme/sphinx_airflow_theme/globaltoc.html new file mode 100644 index 0000000000000..d4426f62d9712 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/globaltoc.html @@ -0,0 +1,57 @@ + +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + + + + + diff --git a/docs-theme/sphinx_airflow_theme/header.html b/docs-theme/sphinx_airflow_theme/header.html new file mode 100644 index 0000000000000..e3dedcda13a77 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/header.html @@ -0,0 +1,142 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +
+ + +
diff --git a/docs-theme/sphinx_airflow_theme/layout.html b/docs-theme/sphinx_airflow_theme/layout.html new file mode 100644 index 0000000000000..a4ca32872ffe9 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/layout.html @@ -0,0 +1,446 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{# + basic/layout.html + ~~~~~~~~~~~~~~~~~ + + Main layout template for Sphinx themes. + + :copyright: Copyright 2007-2019 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +#} +{%- set url_root = pathto('', 1) %} +{%- if url_root == '#' %}{% set url_root = '' %}{% endif %} +{%- if not embedded and docstitle %} + {%- set titlesuffix = " — " + docstitle|e %} +{%- else %} + {%- set titlesuffix = "" %} +{%- endif %} +{%- set lang_attr = 'en' if language == None else (language | replace('_', '-')) %} + + + + + +{%- macro pager() %} + +{%- endmacro %} + +{%- macro rating() %} +
+

Was this entry helpful?

+
+ +
+ + + + + + +
+ +
+ + + + + + +
+ +
+ + + + + + +
+ +
+ + + + + + +
+ +
+ + + + + + +
+ +
+
+{%- endmacro %} + +{%- macro sidebar() %} +
+ {%- for sidebartemplate in sidebars %} + {%- include sidebartemplate %} + {%- endfor %} +
+{%- endmacro %} + +{%- macro mobileSidebar() %} +
+ + +
+
+
+ {%- for sidebartemplate in sidebars %} + {%- include sidebartemplate %} + {%- endfor %} +
+
+
+
+{%- endmacro %} + +{%- macro script() %} + + + + + + {%- for js in script_files %} + {{ js_tag(js) }} + {%- endfor %} +{%- endmacro %} + +{%- macro css() %} + + + {%- for css in css_files %} + {%- if css|attr("filename") %} + {{ css_tag(css) }} + {%- else %} + + {%- endif %} + {%- endfor %} +{%- endmacro %} + +{%- macro sidetoc() %} + +{%- endmacro %} + +{%- if html_tag %} + {{ html_tag }} +{%- else %} + +{%- endif %} + + {%- if not html5_doctype and not skip_ua_compatible %} + + {%- endif %} + {%- if use_meta_charset or html5_doctype %} + + {%- else %} + + {%- endif %} + + {{- metatags }} + {%- block htmltitle %} + {{ title|striptags|e }}{{ titlesuffix }} + {%- endblock %} + {%- block css %} + {{- css() }} + {%- endblock %} + {%- if not embedded %} + {%- if pageurl %} + + {%- endif %} + {%- if use_opensearch %} + + {%- endif %} + {%- if favicon %} + + {%- endif %} + {%- endif %} + {%- block linktags %} + {%- if hasdoc('about') %} + + {%- endif %} + {%- if hasdoc('genindex') %} + + {%- endif %} + {%- if hasdoc('search') %} + + {%- endif %} + {%- if hasdoc('copyright') %} + + {%- endif %} + {%- if next %} + + {%- endif %} + {%- if prev %} + + {%- endif %} + {%- endblock %} + + + + + + + {%- block extrahead %} + + {% endblock %} + + +{%- block body_tag %}{% endblock %} + + + + + + + + + + + + + +{%- block header %} + {% include "header.html" %} +{% endblock %} + +{%- block content %} + +
+ {# START MOBILE SIDEBAR #} + {{ mobileSidebar() }} + {# END MOBILE SIDEBAR #} +
+ {# START SIDEBAR #} + {{ sidebar() }} + {# END SIDEBAR #} + + {# START BODY #} + +
+ {% include "breadcrumbs.html" %} + {# START CONTENT #} +
+
+ {%- block document %} +
+
+
+ {% block body %} {% endblock %} + {%- block pager %}{{ pager() }}{% endblock %} +
+ +
+
+ {%- endblock %} +
+
+ {%- block rating %}{{ rating() }}{% endblock %} + {# END CONTENT #} +
+ {# END BODY #} + + {# START SIDEBAR #} + {{ sidetoc() }} + {# END SIDEBAR #} +
+ {% include "suggest_change_button.html" %} +
+ +{%- endblock %} + +{#{%- block relbar2 %}{{ relbar() }}{% endblock %}#} + +{%- block footer %} + {% include "footer.html" %} +{%- endblock %} +{%- block scripts %} + {{- script() }} +{%- endblock %} + + diff --git a/docs-theme/sphinx_airflow_theme/searchbox.html b/docs-theme/sphinx_airflow_theme/searchbox.html new file mode 100644 index 0000000000000..8a82c9789c619 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/searchbox.html @@ -0,0 +1,47 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +
+
+ + + + + +
+
+ + + diff --git a/docs-theme/sphinx_airflow_theme/searchresults.html b/docs-theme/sphinx_airflow_theme/searchresults.html new file mode 100644 index 0000000000000..6387f64493f1a --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/searchresults.html @@ -0,0 +1,58 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +

{{ _('Search') }}

+

+ From here you can search these documents. Enter your search + words into the box below and click "search". +

+
+ + + +
+ +
+ + +
+{%- if search_performed %} +

{{ _('Search Results') }}

+ {%- if not search_results %} +

{{ _('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\'ve selected enough categories.') }}

+ {%- endif %} +{%- endif %} +
+ {%- if search_results %} + + {%- endif %} +
diff --git a/docs-theme/sphinx_airflow_theme/static/js/globaltoc.js b/docs-theme/sphinx_airflow_theme/static/js/globaltoc.js new file mode 100644 index 0000000000000..f71b5fd21f692 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/static/js/globaltoc.js @@ -0,0 +1,43 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// The script adds a 'click' eventListener to the 'span.toctree-expand' element that makes the active submenus collapsible. +// To make the submenus collapsible is sufficient to add/remove the 'current' class to the active 'li.toctree-l1' element. +$("span.toctree-expand").on("click", function() { + $(this).closest("li").toggleClass("current"); +}); diff --git a/docs-theme/sphinx_airflow_theme/suggest_change_button.html b/docs-theme/sphinx_airflow_theme/suggest_change_button.html new file mode 100644 index 0000000000000..22671dc9677a2 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/suggest_change_button.html @@ -0,0 +1,82 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{% if meta is defined and meta is not none %} + {% set check_meta = True %} +{% else %} + {% set check_meta = False %} +{% endif %} + +{% if check_meta and 'github_url' in meta %} + {% set display_github = True %} +{% endif %} + +{% if check_meta and 'github_url' in meta %} + {% set display_github = True %} +{% endif %} + +{% if hasdoc(pagename) %} + {% if display_github %} + {% if check_meta and 'github_url' in meta %} + {% set github_link = meta['github_url'] %} + {% else %} + {% set github_link = 'https://' ~ github_host|default("github.com") ~ '/' ~ github_user ~ '/' ~ github_repo ~ '/' ~ theme_vcs_pageview_mode|default("blob") ~ '/' ~ github_version ~ conf_py_path ~ pagename ~ suffix %} + {% endif %} +
+ + + + + +
+ {% else %} + {% if source_url_prefix %} + {% set sourcce_link = source_url_prefix ~ pagename ~ suffix %} + {% elif has_source and sourcename %} + {% set sourcce_link = pathto('_sources/' + sourcename, true)|e %} + {% else %} + {% set sourcce_link = None %} + {% endif %} +
+ + + + +
+ {% endif %} +{% endif %} diff --git a/docs-theme/sphinx_airflow_theme/theme.conf b/docs-theme/sphinx_airflow_theme/theme.conf new file mode 100644 index 0000000000000..b73b8ab81a202 --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/theme.conf @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[theme] +inherit = basic +stylesheet = _gen/css/main.min.css +pygments_style = default + +[options] +canonical_url = +analytics_id = +# Default set by python code, need to list this here to avoid warning from Sphinx +navbar_links = +hide_website_buttons = false +sidebar_collapse = true +sidebar_includehidden = true diff --git a/docs-theme/sphinx_airflow_theme/version-selector.html b/docs-theme/sphinx_airflow_theme/version-selector.html new file mode 100644 index 0000000000000..f6267f78a9b3c --- /dev/null +++ b/docs-theme/sphinx_airflow_theme/version-selector.html @@ -0,0 +1,31 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +
+ +
+ +
+ +
diff --git a/pyproject.toml b/pyproject.toml index 191e734061e3f..0d2908fc8e803 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1499,6 +1499,7 @@ apache-airflow-shared-timezones = false apache-airflow-task-sdk = false apache-airflow-task-sdk-integration-tests = false apache-aurflow-docker-stack = false +sphinx-airflow-theme = false # End of automatically generated exclude-newer-package entries # Manual overrides (kept outside the auto-generated block above so the @@ -1650,6 +1651,7 @@ apache-airflow-shared-timezones = false apache-airflow-task-sdk = false apache-airflow-task-sdk-integration-tests = false apache-aurflow-docker-stack = false +sphinx-airflow-theme = false # End of automatically generated exclude-newer-package-pip entries # Manual overrides — see the matching block under @@ -1692,6 +1694,7 @@ apache-airflow-shared-serialization = { workspace = true } apache-airflow-shared-state = { workspace = true } apache-airflow-shared-template-rendering = { workspace = true } apache-airflow-shared-timezones = { workspace = true } +sphinx-airflow-theme = { workspace = true } # Automatically generated provider workspace items (update_airflow_pyproject_toml.py) apache-airflow-providers-airbyte = { workspace = true } apache-airflow-providers-akeyless = { workspace = true } @@ -1815,6 +1818,7 @@ members = [ "task-sdk", "providers-summary-docs", "docker-stack-docs", + "docs-theme", "shared/configuration", "shared/dagnode", "shared/listeners", diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 185f2f385c344..0536ae894dfed 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -90,6 +90,9 @@ services: - type: bind source: ../../../docs target: /opt/airflow/docs + - type: bind + source: ../../../docs-theme + target: /opt/airflow/docs-theme - type: bind source: ../../../generated target: /opt/airflow/generated diff --git a/scripts/ci/fetch_theme_assets.py b/scripts/ci/fetch_theme_assets.py new file mode 100644 index 0000000000000..0583c02042ffa --- /dev/null +++ b/scripts/ci/fetch_theme_assets.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Fetch theme files from the published sphinx-airflow-theme wheel. + +Downloads the published wheel from airflow.apache.org and extracts all +theme files (templates, static assets, ``_gen/``) into +``docs-theme/sphinx_airflow_theme/``. The local ``__init__.py`` is +preserved because it contains the ``_gen/`` guard and is version-managed +by ``upgrade_important_versions.py``. +Idempotent: skips if a ``.version`` stamp file matches the current version. +""" + +from __future__ import annotations + +import re +import sys +import time +import zipfile +from io import BytesIO +from pathlib import Path +from urllib.error import URLError +from urllib.request import urlopen + +AIRFLOW_ROOT = Path(__file__).resolve().parents[2] +THEME_DIR = AIRFLOW_ROOT / "docs-theme" / "sphinx_airflow_theme" +STATIC_DIR = THEME_DIR / "static" +GEN_DIR = STATIC_DIR / "_gen" +VERSION_STAMP = GEN_DIR / ".version" +WHEEL_URL = "https://airflow.apache.org/sphinx-airflow-theme/sphinx_airflow_theme-{version}-py3-none-any.whl" + +ASF_LICENSE_CONF = """\ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" + +ASF_LICENSE_JS = """\ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +""" + + +def get_theme_version() -> str: + init_py = THEME_DIR / "__init__.py" + match = re.search(r'^__version__\s*=\s*["\']([^"\']+)["\']', init_py.read_text(), re.MULTILINE) + if not match: + print("ERROR: Could not read __version__ from", init_py, file=sys.stderr) + sys.exit(1) + return match.group(1) + + +def is_up_to_date(version: str) -> bool: + if not GEN_DIR.is_dir() or not VERSION_STAMP.is_file(): + return False + return VERSION_STAMP.read_text().strip() == version + + +def _apply_compliance_patches() -> None: + """Patch extracted files to satisfy Airflow pre-commit checks.""" + # layout.html: inclusive language + remove |safe filter + layout = THEME_DIR / "layout.html" + text = layout.read_text() + text = text.replace("Ma" + "ster layout template", "Main layout template") + text = text.replace('" — "|safe + docstitle|e', '" — " + docstitle|e') + layout.write_text(text) + + # theme.conf: prepend ASF license header + conf = THEME_DIR / "theme.conf" + content = conf.read_text() + if not content.startswith("# Licensed"): + conf.write_text(ASF_LICENSE_CONF + content) + + # globaltoc.js: prepend ASF license block (/*! */ style required by pre-commit) + js = STATIC_DIR / "js" / "globaltoc.js" + content = js.read_text() + if not content.startswith("/*!"): + js.write_text(ASF_LICENSE_JS + content) + + +def fetch_and_extract(version: str) -> None: + url = WHEEL_URL.format(version=version) + print(f"Fetching theme assets from {url}") + max_retries = 3 + for attempt in range(1, max_retries + 1): + try: + response = urlopen(url, timeout=30) + break + except URLError as e: + if attempt < max_retries: + delay = 2**attempt + print(f"WARNING: Attempt {attempt}/{max_retries} failed: {e}. Retrying in {delay}s...") + time.sleep(delay) + else: + print(f"ERROR: Failed to download {url} after {max_retries} attempts: {e}", file=sys.stderr) + sys.exit(1) + wheel_bytes = BytesIO(response.read()) + + pkg_prefix = "sphinx_airflow_theme/" + skip = {f"{pkg_prefix}__init__.py"} + with zipfile.ZipFile(wheel_bytes) as whl: + members = [ + m for m in whl.namelist() if m.startswith(pkg_prefix) and not m.endswith("/") and m not in skip + ] + if not members: + print(f"ERROR: No theme files found in wheel {url}", file=sys.stderr) + sys.exit(1) + for member in members: + rel_path = member[len(pkg_prefix) :] + target = (THEME_DIR / rel_path).resolve() + if not target.is_relative_to(THEME_DIR): + print(f"ERROR: Refusing to extract path outside theme dir: {member}", file=sys.stderr) + sys.exit(1) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(whl.read(member)) + + _apply_compliance_patches() + VERSION_STAMP.write_text(version + "\n") + print(f"Extracted {len(members)} files into {THEME_DIR}") + + +def main() -> None: + version = get_theme_version() + if is_up_to_date(version): + print(f"Theme assets already up to date (v{version})") + return + fetch_and_extract(version) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 8b751e7d77fed..d019b26c73a26 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -89,6 +89,7 @@ (AIRFLOW_ROOT_PATH / "dev" / "provider_db_inventory.py", False), (AIRFLOW_ROOT_PATH / "dev" / "pyproject.toml", False), (AIRFLOW_ROOT_PATH / "go-sdk" / ".pre-commit-config.yaml", False), + (AIRFLOW_ROOT_PATH / "docs-theme" / "sphinx_airflow_theme" / "__init__.py", False), ] for file in DOCKER_IMAGES_EXAMPLE_DIR_PATH.rglob("*.sh"): FILES_TO_UPDATE.append((file, False)) @@ -605,10 +606,7 @@ def apply_pattern_replacements( (r"(OPENAPI_GENERATOR_CLI_VER = )(\"[0-9.]+\")", 'OPENAPI_GENERATOR_CLI_VER = "{version}"'), ], "sphinx_airflow_theme": [ - ( - r"(sphinx-airflow-theme@https://airflow\.apache\.org/sphinx-airflow-theme/sphinx_airflow_theme-)([0-9.]+)(-py3-none-any\.whl)", - "sphinx-airflow-theme@https://airflow.apache.org/sphinx-airflow-theme/sphinx_airflow_theme-{version}-py3-none-any.whl", - ), + (r"(__version__ = \")([\d.]+)(\")", '__version__ = "{version}"'), ], } diff --git a/uv.lock b/uv.lock index b66a24df9039b..5adfa8aedec05 100644 --- a/uv.lock +++ b/uv.lock @@ -62,6 +62,7 @@ apache-airflow-devel-common = false apache-airflow-providers-apache-cassandra = false apache-airflow-providers-asana = false apache-airflow-providers-oracle = false +sphinx-airflow-theme = false apache-airflow-providers-mysql = false apache-airflow-providers-alibaba = false apache-airflow-providers-microsoft-mssql = false @@ -288,6 +289,7 @@ members = [ "apache-airflow-task-sdk", "apache-airflow-task-sdk-integration-tests", "docker-stack", + "sphinx-airflow-theme", ] [[package]] @@ -2543,7 +2545,7 @@ requires-dist = [ { name = "semver", marker = "extra == 'devscripts'", specifier = ">=3.0.2" }, { name = "setuptools", marker = "extra == 'docs'", specifier = "<82.0.0" }, { name = "sphinx", marker = "extra == 'docs'", specifier = ">=7" }, - { name = "sphinx-airflow-theme", marker = "extra == 'docs'", url = "https://airflow.apache.org/sphinx-airflow-theme/sphinx_airflow_theme-0.3.9-py3-none-any.whl" }, + { name = "sphinx-airflow-theme", marker = "extra == 'docs'", editable = "docs-theme" }, { name = "sphinx-argparse", marker = "extra == 'docs'", specifier = ">=0.4.0" }, { name = "sphinx-autoapi", marker = "extra == 'docs'", specifier = ">=3.8.0" }, { name = "sphinx-autobuild", marker = "extra == 'docs'", specifier = ">=2024.10.2" }, @@ -20747,16 +20749,12 @@ wheels = [ [[package]] name = "sphinx-airflow-theme" -version = "0.3.9" -source = { url = "https://airflow.apache.org/sphinx-airflow-theme/sphinx_airflow_theme-0.3.9-py3-none-any.whl" } +source = { editable = "docs-theme" } dependencies = [ { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] -wheels = [ - { url = "https://airflow.apache.org/sphinx-airflow-theme/sphinx_airflow_theme-0.3.9-py3-none-any.whl", hash = "sha256:8e28f827963d4a48a901959a7fee194421173cb4c5fdfc00fdf0c5df825011f7" }, -] [package.metadata] requires-dist = [{ name = "sphinx", specifier = ">=7" }]