Generalize import-flux2-assets.sh script

This is the first step towards supporting multiple versions of Flux to
be documented in parallel. The script has been condensed to only
require a single input parameter denoting the desired Flux major and
minor version for which to build the documentation.

This parameter is read from the environment variable `BRANCH` (which
Netlify) automatically provides and has to be either `main` or have
the format `vX-Y` which `X` denoting the major and `Y` denoting the
minor version desired to build the docs for.

The intent is to have branches following this naming pattern that
Netlify will directly deploy under the URL vX-Y.fluxcd.io.

The changed import-flux2-assets.sh script now conducts the following
steps in order to generate the docs for a specific version:

1. Find most recent patch version of Flux matching the given `BRANCH`.
2. For each controller fetch the `kustomization.yaml` from
   https://github.com/fluxcd/flux2/blob/FLUX_VERSION/manifests/bases/CONTROLLER/kustomization.yaml
3. From there fetch the `*.crds.yaml` file and the controller version.
4. For each CRD group version fetch the API docs from the controller
   repository as well as the spec docs.

Some special cases remain, e.g. for IRC/IAC which are supposed to be
documented under the same main navigation entry and for NC's `Event`
type which is not a CRD.

refs #1534

Co-authored-by: Kingdon Barrett <kingdon@weave.works>
Signed-off-by: Max Jonas Werner <mail@makk.es>
Signed-off-by: Kingdon Barrett <kingdon@weave.works>

.gitignore

@@ -15,3 +15,6 @@ static/install.sh
 
 # local GH token, used for some scripts
 .gh-token
+
+# build directories
+/bin

Makefile

@@ -13,18 +13,33 @@ HUGO_BIND_ADDRESS          ?= 127.0.0.1
 BUILDER_CLI                := docker
 # BUILDER_CLI                := okteto
 LYCHEE_IMAGE_NAME          ?= lycheeverse/lychee:202105190720247e4977
+YQ_VERSION                 ?= v4.34.2
+BRANCH                     ?= main
+
+REPO_ROOT := $(shell git rev-parse --show-toplevel)
+OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+BIN_DIR := $(REPO_ROOT)/bin
 
 help:  ## Display this help menu.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 .PHONY: help
 
 prereqs:
 	npm install
+	python3 -m pip install -r requirements.txt
+
+.PHONY: yq
+yq: $(BIN_DIR)/yq-$(YQ_VERSION)
+
+$(BIN_DIR)/yq-$(YQ_VERSION):
+	mkdir -p $(BIN_DIR) && wget https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(OS)_$(shell go env GOARCH) -O $(BIN_DIR)/yq-$(YQ_VERSION) && \
+	    chmod +x $(BIN_DIR)/yq-$(YQ_VERSION) && \
+	    cp $(BIN_DIR)/yq-$(YQ_VERSION) $(BIN_DIR)/yq
 
-gen-content: ## Generates content from external sources.
+gen-content: yq ## Generates content from external sources.
 	hack/gen-content.py
 	hack/import-calendar.py
-	hack/import-flux2-assets.sh
+	PATH=$(BIN_DIR):$(PATH) BRANCH=$(BRANCH) hack/import-flux2-assets.sh
 
 serve: gen-content prereqs ## Spawns a development server.
 	hugo server \
@@ -63,7 +78,7 @@ docker-preview: docker-serve
 .PHONY: docker-serve
 docker-serve:
 	docker run -v $(shell pwd):/site -p 1313:1313 -it $(SUPPORT_IMAGE_NAME) \
-		make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0
+		make \"MAKEFLAGS=$(MAKEFLAGS)\" serve HUGO_BIND_ADDRESS=0.0.0.0 GITHUB_TOKEN=$(GITHUB_TOKEN) BRANCH=$(BRANCH)
 
 .PHONY: docker-push docker-push-hugo docker-push-support
 docker-push: docker-push-hugo docker-push-support

README.md

@@ -32,11 +32,6 @@ In order to run the Flux site locally, you need to install:
 
 Once those tools are installed, fetch the assets necessary to run the site:
 
-```cli
-npm install
-python3 -m pip install -r requirements.txt
-```
-
 Then run the site in "server" mode:
 
 ```cli
@@ -45,13 +40,19 @@ make serve
 
 Navigate to <http://localhost:1313> to see the site running in your browser. As you make updates to the site, the browser will immediately update to reflect those changes.
 
+> **Note**: Sometimes you need to clear the output of previous builds, e.g. when the structure of imported documentation has changed. If you see duplicate pages being served, run
+> ```
+> git clean -fx -- content/
+> ```
+> and then run `make serve` again. The duplicates will then be gone.
+
 ## Publishing the site
 
 The Flux website is published automatically by [Netlify](https://netlify.com) when changes are pushed to the `main` branch. The site does not need to be published manually.
 
 ### Preview builds
 
-When you submit a pull request to this repository, Netlify builds a "deploy preview" of your changes. You can see that preview by clicking on the **deploy/netlify** link in the pull request window.
+When you submit a pull request to this repository, Netlify builds a "deploy preview" of your changes. You can see that preview by clicking on the **Details** link of the **netlify/fluxcd/deploy-preview** check at the bottom of the pull request page.
 
 ## Local Development (docker)
 

content/en/flux/components/helm/.gitignore

@@ -1,2 +1,2 @@
-api.md
+api/v*.md
 helmreleases.md

content/en/flux/components/helm/api/_index.md

@@ -0,0 +1,5 @@
+---
+title: API Reference
+linkTitle: API Reference
+weight: 1000
+---

content/en/flux/components/helm/options.md

@@ -2,7 +2,7 @@
 title: Controller Options
 linkTitle: Controller Options
 description: "Controller command flags and defaults."
-weight: 1000
+weight: 1
 ---
 
 To customise the controller options at install time,

content/en/flux/components/image/.gitignore

@@ -1,5 +1,5 @@
-automation-api.md
+automation-api/v*.md
+reflector-api/v*.md
 imagepolicies.md
 imagerepositories.md
 imageupdateautomations.md
-reflector-api.md

content/en/flux/components/image/automation-api/_index.md

@@ -0,0 +1,5 @@
+---
+title: Image Update Automation API Reference
+linkTitle: Image Update Automation API Reference
+weight: 70
+---

content/en/flux/components/image/options.md

@@ -2,7 +2,7 @@
 title: Controller Options
 linkTitle: Controller Options
 description: "Controller command flags and defaults."
-weight: 1000
+weight: 1
 ---
 
 To customise the controller options at install time,

content/en/flux/components/image/reflector-api/_index.md

@@ -0,0 +1,5 @@
+---
+title: Image Reflector API Reference
+linkTitle: Image Reflector API Reference
+weight: 70
+---

content/en/flux/components/kustomize/.gitignore

@@ -1,2 +1,2 @@
 api/v*.md
-kustomization.md
+kustomizations.md

content/en/flux/components/notification/.gitignore

@@ -1,5 +1,5 @@
-alert.md
+alerts.md
 api/v*.md
-event.md
-provider.md
-receiver.md
+events.md
+providers.md
+receivers.md

hack/import-flux2-assets.sh

@@ -1,9 +1,11 @@
-#!/bin/sh -x
+#!/usr/bin/env bash
+
+set -euxo pipefail
 
 COMPONENTS_DIR="content/en/flux/components"
 FLUX_DIR="content/en/flux/cmd"
 
-if [ -z "${GITHUB_USER}" ]; then
+if [ -z "${GITHUB_USER:-}" ]; then
     GITHUB_USER=fluxcdbot
 fi
 
@@ -19,7 +21,7 @@ fatal() {
 
 # Set os, fatal if operating system not supported
 setup_verify_os() {
-    if [ -z "${OS}" ]; then
+    if [ -z "${OS:-}" ]; then
         OS=$(uname)
     fi
     case ${OS} in
@@ -36,7 +38,7 @@ setup_verify_os() {
 
 # Set arch, fatal if architecture not supported
 setup_verify_arch() {
-    if [ -z "${ARCH}" ]; then
+    if [ -z "${ARCH:-}" ]; then
         ARCH=$(uname -m)
     fi
     case ${ARCH} in
@@ -58,22 +60,10 @@ setup_verify_arch() {
 }
 
 
-controller_version() {
-  url=https://api.github.com/repos/fluxcd/$1/releases
-  out=$(curl -w "%{http_code}" -o /tmp/releases -u "$GITHUB_USER:$GITHUB_TOKEN" -s "$url")
-  if [ "${out}" -ne "200" ] ; then
-      echo "error fetching releases from $url: $out"
-      cat /tmp/releases
-      exit 1
-  fi
-
-  jq -r '.[] | .tag_name' < /tmp/releases | sed 's/.*\///' | awk '{ if ($1 ~ /-/) print; else print $0"_" ; }' | sort -rV | sed 's/_$//' | head -n1
-}
-
 gen_crd_doc() {
   URL="$1"
   DEST="$2"
-  HUGETABLE="$3"
+  HUGETABLE="${3:-}"
 
   TMP="$(mktemp)"
   curl -u "$GITHUB_USER:$GITHUB_TOKEN" -# -Lf "$URL" > "$TMP"
@@ -85,7 +75,7 @@ gen_crd_doc() {
   # 3. While others go     # Here is the title you're looking for...
   #
 
-  FIRST_LINE="$(grep -vE "^<!--" "$TMP" | head -n1)"
+  FIRST_LINE="$(grep -vEm1 "^<!--" "$TMP")"
   if echo "$FIRST_LINE" | grep -q "<h1>" ; then
     TITLE="$(echo "$FIRST_LINE" | cut -d'<' -f2 | cut -d'>' -f2 | sed 's/^\#\ //')"
   elif echo "$FIRST_LINE" | grep -E "^# "; then
@@ -95,7 +85,7 @@ gen_crd_doc() {
     exit 1
   fi
 
-  WEIGHT="$(grep -E '^<!-- menuweight:[[:digit:]]+ -->$' "$TMP" | cut -d' ' -f2|cut -d':' -f2)"
+  WEIGHT="$(grep -E '^<!-- menuweight:[[:digit:]]+ -->$' "$TMP" | cut -d' ' -f2|cut -d':' -f2 || true)"
   if [ -z "${WEIGHT}" ] ; then
     WEIGHT=0
   fi
@@ -119,55 +109,47 @@ gen_crd_doc() {
   fi
 }
 
-{
-  # source-controller CRDs
-  SOURCE_VER="$(controller_version source-controller)"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/api/v1/source.md" "$COMPONENTS_DIR/source/api/v1.md" "HUGETABLE"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/api/v1beta2/source.md" "$COMPONENTS_DIR/source/api/v1beta2.md" "HUGETABLE"
-  SOURCE_CRD_VER="v1"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/gitrepositories.md" "$COMPONENTS_DIR/source/gitrepositories.md"
-  SOURCE_CRD_VER="v1beta2"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/ocirepositories.md" "$COMPONENTS_DIR/source/ocirepositories.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/helmrepositories.md" "$COMPONENTS_DIR/source/helmrepositories.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/helmcharts.md" "$COMPONENTS_DIR/source/helmcharts.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/source-controller/$SOURCE_VER/docs/spec/$SOURCE_CRD_VER/buckets.md" "$COMPONENTS_DIR/source/buckets.md"
+function all_versions {
+  for crd in $1 ; do
+    echo "${crd##*,}"
+  done | sort | uniq
 }
 
-{
-  # kustomize-controller CRDs
-  KUSTOMIZE_VER="$(controller_version kustomize-controller)"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/kustomize-controller/$KUSTOMIZE_VER/docs/api/v1/kustomize.md" "$COMPONENTS_DIR/kustomize/api/v1.md" "HUGETABLE"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/kustomize-controller/$KUSTOMIZE_VER/docs/spec/v1/kustomization.md" "$COMPONENTS_DIR/kustomize/kustomization.md"
-}
+function gen_ctrl_docs {
+  flux_version=${1}
+  ctrl=${2}
+  ctrl_short=${ctrl%%-controller}
+  ctrl_out=${ctrl%%-*} # this is necessary to collect IAC and IRC together in the "image" folder
 
-{
-  # helm-controller CRDs
-  HELM_VER="$(controller_version helm-controller)"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/helm-controller/$HELM_VER/docs/api/helmrelease.md" "$COMPONENTS_DIR/helm/api.md" "HUGETABLE"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/helm-controller/$HELM_VER/docs/spec/v2beta1/helmreleases.md" "$COMPONENTS_DIR/helm/helmreleases.md"
-}
+  api_out="api"
+  if [ "${ctrl_short}" = "image-reflector" ] ; then
+    api_out="reflector-api"
+  fi
+  if [ "${ctrl_short}" = "image-automation" ] ; then
+    api_out="automation-api"
+  fi
 
-{
-  # notification-controller CRDs
-  NOTIFICATION_VER="$(controller_version notification-controller)"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/api/v1/notification.md" "$COMPONENTS_DIR/notification/api/v1.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/api/v1beta2/notification.md" "$COMPONENTS_DIR/notification/api/v1beta2.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/events.md" "$COMPONENTS_DIR/notification/event.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/alerts.md" "$COMPONENTS_DIR/notification/alert.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1beta2/providers.md" "$COMPONENTS_DIR/notification/provider.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/notification-controller/$NOTIFICATION_VER/docs/spec/v1/receivers.md" "$COMPONENTS_DIR/notification/receiver.md"
-}
+  ks_url=$(curl -sL "https://raw.githubusercontent.com/fluxcd/flux2/${flux_version}/manifests/bases/${ctrl}/kustomization.yaml" | yq '.resources[]|select(. == "*crds.yaml*")')
+  ctrl_version=$(echo "${ks_url}" | cut -d/ -f8)
 
-{
-  # image-*-controller CRDs; these use the same API group
-  IMG_REFL_VER="$(controller_version image-reflector-controller)"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/api/image-reflector.md" "$COMPONENTS_DIR/image/reflector-api.md" "HUGETABLE"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/spec/v1beta2/imagerepositories.md" "$COMPONENTS_DIR/image/imagerepositories.md"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-reflector-controller/$IMG_REFL_VER/docs/spec/v1beta2/imagepolicies.md" "$COMPONENTS_DIR/image/imagepolicies.md"
-
-  IMG_AUTO_VER="$(controller_version image-automation-controller)"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/api/image-automation.md" "$COMPONENTS_DIR/image/automation-api.md" "HUGETABLE"
-  gen_crd_doc "https://raw.githubusercontent.com/fluxcd/image-automation-controller/$IMG_AUTO_VER/docs/spec/v1beta1/imageupdateautomations.md" "$COMPONENTS_DIR/image/imageupdateautomations.md"
+  crds=$(curl -sL "${ks_url}" | yq ea '[[.metadata.name, .spec.versions[] | select(.storage == "true").name]]' -o csv)
+  for api_version in $(all_versions "${crds}") ; do
+    doc_url=https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/api/${api_version}/${ctrl_short}.md
+    gen_crd_doc "${doc_url}" "$COMPONENTS_DIR/${ctrl_out}/${api_out}/${api_version}.md" "HUGETABLE"
+
+  done
+
+  for crd in ${crds} ; do
+    name=${crd%%,*}
+    name=${name%%.*}
+    version=${crd##*,}
+    gen_crd_doc "https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/spec/${version}/${name}.md" "$COMPONENTS_DIR/${ctrl_out}/${name}.md"
+  done
+
+  # special case: n-c's `Events` type is not a CRD but needs to be documented, too.
+  if [ "${ctrl}" = "notification-controller" ] ; then
+    gen_crd_doc "https://raw.githubusercontent.com/fluxcd/${ctrl}/${ctrl_version}/docs/spec/v1beta2/events.md" "$COMPONENTS_DIR/${ctrl_out}/events.md"
+  fi
 }
 
 {
@@ -176,11 +158,31 @@ gen_crd_doc() {
   setup_verify_arch
 
   TMP="$(mktemp -d)"
-  TMP_METADATA="$TMP/flux.json"
   TMP_BIN="$TMP/flux.tar.gz"
 
-  curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_METADATA}" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases/latest"
-  VERSION_FLUX=$(grep '"tag_name":' "${TMP_METADATA}" | sed -E 's/.*"([^"]+)".*/\1/' | cut -c 2-)
+  if [ -z "${BRANCH:-}" ] ; then
+    fatal "BRANCH environment variable not set"
+  fi
+
+  if [[ "${BRANCH}" =~ ^pull/[[:digit:]]*/head$ ]] ; then
+    BRANCH=$(curl -sSfL "https://api.github.com/repos/fluxcd/website/pulls/$(echo ${BRANCH}|cut -d/ -f2)" | jq .base.ref -r)
+  fi
+
+  VERSION_FLUX=
+  for tag in $(curl -u "$GITHUB_USER:$GITHUB_TOKEN" --retry 3 -sSfL "https://api.github.com/repos/fluxcd/flux2/releases" | jq .[].tag_name -r) ; do
+    if [ "${BRANCH}" = "main" ] ; then
+      VERSION_FLUX="${tag#v}"
+      break
+    fi
+    if [ "${tag%.*}" = "${BRANCH/-/.}" ] ; then
+      VERSION_FLUX="${tag#v}"
+      break
+    fi
+  done
+
+  if [ -z "${VERSION_FLUX}" ] ; then
+    fatal "No Flux version found matching branch '${BRANCH}'"
+  fi
 
   curl -u "$GITHUB_USER:$GITHUB_TOKEN" -o "${TMP_BIN}" --retry 3 -sSfL "https://github.com/fluxcd/flux2/releases/download/v${VERSION_FLUX}/flux_${VERSION_FLUX}_${OS}_${ARCH}.tar.gz"
   tar xfz "${TMP_BIN}" -C "${TMP}"
@@ -191,11 +193,37 @@ gen_crd_doc() {
   rm -rf "$TMP"
 }
 
+{
+  # source-controller CRDs
+  gen_ctrl_docs "v${VERSION_FLUX}" "source-controller"
+}
+
+{
+  # kustomize-controller CRDs
+  gen_ctrl_docs "v${VERSION_FLUX}" "kustomize-controller"
+}
+
+{
+  # helm-controller CRDs
+  gen_ctrl_docs "v${VERSION_FLUX}" "helm-controller"
+}
+
+{
+  # notification-controller CRDs
+  gen_ctrl_docs "v${VERSION_FLUX}" "notification-controller"
+}
+
+{
+  # image-*-controller CRDs; these use the same API group
+  gen_ctrl_docs "v${VERSION_FLUX}" "image-reflector-controller"
+  gen_ctrl_docs "v${VERSION_FLUX}" "image-automation-controller"
+}
+
 {
   # provide Flux install script
   if [ ! -d static ]; then
     mkdir static
   fi
-  curl -u "$GITHUB_USER:$GITHUB_TOKEN" -s -# -Lf https://raw.githubusercontent.com/fluxcd/flux2/main/install/flux.sh -o static/install.sh
+  curl -u "$GITHUB_USER:$GITHUB_TOKEN" -s -# -Lf "https://raw.githubusercontent.com/fluxcd/flux2/v${VERSION_FLUX}/install/flux.sh" -o static/install.sh
 }