Merge pull request #699 from upbound/batch-build

Call `up xpkg batch` to batch-process the smaller provider packages in the build pipelines

.github/workflows/publish-service-artifacts.yml

@@ -10,13 +10,23 @@ on:
       version:
         description: 'Provider version (e.g. v0.1.0)'
         required: true
+      size:
+        description: "Number of smaller provider packages to build and push with each build job"
+        default: '30'
+        required: true
+      concurrency:
+        description: "Number of parallel package builds within each build job"
+        default: '1'
+        required: false
 
 jobs:
   publish-service-artifacts:
     uses: upbound/uptest/.github/workflows/provider-publish-service-artifacts.yml@main
     with:
       subpackages: ${{ github.event.inputs.subpackages }}
       version: ${{ github.event.inputs.version }}
+      size: ${{ github.event.inputs.size }}
+      concurrency: ${{ github.event.inputs.concurrency }}
     secrets:
       UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR_RC }}
       UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW_RC }}

Makefile

@@ -1,9 +1,11 @@
 # ====================================================================================
 # Setup Project
 
-PROJECT_NAME := provider-aws
+PROVIDER_NAME := aws
+PROJECT_NAME := provider-$(PROVIDER_NAME)
 PROJECT_REPO := github.com/upbound/$(PROJECT_NAME)
 
+export PROVIDER_NAME
 export TERRAFORM_VERSION := 1.2.1
 export TERRAFORM_PROVIDER_SOURCE := hashicorp/aws
 export TERRAFORM_PROVIDER_VERSION := 4.66.0
@@ -56,24 +58,11 @@ export SUBPACKAGES := $(SUBPACKAGES)
 
 -include build/makelib/golang.mk
 
-# ====================================================================================
-# Setup XPKG
-
-XPKG_REG_ORGS ?= xpkg.upbound.io/upbound
-# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are
-# inferred.
-XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/upbound
-
-export XPKG_REG_ORGS := $(XPKG_REG_ORGS)
-export XPKG_REG_ORGS_NO_PROMOTE := $(XPKG_REG_ORGS_NO_PROMOTE)
-
--include build/makelib/xpkg.mk
-
 # ====================================================================================
 # Setup Kubernetes tools
 
 KIND_VERSION = v0.15.0
-UP_VERSION = v0.16.1
+UP_VERSION = v0.17.0
 UP_CHANNEL = stable
 UPTEST_VERSION = v0.5.0
 
@@ -87,8 +76,24 @@ export UP_CHANNEL := $(UP_CHANNEL)
 
 REGISTRY_ORGS ?= xpkg.upbound.io/upbound
 IMAGES = provider-aws
+BATCH_PLATFORMS ?= linux_amd64,linux_arm64
+export BATCH_PLATFORMS := $(BATCH_PLATFORMS)
+
 -include build/makelib/imagelight.mk
 
+# ====================================================================================
+# Setup XPKG
+
+XPKG_REG_ORGS ?= xpkg.upbound.io/upbound
+# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are
+# inferred.
+XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/upbound
+
+export XPKG_REG_ORGS := $(XPKG_REG_ORGS)
+export XPKG_REG_ORGS_NO_PROMOTE := $(XPKG_REG_ORGS_NO_PROMOTE)
+
+-include build/makelib/xpkg.mk
+
 # ====================================================================================
 # Targets
 
@@ -185,7 +190,10 @@ uptest: $(UPTEST) $(KUBECTL) $(KUTTL)
 uptest-local:
 	@$(WARN) "this target is deprecated, please use 'make uptest' instead"
 
-local-deploy: build controlplane.up local.xpkg.deploy.provider.$(PROJECT_NAME)-monolith
+build-monolith:
+	@$(MAKE) build SUBPACKAGES=monolith LOAD_MONOLITH=true
+
+local-deploy: build-monolith controlplane.up local.xpkg.deploy.provider.$(PROJECT_NAME)-monolith
 	@$(INFO) running locally built provider
 	@$(KUBECTL) wait provider.pkg $(PROJECT_NAME)-monolith --for condition=Healthy --timeout 5m
 	@$(KUBECTL) -n upbound-system wait --for=condition=Available deployment --all --timeout=5m

cluster/images/provider-aws/Dockerfile

@@ -27,7 +27,6 @@ FROM alpine:3.16.2 as base
 ARG CROSSPLANE_PROVIDER_VERSION
 ARG TARGETOS
 ARG TARGETARCH
-ARG APIGROUP
 ARG TERRAFORM_VERSION
 ARG TERRAFORM_PROVIDER_SOURCE
 ARG TERRAFORM_PROVIDER_VERSION
@@ -56,6 +55,3 @@ COPY --from=builder /tmp/terraform /usr/local/bin/
 COPY --from=builder /tmp/native/* ${PLUGIN_DIR}/
 
 ENTRYPOINT ["provider"]
-
-FROM base
-ADD "bin/${TARGETOS}_${TARGETARCH}/${APIGROUP}" /usr/local/bin/provider

cluster/images/provider-aws/Makefile

@@ -9,65 +9,32 @@ include ../../../build/makelib/common.mk
 include ../../../build/makelib/imagelight.mk
 
 # ====================================================================================
-# XPKG
-
--include ../../../build/makelib/xpkg.mk
+# Setup k8s tools
 
 -include ../../../build/makelib/k8s_tools.mk
 
 # ====================================================================================
-# Targets
-
-img.build:
-	@$(INFO) Subpackages to build: $(SUBPACKAGES)
-	@for g in $(SUBPACKAGES); do \
-		IMAGE=$$(echo $(IMAGE) | sed -r "s/(.*)-/\1-$${g}-/"); \
-		$(INFO) Building image $${IMAGE}; \
-		$(MAKE) BUILD_ARGS="--load" IMAGE=$${IMAGE} APIGROUP=$${g} XPKG_REG_ORGS=$(XPKG_REG_ORGS) img.build.shared; \
-		$(OK) docker build $${IMAGE}; \
-	done
+# Setup xpkg
 
-# make BRANCH_NAME=main SUBPACKAGES=cloudplatform publish
-img.publish: $(UP)
-	@$(INFO) Subpackages to publish: $(SUBPACKAGES)
-	@for g in $(SUBPACKAGES); do \
-		PACKAGE=$(XPKG_REG_ORGS)/$(PROJECT_NAME)-$${g}:$(VERSION); \
-		if [[ $${g} == "monolith" ]]; then PACKAGE=$(XPKG_REG_ORGS)/$(PROJECT_NAME):$(VERSION); fi; \
-		$(INFO) Pushing package $${PACKAGE}; \
-		$(UP) xpkg push $${PACKAGE} \
-				$(foreach p,$(XPKG_LINUX_PLATFORMS),--package $(XPKG_OUTPUT_DIR)/$(p)/$(PROJECT_NAME)-$${g}-$(VERSION).xpkg ) || $(FAIL); \
-		$(OK) Done processing $${PACKAGE}. Moving to the next package...; \
-	done || $(FAIL)
+-include ../../../build/makelib/xpkg.mk
 
-ifneq (,$(filter $(APIGROUP),monolith config))
-    ARG_XPKG_AUTH_EXT := "XPKG_AUTH_EXT=$(ROOT_DIR)/package/auth.yaml"
-endif
+# ====================================================================================
+# Targets
 
-DEP_CONSTRAINT := >= 0.0.0
-ifeq (-,$(findstring -,$(VERSION)))
-    DEP_CONSTRAINT = >= 0.0.0-0
-endif
+img.build:
+	@$(INFO) Family base image to build: $(IMAGE)
+	@$(INFO) Building image $${IMAGE}; \
+	$(MAKE) BUILD_ARGS="--load" IMAGE=$${IMAGE} XPKG_REG_ORGS=$(XPKG_REG_ORGS) img.build.shared; \
+	if [[ "$${LOAD_MONOLITH}" == "true" ]]; then \
+	  $(MAKE) batch-process SUBPACKAGES=monolith BATCH_PLATFORMS=$(PLATFORM) BUILD_ONLY=true STORE_PACKAGE=monolith && \
+	  export t=$$(docker load -qi "$(XPKG_OUTPUT_DIR)/$(PLATFORM)/$(PROJECT_NAME)-monolith-$(VERSION).xpkg") && \
+	  docker tag $${t##*:} $(BUILD_REGISTRY)/$(PROJECT_NAME)-monolith-$(ARCH); \
+	fi || $(FAIL)
+	@$(OK) docker build $${IMAGE};
 
-img.build.shared: $(UP)
-	@mkdir -p $(IMAGE_TEMP_DIR)/package/crds || $(FAIL)
-	@mkdir $(IMAGE_TEMP_DIR)/examples || $(FAIL)
-	@if [[ $(APIGROUP) == "config" ]]; then \
-		sed -e "s/{{ APIGROUP }}/$(APIGROUP)/g" $(ROOT_DIR)/package/crossplane-config.yaml > $(IMAGE_TEMP_DIR)/package/crossplane.yaml && \
-		cp $(ROOT_DIR)/package/crds/aws.* $(IMAGE_TEMP_DIR)/package/crds/ && \
-		cp -r $(ROOT_DIR)/examples/providerconfig $(IMAGE_TEMP_DIR)/examples/; \
-	elif [[ $(APIGROUP) == "monolith" ]]; then \
-		cp $(ROOT_DIR)/package/crossplane-monolith.yaml $(IMAGE_TEMP_DIR)/package/crossplane.yaml && \
-		cp -r $(ROOT_DIR)/package/crds $(IMAGE_TEMP_DIR)/package/ && \
-		cp -r $(ROOT_DIR)/examples $(IMAGE_TEMP_DIR)/; \
-	else \
-		sed -e "s|{{ APIGROUP }}|$(APIGROUP)|g" $(ROOT_DIR)/package/crossplane.yaml | sed -e "s|{{ XPKG_REG_ORGS }}|$(XPKG_REG_ORGS)|g" | sed -e "s|{{ VERSION }}|$(VERSION)|g" | sed -e "s|{{ DEP_CONSTRAINT }}|$(DEP_CONSTRAINT)|g" > $(IMAGE_TEMP_DIR)/package/crossplane.yaml && \
-		cp $(ROOT_DIR)/package/crds/$(APIGROUP).* $(IMAGE_TEMP_DIR)/package/crds/ && \
-		cp -r $(ROOT_DIR)/examples/$(APIGROUP) $(IMAGE_TEMP_DIR)/examples/; \
-  	fi || $(FAIL)
+img.build.shared:
 	@cp Dockerfile $(IMAGE_TEMP_DIR) || $(FAIL)
 	@cp terraformrc.hcl $(IMAGE_TEMP_DIR) || $(FAIL)
-	@mkdir -p $(IMAGE_TEMP_DIR)/bin/$(PLATFORM)/ || $(FAIL)
-	@cp -r $(OUTPUT_DIR)/bin/$(PLATFORM)/$(APIGROUP) $(IMAGE_TEMP_DIR)/bin/$(PLATFORM)/ || $(FAIL)
 	@docker buildx build $(BUILD_ARGS) \
 		--platform $(IMAGE_PLATFORMS) \
 		--build-arg TERRAFORM_VERSION=$(TERRAFORM_VERSION) \
@@ -76,14 +43,43 @@ img.build.shared: $(UP)
 		--build-arg TERRAFORM_PROVIDER_DOWNLOAD_NAME=$(TERRAFORM_PROVIDER_DOWNLOAD_NAME) \
 		--build-arg TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX=$(TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX) \
 		--build-arg CROSSPLANE_PROVIDER_VERSION=$(VERSION) \
-		--build-arg APIGROUP=$(APIGROUP) \
 		-t $(IMAGE) \
 		$(IMAGE_TEMP_DIR) || $(FAIL)
-	@$(MAKE) UP=$(UP) XPKG_REG_ORGS=$(XPKG_REG_ORGS) XPKG_REG_ORGS_NO_PROMOTE=$(XPKG_REG_ORGS_NO_PROMOTE) XPKGS=$(PROJECT_NAME)-$(APIGROUP) XPKG_DIR=$(IMAGE_TEMP_DIR)/package XPKG_EXAMPLES_DIR=$(IMAGE_TEMP_DIR)/examples $(ARG_XPKG_AUTH_EXT) build.xpkg || $(FAIL)
 
-build.xpkg: do.build.xpkgs
+img.publish: batch-process
 
 img.promote:
 	@$(INFO) Skipping image promotion from $(FROM_IMAGE) to $(TO_IMAGE)
 	@echo Promote is deferred to xpkg machinery
 	@$(OK) Image promotion skipped for $(FROM_IMAGE) to $(TO_IMAGE)
+
+CONCURRENCY ?= 30
+DEP_CONSTRAINT := >= 0.0.0
+ifeq (-,$(findstring -,$(VERSION)))
+    DEP_CONSTRAINT = >= 0.0.0-0
+endif
+BUILD_ONLY ?= false
+STORE_PACKAGE ?= ""
+batch-process: $(UP)
+	@$(INFO) Batch processing smaller provider packages for: $(SUBPACKAGES)
+	@mkdir -p "$(XPKG_OUTPUT_DIR)/$(PLATFORM)" && \
+	$(UP) xpkg batch --smaller-providers $$(echo -n $(SUBPACKAGES) | tr ' ' ',') \
+		--family-base-image $(BUILD_REGISTRY)/$(PROJECT_NAME) \
+		--platform $(BATCH_PLATFORMS) \
+		--provider-name $(PROJECT_NAME) \
+		--family-package-url-format $(XPKG_REG_ORGS)/%s:$(VERSION) \
+		--package-repo-override monolith=$(PROJECT_NAME) --package-repo-override config=provider-$(PROVIDER_NAME)-config \
+		--provider-bin-root $(OUTPUT_DIR)/bin \
+		--output-dir $(XPKG_OUTPUT_DIR) \
+		--store-packages $(STORE_PACKAGE) \
+		--build-only=$(BUILD_ONLY) \
+		--examples-root $(ROOT_DIR)/examples \
+		--examples-group-override monolith=* --examples-group-override config=providerconfig \
+		--auth-ext $(ROOT_DIR)/package/auth.yaml \
+		--crd-root $(ROOT_DIR)/package/crds \
+		--crd-group-override monolith=* --crd-group-override config=$(PROVIDER_NAME) \
+		--package-metadata-template $(ROOT_DIR)/package/crossplane.yaml.tmpl \
+		--template-var XpkgRegOrg=$(XPKG_REG_ORGS) --template-var DepConstraint="$(DEP_CONSTRAINT)" --template-var ProviderName=$(PROVIDER_NAME) \
+		--concurrency $(CONCURRENCY) \
+		--push-retry 10 || $(FAIL)
+	@$(OK) Done processing smaller provider packages for: $(SUBPACKAGES)

package/crossplane.yaml → package/crossplane.yaml.tmpl

@@ -1,25 +1,29 @@
 apiVersion: meta.pkg.crossplane.io/v1alpha1
 kind: Provider
 metadata:
-  name: provider-aws-{{ APIGROUP }}
+  name: {{ .Name }}
+{{ if ne .Service "monolith" }}
   labels:
-    pkg.crossplane.io/provider-family: aws
+    pkg.crossplane.io/provider-family: provider-{{ .ProviderName }}-config
+{{ end }}
   annotations:
     meta.crossplane.io/maintainer: Upbound <support@upbound.io>
-    meta.crossplane.io/source: github.com/upbound/provider-aws
+    meta.crossplane.io/source: github.com/upbound/provider-{{ .ProviderName }}
     meta.crossplane.io/description: |
       Upbound's official Crossplane provider to manage Amazon Web Services (AWS)
-      {{ APIGROUP }} services in Kubernetes.
+      {{ .Service }} services in Kubernetes.
     meta.crossplane.io/readme: |
       Provider AWS is a Crossplane provider for [Amazon Web Services
       (AWS)](https://aws.amazon.com/) developed and supported by Upbound.
       Available resources and their fields can be found in the [Upbound
-      Marketplace](https://marketplace.upbound.io/providers/upbound/provider-aws).
+      Marketplace](https://marketplace.upbound.io/providers/upbound/provider-{{ .ProviderName }}).
       If you encounter an issue please reach out on support@upbound.io email
-      address. This is a subpackage for the {{ APIGROUP }} API group.
-    friendly-name.meta.crossplane.io: Provider AWS ({{ APIGROUP }})
-    auth.upbound.io/group: aws.upbound.io
+      address. This is a subpackage for the {{ .Service }} API group.
+    friendly-name.meta.crossplane.io: Provider AWS ({{ .Service }})
+    auth.upbound.io/group: {{ .ProviderName }}.upbound.io
+{{ if and (ne .Service "config") (ne .Service "monolith") -}}
 spec:
   dependsOn:
-    - provider: {{ XPKG_REG_ORGS }}/provider-aws-config
-      version: "{{ DEP_CONSTRAINT }}"
+    - provider: {{ .XpkgRegOrg }}/provider-{{ .ProviderName }}-config
+      version: "{{ .DepConstraint }}"
+{{ end }}