Add split Go SDK (#1934)

* Spike generating a single namespace SDK * Add hacks to flatten single module packages - Add go.mod generation. - Move module init code. * Generate example of what the SDK split would look like Generate new Go pulumi-azure-native-sdk - Append go.mod files after codegen and before emit. - Write into a new ignored directory - this will then be published into a second repository. - Revert previous experiments. Note: the root package name is now `pulumiazurenativesdk` rather than `azure`. I think this should be fine though. * Integrating into build & release pipeline - Use module replacement locally, then prepare the Go SDK for release by stripping replacements. - Organise pulumi-azure-native-sdk into sdk folder. - Sanitise semver version for go modules. - Commit to SDK repo on publish. * Add go-split SDK generation to acceptance tests * Remove go.sum files before publishing * Improve go.mod generation - Remove temporary comments. - Use text templates instead of sprintf. - Use filepath instead of regex. * Limit find depth for prepublish * Fix GH workflow steps * Create dir before untar * Fix yaml scalars & prerelease * Put step back that got lost * Fix go release issues * Tweak build & release pipelines * Make prepublish_go portable - `-maxdepth` must be first in linux * Fix git push for Go SDK publish * Tweak go module versioning - Maintain the commit hash from the build of the version number. - Add simple unit test suite of examples. - Output version to a file in the go SDK. - Add readme to the Go SDK. - When publishing Go, use the version from the version.txt to keep it all aligned. * Include licence in split-SDK *Fix to use US spelling * Add additional documentation
pulumi · Sep 13, 2022 · d18468f · d18468f
1 parent 1950780
commit d18468f
Show file tree

Hide file tree

Showing 10 changed files with 361 additions and 3 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -230,6 +230,15 @@ jobs:
       with:
         name: ${{ matrix.language  }}-sdk.tar.gz
         path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz
+    - name: Tar split Go SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: tar -zcf sdk/pulumi-azure-native-sdk.tar.gz -C sdk/pulumi-azure-native-sdk .
+    - name: Upload split Go artifacts
+      uses: actions/upload-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz
     - if: failure() && github.event_name == 'push'
       name: Notify Slack
       uses: 8398a7/action-slack@v3
@@ -321,6 +330,17 @@ jobs:
     - name: UnTar SDK folder
       run: tar -zxf ${{ github.workspace}}/sdk/${{ matrix.language}}.tar.gz -C ${{
         github.workspace}}/sdk/${{ matrix.language}}
+    - name: Download Go split SDK
+      uses: actions/download-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: UnTar Go split SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Install Node dependencies
@@ -504,6 +524,39 @@ jobs:
         arguments: publishToSonatype closeAndReleaseSonatypeStagingRepository
         build-root-directory: ./sdk/java
         gradle-version: 7.4.1
+    - name: Checkout Go SDK repo
+      uses: actions/checkout@v2
+      with:
+        token: ${{ secrets.PULUMI_BOT_TOKEN }}
+        repository: pulumi/pulumi-azure-native-sdk
+        path: sdk/pulumi-azure-native-sdk
+        fetch-depth: 0
+    - name: Prepare Go SDK repo
+      run: |
+        cd sdk/pulumi-azure-native-sdk
+        find . -maxdepth 1 -not -name '.*' -exec rm -rf {} \;
+    - name: Download Go split SDK
+      uses: actions/download-artifact@v2
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: UnTar Go split SDK folder
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+    - name: Publish Split Go SDK
+      run: |
+        make prepublish_go
+        cd sdk/pulumi-azure-native-sdk
+        VERSION="$(cat version.txt)"
+        git config user.name "Pulumi Bot"
+        git config user.email "bot@pulumi.com"
+        git add .
+        git commit -m "$VERSION"
+        git tag "$VERSION"
+        # Find go.mod's, get parent dir name, concat with version, set as a tag
+        find */go.mod -type f -exec sh -c "dirname {} | xargs basename | xargs printf "%s/$VERSION" | xargs git tag" \;
+        git push --tags
     - if: failure() && github.event_name == 'push'
       name: Notify Slack
       uses: 8398a7/action-slack@v3

diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml
@@ -221,6 +221,15 @@ jobs:
       with:
         name: ${{ matrix.language  }}-sdk.tar.gz
         path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz
+    - name: Tar split Go SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: tar -zcf sdk/pulumi-azure-native-sdk.tar.gz -C sdk/pulumi-azure-native-sdk .
+    - name: Upload split Go artifacts
+      uses: actions/upload-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz
     - if: failure() && github.event_name == 'push'
       name: Notify Slack
       uses: 8398a7/action-slack@v3
@@ -312,6 +321,17 @@ jobs:
     - name: UnTar SDK folder
       run: tar -zxf ${{ github.workspace}}/sdk/${{ matrix.language}}.tar.gz -C ${{
         github.workspace}}/sdk/${{ matrix.language}}
+    - name: Download Go split SDK
+      uses: actions/download-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: UnTar Go split SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Install Node dependencies
@@ -495,6 +515,40 @@ jobs:
         arguments: publishToSonatype closeAndReleaseSonatypeStagingRepository
         build-root-directory: ./sdk/java
         gradle-version: 7.4.1
+    - name: Checkout Go SDK repo
+      uses: actions/checkout@v2
+      with:
+        token: ${{ secrets.PULUMI_BOT_TOKEN }}
+        repository: pulumi/pulumi-azure-native-sdk
+        path: sdk/pulumi-azure-native-sdk
+        fetch-depth: 0
+    - name: Prepare Go SDK repo
+      run: |
+        cd sdk/pulumi-azure-native-sdk
+        git checkout master
+        find . -maxdepth 1 -not -name '.*' -exec rm -rf {} \;
+    - name: Download Go split SDK
+      uses: actions/download-artifact@v2
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: UnTar Go split SDK folder
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+    - name: Publish Split Go SDK
+      run: |
+        make prepublish_go
+        cd sdk/pulumi-azure-native-sdk
+        VERSION="$(cat version.txt)"
+        git config user.name "Pulumi Bot"
+        git config user.email "bot@pulumi.com"
+        git add .
+        git commit -m "$VERSION"
+        git tag "$VERSION"
+        # Find go.mod's, get parent dir name, concat with version, set as a tag
+        find */go.mod -type f -exec sh -c "dirname {} | xargs basename | xargs printf "%s/$VERSION" | xargs git tag" \;
+        git push --tags
     - if: failure() && github.event_name == 'push'
       name: Notify Slack
       uses: 8398a7/action-slack@v3

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -222,6 +222,15 @@ jobs:
       with:
         name: ${{ matrix.language  }}-sdk.tar.gz
         path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz
+    - name: Tar split Go SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: tar -zcf sdk/pulumi-azure-native-sdk.tar.gz -C sdk/pulumi-azure-native-sdk .
+    - name: Upload split Go artifacts
+      uses: actions/upload-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz
     - if: failure() && github.event_name == 'push'
       name: Notify Slack
       uses: 8398a7/action-slack@v3
@@ -313,6 +322,17 @@ jobs:
     - name: UnTar SDK folder
       run: tar -zxf ${{ github.workspace}}/sdk/${{ matrix.language}}.tar.gz -C ${{
         github.workspace}}/sdk/${{ matrix.language}}
+    - name: Download Go split SDK
+      uses: actions/download-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: UnTar Go split SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Install Node dependencies
@@ -502,6 +522,40 @@ jobs:
         arguments: publishToSonatype closeAndReleaseSonatypeStagingRepository
         build-root-directory: ./sdk/java
         gradle-version: 7.4.1
+    - name: Checkout Go SDK repo
+      uses: actions/checkout@v2
+      with:
+        token: ${{ secrets.PULUMI_BOT_TOKEN }}
+        repository: pulumi/pulumi-azure-native-sdk
+        path: sdk/pulumi-azure-native-sdk
+        fetch-depth: 0
+    - name: Prepare Go SDK repo
+      run: |
+        cd sdk/pulumi-azure-native-sdk
+        git checkout master
+        find . -maxdepth 1 -not -name '.*' -exec rm -rf {} \;
+    - name: Download Go split SDK
+      uses: actions/download-artifact@v2
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: UnTar Go split SDK folder
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+    - name: Publish Split Go SDK
+      run: |
+        make prepublish_go
+        cd sdk/pulumi-azure-native-sdk
+        VERSION="$(cat version.txt)"
+        git config user.name "Pulumi Bot"
+        git config user.email "bot@pulumi.com"
+        git add .
+        git commit -m "$VERSION"
+        git tag "$VERSION"
+        # Find go.mod's, get parent dir name, concat with version, set as a tag
+        find */go.mod -type f -exec sh -c "dirname {} | xargs basename | xargs printf "%s/$VERSION" | xargs git tag" \;
+        git push --tags
     - if: failure() && github.event_name == 'push'
       name: Notify Slack
       uses: 8398a7/action-slack@v3

diff --git a/.github/workflows/run-acceptance-tests.yml b/.github/workflows/run-acceptance-tests.yml
@@ -251,6 +251,15 @@ jobs:
       with:
         name: ${{ matrix.language  }}-sdk.tar.gz
         path: ${{ github.workspace}}/sdk/${{ matrix.language }}.tar.gz
+    - name: Tar split Go SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: tar -zcf sdk/pulumi-azure-native-sdk.tar.gz -C sdk/pulumi-azure-native-sdk .
+    - name: Upload split Go artifacts
+      uses: actions/upload-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz
     - if: failure() && github.event_name == 'push'
       name: Notify Slack
       uses: 8398a7/action-slack@v3
@@ -345,6 +354,17 @@ jobs:
     - name: UnTar SDK folder
       run: tar -zxf ${{ github.workspace}}/sdk/${{ matrix.language}}.tar.gz -C ${{
         github.workspace}}/sdk/${{ matrix.language}}
+    - name: Download Go split SDK
+      uses: actions/download-artifact@v2
+      if: ${{ matrix.language == 'go' }}
+      with:
+        name: pulumi-azure-native-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: UnTar Go split SDK folder
+      if: ${{ matrix.language == 'go' }}
+      run: |
+        mkdir -p ${{github.workspace}}/sdk/pulumi-azure-native-sdk
+        tar -zxf ${{ github.workspace}}/sdk/pulumi-azure-native-sdk.tar.gz -C ${{github.workspace}}/sdk/pulumi-azure-native-sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Install Node dependencies

diff --git a/.gitignore b/.gitignore
@@ -31,3 +31,4 @@ sdk/java/.gradle
 sdk/java/gradle
 sdk/java/gradlew
 sdk/java/gradlew.bat
+pulumi-azure-native-sdk
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -109,3 +109,9 @@ The versioner tool has been added to break apart the process of reporting and co
 Supporting every version of every API causes the SDK size to be very large. One method we've used to reduce the required size is to only have a single set of types for each API which are re-used across all versions. However, not all the types are compatible and therefore a type with the same name might be incompatible in the next version. This incompatibility is currently detected but ignored on a case-by-case basis for now.
 
 Ideally, at the point a new incompatible version of the type is added, it should then be created with a unique, stable name. Alternatively, we could create a union of the two possible types
+
+## New Go SDK
+
+As the size of the Go SDK is close to exceeding the limit of 512Mb, we've created a new SDK which we're publishing in parallel which defined a Go module per Azure namespace rather than a single root Go module. The additional go modules are auto-generated with the required dependencies in [provider/cmd/pulumi-gen-azure-native/main.go](./provider/cmd/pulumi-gen-azure-native/main.go#L312).
+
+This new SDK is published to its own repository at [github.com/pulumi/pulumi-azure-native-sdk](https://github.com/pulumi/pulumi-azure-native-sdk). This is separate partly due to the large number of tags which are created in this new repository per release, but also to remove the need to commit the SDK code into this repository.
diff --git a/Makefile b/Makefile
@@ -41,7 +41,7 @@ ensure:: init_submodules
 
 
 local_generate_code:: clean bin/pulumi-java-gen
-	$(WORKING_DIR)/bin/$(CODEGEN) schema,nodejs,dotnet,python,go $(VERSION)
+	$(WORKING_DIR)/bin/$(CODEGEN) schema,nodejs,dotnet,python,go,go-split $(VERSION)
 	$(WORKING_DIR)/bin/$(JAVA_GEN) generate --schema $(WORKING_DIR)/provider/cmd/$(PROVIDER)/schema.json --out sdk/java --build gradle-nexus
 	cd ${PACKDIR}/go/ && find . -type f -exec sed -i '' -e '/^\/\/.*/g' {} \;
 	cd ${PACKDIR}/dotnet/ && \
@@ -53,7 +53,7 @@ local_generate_code:: clean bin/pulumi-java-gen
 	echo "Finished generating."
 
 local_generate:: clean bin/pulumi-java-gen
-	$(WORKING_DIR)/bin/$(CODEGEN) schema,docs,nodejs,dotnet,python,go $(VERSION)
+	$(WORKING_DIR)/bin/$(CODEGEN) schema,docs,nodejs,dotnet,python,go,go-split $(VERSION)
 	$(WORKING_DIR)/bin/$(JAVA_GEN) generate --schema $(WORKING_DIR)/provider/cmd/$(PROVIDER)/schema.json --out sdk/java --build gradle-nexus
 	cd ${PACKDIR}/go/ && find . -type f -exec sed -i '' -e '/^\/\/.*/g' {} \;
 	cd ${PACKDIR}/dotnet/ && \
@@ -190,13 +190,26 @@ bin/pulumi-java-gen::
 	$(shell pulumictl download-binary -n pulumi-language-java -v $(JAVA_GEN_VERSION) -r pulumi/pulumi-java)
 
 generate_go::
-	$(WORKING_DIR)/bin/$(CODEGEN) go $(VERSION)
+	$(WORKING_DIR)/bin/$(CODEGEN) go,go-split $(VERSION)
+
 	cd ${PACKDIR}/go/ && find . -type f -exec sed -i '' -e '/^\/\/.*/g' {} \;
 
 build_go::
 	# Only building the top level packages and building 1 package at a time to avoid OOMing
 	cd sdk/ && \
 	GOGC=50 go list github.com/pulumi/pulumi-azure-native/sdk/go/azure/... | grep -v "latest\|\/v.*"$ | xargs -L 1 go build
+	find sdk/pulumi-azure-native-sdk -type d -maxdepth 1 -exec sh -c "cd \"{}\" && go mod tidy && go build" \;
+
+prepublish_go:
+	@# Remove go module replacements which are added for local testing
+	@# Note: must use `sed -i -e` to be portable - but leaves go.mod-e behind on macos
+	find sdk/pulumi-azure-native-sdk -maxdepth 2 -type f -name go.mod -exec sed -i -e '/replace github\.com\/pulumi\/pulumi-azure-native-sdk /d' {} \;
+	@# Remove sed backup files if using older sed versions
+	find sdk/pulumi-azure-native-sdk -maxdepth 2 -type f -name go.mod-e -delete
+	@# Delete go.sum files as these are not used at the point of publishing.
+	@# This is because we depend on the root package which will come from the same release commit, that doesn't yet exist.
+	find sdk/pulumi-azure-native-sdk -maxdepth 2 -type f -name go.sum -delete
+	cp README.md LICENSE sdk/pulumi-azure-native-sdk/
 
 clean::
 	rm -rf $$(find sdk/nodejs -mindepth 1 -maxdepth 1 ! -name "go.mod")