Merge pull request #15130 from SimonRichardson/build-jujud-snap

[JUJU-2627] Build jujud snap

Makefile

@@ -227,8 +227,8 @@ define run_cgo_build
 			-mod=$(JUJU_GOMOD_MODE) \
 			-o ${BBIN_DIR} \
 			-tags "$(BUILD_TAGS)" \
-			$(COMPILE_FLAGS) \
-			-ldflags $(LINK_FLAGS) \
+			${COMPILE_FLAGS} \
+			-ldflags ${CGO_LINK_FLAGS} \
 			-v ${PACKAGE}
 endef
 

snap/plugins/juju_dqlite.py

@@ -0,0 +1,57 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2020 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""The juju-dqlite plugin used for snapping juju.
+
+This plugin uses the common plugin keywords as well as those for "sources".
+For more information check the 'plugins' topic for the former and the
+'sources' topic for the latter.
+"""
+from typing import Any, Dict, List, Set
+
+from snapcraft.plugins.v2 import PluginV2
+
+
+class PluginImpl(PluginV2):
+    @classmethod
+    def get_schema(cls) -> Dict[str, Any]:
+        return {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {
+                "curl-channel": {
+                    "type": "string",
+                    "default": "latest/stable",
+                },
+            },
+            "required": ["source"],
+        }
+
+    def get_build_snaps(self) -> Set[str]:
+        return {f"curl/{self.options.curl_channel}"}
+
+    def get_build_packages(self) -> Set[str]:
+        return {"gcc"}
+
+    def get_build_environment(self) -> Dict[str, str]:
+        return {}
+
+    def get_build_commands(self) -> List[str]:
+        return [
+            f'curl -o ${{SNAPCRAFT_PART_INSTALL}}/dqlite.tar.bz2 https://dqlite-static-libs.s3.amazonaws.com/latest-dqlite-deps-${{SNAP_ARCH}}.tar.bz2',
+            f'tar -C ${{SNAPCRAFT_PART_INSTALL}} -xjf ${{SNAPCRAFT_PART_INSTALL}}/dqlite.tar.bz2',
+        ]

snap/plugins/juju_go.py

@@ -43,7 +43,10 @@ def get_schema(cls) -> Dict[str, Any]:
             "type": "object",
             "additionalProperties": False,
             "properties": {
-                "go-channel": {"type": "string", "default": "latest/stable"},
+                "go-channel": {
+                    "type": "string",
+                    "default": "latest/stable",
+                },
                 "go-buildtags": {
                     "type": "array",
                     "uniqueItems": True,
@@ -70,6 +73,30 @@ def get_schema(cls) -> Dict[str, Any]:
                     "type": "boolean",
                     "default": False,
                 },
+                "go-cgo-enabled": {
+                    "type": "string",
+                    "default": "0",
+                },
+                "go-cgo-cc": {
+                    "type": "string",
+                    "default": "gcc",
+                },
+                "go-cgo-cflags": {
+                    "type": "string",
+                    "default": "",
+                },
+                "go-cgo-ldflags": {
+                    "type": "string",
+                    "default": "",
+                },
+                "go-cgo-ldflags-allow": {
+                    "type": "string",
+                    "default": "",
+                },
+                "go-cgo-ld-library-path": {
+                    "type": "string",
+                    "default": "",
+                },
             },
             "required": ["source"],
         }
@@ -78,27 +105,37 @@ def get_build_snaps(self) -> Set[str]:
         return {f"go/{self.options.go_channel}"}
 
     def get_build_packages(self) -> Set[str]:
+        if self.options.go_cgo_cc == "musl-gcc":
+            return {"musl-tools", "musl-dev"}
         return {"gcc"}
 
     def get_build_environment(self) -> Dict[str, str]:
         env = {
             "GOBIN": "${SNAPCRAFT_PART_INSTALL}/bin",
+            "CGO_ENABLED": self.options.go_cgo_enabled,
         }
-        if self.options.go_static:
+
+        if self.options.go_cgo_enabled == "1":
             env.update({
-                "CGO_ENABLED": "0"
+                "SNAPCRAFT_GO_CGO_CFLAGS": f"-I${{SNAPCRAFT_STAGE}}/include {self.options.go_cgo_cflags}",
+                "SNAPCRAFT_GO_CGO_LDFLAGS": f"-L${{SNAPCRAFT_STAGE}} {self.options.go_cgo_ldflags}",
+                "SNAPCRAFT_GO_CGO_LDFLAGS_ALLOW": self.options.go_cgo_ldflags_allow,
+                "SNAPCRAFT_GO_LD_LIBRARY_PATH": f"${{SNAPCRAFT_STAGE}} {self.options.go_cgo_ld_library_path}",
             })
 
         ld_flags = ''
         if self.options.go_strip:
             ld_flags += '-s -w '
         if self.options.go_static:
             ld_flags += '-extldflags "-static" '
+        if self.options.go_cgo_enabled == "1":
+            ld_flags += '-linkmode "external" '
         ld_flags = ld_flags.strip()
 
         if len(self.options.go_external_strings) > 0:
             for k, v in self.options.go_external_strings.items():
                 ld_flags += f' -X {k}={v}'
+
         env.update({
             "SNAPCRAFT_GO_LDFLAGS": f'{ld_flags}'
         })
@@ -114,7 +151,15 @@ def get_build_commands(self) -> List[str]:
         for go_package in self.options.go_packages:
             cmd += f" {go_package}"
 
-        return [
-            "go mod download",
-            cmd,
-        ]
+        cmds = []
+        cmds.append("go mod download")
+
+        if self.options.go_cgo_enabled == "1":
+            cmds.append(f'export CGO_CFLAGS="${{SNAPCRAFT_GO_CGO_CFLAGS}}"')
+            cmds.append(f'export CGO_LDFLAGS="${{SNAPCRAFT_GO_CGO_LDFLAGS}}"')
+            cmds.append(f'export CGO_LDFLAGS_ALLOW="${{SNAPCRAFT_GO_CGO_LDFLAGS_ALLOW}}"')
+            cmds.append(f'export LD_LIBRARY_PATH="${{SNAPCRAFT_GO_LD_LIBRARY_PATH}}"')
+            cmds.append(f'export CC={self.options.go_cgo_cc}')
+
+        cmds.append(cmd)
+        return cmds

snap/snapcraft.yaml

@@ -69,8 +69,44 @@ parts:
   wrappers:
     plugin: dump
     source: snap/local
+
+  dqlite:
+    plugin: juju-dqlite
+    source: .
+    organize:
+      juju-dqlite-static-lib-deps/*: .
+
+  jujud:
+    after:
+      - dqlite
+    plugin: juju-go
+    go-channel: 1.18/stable
+    source: .
+    go-packages:
+      - github.com/juju/juju/cmd/jujud
+    go-external-strings:
+      github.com/juju/juju/version.GitCommit: ""
+      github.com/juju/juju/version.GitTreeState: ""
+      github.com/juju/juju/version.build: ""
+    go-static: true
+    go-strip: true
+    go-cgo-enabled: "1"
+    go-cgo-cc: "musl-gcc"
+    go-cgo-cflags: "-I/usr/local/musl/include"
+    go-cgo-ldflags: "-luv -lraft -ldqlite -llz4 -lsqlite3"
+    go-cgo-ldflags-allow: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
+    override-pull: |
+      snapcraftctl pull
+
+      mkdir -p /usr/local/musl/include
+
+      ln -s /usr/include/$(uname -m)-linux-gnu/asm /usr/local/musl/include/asm || true
+      ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic || true
+      ln -s /usr/include/linux /usr/local/musl/include/linux || true
+
   juju:
-    # TODO(hpidcock): move to upstream go plugin when it has the features we need.
+    after:
+      - jujud
     plugin: juju-go
     go-channel: 1.18/stable
     # The source can be your local tree or github
@@ -89,41 +125,48 @@ parts:
     # apply patches before building
     go-packages:
       - github.com/juju/juju/cmd/juju
-      # If you are releasing a build with public streams, you don't need to build the agent
-      # Instead, you should use the released agent
       - github.com/juju/juju/cmd/jujuc
-      - github.com/juju/juju/cmd/jujud
       - github.com/juju/juju/cmd/plugins/juju-metadata
-    # go-external-strings is not supported by the standard go plugin.
-    # these strings are filled in by CI.
     go-external-strings:
       github.com/juju/juju/version.GitCommit: ""
       github.com/juju/juju/version.GitTreeState: ""
       github.com/juju/juju/version.build: ""
-    # go-static is not supported by the standard go plugin.
     go-static: true
-    # go-strip is not supported by the standard go plugin.
     go-strip: true
     override-build: |
       snapcraftctl build
 
-      mkdir -p $SNAPCRAFT_PART_INSTALL/bash_completions
-      cp -a etc/bash_completion.d/juju* $SNAPCRAFT_PART_INSTALL/bash_completions/.
+      mkdir -p ${SNAPCRAFT_PART_INSTALL}/bash_completions
+      cp -a etc/bash_completion.d/juju* ${SNAPCRAFT_PART_INSTALL}/bash_completions/.
       # If you are releasing a build with public streams, copy in the agent directly
       # If needed, grab the agent from streams
-      # curl http://streams.canonical.com/juju/tools/agent/$SNAPCRAFT_PROJECT_VERSION/juju-$SNAPCRAFT_PROJECT_VERSION-ubuntu-amd64.tgz | tar xz -C $SNAPCRAFT_PART_INSTALL/bin/
-      jujud=$SNAPCRAFT_PART_INSTALL/bin/jujud
+      # curl http://streams.canonical.com/juju/tools/agent/$SNAPCRAFT_PROJECT_VERSION/juju-$SNAPCRAFT_PROJECT_VERSION-ubuntu-amd64.tgz | tar xz -C ${SNAPCRAFT_PART_INSTALL}/bin/
+      jujud=${SNAPCRAFT_STAGE}/bin/jujud
       version=$($jujud version)
       hash=$(sha256sum $jujud | cut -d " " -f 1)
       cat > jujud-versions.yaml <<EOF
       versions:
         - version: $version
           sha256: $hash
       EOF
-      cp -a jujud-versions.yaml $SNAPCRAFT_PART_INSTALL/bin
+      cp -a jujud-versions.yaml ${SNAPCRAFT_PART_INSTALL}/bin
     stage-packages:
       - openssh-client
 
+  dqlite-cleanup:
+    # This ensures that we clean up the dqlite clean up files.
+    after:
+      - juju
+    plugin: nil
+    source: .
+    override-prime: |
+      snapcraftctl prime
+
+      rm -rf ${SNAPCRAFT_PRIME}/include || true
+      rm -rf ${SNAPCRAFT_PRIME}/juju-dqlite-static-lib-deps || true
+      rm -rf ${SNAPCRAFT_PRIME}/dqlite.tar.bz2 || true
+      rm -f ${SNAPCRAFT_PRIME}/lib* || true
+
 hooks:
   connect-plug-peers: {}
   disconnect-plug-peers: {}