Skip to content

Commit

Permalink
travis, build, internal: use own Go bundle for PPA builds (ethereum#2…
Browse files Browse the repository at this point in the history
…0240)

* build: bump PPAs to Go 1.13 (via longsleep), keep Trusty on 1.11

* travis, build, vendor: use own Go bundle for PPA builds

* travis, build, internal, vendor: smarter Go bundler, own untar

* build: updated ci-notes with new Go bundling, only make, don't test
  • Loading branch information
karalabe authored and millken committed Feb 25, 2021
1 parent c38194e commit 3cae297
Show file tree
Hide file tree
Showing 7 changed files with 193 additions and 32 deletions.
5 changes: 4 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,12 @@ jobs:
- fakeroot
- python-bzrlib
- python-paramiko
cache:
directories:
- $HOME/.gobundle
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz

# This builder does the Linux Azure uploads
- stage: build
Expand Down
17 changes: 8 additions & 9 deletions build/ci-notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,30 +22,29 @@ variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.

We want to build go-ethereum with the most recent version of Go, irrespective of the Go
version that is available in the main Ubuntu repository. In order to make this possible,
our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
golang-1.11, which is co-installable alongside the regular golang package. PPA dependencies
can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
we bundle the entire Go sources into our own source archive and start the built job by
compiling Go and then using that to build go-ethereum. On Trusty we have a special case
requiring the `~gophers/ubuntu/archive` PPA since Trusty can't even build Go itself. PPA
deps are set at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies

## Building Packages Locally (for testing)

You need to run Ubuntu to do test packaging.

Add the gophers PPA and install Go 1.11 and Debian packaging tools:
Install any version of Go and Debian packaging tools:

$ sudo apt-add-repository ppa:gophers/ubuntu/archive
$ sudo apt-get update
$ sudo apt-get install build-essential golang-1.11 devscripts debhelper python-bzrlib python-paramiko
$ sudo apt-get install build-essential golang-go devscripts debhelper python-bzrlib python-paramiko

Create the source packages:

$ go run build/ci.go debsrc -workdir dist

Then go into the source package directory for your running distribution and build the package:

$ cd dist/ethereum-unstable-1.6.0+xenial
$ cd dist/ethereum-unstable-1.9.6+bionic
$ dpkg-buildpackage

Built packages are placed in the dist/ directory.

$ cd ..
$ dpkg-deb -c geth-unstable_1.6.0+xenial_amd64.deb
$ dpkg-deb -c geth-unstable_1.9.6+bionic_amd64.deb
70 changes: 50 additions & 20 deletions build/ci.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import (
"strings"
"time"

"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/internal/build"
"github.com/ethereum/go-ethereum/params"
)
Expand Down Expand Up @@ -138,7 +139,18 @@ var (
// Note: zesty is unsupported because it was officially deprecated on Launchpad.
// Note: artful is unsupported because it was officially deprecated on Launchpad.
// Note: cosmic is unsupported because it was officially deprecated on Launchpad.
debDistros = []string{"trusty", "xenial", "bionic", "disco", "eoan"}
debDistroGoBoots = map[string]string{
"trusty": "golang-1.11",
"xenial": "golang-go",
"bionic": "golang-go",
"disco": "golang-go",
"eoan": "golang-go",
}

debGoBootPaths = map[string]string{
"golang-1.11": "/usr/lib/go-1.11",
"golang-go": "/usr/lib/go",
}
)

var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
Expand Down Expand Up @@ -459,11 +471,14 @@ func maybeSkipArchive(env build.Environment) {
// Debian Packaging
func doDebianSource(cmdline []string) {
var (
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`)
gobundle = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`)
gohash = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`)
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
)
flag.CommandLine.Parse(cmdline)
*workdir = makeWorkdir(*workdir)
Expand All @@ -476,12 +491,25 @@ func doDebianSource(cmdline []string) {
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
}

// Download and verify the Go source package
if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil {
log.Fatalf("Failed to ensure Go source package: %v", err)
}
// Create Debian packages and upload them
for _, pkg := range debPackages {
for _, distro := range debDistros {
meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
for distro, goboot := range debDistroGoBoots {
// Prepare the debian package with the go-ethereum sources
meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
pkgdir := stageDebianSource(*workdir, meta)

// Ship the Go sources along so we have a proper thing to build with
if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil {
log.Fatalf("Failed to extract Go sources: %v", err)
}
if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
log.Fatalf("Failed to rename Go source folder: %v", err)
}
// Run the packaging and upload to the PPA
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
debuild.Dir = pkgdir
build.MustRun(debuild)
Expand Down Expand Up @@ -561,7 +589,9 @@ type debPackage struct {
}

type debMetadata struct {
Env build.Environment
Env build.Environment
GoBootPackage string
GoBootPath string

PackageName string

Expand Down Expand Up @@ -590,19 +620,21 @@ func (d debExecutable) Package() string {
return d.BinaryName
}

func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
if author == "" {
// No signing key, use default author.
author = "Ethereum Builds <fjl@ethereum.org>"
}
return debMetadata{
PackageName: name,
Env: env,
Author: author,
Distro: distro,
Version: version,
Time: t.Format(time.RFC1123Z),
Executables: exes,
GoBootPackage: goboot,
GoBootPath: debGoBootPaths[goboot],
PackageName: name,
Env: env,
Author: author,
Distro: distro,
Version: version,
Time: t.Format(time.RFC1123Z),
Executables: exes,
}
}

Expand Down Expand Up @@ -667,7 +699,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
if err := os.Mkdir(pkgdir, 0755); err != nil {
log.Fatal(err)
}

// Copy the source code.
build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))

Expand All @@ -685,7 +716,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe)
build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe)
}

return pkgdir
}

Expand Down
2 changes: 1 addition & 1 deletion build/deb/ethereum/deb.control
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Source: {{.Name}}
Section: science
Priority: extra
Maintainer: {{.Author}}
Build-Depends: debhelper (>= 8.0.0), golang-1.11
Build-Depends: debhelper (>= 8.0.0), {{.GoBootPackage}}
Standards-Version: 3.9.5
Homepage: https://ethereum.org
Vcs-Git: git://github.com/ethereum/go-ethereum.git
Expand Down
4 changes: 3 additions & 1 deletion build/deb/ethereum/deb.rules
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@

# Launchpad rejects Go's access to $HOME/.cache, use custom folder
export GOCACHE=/tmp/go-build
export GOROOT_BOOTSTRAP={{.GoBootPath}}

override_dh_auto_build:
build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
(cd .go/src && ./make.bash)
build/env.sh .go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}

override_dh_auto_test:

Expand Down
46 changes: 46 additions & 0 deletions internal/build/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,3 +183,49 @@ func (a *TarballArchive) Close() error {
}
return a.file.Close()
}

func ExtractTarballArchive(archive string, dest string) error {
// We're only interested in gzipped archives, wrap the reader now
ar, err := os.Open(archive)
if err != nil {
return err
}
defer ar.Close()

gzr, err := gzip.NewReader(ar)
if err != nil {
return err
}
defer gzr.Close()

// Iterate over all the files in the tarball
tr := tar.NewReader(gzr)
for {
// Fetch the next tarball header and abort if needed
header, err := tr.Next()
if err != nil {
if err == io.EOF {
return nil
}
return err
}
// Figure out the target and create it
target := filepath.Join(dest, header.Name)

switch header.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0755); err != nil {
return err
}
case tar.TypeReg:
file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return err
}
if _, err := io.Copy(file, tr); err != nil {
return err
}
file.Close()
}
}
}
81 changes: 81 additions & 0 deletions internal/build/gosrc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package build

import (
"bytes"
"crypto/sha256"
"fmt"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
)

// EnsureGoSources ensures that path contains a file with the given SHA256 hash,
// and if not, it downloads a fresh Go source package from upstream and replaces
// path with it (if the hash matches).
func EnsureGoSources(version string, hash []byte, path string) error {
// Sanity check the destination path to ensure we don't do weird things
if !strings.HasSuffix(path, ".tar.gz") {
return fmt.Errorf("destination path (%s) must end with .tar.gz", path)
}
// If the file exists, validate it's hash
if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all
hasher := sha256.New()
hasher.Write(archive)
have := hasher.Sum(nil)

if bytes.Equal(have, hash) {
fmt.Printf("Go %s [%x] available at %s\n", version, hash, path)
return nil
}
fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path)
if err := os.Remove(path); err != nil {
return err
}
}
// Archive missing or bad hash, download a new one
fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path)

res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version))
if err != nil || res.StatusCode != http.StatusOK {
return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err)
}
defer res.Body.Close()

archive, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
// Sanity check the downloaded archive, save if checks out
hasher := sha256.New()
hasher.Write(archive)

if have := hasher.Sum(nil); !bytes.Equal(have, hash) {
return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash)
}
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
if err := ioutil.WriteFile(path, archive, 0644); err != nil {
return err
}
fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path)
return nil
}

0 comments on commit 3cae297

Please sign in to comment.