API for binlog binary contents

RELEASE_VERSION

@@ -1 +1 @@
-1.2.0
+1.2.1

go/cmd/orchestrator-agent/main.go

@@ -29,6 +29,8 @@ import (
 	"github.com/outbrain/orchestrator-agent/go/config"
 )
 
+var AppVersion string
+
 func acceptSignal() {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGHUP)
@@ -57,7 +59,11 @@ func main() {
 		log.SetPrintStackTrace(*stack)
 	}
 
-	log.Info("starting")
+	if AppVersion == "" {
+		AppVersion = "local-build"
+	}
+
+	log.Info("starting orchestrator-agent %s", AppVersion)
 
 	if len(*configFile) > 0 {
 		config.ForceRead(*configFile)

go/http/api.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"path"
 	"strconv"
@@ -473,7 +474,7 @@ func (this *HttpAPI) RelayLogEndCoordinates(params martini.Params, r render.Rend
 	r.JSON(200, coordinates)
 }
 
-// BinlogContents returns contents of binary log entries
+// RelaylogContentsTail returns contents of relay logs, from given position to the very last entry
 func (this *HttpAPI) RelaylogContentsTail(params martini.Params, r render.Render, req *http.Request) {
 	if err := this.validateToken(r, req); err != nil {
 		return
@@ -500,16 +501,18 @@ func (this *HttpAPI) RelaylogContentsTail(params martini.Params, r render.Render
 		}
 	}
 
-	output, err := osagent.MySQLBinlogContents(parseRelaylogs, startPosition, 0)
+	output, err := osagent.MySQLBinlogBinaryContents(parseRelaylogs, startPosition, 0)
 	if err != nil {
 		r.JSON(500, &APIResponse{Code: ERROR, Message: err.Error()})
 		return
 	}
 	r.JSON(200, output)
 }
 
-// BinlogContents returns contents of binary log entries
-func (this *HttpAPI) BinlogContents(params martini.Params, r render.Render, req *http.Request) {
+// binlogContents returns contents of binary log entries
+func (this *HttpAPI) binlogContents(params martini.Params, r render.Render, req *http.Request,
+	contentsFunc func(binlogFiles []string, startPosition int64, stopPosition int64) (string, error),
+) {
 	if err := this.validateToken(r, req); err != nil {
 		return
 	}
@@ -537,6 +540,16 @@ func (this *HttpAPI) BinlogContents(params martini.Params, r render.Render, req
 	r.JSON(200, output)
 }
 
+// BinlogContents returns contents of binary log entries
+func (this *HttpAPI) BinlogContents(params martini.Params, r render.Render, req *http.Request) {
+	this.binlogContents(params, r, req, osagent.MySQLBinlogContents)
+}
+
+// BinlogBinaryContents returns contents of binary log entries
+func (this *HttpAPI) BinlogBinaryContents(params martini.Params, r render.Render, req *http.Request) {
+	this.binlogContents(params, r, req, osagent.MySQLBinlogBinaryContents)
+}
+
 func (this *HttpAPI) RunCommand(params martini.Params, r render.Render, req *http.Request) {
 	if err := this.validateToken(r, req); err != nil {
 		return
@@ -592,7 +605,19 @@ func (this *HttpAPI) RegisterRequests(m *martini.ClassicMartini) {
 	m.Get("/api/mysql-relay-log-files", this.RelayLogFiles)
 	m.Get("/api/mysql-relay-log-end-coordinates", this.RelayLogEndCoordinates)
 	m.Get("/api/mysql-binlog-contents", this.BinlogContents)
+	m.Get("/api/mysql-binlog-binary-contents", this.BinlogBinaryContents)
 	m.Get("/api/mysql-relaylog-contents-tail/:relaylog/:start", this.RelaylogContentsTail)
 	m.Get("/api/custom-commands/:cmd", this.RunCommand)
 	m.Get(config.Config.StatusEndpoint, this.Status)
+
+	// list all the /debug/ endpoints we want
+	m.Get("/debug/pprof", pprof.Index)
+	m.Get("/debug/pprof/cmdline", pprof.Cmdline)
+	m.Get("/debug/pprof/profile", pprof.Profile)
+	m.Get("/debug/pprof/symbol", pprof.Symbol)
+	m.Post("/debug/pprof/symbol", pprof.Symbol)
+	m.Get("/debug/pprof/block", pprof.Handler("block").ServeHTTP)
+	m.Get("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
+	m.Get("/debug/pprof/goroutine", pprof.Handler("goroutine").ServeHTTP)
+	m.Get("/debug/pprof/threadcreate", pprof.Handler("threadcreate").ServeHTTP)
 }

go/osagent/osagent.go

@@ -142,6 +142,55 @@ func MySQLBinlogContents(binlogFiles []string, startPosition int64, stopPosition
 	return string(output), err
 }
 
+func MySQLBinlogBinaryContents(binlogFiles []string, startPosition int64, stopPosition int64) (string, error) {
+	if len(binlogFiles) == 0 {
+		return "", log.Errorf("No binlog files provided in MySQLBinlogContents")
+	}
+	binlogHeaderTmpFile, err := ioutil.TempFile("", "orchestrator-agent-binlog-header-size-")
+	if err != nil {
+		return "", log.Errore(err)
+	}
+	{
+		// magic header
+		// There are the first 4 bytes, and then there's also the first entry (the format-description).
+		// We need both from the first log file.
+		// Typically, the format description ends at pos 120, but let's verify...
+
+		cmd := fmt.Sprintf("mysqlbinlog %s --start-position=4 | head | egrep -o 'end_log_pos [^ ]+' | head -1 | awk '{print $2}' > %s", binlogFiles[0], binlogHeaderTmpFile.Name())
+		if _, err := commandOutput(sudoCmd(cmd)); err != nil {
+			return "", err
+		}
+	}
+	tmpFile, err := ioutil.TempFile("", "orchestrator-agent-binlog-contents-")
+	if err != nil {
+		return "", log.Errore(err)
+	}
+	if startPosition != 0 {
+		cmd := fmt.Sprintf("cat %s | head -c$(cat %s) >> %s", binlogFiles[0], binlogHeaderTmpFile.Name(), tmpFile.Name())
+		if _, err := commandOutput(sudoCmd(cmd)); err != nil {
+			return "", err
+		}
+	}
+	for i, binlogFile := range binlogFiles {
+		cmd := fmt.Sprintf("cat %s", binlogFile)
+
+		if i == len(binlogFiles)-1 && stopPosition != 0 {
+			cmd = fmt.Sprintf("%s | head -c %d", cmd, stopPosition)
+		}
+		if i == 0 && startPosition != 0 {
+			cmd = fmt.Sprintf("%s | tail -c+%d", cmd, (startPosition + 1))
+		}
+		cmd = fmt.Sprintf("%s >> %s", cmd, tmpFile.Name())
+		if _, err := commandOutput(sudoCmd(cmd)); err != nil {
+			return "", err
+		}
+	}
+
+	cmd := fmt.Sprintf("cat %s | gzip | base64", tmpFile.Name())
+	output, err := commandOutput(cmd)
+	return string(output), err
+}
+
 // Equals tests equality of this corrdinate and another one.
 func (this *LogicalVolume) IsSnapshotValid() bool {
 	if !this.IsSnapshot {

script/bootstrap

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+# Make sure we have the version of Go we want to depend on, either from the
+# system or one we grab ourselves.
+. script/ensure-go-installed
+
+# Since we want to be able to build this outside of GOPATH, we set it
+# up so it points back to us and go is none the wiser
+
+set -x
+rm -rf .gopath
+mkdir -p .gopath/src/github.com/outbrain
+ln -s "$PWD" .gopath/src/github.com/outbrain/orchestrator-agent
+export GOPATH=$PWD/.gopath:$GOPATH

script/build

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+. script/bootstrap
+
+mkdir -p bin
+bindir="$PWD"/bin
+scriptdir="$PWD"/script
+
+# We have a few binaries that we want to build, so let's put them into bin/
+
+version=$(git rev-parse HEAD)
+describe=$(git describe --tags --always --dirty)
+
+export GOPATH="$PWD/.gopath"
+cd .gopath/src/github.com/outbrain/orchestrator-agent
+
+# We put the binaries directly into the bindir, because we have no need for shim wrappers
+go build -o "$bindir/orchestrator-agent" -ldflags "-X main.AppVersion=${version} -X main.BuildDescribe=${describe}" ./go/cmd/orchestrator-agent/main.go

script/cibuild

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+. script/bootstrap
+
+echo "Verifying code is formatted via 'gofmt -s -w  go/'"
+gofmt -s -w  go/
+git diff --exit-code --quiet
+
+echo "Building"
+script/build
+
+cd .gopath/src/github.com/outbrain/orchestrator-agent
+
+echo "Running unit tests"
+go test ./go/...

script/cibuild-orchestrator-agent-build-deploy-tarball

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+set -e
+
+script/cibuild
+
+# Get a fresh directory and make sure to delete it afterwards
+build_dir=tmp/build
+rm -rf $build_dir
+mkdir -p $build_dir
+trap "rm -rf $build_dir" EXIT
+
+commit_sha=$(git rev-parse HEAD)
+
+if [ $(uname -s) = "Darwin" ]; then
+    build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+else
+    build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+fi
+
+tarball=$build_dir/${commit_sha}-${build_arch}.tar
+
+# Create the tarball
+tar cvf $tarball --mode="ugo=rx" bin/
+
+# Compress it and copy it to the directory for the CI to upload it
+gzip $tarball
+mkdir -p "$BUILD_ARTIFACT_DIR"/orchestrator-agent
+cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/orchestrator-agent/
+
+### HACK HACK HACK ###
+# blame @carlosmn
+# We don't have any jessie machines for building, but a pure-Go binary depends
+# on a version of libc and ld which are widely available, so we can copy the
+# tarball over with jessie in its name so we can deploy it on jessie machines.
+jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-precise-/-jessie-/)
+cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/orchestrator-agent/${jessie_tarball_name}.gz"

script/ensure-go-installed

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+GO_VERSION=go1.7
+
+GO_PKG_DARWIN=${GO_VERSION}.darwin-amd64.pkg
+GO_PKG_DARWIN_SHA=e7089843bc7148ffcc147759985b213604d22bb9fd19bd930b515aa981bf1b22
+
+GO_PKG_LINUX=${GO_VERSION}.linux-amd64.tar.gz
+GO_PKG_LINUX_SHA=702ad90f705365227e902b42d91dd1a40e48ca7f67a2f4b2fd052aaa4295cd95
+
+export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+cd $ROOTDIR
+
+# If Go isn't installed globally, setup environment variables for local install.
+if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then
+  GODIR="$ROOTDIR/.vendor/go17"
+
+  if [ $(uname -s) = "Darwin" ]; then
+    export GOROOT="$GODIR/usr/local/go"
+  else
+    export GOROOT="$GODIR/go"
+  fi
+
+  export PATH="$GOROOT/bin:$PATH"
+fi
+
+# Check if local install exists, and install otherwise.
+if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then
+  [ -d "$GODIR" ] && rm -rf $GODIR
+  mkdir -p "$GODIR"
+  cd "$GODIR";
+
+  if [ $(uname -s) = "Darwin" ]; then
+    curl -L -O https://storage.googleapis.com/golang/$GO_PKG_DARWIN
+    shasum -a256 $GO_PKG_DARWIN | grep $GO_PKG_DARWIN_SHA
+    xar -xf $GO_PKG_DARWIN
+    cpio -i < com.googlecode.go.pkg/Payload
+  else
+    curl -L -O https://storage.googleapis.com/golang/$GO_PKG_LINUX
+    shasum -a256 $GO_PKG_LINUX | grep $GO_PKG_LINUX_SHA
+    tar xf $GO_PKG_LINUX
+  fi
+
+  # Prove we did something right
+  echo "$GO_VERSION installed in $GODIR: Go Binary: $(which go)"
+fi
+
+cd $ROOTDIR
+
+# Configure the new go to be the first go found
+export GOPATH=$ROOTDIR/.vendor

script/go

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+. script/bootstrap
+
+mkdir -p bin
+bindir="$PWD"/bin
+
+cd .gopath/src/github.com/outbrain/orchestrator-agent
+go "$@"