Finish kernel support in Go update payload implementation

update/_apply/main.go

@@ -10,6 +10,7 @@ import (
 func main() {
 	u := update.Updater{
 		DstPartition: "out",
+		DstKernel:    "vmlinuz",
 	}
 
 	if err := u.OpenPayload(os.Args[1]); err != nil {

update/_gen/main.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/coreos/mantle/update/generator"
+)
+
+func main() {
+	var g generator.Generator
+
+	part, err := generator.FullUpdate(os.Args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	if err := g.Partition(part); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	if len(os.Args) > 2 {
+		kern, err := generator.FullUpdate(os.Args[2])
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		if err := g.Kernel(kern); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+	}
+
+	if err := g.Write("out.gz"); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

update/generator/full.go

@@ -16,8 +16,6 @@ package generator
 
 import (
 	"crypto/sha256"
-	"errors"
-	"fmt"
 	"io"
 	"os"
 
@@ -27,10 +25,6 @@ import (
 	"github.com/coreos/mantle/update/metadata"
 )
 
-var (
-	errShortRead = errors.New("read an incomplete block")
-)
-
 // FullUpdate generates an update Procedure for the given file, embedding its
 // entire contents in the payload so it does not depend any previous state.
 func FullUpdate(path string) (*Procedure, error) {
@@ -56,9 +50,6 @@ func FullUpdate(path string) (*Procedure, error) {
 	}
 	if err != nil && err != io.EOF {
 		payload.Close()
-		if err == errShortRead {
-			err = fmt.Errorf("%s: %v", path, err)
-		}
 		return nil, err
 	}
 
@@ -80,13 +71,18 @@ type fullScanner struct {
 	payload    io.Writer
 	source     io.Reader
 	offset     uint64
+	eof        error
 	operations []*metadata.InstallOperation
 }
 
 func (f *fullScanner) readChunk() ([]byte, error) {
+	if f.eof != nil {
+		return nil, f.eof
+	}
 	chunk := make([]byte, ChunkSize)
 	n, err := io.ReadFull(f.source, chunk)
-	if (err == io.EOF || err == io.ErrUnexpectedEOF) && n != 0 {
+	if err == io.ErrUnexpectedEOF {
+		f.eof = io.EOF
 		err = nil
 	}
 	return chunk[:n], err
@@ -97,12 +93,9 @@ func (f *fullScanner) Scan() error {
 	if err != nil {
 		return err
 	}
-	if len(chunk)%BlockSize != 0 {
-		return errShortRead
-	}
 
 	startBlock := uint64(f.offset) / BlockSize
-	numBlocks := uint64(len(chunk)) / BlockSize
+	numBlocks := (uint64(len(chunk)) + BlockSize - 1) / BlockSize
 	f.offset += uint64(len(chunk))
 
 	// Try bzip2 compressing the data, hopefully it will shrink!

update/generator/full_test.go

@@ -149,8 +149,13 @@ func TestFullUpdateScanUnaligned(t *testing.T) {
 		source:  bytes.NewReader(testUnaligned),
 	}
 
-	if err := scanner.Scan(); err != errShortRead {
-		t.Fatalf("expected errShortRead, got %v", err)
+	if err := scanner.Scan(); err != nil {
+		t.Fatalf("expected nil, got %v", err)
+	}
+
+	const expect = BlockSize + 1
+	if scanner.offset != expect {
+		t.Errorf("expected %d bytes, got %d", expect, scanner.offset)
 	}
 }
 

update/generator/generator.go

@@ -17,7 +17,9 @@ package generator
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
+	"math"
 	"os"
 
 	"github.com/coreos/pkg/capnslog"
@@ -65,6 +67,10 @@ func (g *Generator) Partition(proc *Procedure) error {
 		return ErrProcedureExists
 	}
 
+	if err := checkBlockAlignment(proc); err != nil {
+		return err
+	}
+
 	g.AddCloser(proc)
 	g.manifest.PartitionOperations = proc.Operations
 	g.manifest.OldPartitionInfo = proc.OldInfo
@@ -73,12 +79,50 @@ func (g *Generator) Partition(proc *Procedure) error {
 	return nil
 }
 
+// sanity check for block alignment in partitions.
+func checkBlockAlignment(proc *Procedure) error {
+	if proc.OldInfo.GetSize()%BlockSize != 0 {
+		return fmt.Errorf("generator: old size %d not divisble by %d",
+			proc.OldInfo.Size, BlockSize)
+	}
+	if proc.NewInfo.GetSize()%BlockSize != 0 {
+		return fmt.Errorf("generator: new size %d not divisble by %d",
+			proc.NewInfo.Size, BlockSize)
+	}
+	return nil
+}
+
+func (g *Generator) Kernel(proc *Procedure) error {
+	proc.Type = metadata.InstallProcedure_KERNEL.Enum()
+	return g.addProc(proc)
+}
+
+func (g *Generator) addProc(proc *Procedure) error {
+	for _, other := range g.manifest.Procedures {
+		if *other.Type == *proc.Type {
+			return ErrProcedureExists
+		}
+	}
+
+	g.AddCloser(proc)
+	g.manifest.Procedures = append(g.manifest.Procedures,
+		&proc.InstallProcedure)
+	g.payloads = append(g.payloads, proc)
+
+	return nil
+}
+
 // Write finalizes the payload, writing it out to the given file path.
 func (g *Generator) Write(path string) (err error) {
+	g.manifest.BlockSize = proto.Uint32(BlockSize)
+
 	if err = g.updateOffsets(); err != nil {
 		return
 	}
 
+	// for compatibility with old update_engine versions
+	g.addNoops()
+
 	plog.Infof("Writing payload to %s", path)
 
 	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
@@ -138,6 +182,61 @@ func (g *Generator) updateOffsets() error {
 	return err
 }
 
+// The sparse hole was a feature of update_engine, it is not used
+// in this new code outside of the noop compatibility goo.
+const sparseHole = math.MaxUint64
+
+// Translate a normal install operation to a dummy that discards data.
+func opToNoop(op *metadata.InstallOperation) *metadata.InstallOperation {
+	blocks := (uint64(*op.DataLength) + BlockSize - 1) / BlockSize
+	sum := make([]byte, len(op.DataSha256Hash))
+	copy(sum, op.DataSha256Hash)
+
+	return &metadata.InstallOperation{
+		Type:           metadata.InstallOperation_REPLACE.Enum(),
+		DataOffset:     proto.Uint32(*op.DataOffset),
+		DataLength:     proto.Uint32(*op.DataLength),
+		DataSha256Hash: sum,
+		DstExtents: []*metadata.Extent{&metadata.Extent{
+			StartBlock: proto.Uint64(sparseHole),
+			NumBlocks:  proto.Uint64(blocks),
+		}},
+	}
+}
+
+// Fill in the dummy noop_operations list for compatibility with old
+// update_engine versions that didn't support procedures and handled
+// signature data weirdly.
+func (g *Generator) addNoops() {
+	// Translate the new procedures list to noop operations.
+	for _, proc := range g.manifest.Procedures {
+		for _, op := range proc.Operations {
+			if op.GetDataLength() == 0 {
+				continue
+			}
+
+			g.manifest.NoopOperations = append(
+				g.manifest.NoopOperations, opToNoop(op))
+		}
+	}
+
+	// Create a dummy noop operation to cover trailing signature data.
+	// Yes, the manifest inconsistently uses 32 and 64 bit values...
+	offset := uint32(*g.manifest.SignaturesOffset)
+	length := uint32(*g.manifest.SignaturesSize)
+	blocks := (*g.manifest.SignaturesSize + BlockSize - 1) / BlockSize
+	g.manifest.NoopOperations = append(g.manifest.NoopOperations,
+		&metadata.InstallOperation{
+			Type:       metadata.InstallOperation_REPLACE.Enum(),
+			DataOffset: proto.Uint32(offset),
+			DataLength: proto.Uint32(length),
+			DstExtents: []*metadata.Extent{&metadata.Extent{
+				StartBlock: proto.Uint64(sparseHole),
+				NumBlocks:  proto.Uint64(blocks),
+			}},
+		})
+}
+
 func (g *Generator) writeHeader(w io.Writer) error {
 	manifestSize := proto.Size(&g.manifest)
 	header := metadata.DeltaArchiveHeader{