Skip to content

Commit

Permalink
Write files to zipper as we discover them
Browse files Browse the repository at this point in the history 
This change modifies the tile writer to add files to the zipper as it
walks the tree rather than building up a map of files to add to the
zipper. This change reduces the number of open files to a constant
number (hopefully 1) at any given point in time.

[finishes #152737218]

Signed-off-by: Kalai Wei <kawei@pivotal.io>
  • Loading branch information
@jfmyers9
jfmyers9 authored and Kalai Wei committed Nov 10, 2017
1 parent 11e7973 commit 9f159fa
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 86 deletions.
124 changes: 58 additions & 66 deletions builder/tile_writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -70,60 +70,27 @@ func (w TileWriter) Write(generatedMetadataContents []byte, config commands.Bake
return err
}

files := map[string]io.Reader{}
fileModes := map[string]os.FileMode{}
err = w.addToZipper(filepath.Join("metadata", fmt.Sprintf("%s.yml", config.ProductName)), bytes.NewBuffer(generatedMetadataContents), config.OutputFile)
if err != nil {
return err
}

files[filepath.Join("metadata", fmt.Sprintf("%s.yml", config.ProductName))] = bytes.NewBuffer(generatedMetadataContents)
err = w.addMigrations(config.MigrationDirectories, config.OutputFile)
if err != nil {
return err
}

if len(config.ReleaseDirectories) > 0 {
for _, releasesDirectory := range config.ReleaseDirectories {
err = w.addReleaseTarballs(files, releasesDirectory, config.StubReleases)
if err != nil {
return err
}
}
}

if len(config.MigrationDirectories) > 0 {
for _, migrationsDir := range config.MigrationDirectories {
err = w.addMigrations(files, migrationsDir)
err = w.addReleaseTarballs(releasesDirectory, config.StubReleases, config.OutputFile)
if err != nil {
return err
}
}
}

for _, embedPath := range config.EmbedPaths {
err = w.addEmbeddedPath(files, embedPath, fileModes)
if err != nil {
return err
}
}

var paths []string
for path := range files {
paths = append(paths, path)
}

sort.Strings(paths)

if !w.containsMigrations(paths) {
err = w.addEmptyMigrationsDirectory(config.OutputFile)
if err != nil {
return err
}
}

for _, path := range paths {
w.logger.Printf("Adding %s to %s...", path, config.OutputFile)

var err error
if mode, ok := fileModes[path]; ok {
err = w.zipper.AddWithMode(path, files[path], mode)
} else {
err = w.zipper.Add(path, files[path])
}

err = w.addEmbeddedPath(embedPath, config.OutputFile)
if err != nil {
return err
}
Expand All @@ -145,13 +112,12 @@ func (w TileWriter) Write(generatedMetadataContents []byte, config commands.Bake
return nil
}

func (w TileWriter) addReleaseTarballs(files map[string]io.Reader, releasesDir string, stubReleases bool) error {
func (w TileWriter) addReleaseTarballs(releasesDir string, stubReleases bool, outputFile string) error {
return w.filesystem.Walk(releasesDir, func(filePath string, info os.FileInfo, err error) error {
isTarball, _ := regexp.MatchString("tgz$|tar.gz$", filePath)
if !isTarball {
return nil
}
var file io.Reader = strings.NewReader("")

if err != nil {
return err
Expand All @@ -161,19 +127,20 @@ func (w TileWriter) addReleaseTarballs(files map[string]io.Reader, releasesDir s
return nil
}

var file io.ReadCloser = ioutil.NopCloser(strings.NewReader(""))
if !stubReleases {
file, err = w.filesystem.Open(filePath)
if err != nil {
return err
}
defer file.Close()
}
files[filepath.Join("releases", filepath.Base(filePath))] = file

return nil
return w.addToZipper(filepath.Join("releases", filepath.Base(filePath)), file, outputFile)
})
}

func (w TileWriter) addEmbeddedPath(files map[string]io.Reader, pathToEmbed string, fileModes map[string]os.FileMode) error {
func (w TileWriter) addEmbeddedPath(pathToEmbed, outputFile string) error {
return w.filesystem.Walk(pathToEmbed, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
Expand All @@ -187,39 +154,64 @@ func (w TileWriter) addEmbeddedPath(files map[string]io.Reader, pathToEmbed stri
if err != nil {
return err
}
defer file.Close()

relativePath, err := filepath.Rel(pathToEmbed, filePath)
if err != nil {
return err //not tested
}

entryPath := filepath.Join("embed", filepath.Join(filepath.Base(pathToEmbed), relativePath))

files[entryPath] = file
fileModes[entryPath] = info.Mode()

return nil
return w.addToZipperWithMode(entryPath, file, info.Mode(), outputFile)
})
}

func (w TileWriter) addMigrations(files map[string]io.Reader, migrationsDir string) error {
return w.filesystem.Walk(migrationsDir, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
func (w TileWriter) addMigrations(migrationsDir []string, outputFile string) error {
var found bool

if info.IsDir() {
return nil
}
for _, migrationDir := range migrationsDir {
err := w.filesystem.Walk(migrationDir, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}

if info.IsDir() {
return nil
}

found = true

file, err := w.filesystem.Open(filePath)
if err != nil {
return err
}
defer file.Close()

return w.addToZipper(filepath.Join("migrations", "v1", filepath.Base(filePath)), file, outputFile)
})

file, err := w.filesystem.Open(filePath)
if err != nil {
return err
}
}

files[filepath.Join("migrations", "v1", filepath.Base(filePath))] = file
return nil
})
if !found {
return w.addEmptyMigrationsDirectory(outputFile)
}

return nil
}

func (w TileWriter) addToZipper(path string, contents io.Reader, outputFile string) error {
w.logger.Printf("Adding %s to %s...", path, outputFile)

return w.zipper.Add(path, contents)
}

func (w TileWriter) addToZipperWithMode(path string, contents io.Reader, mode os.FileMode, outputFile string) error {
w.logger.Printf("Adding %s to %s...", path, outputFile)

return w.zipper.AddWithMode(path, contents, mode)
}

func (w TileWriter) containsMigrations(entries []string) bool {
Expand Down
36 changes: 16 additions & 20 deletions builder/tile_writer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,8 @@ var _ = Describe("TileWriter", func() {

Expect(logger.PrintfCall.Receives.LogLines).To(Equal([]string{
fmt.Sprintf("Building %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding metadata/cool-product-name.yml to %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding releases/release-1.tgz to %s...", outputFile),
fmt.Sprintf("Adding releases/release-2.tgz to %s...", outputFile),
fmt.Sprintf("Calculating md5 sum of %s...", outputFile),
Expand Down Expand Up @@ -250,8 +250,8 @@ var _ = Describe("TileWriter", func() {

Expect(logger.PrintfCall.Receives.LogLines).To(Equal([]string{
fmt.Sprintf("Building %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding metadata/cool-product-name.yml to %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding releases/release-1.tgz to %s...", outputFile),
fmt.Sprintf("Adding releases/release-2.tgz to %s...", outputFile),
fmt.Sprintf("Calculating md5 sum of %s...", outputFile),
Expand All @@ -278,13 +278,10 @@ var _ = Describe("TileWriter", func() {

filesystem.WalkStub = func(root string, walkFn filepath.WalkFunc) error {
if root == "/some/path/releases" {
walkFn("/some/path/releases", dirInfo, nil)
walkFn("/some/path/releases/release-1.tgz", releaseInfo, nil)
walkFn("/some/path/releases/release-2.tgz", releaseInfo, nil)
walkFn(root, dirInfo, nil)
walkFn(filepath.Join(root, "release-1.tgz"), releaseInfo, nil)
walkFn(filepath.Join(root, "release-2.tgz"), releaseInfo, nil)
} else if root == "/some/path/to-embed/my-file.txt" {
walkFn("/some/path/to-embed", dirInfo, nil)
walkFn("/some/path/to-embed/my-file.txt", embedFileInfo, nil)
walkFn(root, embedFileInfo, nil)
}
return nil
Expand All @@ -295,7 +292,7 @@ var _ = Describe("TileWriter", func() {
return NewBuffer(bytes.NewBufferString("contents-of-embedded-file")), nil
}

return nil, nil
return NewBuffer(bytes.NewBufferString("contents-of-non-embedded-file")), nil
}
})

Expand All @@ -315,11 +312,11 @@ var _ = Describe("TileWriter", func() {

Expect(logger.PrintfCall.Receives.LogLines).To(Equal([]string{
fmt.Sprintf("Building %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding embed/my-file.txt to %s...", outputFile),
fmt.Sprintf("Adding metadata/cool-product-name.yml to %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding releases/release-1.tgz to %s...", outputFile),
fmt.Sprintf("Adding releases/release-2.tgz to %s...", outputFile),
fmt.Sprintf("Adding embed/my-file.txt to %s...", outputFile),
fmt.Sprintf("Calculating md5 sum of %s...", outputFile),
"Calculated md5 sum: ",
}))
Expand All @@ -346,15 +343,13 @@ var _ = Describe("TileWriter", func() {

filesystem.WalkStub = func(root string, walkFn filepath.WalkFunc) error {
if root == "/some/path/releases" {
walkFn("/some/path/releases", dirInfo, nil)
walkFn("/some/path/releases/release-1.tgz", releaseInfo, nil)
walkFn("/some/path/releases/release-2.tgz", releaseInfo, nil)
walkFn(root, dirInfo, nil)
walkFn(filepath.Join(root, "release-1.tgz"), releaseInfo, nil)
walkFn(filepath.Join(root, "release-2.tgz"), releaseInfo, nil)
} else if root == "/some/path/to-embed" {
walkFn("/some/path/to-embed", dirInfo, nil)
walkFn("/some/path/to-embed/my-file-1.txt", embedFileInfo, nil)
walkFn("/some/path/to-embed/my-file-2.txt", embedFileInfo, nil)
walkFn(root, dirInfo, nil)
walkFn(filepath.Join(root, "my-file-1.txt"), embedFileInfo, nil)
walkFn(filepath.Join(root, "my-file-2.txt"), embedFileInfo, nil)
}
return nil
}
Expand All @@ -365,7 +360,8 @@ var _ = Describe("TileWriter", func() {
} else if path == "/some/path/to-embed/my-file-2.txt" {
return NewBuffer(bytes.NewBufferString("contents-of-embedded-file-2")), nil
}
return nil, nil

return NewBuffer(bytes.NewBufferString("contents-of-non-embedded-file")), nil
}
})

Expand All @@ -385,12 +381,12 @@ var _ = Describe("TileWriter", func() {

Expect(logger.PrintfCall.Receives.LogLines).To(Equal([]string{
fmt.Sprintf("Building %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding embed/to-embed/my-file-1.txt to %s...", outputFile),
fmt.Sprintf("Adding embed/to-embed/my-file-2.txt to %s...", outputFile),
fmt.Sprintf("Adding metadata/cool-product-name.yml to %s...", outputFile),
fmt.Sprintf("Creating empty migrations folder in %s...", outputFile),
fmt.Sprintf("Adding releases/release-1.tgz to %s...", outputFile),
fmt.Sprintf("Adding releases/release-2.tgz to %s...", outputFile),
fmt.Sprintf("Adding embed/to-embed/my-file-1.txt to %s...", outputFile),
fmt.Sprintf("Adding embed/to-embed/my-file-2.txt to %s...", outputFile),
fmt.Sprintf("Calculating md5 sum of %s...", outputFile),
"Calculated md5 sum: ",
}))
Expand Down

0 comments on commit 9f159fa

Please sign in to comment.