diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 35b356048..0d6883276 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/create-debugging-artifact.yml b/.github/workflows/create-debugging-artifact.yml index 011b7e016..ca3c0c47d 100644 --- a/.github/workflows/create-debugging-artifact.yml +++ b/.github/workflows/create-debugging-artifact.yml @@ -19,16 +19,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 submodules: 'recursive' token: ${{ secrets.RELEEN_GITHUB_TOKEN }} - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: "1.21" + go-version-file: go.mod check-latest: true - name: Build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 386b37e90..434a55b54 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,17 +15,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: 'recursive' token: ${{ secrets.RELEEN_GITHUB_TOKEN }} - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: "1.21" - check-latest: true + go-version-file: go.mod - name: Build run: go build ./... diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82b970299..4a99a1748 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,7 @@ name: test on: push: - branches: - - main + branches: ["main"] pull_request: jobs: test: @@ -10,17 +9,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: 'recursive' token: ${{ secrets.RELEEN_GITHUB_TOKEN }} - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: "1.21" - check-latest: true + go-version-file: go.mod - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/README.md b/README.md index 613b0c692..2070c2e03 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ These are the mappings from bake flag to each field in a bake_configurations ele | `"properties_directories"` | `--properties-directory=` | This may be a list of directories. | | `"runtime_configurations_directories"` | `--runtime-configs-directory=` | This may be a list of directories. | | `"bosh_variables_directories"` | `--bosh-variables-directory=` | This may be a list of directories. | -| `"embed_paths_directories"` | `--embed=` | This may be a list of directories. | +| `"embed_files"` | `--embed=` | This may be a list of filepaths. | | `"variable_files"` | `--variables-file=` | This may be a list of filepaths. | ### The Lock File [(source)](https://pkg.go.dev/github.com/pivotal-cf/kiln/pkg/cargo#Kilnfile) diff --git a/go.mod b/go.mod index be7a9852e..7231421f3 100644 --- a/go.mod +++ b/go.mod @@ -33,10 +33,10 @@ require ( github.com/pivotal-cf/om v0.0.0-20230707145702-e2ef8fd451b1 github.com/snabb/httpreaderat v1.0.1 github.com/stretchr/testify v1.8.4 - golang.org/x/crypto v0.19.0 + golang.org/x/crypto v0.21.0 golang.org/x/oauth2 v0.16.0 golang.org/x/sync v0.6.0 - golang.org/x/term v0.17.0 + golang.org/x/term v0.18.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -121,7 +121,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.18.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 19035109c..bf9bbbe67 100644 --- a/go.sum +++ b/go.sum @@ -633,6 +633,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -804,6 +806,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -813,6 +817,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/acceptance/bake/bake_test.go b/internal/acceptance/bake/bake_test.go index e83e86750..089af17d3 100644 --- a/internal/acceptance/bake/bake_test.go +++ b/internal/acceptance/bake/bake_test.go @@ -151,6 +151,10 @@ var _ = Describe("bake command", func() { file, err := bakedTile.Open("metadata/metadata.yml") Expect(err).NotTo(HaveOccurred()) + info, err := file.Stat() + Expect(err).NotTo(HaveOccurred()) + Expect(info.ModTime()).To(Equal(time.Unix(0, 0).In(time.UTC))) + metadataContents, err := io.ReadAll(file) Expect(err).NotTo(HaveOccurred()) @@ -874,7 +878,6 @@ some_runtime_configs: serial: false selected_value: "235" kiln_metadata: - kiln_version: 0.0.0-dev.0+acceptance metadata_git_sha: %s ` @@ -899,7 +902,6 @@ stemcell_criteria: requires_cpi: false enable_patch_security_updates: true kiln_metadata: - kiln_version: 0.0.0-dev.0+acceptance metadata_git_sha: %s ` @@ -921,7 +923,6 @@ additional_stemcells_criteria: - os: windows version: "2019.4" kiln_metadata: - kiln_version: 0.0.0-dev.0+acceptance metadata_git_sha: %s ` @@ -940,6 +941,5 @@ stemcell_criteria: os: ubuntu-trusty version: "3215.4" kiln_metadata: - kiln_version: 0.0.0-dev.0+acceptance metadata_git_sha: %s ` diff --git a/internal/acceptance/workflows/acceptance_test.go b/internal/acceptance/workflows/acceptance_test.go index a23c94fbe..d3383d376 100644 --- a/internal/acceptance/workflows/acceptance_test.go +++ b/internal/acceptance/workflows/acceptance_test.go @@ -25,12 +25,6 @@ import ( "github.com/pivotal-cf/kiln/internal/acceptance/workflows/scenario" ) -func TestMain(m *testing.M) { - code := m.Run() - _ = exec.Command("git", "submodule", "update", "--init", "--recursive", "hello-tile").Run() - os.Exit(code) -} - func Test_baking_a_tile(t *testing.T) { // t.SkipNow() setupAndRunFeatureTest(t) diff --git a/internal/acceptance/workflows/baking_a_tile.feature b/internal/acceptance/workflows/baking_a_tile.feature index 65ba0da64..e346af953 100644 --- a/internal/acceptance/workflows/baking_a_tile.feature +++ b/internal/acceptance/workflows/baking_a_tile.feature @@ -14,9 +14,10 @@ Feature: As a developer, I want to bake a tile | releases/hello-release-0.2.3.tgz | And "bake_records/0.2.0-dev.json" contains substring: "version": "0.2.0-dev" And "bake_records/0.2.0-dev.json" contains substring: "source_revision": "bc3ac24e192ba06a2eca19381ad785ec7069e0d0" + And "bake_records/0.2.0-dev.json" contains substring: "tile_directory": "." And "bake_records/0.2.0-dev.json" contains substring: "kiln_version": "0.0.0+acceptance-tests" - And "bake_records/0.2.0-dev.json" contains substring: "file_checksum": "342fd94f1f3ccaaa1b964977067a1d89915b2c6957242d18bd1ef39babcda915" - And "tile-0.2.0-dev.pivotal" has sha256 sum "342fd94f1f3ccaaa1b964977067a1d89915b2c6957242d18bd1ef39babcda915" + And "bake_records/0.2.0-dev.json" contains substring: "file_checksum": "5f8abc7a3272a70fa716cdf120f6976f6b78e16a01a4b3e085ced7f51d6c7691" + And "tile-0.2.0-dev.pivotal" has sha256 sum "5f8abc7a3272a70fa716cdf120f6976f6b78e16a01a4b3e085ced7f51d6c7691" Scenario: it reads directory configuration from Kilnfile Given I have a tile source directory "testdata/tiles/non-standard-paths" @@ -24,3 +25,14 @@ Feature: As a developer, I want to bake a tile | bake | | --stub-releases | Then a Tile is created + + Scenario: it bakes a tile from a bake record + Given I have a tile source directory "testdata/tiles/bake-record" + When I invoke kiln + | re-bake | + | --output-file=tile-0.1.0.pivotal | + | tile/bake_records/0.1.0.json | + Then a Tile is created + And the Tile contains + | metadata/metadata.yml | + | releases/bpm-1.1.21.tgz | \ No newline at end of file diff --git a/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go b/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go index d65b44e37..4f0afb86b 100644 --- a/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go +++ b/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go @@ -92,7 +92,12 @@ func iHaveATileDirectory(ctx context.Context, tileDirectory string) (context.Con return ctx, fmt.Errorf("failed to create new copy of tile directory: %w", err) } - dir, err := copyTileDirectory(tmpDir, tileDirectory) + resolvedDir, err := filepath.EvalSymlinks(tmpDir) + if err != nil { + return ctx, fmt.Errorf("failed to resolve symlinks for test tile directory: %w", err) + } + + dir, err := copyTileDirectory(resolvedDir, tileDirectory) if err != nil { return ctx, err } diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/.gitignore b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/.gitignore new file mode 100644 index 000000000..e36ada76b --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/.gitignore @@ -0,0 +1,3 @@ +releases/*.tgz +*.pivotal +bake_records/*.json \ No newline at end of file diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/Kilnfile b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/Kilnfile new file mode 100644 index 000000000..6ce4e51f7 --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/Kilnfile @@ -0,0 +1,6 @@ +--- +slug: min +release_sources: + - type: bosh.io +releases: + - name: bpm diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/Kilnfile.lock b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/Kilnfile.lock new file mode 100644 index 000000000..ca38b36a3 --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/Kilnfile.lock @@ -0,0 +1,9 @@ +releases: + - name: bpm + sha1: e8abe19ec186962828de843f8f281cddb6141904 + version: 1.1.21 + remote_source: bosh.io + remote_path: https://bosh.io/d/github.com/cloudfoundry/bpm-release?v=1.1.21 +stemcell_criteria: + os: ubuntu-xenial + version: "621.0" diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/bake_records/0.1.0.json b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/bake_records/0.1.0.json new file mode 100644 index 000000000..205472bff --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/bake_records/0.1.0.json @@ -0,0 +1,7 @@ +{ + "source_revision": "4b872e02af39ac7a025af39e1da55b3e0a3cbfe0", + "version": "0.1.0", + "kiln_version": "0.0.0+acceptance-tests", + "tile_directory": "tile", + "file_checksum": "781290c114d1e06f44e4bf05c22b5b03ee576f6526b28b103417050934f878b3" +} \ No newline at end of file diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/base.yml b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/base.yml new file mode 100644 index 000000000..039df82f7 --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/base.yml @@ -0,0 +1,84 @@ +--- +name: bpm +label: BPM +description: It just installs the BOSH Package Manager on an instance +icon_image: $( icon ) + +metadata_version: "2.7.0" +minimum_version_for_upgrade: 0.1.0 +product_version: $( version ) +provides_product_versions: + - name: bpm + version: $( version ) + +rank: 90 +serial: false + +releases: + - $( release "bpm" ) + +stemcell_criteria: $( stemcell ) + +job_types: + - name: hello-server + label: Server + resource_label: Server + description: HTTP Server + + templates: + - name: bpm + release: bpm + manifest: {} + + static_ip: 1 + dynamic_ip: 0 + + max_in_flight: 1 + single_az_only: true + + instance_definition: + name: instances + type: integer + label: Instances + configurable: true + default: 1 + constraints: + min: 0 + max: 1 + + resource_definitions: + - name: ram + type: integer + label: RAM + configurable: true + default: 1024 + constraints: + min: 1024 + + - name: ephemeral_disk + type: integer + label: Ephemeral Disk + configurable: true + default: 4000 + constraints: + min: 2000 + + - name: persistent_disk + type: integer + label: Persistent Disk + configurable: false + default: 4000 + constraints: + min: 2000 + + - name: cpu + type: integer + label: CPU + configurable: true + default: 1 + constraints: + min: 1 + +runtime_configs: [] +property_blueprints: [] +form_types: [] diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/icon.png b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/icon.png new file mode 100644 index 000000000..3d878e526 Binary files /dev/null and b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/icon.png differ diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/releases/.gitkeep b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/releases/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/acceptance/workflows/testdata/tiles/bake-record/tile/version b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/version new file mode 100644 index 000000000..6c6aa7cb0 --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/bake-record/tile/version @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file diff --git a/internal/acceptance/workflows/using_kiln.feature b/internal/acceptance/workflows/using_kiln.feature index 1811295a6..77de23282 100644 --- a/internal/acceptance/workflows/using_kiln.feature +++ b/internal/acceptance/workflows/using_kiln.feature @@ -33,6 +33,7 @@ Feature: As a developer, I want the Kiln CLI to be usable Examples: | command | | bake | + | re-bake | | fetch | | find-release-version | | find-stemcell-version | diff --git a/internal/builder/interpolator.go b/internal/builder/interpolator.go index a17ad3b12..fe81b15e8 100644 --- a/internal/builder/interpolator.go +++ b/internal/builder/interpolator.go @@ -37,7 +37,6 @@ type Interpolator struct{} type InterpolateInput struct { Version string - KilnVersion string BOSHVariables map[string]any Variables map[string]any ReleaseManifests map[string]any diff --git a/internal/builder/kiln_metadata.go b/internal/builder/kiln_metadata.go index 12016e139..92d00862c 100644 --- a/internal/builder/kiln_metadata.go +++ b/internal/builder/kiln_metadata.go @@ -10,14 +10,12 @@ import ( type KilnMetadata struct { MetadataGitSHA string `yaml:"metadata_git_sha"` - KilnVersion string `yaml:"kiln_version"` TileName string `yaml:"tile_name,omitempty"` } func newKilnMetadata(input InterpolateInput) KilnMetadata { m := KilnMetadata{ - KilnVersion: input.KilnVersion, MetadataGitSHA: input.MetadataGitSHA, } diff --git a/internal/builder/kiln_metadata_test.go b/internal/builder/kiln_metadata_test.go index 64de3e3cd..e5fb22ce6 100644 --- a/internal/builder/kiln_metadata_test.go +++ b/internal/builder/kiln_metadata_test.go @@ -22,7 +22,6 @@ func Test_setKilnMetadata(t *testing.T) { result, err := setKilnMetadata(inputMetadataYML, KilnMetadata{ MetadataGitSHA: "some-commit-sha", - KilnVersion: "some-kiln-version", }) require.NoError(t, err) assert.Equal(t, string(outputMetadataYML), string(result)) @@ -33,7 +32,6 @@ func Test_setKilnMetadata(t *testing.T) { func Test_newKilnMetadata(t *testing.T) { t.Run("when tile name is set", func(t *testing.T) { input := InterpolateInput{ - KilnVersion: "some-version", MetadataGitSHA: "some-sha", Variables: map[string]any{ TileNameVariable: "some-tile", @@ -43,7 +41,6 @@ func Test_newKilnMetadata(t *testing.T) { km := newKilnMetadata(input) require.Equal(t, KilnMetadata{ - KilnVersion: "some-version", MetadataGitSHA: "some-sha", TileName: "some-tile", }, km) @@ -51,7 +48,6 @@ func Test_newKilnMetadata(t *testing.T) { t.Run("when no tile name is set", func(t *testing.T) { input := InterpolateInput{ - KilnVersion: "some-version", MetadataGitSHA: "some-sha", Variables: nil, } @@ -59,7 +55,6 @@ func Test_newKilnMetadata(t *testing.T) { km := newKilnMetadata(input) require.Equal(t, KilnMetadata{ - KilnVersion: "some-version", MetadataGitSHA: "some-sha", TileName: "", }, km) @@ -67,7 +62,6 @@ func Test_newKilnMetadata(t *testing.T) { t.Run("when no tile name is the wrong type", func(t *testing.T) { input := InterpolateInput{ - KilnVersion: "some-version", MetadataGitSHA: "some-sha", Variables: map[string]any{ TileNameVariable: 11, @@ -77,7 +71,6 @@ func Test_newKilnMetadata(t *testing.T) { km := newKilnMetadata(input) require.Equal(t, KilnMetadata{ - KilnVersion: "some-version", MetadataGitSHA: "some-sha", TileName: "", }, km) diff --git a/internal/builder/metadata_git_sha.go b/internal/builder/metadata_git_sha.go index a55cb8258..20fb11bf7 100644 --- a/internal/builder/metadata_git_sha.go +++ b/internal/builder/metadata_git_sha.go @@ -5,9 +5,7 @@ import ( "fmt" "os" "os/exec" - "strconv" "strings" - "time" ) const DirtyWorktreeSHAValue = "DEVELOPMENT" @@ -25,21 +23,6 @@ func GitMetadataSHA(repositoryDirectory string, isDev bool) (string, error) { return gitHeadRevision(repositoryDirectory) } -func ModifiedTime(repositoryDirectory string, isDev bool) (time.Time, error) { - if isDev { - return time.Now(), nil - } - if err := ensureGitExecutableIsFound(); err != nil { - return time.Time{}, err - } - if dirty, err := GitStateIsDirty(repositoryDirectory); err != nil { - return time.Time{}, err - } else if dirty && isDev { - return time.Now(), nil - } - return GitCommitterCommitDate(repositoryDirectory) -} - func GitStateIsDirty(repositoryDirectory string) (bool, error) { gitStatus := exec.Command("git", "status", "--porcelain") gitStatus.Dir = repositoryDirectory @@ -53,20 +36,6 @@ func GitStateIsDirty(repositoryDirectory string) (bool, error) { return false, nil } -func GitCommitterCommitDate(repositoryDirectory string) (time.Time, error) { - cmd := exec.Command("git", "show", "-s", "--format=%ct") - cmd.Dir = repositoryDirectory - output, err := cmd.Output() - if err != nil { - return time.Time{}, err - } - commitTime, err := strconv.ParseInt(strings.TrimSpace(string(output)), 10, 64) - if err != nil { - return time.Time{}, err - } - return time.Unix(commitTime, 0), nil -} - func gitHeadRevision(repositoryDirectory string) (string, error) { var out bytes.Buffer gitRevParseHead := exec.Command("git", "rev-parse", "HEAD") diff --git a/internal/builder/testdata/append_kiln_metadata/output_metadata.yml b/internal/builder/testdata/append_kiln_metadata/output_metadata.yml index f029ed4e3..a34522089 100644 --- a/internal/builder/testdata/append_kiln_metadata/output_metadata.yml +++ b/internal/builder/testdata/append_kiln_metadata/output_metadata.yml @@ -83,4 +83,3 @@ stemcell_criteria: version: "621.256" kiln_metadata: metadata_git_sha: some-commit-sha - kiln_version: some-kiln-version diff --git a/internal/builder/testdata/replace_kiln_metadata/output_metadata.yml b/internal/builder/testdata/replace_kiln_metadata/output_metadata.yml index 3deb47241..c28ec10fd 100644 --- a/internal/builder/testdata/replace_kiln_metadata/output_metadata.yml +++ b/internal/builder/testdata/replace_kiln_metadata/output_metadata.yml @@ -59,7 +59,6 @@ job_types: release: bpm kiln_metadata: metadata_git_sha: some-commit-sha - kiln_version: some-kiln-version label: Hello metadata_version: 2.7.0 minimum_version_for_upgrade: 0.1.0 diff --git a/internal/commands/bake.go b/internal/commands/bake.go index 3b582a246..10871584d 100644 --- a/internal/commands/bake.go +++ b/internal/commands/bake.go @@ -13,6 +13,7 @@ import ( "path/filepath" "slices" "strings" + "time" "github.com/go-git/go-billy/v5" "github.com/pivotal-cf/jhanda" @@ -21,8 +22,8 @@ import ( "github.com/pivotal-cf/kiln/internal/builder" "github.com/pivotal-cf/kiln/internal/commands/flags" "github.com/pivotal-cf/kiln/internal/helper" + "github.com/pivotal-cf/kiln/pkg/bake" "github.com/pivotal-cf/kiln/pkg/cargo" - "github.com/pivotal-cf/kiln/pkg/source" ) //counterfeiter:generate -o ./fakes/interpolator.go --fake-name Interpolator . interpolator @@ -140,7 +141,7 @@ func NewBake(fs billy.Filesystem, releasesService baking.ReleasesService, outLog } } -type writeBakeRecordSignature func(string, string, []byte) error +type writeBakeRecordSignature func(string, string, string, []byte) error type Bake struct { interpolator interpolator @@ -228,21 +229,30 @@ func NewBakeWithInterfaces(interpolator interpolator, tileWriter tileWriter, out var _ writeBakeRecordSignature = writeBakeRecord -func writeBakeRecord(tileFilepath, metadataFilepath string, productTemplate []byte) error { +func writeBakeRecord(kilnVersion, tileFilepath, metadataFilepath string, productTemplate []byte) error { tileSum, err := tileChecksum(tileFilepath) if err != nil { return fmt.Errorf("failed to calculate checksum: %w", err) } - b, err := source.NewBakeRecord(tileSum, productTemplate) + b, err := bake.NewRecord(tileSum, productTemplate) if err != nil { return fmt.Errorf("failed to create bake record: %w", err) } + + b.KilnVersion = kilnVersion + abs, err := filepath.Abs(metadataFilepath) if err != nil { return fmt.Errorf("failed to find tile root for bake records: %w", err) } - dir := filepath.Dir(abs) - if err := b.WriteFile(dir); err != nil { + tileDir := filepath.Dir(abs) + + b, err = b.SetTileDirectory(tileDir) + if err != nil { + return err + } + + if err := b.WriteFile(tileDir); err != nil { return fmt.Errorf("failed to write bake record: %w", err) } return nil @@ -523,13 +533,9 @@ func (b Bake) Execute(args []string) error { return fmt.Errorf("failed to read metadata: %s", err) } - modTime, err := builder.ModifiedTime(filepath.Dir(b.Options.Kilnfile), isDevBuild) - if err != nil { - return fmt.Errorf("failed to read modified date from commit: %s", err) - } + modTime := time.Unix(0, 0).In(time.UTC) input := builder.InterpolateInput{ - KilnVersion: b.KilnVersion, Version: b.Options.Version, Variables: templateVariables, BOSHVariables: boshVariables, @@ -575,7 +581,7 @@ func (b Bake) Execute(args []string) error { } if b.Options.IsFinal { - if err := b.writeBakeRecord(b.Options.OutputFile, b.Options.Metadata, interpolatedMetadata); err != nil { + if err := b.writeBakeRecord(b.KilnVersion, b.Options.OutputFile, b.Options.Metadata, interpolatedMetadata); err != nil { return err } } diff --git a/internal/commands/bake_test.go b/internal/commands/bake_test.go index a9c022a88..7b02eff46 100644 --- a/internal/commands/bake_test.go +++ b/internal/commands/bake_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/pivotal-cf/kiln/pkg/bake" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/pivotal-cf-experimental/gomegamatchers" @@ -22,7 +24,6 @@ import ( "github.com/pivotal-cf/kiln/internal/commands/fakes" "github.com/pivotal-cf/kiln/internal/commands/flags" "github.com/pivotal-cf/kiln/pkg/proofing" - "github.com/pivotal-cf/kiln/pkg/source" ) var _ = Describe("Bake", func() { @@ -1013,13 +1014,14 @@ release_sources: }) type fakeWriteBakeRecordFunc struct { - tilePath, recordPath string - productTemplate []byte + kilnVersion, tilePath, recordPath string + productTemplate []byte err error } -func (f *fakeWriteBakeRecordFunc) call(tilePath, recordPath string, productTemplate []byte) error { +func (f *fakeWriteBakeRecordFunc) call(kilnVersion, tilePath, recordPath string, productTemplate []byte) error { + f.kilnVersion = kilnVersion f.tilePath = tilePath f.recordPath = recordPath f.productTemplate = productTemplate @@ -1034,7 +1036,7 @@ func TestBakeDescription(t *testing.T) { t.Fatalf("expected Options struct field %s", fieldName) } description := field.Tag.Get("description") - if !strings.Contains(description, source.BakeRecordsDirectory) { - t.Errorf("expected description to mention bake records directory %q", source.BakeRecordsDirectory) + if !strings.Contains(description, bake.RecordsDirectory) { + t.Errorf("expected description to mention bake records directory %q", bake.RecordsDirectory) } } diff --git a/internal/commands/rebake.go b/internal/commands/rebake.go new file mode 100644 index 000000000..e5aa29ad3 --- /dev/null +++ b/internal/commands/rebake.go @@ -0,0 +1,89 @@ +package commands + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/pivotal-cf/jhanda" + + "github.com/pivotal-cf/kiln/internal/builder" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +type ReBake struct { + bake jhanda.Command + Options struct { + OutputFile string `short:"o" long:"output-file" required:"true" description:"path to where the tile will be output"` + } +} + +func NewReBake(bake jhanda.Command) ReBake { + return ReBake{bake: bake} +} + +func (cmd ReBake) Execute(args []string) error { + records, err := jhanda.Parse(&cmd.Options, args) + if err != nil { + return err + } + if len(records) != 1 { + return fmt.Errorf("please add exactly one required bake record argument: %d bake arguments passed", len(records)) + } + recordBuffer, err := os.ReadFile(records[0]) + if err != nil { + return fmt.Errorf("failed to read bake record file: %w", err) + } + + var record bake.Record + if err := json.Unmarshal(recordBuffer, &record); err != nil { + return fmt.Errorf("failed to parse bake record: %w", err) + } + + workingDirectorySHA, err := builder.GitMetadataSHA(".", false) + if err != nil { + return err + } + + if got, exp := workingDirectorySHA, record.SourceRevision; got != exp { + return fmt.Errorf("expected the current worktree to be checked out at the source revision from the record %s but the current head is %s", exp, got) + } + + tileDir := filepath.FromSlash(record.TileDirectory) + + bakeFlags := []string{ + "--version", record.Version, + "--kilnfile", filepath.Join(tileDir, "Kilnfile"), + "--output-file", cmd.Options.OutputFile, + } + + if record.TileName != "" { + bakeFlags = append(bakeFlags, strings.Join([]string{"--variable", builder.TileNameVariable, record.TileName}, "=")) + } + + if err := cmd.bake.Execute(bakeFlags); err != nil { + return err + } + + newRecord, err := bake.NewRecordFromFile(cmd.Options.OutputFile) + if err != nil { + return err + } + + if !record.IsEquivalent(newRecord, log.New(os.Stderr, "bake record diff: ", 0)) { + return fmt.Errorf("expected tile bake records to be equivilant") + } + + return nil +} + +func (cmd ReBake) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "re-bake (aka record bake) builds a tile from a bake record. You must check out the repository to the revision of the source_revision in the bake record before running this command.", + ShortDescription: "re-bake constructs a tile from a bake record", + Flags: &cmd.Options, + } +} diff --git a/main.go b/main.go index 75e3ff751..52e9d7303 100644 --- a/main.go +++ b/main.go @@ -79,6 +79,7 @@ func main() { bakeCommand := commands.NewBake(fs, releasesService, outLogger, errLogger, fetch) bakeCommand.KilnVersion = version commandSet["bake"] = bakeCommand + commandSet["re-bake"] = commands.NewReBake(bakeCommand) commandSet["test"] = commands.NewTileTest() commandSet["help"] = commands.NewHelp(os.Stdout, globalFlagsUsage, commandSet) diff --git a/pkg/bake/record.go b/pkg/bake/record.go new file mode 100644 index 000000000..2b8425442 --- /dev/null +++ b/pkg/bake/record.go @@ -0,0 +1,248 @@ +package bake + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/fs" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "slices" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/Masterminds/semver/v3" + + "github.com/pivotal-cf/kiln/internal/builder" + "github.com/pivotal-cf/kiln/pkg/tile" +) + +// RecordsDirectory should be a sibling to Kilnfile or base.yml +const RecordsDirectory = "bake_records" + +// Record is created by the function +type Record struct { + // SourceRevision is the commit checked out when the build was run + SourceRevision string `yaml:"source_revision" json:"source_revision"` + + // Version is the tile version used for product template field `product_version` + Version string `yaml:"version" json:"version"` + + // KilnVersion is the Kiln CLI version + KilnVersion string `yaml:"kiln_version,omitempty" json:"kiln_version,omitempty"` + + // TileName might record the tile name used in baking because sometimes multiple tiles can be generated from the same tile source directory + // An example of this is the two Tanzu Application Service tiles with different topologies listed on TanzuNetwork. + TileName string `yaml:"tile_name,omitempty" json:"tile_name,omitempty"` + + // FileChecksum may be the SHA256 checksum of the baked tile. + FileChecksum string `yaml:"file_checksum,omitempty" json:"file_checksum,omitempty"` + + // TileDirectory may be the directory containing tile source. + TileDirectory string `yaml:"tile_directory,omitempty" json:"tile_directory,omitempty"` +} + +// NewRecord parses build information from an OpsManger Product Template (aka metadata/metadata.yml) +func NewRecord(fileChecksum string, productTemplateBytes []byte) (Record, error) { + var productTemplate struct { + ProductVersion string `yaml:"product_version"` + KilnMetadata builder.KilnMetadata `yaml:"kiln_metadata"` + } + + err := yaml.Unmarshal(productTemplateBytes, &productTemplate) + if err != nil { + return Record{}, err + } + + if productTemplate.KilnMetadata.MetadataGitSHA == "" { + return Record{}, fmt.Errorf("failed to parse build information from product template: kiln_metadata.metadata_git_sha not found") + } + + return Record{ + SourceRevision: productTemplate.KilnMetadata.MetadataGitSHA, + Version: productTemplate.ProductVersion, + TileName: productTemplate.KilnMetadata.TileName, + FileChecksum: fileChecksum, + }, nil +} + +// NewRecordFromFile parses the product template and pulls the kiln metadata out. +// The SHA256 sum is also calculated for the file. +func NewRecordFromFile(tileFilepath string) (Record, error) { + metadata, err := tile.ReadMetadataFromFile(tileFilepath) + if err != nil { + return Record{}, err + } + checksum, err := fileChecksum(tileFilepath) + if err != nil { + return Record{}, err + } + return NewRecord(checksum, metadata) +} + +func fileChecksum(name string) (string, error) { + f, err := os.Open(name) + if err != nil { + return "", err + } + defer closeAndIgnoreError(f) + sum := sha256.New() + _, err = io.Copy(sum, f) + return hex.EncodeToString(sum.Sum(nil)), err +} + +func ReadRecords(dir fs.FS) ([]Record, error) { + infos, err := fs.ReadDir(dir, RecordsDirectory) + if err != nil { + return nil, err + } + builds := make([]Record, 0, len(infos)) + for _, info := range infos { + buf, err := fs.ReadFile(dir, path.Join(RecordsDirectory, info.Name())) + if err != nil { + return nil, err + } + var build Record + if err := json.Unmarshal(buf, &build); err != nil { + return nil, err + } + builds = append(builds, build) + } + slices.SortFunc(builds, compareMultiple(Record.CompareVersion, Record.CompareTileName)) + return builds, nil +} + +func (record Record) Name() string { + if record.TileName != "" { + return path.Join(record.TileName, record.Version) + } + return record.Version +} + +func (record Record) CompareVersion(o Record) int { + bv, err := semver.NewVersion(record.Version) + if err != nil { + return strings.Compare(record.Version, o.Version) + } + ov, err := semver.NewVersion(o.Version) + if err != nil { + return strings.Compare(record.Version, o.Version) + } + return bv.Compare(ov) +} + +func (record Record) CompareTileName(o Record) int { + return strings.Compare(record.TileName, o.TileName) +} + +func (record Record) IsDevBuild() bool { + return record.SourceRevision == builder.DirtyWorktreeSHAValue +} + +func (record Record) IsEquivalent(other Record, logger *log.Logger) bool { + if logger == nil { + logger = log.New(io.Discard, "", 0) + } + + // if tile records differ on the following fields the tiles are still close enough + record.KilnVersion = "" + other.KilnVersion = "" + record.TileDirectory = "" + other.TileDirectory = "" + record.TileName = "" + other.TileName = "" + + if exp, got := record.Version, other.Version; exp != got { + logger.Printf("tile versions are not the same: expected %q but got %q", exp, got) + } + + if exp, got := record.SourceRevision, other.SourceRevision; exp != got { + logger.Printf("tile source revisions are not the same: expected %q but got %q", exp, got) + } + + if exp, got := record.FileChecksum, other.FileChecksum; exp != got { + logger.Printf("tile file checksums are not the same: expected %q but got %q", exp, got) + } + + return record == other +} + +func (record Record) WriteFile(tileSourceDirectory string) error { + if record.Version == "" { + return fmt.Errorf("missing required version field") + } + if record.IsDevBuild() { + return fmt.Errorf("will not write development builds to %s directory", RecordsDirectory) + } + if err := os.MkdirAll(filepath.Join(tileSourceDirectory, RecordsDirectory), 0o766); err != nil { + return err + } + buf, err := json.MarshalIndent(record, "", " ") + if err != nil { + return err + } + fileName := record.Version + ".json" + if record.TileName != "" { + fileName = record.TileName + "-" + fileName + } + outputFilepath := filepath.Join(tileSourceDirectory, RecordsDirectory, fileName) + if _, err := os.Stat(outputFilepath); err == nil { + return fmt.Errorf("tile bake record already exists for %s", record.Name()) + } + return os.WriteFile(outputFilepath, buf, 0o644) +} + +func (record Record) SetTileDirectory(tileSourceDirectory string) (Record, error) { + absoluteTileSourceDirectory, err := filepath.Abs(tileSourceDirectory) + if err != nil { + return record, err + } + repoRoot, err := repositoryRoot(tileSourceDirectory) + if err != nil { + return record, err + } + repoRoot += string(filepath.Separator) + absoluteTileSourceDirectory += string(filepath.Separator) + if !strings.HasPrefix(absoluteTileSourceDirectory, repoRoot) { + return record, fmt.Errorf("expected tile directory %q to either be or be a child of the repository root directory %q", absoluteTileSourceDirectory, repoRoot) + } + relativeTilePath := strings.TrimPrefix(absoluteTileSourceDirectory, repoRoot) + record.TileDirectory = filepath.ToSlash(filepath.Clean(relativeTilePath)) + return record, nil +} + +func repositoryRoot(dir string) (string, error) { + var out bytes.Buffer + cmd := exec.Command("git", "rev-parse", "--show-toplevel") + cmd.Dir = dir + cmd.Stdout = &out + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to get HEAD revision hash: %w", err) + } + return strings.TrimSpace(out.String()), nil +} + +func compareMultiple[T any](cmp ...func(a, b T) int) func(a, b T) int { + return func(a, b T) int { + var result int + for _, c := range cmp { + result = c(a, b) + if result != 0 { + break + } + } + return result + } +} + +func closeAndIgnoreError(closer io.Closer) { + _ = closer.Close() +} diff --git a/pkg/bake/record_test.go b/pkg/bake/record_test.go new file mode 100644 index 000000000..9d28266c6 --- /dev/null +++ b/pkg/bake/record_test.go @@ -0,0 +1,287 @@ +package bake_test + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/pivotal-cf/kiln/internal/builder" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +func TestNewRecordFromFile(t *testing.T) { + tilePath := filepath.Join("testdata", "tile.pivotal") + record, err := bake.NewRecordFromFile(tilePath) + require.NoError(t, err) + assert.Equal(t, bake.Record{ + Version: "0.2.0-dev", + SourceRevision: "5874e0f81d0af47922716a7c69a08bcdead13348", + FileChecksum: "7490ba0b736c262ee7dc433c423c4f95ad838b014769d8465c50e445967d2735", + }, record) +} + +func TestNewRecord(t *testing.T) { + t.Run("when creating a bake record from a product template", func(t *testing.T) { + // language=yaml + b, err := bake.NewRecord("some-peach-jam", []byte(` +product_name: p-each +product_version: some-product-version +kiln_metadata: + kiln_version: some-kiln-version + metadata_git_sha: some-tile-source-revision + tile_name: srt +`)) + require.NoError(t, err) + require.Equal(t, bake.Record{ + Version: "some-product-version", + SourceRevision: "some-tile-source-revision", + TileName: "srt", + FileChecksum: "some-peach-jam", + }, b) + }) + + t.Run("when the product template is missing kiln_metadata", func(t *testing.T) { + // language=yaml + _, err := bake.NewRecord("some-peach-jam", []byte(` +product_name: p-each +product_version: some-product-version +`)) + require.ErrorContains(t, err, "kiln_metadata") + }) + + t.Run("write one file", func(t *testing.T) { + dir := t.TempDir() + + b := bake.Record{ + TileName: "p-each", + SourceRevision: "some-revision", + Version: "1.2.3", + KilnVersion: "some-version", + } + + require.NoError(t, b.WriteFile(dir)) + + buf, err := os.ReadFile(filepath.Join(dir, bake.RecordsDirectory, "p-each-1.2.3.json")) + require.NoError(t, err) + + require.JSONEq(t, `{"source_revision":"some-revision", "tile_name":"p-each", "version":"1.2.3", "kiln_version": "some-version"}`, string(buf)) + }) + + t.Run("when the record is missing the version field", func(t *testing.T) { + dir := t.TempDir() + + b := bake.Record{ + Version: "", + } + + require.ErrorContains(t, b.WriteFile(dir), "version") + }) + + t.Run("when a record is marked as developement", func(t *testing.T) { + dir := t.TempDir() + + b := bake.Record{ + Version: "1.2.3", + SourceRevision: builder.DirtyWorktreeSHAValue, + } + + require.ErrorContains(t, b.WriteFile(dir), "will not write development") + }) + + t.Run("write only required some fields", func(t *testing.T) { + dir := t.TempDir() + + b := bake.Record{ + Version: "some-version", + } + + require.NoError(t, b.WriteFile(dir)) + + buf, err := os.ReadFile(filepath.Join(dir, bake.RecordsDirectory, "some-version.json")) + require.NoError(t, err) + + require.JSONEq(t, `{"source_revision":"", "version":"some-version"}`, string(buf)) + }) + + t.Run("when a build record with the same version already exists", func(t *testing.T) { + dir := t.TempDir() + + b := bake.Record{ + TileName: "some-tile", + Version: "some-version", + } + + require.NoError(t, b.WriteFile(dir)) + require.ErrorContains(t, b.WriteFile(dir), "tile bake record already exists for some-tile/some-version") + }) + + t.Run("when read builds", func(t *testing.T) { + dir := t.TempDir() + + bs := []bake.Record{ + { // non standard semver + TileName: "p-each", + SourceRevision: "some-hash-000", + KilnVersion: "some-kiln-version", + Version: "0.1.0.0", + FileChecksum: "some-hash-browns", + }, + { + TileName: "p-each", + SourceRevision: "some-hash-000", + KilnVersion: "some-kiln-version", + Version: "0.1.0.2", + FileChecksum: "some-hash-browns", + }, + { + TileName: "p-each", + SourceRevision: "some-hash-000", + KilnVersion: "some-kiln-version", + Version: "1.1.0", + FileChecksum: "some-hash-browns", + }, + { + TileName: "p-each", + SourceRevision: "some-hash-002", + KilnVersion: "some-kiln-version", + Version: "1.2.0", + FileChecksum: "some-hash-browns", + }, + { + TileName: "p-each", + SourceRevision: "some-hash-003", + KilnVersion: "some-kiln-version", + Version: "2.0.0", + FileChecksum: "some-hash-browns", + }, + { + TileName: "p-ear", + SourceRevision: "some-hash-004", + KilnVersion: "some-kiln-version", + Version: "2.0.0", + FileChecksum: "some-hash-browns", + }, + { + TileName: "p-each", + SourceRevision: "some-hash-005", + KilnVersion: "some-kiln-version", + Version: "2.2.0", + FileChecksum: "some-hash-browns", + }, + } + + for _, b := range bs { + require.NoError(t, b.WriteFile(dir)) + } + + result, err := bake.ReadRecords(os.DirFS(dir)) + require.NoError(t, err) + + require.Equal(t, bs, result, "the builds are in order and contain all the info") + }) +} + +func TestBakeRecord_SetTileDirectory(t *testing.T) { + t.Run("when run in a subdirectory", func(t *testing.T) { + // this test makes use of the go test characteristic that tests are run in the repository root + + repoRoot := createAndMoveToTemporaryTileDirectory(t, "peach", "pear") + createGitRepository(t, repoRoot) + + record, err := bake.Record{}.SetTileDirectory(".") + require.NoError(t, err) + + assert.Equal(t, "peach/pear", record.TileDirectory) + }) + + t.Run("when run in the root", func(t *testing.T) { + // this test makes use of the go test characteristic that tests are run in the repository root + + repoRoot := createAndMoveToTemporaryTileDirectory(t) + createGitRepository(t, repoRoot) + + record, err := bake.Record{}.SetTileDirectory(".") + require.NoError(t, err) + + assert.Equal(t, ".", record.TileDirectory) + }) + + t.Run("when passed a directory not in the repository", func(t *testing.T) { + // this test makes use of the go test characteristic that tests are run in the repository root + + repoRoot := createAndMoveToTemporaryTileDirectory(t) + createGitRepository(t, repoRoot) + + someOtherDir := t.TempDir() + + _, err := bake.Record{}.SetTileDirectory(someOtherDir) + require.Error(t, err, "either be or be a child of the repository root directory") + }) + + t.Run("when passed a sub directory", func(t *testing.T) { + // this test makes use of the go test characteristic that tests are run in the repository root + + repoRoot := createAndMoveToTemporaryTileDirectory(t) + createGitRepository(t, repoRoot) + subDir := filepath.Join("peach", "pear") + require.NoError(t, os.MkdirAll(subDir, 0o766)) + + record, err := bake.Record{}.SetTileDirectory(subDir) + require.NoError(t, err) + + assert.Equal(t, "peach/pear", record.TileDirectory) + }) + + t.Run("when executed from a non repository child directory", func(t *testing.T) { + // this test makes use of the go test characteristic that tests are run in the repository root + + repoDir, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + createGitRepository(t, repoDir) + + record, err := bake.Record{}.SetTileDirectory(repoDir) + require.NoError(t, err) + + assert.Equal(t, ".", record.TileDirectory) + }) +} + +func createAndMoveToTemporaryTileDirectory(t *testing.T, subDirectory ...string) string { + t.Helper() + testDir, err := os.Getwd() + require.NoError(t, err) + repoDir := t.TempDir() + + require.NoError(t, os.Chdir(repoDir)) + + if len(subDirectory) > 0 { + subDir := filepath.Join(subDirectory...) + require.NoError(t, os.MkdirAll(subDir, 0o766)) + require.NoError(t, os.Chdir(subDir)) + } + + t.Cleanup(func() { + if err := os.Chdir(testDir); err != nil { + t.Fatalf("failed to go back to test dir: %s", err) + } + }) + + // EvalSymlinks is required on GOOS=darwin because /var (an ancestor of the returned value from t.TempDir) is a + // symbolic link to /private/var. `git rev-parse --show-toplevel` returns a path with symbolic links resolved. + // To simplify test assertions and the implementation, we resolve thos symbolic links here in the helper. + resolved, err := filepath.EvalSymlinks(repoDir) + require.NoError(t, err) + + return resolved +} + +func createGitRepository(t *testing.T, dir string) { + t.Helper() + gitInit := exec.Command("git", "init") + gitInit.Dir = dir + require.NoError(t, gitInit.Run()) +} diff --git a/pkg/bake/testdata/tile.pivotal b/pkg/bake/testdata/tile.pivotal new file mode 100644 index 000000000..da550e0ab Binary files /dev/null and b/pkg/bake/testdata/tile.pivotal differ diff --git a/pkg/cargo/kilnfile.go b/pkg/cargo/kilnfile.go index 6594797c5..4622543e2 100644 --- a/pkg/cargo/kilnfile.go +++ b/pkg/cargo/kilnfile.go @@ -368,6 +368,6 @@ type BakeConfiguration struct { PropertyDirectories []string `yaml:"properties_directories,omitempty" json:"properties_directories,omitempty"` RuntimeConfigDirectories []string `yaml:"runtime_configurations_directories,omitempty" json:"runtime_configurations_directories,omitempty"` BOSHVariableDirectories []string `yaml:"bosh_variables_directories,omitempty" json:"bosh_variables_directories,omitempty"` - EmbedPaths []string `yaml:"embed_paths_directories,omitempty" json:"embed_paths_directories,omitempty"` + EmbedPaths []string `yaml:"embed_paths,omitempty" json:"embed_paths,omitempty"` VariableFiles []string `yaml:"variable_files,omitempty" json:"variable_files,omitempty"` } diff --git a/pkg/source/bake_record.go b/pkg/source/bake_record.go deleted file mode 100644 index b8e41b550..000000000 --- a/pkg/source/bake_record.go +++ /dev/null @@ -1,148 +0,0 @@ -package source - -import ( - "encoding/json" - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "slices" - "strings" - - "gopkg.in/yaml.v3" - - "github.com/pivotal-cf/kiln/internal/builder" - - "github.com/Masterminds/semver/v3" -) - -// BakeRecordsDirectory should be a sibling to Kilnfile or base.yml -const BakeRecordsDirectory = "bake_records" - -// BakeRecord is created by the function -type BakeRecord struct { - // SourceRevision is the commit checked out when the build was run - SourceRevision string `yaml:"source_revision" json:"source_revision"` - - // Version is the tile version used for product template field `product_version` - Version string `yaml:"version" json:"version"` - - // KilnVersion is the Kiln CLI version - KilnVersion string `yaml:"kiln_version" json:"kiln_version"` - - // TileName might record the tile name used in baking because sometimes multiple tiles can be generated from the same tile source directory - // An example of this is the two Tanzu Application Service tiles with different topologies listed on TanzuNetwork. - TileName string `yaml:"tile_name,omitempty" json:"tile_name,omitempty"` - - // FileChecksum is the SHA256 checksum of the baked tile. - FileChecksum string `yaml:"file_checksum,omitempty" json:"file_checksum,omitempty"` -} - -// NewBakeRecord parses build information from an OpsManger Product Template (aka metadata/metadata.yml) -func NewBakeRecord(fileChecksum string, productTemplateBytes []byte) (BakeRecord, error) { - var productTemplate struct { - ProductVersion string `yaml:"product_version"` - KilnMetadata builder.KilnMetadata `yaml:"kiln_metadata"` - } - - err := yaml.Unmarshal(productTemplateBytes, &productTemplate) - - if productTemplate.KilnMetadata.KilnVersion == "" { - return BakeRecord{}, fmt.Errorf("failed to parse build information from product template: kiln_metadata.kiln_version not found") - } - - return BakeRecord{ - SourceRevision: productTemplate.KilnMetadata.MetadataGitSHA, - Version: productTemplate.ProductVersion, - KilnVersion: productTemplate.KilnMetadata.KilnVersion, - TileName: productTemplate.KilnMetadata.TileName, - FileChecksum: fileChecksum, - }, err -} - -func (b BakeRecord) Name() string { - if b.TileName != "" { - return path.Join(b.TileName, b.Version) - } - return b.Version -} - -func (b BakeRecord) CompareVersion(o BakeRecord) int { - bv, err := semver.NewVersion(b.Version) - if err != nil { - return strings.Compare(b.Version, o.Version) - } - ov, err := semver.NewVersion(o.Version) - if err != nil { - return strings.Compare(b.Version, o.Version) - } - return bv.Compare(ov) -} - -func (b BakeRecord) CompareTileName(o BakeRecord) int { - return strings.Compare(b.TileName, o.TileName) -} - -func (b BakeRecord) IsDevBuild() bool { - return b.SourceRevision == builder.DirtyWorktreeSHAValue -} - -func (b BakeRecord) WriteFile(tileSourceDirectory string) error { - if b.Version == "" { - return fmt.Errorf("missing required version field") - } - if b.IsDevBuild() { - return fmt.Errorf("will not write development builds to %s directory", BakeRecordsDirectory) - } - if err := os.MkdirAll(filepath.Join(tileSourceDirectory, BakeRecordsDirectory), 0o766); err != nil { - return err - } - buf, err := json.MarshalIndent(b, "", " ") - if err != nil { - return err - } - fileName := b.Version + ".json" - if b.TileName != "" { - fileName = b.TileName + "-" + fileName - } - outputFilepath := filepath.Join(tileSourceDirectory, BakeRecordsDirectory, fileName) - if _, err := os.Stat(outputFilepath); err == nil { - return fmt.Errorf("tile bake record already exists for %s", b.Name()) - } - return os.WriteFile(outputFilepath, buf, 0o644) -} - -func ReadBakeRecords(dir fs.FS) ([]BakeRecord, error) { - infos, err := fs.ReadDir(dir, BakeRecordsDirectory) - if err != nil { - return nil, err - } - builds := make([]BakeRecord, 0, len(infos)) - for _, info := range infos { - buf, err := fs.ReadFile(dir, path.Join(BakeRecordsDirectory, info.Name())) - if err != nil { - return nil, err - } - var build BakeRecord - if err := json.Unmarshal(buf, &build); err != nil { - return nil, err - } - builds = append(builds, build) - } - slices.SortFunc(builds, compareMultiple(BakeRecord.CompareVersion, BakeRecord.CompareTileName)) - return builds, nil -} - -func compareMultiple[T any](cmp ...func(a, b T) int) func(a, b T) int { - return func(a, b T) int { - var result int - for _, c := range cmp { - result = c(a, b) - if result != 0 { - break - } - } - return result - } -} diff --git a/pkg/source/bake_record_test.go b/pkg/source/bake_record_test.go deleted file mode 100644 index bb841376f..000000000 --- a/pkg/source/bake_record_test.go +++ /dev/null @@ -1,175 +0,0 @@ -package source_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/pivotal-cf/kiln/internal/builder" - - "github.com/stretchr/testify/require" - - "github.com/pivotal-cf/kiln/pkg/source" -) - -func TestBuild(t *testing.T) { - t.Run("when creating a bake record from a product template", func(t *testing.T) { - // language=yaml - b, err := source.NewBakeRecord("some-peach-jam", []byte(` -product_name: p-each -product_version: some-product-version -kiln_metadata: - kiln_version: some-kiln-version - metadata_git_sha: some-tile-source-revision - tile_name: srt -`)) - require.NoError(t, err) - require.Equal(t, source.BakeRecord{ - Version: "some-product-version", - KilnVersion: "some-kiln-version", - SourceRevision: "some-tile-source-revision", - TileName: "srt", - FileChecksum: "some-peach-jam", - }, b) - }) - - t.Run("when the product template is missing kiln_metadata", func(t *testing.T) { - // language=yaml - _, err := source.NewBakeRecord("some-peach-jam", []byte(` -product_name: p-each -product_version: some-product-version -`)) - require.ErrorContains(t, err, "kiln_metadata") - }) - - t.Run("write one file", func(t *testing.T) { - dir := t.TempDir() - - b := source.BakeRecord{ - TileName: "p-each", - SourceRevision: "some-revision", - Version: "1.2.3", - KilnVersion: "some-version", - } - - require.NoError(t, b.WriteFile(dir)) - - buf, err := os.ReadFile(filepath.Join(dir, source.BakeRecordsDirectory, "p-each-1.2.3.json")) - require.NoError(t, err) - - require.JSONEq(t, `{"source_revision":"some-revision", "tile_name":"p-each", "version":"1.2.3", "kiln_version": "some-version"}`, string(buf)) - }) - - t.Run("when the record is missing the version field", func(t *testing.T) { - dir := t.TempDir() - - b := source.BakeRecord{ - Version: "", - } - - require.ErrorContains(t, b.WriteFile(dir), "version") - }) - - t.Run("when a record is marked as developement", func(t *testing.T) { - dir := t.TempDir() - - b := source.BakeRecord{ - Version: "1.2.3", - SourceRevision: builder.DirtyWorktreeSHAValue, - } - - require.ErrorContains(t, b.WriteFile(dir), "will not write development") - }) - - t.Run("write only required some fields", func(t *testing.T) { - dir := t.TempDir() - - b := source.BakeRecord{ - Version: "some-version", - } - - require.NoError(t, b.WriteFile(dir)) - - buf, err := os.ReadFile(filepath.Join(dir, source.BakeRecordsDirectory, "some-version.json")) - require.NoError(t, err) - - require.JSONEq(t, `{"source_revision":"", "version":"some-version", "kiln_version": ""}`, string(buf)) - }) - - t.Run("when a build record with the same version already exists", func(t *testing.T) { - dir := t.TempDir() - - b := source.BakeRecord{ - TileName: "some-tile", - Version: "some-version", - } - - require.NoError(t, b.WriteFile(dir)) - require.ErrorContains(t, b.WriteFile(dir), "tile bake record already exists for some-tile/some-version") - }) - - t.Run("when read builds", func(t *testing.T) { - dir := t.TempDir() - - bs := []source.BakeRecord{ - { // non standard semver - TileName: "p-each", - SourceRevision: "some-hash-000", - KilnVersion: "some-kiln-version", - Version: "0.1.0.0", - FileChecksum: "some-hash-browns", - }, - { - TileName: "p-each", - SourceRevision: "some-hash-000", - KilnVersion: "some-kiln-version", - Version: "0.1.0.2", - FileChecksum: "some-hash-browns", - }, - { - TileName: "p-each", - SourceRevision: "some-hash-000", - KilnVersion: "some-kiln-version", - Version: "1.1.0", - FileChecksum: "some-hash-browns", - }, - { - TileName: "p-each", - SourceRevision: "some-hash-002", - KilnVersion: "some-kiln-version", - Version: "1.2.0", - FileChecksum: "some-hash-browns", - }, - { - TileName: "p-each", - SourceRevision: "some-hash-003", - KilnVersion: "some-kiln-version", - Version: "2.0.0", - FileChecksum: "some-hash-browns", - }, - { - TileName: "p-ear", - SourceRevision: "some-hash-004", - KilnVersion: "some-kiln-version", - Version: "2.0.0", - FileChecksum: "some-hash-browns", - }, - { - TileName: "p-each", - SourceRevision: "some-hash-005", - KilnVersion: "some-kiln-version", - Version: "2.2.0", - FileChecksum: "some-hash-browns", - }, - } - - for _, b := range bs { - require.NoError(t, b.WriteFile(dir)) - } - - result, err := source.ReadBakeRecords(os.DirFS(dir)) - require.NoError(t, err) - - require.Equal(t, bs, result, "the builds are in order and contain all the info") - }) -}