-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
prototype implementation of core client builder functionality (#1)
* chore: save initial packages, Makefile, and module. Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: add workspace package with interface and initial local workspace type Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * feat: initial implementation for builder capabilties Moves parsing and graph packages under the builder package Adds methods to the Workspace interface Adds JSON parsing concrete implementation Adds initial CLI implemenation Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * feat(cli): adds flags to configure registry client Adds initial flags for insecure, plainHTTP, and alternate auth configs Adds local workspace unit tests Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * feat(parser): adds TemplateFunc type to parser to allow for generic templating conditions Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: add initial Roadmap file Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * feat(cli): adds push and destination flags to make artifact publishing optional Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * feat(parser): adds support for multi-level workspaces fix(parser): handles error before dereferencing template pointer to avoid panic Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * test(parser): adds unit tests for json parser implementation Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * feat(parser): updates parser type determination to use detected content types Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore(parser): clarifies comments in parser package around TemplatingFuncs Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * refactor(registryclient): refactors the oras client implementation to use functional options Wrapping oras as a client implementation needs a unified configuration method for copy and registy options. Use functional options to allow the options to be easily extended. Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: adds comments to orasclient public methods in oras.go Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: fixes spelling errors in builder.go comments Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: Adds basic usage information to README and comments to json.go in parser pkg Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: move building functionality under "build" subcommand Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * refactor: improve usability of builder pkg API Changes Build method to Builder type and Run method to protect against API misuse that can happen when having multiple arguments of the same type. Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore(deps): remove vendor directory for initial review Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: removes all target as the default target in the Makefile Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com> * chore: removed platform specific variablex from build target and updated README with url Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com>
- Loading branch information
Showing
38 changed files
with
3,320 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
# Build artifacts. | ||
/bin/ | ||
|
||
# Test artifacts. | ||
coverage.out | ||
sha256sum.txt | ||
**/test-output | ||
client-workspace | ||
|
||
|
||
# Logs. | ||
|
||
|
||
# Local vendor. Remove when ready to vendor dependencies. | ||
#/vendor/ | ||
|
||
# Local | ||
.idea | ||
tags | ||
*.swp | ||
.vscode |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
GO := go | ||
|
||
GO_BUILD_PACKAGES := ./cmd/... | ||
GO_BUILD_BINDIR :=./bin | ||
|
||
build: | ||
mkdir -p ${GO_BUILD_BINDIR} | ||
$(GO) build -o $(GO_BUILD_BINDIR)/client $(GO_BUILD_PACKAGES) | ||
.PHONY: build | ||
|
||
vendor: | ||
$(GO) mod tidy | ||
$(GO) mod verify | ||
$(GO) mod vendor | ||
.PHONY: vendor | ||
|
||
clean: | ||
@rm -rf ./$(GO_BUILD_BINDIR)/* | ||
.PHONY: clean | ||
|
||
test-unit: | ||
$(GO) test $(GO_BUILD_FLAGS) -coverprofile=coverage.out -race -count=1 ./... | ||
.PHONY: test-unit | ||
|
||
sanity: vendor format vet | ||
git diff --exit-code | ||
.PHONY: sanity | ||
|
||
format: | ||
$(GO) fmt ./... | ||
.PHONY: format | ||
|
||
vet: | ||
$(GO) vet ./... | ||
.PHONY: vet | ||
|
||
all: clean vendor test-unit build | ||
.PHONY: all |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
# Road Map | ||
|
||
## Core Functionality | ||
- Workspace traversal | ||
- Artifact publishing | ||
- JSON File Support | ||
|
||
## Support for Additional Formats | ||
- XML parsing support | ||
- Unstructured data support | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,95 @@ | ||
package builder | ||
|
||
import ( | ||
"bytes" | ||
"context" | ||
"fmt" | ||
"text/template" | ||
|
||
"github.com/opencontainers/go-digest" | ||
|
||
"github.com/uor-framework/client/builder/graph" | ||
"github.com/uor-framework/client/builder/parser" | ||
"github.com/uor-framework/client/util/workspace" | ||
) | ||
|
||
// Builder renders and writes templates from the source workspace. | ||
type Builder struct { | ||
Source workspace.Workspace | ||
} | ||
|
||
// NewBuilder creates an new Builder from the source | ||
// workspace | ||
func NewBuilder(source workspace.Workspace) Builder { | ||
return Builder{source} | ||
} | ||
|
||
// Run traverses the graph to render the file templates to the destination workspace. | ||
func (b Builder) Run(ctx context.Context, g *graph.Graph, destination workspace.Workspace) error { | ||
root, err := g.Root() | ||
if err != nil { | ||
return fmt.Errorf("error calculating root node: %v", err) | ||
} | ||
// Links store the calculated sub problem (i.e. link hashes) | ||
links := make(map[string]interface{}) | ||
return b.makeTemplates(ctx, g, root, destination, links) | ||
} | ||
|
||
// makeTemplates does recursive DFS traversal of the graph to generate digest values and template files. | ||
func (b Builder) makeTemplates(ctx context.Context, g *graph.Graph, start *graph.Node, destination workspace.Workspace, links map[string]interface{}) error { | ||
if start == nil { | ||
return nil | ||
} | ||
|
||
// Template and hash each child node to | ||
// calculate parent node information | ||
for _, n := range start.Nodes { | ||
if _, found := links[n.Name]; found { | ||
continue | ||
} | ||
if err := b.makeTemplates(ctx, g, n, destination, links); err != nil { | ||
return err | ||
} | ||
} | ||
|
||
start.Links = mergeLinkData(start.Links, links) | ||
|
||
buf := new(bytes.Buffer) | ||
if start.Template != (template.Template{}) { | ||
if err := start.Template.Execute(buf, start.Links); err != nil { | ||
return err | ||
} | ||
} else { | ||
if err := b.Source.ReadObject(ctx, start.Name, buf); err != nil { | ||
return err | ||
} | ||
} | ||
|
||
if err := destination.WriteObject(ctx, start.Name, buf.Bytes()); err != nil { | ||
return err | ||
} | ||
|
||
// Must calculate the digest after writing the content of | ||
// the buffer to file because the FromReader method consumes the data. | ||
dgst, err := digest.FromReader(buf) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
templateValue := parser.ConvertFilenameForGoTemplateValue(start.Name) | ||
links[templateValue] = dgst | ||
|
||
return nil | ||
} | ||
|
||
// mergeLinkData will merge any references to in-content links with | ||
// the currently calculated values. | ||
func mergeLinkData(in, curr map[string]interface{}) map[string]interface{} { | ||
for key := range in { | ||
currentVal, ok := curr[key] | ||
if ok { | ||
in[key] = currentVal | ||
} | ||
} | ||
return in | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
/* | ||
Copyright 2022 UOR-Framework Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package builder | ||
|
||
// This package performs the content templating, building, | ||
// and rendering of the client data sets. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
/* | ||
Copyright 2022 UOR-Framework Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package graph | ||
|
||
// This package is used to define the graph and methods | ||
// to get node relationship information. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,126 @@ | ||
package graph | ||
|
||
import ( | ||
"errors" | ||
"fmt" | ||
"sort" | ||
"strings" | ||
"text/template" | ||
) | ||
|
||
// Node defines a single unit containing build information about a file. | ||
type Node struct { | ||
// Unique node name | ||
Name string | ||
// Nodes will describe nodes connected to this one | ||
Nodes map[string]*Node | ||
// Builder specific fields | ||
Template template.Template | ||
Links map[string]interface{} | ||
} | ||
|
||
// NewNode create a empty Node. | ||
func NewNode(name string) *Node { | ||
return &Node{ | ||
Name: name, | ||
Nodes: map[string]*Node{}, | ||
Links: map[string]interface{}{}, | ||
} | ||
} | ||
|
||
// Graph defines a collection of Nodes. | ||
type Graph struct { | ||
// Nodes describes all nodes contained in the graph | ||
Nodes map[string]*Node | ||
} | ||
|
||
// NewGraph creates an empty Graph. | ||
func NewGraph() *Graph { | ||
return &Graph{ | ||
Nodes: map[string]*Node{}, | ||
} | ||
} | ||
|
||
// AddNode adds a new node to the graph. | ||
func (g *Graph) AddNode(name string) { | ||
n := NewNode(name) | ||
g.Nodes[name] = n | ||
} | ||
|
||
// AddNodeTemplate adds a template to the node at the specified key in the graph. | ||
func (g *Graph) AddNodeTemplate(key string, t template.Template) error { | ||
n, found := g.Nodes[key] | ||
if !found { | ||
return fmt.Errorf("node %v not found in graph", key) | ||
} | ||
n.Template = t | ||
g.Nodes[key] = n | ||
return nil | ||
} | ||
|
||
// AddNodeLinkInformation adds link data to the node at the specified key in the graph. | ||
func (g *Graph) AddNodeLinkInformation(key string, links map[string]interface{}) error { | ||
n, found := g.Nodes[key] | ||
if !found { | ||
return fmt.Errorf("node %v not found in graph", key) | ||
} | ||
n.Links = links | ||
g.Nodes[key] = n | ||
return nil | ||
} | ||
|
||
// AddEdge adds an edge between two nodes in the graph | ||
func (g *Graph) AddEdge(origin, destination string) error { | ||
n1 := g.Nodes[origin] | ||
n2 := g.Nodes[destination] | ||
|
||
// return an error if one of the nodes doesn't exist | ||
if n1 == nil || n2 == nil { | ||
return errors.New("not all nodes exist") | ||
} | ||
|
||
// do nothing if the node are already connected | ||
if _, ok := n1.Nodes[n2.Name]; ok { | ||
return nil | ||
} | ||
|
||
n1.Nodes[n2.Name] = n2 | ||
|
||
// Add the nodes to the graph's node map | ||
g.Nodes[n1.Name] = n1 | ||
g.Nodes[n2.Name] = n2 | ||
|
||
return nil | ||
} | ||
|
||
// Root calculates to root node of the graph. | ||
// This is calculated base on existing child nodes. | ||
// This expected only of root node to be found. | ||
func (g *Graph) Root() (*Node, error) { | ||
// FIXME(jpowe432): Optimize or redesign the chain | ||
|
||
childNodes := map[string]int{} | ||
for _, n := range g.Nodes { | ||
for _, ch := range n.Nodes { | ||
childNodes[ch.Name]++ | ||
} | ||
} | ||
var roots []*Node | ||
for _, n := range g.Nodes { | ||
if _, found := childNodes[n.Name]; !found { | ||
roots = append(roots, n) | ||
} | ||
} | ||
if len(roots) == 0 { | ||
return nil, fmt.Errorf("no root found in graph") | ||
} | ||
if len(roots) > 1 { | ||
var rootNames []string | ||
for _, root := range roots { | ||
rootNames = append(rootNames, root.Name) | ||
} | ||
sort.Strings(rootNames) | ||
return nil, fmt.Errorf("multiple roots found in graph: %s", strings.Join(rootNames, ", ")) | ||
} | ||
return roots[0], nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
/* | ||
Copyright 2022 UOR-Framework Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package parser | ||
|
||
// This package is for parsing content to produce | ||
// references to content templates and linkable data. |
Oops, something went wrong.