Skip to content

Commit

Permalink
Template imports
Browse files Browse the repository at this point in the history
  • Loading branch information
@syhpoon
syhpoon committed Feb 13, 2019
1 parent c302a1b commit e12285a
Show file tree
Hide file tree
Showing 16 changed files with 393 additions and 95 deletions.
5 changes: 5 additions & 0 deletions ChangeLog
View file
@@ -1,3 +1,8 @@
# v2.1.0 (2019-02-12)

* Added template import support
* Added new config parameter: `tpl.recursion_limit`

# v2.0.0 (2019-01-23)

* Added dynamic discovery agent
Expand Down
1 change: 1 addition & 0 deletions cmd/run.go
View file
Expand Up @@ -83,6 +83,7 @@ var runCmd = &cobra.Command{
params.TLSCertFile = config.GetString("tls.cert")
params.TLSKeyFile = config.GetString("tls.key")
params.DefaultKeepalive = config.GetDuration("keepalive")
params.RecursionLimit = config.GetInt("tpl.recursion_limit")
params.CengCtx = cengCtx

runLog.Infof("Base directory: %s", params.BaseTplDir)
Expand Down
1 change: 1 addition & 0 deletions pkg/config/default.go
View file
Expand Up @@ -35,6 +35,7 @@ keepalive = "2m"
base_dir = "."
ws_dir = "/tmp/xenvman/ws"
mount_dir = "/tmp/xenvman/mount"
recursion_limit = 1000
[log]
config = "<root>=trace"
Expand Down
4 changes: 3 additions & 1 deletion pkg/conteng/conteng_docker.go
View file
Expand Up @@ -110,7 +110,7 @@ func (de *DockerEngine) CreateNetwork(ctx context.Context,
r, err := de.cl.NetworkCreate(ctx, name, netParams)

if err != nil {
return "", "", errors.Wrapf(err, "Error creating docker network")
return "", "", errors.Wrapf(err, "Error creating docker network: %s", sub)
}

dockerLog.Debugf("Network created: %s - %s :: %s", name, r.ID, sub)
Expand Down Expand Up @@ -367,6 +367,8 @@ func (de *DockerEngine) getSubNet() (string, error) {
var nets []*net.IPNet

for _, addr := range addrs {
dockerLog.Debugf("Inspecting interface %s", addr.String())

_, n, err := net.ParseCIDR(addr.String())

if err != nil {
Expand Down
57 changes: 49 additions & 8 deletions pkg/def/output_env.go
View file
Expand Up @@ -25,6 +25,10 @@
package def

import (
"fmt"
"strconv"
"strings"

"github.com/pkg/errors"
)

Expand All @@ -46,6 +50,8 @@ func NewContainerData(id, hostname string) *ContainerData {
type TplData struct {
// Container name -> <internal port> -> "<host>:<external-port>"
Containers map[string]*ContainerData `json:"containers"`
// Imported templates -> tpl name -> [TplData]
Templates map[string][]*TplData `json:"templates"`
}

type OutputEnv struct {
Expand All @@ -63,18 +69,53 @@ type OutputEnv struct {

func (e *OutputEnv) GetContainer(tplName string, tplIdx int,
contName string) (*ContainerData, error) {
tpls, ok := e.Templates[tplName]

if !ok {
return nil, errors.Errorf("Template not found: %s", tplName)
}
return e.GetContainerByPath([]string{
fmt.Sprintf("%s|%d", tplName, tplIdx),
}, contName)
}

// Path element must be defined in the format: <template>|<index>
func (e *OutputEnv) GetContainerByPath(path []string,
contName string) (*ContainerData, error) {

var data *TplData

tmap := e.Templates

for _, el := range path {
split := strings.Split(el, "|")

if len(split) != 2 {
return nil, errors.Errorf(
"Invalid template path format: %s, expected <template>|<index>",
el)
}

tplName := split[0]
tplIdx, err := strconv.ParseInt(split[1], 10, 32)

if err != nil {
return nil, errors.Wrapf(err, "Invalid template index in %s",
split[1])
}

tpls, ok := tmap[tplName]

if !ok {
return nil, errors.Errorf("Template not found: %s", el)
}

if int(tplIdx) >= len(tpls) {
return nil, errors.Errorf("Template %s index %d not found",
tplName, tplIdx)
}

if tplIdx >= len(tpls) {
return nil, errors.Errorf("Template %s index %d not found",
tplName, tplIdx)
data = tpls[tplIdx]
tmap = data.Templates
}

cont, ok := tpls[tplIdx].Containers[contName]
cont, ok := data.Containers[contName]

if !ok {
return nil, errors.Errorf("Container not found: %s", contName)
Expand Down
134 changes: 86 additions & 48 deletions pkg/env/env.go
View file
Expand Up @@ -90,6 +90,7 @@ type Params struct {
BaseMountDir string
ExportAddress string
DefaultKeepAlive def.Duration
RecursionLimit int
Ctx context.Context
}

Expand Down Expand Up @@ -377,8 +378,9 @@ func (env *Env) buildAndFetch(toBuild map[string]*tpl.BuildImage,
}

type executeResult struct {
t *tpl.Tpl
idx int
t *tpl.Tpl
imprt []*def.Tpl
idx int
}

type executeResults []*executeResult
Expand All @@ -389,7 +391,11 @@ func (er executeResults) Less(i, j int) bool { return er[i].idx < er[j].idx }

// Execute templates in parallel
func (env *Env) executeTemplates(tpls []*def.Tpl,
needDiscovery bool) ([]*tpl.Tpl, error) {
needDiscovery bool, rec int) ([]*tpl.Tpl, error) {

if rec >= env.params.RecursionLimit {
return nil, errors.Errorf("Recursion limit reached")
}

ctx, cancel := context.WithCancel(env.params.Ctx)
defer cancel()
Expand Down Expand Up @@ -451,6 +457,16 @@ func (env *Env) executeTemplates(tpls []*def.Tpl,

for i, r := range results {
templates[i] = r.t

if len(r.imprt) > 0 {
itpls, err := env.executeTemplates(r.imprt, false, rec+1)

if err != nil {
return nil, errors.Wrap(err, "Error executing imported templates")
} else {
templates[i].SetImported(itpls)
}
}
}

return templates, nil
Expand Down Expand Up @@ -567,17 +583,36 @@ func (env *Env) Export() *def.OutputEnv {
env.RLock()
defer env.RUnlock()

return &def.OutputEnv{
Id: env.id,
Name: env.params.EnvDef.Name,
Description: env.params.EnvDef.Description,
WsDir: env.wsDir,
MountDir: env.mountDir,
NetId: env.netId,
Created: env.created.Format(time.RFC3339),
Keepalive: env.keepalive.String(),
ExternalAddress: env.params.ExportAddress,
Templates: env.exportTemplates(env.tpls),
}
}

func (env *Env) exportTemplates(tpls []*tpl.Tpl) map[string][]*def.TplData {
templates := map[string][]*def.TplData{}

for tplName, tpls := range env.ports {
for idx, t := range tpls {
tpld := &def.TplData{
Containers: map[string]*def.ContainerData{},
}
for _, tplobj := range tpls {
name := tplobj.GetName()
idx := tplobj.GetIdx()

for cont, ps := range t {
tpld := &def.TplData{
Containers: map[string]*def.ContainerData{},
Templates: env.exportTemplates(tplobj.GetImported()),
}

if len(env.ports[name]) > idx {
for cont, ps := range env.ports[name][idx] {
if _, ok := tpld.Containers[cont]; !ok {
cid := env.contIds[tplName][idx][cont]
cid := env.contIds[name][idx][cont]

tpld.Containers[cont] = def.NewContainerData(
cid, env.containers[cid].Hostname())
Expand All @@ -587,23 +622,12 @@ func (env *Env) Export() *def.OutputEnv {
tpld.Containers[cont].Ports[fmt.Sprintf("%d", ip)] = int(ep)
}
}

templates[tplName] = append(templates[tplName], tpld)
}
}

return &def.OutputEnv{
Id: env.id,
Name: env.params.EnvDef.Name,
Description: env.params.EnvDef.Description,
WsDir: env.wsDir,
MountDir: env.mountDir,
NetId: env.netId,
Created: env.created.Format(time.RFC3339),
Keepalive: env.keepalive.String(),
ExternalAddress: env.params.ExportAddress,
Templates: templates,
templates[name] = append(templates[name], tpld)
}

return templates
}

func (env *Env) execTpl(tplObj *def.Tpl, idx int,
Expand All @@ -628,12 +652,12 @@ func (env *Env) execTpl(tplObj *def.Tpl, idx int,
params.TplDir = env.params.BaseTplDir
}

t, err := tpl.Execute(env.id, tplObj.Tpl, idx, params)
t, imprt, err := tpl.Execute(env.id, tplObj.Tpl, idx, params)

if err != nil {
errch <- errors.WithStack(err)
} else {
rch <- &executeResult{t: t, idx: idx}
rch <- &executeResult{t: t, imprt: imprt, idx: idx}
}
}

Expand Down Expand Up @@ -702,33 +726,13 @@ func (env *Env) ApplyTemplates(tplDefs []*def.Tpl,

var containers []*tpl.Container

tpls, err := env.executeTemplates(tplDefs, needDiscovery)
tpls, err := env.executeTemplates(tplDefs, needDiscovery, 0)

if err != nil {
return errors.WithStack(err)
}

for _, t := range tpls {
// Collect build images
for _, bimg := range t.GetBuildImages() {
imagesToBuild[bimg.Name()] = bimg

// Collect containers
for _, c := range bimg.Containers() {
containers = append(containers, c)
}
}

// Collect fetch images
for _, fimg := range t.GetFetchImages() {
imagesToFetch[fimg.Name()] = fimg

// Collect containers
for _, c := range fimg.Containers() {
containers = append(containers, c)
}
}
}
env.extractContainers(tpls, &containers, imagesToBuild, imagesToFetch)

imgPorts, err := env.buildAndFetch(imagesToBuild, imagesToFetch,
env.params.ContEng, env.params.Ctx)
Expand Down Expand Up @@ -957,3 +961,37 @@ func (env *Env) ApplyTemplates(tplDefs []*def.Tpl,

return nil
}

func (env *Env) extractContainers(
tpls []*tpl.Tpl, containers *[]*tpl.Container,
imagesToBuild map[string]*tpl.BuildImage,
imagesToFetch map[string]*tpl.FetchImage) {

for _, t := range tpls {
// Collect build images
for _, bimg := range t.GetBuildImages() {
imagesToBuild[bimg.Name()] = bimg

// Collect containers
for _, c := range bimg.Containers() {
*containers = append(*containers, c)
}
}

// Collect fetch images
for _, fimg := range t.GetFetchImages() {
imagesToFetch[fimg.Name()] = fimg

// Collect containers
for _, c := range fimg.Containers() {
*containers = append(*containers, c)
}
}

// Process imported templates as well
if len(t.GetImported()) > 0 {
env.extractContainers(t.GetImported(),
containers, imagesToBuild, imagesToFetch)
}
}
}

0 comments on commit e12285a

Please sign in to comment.