Merge pull request #43 from tarent/lazy-fetch-definitions

Lazy fetch definitions

composition/README.md

@@ -221,6 +221,27 @@ The default is `required=false`, if not specified.
 The alternative content for optional html includes is currently not implemented.
 
 
+
+#### Includes with Paramters
+The HTML Syntax allows to specify includes, which are not preloaded, but will be loaded from the ui service on demand.
+For the case, it is also possible to specify parameters on the include, which allow the ui service to influence the loading of the content containing the fragment.
+
+Example:
+```
+<uic-include src="example.com/foo#content" param-foo="bar" param-bli="bla"/>
+```
+
+In this example, the ui service is requested to load the content associated with the fetch definition named 'example.com/foo' and the parameter map `{foo: bar, bli: bla}`.
+The content will only be loaded, if the there was not an content with the name `example.com/foo` before. Otherwise, the paramters will be ignored.
+
+*Attention:* Parameter names are always converted to lower case names, because of the underlaying html parser,
+
+
+
+
+
+
+
 #### Fetch directive
 It is possible to specifiy a url for additional content to load,
 while the composition takes place. 

composition/content_fetcher.go

@@ -7,12 +7,6 @@ import (
 	"sync"
 )
 
-// IsFetchable returns, whether the fetch definition refers to a fetchable resource
-// or is a local name only.
-func (def *FetchDefinition) IsFetchable() bool {
-	return len(def.URL) > 0
-}
-
 type FetchResult struct {
 	Def     *FetchDefinition
 	Err     error
@@ -33,18 +27,26 @@ func (fr FetchResults) Less(i, j int) bool {
 	return fr[i].Def.Priority < fr[j].Def.Priority
 }
 
+// FetchDefinitionFactory should return a fetch definition for the given name and parameters.
+// This factory method can be used to supply lazy loaded fetch jobs.
+// The FetchDefinition returned has to have the same name as the supplied name parameter.
+// If no fetch definition for the supplied name can be provided by the factory, existing=false is returned, otherwise existing=true.
+type FetchDefinitionFactory func(name string, params Params) (fd *FetchDefinition, existing bool, err error)
+
 // ContentFetcher is a type, which can fetch a set of Content pages in parallel.
 type ContentFetcher struct {
 	activeJobs sync.WaitGroup
 	r          struct {
-		results []*FetchResult
-		mutex   sync.Mutex
+		sheduledFetchDefinitionNames map[string]string
+		results                      []*FetchResult
+		mutex                        sync.Mutex
 	}
 	meta struct {
 		json  map[string]interface{}
 		mutex sync.Mutex
 	}
-	Loader ContentLoader
+	lazyFdFactory FetchDefinitionFactory
+	Loader        ContentLoader
 }
 
 // NewContentFetcher creates a ContentFetcher with an HtmlContentParser as default.
@@ -53,14 +55,25 @@ type ContentFetcher struct {
 func NewContentFetcher(defaultMetaJSON map[string]interface{}) *ContentFetcher {
 	f := &ContentFetcher{}
 	f.r.results = make([]*FetchResult, 0, 0)
+	f.r.sheduledFetchDefinitionNames = make(map[string]string)
 	f.Loader = NewHttpContentLoader()
 	f.meta.json = defaultMetaJSON
 	if f.meta.json == nil {
 		f.meta.json = make(map[string]interface{})
 	}
+	f.lazyFdFactory = func(name string, params Params) (fd *FetchDefinition, exist bool, err error) {
+		return nil, false, nil
+	}
 	return f
 }
 
+// SetFetchDefinitionFactory supplies a factory for lazy evaluated fetch jobs,
+// which will only be loaded if a fragment refrences them.
+// Seting the factory of optional, but if used, has to be done before adding Jobs by AddFetchJob.
+func (fetcher *ContentFetcher) SetFetchDefinitionFactory(factory FetchDefinitionFactory) {
+	fetcher.lazyFdFactory = factory
+}
+
 // Wait blocks until all jobs are done,
 // either successful or with an error result and returns the content and errors.
 // Do we need to return the Results in a special order????
@@ -80,6 +93,8 @@ func (fetcher *ContentFetcher) WaitForResults() []*FetchResult {
 	return results
 }
 
+//func (fetcher *ContentFetcher) AddFetchDefinitionFactory(name string, func(params map[string]string) *FetchDefinition) {
+
 // AddFetchJob adds one job to the fetcher and recursively adds the dependencies also.
 func (fetcher *ContentFetcher) AddFetchJob(d *FetchDefinition) {
 	fetcher.r.mutex.Lock()
@@ -91,9 +106,9 @@ func (fetcher *ContentFetcher) AddFetchJob(d *FetchDefinition) {
 	}
 
 	fetcher.activeJobs.Add(1)
-
 	fetchResult := &FetchResult{Def: d, Hash: hash, Err: errors.New("not fetched")}
 	fetcher.r.results = append(fetcher.r.results, fetchResult)
+	fetcher.r.sheduledFetchDefinitionNames[d.Name] = d.Name
 
 	go func() {
 		defer fetcher.activeJobs.Done()
@@ -115,11 +130,7 @@ func (fetcher *ContentFetcher) AddFetchJob(d *FetchDefinition) {
 
 		if fetchResult.Err == nil {
 			fetcher.addMeta(fetchResult.Content.Meta())
-			for _, dependency := range fetchResult.Content.RequiredContent() {
-				if dependency.IsFetchable() {
-					fetcher.AddFetchJob(dependency)
-				}
-			}
+			fetcher.addDependentFetchJobs(fetchResult.Content)
 		} else {
 			// 404 Error already become logged in logger.go
 			if fetchResult.Content == nil || fetchResult.Content.HttpStatusCode() != 404 {
@@ -132,6 +143,29 @@ func (fetcher *ContentFetcher) AddFetchJob(d *FetchDefinition) {
 	}()
 }
 
+func (fetcher *ContentFetcher) addDependentFetchJobs(content Content) {
+	for _, fetch := range content.RequiredContent() {
+		fetcher.AddFetchJob(fetch)
+	}
+	for dependencyName, params := range content.Dependencies() {
+		_, alreadySheduled := fetcher.r.sheduledFetchDefinitionNames[dependencyName]
+		if !alreadySheduled {
+			lazyFd, existing, err := fetcher.lazyFdFactory(dependencyName, params)
+			if err != nil {
+				logging.Logger.WithError(err).
+					WithField("dependencyName", dependencyName).
+					WithField("params", params).
+					Errorf("failed optaining a fetch definition for dependency %v", dependencyName)
+			}
+			if err == nil && existing {
+				fetcher.AddFetchJob(lazyFd)
+			}
+			// error handling: In the case, the fd could not be loaded, we will do
+			// the error handling in the merging process.
+		}
+	}
+}
+
 func (fetcher *ContentFetcher) Empty() bool {
 	fetcher.r.mutex.Lock()
 	defer fetcher.r.mutex.Unlock()

composition/content_fetcher_test.go

@@ -52,6 +52,48 @@ func Test_ContentFetcher_Empty(t *testing.T) {
 	a.True(fetcher.Empty())
 }
 
+func Test_ContentFetcher_LazyDependencies(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	a := assert.New(t)
+
+	loader := NewMockContentLoader(ctrl)
+
+	parent := NewFetchDefinition("/parent")
+	content := NewMockContent(ctrl)
+	loader.EXPECT().
+		Load(parent).
+		Return(content, nil)
+
+	content.EXPECT().
+		RequiredContent().
+		Return([]*FetchDefinition{})
+	content.EXPECT().
+		Meta().
+		Return(nil)
+	content.EXPECT().
+		Dependencies().
+		Return(map[string]Params{"child": Params{"foo": "bar"}})
+
+	child := getFetchDefinitionMock(ctrl, loader, "/child", nil, time.Millisecond*2, nil)
+
+	fetcher := NewContentFetcher(nil)
+	fetcher.Loader = loader
+	fetcher.SetFetchDefinitionFactory(func(name string, params Params) (fd *FetchDefinition, exist bool, err error) {
+		a.Equal("child", name)
+		a.Equal("bar", params["foo"])
+		return child, true, nil
+	})
+
+	fetcher.AddFetchJob(parent)
+	results := fetcher.WaitForResults()
+
+	a.Equal(2, len(results))
+	a.False(fetcher.Empty())
+	a.Equal("/parent", results[0].Def.URL)
+	a.Equal("/child", results[1].Def.URL)
+}
+
 func getFetchDefinitionMock(ctrl *gomock.Controller, loaderMock *MockContentLoader, url string, requiredContent []*FetchDefinition, loaderBlocking time.Duration, metaJSON map[string]interface{}) *FetchDefinition {
 	fd := NewFetchDefinition(url)
 	fd.Timeout = time.Second * 42
@@ -65,6 +107,10 @@ func getFetchDefinitionMock(ctrl *gomock.Controller, loaderMock *MockContentLoad
 		Meta().
 		Return(metaJSON)
 
+	content.EXPECT().
+		Dependencies().
+		Return(map[string]Params{})
+
 	loaderMock.EXPECT().
 		Load(fd).
 		Do(

composition/html_content_parser.go

@@ -12,12 +12,13 @@ import (
 )
 
 const (
-	UicRemove      = "uic-remove"
-	UicInclude     = "uic-include"
-	UicFetch       = "uic-fetch"
-	UicFragment    = "uic-fragment"
-	UicTail        = "uic-tail"
-	ScriptTypeMeta = "text/uic-meta"
+	UicRemove       = "uic-remove"
+	UicInclude      = "uic-include"
+	UicFetch        = "uic-fetch"
+	UicFragment     = "uic-fragment"
+	UicTail         = "uic-tail"
+	ScriptTypeMeta  = "text/uic-meta"
+	ParamAttrPrefix = "param-"
 )
 
 type HtmlContentParser struct {
@@ -123,8 +124,8 @@ forloop:
 					return err
 				} else {
 					c.body[getFragmentName(attrs)] = f
-					for _, dep := range deps {
-						c.requiredContent[dep.URL] = dep
+					for depName, depParams := range deps {
+						c.dependencies[depName] = depParams
 					}
 				}
 				continue
@@ -134,8 +135,8 @@ forloop:
 					return err
 				} else {
 					c.tail = f
-					for _, dep := range deps {
-						c.requiredContent[dep.URL] = dep
+					for depName, depParams := range deps {
+						c.dependencies[depName] = depParams
 					}
 				}
 				continue
@@ -149,9 +150,10 @@ forloop:
 				}
 			}
 			if string(tag) == UicInclude {
-				if replaceTextStart, replaceTextEnd, err := getInclude(z, attrs); err != nil {
+				if replaceTextStart, replaceTextEnd, dependencyName, dependencyParams, err := getInclude(z, attrs); err != nil {
 					return err
 				} else {
+					c.dependencies[dependencyName] = dependencyParams
 					bodyBuff.WriteString(replaceTextStart)
 					// Enhancement: WriteOut sub tree, to allow alternative content
 					//              for optional includes.
@@ -178,9 +180,9 @@ forloop:
 	return nil
 }
 
-func parseFragment(z *html.Tokenizer) (f Fragment, dependencies []*FetchDefinition, err error) {
+func parseFragment(z *html.Tokenizer) (f Fragment, dependencies map[string]Params, err error) {
 	attrs := make([]html.Attribute, 0, 10)
-	dependencies = make([]*FetchDefinition, 0, 0)
+	dependencies = make(map[string]Params)
 
 	buff := bytes.NewBuffer(nil)
 forloop:
@@ -198,9 +200,10 @@ forloop:
 			break forloop
 		case tt == html.StartTagToken || tt == html.SelfClosingTagToken:
 			if string(tag) == UicInclude {
-				if replaceTextStart, replaceTextEnd, err := getInclude(z, attrs); err != nil {
+				if replaceTextStart, replaceTextEnd, dependencyName, dependencyParams, err := getInclude(z, attrs); err != nil {
 					return nil, nil, err
 				} else {
+					dependencies[dependencyName] = dependencyParams
 					fmt.Fprintf(buff, replaceTextStart)
 					// Enhancement: WriteOut sub tree, to allow alternative content
 					//              for optional includes.
@@ -224,30 +227,39 @@ forloop:
 	return StringFragment(buff.String()), dependencies, nil
 }
 
-func getInclude(z *html.Tokenizer, attrs []html.Attribute) (startMarker, endMarker string, error error) {
+func getInclude(z *html.Tokenizer, attrs []html.Attribute) (startMarker, endMarker, dependencyName string, dependencyParams Params, error error) {
 	var srcString string
 	if url, hasUrl := getAttr(attrs, "src"); !hasUrl {
-		return "", "", fmt.Errorf("include definition without src %s", z.Raw())
+		return "", "", "", nil, fmt.Errorf("include definition without src %s", z.Raw())
 	} else {
 		srcString = strings.TrimSpace(url.Val)
 		if strings.HasPrefix(srcString, "#") {
 			srcString = srcString[1:]
 		}
+		dependencyName = strings.Split(srcString, "#")[0]
+	}
+
+	dependencyParams = Params{}
+	for _, a := range attrs {
+		if strings.HasPrefix(a.Key, ParamAttrPrefix) {
+			key := a.Key[len(ParamAttrPrefix):]
+			dependencyParams[key] = a.Val
+		}
 	}
 
 	required := false
 	if r, hasRequired := getAttr(attrs, "required"); hasRequired {
 		if requiredBool, err := strconv.ParseBool(r.Val); err != nil {
-			return "", "", fmt.Errorf("error parsing bool in %s: %s", z.Raw(), err.Error())
+			return "", "", "", nil, fmt.Errorf("error parsing bool in %s: %s", z.Raw(), err.Error())
 		} else {
 			required = requiredBool
 		}
 	}
 
 	if required {
-		return fmt.Sprintf("§[> %s]§", srcString), "", nil
+		return fmt.Sprintf("§[> %s]§", srcString), "", dependencyName, dependencyParams, nil
 	} else {
-		return fmt.Sprintf("§[#> %s]§", srcString), fmt.Sprintf("§[/%s]§", srcString), nil
+		return fmt.Sprintf("§[#> %s]§", srcString), fmt.Sprintf("§[/%s]§", srcString), dependencyName, dependencyParams, nil
 	}
 }
 

composition/html_content_parser_test.go

@@ -362,6 +362,7 @@ func Test_HtmlContentParser_parseBody(t *testing.T) {
 	parser := &HtmlContentParser{}
 	z := html.NewTokenizer(bytes.NewBufferString(`<body some="attribute">
     <h1>Default Fragment Content</h1><br>
+    <uic-include src="example.com/xyz" required="true" param-foo="bar" param-bazz="buzz"/>
     <ul uic-remove>
       <!-- A Navigation for testing -->
     </ul>
@@ -371,7 +372,7 @@ func Test_HtmlContentParser_parseBody(t *testing.T) {
     </uic-fragment>
     <uic-fragment name="content">
       some content
-      <uic-include src="example.com/foo#content" required="true"/>
+      <uic-include src="example.com/foo#content" required="true" param-bli="bla"/>
       <uic-include src="example.com/optional#content" required="false"/>
       <uic-include src="#local" required="true"/>
     </uic-fragment>
@@ -387,7 +388,7 @@ func Test_HtmlContentParser_parseBody(t *testing.T) {
 	a.NoError(err)
 
 	a.Equal(3, len(c.Body()))
-	eqFragment(t, "<h1>Default Fragment Content</h1><br>", c.Body()[""])
+	eqFragment(t, "<h1>Default Fragment Content</h1><br>\n§[> example.com/xyz]§", c.Body()[""])
 	eqFragment(t, `<h1>Headline</h1> §[#> example.com/optional#content]§§[/example.com/optional#content]§`, c.Body()["headline"])
 	eqFragment(t, `some content`+
 		`§[> example.com/foo#content]§`+
@@ -397,6 +398,9 @@ func Test_HtmlContentParser_parseBody(t *testing.T) {
 
 	eqFragment(t, `some="attribute"`, c.BodyAttributes())
 
+	a.Equal(5, len(c.Dependencies()))
+	a.Equal(c.Dependencies()["example.com/xyz"], Params{"foo": "bar", "bazz": "buzz"})
+	a.Equal(c.Dependencies()["example.com/foo"], Params{"bli": "bla"})
 }
 
 func Test_HtmlContentParser_fetchDependencies(t *testing.T) {

composition/interface_mocks_test.go

@@ -124,6 +124,16 @@ func (_mr *_MockContentRecorder) BodyAttributes() *gomock.Call {
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "BodyAttributes")
 }
 
+func (_m *MockContent) Dependencies() map[string]Params {
+	ret := _m.ctrl.Call(_m, "Dependencies")
+	ret0, _ := ret[0].(map[string]Params)
+	return ret0
+}
+
+func (_mr *_MockContentRecorder) Dependencies() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Dependencies")
+}
+
 func (_m *MockContent) Head() Fragment {
 	ret := _m.ctrl.Call(_m, "Head")
 	ret0, _ := ret[0].(Fragment)