From edef676b7e8aaba0254e41665a7eefca2b7e1b2e Mon Sep 17 00:00:00 2001 From: Christoph Deppisch Date: Thu, 15 Dec 2022 16:17:18 +0100 Subject: [PATCH] fix(#3844): Add service discovery for Kamelet data type converter - Enable service discovery for data type converter resources in order to enable factory finder mechanism when resolving data type implementations - Add proper quarkus-maven-plugin build time properties for quarkus.camel.* properties - Fixes the way camel-quarkus build time properties are set (set properties on the quarkus-maven-plugin instead of using generic Maven system properties) - Explicitly add quarkus.camel.service.discovery.include-patterns for data type converter resources in order to enable lazy loading of Kamelets data type implementations Relates to https://github.com/apache/camel-k/issues/1980 --- pkg/apis/camel/v1/maven_types.go | 7 +- pkg/apis/camel/v1/maven_types_support.go | 41 +++++-- pkg/apis/camel/v1/maven_types_support_test.go | 100 ++++++++++++++++++ pkg/builder/project_test.go | 8 +- pkg/builder/quarkus.go | 50 ++++++--- pkg/cmd/local/local.go | 2 +- pkg/util/camel/camel_dependencies.go | 18 ++-- pkg/util/maven/maven_project.go | 6 +- 8 files changed, 193 insertions(+), 39 deletions(-) create mode 100644 pkg/apis/camel/v1/maven_types_support_test.go diff --git a/pkg/apis/camel/v1/maven_types.go b/pkg/apis/camel/v1/maven_types.go index 1d55cb3fd5..2672089e6b 100644 --- a/pkg/apis/camel/v1/maven_types.go +++ b/pkg/apis/camel/v1/maven_types.go @@ -96,4 +96,9 @@ type Server struct { Configuration Properties `xml:"configuration,omitempty" json:"configuration,omitempty"` } -type Properties map[string]string +type StringOrProperties struct { + Value string `xml:",chardata" json:"-"` + Properties map[string]string `xml:"properties,omitempty" json:"properties,omitempty"` +} + +type Properties map[string]StringOrProperties diff --git a/pkg/apis/camel/v1/maven_types_support.go b/pkg/apis/camel/v1/maven_types_support.go index d4777bffcd..d7152e0a9d 100644 --- a/pkg/apis/camel/v1/maven_types_support.go +++ b/pkg/apis/camel/v1/maven_types_support.go @@ -17,7 +17,9 @@ limitations under the License. package v1 -import "encoding/xml" +import ( + "encoding/xml" +) func (in *MavenArtifact) GetDependencyID() string { switch { @@ -35,23 +37,50 @@ type propertiesEntry struct { func (m Properties) AddAll(properties map[string]string) { for k, v := range properties { - m[k] = v + m.Add(k, v) } } +func (m Properties) Add(key string, value string) { + m[key] = StringOrProperties{Value: value} +} + +func (m Properties) AddProperties(key string, properties map[string]string) { + m[key] = StringOrProperties{Properties: properties} +} + func (m Properties) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if len(m) == 0 { return nil } - err := e.EncodeToken(start) - if err != nil { + if err := e.EncodeToken(start); err != nil { return err } for k, v := range m { - if err := e.Encode(propertiesEntry{XMLName: xml.Name{Local: k}, Value: v}); err != nil { - return err + if v.Value != "" { + if err := e.Encode(propertiesEntry{XMLName: xml.Name{Local: k}, Value: v.Value}); err != nil { + return err + } + } + + if len(v.Properties) > 0 { + nestedPropertyStart := xml.StartElement{Name: xml.Name{Local: k}} + if err := e.EncodeToken(nestedPropertyStart); err != nil { + return err + } + + for key, value := range v.Properties { + if err := e.Encode(propertiesEntry{XMLName: xml.Name{Local: key}, Value: value}); err != nil { + return err + } + } + + if err := e.EncodeToken(nestedPropertyStart.End()); err != nil { + return err + } + } } diff --git a/pkg/apis/camel/v1/maven_types_support_test.go b/pkg/apis/camel/v1/maven_types_support_test.go new file mode 100644 index 0000000000..58073bfce6 --- /dev/null +++ b/pkg/apis/camel/v1/maven_types_support_test.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "bytes" + "encoding/xml" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMarshalEmptyProperties(t *testing.T) { + buf := new(bytes.Buffer) + e := xml.NewEncoder(buf) + + err := Properties{}.MarshalXML(e, xml.StartElement{ + Name: xml.Name{Local: "root"}, + }) + + assert.NoError(t, err) + + err = e.Flush() + + assert.NoError(t, err) + assert.Equal(t, "", buf.String()) +} + +func TestMarshalProperties(t *testing.T) { + buf := new(bytes.Buffer) + e := xml.NewEncoder(buf) + + properties := Properties{} + properties.Add("v1", "foo") + properties.Add("v2", "bar") + + err := properties.MarshalXML(e, xml.StartElement{ + Name: xml.Name{Local: "root"}, + }) + + assert.NoError(t, err) + + err = e.Flush() + + assert.NoError(t, err) + + result := buf.String() + assert.True(t, strings.HasPrefix(result, "")) + assert.True(t, strings.HasSuffix(result, "")) + assert.Contains(t, result, "foo") + assert.Contains(t, result, "bar") +} + +func TestMarshalNestedProperties(t *testing.T) { + buf := new(bytes.Buffer) + e := xml.NewEncoder(buf) + + properties := Properties{} + properties.Add("v1", "foo") + properties.AddProperties("props", map[string]string{ + "prop1": "foo", + "prop2": "baz", + }) + properties.Add("v2", "bar") + + err := properties.MarshalXML(e, xml.StartElement{ + Name: xml.Name{Local: "root"}, + }) + + assert.NoError(t, err) + + err = e.Flush() + + assert.NoError(t, err) + + result := buf.String() + assert.True(t, strings.HasPrefix(result, "")) + assert.True(t, strings.HasSuffix(result, "")) + assert.Contains(t, result, "foo") + assert.Contains(t, result, "") + assert.Contains(t, result, "foo") + assert.Contains(t, result, "baz") + assert.Contains(t, result, "bar") +} diff --git a/pkg/builder/project_test.go b/pkg/builder/project_test.go index 15bd9f2126..18701196be 100644 --- a/pkg/builder/project_test.go +++ b/pkg/builder/project_test.go @@ -452,8 +452,8 @@ func TestInjectServersIntoDefaultMavenSettings(t *testing.T) { ID: "image-repository", Username: "jpoth", Password: "changeit", - Configuration: map[string]string{ - "allowInsecureRegistries": "false", + Configuration: v1.Properties{ + "allowInsecureRegistries": v1.StringOrProperties{Value: "false"}, }, }, } @@ -478,8 +478,8 @@ func TestInjectServersIntoCustomMavenSettings(t *testing.T) { ID: "image-repository", Username: "jpoth", Password: "changeit", - Configuration: map[string]string{ - "allowInsecureRegistries": "false", + Configuration: v1.Properties{ + "allowInsecureRegistries": v1.StringOrProperties{Value: "false"}, }, }, } diff --git a/pkg/builder/quarkus.go b/pkg/builder/quarkus.go index 2061a160fc..7b52e49c49 100644 --- a/pkg/builder/quarkus.go +++ b/pkg/builder/quarkus.go @@ -79,12 +79,9 @@ func loadCamelQuarkusCatalog(ctx *builderContext) error { func generateQuarkusProject(ctx *builderContext) error { p := GenerateQuarkusProjectCommon( - ctx.Build.Runtime.Metadata["camel-quarkus.version"], ctx.Build.Runtime.Version, - ctx.Build.Runtime.Metadata["quarkus.version"]) - - // Add all the properties from the build configuration - p.Properties.AddAll(ctx.Build.Maven.Properties) + ctx.Build.Runtime.Metadata["quarkus.version"], + ctx.Build.Maven.Properties) // Add Maven build extensions p.Build.Extensions = ctx.Build.Maven.Extension @@ -97,23 +94,14 @@ func generateQuarkusProject(ctx *builderContext) error { return nil } -func GenerateQuarkusProjectCommon(camelQuarkusVersion string, runtimeVersion string, quarkusVersion string) maven.Project { +func GenerateQuarkusProjectCommon(runtimeVersion string, quarkusVersion string, buildTimeProperties map[string]string) maven.Project { p := maven.NewProjectWithGAV("org.apache.camel.k.integration", "camel-k-integration", defaults.Version) p.DependencyManagement = &maven.DependencyManagement{Dependencies: make([]maven.Dependency, 0)} p.Dependencies = make([]maven.Dependency, 0) p.Build = &maven.Build{Plugins: make([]maven.Plugin, 0)} - // camel-quarkus does route discovery at startup, but we don't want - // this to happen as routes are loaded at runtime and looking for - // routes at build time may try to load camel-k-runtime routes builder - // proxies which in some case may fail. - p.Properties["quarkus.camel.routes-discovery.enabled"] = "false" - - // disable quarkus banner - p.Properties["quarkus.banner.enabled"] = "false" - // set fast-jar packaging by default, since it gives some startup time improvements - p.Properties["quarkus.package.type"] = "fast-jar" + p.Properties.Add("quarkus.package.type", "fast-jar") // DependencyManagement p.DependencyManagement.Dependencies = append(p.DependencyManagement.Dependencies, @@ -126,6 +114,34 @@ func GenerateQuarkusProjectCommon(camelQuarkusVersion string, runtimeVersion str }, ) + // Add all the properties from the build configuration + p.Properties.AddAll(buildTimeProperties) + + // Quarkus build time properties + buildProperties := make(map[string]string) + + // disable quarkus banner + buildProperties["quarkus.banner.enabled"] = "false" + + // camel-quarkus does route discovery at startup, but we don't want + // this to happen as routes are loaded at runtime and looking for + // routes at build time may try to load camel-k-runtime routes builder + // proxies which in some case may fail. + buildProperties["quarkus.camel.routes-discovery.enabled"] = "false" + + // required for Kamelets utils to resolve data type converters at runtime + buildProperties["quarkus.camel.service.discovery.include-patterns"] = "META-INF/services/org/apache/camel/datatype/converter/*" + + // copy all user defined quarkus.camel build time properties to the quarkus-maven-plugin build properties + for key, value := range buildTimeProperties { + if strings.HasPrefix(key, "quarkus.camel.") { + buildProperties[key] = value + } + } + + configuration := v1.Properties{} + configuration.AddProperties("properties", buildProperties) + // Plugins p.Build.Plugins = append(p.Build.Plugins, maven.Plugin{ @@ -134,9 +150,11 @@ func GenerateQuarkusProjectCommon(camelQuarkusVersion string, runtimeVersion str Version: quarkusVersion, Executions: []maven.Execution{ { + ID: "build-integration", Goals: []string{ "build", }, + Configuration: configuration, }, }, }, diff --git a/pkg/cmd/local/local.go b/pkg/cmd/local/local.go index 353e7a1f97..2cf2d3ca1c 100644 --- a/pkg/cmd/local/local.go +++ b/pkg/cmd/local/local.go @@ -118,9 +118,9 @@ func getTopLevelDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, func getTransitiveDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, dependencies []string, repositories []string) ([]string, error) { project := builder.GenerateQuarkusProjectCommon( - catalog.GetCamelQuarkusVersion(), defaults.DefaultRuntimeVersion, catalog.GetQuarkusVersion(), + make(map[string]string), ) if err := camel.ManageIntegrationDependencies(&project, dependencies, catalog); err != nil { diff --git a/pkg/util/camel/camel_dependencies.go b/pkg/util/camel/camel_dependencies.go index abe230f081..6bd5f27de8 100644 --- a/pkg/util/camel/camel_dependencies.go +++ b/pkg/util/camel/camel_dependencies.go @@ -214,20 +214,22 @@ func addRegistryMavenDependency(project *maven.Project, dependency string) error if isClasspath { outputDirectory = "src/main/resources" } + + properties := v1.Properties{} + properties.Add("outputDirectory", path.Join(outputDirectory, filepath.Dir(outputFileRelativePath))) + properties.Add("outputFileName", filepath.Base(outputFileRelativePath)) + properties.Add("groupId", gav.GroupID) + properties.Add("artifactId", gav.ArtifactID) + properties.Add("version", gav.Version) + properties.Add("type", gav.Type) + exec := maven.Execution{ ID: fmt.Sprint(len(plugin.Executions)), Phase: "validate", Goals: []string{ "artifact", }, - Configuration: map[string]string{ - "outputDirectory": path.Join(outputDirectory, filepath.Dir(outputFileRelativePath)), - "outputFileName": filepath.Base(outputFileRelativePath), - "groupId": gav.GroupID, - "artifactId": gav.ArtifactID, - "version": gav.Version, - "type": gav.Type, - }, + Configuration: properties, } plugin.Executions = append(plugin.Executions, exec) diff --git a/pkg/util/maven/maven_project.go b/pkg/util/maven/maven_project.go index f73eb0d017..ae8bdc05d0 100644 --- a/pkg/util/maven/maven_project.go +++ b/pkg/util/maven/maven_project.go @@ -40,8 +40,8 @@ func NewProjectWithGAV(group string, artifact string, version string) Project { p.GroupID = group p.ArtifactID = artifact p.Version = version - p.Properties = make(map[string]string) - p.Properties["project.build.sourceEncoding"] = "UTF-8" + p.Properties = v1.Properties{} + p.Properties.Add("project.build.sourceEncoding", "UTF-8") return p } @@ -167,7 +167,7 @@ func NewDependency(groupID string, artifactID string, version string) Dependency // The repository can be customized by appending @param to the repository // URL, e.g.: // -// http://my-nexus:8081/repository/publicc@id=my-repo@snapshots +// http://my-nexus:8081/repository/publicc@id=my-repo@snapshots // // That enables snapshots and sets the repository id to `my-repo`. func NewRepository(repo string) v1.Repository {