From 10df4d96f5588d12a2e24d46bdb291087fb2ba91 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Thu, 24 Sep 2020 22:37:23 +0530 Subject: [PATCH] Fix #382: Add support for merging fragment Route spec with default generated Route --- CHANGELOG.md | 1 + .../api/util/KubernetesResourceUtil.java | 51 ++++-- .../generic/DefaultControllerEnricher.java | 18 --- .../generic/openshift/RouteEnricher.java | 143 +++++++++++------ .../generic/openshift/RouteEnricherTest.java | 147 +++++++++++++++--- 5 files changed, 263 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f18d88c384..07d74c6c23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Usage: ``` ### 1.0.1-SNAPSHOT * Fix #381: Remove root as default user in AssemblyConfigurationUtils#getAssemblyConfigurationOrCreateDefault +* Fix #382: Add support for merging fragment Route spec with default generated Route * Fix #358: Prometheus is enabled by default, opt-out via AB_PROMETHEUS_OFF required to disable (like in FMP) * Fix #384: Enricher defined Container environment variables get merged with vars defined in Image Build Configuration diff --git a/jkube-kit/enricher/api/src/main/java/org/eclipse/jkube/kit/enricher/api/util/KubernetesResourceUtil.java b/jkube-kit/enricher/api/src/main/java/org/eclipse/jkube/kit/enricher/api/util/KubernetesResourceUtil.java index 7cd411053d..c6a4dcff7c 100644 --- a/jkube-kit/enricher/api/src/main/java/org/eclipse/jkube/kit/enricher/api/util/KubernetesResourceUtil.java +++ b/jkube-kit/enricher/api/src/main/java/org/eclipse/jkube/kit/enricher/api/util/KubernetesResourceUtil.java @@ -114,8 +114,6 @@ private KubernetesResourceUtil() { } .withCronJobVersion(CRONJOB_VERSION) .withRbacVersioning(RBAC_VERSION); - public static final Set> SIMPLE_FIELD_TYPES = new HashSet<>(); - public static final String CONTAINER_NAME_REGEX = "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"; protected static final String DATE_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ssX"; @@ -566,7 +564,7 @@ public static void mergeSimpleFields(Object targetValues, Object defaultValues) } Class fieldType = targetGetMethod.getReturnType(); - if (!SIMPLE_FIELD_TYPES.contains(fieldType)) { + if (!isSimpleFieldType(fieldType)) { continue; } @@ -727,6 +725,27 @@ private static void ensureHasPort(Container container, ContainerPort port) { ports.add(port); } + private static boolean isSimpleFieldType(Class type) { + ArrayList> simpleFieldTypes = new ArrayList<>(); + simpleFieldTypes.add(String.class); + simpleFieldTypes.add(Double.class); + simpleFieldTypes.add(Float.class); + simpleFieldTypes.add(Long.class); + simpleFieldTypes.add(Integer.class); + simpleFieldTypes.add(Short.class); + simpleFieldTypes.add(Character.class); + simpleFieldTypes.add(Byte.class); + simpleFieldTypes.add(double.class); + simpleFieldTypes.add(float.class); + simpleFieldTypes.add(long.class); + simpleFieldTypes.add(int.class); + simpleFieldTypes.add(short.class); + simpleFieldTypes.add(char.class); + simpleFieldTypes.add(byte.class); + + return simpleFieldTypes.contains(type); + } + public static String getSourceUrlAnnotation(HasMetadata item) { return KubernetesHelper.getOrCreateAnnotations(item).get(Constants.RESOURCE_SOURCE_URL_ANNOTATION); } @@ -965,7 +984,7 @@ private static void mergeMetadata(PodTemplateSpec item1, PodTemplateSpec item2) } } - protected static void mergeMetadata(HasMetadata item1, HasMetadata item2) { + public static void mergeMetadata(HasMetadata item1, HasMetadata item2) { if (item1 != null && item2 != null) { ObjectMeta metadata1 = item1.getMetadata(); ObjectMeta metadata2 = item2.getMetadata(); @@ -983,16 +1002,24 @@ protected static void mergeMetadata(HasMetadata item1, HasMetadata item2) { * when overriding */ private static Map mergeMapsAndRemoveEmptyStrings(Map overrideMap, Map originalMap) { - Map answer = MapUtil.mergeMaps(overrideMap, originalMap); - Set> entries = overrideMap.entrySet(); - for (Map.Entry entry : entries) { - String value = entry.getValue(); - if (value == null || value.isEmpty()) { - String key = entry.getKey(); - answer.remove(key); + if (overrideMap == null && originalMap == null) { + return Collections.emptyMap(); + } else if (originalMap == null) { + return overrideMap; + } else if (overrideMap == null) { + return originalMap; + } else { + Map answer = MapUtil.mergeMaps(overrideMap, originalMap); + Set> entries = overrideMap.entrySet(); + for (Map.Entry entry : entries) { + String value = entry.getValue(); + if (value == null || value.isEmpty()) { + String key = entry.getKey(); + answer.remove(key); + } } + return answer; } - return answer; } // lets use presence of an image name as a clue that we are just enriching things a little diff --git a/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/DefaultControllerEnricher.java b/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/DefaultControllerEnricher.java index d1e65d22d1..3a2da234d9 100644 --- a/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/DefaultControllerEnricher.java +++ b/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/DefaultControllerEnricher.java @@ -166,22 +166,4 @@ private List getContainersFromPodSpec(PodTemplateSpec spec) { return containerNames; } - static { - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(String.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(Double.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(Float.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(Long.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(Integer.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(Short.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(Character.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(Byte.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(double.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(float.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(long.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(int.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(short.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(char.class); - KubernetesResourceUtil.SIMPLE_FIELD_TYPES.add(byte.class); - } - } diff --git a/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricher.java b/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricher.java index d815ca58ee..610d0b1758 100644 --- a/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricher.java +++ b/jkube-kit/enricher/generic/src/main/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricher.java @@ -13,12 +13,12 @@ */ package org.eclipse.jkube.enricher.generic.openshift; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.openshift.api.model.RouteSpec; import org.eclipse.jkube.kit.common.Configs; import org.eclipse.jkube.kit.common.util.FileUtil; import org.eclipse.jkube.kit.config.resource.JKubeAnnotations; @@ -40,6 +40,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import org.apache.commons.lang3.StringUtils; +import org.eclipse.jkube.kit.enricher.api.util.KubernetesResourceUtil; import static org.eclipse.jkube.kit.enricher.api.util.KubernetesResourceUtil.containsLabelInMetadata; import static org.eclipse.jkube.kit.enricher.api.util.KubernetesResourceUtil.removeLabel; @@ -81,25 +82,18 @@ public void create(PlatformMode platformMode, final KubernetesListBuilder listBu } if(platformMode == PlatformMode.openshift && isGenerateRoute()) { - final List routes = new ArrayList<>(); listBuilder.accept(new TypedVisitor() { @Override public void visit(ServiceBuilder serviceBuilder) { - addRoute(listBuilder, serviceBuilder, routes); + addRoute(listBuilder, serviceBuilder); } }); - - if (!routes.isEmpty()) { - Route[] routeArray = new Route[routes.size()]; - routes.toArray(routeArray); - listBuilder.addToItems(routeArray); - } } } - private RoutePort createRoutePort(ServiceBuilder serviceBuilder) { + private static RoutePort createRoutePort(ServiceBuilder serviceBuilder) { RoutePort routePort = null; ServiceSpec spec = serviceBuilder.buildSpec(); if (spec != null) { @@ -157,58 +151,111 @@ private boolean hasExactlyOneServicePort(ServiceBuilder service, String id) { } } - private void addRoute(KubernetesListBuilder listBuilder, ServiceBuilder serviceBuilder, List routes) { + private void addRoute(KubernetesListBuilder listBuilder, ServiceBuilder serviceBuilder) { ObjectMeta serviceMetadata = serviceBuilder.buildMetadata(); if (serviceMetadata != null && StringUtils.isNotBlank(serviceMetadata.getName()) && hasExactlyOneServicePort(serviceBuilder, serviceMetadata.getName()) && isExposedService(serviceMetadata)) { String name = serviceMetadata.getName(); - if (!hasRoute(listBuilder, name)) { - if (StringUtils.isNotBlank(routeDomainPostfix)) { - routeDomainPostfix = prepareHostForRoute(routeDomainPostfix, name); - } else { - routeDomainPostfix = ""; + updateRouteDomainPostFixBasedOnServiceName(name); + Route opinionatedRoute = createOpinionatedRouteFromService(serviceBuilder, routeDomainPostfix); + if (opinionatedRoute != null) { + int routeFromFragmentIndex = getRouteIndexWithName(listBuilder, name); + if (routeFromFragmentIndex > 0) { // Merge fragment with Opinionated Route + Route routeFragment = (Route) listBuilder.buildItems().get(routeFromFragmentIndex); + Route mergedRoute = mergeRoute(routeFragment, opinionatedRoute); + KubernetesResourceUtil.removeItemFromKubernetesBuilder(listBuilder, listBuilder.getItems().get(routeFromFragmentIndex)); + listBuilder.addToItems(mergedRoute); + } else { // No fragment provided. Use Opinionated Route. + listBuilder.addToItems(opinionatedRoute); } + } + } + } - RoutePort routePort = createRoutePort(serviceBuilder); - if (routePort != null) { - RouteBuilder routeBuilder = new RouteBuilder(). - withMetadata(serviceMetadata). - withNewSpec(). - withPort(routePort). - withNewTo().withKind("Service").withName(name).endTo(). - withHost(routeDomainPostfix.isEmpty() ? null : routeDomainPostfix). - endSpec(); - - // removing `expose : true` label from metadata. - removeLabel(routeBuilder.buildMetadata(), EXPOSE_LABEL, "true"); - removeLabel(routeBuilder.buildMetadata(), JKubeAnnotations.SERVICE_EXPOSE_URL.value(), "true"); - routeBuilder.withNewMetadataLike(routeBuilder.buildMetadata()); - routes.add(routeBuilder.build()); - } + private void updateRouteDomainPostFixBasedOnServiceName(String serviceName) { + if (StringUtils.isNotBlank(routeDomainPostfix)) { + routeDomainPostfix = prepareHostForRoute(routeDomainPostfix, serviceName); + } else { + routeDomainPostfix = ""; + } + } + + static Route mergeRoute(Route routeFromFragment, Route opinionatedRoute) { + // Update ApiVersion to route.openshift.io/v1 + if (routeFromFragment.getApiVersion().equals("v1")) { + routeFromFragment.setApiVersion(opinionatedRoute.getApiVersion()); + } + + // Merge metadata + KubernetesResourceUtil.mergeMetadata(routeFromFragment, opinionatedRoute); + + // Merge spec + if (routeFromFragment.getSpec() != null) { + routeFromFragment.setSpec(mergeRouteSpec(routeFromFragment.getSpec(), opinionatedRoute.getSpec())); + } else { + routeFromFragment.setSpec(opinionatedRoute.getSpec()); + } + return routeFromFragment; + } + + static RouteSpec mergeRouteSpec(RouteSpec fragmentSpec, RouteSpec opinionatedSpec) { + KubernetesResourceUtil.mergeSimpleFields(fragmentSpec, opinionatedSpec); + if (fragmentSpec.getAlternateBackends() == null && opinionatedSpec.getAlternateBackends() != null) { + fragmentSpec.setAlternateBackends(opinionatedSpec.getAlternateBackends()); + } + if (fragmentSpec.getPort() == null && opinionatedSpec.getPort() != null) { + fragmentSpec.setPort(opinionatedSpec.getPort()); + } + if (fragmentSpec.getTls() == null && opinionatedSpec.getTls() != null) { + fragmentSpec.setTls(opinionatedSpec.getTls()); + } + if (fragmentSpec.getTo() == null && opinionatedSpec.getTo() != null) { + fragmentSpec.setTo(opinionatedSpec.getTo()); + } + + return fragmentSpec; + } + + static int getRouteIndexWithName(final KubernetesListBuilder listBuilder, final String name) { + int routeInListIndex = -1; + for (int index = 0; index < listBuilder.buildItems().size(); index++) { + HasMetadata item = listBuilder.getItems().get(index); + if (item != null && + item.getMetadata() != null && + item.getMetadata().getName().equals(name) && + item instanceof Route) { + routeInListIndex = index; } } + return routeInListIndex; } - /** - * Returns true if we already have a route created for the given name - */ - private boolean hasRoute(final KubernetesListBuilder listBuilder, final String name) { - final AtomicBoolean answer = new AtomicBoolean(false); - listBuilder.accept(new TypedVisitor() { + static Route createOpinionatedRouteFromService(ServiceBuilder serviceBuilder, String routeDomainPostfix) { + ObjectMeta serviceMetadata = serviceBuilder.buildMetadata(); + if (serviceMetadata != null) { + String name = serviceMetadata.getName(); + RoutePort routePort = createRoutePort(serviceBuilder); + if (routePort != null) { + RouteBuilder routeBuilder = new RouteBuilder(). + withMetadata(serviceMetadata). + withNewSpec(). + withPort(routePort). + withNewTo().withKind("Service").withName(name).endTo(). + withHost(routeDomainPostfix.isEmpty() ? null : routeDomainPostfix). + endSpec(); - @Override - public void visit(RouteBuilder builder) { - ObjectMeta metadata = builder.buildMetadata(); - if (metadata != null && name.equals(metadata.getName())) { - answer.set(true); - } + // removing `expose : true` label from metadata. + removeLabel(routeBuilder.buildMetadata(), EXPOSE_LABEL, "true"); + removeLabel(routeBuilder.buildMetadata(), JKubeAnnotations.SERVICE_EXPOSE_URL.value(), "true"); + routeBuilder.withNewMetadataLike(routeBuilder.buildMetadata()); + return routeBuilder.build(); } - }); - return answer.get(); + } + return null; } - private static boolean isExposedService(ObjectMeta objectMeta) { + static boolean isExposedService(ObjectMeta objectMeta) { return containsLabelInMetadata(objectMeta, EXPOSE_LABEL, "true") || containsLabelInMetadata(objectMeta, JKubeAnnotations.SERVICE_EXPOSE_URL.value(), "true"); } diff --git a/jkube-kit/enricher/generic/src/test/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricherTest.java b/jkube-kit/enricher/generic/src/test/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricherTest.java index 738c00a2da..9eb49a198d 100644 --- a/jkube-kit/enricher/generic/src/test/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricherTest.java +++ b/jkube-kit/enricher/generic/src/test/java/org/eclipse/jkube/enricher/generic/openshift/RouteEnricherTest.java @@ -15,6 +15,8 @@ import java.util.Properties; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.openshift.api.model.Route; import io.fabric8.openshift.api.model.RouteBuilder; import org.eclipse.jkube.kit.config.resource.PlatformMode; import org.eclipse.jkube.kit.config.resource.ProcessorConfig; @@ -25,10 +27,13 @@ import io.fabric8.kubernetes.api.model.ServiceBuilder; import mockit.Expectations; import mockit.Mocked; -import org.junit.Before; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; @SuppressWarnings({"unused", "ResultOfMethodCallIgnored"}) public class RouteEnricherTest { @@ -40,27 +45,12 @@ public class RouteEnricherTest { private ProcessorConfig processorConfig; private KubernetesListBuilder klb; - @Before - public void setUp() { + public void setUpExpectations() { properties = new Properties(); processorConfig = new ProcessorConfig(); klb = new KubernetesListBuilder(); // @formatter:off - klb.addToItems(new ServiceBuilder() - .editOrNewMetadata() - .withName("test-svc") - .addToLabels("expose", "true") - .endMetadata() - .editOrNewSpec() - .addNewPort() - .withName("http") - .withPort(8080) - .withProtocol("TCP") - .withTargetPort(new IntOrString(8080)) - .endPort() - .addToSelector("group", "test") - .withType("LoadBalancer") - .endSpec() + klb.addToItems(getMockServiceBuilder() .build()); new Expectations() {{ context.getProperties(); result = properties; @@ -71,6 +61,8 @@ public void setUp() { @Test public void testCreateWithDefaultsInOpenShiftShouldAddRoute() { + // Given + setUpExpectations(); // When new RouteEnricher(context).create(PlatformMode.openshift, klb); // Then @@ -86,6 +78,7 @@ public void testCreateWithDefaultsInOpenShiftShouldAddRoute() { @Test public void testCreateWithDefaultsInKubernetesShouldNotAddRoute() { // Given + setUpExpectations(); // @formatter:off new Expectations() {{ context.getProperties(); minTimes = 0; @@ -104,6 +97,7 @@ public void testCreateWithDefaultsInKubernetesShouldNotAddRoute() { @Test public void testCreateWithGenerateExtraPropertyInOpenShiftShouldNotAddRoute() { // Given + setUpExpectations(); properties.put("jkube.openshift.generateRoute", "false"); // When new RouteEnricher(context).create(PlatformMode.openshift, klb); @@ -117,6 +111,7 @@ public void testCreateWithGenerateExtraPropertyInOpenShiftShouldNotAddRoute() { @Test public void testCreateWithGenerateEnricherPropertyInOpenShiftShouldNotAddRoute() { // Given + setUpExpectations(); properties.put("jkube.enricher.jkube-openshift-route.generateRoute", "false"); // When new RouteEnricher(context).create(PlatformMode.openshift, klb); @@ -130,6 +125,7 @@ public void testCreateWithGenerateEnricherPropertyInOpenShiftShouldNotAddRoute() @Test public void testCreateWithDefaultsAndRouteDomainInOpenShiftShouldAddRouteWithDomainPostfix() { // Given + setUpExpectations(); // @formatter:off new Expectations() {{ context.getProperties(); minTimes = 0; @@ -151,6 +147,7 @@ public void testCreateWithDefaultsAndRouteDomainInOpenShiftShouldAddRouteWithDom @Test public void testCreateWithDefaultsAndExistingRouteWithMatchingNameInBuilderInOpenShiftShouldReuseExistingRoute() { // Given + setUpExpectations(); klb.addToItems(new RouteBuilder() .editOrNewMetadata() .withName("test-svc") @@ -171,6 +168,118 @@ public void testCreateWithDefaultsAndExistingRouteWithMatchingNameInBuilderInOpe .containsExactly("Service", "Route"); assertThat(klb.build().getItems().get(1)) .extracting("metadata.name", "spec.host", "spec.to.kind", "spec.to.name", "spec.port.targetPort.intVal") - .contains("test-svc", "example.com", null, null, 1337); + .contains("test-svc", "example.com", "Service", "test-svc", 1337); + } + + @Test + public void testCreateOpinionatedRouteFromService() { + // Given + ServiceBuilder serviceBuilder = getMockServiceBuilder(); + + // When + Route route = RouteEnricher.createOpinionatedRouteFromService(serviceBuilder, "example.com"); + + // Then + assertNotNull(route); + assertThat(route) + .extracting("metadata.name", "spec.host", "spec.to.kind", "spec.to.name", "spec.port.targetPort.intVal") + .contains("test-svc", "example.com", "Service", "test-svc", 8080); + } + + @Test + public void testCreateOpinionatedRouteFromServiceWithNullService() { + // Given + ServiceBuilder serviceBuilder = new ServiceBuilder(); + + // When + Route route = RouteEnricher.createOpinionatedRouteFromService(serviceBuilder, "example.com"); + + // Then + assertNull(route); + } + + @Test + public void testIsExposedService() { + assertTrue(RouteEnricher.isExposedService(new ObjectMetaBuilder().addToLabels("expose", "true").build())); + assertTrue(RouteEnricher.isExposedService(new ObjectMetaBuilder().addToLabels("jkube.io/exposeUrl", "true").build())); + } + + @Test + public void testMergeRouteWithEmptyFragment() { + // Given + Route opinionatedRoute = getMockOpinionatedRoute(); + Route fragmentRoute = new RouteBuilder().build(); + + // When + Route result = RouteEnricher.mergeRoute(fragmentRoute, opinionatedRoute); + + // Then + assertNotNull(result); + assertEquals(opinionatedRoute, result); + } + + @Test + public void testMergeRouteWithNonEmptyFragment() { + // Given + Route opinionatedRoute = getMockOpinionatedRoute(); + Route fragmentRoute = new RouteBuilder() + .withNewSpec() + .withNewTls() + .withInsecureEdgeTerminationPolicy("Redirect") + .withTermination("edge") + .endTls() + .endSpec() + .build(); + + // When + Route result = RouteEnricher.mergeRoute(fragmentRoute, opinionatedRoute); + + // Then + assertNotNull(result); + assertThat(result) + .extracting("metadata.name", "spec.host", "spec.to.kind", "spec.to.name", + "spec.port.targetPort.intVal", "spec.tls.insecureEdgeTerminationPolicy", "spec.tls.termination") + .contains("test-svc", "example.com", "Service", "test-svc", + 8080, "Redirect", "edge"); + } + + private Route getMockOpinionatedRoute() { + return new RouteBuilder() + .withNewMetadata().withName("test-svc").endMetadata() + .withNewSpec() + .withNewPort() + .withNewTargetPort().withIntVal(8080).endTargetPort() + .endPort() + .withHost("example.com") + .withNewTo().withKind("Service").withName("test-svc").endTo() + .withNewTls() + .withInsecureEdgeTerminationPolicy("Redirect") + .withTermination("edge") + .endTls() + .addNewAlternateBackend() + .withKind("Service") + .withName("test-svc-2") + .withWeight(10) + .endAlternateBackend() + .endSpec() + .build(); + } + + private ServiceBuilder getMockServiceBuilder() { + return new ServiceBuilder() + .editOrNewMetadata() + .withName("test-svc") + .addToLabels("expose", "true") + .endMetadata() + .editOrNewSpec() + .addNewPort() + .withName("http") + .withPort(8080) + .withProtocol("TCP") + .withTargetPort(new IntOrString(8080)) + .endPort() + .addToSelector("group", "test") + .withType("LoadBalancer") + .endSpec(); } }