From dbde53326d51e51361d154d674b7d8d47c4c0c16 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 5 Oct 2016 16:47:14 -0400 Subject: [PATCH 01/33] MINIFI-115 Bump NiFi versions to 1.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f44d6aaf7..2960e5b34 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ limitations under the License. UTF-8 2016 1.7.12 - 0.6.1 + 1.0.0 9.2.11.v20150529 From 77e55c32ebf188294d50b4e273d9728ecf89595d Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 5 Oct 2016 16:51:17 -0400 Subject: [PATCH 02/33] MINIFI-115 Remove provenance reporting bundle to use the NiFi equivalent. --- .../minifi-provenance-reporting-nar/pom.xml | 40 -- .../src/main/resources/META-INF/NOTICE | 15 - .../minifi-provenance-reporting-task/pom.xml | 82 ---- .../reporting/ProvenanceReportingTask.java | 462 ------------------ .../org.apache.nifi.reporting.ReportingTask | 16 - .../TestProvenanceReportingTask.java | 186 ------- .../pom.xml | 46 -- 7 files changed, 847 deletions(-) delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml deleted file mode 100644 index 88b5b9002..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - 4.0.0 - - org.apache.nifi.minifi - minifi-provenance-reporting-bundle - 0.1.0-SNAPSHOT - - - minifi-provenance-reporting-nar - nar - - true - true - - - - org.apache.nifi.minifi - minifi-provenance-reporting-task - - - org.apache.nifi - nifi-standard-services-api-nar - nar - - - diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index be55e5968..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,15 +0,0 @@ -minifi-provenance-reporting-task-nar -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -************************ -Common Development and Distribution License 1.1 -************************ - -The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details. - - (CDDL 1.1) (GPL2 w/ CPE) JSON Processing API (javax.json:javax.json-api:jar:1.0 - http://json-processing-spec.java.net) - (CDDL 1.1) (GPL2 w/ CPE) JSON Processing Default Provider (org.glassfish:javax.json:jar:1.0.4 - https://jsonp.java.net) - diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml deleted file mode 100644 index 837b94f46..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - 4.0.0 - - org.apache.nifi.minifi - minifi-provenance-reporting-bundle - 0.1.0-SNAPSHOT - - - minifi-provenance-reporting-task - Publishes MiNiFi metrics to NiFi via S2S - 0.1.0-SNAPSHOT - - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-processor-utils - - - org.apache.nifi - nifi-utils - - - org.apache.nifi - nifi-ssl-context-service-api - - - org.apache.nifi - nifi-site-to-site-client - - - org.glassfish - javax.json - 1.0.4 - - - javax.json - javax.json-api - 1.0 - - - - org.mockito - mockito-all - test - - - org.apache.nifi - nifi-data-provenance-utils - test - - - org.apache.nifi - nifi-mock - test - - - junit - junit - 4.12 - test - - - diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java deleted file mode 100644 index a0d1776ff..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.provenance.reporting; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonArrayBuilder; -import javax.json.JsonBuilderFactory; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import javax.net.ssl.SSLContext; - -import org.apache.nifi.annotation.behavior.Stateful; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.annotation.lifecycle.OnStopped; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.components.Validator; -import org.apache.nifi.components.state.Scope; -import org.apache.nifi.components.state.StateManager; -import org.apache.nifi.components.state.StateMap; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.status.PortStatus; -import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.controller.status.ProcessorStatus; -import org.apache.nifi.controller.status.RemoteProcessGroupStatus; -import org.apache.nifi.events.EventReporter; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.provenance.ProvenanceEventRecord; -import org.apache.nifi.remote.Transaction; -import org.apache.nifi.remote.TransferDirection; -import org.apache.nifi.remote.client.SiteToSiteClient; -import org.apache.nifi.reporting.AbstractReportingTask; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.Severity; -import org.apache.nifi.ssl.SSLContextService; -import org.apache.nifi.ssl.SSLContextService.ClientAuth; - -@Tags({"provenance", "lineage", "tracking", "site", "site to site"}) -@CapabilityDescription("Publishes Provenance events using the Site To Site protocol.") -@Stateful(scopes = Scope.LOCAL, description = "Stores the Reporting Task's last event Id so that on restart of MiNiFi the task knows where it left off.") -public class ProvenanceReportingTask extends AbstractReportingTask { - private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; - private static final String LAST_EVENT_ID_KEY = "last_event_id"; - - static final PropertyDescriptor DESTINATION_URL = new PropertyDescriptor.Builder() - .name("Destination URL") - .description("The URL to post the Provenance Events to.") - .required(true) - .expressionLanguageSupported(true) - .addValidator(StandardValidators.URL_VALIDATOR) - .build(); - static final PropertyDescriptor PORT_NAME = new PropertyDescriptor.Builder() - .name("Input Port Name") - .description("The name of the Input Port to delivery Provenance Events to.") - .required(true) - .expressionLanguageSupported(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - static final PropertyDescriptor SSL_CONTEXT = new PropertyDescriptor.Builder() - .name("SSL Context Service") - .description("The SSL Context Service to use when communicating with the destination. If not specified, communications will not be secure.") - .required(false) - .identifiesControllerService(SSLContextService.class) - .build(); - static final PropertyDescriptor MINIFI_URL = new PropertyDescriptor.Builder() - .name("MiNiFi URL") - .description("The URL of this MiNiFi instance. This is used to include the Content URI to send to the destination.") - .required(true) - .expressionLanguageSupported(true) - .defaultValue("http://${hostname(true)}:8080/nifi") - .addValidator(new NiFiUrlValidator()) - .build(); - static final PropertyDescriptor COMPRESS = new PropertyDescriptor.Builder() - .name("Compress Events") - .description("Indicates whether or not to compress the events when being sent.") - .required(true) - .allowableValues("true", "false") - .defaultValue("true") - .build(); - static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() - .name("Communications Timeout") - .description("Specifies how long to wait to a response from the destination before deciding that an error has occurred and canceling the transaction") - .required(true) - .defaultValue("30 secs") - .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) - .build(); - static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() - .name("Batch Size") - .description("Specifies how many records to send in a single batch, at most.") - .required(true) - .defaultValue("1000") - .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) - .build(); - - private volatile long firstEventId = -1L; - private volatile SiteToSiteClient siteToSiteClient; - - @Override - protected List getSupportedPropertyDescriptors() { - final List properties = new ArrayList<>(); - properties.add(DESTINATION_URL); - properties.add(PORT_NAME); - properties.add(SSL_CONTEXT); - properties.add(MINIFI_URL); - properties.add(COMPRESS); - properties.add(TIMEOUT); - properties.add(BATCH_SIZE); - return properties; - } - - @OnScheduled - public void setup(final ConfigurationContext context) throws IOException { - final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class); - final SSLContext sslContext = sslContextService == null ? null : sslContextService.createSSLContext(ClientAuth.REQUIRED); - final EventReporter eventReporter = new EventReporter() { - @Override - public void reportEvent(final Severity severity, final String category, final String message) { - switch (severity) { - case WARNING: - getLogger().warn(message); - break; - case ERROR: - getLogger().error(message); - break; - default: - break; - } - } - }; - - final String destinationUrlPrefix = context.getProperty(DESTINATION_URL).evaluateAttributeExpressions().getValue(); - final String destinationUrl = destinationUrlPrefix + (destinationUrlPrefix.endsWith("/") ? "nifi" : "/nifi"); - - siteToSiteClient = new SiteToSiteClient.Builder() - .url(destinationUrl) - .portName(context.getProperty(PORT_NAME).getValue()) - .useCompression(context.getProperty(COMPRESS).asBoolean()) - .eventReporter(eventReporter) - .sslContext(sslContext) - .timeout(context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS) - .build(); - } - - @OnStopped - public void shutdown() throws IOException { - final SiteToSiteClient client = getClient(); - if (client != null) { - client.close(); - } - } - - // this getter is intended explicitly for testing purposes - protected SiteToSiteClient getClient() { - return this.siteToSiteClient; - } - - private String getComponentName(final ProcessGroupStatus status, final ProvenanceEventRecord event) { - if (status == null) { - return null; - } - - final String componentId = event.getComponentId(); - if (status.getId().equals(componentId)) { - return status.getName(); - } - - for (final ProcessorStatus procStatus : status.getProcessorStatus()) { - if (procStatus.getId().equals(componentId)) { - return procStatus.getName(); - } - } - - for (final PortStatus portStatus : status.getInputPortStatus()) { - if (portStatus.getId().equals(componentId)) { - return portStatus.getName(); - } - } - - for (final PortStatus portStatus : status.getOutputPortStatus()) { - if (portStatus.getId().equals(componentId)) { - return portStatus.getName(); - } - } - - for (final RemoteProcessGroupStatus rpgStatus : status.getRemoteProcessGroupStatus()) { - if (rpgStatus.getId().equals(componentId)) { - return rpgStatus.getName(); - } - } - - for (final ProcessGroupStatus childGroup : status.getProcessGroupStatus()) { - final String componentName = getComponentName(childGroup, event); - if (componentName != null) { - return componentName; - } - } - - return null; - } - - @Override - public void onTrigger(final ReportingContext context) { - final ProcessGroupStatus procGroupStatus = context.getEventAccess().getControllerStatus(); - final String rootGroupName = procGroupStatus == null ? null : procGroupStatus.getName(); - - Long currMaxId = context.getEventAccess().getProvenanceRepository().getMaxEventId(); - - if(currMaxId == null) { - getLogger().debug("No events to send because no events have been created yet."); - return; - } - - if (firstEventId < 0) { - Map state; - try { - state = context.getStateManager().getState(Scope.LOCAL).toMap(); - } catch (IOException e) { - getLogger().error("Failed to get state at start up due to {}:"+e.getMessage(), e); - return; - } - if (state.containsKey(LAST_EVENT_ID_KEY)) { - firstEventId = Long.parseLong(state.get(LAST_EVENT_ID_KEY)) + 1; - } - - if(currMaxId < firstEventId){ - getLogger().debug("Current provenance max id is {} which is less than what was stored in state as the last queried event, which was {}. This means the provenance restarted its " + - "ids. Restarting querying from the beginning.", new Object[]{currMaxId, firstEventId}); - firstEventId = -1; - } - } - - if (currMaxId == (firstEventId - 1)) { - getLogger().debug("No events to send due to the current max id being equal to the last id that was queried."); - return; - } - - final List events; - try { - events = context.getEventAccess().getProvenanceEvents(firstEventId, context.getProperty(BATCH_SIZE).asInteger()); - } catch (final IOException ioe) { - getLogger().error("Failed to retrieve Provenance Events from repository due to {}:"+ioe.getMessage(), ioe); - return; - } - - if (events == null || events.isEmpty()) { - getLogger().debug("No events to send due to 'events' being null or empty."); - return; - } - - final long start = System.nanoTime(); - final Map config = Collections.emptyMap(); - final JsonBuilderFactory factory = Json.createBuilderFactory(config); - final JsonObjectBuilder builder = factory.createObjectBuilder(); - - final String nifiUrl = context.getProperty(MINIFI_URL).evaluateAttributeExpressions().getValue(); - URL url; - try { - url = new URL(nifiUrl); - } catch (final MalformedURLException e1) { - // already validated - throw new AssertionError(); - } - - final String hostname = url.getHost(); - - final JsonArrayBuilder arrayBuilder = factory.createArrayBuilder(); - for (final ProvenanceEventRecord event : events) { - arrayBuilder.add(serialize(factory, builder, event, getComponentName(procGroupStatus, event), hostname, url, rootGroupName)); - } - final JsonArray jsonArray = arrayBuilder.build(); - - try { - final Transaction transaction = getClient().createTransaction(TransferDirection.SEND); - if (transaction == null) { - getLogger().debug("All destination nodes are penalized; will attempt to send data later"); - return; - } - - final Map attributes = new HashMap<>(); - final String transactionId = UUID.randomUUID().toString(); - attributes.put("reporting.task.transaction.id", transactionId); - - final byte[] data = jsonArray.toString().getBytes(StandardCharsets.UTF_8); - transaction.send(data, attributes); - transaction.confirm(); - transaction.complete(); - - final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); - getLogger().info("Successfully sent {} Provenance Events to destination in {} ms; Transaction ID = {}; First Event ID = {}", - new Object[] {events.size(), transferMillis, transactionId, events.get(0).getEventId()}); - } catch (final IOException e) { - throw new ProcessException("Failed to send Provenance Events to destination due to IOException:" + e.getMessage(), e); - } - - final ProvenanceEventRecord lastEvent = events.get(events.size() - 1); - final String lastEventId = String.valueOf(lastEvent.getEventId()); - try { - StateManager stateManager = context.getStateManager(); - StateMap stateMap = stateManager.getState(Scope.LOCAL); - Map newMapOfState = new HashMap<>(); - newMapOfState.put(LAST_EVENT_ID_KEY, lastEventId); - stateManager.replace(stateMap, newMapOfState, Scope.LOCAL); - } catch (final IOException ioe) { - getLogger().error("Failed to update state to {} due to {}; this could result in events being re-sent after a restart of MiNiFi. The message of {} was: {}", - new Object[] {lastEventId, ioe, ioe, ioe.getMessage()}, ioe); - } - - firstEventId = lastEvent.getEventId() + 1; - } - - static JsonObject serialize(final JsonBuilderFactory factory, final JsonObjectBuilder builder, final ProvenanceEventRecord event, - final String componentName, final String hostname, final URL nifiUrl, final String applicationName) { - addField(builder, "eventId", UUID.randomUUID().toString()); - addField(builder, "eventOrdinal", event.getEventId()); - addField(builder, "eventType", event.getEventType().name()); - addField(builder, "timestampMillis", event.getEventTime()); - - final DateFormat df = new SimpleDateFormat(TIMESTAMP_FORMAT); - df.setTimeZone(TimeZone.getTimeZone("Z")); - addField(builder, "timestamp", df.format(event.getEventTime())); - - addField(builder, "durationMillis", event.getEventDuration()); - addField(builder, "lineageStart", event.getLineageStartDate()); - - final Set lineageIdentifiers = new HashSet<>(); - if (event.getLineageIdentifiers() != null) { - lineageIdentifiers.addAll(event.getLineageIdentifiers()); - } - lineageIdentifiers.add(event.getFlowFileUuid()); - addField(builder, factory, "lineageIdentifiers", lineageIdentifiers); - addField(builder, "details", event.getDetails()); - addField(builder, "componentId", event.getComponentId()); - addField(builder, "componentType", event.getComponentType()); - addField(builder, "componentName", componentName); - addField(builder, "entityId", event.getFlowFileUuid()); - addField(builder, "entityType", "org.apache.nifi.flowfile.FlowFile"); - addField(builder, "entitySize", event.getFileSize()); - addField(builder, "previousEntitySize", event.getPreviousFileSize()); - addField(builder, factory, "updatedAttributes", event.getUpdatedAttributes()); - addField(builder, factory, "previousAttributes", event.getPreviousAttributes()); - - addField(builder, "actorHostname", hostname); - if (nifiUrl != null) { - final String urlPrefix = nifiUrl.toString().replace(nifiUrl.getPath(), ""); - final String contentUriBase = urlPrefix + "/nifi-api/controller/provenance/events/" + event.getEventId() + "/content/"; - addField(builder, "contentURI", contentUriBase + "output"); - addField(builder, "previousContentURI", contentUriBase + "input"); - } - - addField(builder, factory, "parentIds", event.getParentUuids()); - addField(builder, factory, "childIds", event.getChildUuids()); - addField(builder, "transitUri", event.getTransitUri()); - addField(builder, "remoteIdentifier", event.getSourceSystemFlowFileIdentifier()); - addField(builder, "alternateIdentifier", event.getAlternateIdentifierUri()); - addField(builder, "platform", "minifi"); - addField(builder, "application", applicationName); - - return builder.build(); - } - - private static void addField(final JsonObjectBuilder builder, final JsonBuilderFactory factory, final String key, final Map values) { - if (values == null) { - return; - } - - final JsonObjectBuilder mapBuilder = factory.createObjectBuilder(); - for (final Map.Entry entry : values.entrySet()) { - if (entry.getKey() == null || entry.getValue() == null) { - continue; - } - - mapBuilder.add(entry.getKey(), entry.getValue()); - } - - builder.add(key, mapBuilder); - } - - private static void addField(final JsonObjectBuilder builder, final String key, final Long value) { - if (value != null) { - builder.add(key, value.longValue()); - } - } - - private static void addField(final JsonObjectBuilder builder, final JsonBuilderFactory factory, final String key, final Collection values) { - if (values == null) { - return; - } - - builder.add(key, createJsonArray(factory, values)); - } - - private static void addField(final JsonObjectBuilder builder, final String key, final String value) { - if (value == null) { - return; - } - - builder.add(key, value); - } - - private static JsonArrayBuilder createJsonArray(JsonBuilderFactory factory, final Collection values) { - final JsonArrayBuilder builder = factory.createArrayBuilder(); - for (final String value : values) { - if (value != null) { - builder.add(value); - } - } - return builder; - } - - - private static class NiFiUrlValidator implements Validator { - @Override - public ValidationResult validate(final String subject, final String input, final ValidationContext context) { - final String value = context.newPropertyValue(input).evaluateAttributeExpressions().getValue(); - try { - new URL(value); - } catch (final Exception e) { - return new ValidationResult.Builder().input(input).subject(subject).valid(false).explanation("Not a valid URL").build(); - } - - return new ValidationResult.Builder().input(input).subject(subject).valid(true).build(); - } - } -} diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask deleted file mode 100644 index 331d75931..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.nifi.minifi.provenance.reporting.ProvenanceReportingTask \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java deleted file mode 100644 index 97291c018..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.provenance.reporting; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.provenance.ProvenanceEventBuilder; -import org.apache.nifi.provenance.ProvenanceEventRecord; -import org.apache.nifi.provenance.ProvenanceEventRepository; -import org.apache.nifi.provenance.ProvenanceEventType; -import org.apache.nifi.provenance.StandardProvenanceEventRecord; -import org.apache.nifi.remote.Transaction; -import org.apache.nifi.remote.TransferDirection; -import org.apache.nifi.remote.client.SiteToSiteClient; -import org.apache.nifi.reporting.EventAccess; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.ReportingInitializationContext; -import org.apache.nifi.state.MockStateManager; -import org.apache.nifi.stream.io.ByteArrayInputStream; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.MockPropertyValue; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import javax.json.Json; -import javax.json.JsonObject; -import javax.json.JsonReader; - -public class TestProvenanceReportingTask { - - @Test - public void testSerializedForm() throws IOException, InitializationException { - final String uuid = "10000000-0000-0000-0000-000000000000"; - final Map attributes = new HashMap<>(); - attributes.put("abc", "xyz"); - attributes.put("xyz", "abc"); - attributes.put("filename", "file-" + uuid); - - final Map prevAttrs = new HashMap<>(); - attributes.put("filename", "1234.xyz"); - - final Set lineageIdentifiers = new HashSet<>(); - lineageIdentifiers.add("123"); - lineageIdentifiers.add("321"); - - final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); - builder.setEventTime(System.currentTimeMillis()); - builder.setEventType(ProvenanceEventType.RECEIVE); - builder.setTransitUri("nifi://unit-test"); - attributes.put("uuid", uuid); - builder.fromFlowFile(createFlowFile(3L, attributes)); - builder.setAttributes(prevAttrs, attributes); - builder.setComponentId("1234"); - builder.setComponentType("dummy processor"); - builder.setLineageIdentifiers(lineageIdentifiers); - final ProvenanceEventRecord event = builder.build(); - - final List dataSent = new ArrayList<>(); - final ProvenanceReportingTask task = new ProvenanceReportingTask() { - @SuppressWarnings("unchecked") - @Override - protected SiteToSiteClient getClient() { - final SiteToSiteClient client = Mockito.mock(SiteToSiteClient.class); - final Transaction transaction = Mockito.mock(Transaction.class); - - try { - Mockito.doAnswer(new Answer() { - @Override - public Object answer(final InvocationOnMock invocation) throws Throwable { - final byte[] data = invocation.getArgumentAt(0, byte[].class); - dataSent.add(data); - return null; - } - }).when(transaction).send(Mockito.any(byte[].class), Mockito.any(Map.class)); - - Mockito.when(client.createTransaction(Mockito.any(TransferDirection.class))).thenReturn(transaction); - } catch (final Exception e) { - e.printStackTrace(); - Assert.fail(e.toString()); - } - - return client; - } - }; - - final List events = new ArrayList<>(); - events.add(event); - - final Map properties = new HashMap<>(); - for (final PropertyDescriptor descriptor : task.getSupportedPropertyDescriptors()) { - properties.put(descriptor, descriptor.getDefaultValue()); - } - properties.put(ProvenanceReportingTask.BATCH_SIZE, "1000"); - - final ReportingContext context = Mockito.mock(ReportingContext.class); - Mockito.when(context.getStateManager()) - .thenReturn(new MockStateManager(task)); - Mockito.doAnswer(new Answer() { - @Override - public PropertyValue answer(final InvocationOnMock invocation) throws Throwable { - final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class); - return new MockPropertyValue(properties.get(descriptor), null); - } - }).when(context).getProperty(Mockito.any(PropertyDescriptor.class)); - - final EventAccess eventAccess = Mockito.mock(EventAccess.class); - Mockito.doAnswer(new Answer>() { - @Override - public List answer(final InvocationOnMock invocation) throws Throwable { - final long startId = invocation.getArgumentAt(0, long.class); - final int maxRecords = invocation.getArgumentAt(1, int.class); - - final List eventsToReturn = new ArrayList<>(); - for (int i = (int) Math.max(0, startId); i < (int) (startId + maxRecords) && i < events.size(); i++) { - eventsToReturn.add(events.get(i)); - } - return eventsToReturn; - } - }).when(eventAccess).getProvenanceEvents(Mockito.anyLong(), Mockito.anyInt()); - - final ProvenanceEventRepository provenanceRepository = Mockito.mock(ProvenanceEventRepository.class); - Mockito.doAnswer(new Answer() { - @Override - public Long answer(final InvocationOnMock invocation) throws Throwable { - return 1L; - } - }).when(provenanceRepository).getMaxEventId(); - - Mockito.when(context.getEventAccess()).thenReturn(eventAccess); - Mockito.when(eventAccess.getProvenanceRepository()).thenReturn(provenanceRepository); - - final ComponentLog logger = Mockito.mock(ComponentLog.class); - final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class); - Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString()); - Mockito.when(initContext.getLogger()).thenReturn(logger); - - - task.initialize(initContext); - task.onTrigger(context); - - assertEquals(1, dataSent.size()); - final String msg = new String(dataSent.get(0), StandardCharsets.UTF_8); - JsonReader jsonReader = Json.createReader(new ByteArrayInputStream(msg.getBytes())); - JsonObject msgArray = jsonReader.readArray().getJsonObject(0).getJsonObject("updatedAttributes"); - assertEquals(msgArray.getString("abc"), events.get(0).getAttributes().get("abc")); - } - - public static FlowFile createFlowFile(final long id, final Map attributes) { - MockFlowFile mockFlowFile = new MockFlowFile(id); - mockFlowFile.putAttributes(attributes); - return mockFlowFile; - } -} diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml deleted file mode 100644 index f3e70c706..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - 4.0.0 - - org.apache.nifi.minifi - minifi-nar-bundles - 0.1.0-SNAPSHOT - - - minifi-provenance-reporting-bundle - pom - - - minifi-provenance-reporting-task - minifi-provenance-reporting-nar - - - - - - org.apache.nifi.minifi - minifi-provenance-reporting-task - 0.1.0-SNAPSHOT - - - org.glassfish.jersey.core - jersey-client - 2.19 - - - - From 9af32332eb634e95c36af14a368954b8541c6097 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 5 Oct 2016 16:54:06 -0400 Subject: [PATCH 03/33] Add dependency on nifi-framework-api and change AuthorityProvider to Authorizer. --- .../minifi-framework/minifi-nar-utils/pom.xml | 7 ++++++- .../main/java/org/apache/nifi/nar/ExtensionManager.java | 4 ++-- .../org/apache/nifi/nar/NarThreadContextClassLoader.java | 4 ++-- pom.xml | 5 +++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/pom.xml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/pom.xml index ad9cf8d8d..50fdeb56c 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/pom.xml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/pom.xml @@ -15,7 +15,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> - + minifi-framework org.apache.nifi.minifi @@ -35,6 +36,10 @@ limitations under the License. org.apache.nifi nifi-api + + org.apache.nifi + nifi-framework-api + \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java index db0b35eb5..2932862e0 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java @@ -24,7 +24,7 @@ import java.util.Set; import org.apache.nifi.authentication.LoginIdentityProvider; -import org.apache.nifi.authorization.AuthorityProvider; +import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.controller.repository.ContentRepository; import org.apache.nifi.controller.repository.FlowFileRepository; @@ -58,7 +58,7 @@ public class ExtensionManager { definitionMap.put(FlowFilePrioritizer.class, new HashSet()); definitionMap.put(ReportingTask.class, new HashSet()); definitionMap.put(ControllerService.class, new HashSet()); - definitionMap.put(AuthorityProvider.class, new HashSet()); + definitionMap.put(Authorizer.class, new HashSet()); definitionMap.put(LoginIdentityProvider.class, new HashSet()); definitionMap.put(ProvenanceEventRepository.class, new HashSet()); definitionMap.put(ComponentStatusRepository.class, new HashSet()); diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java index c3b17fd42..93f73eb68 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java @@ -17,7 +17,7 @@ package org.apache.nifi.nar; import org.apache.nifi.authentication.LoginIdentityProvider; -import org.apache.nifi.authorization.AuthorityProvider; +import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.components.Validator; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.controller.repository.ContentRepository; @@ -58,7 +58,7 @@ public class NarThreadContextClassLoader extends URLClassLoader { narSpecificClasses.add(OutputStreamCallback.class); narSpecificClasses.add(StreamCallback.class); narSpecificClasses.add(ControllerService.class); - narSpecificClasses.add(AuthorityProvider.class); + narSpecificClasses.add(Authorizer.class); narSpecificClasses.add(LoginIdentityProvider.class); narSpecificClasses.add(ProvenanceEventRepository.class); narSpecificClasses.add(ComponentStatusRepository.class); diff --git a/pom.xml b/pom.xml index 2960e5b34..5fff7cf4b 100644 --- a/pom.xml +++ b/pom.xml @@ -313,6 +313,11 @@ limitations under the License. nifi-api ${org.apache.nifi.version} + + org.apache.nifi + nifi-framework-api + ${org.apache.nifi.version} + org.apache.nifi nifi-utils From 6dd39793c4081a8ef54bf640531a0a7a8a012f03 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Thu, 6 Oct 2016 14:24:23 -0400 Subject: [PATCH 04/33] work in progress --- .../org/apache/nifi/nar/NarUnpackerTest.java | 40 ++++--------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java index 882c8c6fd..7487e1008 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java @@ -33,7 +33,9 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -112,14 +114,14 @@ public void testUnpackNars() { @Test public void testUnpackNarsFromEmptyDir() throws IOException { - NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties"); - final File emptyDir = new File("./target/empty/dir"); emptyDir.delete(); emptyDir.deleteOnExit(); assertTrue(emptyDir.mkdirs()); - properties.setProperty("nifi.nar.library.directory.alt", emptyDir.toString()); + final Map others = new HashMap<>(); + others.put("nifi.nar.library.directory.alt", emptyDir.toString()); + NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", others); final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties); @@ -173,7 +175,7 @@ public void testUnpackNarsFromNonDir() throws IOException { assertNull(extensionMapping); } - private NiFiProperties loadSpecifiedProperties(String propertiesFile) { + private NiFiProperties loadSpecifiedProperties(final String propertiesFile, final Map others) { String filePath; try { filePath = NarUnpackerTest.class.getResource(propertiesFile).toURI().getPath(); @@ -181,34 +183,6 @@ private NiFiProperties loadSpecifiedProperties(String propertiesFile) { throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex); } - System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath); - - NiFiProperties properties = NiFiProperties.getInstance(); - - // clear out existing properties - for (String prop : properties.stringPropertyNames()) { - properties.remove(prop); - } - - InputStream inStream = null; - try { - inStream = new BufferedInputStream(new FileInputStream(filePath)); - properties.load(inStream); - } catch (final Exception ex) { - throw new RuntimeException("Cannot load properties file due to " - + ex.getLocalizedMessage(), ex); - } finally { - if (null != inStream) { - try { - inStream.close(); - } catch (final Exception ex) { - /** - * do nothing * - */ - } - } - } - - return properties; + return NiFiProperties.createBasicNiFiProperties(filePath, others); } } \ No newline at end of file From 520b4214ff5499ccda9002e648bb2c5acfa21553 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Fri, 7 Oct 2016 15:19:26 -0400 Subject: [PATCH 05/33] removing references to provenance reporting bundle. this should default to the bundle introduced in NiFi --- minifi-assembly/pom.xml | 5 ----- .../org/apache/nifi/nar/ExtensionManager.java | 22 +++++++++---------- .../org/apache/nifi/nar/NarUnpackerTest.java | 16 +++++++------- minifi-nar-bundles/pom.xml | 1 - pom.xml | 6 ----- 5 files changed, 19 insertions(+), 31 deletions(-) diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index ac5ed31f6..8a0f4d910 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -116,11 +116,6 @@ limitations under the License. minifi-framework-nar nar - - org.apache.nifi.minifi - minifi-provenance-reporting-nar - nar - org.apache.nifi.minifi minifi-provenance-repository-nar diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java index 2932862e0..f6f42894f 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java @@ -54,17 +54,17 @@ public class ExtensionManager { private static final Map extensionClassloaderLookup = new HashMap<>(); static { - definitionMap.put(Processor.class, new HashSet()); - definitionMap.put(FlowFilePrioritizer.class, new HashSet()); - definitionMap.put(ReportingTask.class, new HashSet()); - definitionMap.put(ControllerService.class, new HashSet()); - definitionMap.put(Authorizer.class, new HashSet()); - definitionMap.put(LoginIdentityProvider.class, new HashSet()); - definitionMap.put(ProvenanceEventRepository.class, new HashSet()); - definitionMap.put(ComponentStatusRepository.class, new HashSet()); - definitionMap.put(FlowFileRepository.class, new HashSet()); - definitionMap.put(FlowFileSwapManager.class, new HashSet()); - definitionMap.put(ContentRepository.class, new HashSet()); + definitionMap.put(Processor.class, new HashSet<>()); + definitionMap.put(FlowFilePrioritizer.class, new HashSet<>()); + definitionMap.put(ReportingTask.class, new HashSet<>()); + definitionMap.put(ControllerService.class, new HashSet<>()); + definitionMap.put(Authorizer.class, new HashSet<>()); + definitionMap.put(LoginIdentityProvider.class, new HashSet<>()); + definitionMap.put(ProvenanceEventRepository.class, new HashSet<>()); + definitionMap.put(ComponentStatusRepository.class, new HashSet<>()); + definitionMap.put(FlowFileRepository.class, new HashSet<>()); + definitionMap.put(FlowFileSwapManager.class, new HashSet<>()); + definitionMap.put(ContentRepository.class, new HashSet<>()); } /** diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java index 7487e1008..4587e5e2c 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java @@ -21,11 +21,8 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.BufferedInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.net.URISyntaxException; import java.nio.file.FileVisitResult; import java.nio.file.Files; @@ -33,6 +30,7 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -83,7 +81,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) @Test public void testUnpackNars() { - NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties"); + NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", Collections.EMPTY_MAP); assertEquals("./target/NarUnpacker/lib/", properties.getProperty("nifi.nar.library.directory")); @@ -143,8 +141,9 @@ public void testUnpackNarsFromNonExistantDir() { nonExistantDir.delete(); nonExistantDir.deleteOnExit(); - NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties"); - properties.setProperty("nifi.nar.library.directory.alt", nonExistantDir.toString()); + final Map others = new HashMap<>(); + others.put("nifi.nar.library.directory.alt", nonExistantDir.toString()); + NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", others); final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties); @@ -167,8 +166,9 @@ public void testUnpackNarsFromNonDir() throws IOException { nonDir.createNewFile(); nonDir.deleteOnExit(); - NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties"); - properties.setProperty("nifi.nar.library.directory.alt", nonDir.toString()); + final Map others = new HashMap<>(); + others.put("nifi.nar.library.directory.alt", nonDir.toString()); + NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", others); final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties); diff --git a/minifi-nar-bundles/pom.xml b/minifi-nar-bundles/pom.xml index f0fd400a2..0721b41d7 100644 --- a/minifi-nar-bundles/pom.xml +++ b/minifi-nar-bundles/pom.xml @@ -26,7 +26,6 @@ minifi-framework-bundle minifi-provenance-repository-bundle - minifi-provenance-reporting-bundle minifi-update-attribute-nar minifi-standard-nar diff --git a/pom.xml b/pom.xml index 5fff7cf4b..a7205467d 100644 --- a/pom.xml +++ b/pom.xml @@ -166,12 +166,6 @@ limitations under the License. 0.1.0-SNAPSHOT nar - - org.apache.nifi.minifi - minifi-provenance-reporting-nar - 0.1.0-SNAPSHOT - nar - org.apache.nifi.minifi minifi-resources From ea184cc9449bc893be1c15dcde992ff487a60411 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Fri, 7 Oct 2016 15:55:33 -0400 Subject: [PATCH 06/33] Adjust provenance repository to new API. --- minifi-api/pom.xml | 4 + .../MiNiFiPersistentProvenanceRepository.java | 98 +++++++++++++------ ...tMiNiFiPersistentProvenanceRepository.java | 28 +++--- .../org/apache/nifi/provenance/TestUtil.java | 13 ++- 4 files changed, 94 insertions(+), 49 deletions(-) diff --git a/minifi-api/pom.xml b/minifi-api/pom.xml index 111144b73..98a78afd3 100644 --- a/minifi-api/pom.xml +++ b/minifi-api/pom.xml @@ -30,5 +30,9 @@ limitations under the License. org.apache.nifi nifi-api + + org.apache.nifi + nifi-framework-api + \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java index ea66c388a..6fe264af6 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java @@ -23,6 +23,7 @@ import java.io.FileFilter; import java.io.FileNotFoundException; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -54,10 +55,14 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Pattern; +import org.apache.nifi.authorization.Authorizer; +import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.events.EventReporter; import org.apache.nifi.processor.DataUnit; import org.apache.nifi.provenance.expiration.ExpirationAction; import org.apache.nifi.provenance.expiration.FileRemovalAction; +import org.apache.nifi.provenance.lineage.ComputeLineageSubmission; +import org.apache.nifi.provenance.lineage.LineageComputationType; import org.apache.nifi.provenance.lucene.LuceneUtil; import org.apache.nifi.provenance.search.Query; import org.apache.nifi.provenance.search.QuerySubmission; @@ -77,7 +82,7 @@ // TODO: When API, FlowController, and supporting classes are refactored/reimplemented migrate this class and its accompanying imports to minifi package structure -public class MiNiFiPersistentProvenanceRepository implements ProvenanceEventRepository { +public class MiNiFiPersistentProvenanceRepository implements ProvenanceRepository { public static final String EVENT_CATEGORY = "Provenance Repository"; private static final String FILE_EXTENSION = ".prov"; @@ -132,7 +137,20 @@ public class MiNiFiPersistentProvenanceRepository implements ProvenanceEventRepo private Long maxId = null; public MiNiFiPersistentProvenanceRepository() throws IOException { - this(createRepositoryConfiguration(), 10000); + maxPartitionMillis = 0; + maxPartitionBytes = 0; + writers = null; + configuration = null; + alwaysSync = false; + rolloverCheckMillis = 0; + maxAttributeChars = 0; + scheduledExecService = null; + rolloverExecutor = null; + eventReporter = null; + } + + public MiNiFiPersistentProvenanceRepository(final NiFiProperties nifiProperties) throws IOException { + this(createRepositoryConfiguration(nifiProperties), 10000); } public MiNiFiPersistentProvenanceRepository(final RepositoryConfiguration configuration, final int rolloverCheckMillis) throws IOException { @@ -170,7 +188,7 @@ public MiNiFiPersistentProvenanceRepository(final RepositoryConfiguration config } @Override - public void initialize(final EventReporter eventReporter) throws IOException { + public void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory) throws IOException { writeLock.lock(); try { if (initialized.getAndSet(true)) { @@ -237,8 +255,7 @@ public void run() { } } - private static RepositoryConfiguration createRepositoryConfiguration() throws IOException { - final NiFiProperties properties = NiFiProperties.getInstance(); + private static RepositoryConfiguration createRepositoryConfiguration(final NiFiProperties properties) throws IOException { final Map storageDirectories = properties.getProvenanceRepositoryPaths(); if (storageDirectories.isEmpty()) { storageDirectories.put("provenance_repository", Paths.get("provenance_repository")); @@ -336,6 +353,11 @@ public void registerEvents(final Iterable events) { persistRecord(events); } + @Override + public List getEvents(final long firstRecordId, final int maxRecords, final NiFiUser user) throws IOException { + throw new MethodNotSupportedException("Cannot list events for a specified user."); + } + @Override public List getEvents(final long firstRecordId, final int maxRecords) throws IOException { final List records = new ArrayList<>(maxRecords); @@ -586,7 +608,7 @@ private void persistRecord(final Iterable records) { final int numDirty = dirtyWriterCount.get(); if (numDirty >= recordWriters.length) { throw new IllegalStateException("Cannot update repository because all partitions are unusable at this time. Writing to the repository would cause corruption. " - + "This most often happens as a result of the repository running out of disk space or the JMV running out of memory."); + + "This most often happens as a result of the repository running out of disk space or the JMV running out of memory."); } final long idx = writerIndex.getAndIncrement(); @@ -826,15 +848,15 @@ public int compare(final File o1, final File o2) { removed.add(baseName); } catch (final FileNotFoundException fnf) { logger.warn("Failed to perform Expiration Action {} on Provenance Event file {} because the file no longer exists; will not " - + "perform additional Expiration Actions on this file", currentAction, file); + + "perform additional Expiration Actions on this file", currentAction, file); removed.add(baseName); } catch (final Throwable t) { logger.warn("Failed to perform Expiration Action {} on Provenance Event file {} due to {}; will not perform additional " - + "Expiration Actions on this file at this time", currentAction, file, t.toString()); + + "Expiration Actions on this file at this time", currentAction, file, t.toString()); logger.warn("", t); eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to perform Expiration Action " + currentAction + - " on Provenance Event file " + file + " due to " + t.toString() + "; will not perform additional Expiration Actions " + - "on this file at this time"); + " on Provenance Event file " + file + " due to " + t.toString() + "; will not perform additional Expiration Actions " + + "on this file at this time"); } } @@ -1064,10 +1086,10 @@ public void run() { // that is no longer the case. if (journalFileCount > journalCountThreshold || repoSize > sizeThreshold) { logger.warn("The rate of the dataflow is exceeding the provenance recording rate. " - + "Slowing down flow to accommodate. Currently, there are {} journal files ({} bytes) and " - + "threshold for blocking is {} ({} bytes)", journalFileCount, repoSize, journalCountThreshold, sizeThreshold); + + "Slowing down flow to accommodate. Currently, there are {} journal files ({} bytes) and " + + "threshold for blocking is {} ({} bytes)", journalFileCount, repoSize, journalCountThreshold, sizeThreshold); eventReporter.reportEvent(Severity.WARNING, "Provenance Repository", "The rate of the dataflow is " - + "exceeding the provenance recording rate. Slowing down flow to accommodate"); + + "exceeding the provenance recording rate. Slowing down flow to accommodate"); while (journalFileCount > journalCountThreshold || repoSize > sizeThreshold) { // if a shutdown happens while we are in this loop, kill the rollover thread and break @@ -1097,15 +1119,15 @@ public void run() { } logger.debug("Provenance Repository is still behind. Keeping flow slowed down " - + "to accommodate. Currently, there are {} journal files ({} bytes) and " - + "threshold for blocking is {} ({} bytes)", journalFileCount, repoSize, journalCountThreshold, sizeThreshold); + + "to accommodate. Currently, there are {} journal files ({} bytes) and " + + "threshold for blocking is {} ({} bytes)", journalFileCount, repoSize, journalCountThreshold, sizeThreshold); journalFileCount = getJournalCount(); repoSize = getSize(getLogFiles(), 0L); } logger.info("Provenance Repository has now caught up with rolling over journal files. Current number of " - + "journal files to be rolled over is {}", journalFileCount); + + "journal files to be rolled over is {}", journalFileCount); } // we've finished rolling over successfully. Create new writers and reset state. @@ -1240,26 +1262,26 @@ public int compare(final File o1, final File o2) { // we have all "partial" files and there is already a merged file. Delete the data from the index // because the merge file may not be fully merged. We will re-merge. logger.warn("Merged Journal File {} already exists; however, all partial journal files also exist " - + "so assuming that the merge did not finish. Repeating procedure in order to ensure consistency."); + + "so assuming that the merge did not finish. Repeating procedure in order to ensure consistency."); // Since we only store the file's basename, block offset, and event ID, and because the newly created file could end up on // a different Storage Directory than the original, we need to ensure that we delete both the partially merged // file and the TOC file. Otherwise, we could get the wrong copy and have issues retrieving events. if (!suggestedMergeFile.delete()) { logger.error("Failed to delete partially written Provenance Journal File {}. This may result in events from this journal " - + "file not being able to be displayed. This file should be deleted manually.", suggestedMergeFile); + + "file not being able to be displayed. This file should be deleted manually.", suggestedMergeFile); } final File tocFile = TocUtil.getTocFile(suggestedMergeFile); if (tocFile.exists() && !tocFile.delete()) { logger.error("Failed to delete .toc file {}; this may result in not being able to read the Provenance Events from the {} Journal File. " - + "This can be corrected by manually deleting the {} file", tocFile, suggestedMergeFile, tocFile); + + "This can be corrected by manually deleting the {} file", tocFile, suggestedMergeFile, tocFile); } } } else { logger.warn("Cannot merge journal files {} because expected first file to end with extension '.0' " - + "but it did not; assuming that the files were already merged but only some finished deletion " - + "before restart. Deleting remaining partial journal files.", journalFiles); + + "but it did not; assuming that the files were already merged but only some finished deletion " + + "before restart. Deleting remaining partial journal files.", journalFiles); for (final File file : journalFiles) { if (!file.delete() && file.exists()) { @@ -1320,14 +1342,14 @@ record = reader.nextRecord(); } catch (final EOFException eof) { } catch (final Exception e) { logger.warn("Failed to generate Provenance Event Record from Journal due to " + e + "; it's possible that the record wasn't " - + "completely written to the file. This record will be skipped."); + + "completely written to the file. This record will be skipped."); if (logger.isDebugEnabled()) { logger.warn("", e); } if (eventReporter != null) { eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to read Provenance Event Record from Journal due to " + e + - "; it's possible that hte record wasn't completely written to the file. This record will be skipped."); + "; it's possible that hte record wasn't completely written to the file. This record will be skipped."); } } @@ -1393,7 +1415,7 @@ record = reader.nextRecord(); if (eventReporter != null) { eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to remove temporary journal file " + - journalFile.getAbsolutePath() + "; this file should be cleaned up manually"); + journalFile.getAbsolutePath() + "; this file should be cleaned up manually"); } } @@ -1403,7 +1425,7 @@ record = reader.nextRecord(); if (eventReporter != null) { eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, "Failed to remove temporary journal TOC file " + - tocFile.getAbsolutePath() + "; this file should be cleaned up manually"); + tocFile.getAbsolutePath() + "; this file should be cleaned up manually"); } } } @@ -1598,12 +1620,12 @@ protected MethodNotSupportedException(String message, Throwable cause, boolean e } @Override - public QuerySubmission submitQuery(Query query) { + public QuerySubmission submitQuery(Query query, NiFiUser niFiUser) { throw new MethodNotSupportedException("Querying and indexing is not available for implementation " + this.getClass().getName()); } @Override - public QuerySubmission retrieveQuerySubmission(String queryIdentifier) { + public QuerySubmission retrieveQuerySubmission(String queryIdentifier, NiFiUser niFiUser) { throw new MethodNotSupportedException("Querying and indexing is not available for implementation " + this.getClass().getName()); } @@ -1618,23 +1640,37 @@ public List getSearchableAttributes() { } @Override - public AsyncLineageSubmission submitLineageComputation(final String flowFileUuid) { + public AsyncLineageSubmission submitLineageComputation(final String flowFileUuid, NiFiUser niFiUser) { + throw new MethodNotSupportedException("Computation of lineage is not available for implementation " + this.getClass().getName()); + } + + @Override + public ComputeLineageSubmission submitLineageComputation(long eventId, NiFiUser user) { throw new MethodNotSupportedException("Computation of lineage is not available for implementation " + this.getClass().getName()); } @Override - public AsyncLineageSubmission submitExpandChildren(final long eventId) { + public AsyncLineageSubmission submitExpandChildren(final long eventId, NiFiUser niFiUser) { throw new MethodNotSupportedException("Computation of lineage is not available for implementation " + this.getClass().getName()); } @Override - public AsyncLineageSubmission submitExpandParents(final long eventId) { + public AsyncLineageSubmission submitExpandParents(final long eventId, NiFiUser niFiUser) { throw new MethodNotSupportedException("Computation of lineage is not available for implementation " + this.getClass().getName()); } @Override - public AsyncLineageSubmission retrieveLineageSubmission(final String lineageIdentifier) { + public AsyncLineageSubmission retrieveLineageSubmission(final String lineageIdentifier, NiFiUser niFiUser) { throw new MethodNotSupportedException("Computation of lineage is not available for implementation " + this.getClass().getName()); } + @Override + public ProvenanceEventRecord getEvent(final long id, final NiFiUser user) throws IOException { + throw new MethodNotSupportedException("Cannot handle user authorization requests."); + } + + @Override + public ProvenanceEventRepository getProvenanceEventRepository() { + return this; + } } diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestMiNiFiPersistentProvenanceRepository.java b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestMiNiFiPersistentProvenanceRepository.java index 118a2a66d..fbeeeb61d 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestMiNiFiPersistentProvenanceRepository.java +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestMiNiFiPersistentProvenanceRepository.java @@ -148,7 +148,7 @@ public void testAddAndRecover() throws IOException, InterruptedException { config.setMaxEventFileCapacity(1L); config.setMaxEventFileLife(1, TimeUnit.SECONDS); repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final Map attributes = new HashMap<>(); attributes.put("abc", "xyz"); @@ -176,7 +176,7 @@ public void testAddAndRecover() throws IOException, InterruptedException { Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.) repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final List recoveredRecords = repo.getEvents(0L, 12); Assert.assertEquals("Did not establish the correct, Max Event Id through recovery after reloading", 9, repo.getMaxEventId().intValue()); @@ -198,7 +198,7 @@ public void testCompressOnRollover() throws IOException, InterruptedException, P config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setCompressOnRollover(true); repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map attributes = new HashMap<>(); @@ -236,7 +236,7 @@ public void testLineageRequestNotSupported() throws IOException, InterruptedExce config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final String uuid = "00000000-0000-0000-0000-000000000001"; final Map attributes = new HashMap<>(); @@ -244,7 +244,7 @@ public void testLineageRequestNotSupported() throws IOException, InterruptedExce attributes.put("uuid", uuid); attributes.put("filename", "file-" + uuid); - repo.submitLineageComputation(uuid); + repo.submitLineageComputation(uuid, null); } @@ -253,7 +253,7 @@ public void testCorrectProvenanceEventIdOnRestore() throws IOException { final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(1, TimeUnit.SECONDS); repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map attributes = new HashMap<>(); @@ -279,7 +279,7 @@ public void testCorrectProvenanceEventIdOnRestore() throws IOException { repo.close(); final MiNiFiPersistentProvenanceRepository secondRepo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - secondRepo.initialize(getEventReporter()); + secondRepo.initialize(getEventReporter(), null, null); try { final ProvenanceEventRecord event11 = builder.build(); @@ -336,7 +336,7 @@ private File prepCorruptedEventFileTests() throws Exception { config.setDesiredIndexSize(10); repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); String uuid = UUID.randomUUID().toString(); for (int i = 0; i < 20; i++) { @@ -369,7 +369,7 @@ protected int getJournalCount() { return journalCountRef.get(); } }; - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final Map attributes = new HashMap<>(); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); @@ -422,7 +422,7 @@ public void testMergeJournals() throws IOException, InterruptedException { final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(3, TimeUnit.SECONDS); repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final Map attributes = new HashMap<>(); @@ -473,7 +473,7 @@ public void testTruncateAttributes() throws IOException, InterruptedException { config.setMaxAttributeChars(50); config.setMaxEventFileLife(3, TimeUnit.SECONDS); repo = new MiNiFiPersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final Map attributes = new HashMap<>(); attributes.put("75chars", "123456789012345678901234567890123456789012345678901234567890123456789012345"); @@ -520,7 +520,7 @@ protected RecordWriter[] createWriters(RepositoryConfiguration config, long init }; // initialize with our event reporter - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); // create some events in the journal files. final Map attributes = new HashMap<>(); @@ -597,7 +597,7 @@ public Object answer(final InvocationOnMock invocation) throws Throwable { return spiedWriters; } }; - repo.initialize(getEventReporter()); + repo.initialize(getEventReporter(), null, null); final Map attributes = new HashMap<>(); attributes.put("75chars", "123456789012345678901234567890123456789012345678901234567890123456789012345"); @@ -658,7 +658,7 @@ protected Set recoverJournalFiles() throws IOException { }; try { - recoveryRepo.initialize(getEventReporter()); + recoveryRepo.initialize(getEventReporter(), null, null); } finally { recoveryRepo.close(); } diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java index 26766d68e..493045504 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java @@ -39,13 +39,13 @@ public long getEntryDate() { } @Override - public Set getLineageIdentifiers() { - return new HashSet(); + public long getLineageStartDate() { + return System.currentTimeMillis(); } @Override - public long getLineageStartDate() { - return System.currentTimeMillis(); + public long getLineageStartIndex() { + return 0; } @Override @@ -53,6 +53,11 @@ public Long getLastQueueDate() { return System.currentTimeMillis(); } + @Override + public long getQueueDateIndex() { + return 0; + } + @Override public boolean isPenalized() { return false; From af4c0511ed4e9d47e435d83e81884a9b291d8a02 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 10 Oct 2016 09:23:16 -0400 Subject: [PATCH 07/33] Adjusting ConfigMain to use ComponentDTO. --- .../apache/nifi/minifi/toolkit/configuration/ConfigMain.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java index b726c5a1c..475ad1278 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java @@ -24,10 +24,10 @@ import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.apache.nifi.minifi.commons.schema.serialization.SchemaSaver; import org.apache.nifi.minifi.toolkit.configuration.dto.ConfigSchemaFunction; +import org.apache.nifi.web.api.dto.ComponentDTO; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.NiFiComponentDTO; import org.apache.nifi.web.api.dto.PortDTO; import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; From 8b42bd579006782784d8fdb37c7721fa60fe770b Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 10 Oct 2016 10:35:44 -0400 Subject: [PATCH 08/33] Adjusting handling of remote authorization failures for RPGs and updating status code. --- .../rpg/RemoteProcessGroupStatusBean.java | 15 +---- .../status/util/StatusReportPopulator.java | 6 +- .../commons/status/TestStatusReport.java | 4 +- .../org/apache/nifi/minifi/MiNiFiServer.java | 56 ++++++++++++++++--- .../minifi/status/StatusRequestParser.java | 5 -- .../status/TestStatusConfigReporter.java | 32 ++--------- 6 files changed, 59 insertions(+), 59 deletions(-) diff --git a/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupStatusBean.java b/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupStatusBean.java index b4ee5ce3e..3acedc091 100644 --- a/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupStatusBean.java +++ b/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupStatusBean.java @@ -19,13 +19,13 @@ import org.apache.nifi.minifi.commons.status.common.BulletinStatus; +import java.io.Serializable; import java.util.List; -public class RemoteProcessGroupStatusBean implements java.io.Serializable { +public class RemoteProcessGroupStatusBean implements Serializable { private String name; private RemoteProcessGroupHealth remoteProcessGroupHealth; private List bulletinList; - private List authorizationIssues; private List inputPortStatusList; private RemoteProcessGroupStats remoteProcessGroupStats; @@ -56,14 +56,6 @@ public void setBulletinList(List bulletinList) { this.bulletinList = bulletinList; } - public List getAuthorizationIssues() { - return authorizationIssues; - } - - public void setAuthorizationIssues(List authorizationIssues) { - this.authorizationIssues = authorizationIssues; - } - public List getInputPortStatusList() { return inputPortStatusList; } @@ -90,7 +82,6 @@ public boolean equals(Object o) { if (getName() != null ? !getName().equals(that.getName()) : that.getName() != null) return false; if (getRemoteProcessGroupHealth() != null ? !getRemoteProcessGroupHealth().equals(that.getRemoteProcessGroupHealth()) : that.getRemoteProcessGroupHealth() != null) return false; if (getBulletinList() != null ? !getBulletinList().equals(that.getBulletinList()) : that.getBulletinList() != null) return false; - if (getAuthorizationIssues() != null ? !getAuthorizationIssues().equals(that.getAuthorizationIssues()) : that.getAuthorizationIssues() != null) return false; if (getInputPortStatusList() != null ? !getInputPortStatusList().equals(that.getInputPortStatusList()) : that.getInputPortStatusList() != null) return false; return getRemoteProcessGroupStats() != null ? getRemoteProcessGroupStats().equals(that.getRemoteProcessGroupStats()) : that.getRemoteProcessGroupStats() == null; @@ -101,7 +92,6 @@ public int hashCode() { int result = getName() != null ? getName().hashCode() : 0; result = 31 * result + (getRemoteProcessGroupHealth() != null ? getRemoteProcessGroupHealth().hashCode() : 0); result = 31 * result + (getBulletinList() != null ? getBulletinList().hashCode() : 0); - result = 31 * result + (getAuthorizationIssues() != null ? getAuthorizationIssues().hashCode() : 0); result = 31 * result + (getInputPortStatusList() != null ? getInputPortStatusList().hashCode() : 0); result = 31 * result + (getRemoteProcessGroupStats() != null ? getRemoteProcessGroupStats().hashCode() : 0); return result; @@ -113,7 +103,6 @@ public String toString() { "name='" + name + '\'' + ", remoteProcessGroupHealth=" + remoteProcessGroupHealth + ", bulletinList=" + bulletinList + - ", authorizationIssues=" + authorizationIssues + ", inputPortStatusList=" + inputPortStatusList + ", remoteProcessGroupStats=" + remoteProcessGroupStats + '}'; diff --git a/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/util/StatusReportPopulator.java b/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/util/StatusReportPopulator.java index ff3d2cca1..8bdee994f 100644 --- a/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/util/StatusReportPopulator.java +++ b/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/util/StatusReportPopulator.java @@ -134,19 +134,15 @@ public static void addConnectionStatus(FlowStatusReport flowStatusReport, boolea flowStatusReport.setConnectionStatusList(Collections.singletonList(expectedConnectionStatus)); } - public static void addExpectedRemoteProcessGroupStatus(FlowStatusReport flowStatusReport, boolean addHealth, boolean addAuthorizationIssues, boolean addInputPort, + public static void addExpectedRemoteProcessGroupStatus(FlowStatusReport flowStatusReport, boolean addHealth, boolean addInputPort, boolean addStats, boolean addBulletins, boolean populateBulletins) { RemoteProcessGroupStatusBean expectedRemoteProcessGroupStatus = new RemoteProcessGroupStatusBean(); expectedRemoteProcessGroupStatus.setName("rpg1"); - if (addAuthorizationIssues) { - expectedRemoteProcessGroupStatus.setAuthorizationIssues(Collections.singletonList("auth issue")); - } if (addHealth) { RemoteProcessGroupHealth remoteProcessGroupHealth = new RemoteProcessGroupHealth(); remoteProcessGroupHealth.setTransmissionStatus("Transmitting"); remoteProcessGroupHealth.setHasBulletins(populateBulletins); - remoteProcessGroupHealth.setHasAuthorizationIssues(addAuthorizationIssues); remoteProcessGroupHealth.setActivePortCount(1); remoteProcessGroupHealth.setInactivePortCount(2); expectedRemoteProcessGroupStatus.setRemoteProcessGroupHealth(remoteProcessGroupHealth); diff --git a/minifi-commons/minifi-utils/src/test/java/org/apache/nifi/minifi/commons/status/TestStatusReport.java b/minifi-commons/minifi-utils/src/test/java/org/apache/nifi/minifi/commons/status/TestStatusReport.java index fc697e330..cae5b3a1c 100644 --- a/minifi-commons/minifi-utils/src/test/java/org/apache/nifi/minifi/commons/status/TestStatusReport.java +++ b/minifi-commons/minifi-utils/src/test/java/org/apache/nifi/minifi/commons/status/TestStatusReport.java @@ -47,7 +47,7 @@ public void verifySerializableFullyPopulated() throws IOException, ClassNotFound addReportingTaskStatus(original, true, true, true, true); addConnectionStatus(original, true, true); addProcessorStatus(original, true, true, true, true, true); - addExpectedRemoteProcessGroupStatus(original, true, true, true, true, true, true); + addExpectedRemoteProcessGroupStatus(original, true, true, true, true, true); byte[] byteArrayCopy = serialize(original); FlowStatusReport copy = unSerialize(byteArrayCopy, FlowStatusReport.class); @@ -63,7 +63,7 @@ public void verifySerializableSomeNull() throws IOException, ClassNotFoundExcept addInstanceStatus(original, true, true, true, true); addSystemDiagnosticStatus(original, true, true, true, true, true); addProcessorStatus(original, true, true, true, true, true); - addExpectedRemoteProcessGroupStatus(original, true, true, true, true, true, true); + addExpectedRemoteProcessGroupStatus(original, true, true, true, true, true); byte[] byteArrayCopy = serialize(original); FlowStatusReport copy = unSerialize(byteArrayCopy, FlowStatusReport.class); diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java index ba22826d9..2c032ffd3 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java @@ -17,18 +17,28 @@ package org.apache.nifi.minifi; import org.apache.nifi.admin.service.AuditService; -import org.apache.nifi.admin.service.UserService; import org.apache.nifi.admin.service.impl.StandardAuditService; -import org.apache.nifi.admin.service.impl.StandardUserService; +import org.apache.nifi.authorization.AuthorizationRequest; +import org.apache.nifi.authorization.AuthorizationResult; +import org.apache.nifi.authorization.Authorizer; +import org.apache.nifi.authorization.AuthorizerConfigurationContext; +import org.apache.nifi.authorization.AuthorizerInitializationContext; +import org.apache.nifi.authorization.exception.AuthorizationAccessException; +import org.apache.nifi.authorization.exception.AuthorizerCreationException; +import org.apache.nifi.authorization.exception.AuthorizerDestructionException; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.StandardFlowService; import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.repository.RingBufferEventRepository; import org.apache.nifi.encrypt.StringEncryptor; +import org.apache.nifi.events.VolatileBulletinRepository; import org.apache.nifi.minifi.commons.status.FlowStatusReport; import org.apache.nifi.minifi.status.StatusConfigReporter; import org.apache.nifi.minifi.status.StatusRequestException; +import org.apache.nifi.registry.VariableRegistry; +import org.apache.nifi.reporting.BulletinRepository; import org.apache.nifi.services.FlowService; +import org.apache.nifi.util.FileBasedVariableRegistry; import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +50,7 @@ public class MiNiFiServer { private static final Logger logger = LoggerFactory.getLogger(MiNiFiServer.class); private final NiFiProperties props; private FlowService flowService; + private FlowController flowController; /** * @@ -55,20 +66,47 @@ public void start() { FlowFileEventRepository flowFileEventRepository = new RingBufferEventRepository(5); AuditService auditService = new StandardAuditService(); - UserService userService = new StandardUserService(); - StringEncryptor encryptor = StringEncryptor.createEncryptor(); + Authorizer authorizer = new Authorizer() { + @Override + public AuthorizationResult authorize(AuthorizationRequest request) throws AuthorizationAccessException { + return AuthorizationResult.approved(); + } + + @Override + public void initialize(AuthorizerInitializationContext initializationContext) throws AuthorizerCreationException { + + } + + @Override + public void onConfigured(AuthorizerConfigurationContext configurationContext) throws AuthorizerCreationException { + + } + + @Override + public void preDestruction() throws AuthorizerDestructionException { + + } + }; + StringEncryptor encryptor = StringEncryptor.createEncryptor(props); + VariableRegistry variableRegistry = new FileBasedVariableRegistry(props.getVariableRegistryPropertiesPaths()); + BulletinRepository bulletinRepository = new VolatileBulletinRepository(); FlowController flowController = FlowController.createStandaloneInstance( flowFileEventRepository, props, - userService, + authorizer, auditService, - encryptor); + encryptor, + bulletinRepository, + variableRegistry + ); flowService = StandardFlowService.createStandaloneInstance( flowController, props, - encryptor); + encryptor, + null, // revision manager + authorizer); // start and load the flow flowService.start(); @@ -76,6 +114,8 @@ public void start() { flowController.onFlowInitialized(true); flowController.getGroup(flowController.getRootGroupId()).startProcessing(); + this.flowController = flowController; + logger.info("Flow loaded successfully."); } catch (Exception e) { // ensure the flow service is terminated @@ -107,6 +147,6 @@ public void stop() { } public FlowStatusReport getStatusReport(String requestString) throws StatusRequestException { - return StatusConfigReporter.getStatus(flowService.getController(), requestString, logger); + return StatusConfigReporter.getStatus(this.flowController, requestString, logger); } } diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/status/StatusRequestParser.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/status/StatusRequestParser.java index 061120ce5..74d3b1f38 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/status/StatusRequestParser.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/status/StatusRequestParser.java @@ -122,7 +122,6 @@ static RemoteProcessGroupStatusBean parseRemoteProcessGroupStatusRequest(RemoteP new BulletinQuery.Builder() .sourceIdMatches(inputRemoteProcessGroupStatus.getId()) .build()); - List authorizationIssues = inputRemoteProcessGroupStatus.getAuthorizationIssues(); for (String statusType : statusSplits) { switch (statusType.toLowerCase().trim()) { @@ -130,7 +129,6 @@ static RemoteProcessGroupStatusBean parseRemoteProcessGroupStatusRequest(RemoteP RemoteProcessGroupHealth remoteProcessGroupHealth = new RemoteProcessGroupHealth(); remoteProcessGroupHealth.setTransmissionStatus(inputRemoteProcessGroupStatus.getTransmissionStatus().name()); - remoteProcessGroupHealth.setHasAuthorizationIssues(!authorizationIssues.isEmpty()); remoteProcessGroupHealth.setActivePortCount(inputRemoteProcessGroupStatus.getActiveRemotePortCount()); remoteProcessGroupHealth.setInactivePortCount(inputRemoteProcessGroupStatus.getInactiveRemotePortCount()); remoteProcessGroupHealth.setHasBulletins(!bulletinList.isEmpty()); @@ -140,9 +138,6 @@ static RemoteProcessGroupStatusBean parseRemoteProcessGroupStatusRequest(RemoteP case "bulletins": remoteProcessGroupStatusBean.setBulletinList(transformBulletins(bulletinList)); break; - case "authorizationissues": - remoteProcessGroupStatusBean.setAuthorizationIssues(authorizationIssues); - break; case "inputports": List inputPortStatusList = new LinkedList<>(); RemoteProcessGroup remoteProcessGroup = flowController.getGroup(rootGroupId).getRemoteProcessGroup(inputRemoteProcessGroupStatus.getId()); diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java index 027d685f4..d3f8f68dc 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java @@ -435,7 +435,7 @@ public void remoteProcessGroupStatusHealth() throws Exception { FlowStatusReport expected = new FlowStatusReport(); expected.setErrorsGeneratingReport(Collections.EMPTY_LIST); - addExpectedRemoteProcessGroupStatus(expected, true, false, false, false, false, false); + addExpectedRemoteProcessGroupStatus(expected, true, false, false, false, false); assertEquals(expected, actual); } @@ -450,22 +450,7 @@ public void remoteProcessGroupStatusBulletins() throws Exception { FlowStatusReport expected = new FlowStatusReport(); expected.setErrorsGeneratingReport(Collections.EMPTY_LIST); - addExpectedRemoteProcessGroupStatus(expected, false, false, false, false, true, true); - - assertEquals(expected, actual); - } - - @Test - public void remoteProcessGroupStatusAuthorizationIssues() throws Exception { - populateRemoteProcessGroup(false, true); - - String statusRequest = "remoteProcessGroup:all:authorizationissues"; - FlowStatusReport actual = StatusConfigReporter.getStatus(mockFlowController, statusRequest, LoggerFactory.getLogger(TestStatusConfigReporter.class)); - - FlowStatusReport expected = new FlowStatusReport(); - expected.setErrorsGeneratingReport(Collections.EMPTY_LIST); - - addExpectedRemoteProcessGroupStatus(expected, false, true, false, false, false, false); + addExpectedRemoteProcessGroupStatus(expected, false, false, false, true, true); assertEquals(expected, actual); } @@ -480,7 +465,7 @@ public void remoteProcessGroupStatusInputPorts() throws Exception { FlowStatusReport expected = new FlowStatusReport(); expected.setErrorsGeneratingReport(Collections.EMPTY_LIST); - addExpectedRemoteProcessGroupStatus(expected, false, false, true, false, false, false); + addExpectedRemoteProcessGroupStatus(expected, false, true, false, false, false); assertEquals(expected, actual); } @@ -495,7 +480,7 @@ public void remoteProcessGroupStatusStats() throws Exception { FlowStatusReport expected = new FlowStatusReport(); expected.setErrorsGeneratingReport(Collections.EMPTY_LIST); - addExpectedRemoteProcessGroupStatus(expected, false, false, false, true, false, false); + addExpectedRemoteProcessGroupStatus(expected, false, false, true, false, false); assertEquals(expected, actual); } @@ -511,7 +496,7 @@ public void remoteProcessGroupStatusAll() throws Exception { FlowStatusReport expected = new FlowStatusReport(); expected.setErrorsGeneratingReport(Collections.EMPTY_LIST); - addExpectedRemoteProcessGroupStatus(expected, true, true, true, true, true, true); + addExpectedRemoteProcessGroupStatus(expected, true, true, true, true, true); assertEquals(expected, actual); } @@ -543,7 +528,7 @@ public void statusEverything() throws Exception { addReportingTaskStatus(expected, true, true, true, false); addConnectionStatus(expected, true, true); addProcessorStatus(expected, true, true, true, true, false); - addExpectedRemoteProcessGroupStatus(expected, true, true, true, true, true, false); + addExpectedRemoteProcessGroupStatus(expected, true, true, true, true, false); assertEquals(expected, actual); } @@ -714,11 +699,6 @@ private void populateRemoteProcessGroup(boolean addBulletins, boolean addAuthIss RemoteProcessGroupStatus remoteProcessGroupStatus = new RemoteProcessGroupStatus(); addRemoteProcessGroupStatus(remoteProcessGroupStatus); - if (addAuthIssues) { - remoteProcessGroupStatus.setAuthorizationIssues(Collections.singletonList("auth issue")); - } else { - remoteProcessGroupStatus.setAuthorizationIssues(Collections.EMPTY_LIST); - } if (addBulletins) { addBulletins("Bulletin message", remoteProcessGroupStatus.getId()); } From 248c4c2d7e6e60a2d2f0ff311e63f11ff3f8cab6 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 10 Oct 2016 10:38:52 -0400 Subject: [PATCH 09/33] Completed all code changes to support libraries. Now issues at runtime with needed dependencies. --- .../src/main/java/org/apache/nifi/minifi/MiNiFi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java index b2b025b58..86dab0bc7 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java @@ -229,7 +229,7 @@ MiNiFiServer getMinifiServer() { public static void main(String[] args) { logger.info("Launching MiNiFi..."); try { - NiFiProperties niFiProperties = NiFiProperties.getInstance(); + NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, null); new MiNiFi(niFiProperties); } catch (final Throwable t) { logger.error("Failure to launch MiNiFi due to " + t, t); From a294298b09609c92b20dfb1d2abd60a4d75fdfee Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 10 Oct 2016 10:43:06 -0400 Subject: [PATCH 10/33] Provide nifi-framework-api to provide FlowFilePrioritizer interface. --- minifi-assembly/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 8a0f4d910..ff06932f7 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -143,6 +143,10 @@ limitations under the License. org.apache.nifi nifi-api + + org.apache.nifi + nifi-framework-api + org.apache.nifi nifi-framework-core-api From 3acc3e925835d116b57e5da67e6d28aadb5117e5 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 10 Oct 2016 12:35:17 -0400 Subject: [PATCH 11/33] Adjusting classloading and adjusting Persistent Prov repo. --- minifi-assembly/pom.xml | 4 + .../org/apache/nifi/nar/ExtensionManager.java | 9 +- .../org/apache/nifi/nar/NarClassLoaders.java | 128 ++++++++++++------ .../org/apache/nifi/nar/NarCloseable.java | 51 ++++++- .../nifi/nar/NarThreadContextClassLoader.java | 34 ++++- .../java/org/apache/nifi/minifi/MiNiFi.java | 6 +- ...pache.nifi.provenance.ProvenanceRepository | 15 ++ 7 files changed, 196 insertions(+), 51 deletions(-) create mode 100644 minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index ff06932f7..7ba58e544 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -143,6 +143,10 @@ limitations under the License. org.apache.nifi nifi-api + + org.apache.nifi + nifi-ssl-context-service-api + org.apache.nifi nifi-framework-api diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java index f6f42894f..787fb3c7b 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionManager.java @@ -32,7 +32,7 @@ import org.apache.nifi.controller.status.history.ComponentStatusRepository; import org.apache.nifi.flowfile.FlowFilePrioritizer; import org.apache.nifi.processor.Processor; -import org.apache.nifi.provenance.ProvenanceEventRepository; +import org.apache.nifi.provenance.ProvenanceRepository; import org.apache.nifi.reporting.ReportingTask; import org.slf4j.Logger; @@ -60,7 +60,7 @@ public class ExtensionManager { definitionMap.put(ControllerService.class, new HashSet<>()); definitionMap.put(Authorizer.class, new HashSet<>()); definitionMap.put(LoginIdentityProvider.class, new HashSet<>()); - definitionMap.put(ProvenanceEventRepository.class, new HashSet<>()); + definitionMap.put(ProvenanceRepository.class, new HashSet<>()); definitionMap.put(ComponentStatusRepository.class, new HashSet<>()); definitionMap.put(FlowFileRepository.class, new HashSet<>()); definitionMap.put(FlowFileSwapManager.class, new HashSet<>()); @@ -69,8 +69,9 @@ public class ExtensionManager { /** * Loads all FlowFileProcessor, FlowFileComparator, ReportingTask class types that can be found on the bootstrap classloader and by creating classloaders for all NARs found within the classpath. + * @param extensionLoaders the loaders to scan through in search of extensions */ - public static void discoverExtensions() { + public static void discoverExtensions(final Set extensionLoaders) { final ClassLoader systemClassLoader = ClassLoader.getSystemClassLoader(); // get the current context class loader @@ -80,7 +81,7 @@ public static void discoverExtensions() { loadExtensions(systemClassLoader); // consider each nar class loader - for (final ClassLoader ncl : NarClassLoaders.getExtensionClassLoaders()) { + for (final ClassLoader ncl : extensionLoaders) { // Must set the context class loader to the nar classloader itself // so that static initialization techniques that depend on the context class loader will work properly diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java index a4525b09b..327712c24 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java @@ -44,28 +44,85 @@ public final class NarClassLoaders { public static final String FRAMEWORK_NAR_ID = "minifi-framework-nar"; + private static volatile NarClassLoaders ncl; + private volatile InitContext initContext; private static final Logger logger = LoggerFactory.getLogger(NarClassLoaders.class); - private static final AtomicBoolean initialized = new AtomicBoolean(false); - private static final AtomicReference> extensionClassLoaders = new AtomicReference<>(); - private static final AtomicReference frameworkClassLoader = new AtomicReference<>(); + + private final static class InitContext { + + private final File frameworkWorkingDir; + private final File extensionWorkingDir; + private final ClassLoader frameworkClassLoader; + private final Map extensionClassLoaders; + + private InitContext( + final File frameworkDir, + final File extensionDir, + final ClassLoader frameworkClassloader, + final Map extensionClassLoaders) { + this.frameworkWorkingDir = frameworkDir; + this.extensionWorkingDir = extensionDir; + this.frameworkClassLoader = frameworkClassloader; + this.extensionClassLoaders = extensionClassLoaders; + } + } + + private NarClassLoaders() { + } /** - * Loads the extensions class loaders from the specified working directory. - * Loading is only performed during the initial invocation of load. - * Subsequent attempts will be ignored. - * + * @return The singleton instance of the NarClassLoaders + */ + public static NarClassLoaders getInstance() { + NarClassLoaders result = ncl; + if (result == null) { + synchronized (NarClassLoaders.class) { + result = ncl; + if (result == null) { + ncl = result = new NarClassLoaders(); + } + } + } + return result; + } + + /** + * Initializes and loads the NarClassLoaders. This method must be called + * before the rest of the methods to access the classloaders are called and + * it can be safely called any number of times provided the same framework + * and extension working dirs are used. * - * @param properties properties object to initialize with - * @throws java.io.IOException ioe - * @throws java.lang.ClassNotFoundException cfne - * @throws IllegalStateException if the class loaders have already been - * created + * @param frameworkWorkingDir where to find framework artifacts + * @param extensionsWorkingDir where to find extension artifacts + * @throws java.io.IOException if any issue occurs while exploding nar working directories. + * @throws java.lang.ClassNotFoundException if unable to load class definition + * @throws IllegalStateException already initialized with a given pair of + * directories cannot reinitialize or use a different pair of directories. */ - public static void load(final NiFiProperties properties) throws IOException, ClassNotFoundException { - if (initialized.getAndSet(true)) { - throw new IllegalStateException("Extensions class loaders have already been loaded."); + public void init(final File frameworkWorkingDir, final File extensionsWorkingDir) throws IOException, ClassNotFoundException { + if (frameworkWorkingDir == null || extensionsWorkingDir == null) { + throw new NullPointerException("cannot have empty arguments"); } + InitContext ic = initContext; + if (ic == null) { + synchronized (this) { + ic = initContext; + if (ic == null) { + initContext = ic = load(frameworkWorkingDir, extensionsWorkingDir); + } + } + } + boolean matching = initContext.extensionWorkingDir.equals(extensionsWorkingDir) + && initContext.frameworkWorkingDir.equals(frameworkWorkingDir); + if (!matching) { + throw new IllegalStateException("Cannot reinitialize and extension/framework directories cannot change"); + } + } + /** + * Should be called at most once. + */ + private InitContext load(final File frameworkWorkingDir, final File extensionsWorkingDir) throws IOException, ClassNotFoundException { // get the system classloader final ClassLoader systemClassLoader = ClassLoader.getSystemClassLoader(); @@ -76,19 +133,16 @@ public static void load(final NiFiProperties properties) throws IOException, Cla final Map extensionDirectoryClassLoaderLookup = new LinkedHashMap<>(); final Map narIdClassLoaderLookup = new HashMap<>(); - final File frameworkWorkingDirectory = properties.getFrameworkWorkingDirectory(); - final File extensionsWorkingDirectory = properties.getExtensionsWorkingDirectory(); - // make sure the nar directory is there and accessible - FileUtils.ensureDirectoryExistAndCanAccess(frameworkWorkingDirectory); - FileUtils.ensureDirectoryExistAndCanAccess(extensionsWorkingDirectory); + FileUtils.ensureDirectoryExistAndCanAccess(frameworkWorkingDir); + FileUtils.ensureDirectoryExistAndCanAccess(extensionsWorkingDir); final List narWorkingDirContents = new ArrayList<>(); - final File[] frameworkWorkingDirContents = frameworkWorkingDirectory.listFiles(); + final File[] frameworkWorkingDirContents = frameworkWorkingDir.listFiles(); if (frameworkWorkingDirContents != null) { narWorkingDirContents.addAll(Arrays.asList(frameworkWorkingDirContents)); } - final File[] extensionsWorkingDirContents = extensionsWorkingDirectory.listFiles(); + final File[] extensionsWorkingDirContents = extensionsWorkingDir.listFiles(); if (extensionsWorkingDirContents != null) { narWorkingDirContents.addAll(Arrays.asList(extensionsWorkingDirContents)); } @@ -145,11 +199,7 @@ public static void load(final NiFiProperties properties) throws IOException, Cla } } - // set the framework class loader - frameworkClassLoader.set(narIdClassLoaderLookup.get(FRAMEWORK_NAR_ID)); - - // set the extensions class loader map - extensionClassLoaders.set(new LinkedHashMap<>(extensionDirectoryClassLoaderLookup)); + return new InitContext(frameworkWorkingDir, extensionsWorkingDir, narIdClassLoaderLookup.get(FRAMEWORK_NAR_ID), new LinkedHashMap<>(extensionDirectoryClassLoaderLookup)); } /** @@ -199,12 +249,12 @@ private static NarDetails getNarDetails(final File narDirectory) throws IOExcept * @throws IllegalStateException if the frame class loader has not been * loaded */ - public static ClassLoader getFrameworkClassLoader() { - if (!initialized.get()) { + public ClassLoader getFrameworkClassLoader() { + if (initContext == null) { throw new IllegalStateException("Framework class loader has not been loaded."); } - return frameworkClassLoader.get(); + return initContext.frameworkClassLoader; } /** @@ -213,14 +263,17 @@ public static ClassLoader getFrameworkClassLoader() { * null when no class loader exists for the specified working directory * @throws IllegalStateException if the class loaders have not been loaded */ - public static ClassLoader getExtensionClassLoader(final File extensionWorkingDirectory) { - if (!initialized.get()) { + public ClassLoader getExtensionClassLoader(final File extensionWorkingDirectory) { + if (initContext == null) { throw new IllegalStateException("Extensions class loaders have not been loaded."); } try { - return extensionClassLoaders.get().get(extensionWorkingDirectory.getCanonicalPath()); + return initContext.extensionClassLoaders.get(extensionWorkingDirectory.getCanonicalPath()); } catch (final IOException ioe) { + if(logger.isDebugEnabled()){ + logger.debug("Unable to get extension classloader for working directory '{}'", extensionWorkingDirectory); + } return null; } } @@ -229,12 +282,12 @@ public static ClassLoader getExtensionClassLoader(final File extensionWorkingDir * @return the extension class loaders * @throws IllegalStateException if the class loaders have not been loaded */ - public static Set getExtensionClassLoaders() { - if (!initialized.get()) { + public Set getExtensionClassLoaders() { + if (initContext == null) { throw new IllegalStateException("Extensions class loaders have not been loaded."); } - return new LinkedHashSet<>(extensionClassLoaders.get().values()); + return new LinkedHashSet<>(initContext.extensionClassLoaders.values()); } private static class NarDetails { @@ -267,7 +320,4 @@ public void setNarWorkingDirectory(File narWorkingDirectory) { this.narWorkingDirectory = narWorkingDirectory; } } - - private NarClassLoaders() { - } } diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarCloseable.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarCloseable.java index c0b43dc7b..639d0326f 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarCloseable.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarCloseable.java @@ -16,6 +16,9 @@ */ package org.apache.nifi.nar; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.Closeable; /** @@ -23,10 +26,54 @@ */ public class NarCloseable implements Closeable { - public static org.apache.nifi.nar.NarCloseable withNarLoader() { + private static final Logger logger = LoggerFactory.getLogger(NarCloseable.class); + + public static NarCloseable withNarLoader() { final ClassLoader current = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(NarThreadContextClassLoader.getInstance()); - return new org.apache.nifi.nar.NarCloseable(current); + return new NarCloseable(current); + } + + /** + * Sets the current thread context class loader to the specific appropriate + * Nar class loader for the given configurable component. Restores to the + * previous classloader once complete. If the given class is not assignable + * from ConfigurableComponent then the NarThreadContextClassLoader is used. + * + * @param componentClass componentClass + * @return NarCloseable with current thread context classloader jailed to + * the nar of the component + */ + public static NarCloseable withComponentNarLoader(final Class componentClass) { + final ClassLoader current = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(componentClass.getClassLoader()); + return new NarCloseable(current); + } + + /** + * Creates a Closeable object that can be used to to switch to current class + * loader to the framework class loader and will automatically set the + * ClassLoader back to the previous class loader when closed + * + * @return a NarCloseable + */ + public static NarCloseable withFrameworkNar() { + final ClassLoader frameworkClassLoader; + try { + frameworkClassLoader = NarClassLoaders.getInstance().getFrameworkClassLoader(); + } catch (final Exception e) { + // This should never happen in a running instance, but it will occur in unit tests + logger.error("Unable to access Framework ClassLoader due to " + e + ". Will continue without changing ClassLoaders."); + if (logger.isDebugEnabled()) { + logger.error("", e); + } + + return new NarCloseable(null); + } + + final ClassLoader current = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(frameworkClassLoader); + return new NarCloseable(current); } private final ClassLoader toSet; diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java index 93f73eb68..9306e44d4 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java @@ -30,10 +30,14 @@ import org.apache.nifi.processor.io.OutputStreamCallback; import org.apache.nifi.processor.io.StreamCallback; import org.apache.nifi.provenance.ProvenanceEventRepository; +import org.apache.nifi.provenance.ProvenanceRepository; import org.apache.nifi.reporting.ReportingTask; +import org.apache.nifi.util.NiFiProperties; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; @@ -60,7 +64,7 @@ public class NarThreadContextClassLoader extends URLClassLoader { narSpecificClasses.add(ControllerService.class); narSpecificClasses.add(Authorizer.class); narSpecificClasses.add(LoginIdentityProvider.class); - narSpecificClasses.add(ProvenanceEventRepository.class); + narSpecificClasses.add(ProvenanceRepository.class); narSpecificClasses.add(ComponentStatusRepository.class); narSpecificClasses.add(FlowFileRepository.class); narSpecificClasses.add(FlowFileSwapManager.class); @@ -165,7 +169,22 @@ Class[] getExecutionStack() { } } - public static T createInstance(final String implementationClassName, final Class typeDefinition) throws InstantiationException, IllegalAccessException, ClassNotFoundException { + /** + * Constructs an instance of the given type using either default no args + * constructor or a constructor which takes a NiFiProperties object + * (preferred). + * + * @param the type to create an instance for + * @param implementationClassName the implementation class name + * @param typeDefinition the type definition + * @param nifiProperties the NiFiProperties instance + * @return constructed instance + * @throws InstantiationException if there is an error instantiating the class + * @throws IllegalAccessException if there is an error accessing the type + * @throws ClassNotFoundException if the class cannot be found + */ + public static T createInstance(final String implementationClassName, final Class typeDefinition, final NiFiProperties nifiProperties) + throws InstantiationException, IllegalAccessException, ClassNotFoundException { final ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(NarThreadContextClassLoader.getInstance()); try { @@ -181,7 +200,16 @@ public static T createInstance(final String implementationClassName, final C Thread.currentThread().setContextClassLoader(detectedClassLoaderForType); final Class desiredClass = rawClass.asSubclass(typeDefinition); - return typeDefinition.cast(desiredClass.newInstance()); + if(nifiProperties == null){ + return typeDefinition.cast(desiredClass.newInstance()); + } + Constructor constructor = null; + try { + constructor = desiredClass.getConstructor(NiFiProperties.class); + return typeDefinition.cast(constructor.newInstance(nifiProperties)); + } catch (final NoSuchMethodException | InvocationTargetException ex) { + return typeDefinition.cast(desiredClass.newInstance()); + } } finally { Thread.currentThread().setContextClassLoader(originalClassLoader); } diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java index 86dab0bc7..573b13956 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java @@ -108,16 +108,16 @@ public void run() { NarUnpacker.unpackNars(properties); // load the extensions classloaders - NarClassLoaders.load(properties); + NarClassLoaders.getInstance().init(properties.getFrameworkWorkingDirectory(), properties.getExtensionsWorkingDirectory()); // load the framework classloader - final ClassLoader frameworkClassLoader = NarClassLoaders.getFrameworkClassLoader(); + final ClassLoader frameworkClassLoader = NarClassLoaders.getInstance().getFrameworkClassLoader(); if (frameworkClassLoader == null) { throw new IllegalStateException("Unable to find the framework NAR ClassLoader."); } // discover the extensions - ExtensionManager.discoverExtensions(); + ExtensionManager.discoverExtensions(NarClassLoaders.getInstance().getExtensionClassLoaders()); ExtensionManager.logClassLoaderMapping(); // load the server from the framework classloader diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository new file mode 100644 index 000000000..71f8c062b --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/resources/META-INF/services/org.apache.nifi.provenance.ProvenanceRepository @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository \ No newline at end of file From 482bbd43a7892384da86f6196937751091ccd41a Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 10 Oct 2016 12:56:59 -0400 Subject: [PATCH 12/33] Include httpclient as it is needed for Site to Site. --- minifi-assembly/src/main/assembly/dependencies.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index a774e493b..9853a5155 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -46,7 +46,6 @@ h2 netty jaxb-impl - httpclient mail log4j lucene-queryparser From ad6cb2536196f70ee6dea6fa627b16618cc57563 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 10 Oct 2016 14:17:07 -0400 Subject: [PATCH 13/33] Creating a configexception to allow removal of ZooKeeper dependency. --- .../src/main/assembly/dependencies.xml | 2 ++ .../server/quorum/QuorumPeerConfig.java | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index 9853a5155..db2d18559 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -37,6 +37,7 @@ minifi-resources spring-aop + zookeeper spring-context spring-security-core spring-beans @@ -49,6 +50,7 @@ mail log4j lucene-queryparser + lucene-core commons-net diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java new file mode 100644 index 000000000..443b8f4c2 --- /dev/null +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.quorum; + + +public class QuorumPeerConfig { + + @SuppressWarnings("serial") + public static class ConfigException extends Exception { + public ConfigException(String msg) { + super(msg); + } + public ConfigException(String msg, Exception e) { + super(msg, e); + } + } + +} From c840cf570fb713a68d28a002ce0a04b41efdba31 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 12 Oct 2016 16:03:55 -0400 Subject: [PATCH 14/33] Excluding lucene-analyzers-common. --- minifi-assembly/src/main/assembly/dependencies.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index db2d18559..5503f6034 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -49,6 +49,7 @@ jaxb-impl mail log4j + lucene-analyzers-common lucene-queryparser lucene-core commons-net From bf59001e6b9581c751a1bff8cb908bc3e7f04f68 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 17 Oct 2016 11:38:17 -0400 Subject: [PATCH 15/33] Removing reliance on LuceneUtil. --- .../org/apache/nifi/minifi/MiNiFiServer.java | 6 ++--- .../MiNiFiPersistentProvenanceRepository.java | 27 +++++++++---------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java index 2c032ffd3..ad82039f0 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiServer.java @@ -74,17 +74,17 @@ public AuthorizationResult authorize(AuthorizationRequest request) throws Author @Override public void initialize(AuthorizerInitializationContext initializationContext) throws AuthorizerCreationException { - + // do nothing } @Override public void onConfigured(AuthorizerConfigurationContext configurationContext) throws AuthorizerCreationException { - + // do nothing } @Override public void preDestruction() throws AuthorizerDestructionException { - + // do nothing } }; StringEncryptor encryptor = StringEncryptor.createEncryptor(props); diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java index 6fe264af6..6ccb4c20d 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java @@ -55,6 +55,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.events.EventReporter; @@ -62,8 +63,6 @@ import org.apache.nifi.provenance.expiration.ExpirationAction; import org.apache.nifi.provenance.expiration.FileRemovalAction; import org.apache.nifi.provenance.lineage.ComputeLineageSubmission; -import org.apache.nifi.provenance.lineage.LineageComputationType; -import org.apache.nifi.provenance.lucene.LuceneUtil; import org.apache.nifi.provenance.search.Query; import org.apache.nifi.provenance.search.QuerySubmission; import org.apache.nifi.provenance.search.SearchableField; @@ -533,7 +532,7 @@ public int compare(final Long o1, final Long o2) { continue; } - final String basename = LuceneUtil.substringBefore(recoveredJournal.getName(), "."); + final String basename = StringUtils.substringBefore(recoveredJournal.getName(), "."); try { final long minId = Long.parseLong(basename); @@ -776,8 +775,8 @@ synchronized void purgeOldEvents() throws IOException { final Comparator sortByBasenameComparator = new Comparator() { @Override public int compare(final File o1, final File o2) { - final String baseName1 = LuceneUtil.substringBefore(o1.getName(), "."); - final String baseName2 = LuceneUtil.substringBefore(o2.getName(), "."); + final String baseName1 = StringUtils.substringBefore(o1.getName(), "."); + final String baseName2 = StringUtils.substringBefore(o2.getName(), "."); Long id1 = null; Long id2 = null; @@ -831,7 +830,7 @@ public int compare(final File o1, final File o2) { // Age off the data. final Set removed = new LinkedHashSet<>(); for (File file : uniqueFilesToPurge) { - final String baseName = LuceneUtil.substringBefore(file.getName(), "."); + final String baseName = StringUtils.substringBefore(file.getName(), "."); ExpirationAction currentAction = null; try { for (final ExpirationAction action : expirationActions) { @@ -874,7 +873,7 @@ public int compare(final File o1, final File o2) { while (itr.hasNext()) { final Map.Entry entry = itr.next(); final String filename = entry.getValue().toFile().getName(); - final String baseName = LuceneUtil.substringBefore(filename, "."); + final String baseName = StringUtils.substringBefore(filename, "."); if (removed.contains(baseName)) { itr.remove(); @@ -994,7 +993,7 @@ private void rollover(final boolean force) throws IOException { if (journalsToMerge.isEmpty()) { logger.debug("No journals to merge; all RecordWriters were already closed"); } else { - logger.debug("Going to merge {} files for journals starting with ID {}", journalsToMerge.size(), LuceneUtil.substringBefore(journalsToMerge.get(0).getName(), ".")); + logger.debug("Going to merge {} files for journals starting with ID {}", journalsToMerge.size(), StringUtils.substringBefore(journalsToMerge.get(0).getName(), ".")); } } @@ -1032,7 +1031,7 @@ public void run() { // We need to make sure that another thread doesn't also update the map at the same time. We cannot // use the write lock when purging old events, and we want to use the same approach here. boolean updated = false; - final Long fileFirstEventId = Long.valueOf(LuceneUtil.substringBefore(fileRolledOver.getName(), ".")); + final Long fileFirstEventId = Long.valueOf(StringUtils.substringBefore(fileRolledOver.getName(), ".")); while (!updated) { final SortedMap existingPathMap = idToPathMap.get(); final SortedMap newIdToPathMap = new TreeMap<>(new PathMapComparator()); @@ -1165,7 +1164,7 @@ protected Set recoverJournalFiles() throws IOException { continue; } - final String basename = LuceneUtil.substringBefore(journalFile.getName(), "."); + final String basename = StringUtils.substringBefore(journalFile.getName(), "."); List files = journalMap.get(basename); if (files == null) { files = new ArrayList<>(); @@ -1193,7 +1192,7 @@ static File getMergeFile(final List journalFiles, final File storageDir) { // verify that all Journal files have the same basename String canonicalBaseName = null; for (final File journal : journalFiles) { - final String basename = LuceneUtil.substringBefore(journal.getName(), "."); + final String basename = StringUtils.substringBefore(journal.getName(), "."); if (canonicalBaseName == null) { canonicalBaseName = basename; } @@ -1239,8 +1238,8 @@ File mergeJournals(final List journalFiles, final File suggestedMergeFile, Collections.sort(journalFiles, new Comparator() { @Override public int compare(final File o1, final File o2) { - final String suffix1 = LuceneUtil.substringAfterLast(o1.getName(), "."); - final String suffix2 = LuceneUtil.substringAfterLast(o2.getName(), "."); + final String suffix1 = StringUtils.substringAfterLast(o1.getName(), "."); + final String suffix2 = StringUtils.substringAfterLast(o2.getName(), "."); try { final int journalIndex1 = Integer.parseInt(suffix1); @@ -1253,7 +1252,7 @@ public int compare(final File o1, final File o2) { }); final String firstJournalFile = journalFiles.get(0).getName(); - final String firstFileSuffix = LuceneUtil.substringAfterLast(firstJournalFile, "."); + final String firstFileSuffix = StringUtils.substringAfterLast(firstJournalFile, "."); final boolean allPartialFiles = firstFileSuffix.equals("0"); // check if we have all of the "partial" files for the journal. From b48c7eb7e3e1f589b245b0bb56b69bf62715c9ac Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 17 Oct 2016 12:08:11 -0400 Subject: [PATCH 16/33] Including lucene core needed for utility methods. Removing extraneous logging message. --- minifi-assembly/src/main/assembly/dependencies.xml | 1 - .../nifi/provenance/MiNiFiPersistentProvenanceRepository.java | 1 - 2 files changed, 2 deletions(-) diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index 5503f6034..e0142c879 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -51,7 +51,6 @@ log4j lucene-analyzers-common lucene-queryparser - lucene-core commons-net diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java index 6ccb4c20d..90be86383 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java @@ -1224,7 +1224,6 @@ static File getMergeFile(final List journalFiles, final File storageDir) { * @throws IOException if a problem occurs writing to the mergedFile, reading from a journal */ File mergeJournals(final List journalFiles, final File suggestedMergeFile, final EventReporter eventReporter) throws IOException { - logger.warn("Merging {} to {}", journalFiles, suggestedMergeFile); if (this.closed.get()) { logger.info("Provenance Repository has been closed; will not merge journal files to {}", suggestedMergeFile); return null; From d2fbab55c83dfa2f7b60215b55cde2f09d20ac46 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 17 Oct 2016 12:39:49 -0400 Subject: [PATCH 17/33] Adding site-to-site-reporting nar to assembly. --- minifi-assembly/pom.xml | 5 +++++ pom.xml | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 7ba58e544..8c4caa455 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -229,6 +229,11 @@ limitations under the License. jetty-server compile + + org.apache.nifi + nifi-site-to-site-reporting-nar + nar + diff --git a/pom.xml b/pom.xml index a7205467d..8f9285ec4 100644 --- a/pom.xml +++ b/pom.xml @@ -502,6 +502,12 @@ limitations under the License. commons-io 2.5 + + org.apache.nifi + nifi-site-to-site-reporting-nar + ${org.apache.nifi.version} + nar + From 592d9bca37553d83e706cd2aa38060e4f357c92b Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 19 Oct 2016 09:43:04 -0400 Subject: [PATCH 18/33] Removing unused imports causing contrib-check to fail. --- .../src/main/java/org/apache/nifi/nar/NarClassLoaders.java | 3 --- .../java/org/apache/nifi/nar/NarThreadContextClassLoader.java | 1 - .../nifi/provenance/MiNiFiPersistentProvenanceRepository.java | 1 - .../src/test/java/org/apache/nifi/provenance/TestUtil.java | 2 -- 4 files changed, 7 deletions(-) diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java index 327712c24..921972107 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java @@ -28,12 +28,9 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.jar.Attributes; import java.util.jar.Manifest; import org.apache.nifi.util.FileUtils; -import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java index 9306e44d4..5c70927a0 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java @@ -29,7 +29,6 @@ import org.apache.nifi.processor.io.InputStreamCallback; import org.apache.nifi.processor.io.OutputStreamCallback; import org.apache.nifi.processor.io.StreamCallback; -import org.apache.nifi.provenance.ProvenanceEventRepository; import org.apache.nifi.provenance.ProvenanceRepository; import org.apache.nifi.reporting.ReportingTask; import org.apache.nifi.util.NiFiProperties; diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java index 90be86383..a015dca95 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/MiNiFiPersistentProvenanceRepository.java @@ -23,7 +23,6 @@ import java.io.FileFilter; import java.io.FileNotFoundException; import java.io.IOException; -import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java index 493045504..935d43996 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestUtil.java @@ -17,9 +17,7 @@ package org.apache.nifi.provenance; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import org.apache.nifi.flowfile.FlowFile; From 4dafa8eccbd40c92af9fb3403079a4edb6c4833d Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Fri, 21 Oct 2016 16:43:15 -0400 Subject: [PATCH 19/33] Adding comment and SuppressWarning for unused QuorumPeerConfig --- .../apache/zookeeper/server/quorum/QuorumPeerConfig.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java index 443b8f4c2..294b66fcd 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java @@ -19,6 +19,13 @@ package org.apache.zookeeper.server.quorum; +/** + * This class is provided such that the ConfigException can be provided to the FlowController currently reused from NiFi. + * By creating the class within the MiNiFi codebase it is possible to remove the dependency on ZooKeeper. + * + * This totality of this class is eligible for removal when a FlowController explicitly for MiNiFi is introduced. + */ +@SuppressWarnings(value = "unused") public class QuorumPeerConfig { @SuppressWarnings("serial") From ddaffcd8883a2911545da1a334210d74996f16c0 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 24 Oct 2016 10:20:11 -0400 Subject: [PATCH 20/33] Removing remaining references to authorization issues in docs and associated status classes. --- .../commons/status/rpg/RemoteProcessGroupHealth.java | 12 ------------ minifi-docs/src/main/markdown/System_Admin_Guide.md | 5 ++--- .../nifi/minifi/status/TestStatusConfigReporter.java | 4 ++-- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupHealth.java b/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupHealth.java index 0994b5344..0853e92a5 100644 --- a/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupHealth.java +++ b/minifi-commons/minifi-utils/src/main/java/org/apache/nifi/minifi/commons/status/rpg/RemoteProcessGroupHealth.java @@ -20,7 +20,6 @@ public class RemoteProcessGroupHealth implements java.io.Serializable { private String transmissionStatus; private boolean hasBulletins; - private boolean hasAuthorizationIssues; private int activePortCount; private int inactivePortCount; @@ -43,14 +42,6 @@ public void setHasBulletins(boolean hasBulletins) { this.hasBulletins = hasBulletins; } - public boolean isHasAuthorizationIssues() { - return hasAuthorizationIssues; - } - - public void setHasAuthorizationIssues(boolean hasAuthorizationIssues) { - this.hasAuthorizationIssues = hasAuthorizationIssues; - } - public int getActivePortCount() { return activePortCount; } @@ -75,7 +66,6 @@ public boolean equals(Object o) { RemoteProcessGroupHealth that = (RemoteProcessGroupHealth) o; if (isHasBulletins() != that.isHasBulletins()) return false; - if (isHasAuthorizationIssues() != that.isHasAuthorizationIssues()) return false; if (getActivePortCount() != that.getActivePortCount()) return false; if (getInactivePortCount() != that.getInactivePortCount()) return false; return getTransmissionStatus() != null ? getTransmissionStatus().equals(that.getTransmissionStatus()) : that.getTransmissionStatus() == null; @@ -86,7 +76,6 @@ public boolean equals(Object o) { public int hashCode() { int result = getTransmissionStatus() != null ? getTransmissionStatus().hashCode() : 0; result = 31 * result + (isHasBulletins() ? 1 : 0); - result = 31 * result + (isHasAuthorizationIssues() ? 1 : 0); result = 31 * result + getActivePortCount(); result = 31 * result + getInactivePortCount(); return result; @@ -97,7 +86,6 @@ public String toString() { return "{" + "transmissionStatus='" + transmissionStatus + '\'' + ", hasBulletins=" + hasBulletins + - ", hasAuthorizationIssues=" + hasAuthorizationIssues + ", activePortCount=" + activePortCount + ", inactivePortCount=" + inactivePortCount + '}'; diff --git a/minifi-docs/src/main/markdown/System_Admin_Guide.md b/minifi-docs/src/main/markdown/System_Admin_Guide.md index eaa86979c..361e6011d 100644 --- a/minifi-docs/src/main/markdown/System_Admin_Guide.md +++ b/minifi-docs/src/main/markdown/System_Admin_Guide.md @@ -229,14 +229,13 @@ Option | Description ------ | ----------- health | The connections's queued bytes and queued FlowFile count. bulletins | A list of all the current bulletins (if there are any). -authorizationIssues | A list of all the current authorization issues (if there are any). inputPorts | A list of every input port for this RPG and their status. Their status includes it's name, whether the target exit and whether it's currently running. stats | The current stats of the RPG. This includes the active threads, sent content size and count. -An example query to get the health, bulletins, authorization issues, input ports and stats of all the RPGS is below. +An example query to get the health, bulletins, input ports and stats of all the RPGS is below. ``` -minifi.sh flowStatus remoteprocessinggroup:all:health,bulletins,authorizationIssues,inputports,stats +minifi.sh flowStatus remoteprocessinggroup:all:health,bulletins,inputports,stats ``` ### Controller Services diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java index d3f8f68dc..37483af86 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/status/TestStatusConfigReporter.java @@ -490,7 +490,7 @@ public void remoteProcessGroupStatusStats() throws Exception { public void remoteProcessGroupStatusAll() throws Exception { populateRemoteProcessGroup(true, true); - String statusRequest = "remoteProcessGroup:all:health, authorizationissues, bulletins, inputPorts, stats"; + String statusRequest = "remoteProcessGroup:all:health, bulletins, inputPorts, stats"; FlowStatusReport actual = StatusConfigReporter.getStatus(mockFlowController, statusRequest, LoggerFactory.getLogger(TestStatusConfigReporter.class)); FlowStatusReport expected = new FlowStatusReport(); @@ -515,7 +515,7 @@ public void statusEverything() throws Exception { String statusRequest = "controllerServices:bulletins,health; processor:all:health,stats,bulletins; instance:bulletins,health,stats ; systemDiagnostics:garbagecollection, heap, " + "processorstats, contentrepositoryusage, flowfilerepositoryusage; connection:all:health,stats; provenanceReporting:health,bulletins; remoteProcessGroup:all:health, " + - "authorizationissues, bulletins, inputPorts, stats"; + "bulletins, inputPorts, stats"; FlowStatusReport actual = StatusConfigReporter.getStatus(mockFlowController, statusRequest, LoggerFactory.getLogger(TestStatusConfigReporter.class)); From a0d2668c72ebc0841f6533a6c1e28e3bcac879eb Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 24 Oct 2016 15:59:13 -0400 Subject: [PATCH 21/33] Revert "MINIFI-115 Remove provenance reporting bundle to use the NiFi equivalent." This reverts commit a2ffa018d4b8c80d35f8cf18532843d63e19a7a9. --- .../minifi-provenance-reporting-nar/pom.xml | 40 ++ .../src/main/resources/META-INF/NOTICE | 15 + .../minifi-provenance-reporting-task/pom.xml | 82 ++++ .../reporting/ProvenanceReportingTask.java | 462 ++++++++++++++++++ .../org.apache.nifi.reporting.ReportingTask | 16 + .../TestProvenanceReportingTask.java | 186 +++++++ .../pom.xml | 46 ++ 7 files changed, 847 insertions(+) create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml new file mode 100644 index 000000000..88b5b9002 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + org.apache.nifi.minifi + minifi-provenance-reporting-bundle + 0.1.0-SNAPSHOT + + + minifi-provenance-reporting-nar + nar + + true + true + + + + org.apache.nifi.minifi + minifi-provenance-reporting-task + + + org.apache.nifi + nifi-standard-services-api-nar + nar + + + diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000..be55e5968 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,15 @@ +minifi-provenance-reporting-task-nar +Copyright 2015-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +************************ +Common Development and Distribution License 1.1 +************************ + +The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details. + + (CDDL 1.1) (GPL2 w/ CPE) JSON Processing API (javax.json:javax.json-api:jar:1.0 - http://json-processing-spec.java.net) + (CDDL 1.1) (GPL2 w/ CPE) JSON Processing Default Provider (org.glassfish:javax.json:jar:1.0.4 - https://jsonp.java.net) + diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml new file mode 100644 index 000000000..837b94f46 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml @@ -0,0 +1,82 @@ + + + + 4.0.0 + + org.apache.nifi.minifi + minifi-provenance-reporting-bundle + 0.1.0-SNAPSHOT + + + minifi-provenance-reporting-task + Publishes MiNiFi metrics to NiFi via S2S + 0.1.0-SNAPSHOT + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-processor-utils + + + org.apache.nifi + nifi-utils + + + org.apache.nifi + nifi-ssl-context-service-api + + + org.apache.nifi + nifi-site-to-site-client + + + org.glassfish + javax.json + 1.0.4 + + + javax.json + javax.json-api + 1.0 + + + + org.mockito + mockito-all + test + + + org.apache.nifi + nifi-data-provenance-utils + test + + + org.apache.nifi + nifi-mock + test + + + junit + junit + 4.12 + test + + + diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java new file mode 100644 index 000000000..a0d1776ff --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java @@ -0,0 +1,462 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.provenance.reporting; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.net.ssl.SSLContext; + +import org.apache.nifi.annotation.behavior.Stateful; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.components.state.StateMap; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.controller.status.PortStatus; +import org.apache.nifi.controller.status.ProcessGroupStatus; +import org.apache.nifi.controller.status.ProcessorStatus; +import org.apache.nifi.controller.status.RemoteProcessGroupStatus; +import org.apache.nifi.events.EventReporter; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.provenance.ProvenanceEventRecord; +import org.apache.nifi.remote.Transaction; +import org.apache.nifi.remote.TransferDirection; +import org.apache.nifi.remote.client.SiteToSiteClient; +import org.apache.nifi.reporting.AbstractReportingTask; +import org.apache.nifi.reporting.ReportingContext; +import org.apache.nifi.reporting.Severity; +import org.apache.nifi.ssl.SSLContextService; +import org.apache.nifi.ssl.SSLContextService.ClientAuth; + +@Tags({"provenance", "lineage", "tracking", "site", "site to site"}) +@CapabilityDescription("Publishes Provenance events using the Site To Site protocol.") +@Stateful(scopes = Scope.LOCAL, description = "Stores the Reporting Task's last event Id so that on restart of MiNiFi the task knows where it left off.") +public class ProvenanceReportingTask extends AbstractReportingTask { + private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + private static final String LAST_EVENT_ID_KEY = "last_event_id"; + + static final PropertyDescriptor DESTINATION_URL = new PropertyDescriptor.Builder() + .name("Destination URL") + .description("The URL to post the Provenance Events to.") + .required(true) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.URL_VALIDATOR) + .build(); + static final PropertyDescriptor PORT_NAME = new PropertyDescriptor.Builder() + .name("Input Port Name") + .description("The name of the Input Port to delivery Provenance Events to.") + .required(true) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + static final PropertyDescriptor SSL_CONTEXT = new PropertyDescriptor.Builder() + .name("SSL Context Service") + .description("The SSL Context Service to use when communicating with the destination. If not specified, communications will not be secure.") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + static final PropertyDescriptor MINIFI_URL = new PropertyDescriptor.Builder() + .name("MiNiFi URL") + .description("The URL of this MiNiFi instance. This is used to include the Content URI to send to the destination.") + .required(true) + .expressionLanguageSupported(true) + .defaultValue("http://${hostname(true)}:8080/nifi") + .addValidator(new NiFiUrlValidator()) + .build(); + static final PropertyDescriptor COMPRESS = new PropertyDescriptor.Builder() + .name("Compress Events") + .description("Indicates whether or not to compress the events when being sent.") + .required(true) + .allowableValues("true", "false") + .defaultValue("true") + .build(); + static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("Communications Timeout") + .description("Specifies how long to wait to a response from the destination before deciding that an error has occurred and canceling the transaction") + .required(true) + .defaultValue("30 secs") + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() + .name("Batch Size") + .description("Specifies how many records to send in a single batch, at most.") + .required(true) + .defaultValue("1000") + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .build(); + + private volatile long firstEventId = -1L; + private volatile SiteToSiteClient siteToSiteClient; + + @Override + protected List getSupportedPropertyDescriptors() { + final List properties = new ArrayList<>(); + properties.add(DESTINATION_URL); + properties.add(PORT_NAME); + properties.add(SSL_CONTEXT); + properties.add(MINIFI_URL); + properties.add(COMPRESS); + properties.add(TIMEOUT); + properties.add(BATCH_SIZE); + return properties; + } + + @OnScheduled + public void setup(final ConfigurationContext context) throws IOException { + final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class); + final SSLContext sslContext = sslContextService == null ? null : sslContextService.createSSLContext(ClientAuth.REQUIRED); + final EventReporter eventReporter = new EventReporter() { + @Override + public void reportEvent(final Severity severity, final String category, final String message) { + switch (severity) { + case WARNING: + getLogger().warn(message); + break; + case ERROR: + getLogger().error(message); + break; + default: + break; + } + } + }; + + final String destinationUrlPrefix = context.getProperty(DESTINATION_URL).evaluateAttributeExpressions().getValue(); + final String destinationUrl = destinationUrlPrefix + (destinationUrlPrefix.endsWith("/") ? "nifi" : "/nifi"); + + siteToSiteClient = new SiteToSiteClient.Builder() + .url(destinationUrl) + .portName(context.getProperty(PORT_NAME).getValue()) + .useCompression(context.getProperty(COMPRESS).asBoolean()) + .eventReporter(eventReporter) + .sslContext(sslContext) + .timeout(context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS) + .build(); + } + + @OnStopped + public void shutdown() throws IOException { + final SiteToSiteClient client = getClient(); + if (client != null) { + client.close(); + } + } + + // this getter is intended explicitly for testing purposes + protected SiteToSiteClient getClient() { + return this.siteToSiteClient; + } + + private String getComponentName(final ProcessGroupStatus status, final ProvenanceEventRecord event) { + if (status == null) { + return null; + } + + final String componentId = event.getComponentId(); + if (status.getId().equals(componentId)) { + return status.getName(); + } + + for (final ProcessorStatus procStatus : status.getProcessorStatus()) { + if (procStatus.getId().equals(componentId)) { + return procStatus.getName(); + } + } + + for (final PortStatus portStatus : status.getInputPortStatus()) { + if (portStatus.getId().equals(componentId)) { + return portStatus.getName(); + } + } + + for (final PortStatus portStatus : status.getOutputPortStatus()) { + if (portStatus.getId().equals(componentId)) { + return portStatus.getName(); + } + } + + for (final RemoteProcessGroupStatus rpgStatus : status.getRemoteProcessGroupStatus()) { + if (rpgStatus.getId().equals(componentId)) { + return rpgStatus.getName(); + } + } + + for (final ProcessGroupStatus childGroup : status.getProcessGroupStatus()) { + final String componentName = getComponentName(childGroup, event); + if (componentName != null) { + return componentName; + } + } + + return null; + } + + @Override + public void onTrigger(final ReportingContext context) { + final ProcessGroupStatus procGroupStatus = context.getEventAccess().getControllerStatus(); + final String rootGroupName = procGroupStatus == null ? null : procGroupStatus.getName(); + + Long currMaxId = context.getEventAccess().getProvenanceRepository().getMaxEventId(); + + if(currMaxId == null) { + getLogger().debug("No events to send because no events have been created yet."); + return; + } + + if (firstEventId < 0) { + Map state; + try { + state = context.getStateManager().getState(Scope.LOCAL).toMap(); + } catch (IOException e) { + getLogger().error("Failed to get state at start up due to {}:"+e.getMessage(), e); + return; + } + if (state.containsKey(LAST_EVENT_ID_KEY)) { + firstEventId = Long.parseLong(state.get(LAST_EVENT_ID_KEY)) + 1; + } + + if(currMaxId < firstEventId){ + getLogger().debug("Current provenance max id is {} which is less than what was stored in state as the last queried event, which was {}. This means the provenance restarted its " + + "ids. Restarting querying from the beginning.", new Object[]{currMaxId, firstEventId}); + firstEventId = -1; + } + } + + if (currMaxId == (firstEventId - 1)) { + getLogger().debug("No events to send due to the current max id being equal to the last id that was queried."); + return; + } + + final List events; + try { + events = context.getEventAccess().getProvenanceEvents(firstEventId, context.getProperty(BATCH_SIZE).asInteger()); + } catch (final IOException ioe) { + getLogger().error("Failed to retrieve Provenance Events from repository due to {}:"+ioe.getMessage(), ioe); + return; + } + + if (events == null || events.isEmpty()) { + getLogger().debug("No events to send due to 'events' being null or empty."); + return; + } + + final long start = System.nanoTime(); + final Map config = Collections.emptyMap(); + final JsonBuilderFactory factory = Json.createBuilderFactory(config); + final JsonObjectBuilder builder = factory.createObjectBuilder(); + + final String nifiUrl = context.getProperty(MINIFI_URL).evaluateAttributeExpressions().getValue(); + URL url; + try { + url = new URL(nifiUrl); + } catch (final MalformedURLException e1) { + // already validated + throw new AssertionError(); + } + + final String hostname = url.getHost(); + + final JsonArrayBuilder arrayBuilder = factory.createArrayBuilder(); + for (final ProvenanceEventRecord event : events) { + arrayBuilder.add(serialize(factory, builder, event, getComponentName(procGroupStatus, event), hostname, url, rootGroupName)); + } + final JsonArray jsonArray = arrayBuilder.build(); + + try { + final Transaction transaction = getClient().createTransaction(TransferDirection.SEND); + if (transaction == null) { + getLogger().debug("All destination nodes are penalized; will attempt to send data later"); + return; + } + + final Map attributes = new HashMap<>(); + final String transactionId = UUID.randomUUID().toString(); + attributes.put("reporting.task.transaction.id", transactionId); + + final byte[] data = jsonArray.toString().getBytes(StandardCharsets.UTF_8); + transaction.send(data, attributes); + transaction.confirm(); + transaction.complete(); + + final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + getLogger().info("Successfully sent {} Provenance Events to destination in {} ms; Transaction ID = {}; First Event ID = {}", + new Object[] {events.size(), transferMillis, transactionId, events.get(0).getEventId()}); + } catch (final IOException e) { + throw new ProcessException("Failed to send Provenance Events to destination due to IOException:" + e.getMessage(), e); + } + + final ProvenanceEventRecord lastEvent = events.get(events.size() - 1); + final String lastEventId = String.valueOf(lastEvent.getEventId()); + try { + StateManager stateManager = context.getStateManager(); + StateMap stateMap = stateManager.getState(Scope.LOCAL); + Map newMapOfState = new HashMap<>(); + newMapOfState.put(LAST_EVENT_ID_KEY, lastEventId); + stateManager.replace(stateMap, newMapOfState, Scope.LOCAL); + } catch (final IOException ioe) { + getLogger().error("Failed to update state to {} due to {}; this could result in events being re-sent after a restart of MiNiFi. The message of {} was: {}", + new Object[] {lastEventId, ioe, ioe, ioe.getMessage()}, ioe); + } + + firstEventId = lastEvent.getEventId() + 1; + } + + static JsonObject serialize(final JsonBuilderFactory factory, final JsonObjectBuilder builder, final ProvenanceEventRecord event, + final String componentName, final String hostname, final URL nifiUrl, final String applicationName) { + addField(builder, "eventId", UUID.randomUUID().toString()); + addField(builder, "eventOrdinal", event.getEventId()); + addField(builder, "eventType", event.getEventType().name()); + addField(builder, "timestampMillis", event.getEventTime()); + + final DateFormat df = new SimpleDateFormat(TIMESTAMP_FORMAT); + df.setTimeZone(TimeZone.getTimeZone("Z")); + addField(builder, "timestamp", df.format(event.getEventTime())); + + addField(builder, "durationMillis", event.getEventDuration()); + addField(builder, "lineageStart", event.getLineageStartDate()); + + final Set lineageIdentifiers = new HashSet<>(); + if (event.getLineageIdentifiers() != null) { + lineageIdentifiers.addAll(event.getLineageIdentifiers()); + } + lineageIdentifiers.add(event.getFlowFileUuid()); + addField(builder, factory, "lineageIdentifiers", lineageIdentifiers); + addField(builder, "details", event.getDetails()); + addField(builder, "componentId", event.getComponentId()); + addField(builder, "componentType", event.getComponentType()); + addField(builder, "componentName", componentName); + addField(builder, "entityId", event.getFlowFileUuid()); + addField(builder, "entityType", "org.apache.nifi.flowfile.FlowFile"); + addField(builder, "entitySize", event.getFileSize()); + addField(builder, "previousEntitySize", event.getPreviousFileSize()); + addField(builder, factory, "updatedAttributes", event.getUpdatedAttributes()); + addField(builder, factory, "previousAttributes", event.getPreviousAttributes()); + + addField(builder, "actorHostname", hostname); + if (nifiUrl != null) { + final String urlPrefix = nifiUrl.toString().replace(nifiUrl.getPath(), ""); + final String contentUriBase = urlPrefix + "/nifi-api/controller/provenance/events/" + event.getEventId() + "/content/"; + addField(builder, "contentURI", contentUriBase + "output"); + addField(builder, "previousContentURI", contentUriBase + "input"); + } + + addField(builder, factory, "parentIds", event.getParentUuids()); + addField(builder, factory, "childIds", event.getChildUuids()); + addField(builder, "transitUri", event.getTransitUri()); + addField(builder, "remoteIdentifier", event.getSourceSystemFlowFileIdentifier()); + addField(builder, "alternateIdentifier", event.getAlternateIdentifierUri()); + addField(builder, "platform", "minifi"); + addField(builder, "application", applicationName); + + return builder.build(); + } + + private static void addField(final JsonObjectBuilder builder, final JsonBuilderFactory factory, final String key, final Map values) { + if (values == null) { + return; + } + + final JsonObjectBuilder mapBuilder = factory.createObjectBuilder(); + for (final Map.Entry entry : values.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) { + continue; + } + + mapBuilder.add(entry.getKey(), entry.getValue()); + } + + builder.add(key, mapBuilder); + } + + private static void addField(final JsonObjectBuilder builder, final String key, final Long value) { + if (value != null) { + builder.add(key, value.longValue()); + } + } + + private static void addField(final JsonObjectBuilder builder, final JsonBuilderFactory factory, final String key, final Collection values) { + if (values == null) { + return; + } + + builder.add(key, createJsonArray(factory, values)); + } + + private static void addField(final JsonObjectBuilder builder, final String key, final String value) { + if (value == null) { + return; + } + + builder.add(key, value); + } + + private static JsonArrayBuilder createJsonArray(JsonBuilderFactory factory, final Collection values) { + final JsonArrayBuilder builder = factory.createArrayBuilder(); + for (final String value : values) { + if (value != null) { + builder.add(value); + } + } + return builder; + } + + + private static class NiFiUrlValidator implements Validator { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + final String value = context.newPropertyValue(input).evaluateAttributeExpressions().getValue(); + try { + new URL(value); + } catch (final Exception e) { + return new ValidationResult.Builder().input(input).subject(subject).valid(false).explanation("Not a valid URL").build(); + } + + return new ValidationResult.Builder().input(input).subject(subject).valid(true).build(); + } + } +} diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask new file mode 100644 index 000000000..331d75931 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.minifi.provenance.reporting.ProvenanceReportingTask \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java new file mode 100644 index 000000000..97291c018 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.provenance.reporting; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.provenance.ProvenanceEventBuilder; +import org.apache.nifi.provenance.ProvenanceEventRecord; +import org.apache.nifi.provenance.ProvenanceEventRepository; +import org.apache.nifi.provenance.ProvenanceEventType; +import org.apache.nifi.provenance.StandardProvenanceEventRecord; +import org.apache.nifi.remote.Transaction; +import org.apache.nifi.remote.TransferDirection; +import org.apache.nifi.remote.client.SiteToSiteClient; +import org.apache.nifi.reporting.EventAccess; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.reporting.ReportingContext; +import org.apache.nifi.reporting.ReportingInitializationContext; +import org.apache.nifi.state.MockStateManager; +import org.apache.nifi.stream.io.ByteArrayInputStream; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.MockPropertyValue; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonReader; + +public class TestProvenanceReportingTask { + + @Test + public void testSerializedForm() throws IOException, InitializationException { + final String uuid = "10000000-0000-0000-0000-000000000000"; + final Map attributes = new HashMap<>(); + attributes.put("abc", "xyz"); + attributes.put("xyz", "abc"); + attributes.put("filename", "file-" + uuid); + + final Map prevAttrs = new HashMap<>(); + attributes.put("filename", "1234.xyz"); + + final Set lineageIdentifiers = new HashSet<>(); + lineageIdentifiers.add("123"); + lineageIdentifiers.add("321"); + + final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); + builder.setEventTime(System.currentTimeMillis()); + builder.setEventType(ProvenanceEventType.RECEIVE); + builder.setTransitUri("nifi://unit-test"); + attributes.put("uuid", uuid); + builder.fromFlowFile(createFlowFile(3L, attributes)); + builder.setAttributes(prevAttrs, attributes); + builder.setComponentId("1234"); + builder.setComponentType("dummy processor"); + builder.setLineageIdentifiers(lineageIdentifiers); + final ProvenanceEventRecord event = builder.build(); + + final List dataSent = new ArrayList<>(); + final ProvenanceReportingTask task = new ProvenanceReportingTask() { + @SuppressWarnings("unchecked") + @Override + protected SiteToSiteClient getClient() { + final SiteToSiteClient client = Mockito.mock(SiteToSiteClient.class); + final Transaction transaction = Mockito.mock(Transaction.class); + + try { + Mockito.doAnswer(new Answer() { + @Override + public Object answer(final InvocationOnMock invocation) throws Throwable { + final byte[] data = invocation.getArgumentAt(0, byte[].class); + dataSent.add(data); + return null; + } + }).when(transaction).send(Mockito.any(byte[].class), Mockito.any(Map.class)); + + Mockito.when(client.createTransaction(Mockito.any(TransferDirection.class))).thenReturn(transaction); + } catch (final Exception e) { + e.printStackTrace(); + Assert.fail(e.toString()); + } + + return client; + } + }; + + final List events = new ArrayList<>(); + events.add(event); + + final Map properties = new HashMap<>(); + for (final PropertyDescriptor descriptor : task.getSupportedPropertyDescriptors()) { + properties.put(descriptor, descriptor.getDefaultValue()); + } + properties.put(ProvenanceReportingTask.BATCH_SIZE, "1000"); + + final ReportingContext context = Mockito.mock(ReportingContext.class); + Mockito.when(context.getStateManager()) + .thenReturn(new MockStateManager(task)); + Mockito.doAnswer(new Answer() { + @Override + public PropertyValue answer(final InvocationOnMock invocation) throws Throwable { + final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class); + return new MockPropertyValue(properties.get(descriptor), null); + } + }).when(context).getProperty(Mockito.any(PropertyDescriptor.class)); + + final EventAccess eventAccess = Mockito.mock(EventAccess.class); + Mockito.doAnswer(new Answer>() { + @Override + public List answer(final InvocationOnMock invocation) throws Throwable { + final long startId = invocation.getArgumentAt(0, long.class); + final int maxRecords = invocation.getArgumentAt(1, int.class); + + final List eventsToReturn = new ArrayList<>(); + for (int i = (int) Math.max(0, startId); i < (int) (startId + maxRecords) && i < events.size(); i++) { + eventsToReturn.add(events.get(i)); + } + return eventsToReturn; + } + }).when(eventAccess).getProvenanceEvents(Mockito.anyLong(), Mockito.anyInt()); + + final ProvenanceEventRepository provenanceRepository = Mockito.mock(ProvenanceEventRepository.class); + Mockito.doAnswer(new Answer() { + @Override + public Long answer(final InvocationOnMock invocation) throws Throwable { + return 1L; + } + }).when(provenanceRepository).getMaxEventId(); + + Mockito.when(context.getEventAccess()).thenReturn(eventAccess); + Mockito.when(eventAccess.getProvenanceRepository()).thenReturn(provenanceRepository); + + final ComponentLog logger = Mockito.mock(ComponentLog.class); + final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class); + Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString()); + Mockito.when(initContext.getLogger()).thenReturn(logger); + + + task.initialize(initContext); + task.onTrigger(context); + + assertEquals(1, dataSent.size()); + final String msg = new String(dataSent.get(0), StandardCharsets.UTF_8); + JsonReader jsonReader = Json.createReader(new ByteArrayInputStream(msg.getBytes())); + JsonObject msgArray = jsonReader.readArray().getJsonObject(0).getJsonObject("updatedAttributes"); + assertEquals(msgArray.getString("abc"), events.get(0).getAttributes().get("abc")); + } + + public static FlowFile createFlowFile(final long id, final Map attributes) { + MockFlowFile mockFlowFile = new MockFlowFile(id); + mockFlowFile.putAttributes(attributes); + return mockFlowFile; + } +} diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml new file mode 100644 index 000000000..f3e70c706 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + org.apache.nifi.minifi + minifi-nar-bundles + 0.1.0-SNAPSHOT + + + minifi-provenance-reporting-bundle + pom + + + minifi-provenance-reporting-task + minifi-provenance-reporting-nar + + + + + + org.apache.nifi.minifi + minifi-provenance-reporting-task + 0.1.0-SNAPSHOT + + + org.glassfish.jersey.core + jersey-client + 2.19 + + + + From 0dc9f65adf0373b22644899696086553ec0741c9 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 24 Oct 2016 16:09:45 -0400 Subject: [PATCH 22/33] Reverting back to including a separate minifi-provenance-reporting nar that makes use of the nifi-site-to-site reporting jar. Marking dependencies provided that are provided by lib. --- minifi-assembly/pom.xml | 5 - .../minifi-provenance-reporting-task/pom.xml | 82 ---- .../reporting/ProvenanceReportingTask.java | 462 ------------------ .../org.apache.nifi.reporting.ReportingTask | 16 - .../TestProvenanceReportingTask.java | 186 ------- .../pom.xml | 9 +- .../minifi-standard-nar/pom.xml | 18 + 7 files changed, 26 insertions(+), 752 deletions(-) delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 8c4caa455..7ba58e544 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -229,11 +229,6 @@ limitations under the License. jetty-server compile - - org.apache.nifi - nifi-site-to-site-reporting-nar - nar - diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml deleted file mode 100644 index 837b94f46..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - 4.0.0 - - org.apache.nifi.minifi - minifi-provenance-reporting-bundle - 0.1.0-SNAPSHOT - - - minifi-provenance-reporting-task - Publishes MiNiFi metrics to NiFi via S2S - 0.1.0-SNAPSHOT - - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-processor-utils - - - org.apache.nifi - nifi-utils - - - org.apache.nifi - nifi-ssl-context-service-api - - - org.apache.nifi - nifi-site-to-site-client - - - org.glassfish - javax.json - 1.0.4 - - - javax.json - javax.json-api - 1.0 - - - - org.mockito - mockito-all - test - - - org.apache.nifi - nifi-data-provenance-utils - test - - - org.apache.nifi - nifi-mock - test - - - junit - junit - 4.12 - test - - - diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java deleted file mode 100644 index a0d1776ff..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/java/org/apache/nifi/minifi/provenance/reporting/ProvenanceReportingTask.java +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.provenance.reporting; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonArrayBuilder; -import javax.json.JsonBuilderFactory; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import javax.net.ssl.SSLContext; - -import org.apache.nifi.annotation.behavior.Stateful; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.annotation.lifecycle.OnStopped; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.components.Validator; -import org.apache.nifi.components.state.Scope; -import org.apache.nifi.components.state.StateManager; -import org.apache.nifi.components.state.StateMap; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.status.PortStatus; -import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.controller.status.ProcessorStatus; -import org.apache.nifi.controller.status.RemoteProcessGroupStatus; -import org.apache.nifi.events.EventReporter; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.provenance.ProvenanceEventRecord; -import org.apache.nifi.remote.Transaction; -import org.apache.nifi.remote.TransferDirection; -import org.apache.nifi.remote.client.SiteToSiteClient; -import org.apache.nifi.reporting.AbstractReportingTask; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.Severity; -import org.apache.nifi.ssl.SSLContextService; -import org.apache.nifi.ssl.SSLContextService.ClientAuth; - -@Tags({"provenance", "lineage", "tracking", "site", "site to site"}) -@CapabilityDescription("Publishes Provenance events using the Site To Site protocol.") -@Stateful(scopes = Scope.LOCAL, description = "Stores the Reporting Task's last event Id so that on restart of MiNiFi the task knows where it left off.") -public class ProvenanceReportingTask extends AbstractReportingTask { - private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; - private static final String LAST_EVENT_ID_KEY = "last_event_id"; - - static final PropertyDescriptor DESTINATION_URL = new PropertyDescriptor.Builder() - .name("Destination URL") - .description("The URL to post the Provenance Events to.") - .required(true) - .expressionLanguageSupported(true) - .addValidator(StandardValidators.URL_VALIDATOR) - .build(); - static final PropertyDescriptor PORT_NAME = new PropertyDescriptor.Builder() - .name("Input Port Name") - .description("The name of the Input Port to delivery Provenance Events to.") - .required(true) - .expressionLanguageSupported(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - static final PropertyDescriptor SSL_CONTEXT = new PropertyDescriptor.Builder() - .name("SSL Context Service") - .description("The SSL Context Service to use when communicating with the destination. If not specified, communications will not be secure.") - .required(false) - .identifiesControllerService(SSLContextService.class) - .build(); - static final PropertyDescriptor MINIFI_URL = new PropertyDescriptor.Builder() - .name("MiNiFi URL") - .description("The URL of this MiNiFi instance. This is used to include the Content URI to send to the destination.") - .required(true) - .expressionLanguageSupported(true) - .defaultValue("http://${hostname(true)}:8080/nifi") - .addValidator(new NiFiUrlValidator()) - .build(); - static final PropertyDescriptor COMPRESS = new PropertyDescriptor.Builder() - .name("Compress Events") - .description("Indicates whether or not to compress the events when being sent.") - .required(true) - .allowableValues("true", "false") - .defaultValue("true") - .build(); - static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() - .name("Communications Timeout") - .description("Specifies how long to wait to a response from the destination before deciding that an error has occurred and canceling the transaction") - .required(true) - .defaultValue("30 secs") - .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) - .build(); - static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() - .name("Batch Size") - .description("Specifies how many records to send in a single batch, at most.") - .required(true) - .defaultValue("1000") - .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) - .build(); - - private volatile long firstEventId = -1L; - private volatile SiteToSiteClient siteToSiteClient; - - @Override - protected List getSupportedPropertyDescriptors() { - final List properties = new ArrayList<>(); - properties.add(DESTINATION_URL); - properties.add(PORT_NAME); - properties.add(SSL_CONTEXT); - properties.add(MINIFI_URL); - properties.add(COMPRESS); - properties.add(TIMEOUT); - properties.add(BATCH_SIZE); - return properties; - } - - @OnScheduled - public void setup(final ConfigurationContext context) throws IOException { - final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class); - final SSLContext sslContext = sslContextService == null ? null : sslContextService.createSSLContext(ClientAuth.REQUIRED); - final EventReporter eventReporter = new EventReporter() { - @Override - public void reportEvent(final Severity severity, final String category, final String message) { - switch (severity) { - case WARNING: - getLogger().warn(message); - break; - case ERROR: - getLogger().error(message); - break; - default: - break; - } - } - }; - - final String destinationUrlPrefix = context.getProperty(DESTINATION_URL).evaluateAttributeExpressions().getValue(); - final String destinationUrl = destinationUrlPrefix + (destinationUrlPrefix.endsWith("/") ? "nifi" : "/nifi"); - - siteToSiteClient = new SiteToSiteClient.Builder() - .url(destinationUrl) - .portName(context.getProperty(PORT_NAME).getValue()) - .useCompression(context.getProperty(COMPRESS).asBoolean()) - .eventReporter(eventReporter) - .sslContext(sslContext) - .timeout(context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS) - .build(); - } - - @OnStopped - public void shutdown() throws IOException { - final SiteToSiteClient client = getClient(); - if (client != null) { - client.close(); - } - } - - // this getter is intended explicitly for testing purposes - protected SiteToSiteClient getClient() { - return this.siteToSiteClient; - } - - private String getComponentName(final ProcessGroupStatus status, final ProvenanceEventRecord event) { - if (status == null) { - return null; - } - - final String componentId = event.getComponentId(); - if (status.getId().equals(componentId)) { - return status.getName(); - } - - for (final ProcessorStatus procStatus : status.getProcessorStatus()) { - if (procStatus.getId().equals(componentId)) { - return procStatus.getName(); - } - } - - for (final PortStatus portStatus : status.getInputPortStatus()) { - if (portStatus.getId().equals(componentId)) { - return portStatus.getName(); - } - } - - for (final PortStatus portStatus : status.getOutputPortStatus()) { - if (portStatus.getId().equals(componentId)) { - return portStatus.getName(); - } - } - - for (final RemoteProcessGroupStatus rpgStatus : status.getRemoteProcessGroupStatus()) { - if (rpgStatus.getId().equals(componentId)) { - return rpgStatus.getName(); - } - } - - for (final ProcessGroupStatus childGroup : status.getProcessGroupStatus()) { - final String componentName = getComponentName(childGroup, event); - if (componentName != null) { - return componentName; - } - } - - return null; - } - - @Override - public void onTrigger(final ReportingContext context) { - final ProcessGroupStatus procGroupStatus = context.getEventAccess().getControllerStatus(); - final String rootGroupName = procGroupStatus == null ? null : procGroupStatus.getName(); - - Long currMaxId = context.getEventAccess().getProvenanceRepository().getMaxEventId(); - - if(currMaxId == null) { - getLogger().debug("No events to send because no events have been created yet."); - return; - } - - if (firstEventId < 0) { - Map state; - try { - state = context.getStateManager().getState(Scope.LOCAL).toMap(); - } catch (IOException e) { - getLogger().error("Failed to get state at start up due to {}:"+e.getMessage(), e); - return; - } - if (state.containsKey(LAST_EVENT_ID_KEY)) { - firstEventId = Long.parseLong(state.get(LAST_EVENT_ID_KEY)) + 1; - } - - if(currMaxId < firstEventId){ - getLogger().debug("Current provenance max id is {} which is less than what was stored in state as the last queried event, which was {}. This means the provenance restarted its " + - "ids. Restarting querying from the beginning.", new Object[]{currMaxId, firstEventId}); - firstEventId = -1; - } - } - - if (currMaxId == (firstEventId - 1)) { - getLogger().debug("No events to send due to the current max id being equal to the last id that was queried."); - return; - } - - final List events; - try { - events = context.getEventAccess().getProvenanceEvents(firstEventId, context.getProperty(BATCH_SIZE).asInteger()); - } catch (final IOException ioe) { - getLogger().error("Failed to retrieve Provenance Events from repository due to {}:"+ioe.getMessage(), ioe); - return; - } - - if (events == null || events.isEmpty()) { - getLogger().debug("No events to send due to 'events' being null or empty."); - return; - } - - final long start = System.nanoTime(); - final Map config = Collections.emptyMap(); - final JsonBuilderFactory factory = Json.createBuilderFactory(config); - final JsonObjectBuilder builder = factory.createObjectBuilder(); - - final String nifiUrl = context.getProperty(MINIFI_URL).evaluateAttributeExpressions().getValue(); - URL url; - try { - url = new URL(nifiUrl); - } catch (final MalformedURLException e1) { - // already validated - throw new AssertionError(); - } - - final String hostname = url.getHost(); - - final JsonArrayBuilder arrayBuilder = factory.createArrayBuilder(); - for (final ProvenanceEventRecord event : events) { - arrayBuilder.add(serialize(factory, builder, event, getComponentName(procGroupStatus, event), hostname, url, rootGroupName)); - } - final JsonArray jsonArray = arrayBuilder.build(); - - try { - final Transaction transaction = getClient().createTransaction(TransferDirection.SEND); - if (transaction == null) { - getLogger().debug("All destination nodes are penalized; will attempt to send data later"); - return; - } - - final Map attributes = new HashMap<>(); - final String transactionId = UUID.randomUUID().toString(); - attributes.put("reporting.task.transaction.id", transactionId); - - final byte[] data = jsonArray.toString().getBytes(StandardCharsets.UTF_8); - transaction.send(data, attributes); - transaction.confirm(); - transaction.complete(); - - final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); - getLogger().info("Successfully sent {} Provenance Events to destination in {} ms; Transaction ID = {}; First Event ID = {}", - new Object[] {events.size(), transferMillis, transactionId, events.get(0).getEventId()}); - } catch (final IOException e) { - throw new ProcessException("Failed to send Provenance Events to destination due to IOException:" + e.getMessage(), e); - } - - final ProvenanceEventRecord lastEvent = events.get(events.size() - 1); - final String lastEventId = String.valueOf(lastEvent.getEventId()); - try { - StateManager stateManager = context.getStateManager(); - StateMap stateMap = stateManager.getState(Scope.LOCAL); - Map newMapOfState = new HashMap<>(); - newMapOfState.put(LAST_EVENT_ID_KEY, lastEventId); - stateManager.replace(stateMap, newMapOfState, Scope.LOCAL); - } catch (final IOException ioe) { - getLogger().error("Failed to update state to {} due to {}; this could result in events being re-sent after a restart of MiNiFi. The message of {} was: {}", - new Object[] {lastEventId, ioe, ioe, ioe.getMessage()}, ioe); - } - - firstEventId = lastEvent.getEventId() + 1; - } - - static JsonObject serialize(final JsonBuilderFactory factory, final JsonObjectBuilder builder, final ProvenanceEventRecord event, - final String componentName, final String hostname, final URL nifiUrl, final String applicationName) { - addField(builder, "eventId", UUID.randomUUID().toString()); - addField(builder, "eventOrdinal", event.getEventId()); - addField(builder, "eventType", event.getEventType().name()); - addField(builder, "timestampMillis", event.getEventTime()); - - final DateFormat df = new SimpleDateFormat(TIMESTAMP_FORMAT); - df.setTimeZone(TimeZone.getTimeZone("Z")); - addField(builder, "timestamp", df.format(event.getEventTime())); - - addField(builder, "durationMillis", event.getEventDuration()); - addField(builder, "lineageStart", event.getLineageStartDate()); - - final Set lineageIdentifiers = new HashSet<>(); - if (event.getLineageIdentifiers() != null) { - lineageIdentifiers.addAll(event.getLineageIdentifiers()); - } - lineageIdentifiers.add(event.getFlowFileUuid()); - addField(builder, factory, "lineageIdentifiers", lineageIdentifiers); - addField(builder, "details", event.getDetails()); - addField(builder, "componentId", event.getComponentId()); - addField(builder, "componentType", event.getComponentType()); - addField(builder, "componentName", componentName); - addField(builder, "entityId", event.getFlowFileUuid()); - addField(builder, "entityType", "org.apache.nifi.flowfile.FlowFile"); - addField(builder, "entitySize", event.getFileSize()); - addField(builder, "previousEntitySize", event.getPreviousFileSize()); - addField(builder, factory, "updatedAttributes", event.getUpdatedAttributes()); - addField(builder, factory, "previousAttributes", event.getPreviousAttributes()); - - addField(builder, "actorHostname", hostname); - if (nifiUrl != null) { - final String urlPrefix = nifiUrl.toString().replace(nifiUrl.getPath(), ""); - final String contentUriBase = urlPrefix + "/nifi-api/controller/provenance/events/" + event.getEventId() + "/content/"; - addField(builder, "contentURI", contentUriBase + "output"); - addField(builder, "previousContentURI", contentUriBase + "input"); - } - - addField(builder, factory, "parentIds", event.getParentUuids()); - addField(builder, factory, "childIds", event.getChildUuids()); - addField(builder, "transitUri", event.getTransitUri()); - addField(builder, "remoteIdentifier", event.getSourceSystemFlowFileIdentifier()); - addField(builder, "alternateIdentifier", event.getAlternateIdentifierUri()); - addField(builder, "platform", "minifi"); - addField(builder, "application", applicationName); - - return builder.build(); - } - - private static void addField(final JsonObjectBuilder builder, final JsonBuilderFactory factory, final String key, final Map values) { - if (values == null) { - return; - } - - final JsonObjectBuilder mapBuilder = factory.createObjectBuilder(); - for (final Map.Entry entry : values.entrySet()) { - if (entry.getKey() == null || entry.getValue() == null) { - continue; - } - - mapBuilder.add(entry.getKey(), entry.getValue()); - } - - builder.add(key, mapBuilder); - } - - private static void addField(final JsonObjectBuilder builder, final String key, final Long value) { - if (value != null) { - builder.add(key, value.longValue()); - } - } - - private static void addField(final JsonObjectBuilder builder, final JsonBuilderFactory factory, final String key, final Collection values) { - if (values == null) { - return; - } - - builder.add(key, createJsonArray(factory, values)); - } - - private static void addField(final JsonObjectBuilder builder, final String key, final String value) { - if (value == null) { - return; - } - - builder.add(key, value); - } - - private static JsonArrayBuilder createJsonArray(JsonBuilderFactory factory, final Collection values) { - final JsonArrayBuilder builder = factory.createArrayBuilder(); - for (final String value : values) { - if (value != null) { - builder.add(value); - } - } - return builder; - } - - - private static class NiFiUrlValidator implements Validator { - @Override - public ValidationResult validate(final String subject, final String input, final ValidationContext context) { - final String value = context.newPropertyValue(input).evaluateAttributeExpressions().getValue(); - try { - new URL(value); - } catch (final Exception e) { - return new ValidationResult.Builder().input(input).subject(subject).valid(false).explanation("Not a valid URL").build(); - } - - return new ValidationResult.Builder().input(input).subject(subject).valid(true).build(); - } - } -} diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask deleted file mode 100644 index 331d75931..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.nifi.minifi.provenance.reporting.ProvenanceReportingTask \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java deleted file mode 100644 index 97291c018..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-task/src/test/java/org/apache/nifi/minifi/provenance/reporting/TestProvenanceReportingTask.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.provenance.reporting; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.provenance.ProvenanceEventBuilder; -import org.apache.nifi.provenance.ProvenanceEventRecord; -import org.apache.nifi.provenance.ProvenanceEventRepository; -import org.apache.nifi.provenance.ProvenanceEventType; -import org.apache.nifi.provenance.StandardProvenanceEventRecord; -import org.apache.nifi.remote.Transaction; -import org.apache.nifi.remote.TransferDirection; -import org.apache.nifi.remote.client.SiteToSiteClient; -import org.apache.nifi.reporting.EventAccess; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.ReportingInitializationContext; -import org.apache.nifi.state.MockStateManager; -import org.apache.nifi.stream.io.ByteArrayInputStream; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.MockPropertyValue; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import javax.json.Json; -import javax.json.JsonObject; -import javax.json.JsonReader; - -public class TestProvenanceReportingTask { - - @Test - public void testSerializedForm() throws IOException, InitializationException { - final String uuid = "10000000-0000-0000-0000-000000000000"; - final Map attributes = new HashMap<>(); - attributes.put("abc", "xyz"); - attributes.put("xyz", "abc"); - attributes.put("filename", "file-" + uuid); - - final Map prevAttrs = new HashMap<>(); - attributes.put("filename", "1234.xyz"); - - final Set lineageIdentifiers = new HashSet<>(); - lineageIdentifiers.add("123"); - lineageIdentifiers.add("321"); - - final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); - builder.setEventTime(System.currentTimeMillis()); - builder.setEventType(ProvenanceEventType.RECEIVE); - builder.setTransitUri("nifi://unit-test"); - attributes.put("uuid", uuid); - builder.fromFlowFile(createFlowFile(3L, attributes)); - builder.setAttributes(prevAttrs, attributes); - builder.setComponentId("1234"); - builder.setComponentType("dummy processor"); - builder.setLineageIdentifiers(lineageIdentifiers); - final ProvenanceEventRecord event = builder.build(); - - final List dataSent = new ArrayList<>(); - final ProvenanceReportingTask task = new ProvenanceReportingTask() { - @SuppressWarnings("unchecked") - @Override - protected SiteToSiteClient getClient() { - final SiteToSiteClient client = Mockito.mock(SiteToSiteClient.class); - final Transaction transaction = Mockito.mock(Transaction.class); - - try { - Mockito.doAnswer(new Answer() { - @Override - public Object answer(final InvocationOnMock invocation) throws Throwable { - final byte[] data = invocation.getArgumentAt(0, byte[].class); - dataSent.add(data); - return null; - } - }).when(transaction).send(Mockito.any(byte[].class), Mockito.any(Map.class)); - - Mockito.when(client.createTransaction(Mockito.any(TransferDirection.class))).thenReturn(transaction); - } catch (final Exception e) { - e.printStackTrace(); - Assert.fail(e.toString()); - } - - return client; - } - }; - - final List events = new ArrayList<>(); - events.add(event); - - final Map properties = new HashMap<>(); - for (final PropertyDescriptor descriptor : task.getSupportedPropertyDescriptors()) { - properties.put(descriptor, descriptor.getDefaultValue()); - } - properties.put(ProvenanceReportingTask.BATCH_SIZE, "1000"); - - final ReportingContext context = Mockito.mock(ReportingContext.class); - Mockito.when(context.getStateManager()) - .thenReturn(new MockStateManager(task)); - Mockito.doAnswer(new Answer() { - @Override - public PropertyValue answer(final InvocationOnMock invocation) throws Throwable { - final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class); - return new MockPropertyValue(properties.get(descriptor), null); - } - }).when(context).getProperty(Mockito.any(PropertyDescriptor.class)); - - final EventAccess eventAccess = Mockito.mock(EventAccess.class); - Mockito.doAnswer(new Answer>() { - @Override - public List answer(final InvocationOnMock invocation) throws Throwable { - final long startId = invocation.getArgumentAt(0, long.class); - final int maxRecords = invocation.getArgumentAt(1, int.class); - - final List eventsToReturn = new ArrayList<>(); - for (int i = (int) Math.max(0, startId); i < (int) (startId + maxRecords) && i < events.size(); i++) { - eventsToReturn.add(events.get(i)); - } - return eventsToReturn; - } - }).when(eventAccess).getProvenanceEvents(Mockito.anyLong(), Mockito.anyInt()); - - final ProvenanceEventRepository provenanceRepository = Mockito.mock(ProvenanceEventRepository.class); - Mockito.doAnswer(new Answer() { - @Override - public Long answer(final InvocationOnMock invocation) throws Throwable { - return 1L; - } - }).when(provenanceRepository).getMaxEventId(); - - Mockito.when(context.getEventAccess()).thenReturn(eventAccess); - Mockito.when(eventAccess.getProvenanceRepository()).thenReturn(provenanceRepository); - - final ComponentLog logger = Mockito.mock(ComponentLog.class); - final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class); - Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString()); - Mockito.when(initContext.getLogger()).thenReturn(logger); - - - task.initialize(initContext); - task.onTrigger(context); - - assertEquals(1, dataSent.size()); - final String msg = new String(dataSent.get(0), StandardCharsets.UTF_8); - JsonReader jsonReader = Json.createReader(new ByteArrayInputStream(msg.getBytes())); - JsonObject msgArray = jsonReader.readArray().getJsonObject(0).getJsonObject("updatedAttributes"); - assertEquals(msgArray.getString("abc"), events.get(0).getAttributes().get("abc")); - } - - public static FlowFile createFlowFile(final long id, final Map attributes) { - MockFlowFile mockFlowFile = new MockFlowFile(id); - mockFlowFile.putAttributes(attributes); - return mockFlowFile; - } -} diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml index f3e70c706..3850f768a 100644 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml @@ -25,10 +25,17 @@ pom - minifi-provenance-reporting-task minifi-provenance-reporting-nar + + + org.apache.nifi + nifi-site-to-site-reporting-task + 1.0.0 + + + diff --git a/minifi-nar-bundles/minifi-standard-nar/pom.xml b/minifi-nar-bundles/minifi-standard-nar/pom.xml index 21342bb01..adb36d9fc 100644 --- a/minifi-nar-bundles/minifi-standard-nar/pom.xml +++ b/minifi-nar-bundles/minifi-standard-nar/pom.xml @@ -52,5 +52,23 @@ limitations under the License. nifi-standard-reporting-tasks ${org.apache.nifi.version} + + org.bouncycastle + bcprov-jdk15on + 1.54 + provided + + + org.bouncycastle + bcpg-jdk15on + 1.54 + provided + + + org.bouncycastle + bcpkix-jdk15on + 1.54 + provided + \ No newline at end of file From f2332c96470ede6995013bd515c148fdcbdabe95 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 26 Oct 2016 15:01:29 -0400 Subject: [PATCH 23/33] Creating custom bundles to reuse libs already provided at the root instead of duplicates with each NAR. --- minifi-assembly/pom.xml | 16 +- .../src/main/assembly/dependencies.xml | 2 + .../minifi-ssl-context-service-nar/pom.xml | 63 ++++ .../src/main/resources/META-INF/LICENSE | 337 ++++++++++++++++++ .../src/main/resources/META-INF/NOTICE | 196 ++++++++++ .../minifi-standard-nar/pom.xml | 31 +- .../minifi-standard-services-api-nar/pom.xml | 82 +++++ .../src/main/resources/META-INF/LICENSE | 337 ++++++++++++++++++ .../src/main/resources/META-INF/NOTICE | 196 ++++++++++ minifi-nar-bundles/pom.xml | 2 + pom.xml | 20 ++ 11 files changed, 1275 insertions(+), 7 deletions(-) create mode 100644 minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml create mode 100644 minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/LICENSE create mode 100644 minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/NOTICE create mode 100644 minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml create mode 100644 minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/LICENSE create mode 100644 minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/NOTICE diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 7ba58e544..93e82f417 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -156,13 +156,15 @@ limitations under the License. nifi-framework-core-api - org.apache.nifi - nifi-standard-services-api-nar + org.apache.nifi.minifi + minifi-standard-services-api-nar + 0.1.0-SNAPSHOT nar - org.apache.nifi - nifi-ssl-context-service-nar + org.apache.nifi.minifi + minifi-ssl-context-service-nar + 0.1.0-SNAPSHOT nar @@ -179,6 +181,12 @@ limitations under the License. + + org.bouncycastle + bcprov-jdk15on + 1.54 + compile + org.eclipse.jetty jetty-servlet diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index e0142c879..e87096786 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -76,6 +76,8 @@ javax.servlet-api commons-io okhttp + bcprov-jdk15on + bcpkix-jdk15on diff --git a/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml b/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml new file mode 100644 index 000000000..d2f89954b --- /dev/null +++ b/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + org.apache.nifi + nifi-ssl-context-bundle + 1.0.0 + + org.apache.nifi.minifi + minifi-ssl-context-service-nar + 0.1.0-SNAPSHOT + nar + + true + true + + + + org.apache.nifi.minifi + minifi-standard-services-api-nar + 0.1.0-SNAPSHOT + nar + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcpg-jdk15on + + + + + org.apache.nifi + nifi-ssl-context-service + + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcpg-jdk15on + + + + + diff --git a/minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/LICENSE b/minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000..a38d06b0b --- /dev/null +++ b/minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,337 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +The binary distribution of this product bundles 'Bouncy Castle JDK 1.5 Provider' +under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +This product bundes 'JCraft Jsch' which is available under a 3-Clause BSD +License. + + Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, + INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This product bundles 'JCraft Jzlib' which is available under a 3-Clause BSD License. + + Copyright (c) 2002-2014 Atsuhiko Yamanaka, JCraft,Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, + INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This product bundles 'asm' which is available under a 3-Clause BSD style license. +For details see http://asm.ow2.org/asmdex-license.html + + Copyright (c) 2012 France Télécom + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles 'jBCrypt' which is available under a BSD license. For details see https://github.com/svenkubiak/jBCrypt/blob/0.4.1/LICENSE + + Copyright (c) 2006 Damien Miller + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/NOTICE b/minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000..5718321dc --- /dev/null +++ b/minifi-nar-bundles/minifi-ssl-context-service-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,196 @@ +minifi-standard-nar +Copyright 2014-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Apache Commons IO + The following NOTICE information applies: + Apache Commons IO + Copyright 2002-2012 The Apache Software Foundation + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2015 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Apache Commons Net + The following NOTICE information applies: + Apache Commons Net + Copyright 2001-2013 The Apache Software Foundation + + (ASLv2) Apache Commons Compress + The following NOTICE information applies: + Apache Commons Compress + Copyright 2002-2014 The Apache Software Foundation + + The files in the package org.apache.commons.compress.archivers.sevenz + were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), + which has been placed in the public domain: + + "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + + (ASLv2) Apache HttpComponents + The following NOTICE information applies: + Apache HttpClient + Copyright 1999-2015 The Apache Software Foundation + + Apache HttpCore + Copyright 2005-2015 The Apache Software Foundation + + This project contains annotations derived from JCIP-ANNOTATIONS + Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net + + (ASLv2) Apache Commons Logging + The following NOTICE information applies: + Apache Commons Logging + Copyright 2003-2014 The Apache Software Foundation + + (ASLv2) Joda Time + The following NOTICE information applies: + This product includes software developed by + Joda.org (http://www.joda.org/). + + (ASLv2) Apache ActiveMQ + The following NOTICE information applies: + ActiveMQ :: Client + Copyright 2005-2015 The Apache Software Foundation + + (ASLv2) Apache Geronimo + The following NOTICE information applies: + Apache Geronimo + Copyright 2003-2008 The Apache Software Foundation + + (ASLv2) Yammer Metrics + The following NOTICE information applies: + Metrics + Copyright 2010-2012 Coda Hale and Yammer, Inc. + + This product includes software developed by Coda Hale and Yammer, Inc. + + This product includes code derived from the JSR-166 project (ThreadLocalRandom), which was released + with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + + (ASLv2) JSON-SMART + The following NOTICE information applies: + Copyright 2011 JSON-SMART authors + + (ASLv2) JsonPath + The following NOTICE information applies: + Copyright 2011 JsonPath authors + + (ASLv2) Apache Commons Codec + The following NOTICE information applies: + Apache Commons Codec + Copyright 2002-2014 The Apache Software Foundation + + src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java + contains test data from http://aspell.net/test/orig/batch0.tab. + Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + + =============================================================================== + + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright: + Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + + (ASLv2) Apache Tika + The following NOTICE information applies: + Apache Tika Core + Copyright 2007-2015 The Apache Software Foundation + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + (ASLv2) Apache Avro + The following NOTICE information applies: + Apache Avro + Copyright 2009-2013 The Apache Software Foundation + + (ASLv2) Snappy Java + The following NOTICE information applies: + This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + +************************ +Common Development and Distribution License 1.1 +************************ + +The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details. + + (CDDL 1.1) (GPL2 w/ CPE) jersey-core (com.sun.jersey:jersey-core:jar:1.19 - https://jersey.java.net/jersey-core/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-client (com.sun.jersey:jersey-client:jar:1.19 - https://jersey.java.net/jersey-client/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-server (com.sun.jersey:jersey-server:jar:1.19 - https://jersey.java.net/jersey-server/) + (CDDL 1.1) (GPL2 w/ CPE) JavaMail API (compat) (javax.mail:mail:jar:1.4.7 - http://kenai.com/projects/javamail/mail) + (CDDL 1.1) (GPL2 w/ CPE) Javax JMS Api (javax.jms:javax.jms-api:jar:2.0.1 - http://java.net/projects/jms-spec/pages/Home) + +***************** +Common Development and Distribution License v1.0: +***************** + +The following binary components are provided under the Common Development and Distribution License v1.0. See project link for details. + + (CDDL 1.0) JavaBeans Activation Framework (JAF) (javax.activation:activation:jar:1.1 - http://java.sun.com/products/javabeans/jaf/index.jsp) + (CDDL 1.0) JSR311 API (javax.ws.rs:jsr311-api:jar:1.1.1 - https://jsr311.dev.java.net) + +***************** +Mozilla Public License v2.0 +***************** + +The following binary components are provided under the Mozilla Public License v2.0. See project link for details. + + (MPL 2.0) Saxon HE (net.sf.saxon:Saxon-HE:jar:9.6.0-5 - http://www.saxonica.com/) + +***************** +Public Domain +***************** + +The following binary components are provided to the 'Public Domain'. See project link for details. + + (Public Domain) XZ for Java (org.tukaani:xz:jar:1.5 - http://tukaani.org/xz/java.html + + + diff --git a/minifi-nar-bundles/minifi-standard-nar/pom.xml b/minifi-nar-bundles/minifi-standard-nar/pom.xml index adb36d9fc..2040c7b1c 100644 --- a/minifi-nar-bundles/minifi-standard-nar/pom.xml +++ b/minifi-nar-bundles/minifi-standard-nar/pom.xml @@ -32,10 +32,10 @@ limitations under the License. - org.apache.nifi - nifi-standard-services-api-nar + org.apache.nifi.minifi + minifi-standard-services-api-nar nar - ${org.apache.nifi.version} + 0.1.0-SNAPSHOT org.apache.nifi @@ -52,6 +52,7 @@ limitations under the License. nifi-standard-reporting-tasks ${org.apache.nifi.version} + org.bouncycastle bcprov-jdk15on @@ -70,5 +71,29 @@ limitations under the License. 1.54 provided + + org.codehaus.jackson + jackson-mapper-asl + 1.9.13 + provided + + + com.fasterxml.jackson.core + jackson-databind + 2.6.1 + provided + + + org.apache.httpcomponents + httpclient + 4.4.1 + provided + + + org.apache.commons + commons-lang3 + 3.4 + provided + \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml new file mode 100644 index 000000000..e3bb0e3c6 --- /dev/null +++ b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + + org.apache.nifi + nifi-standard-services + 1.0.0 + + org.apache.nifi.minifi + minifi-standard-services-api-nar + 0.1.0-SNAPSHOT + nar + + true + true + + + + org.apache.nifi + nifi-ssl-context-service-api + compile + + + org.apache.nifi + nifi-distributed-cache-client-service-api + compile + + + org.apache.nifi + nifi-load-distribution-service-api + compile + + + org.apache.nifi + nifi-http-context-map-api + compile + + + org.apache.nifi + nifi-dbcp-service-api + compile + + + org.apache.nifi + nifi-hbase-client-service-api + compile + + + + org.bouncycastle + bcprov-jdk15on + 1.54 + provided + + + org.bouncycastle + bcpg-jdk15on + 1.54 + provided + + + org.bouncycastle + bcpkix-jdk15on + 1.54 + provided + + + diff --git a/minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/LICENSE b/minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000..a38d06b0b --- /dev/null +++ b/minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,337 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +The binary distribution of this product bundles 'Bouncy Castle JDK 1.5 Provider' +under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +This product bundes 'JCraft Jsch' which is available under a 3-Clause BSD +License. + + Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, + INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This product bundles 'JCraft Jzlib' which is available under a 3-Clause BSD License. + + Copyright (c) 2002-2014 Atsuhiko Yamanaka, JCraft,Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, + INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This product bundles 'asm' which is available under a 3-Clause BSD style license. +For details see http://asm.ow2.org/asmdex-license.html + + Copyright (c) 2012 France Télécom + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles 'jBCrypt' which is available under a BSD license. For details see https://github.com/svenkubiak/jBCrypt/blob/0.4.1/LICENSE + + Copyright (c) 2006 Damien Miller + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/NOTICE b/minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000..5718321dc --- /dev/null +++ b/minifi-nar-bundles/minifi-standard-services-api-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,196 @@ +minifi-standard-nar +Copyright 2014-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Apache Commons IO + The following NOTICE information applies: + Apache Commons IO + Copyright 2002-2012 The Apache Software Foundation + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2015 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Apache Commons Net + The following NOTICE information applies: + Apache Commons Net + Copyright 2001-2013 The Apache Software Foundation + + (ASLv2) Apache Commons Compress + The following NOTICE information applies: + Apache Commons Compress + Copyright 2002-2014 The Apache Software Foundation + + The files in the package org.apache.commons.compress.archivers.sevenz + were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), + which has been placed in the public domain: + + "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + + (ASLv2) Apache HttpComponents + The following NOTICE information applies: + Apache HttpClient + Copyright 1999-2015 The Apache Software Foundation + + Apache HttpCore + Copyright 2005-2015 The Apache Software Foundation + + This project contains annotations derived from JCIP-ANNOTATIONS + Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net + + (ASLv2) Apache Commons Logging + The following NOTICE information applies: + Apache Commons Logging + Copyright 2003-2014 The Apache Software Foundation + + (ASLv2) Joda Time + The following NOTICE information applies: + This product includes software developed by + Joda.org (http://www.joda.org/). + + (ASLv2) Apache ActiveMQ + The following NOTICE information applies: + ActiveMQ :: Client + Copyright 2005-2015 The Apache Software Foundation + + (ASLv2) Apache Geronimo + The following NOTICE information applies: + Apache Geronimo + Copyright 2003-2008 The Apache Software Foundation + + (ASLv2) Yammer Metrics + The following NOTICE information applies: + Metrics + Copyright 2010-2012 Coda Hale and Yammer, Inc. + + This product includes software developed by Coda Hale and Yammer, Inc. + + This product includes code derived from the JSR-166 project (ThreadLocalRandom), which was released + with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + + (ASLv2) JSON-SMART + The following NOTICE information applies: + Copyright 2011 JSON-SMART authors + + (ASLv2) JsonPath + The following NOTICE information applies: + Copyright 2011 JsonPath authors + + (ASLv2) Apache Commons Codec + The following NOTICE information applies: + Apache Commons Codec + Copyright 2002-2014 The Apache Software Foundation + + src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java + contains test data from http://aspell.net/test/orig/batch0.tab. + Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + + =============================================================================== + + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright: + Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + + (ASLv2) Apache Tika + The following NOTICE information applies: + Apache Tika Core + Copyright 2007-2015 The Apache Software Foundation + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + (ASLv2) Apache Avro + The following NOTICE information applies: + Apache Avro + Copyright 2009-2013 The Apache Software Foundation + + (ASLv2) Snappy Java + The following NOTICE information applies: + This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + +************************ +Common Development and Distribution License 1.1 +************************ + +The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details. + + (CDDL 1.1) (GPL2 w/ CPE) jersey-core (com.sun.jersey:jersey-core:jar:1.19 - https://jersey.java.net/jersey-core/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-client (com.sun.jersey:jersey-client:jar:1.19 - https://jersey.java.net/jersey-client/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-server (com.sun.jersey:jersey-server:jar:1.19 - https://jersey.java.net/jersey-server/) + (CDDL 1.1) (GPL2 w/ CPE) JavaMail API (compat) (javax.mail:mail:jar:1.4.7 - http://kenai.com/projects/javamail/mail) + (CDDL 1.1) (GPL2 w/ CPE) Javax JMS Api (javax.jms:javax.jms-api:jar:2.0.1 - http://java.net/projects/jms-spec/pages/Home) + +***************** +Common Development and Distribution License v1.0: +***************** + +The following binary components are provided under the Common Development and Distribution License v1.0. See project link for details. + + (CDDL 1.0) JavaBeans Activation Framework (JAF) (javax.activation:activation:jar:1.1 - http://java.sun.com/products/javabeans/jaf/index.jsp) + (CDDL 1.0) JSR311 API (javax.ws.rs:jsr311-api:jar:1.1.1 - https://jsr311.dev.java.net) + +***************** +Mozilla Public License v2.0 +***************** + +The following binary components are provided under the Mozilla Public License v2.0. See project link for details. + + (MPL 2.0) Saxon HE (net.sf.saxon:Saxon-HE:jar:9.6.0-5 - http://www.saxonica.com/) + +***************** +Public Domain +***************** + +The following binary components are provided to the 'Public Domain'. See project link for details. + + (Public Domain) XZ for Java (org.tukaani:xz:jar:1.5 - http://tukaani.org/xz/java.html + + + diff --git a/minifi-nar-bundles/pom.xml b/minifi-nar-bundles/pom.xml index 0721b41d7..a02b75b4a 100644 --- a/minifi-nar-bundles/pom.xml +++ b/minifi-nar-bundles/pom.xml @@ -28,5 +28,7 @@ minifi-provenance-repository-bundle minifi-update-attribute-nar minifi-standard-nar + minifi-standard-services-api-nar + minifi-ssl-context-service-nar diff --git a/pom.xml b/pom.xml index 8f9285ec4..185ca6d7b 100644 --- a/pom.xml +++ b/pom.xml @@ -509,6 +509,26 @@ limitations under the License. nar + + + org.bouncycastle + bcprov-jdk15on + 1.54 + provided + + + org.bouncycastle + bcpg-jdk15on + 1.54 + provided + + + org.bouncycastle + bcpkix-jdk15on + 1.54 + provided + + junit From cdf8a39b4cf3da3677d5237d01229c2303b6d258 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 26 Oct 2016 17:15:55 -0400 Subject: [PATCH 24/33] Removing duplicated libraries and reusing bootstrap libraries where possible/compatible. --- .../src/main/assembly/dependencies.xml | 8 +++++ .../nifi/minifi/bootstrap/RunMiNiFi.java | 34 ++++++++++++++----- .../minifi-standard-nar/pom.xml | 12 +++++++ 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index e87096786..c19dc5d31 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -35,6 +35,7 @@ minifi-bootstrap minifi-resources + bcprov-jdk15on spring-aop zookeeper @@ -50,6 +51,13 @@ mail log4j lucene-analyzers-common + nifi-utils + logback-classic + logback-core + jetty-server + jetty-util + jetty-http + jetty-io lucene-queryparser commons-net diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java index 52a803ce9..5c3901bba 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java @@ -62,6 +62,10 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.io.input.TeeInputStream; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; @@ -77,6 +81,7 @@ import org.slf4j.LoggerFactory; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.stream.Collectors.toList; /** *

@@ -985,7 +990,14 @@ public Tuple startMiNiFi() throws IOException, Interrup } final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim(); - File libDir = getFile(libFilename, workingDir); + final String bootstrapLibDirectory = "./lib/bootstrap"; + + final File bootstrapLibDir = getFile(bootstrapLibDirectory, workingDir); + final File libDir = getFile(libFilename, workingDir); + + final List classpathDirectories = new ArrayList<>(2); + classpathDirectories.add(bootstrapLibDir); + classpathDirectories.add(libDir); final String confFilename = replaceNull(props.get(CONF_DIR_KEY), "./conf").trim(); File confDir = getFile(confFilename, workingDir); @@ -1011,14 +1023,18 @@ public Tuple startMiNiFi() throws IOException, Interrup } } - final File[] libFiles = libDir.listFiles(new FilenameFilter() { - @Override - public boolean accept(final File dir, final String filename) { - return filename.toLowerCase().endsWith(".jar"); - } - }); + final List libFiles = classpathDirectories + .stream() + .flatMap(new Function>() { + @Override + public Stream apply(File file) { + return Arrays.stream(file.listFiles((dir, name) -> { + return name.toLowerCase().endsWith(".jar"); + })); + } + }).collect(Collectors.toList()); - if (libFiles == null || libFiles.length == 0) { + if (libFiles == null || libFiles.size() == 0) { throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath()); } @@ -1027,7 +1043,7 @@ public boolean accept(final File dir, final String filename) { throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath()); } - final List cpFiles = new ArrayList<>(confFiles.length + libFiles.length); + final List cpFiles = new ArrayList<>(confFiles.length + libFiles.size()); cpFiles.add(confDir.getAbsolutePath()); for (final File file : libFiles) { cpFiles.add(file.getAbsolutePath()); diff --git a/minifi-nar-bundles/minifi-standard-nar/pom.xml b/minifi-nar-bundles/minifi-standard-nar/pom.xml index 2040c7b1c..bfafc2b11 100644 --- a/minifi-nar-bundles/minifi-standard-nar/pom.xml +++ b/minifi-nar-bundles/minifi-standard-nar/pom.xml @@ -53,6 +53,18 @@ limitations under the License. ${org.apache.nifi.version} + + com.sun.jersey + jersey-server + provided + 1.19 + + + com.google.guava + guava + 18.0 + provided + org.bouncycastle bcprov-jdk15on From 108ba8c84e9dc06df629a13277dbb77426d56f60 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Tue, 8 Nov 2016 13:09:31 -0600 Subject: [PATCH 25/33] Migrating shared libraries out of bootstrap into lib. --- minifi-assembly/pom.xml | 5 - .../src/main/assembly/dependencies.xml | 22 +- .../nifi/minifi/bootstrap/RunMiNiFi.java | 34 +- .../minifi-framework-nar/pom.xml | 59 +- .../minifi-framework-core/pom.xml | 42 + .../src/main/resources/bin/minifi.sh | 11 +- pom.xml | 1200 +++++++++++++++++ 7 files changed, 1301 insertions(+), 72 deletions(-) diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 93e82f417..e83e8c16c 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -202,11 +202,6 @@ limitations under the License. jetty-servlets compile - - org.eclipse.jetty - jetty-jsp - compile - javax.servlet.jsp javax.servlet.jsp-api diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index c19dc5d31..01830f26f 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -35,7 +35,6 @@ minifi-bootstrap minifi-resources - bcprov-jdk15on spring-aop zookeeper @@ -51,13 +50,6 @@ mail log4j lucene-analyzers-common - nifi-utils - logback-classic - logback-core - jetty-server - jetty-util - jetty-http - jetty-io lucene-queryparser commons-net @@ -73,19 +65,13 @@ true minifi-bootstrap - slf4j-api - logback-classic - nifi-api + minifi-api + minifi-commons-schema + minifi-utils + snakeyaml nifi-utils - jetty-server - jetty-util - jetty-http - jetty-io - javax.servlet-api commons-io okhttp - bcprov-jdk15on - bcpkix-jdk15on diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java index 5c3901bba..52a803ce9 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java @@ -62,10 +62,6 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.stream.Collector; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.io.input.TeeInputStream; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; @@ -81,7 +77,6 @@ import org.slf4j.LoggerFactory; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static java.util.stream.Collectors.toList; /** *

@@ -990,14 +985,7 @@ public Tuple startMiNiFi() throws IOException, Interrup } final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim(); - final String bootstrapLibDirectory = "./lib/bootstrap"; - - final File bootstrapLibDir = getFile(bootstrapLibDirectory, workingDir); - final File libDir = getFile(libFilename, workingDir); - - final List classpathDirectories = new ArrayList<>(2); - classpathDirectories.add(bootstrapLibDir); - classpathDirectories.add(libDir); + File libDir = getFile(libFilename, workingDir); final String confFilename = replaceNull(props.get(CONF_DIR_KEY), "./conf").trim(); File confDir = getFile(confFilename, workingDir); @@ -1023,18 +1011,14 @@ public Tuple startMiNiFi() throws IOException, Interrup } } - final List libFiles = classpathDirectories - .stream() - .flatMap(new Function>() { - @Override - public Stream apply(File file) { - return Arrays.stream(file.listFiles((dir, name) -> { - return name.toLowerCase().endsWith(".jar"); - })); - } - }).collect(Collectors.toList()); + final File[] libFiles = libDir.listFiles(new FilenameFilter() { + @Override + public boolean accept(final File dir, final String filename) { + return filename.toLowerCase().endsWith(".jar"); + } + }); - if (libFiles == null || libFiles.size() == 0) { + if (libFiles == null || libFiles.length == 0) { throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath()); } @@ -1043,7 +1027,7 @@ public Stream apply(File file) { throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath()); } - final List cpFiles = new ArrayList<>(confFiles.length + libFiles.size()); + final List cpFiles = new ArrayList<>(confFiles.length + libFiles.length); cpFiles.add(confDir.getAbsolutePath()); for (final File file : libFiles) { cpFiles.add(file.getAbsolutePath()); diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml index b46004319..399b72193 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml @@ -34,51 +34,70 @@ limitations under the License. true - + + org.apache.nifi.minifi + minifi-framework-core + provided + + + org.apache.nifi.minifi + minifi-api + provided + + + org.apache.nifi nifi-api - provided + compile org.apache.nifi nifi-runtime - provided + compile org.apache.nifi nifi-nar-utils - provided + compile org.apache.nifi nifi-properties - provided + compile - org.apache.nifi.minifi - minifi-framework-core - provided + org.apache.nifi + nifi-security + compile + 1.0.0 - org.apache.nifi - nifi-administration - provided + org.eclipse.jetty + jetty-server - org.apache.nifi.minifi - minifi-api - provided + org.eclipse.jetty + jetty-servlet - org.apache.nifi - nifi-framework-core - provided + org.eclipse.jetty + jetty-webapp - org.apache.nifi - nifi-framework-core-api - provided + org.eclipse.jetty + jetty-annotations + + + org.eclipse.jetty + jetty-servlets + + + cglib + cglib-nodep + compile + + \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml index 1900ca790..d89f1c553 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml @@ -26,6 +26,48 @@ limitations under the License. minifi-framework-core + + org.eclipse.jetty + jetty-server + 9.3.9.v20160517 + + + + org.eclipse.jetty + jetty-servlet + 9.3.9.v20160517 + + + + org.eclipse.jetty + jetty-webapp + 9.3.9.v20160517 + + + + org.eclipse.jetty + jetty-servlets + 9.3.9.v20160517 + + + + org.eclipse.jetty + jetty-annotations + 9.3.9.v20160517 + + + + org.eclipse.jetty + apache-jsp + 9.3.9.v20160517 + + + + org.eclipse.jetty + apache-jstl + 9.3.9.v20160517 + + org.apache.nifi.minifi minifi-nar-utils diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh index 048247229..45fabc70c 100755 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh @@ -162,6 +162,7 @@ install() { run() { BOOTSTRAP_CONF_DIR="${MINIFI_HOME}/conf" BOOTSTRAP_CONF="${BOOTSTRAP_CONF_DIR}/bootstrap.conf"; + MINIFI_LIBS="${MINIFI_HOME}/lib/*" BOOTSTRAP_LIBS="${MINIFI_HOME}/lib/bootstrap/*" run_as=$(grep run.as "${BOOTSTRAP_CONF}" | cut -d'=' -f2) @@ -181,10 +182,11 @@ run() { BOOTSTRAP_CONF=$(cygpath --path --windows "${BOOTSTRAP_CONF}") BOOTSTRAP_CONF_DIR=$(cygpath --path --windows "${BOOTSTRAP_CONF_DIR}") BOOTSTRAP_LIBS=$(cygpath --path --windows "${BOOTSTRAP_LIBS}") - BOOTSTRAP_CLASSPATH="${BOOTSTRAP_CONF_DIR};${BOOTSTRAP_LIBS}" + MINIFI_LIBS=$(cygpath --path --windows "${MINIFI_LIBS}") + BOOTSTRAP_CLASSPATH="${BOOTSTRAP_CONF_DIR};${BOOTSTRAP_LIBS};${MINIFI_LIBS}" if [ -n "${TOOLS_JAR}" ]; then TOOLS_JAR=$(cygpath --path --windows "${TOOLS_JAR}") - BOOTSTRAP_CLASSPATH="${TOOLS_JAR};${BOOTSTRAP_CLASSPATH}" + BOOTSTRAP_CLASSPATH="${TOOLS_JAR};${BOOTSTRAP_CLASSPATH};${MINIFI_LIBS}" fi else if [ -n "${run_as}" ]; then @@ -195,13 +197,14 @@ run() { exit 1 fi fi; - BOOTSTRAP_CLASSPATH="${BOOTSTRAP_CONF_DIR}:${BOOTSTRAP_LIBS}" + BOOTSTRAP_CLASSPATH="${BOOTSTRAP_CONF_DIR}:${BOOTSTRAP_LIBS}:${MINIFI_LIBS}" if [ -n "${TOOLS_JAR}" ]; then - BOOTSTRAP_CLASSPATH="${TOOLS_JAR}:${BOOTSTRAP_CLASSPATH}" + BOOTSTRAP_CLASSPATH="${TOOLS_JAR}:${BOOTSTRAP_CLASSPATH}:${MINIFI_LIBS}" fi fi echo + echo "Bootstrap Classpath: ${BOOTSTRAP_CLASSPATH}" echo "Java home: ${JAVA_HOME}" echo "MiNiFi home: ${MINIFI_HOME}" echo diff --git a/pom.xml b/pom.xml index 185ca6d7b..be771dd01 100644 --- a/pom.xml +++ b/pom.xml @@ -97,6 +97,21 @@ limitations under the License. 1.7.12 1.0.0 9.2.11.v20150529 + 1.7.12 + 9.3.9.v20160517 + 4.10.4 + 4.2.4.RELEASE + 4.0.3.RELEASE + 1.19 + 2.6.2 + 12.0.1 + 4.2.5 + 2.2.0 + 0.6.0 + 1.2.1 + 2.6.2 + 1.1.2 + 1.0.1 @@ -551,6 +566,1191 @@ limitations under the License. okhttp 3.4.1 + + ch.qos.logback + logback-classic + 1.1.3 + + + ch.qos.logback + jcl-over-slf4j + 1.1.3 + provided + + + org.slf4j + slf4j-api + + + + + org.slf4j + jcl-over-slf4j + ${org.slf4j.version} + provided + + + org.slf4j + log4j-over-slf4j + ${org.slf4j.version} + provided + + + org.slf4j + jul-to-slf4j + ${org.slf4j.version} + provided + + + org.slf4j + slf4j-api + ${org.slf4j.version} + provided + + + junit + junit + 4.12 + + + org.mockito + mockito-core + 1.10.19 + + + org.mockito + mockito-all + 1.10.19 + test + + + org.slf4j + slf4j-simple + ${org.slf4j.version} + + + org.apache.commons + commons-compress + 1.11 + + + org.apache.commons + commons-lang3 + 3.4 + + + org.antlr + antlr-runtime + 3.5.2 + + + org.mongodb + mongo-java-driver + 3.2.2 + + + com.wordnik + swagger-annotations + 1.5.3-M1 + + + org.apache.ignite + ignite-core + 1.6.0 + + + org.apache.ignite + ignite-spring + 1.6.0 + + + org.apache.ignite + ignite-log4j2 + 1.6.0 + + + commons-cli + commons-cli + 1.3.1 + + + commons-codec + commons-codec + 1.10 + + + commons-net + commons-net + 3.3 + + + commons-io + commons-io + 2.5 + + + org.bouncycastle + bcprov-jdk15on + 1.54 + provided + + + org.bouncycastle + bcpg-jdk15on + 1.54 + + + org.bouncycastle + bcpkix-jdk15on + 1.54 + provided + + + com.jcraft + jsch + 0.1.52 + + + org.apache.httpcomponents + httpclient + 4.4.1 + + + javax.mail + mail + 1.4.7 + + + com.github.jponge + lzma-java + 1.3 + + + org.tukaani + xz + 1.5 + + + net.sf.saxon + Saxon-HE + 9.6.0-5 + + + stax + stax-api + 1.0.1 + + + org.quartz-scheduler + quartz + 2.2.1 + + + + c3p0 + c3p0 + + + + + com.sun.jersey.contribs + jersey-spring + ${jersey.version} + + + org.springframework + spring-web + + + org.springframework + spring-core + + + org.springframework + spring-aop + + + org.springframework + spring-context + + + org.springframework + spring-beans + + + + + com.sun.jersey.contribs + jersey-multipart + ${jersey.version} + + + org.springframework + spring-beans + ${spring.version} + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-context-support + ${spring.version} + + + org.springframework + spring-expression + ${spring.version} + + + org.springframework + spring-tx + ${spring.version} + + + org.springframework + spring-core + ${spring.version} + + + + commons-logging + commons-logging + + + + + org.springframework + spring-web + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-jdbc + ${spring.version} + + + org.springframework.security + spring-security-core + ${spring.security.version} + + + org.springframework + spring-web + + + org.springframework + spring-core + + + org.springframework + spring-expression + + + org.springframework + spring-aop + + + org.springframework + spring-context + + + org.springframework + spring-beans + + + + + org.springframework.security + spring-security-web + ${spring.security.version} + + + org.springframework + spring-core + + + org.springframework + spring-tx + + + org.springframework + spring-jdbc + + + org.springframework + spring-expression + + + org.springframework + spring-web + + + org.springframework + spring-aop + + + org.springframework + spring-context + + + org.springframework + spring-beans + + + + + org.springframework.security + spring-security-acl + ${spring.security.version} + + + org.springframework + spring-core + + + org.springframework + spring-tx + + + org.springframework + spring-jdbc + + + org.springframework + spring-aop + + + org.springframework + spring-context + + + + + org.springframework.security + spring-security-config + ${spring.security.version} + + + org.springframework + spring-core + + + org.springframework + spring-aop + + + org.springframework + spring-context + + + org.springframework + spring-beans + + + + + org.springframework.security + spring-security-ldap + ${spring.security.version} + + + org.springframework + spring-core + + + org.springframework + spring-beans + + + org.springframework + spring-context + + + org.springframework + spring-tx + + + + + org.springframework.security.kerberos + spring-security-kerberos-core + 1.0.1.RELEASE + + + org.aspectj + aspectjweaver + 1.8.5 + + + cglib + cglib-nodep + 3.2.2 + + + org.apache.commons + commons-collections4 + 4.0 + + + org.apache.commons + commons-pool2 + 2.4.1 + + + com.jcraft + jzlib + 1.1.3 + + + joda-time + joda-time + 2.8.2 + + + com.yammer.metrics + metrics-ganglia + ${yammer.metrics.version} + + + com.yammer.metrics + metrics-core + ${yammer.metrics.version} + + + javax.jms + javax.jms-api + 2.0.1 + + + org.apache.activemq + activemq-client + 5.12.1 + + + org.apache.activemq + activemq-broker + 5.12.1 + test + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + org.apache.avro + avro + 1.7.7 + + + com.sun.jersey + jersey-server + ${jersey.version} + + + com.sun.jersey + jersey-core + ${jersey.version} + + + com.sun.jersey + jersey-json + ${jersey.version} + + + com.sun.jersey + jersey-servlet + ${jersey.version} + + + com.sun.jersey + jersey-client + ${jersey.version} + + + org.eclipse.jetty + jetty-server + ${jetty.version} + provided + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + provided + + + org.eclipse.jetty + jetty-webapp + ${jetty.version} + provided + + + org.eclipse.jetty + jetty-servlets + ${jetty.version} + provided + + + org.eclipse.jetty + jetty-annotations + ${jetty.version} + provided + + + org.eclipse.jetty + apache-jsp + ${jetty.version} + provided + + + org.eclipse.jetty + apache-jstl + ${jetty.version} + provided + + + javax.servlet + javax.servlet-api + 3.1.0 + provided + + + com.google.guava + guava + 18.0 + + + com.h2database + h2 + 1.3.176 + + + org.jasypt + jasypt + 1.9.2 + + + org.codehaus.jackson + jackson-mapper-asl + 1.9.13 + + + com.fasterxml.jackson.core + jackson-databind + 2.6.1 + + + org.apache.spark + spark-streaming_2.10 + 1.6.0 + + + org.apache.zookeeper + zookeeper + 3.4.6 + + + org.apache.curator + curator-framework + 2.11.0 + + + org.apache.curator + curator-recipes + 2.11.0 + + + org.slf4j + slf4j-log4j12 + ${org.slf4j.version} + + + + + + org.apache.curator + curator-test + 2.9.1 + test + + + org.testng + testng + 6.8.8 + test + + + + + org.jsoup + jsoup + 1.8.3 + + + + org.apache.nifi + nifi-api + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-framework-api + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-site-to-site-client + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-web-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-expression-language + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-custom-ui-utilities + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-ui-extension + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-flowfile-packager + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-socket-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-data-provenance-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-runtime + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-bootstrap + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-toolkit-tls + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-toolkit-encrypt-config + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-toolkit-s2s + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-resources + ${org.apache.nifi.version} + resources + runtime + zip + + + org.apache.nifi + nifi-docs + ${org.apache.nifi.version} + resources + runtime + zip + + + org.apache.nifi + nifi-framework-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-provenance-repository-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-standard-services-api-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-ssl-context-service-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-ssl-context-service-api + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-distributed-cache-services-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-standard-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-jetty-bundle + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-update-attribute-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-hadoop-libraries-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-hadoop-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-html-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-kite-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-mongodb-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-solr-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-kafka-0-8-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-kafka-0-9-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-kafka-0-10-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-ignite-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-http-context-map-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-social-media-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-hl7-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-language-translation-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-enrich-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-aws-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-flume-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-dbcp-service-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-ambari-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-avro-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-media-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-couchbase-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-riemann-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-amqp-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-jms-cf-service-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-jms-processors-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-spring-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-mqtt-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-snmp-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-hbase_1_1_2-client-service-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-hbase-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-azure-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-ldap-iaa-providers-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-kerberos-iaa-providers-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-scripting-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-slack-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-elasticsearch-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-lumberjack-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-email-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-splunk-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-cassandra-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-hive-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-site-to-site-reporting-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-evtx-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-windows-event-log-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-datadog-nar + ${org.apache.nifi.version} + nar + + + org.apache.nifi + nifi-properties + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-properties-loader + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-security-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-logging-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-nar-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-processor-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-hadoop-utils + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-mock + ${org.apache.nifi.version} + test + + + org.apache.nifi + nifi-write-ahead-log + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-dbcp-service + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-dbcp-service-api + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-hbase-client-service-api + ${org.apache.nifi.version} + + + org.apache.nifi + nifi-assembly + ${org.apache.nifi.version} + + + com.jayway.jsonpath + json-path + 2.0.0 + + + org.apache.derby + derby + 10.11.1.1 + + + com.amazonaws + aws-java-sdk + 1.11.8 + + + + + org.codehaus.groovy + groovy-all + 2.4.5 + test + + + org.spockframework + spock-core + 1.0-groovy-2.4 + test + + + org.hamcrest + hamcrest-all + 1.3 + test + From da98aceebc3f5afdbc0a3a67cbbd2250948f8c2c Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Thu, 10 Nov 2016 10:56:42 -0800 Subject: [PATCH 26/33] Adjusting dependencies. --- minifi-assembly/pom.xml | 33 +++++++++++++++++++ .../src/main/assembly/dependencies.xml | 18 ++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index e83e8c16c..67a3ba1f5 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -187,6 +187,39 @@ limitations under the License. 1.54 compile + + org.eclipse.jetty + jetty-util + 9.3.9.v20160517 + compile + + + org.apache.commons + commons-lang3 + + + org.apache.httpcomponents + httpclient + + + com.google.guava + guava + + + org.apache.nifi.minifi + minifi-framework-core + + + + + + + + + + + + org.eclipse.jetty jetty-servlet diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index 01830f26f..bf31ba257 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -36,11 +36,12 @@ minifi-bootstrap minifi-resources - spring-aop + org.apache.nifi:nifi-framework-core:1.0.0 zookeeper + spring-aop spring-context - spring-security-core spring-beans + spring-expression swagger-annotations slf4j-log4j12 aspectjweaver @@ -52,8 +53,21 @@ lucene-analyzers-common lucene-queryparser commons-net + spring-context + spring-security-core + + runtime + false + lib + 0770 + 0660 + + minifi-framework-core + nifi-framework-core-api + + From 06e87704fcec244edfdcb8f00ebd194b4c15de63 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Thu, 10 Nov 2016 14:01:30 -0800 Subject: [PATCH 27/33] Converting NiFiComponentDTO to ComponentDTO as part of 1.0 API change. --- .../nifi/minifi/toolkit/configuration/ConfigMain.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java index 475ad1278..76d1a5fef 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java @@ -115,7 +115,7 @@ public int validate(String[] args) { return ERR_INVALID_CONFIG; } }); - } catch (IOException|SchemaLoaderException e) { + } catch (IOException | SchemaLoaderException e) { return handleErrorLoadingConfiguration(e, ConfigMain::printValidateUsage); } } catch (FileNotFoundException e) { @@ -148,7 +148,7 @@ private static void enrichFlowSnippetDTO(FlowSnippetDTO flowSnippetDTO) { // RPGs with no name get Target URI as name remoteProcessGroups.stream().filter(r -> StringUtil.isNullOrEmpty(r.getName())).forEach(r -> r.setName(r.getTargetUri())); - Map connectableNameMap = getAll(allFlowSnippets, FlowSnippetDTO::getProcessors).collect(Collectors.toMap(NiFiComponentDTO::getId, ProcessorDTO::getName)); + Map connectableNameMap = getAll(allFlowSnippets, FlowSnippetDTO::getProcessors).collect(Collectors.toMap(ComponentDTO::getId, ProcessorDTO::getName)); for (RemoteProcessGroupDTO remoteProcessGroupDTO : remoteProcessGroups) { RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents(); @@ -252,7 +252,7 @@ public int upgrade(String[] args) { } return configSchema; }); - } catch (IOException|SchemaLoaderException e) { + } catch (IOException | SchemaLoaderException e) { return handleErrorLoadingConfiguration(e, ConfigMain::printUpgradeUsage); } } catch (FileNotFoundException e) { From 256998f45e6cfb95416c543023eb39cb634b286e Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Mon, 14 Nov 2016 15:55:33 -0500 Subject: [PATCH 28/33] Adjusting dependency management and assembly inclusions. Removing extraneous provenance reporting files as we are using the standard NiFi reporter. --- minifi-assembly/pom.xml | 23 +- .../src/main/assembly/dependencies.xml | 27 +- .../minifi-framework-nar/pom.xml | 1 - .../minifi-framework-bundle/pom.xml | 1 - .../minifi-provenance-reporting-nar/pom.xml | 40 - .../src/main/resources/META-INF/NOTICE | 15 - .../pom.xml | 53 -- .../minifi-provenance-repository-nar/pom.xml | 2 - .../minifi-ssl-context-service-nar/pom.xml | 8 +- .../minifi-standard-nar/pom.xml | 9 - .../minifi-standard-services-api-nar/pom.xml | 5 +- pom.xml | 695 ++---------------- 12 files changed, 63 insertions(+), 816 deletions(-) delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE delete mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 67a3ba1f5..89ed9e4cb 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -111,6 +111,10 @@ limitations under the License. org.apache.nifi.minifi minifi-runtime + + org.apache.nifi.minifi + minifi-framework-core + org.apache.nifi.minifi minifi-framework-nar @@ -158,13 +162,11 @@ limitations under the License. org.apache.nifi.minifi minifi-standard-services-api-nar - 0.1.0-SNAPSHOT nar org.apache.nifi.minifi minifi-ssl-context-service-nar - 0.1.0-SNAPSHOT nar @@ -184,13 +186,11 @@ limitations under the License. org.bouncycastle bcprov-jdk15on - 1.54 compile org.eclipse.jetty jetty-util - 9.3.9.v20160517 compile @@ -205,21 +205,6 @@ limitations under the License. com.google.guava guava - - org.apache.nifi.minifi - minifi-framework-core - - - - - - - - - - - - org.eclipse.jetty jetty-servlet diff --git a/minifi-assembly/src/main/assembly/dependencies.xml b/minifi-assembly/src/main/assembly/dependencies.xml index bf31ba257..d77b09860 100644 --- a/minifi-assembly/src/main/assembly/dependencies.xml +++ b/minifi-assembly/src/main/assembly/dependencies.xml @@ -33,20 +33,16 @@ 0660 true - minifi-bootstrap + minifi-bootstrap minifi-resources - org.apache.nifi:nifi-framework-core:1.0.0 zookeeper spring-aop spring-context spring-beans - spring-expression swagger-annotations - slf4j-log4j12 aspectjweaver h2 - netty jaxb-impl mail log4j @@ -57,17 +53,6 @@ spring-security-core - - runtime - false - lib - 0770 - 0660 - - minifi-framework-core - nifi-framework-core-api - - @@ -78,14 +63,10 @@ 0660 true - minifi-bootstrap - minifi-api - minifi-commons-schema + minifi-bootstrap minifi-utils - snakeyaml nifi-utils commons-io - okhttp @@ -98,7 +79,7 @@ 0664 true - minifi-resources + minifi-resources true @@ -124,7 +105,7 @@ 0770 true - minifi-resources + minifi-resources true diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml index 399b72193..38159525e 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml @@ -25,7 +25,6 @@ limitations under the License. org.apache.nifi.minifi minifi-framework-nar - 0.1.0-SNAPSHOT nar MiNiFi: Framework NAR diff --git a/minifi-nar-bundles/minifi-framework-bundle/pom.xml b/minifi-nar-bundles/minifi-framework-bundle/pom.xml index 57fbc0776..73a545c45 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/pom.xml +++ b/minifi-nar-bundles/minifi-framework-bundle/pom.xml @@ -26,7 +26,6 @@ limitations under the License. org.apache.nifi.minifi minifi-framework-bundle pom - 0.1.0-SNAPSHOT minifi-framework-nar diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml deleted file mode 100644 index 88b5b9002..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - 4.0.0 - - org.apache.nifi.minifi - minifi-provenance-reporting-bundle - 0.1.0-SNAPSHOT - - - minifi-provenance-reporting-nar - nar - - true - true - - - - org.apache.nifi.minifi - minifi-provenance-reporting-task - - - org.apache.nifi - nifi-standard-services-api-nar - nar - - - diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index be55e5968..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,15 +0,0 @@ -minifi-provenance-reporting-task-nar -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -************************ -Common Development and Distribution License 1.1 -************************ - -The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details. - - (CDDL 1.1) (GPL2 w/ CPE) JSON Processing API (javax.json:javax.json-api:jar:1.0 - http://json-processing-spec.java.net) - (CDDL 1.1) (GPL2 w/ CPE) JSON Processing Default Provider (org.glassfish:javax.json:jar:1.0.4 - https://jsonp.java.net) - diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml deleted file mode 100644 index 3850f768a..000000000 --- a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - 4.0.0 - - org.apache.nifi.minifi - minifi-nar-bundles - 0.1.0-SNAPSHOT - - - minifi-provenance-reporting-bundle - pom - - - minifi-provenance-reporting-nar - - - - - org.apache.nifi - nifi-site-to-site-reporting-task - 1.0.0 - - - - - - - org.apache.nifi.minifi - minifi-provenance-reporting-task - 0.1.0-SNAPSHOT - - - org.glassfish.jersey.core - jersey-client - 2.19 - - - - diff --git a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-provenance-repository-nar/pom.xml b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-provenance-repository-nar/pom.xml index ee397e25f..20c0f774e 100644 --- a/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-provenance-repository-nar/pom.xml +++ b/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-provenance-repository-nar/pom.xml @@ -25,14 +25,12 @@ limitations under the License. org.apache.nifi.minifi minifi-provenance-repository-nar - 0.1.0-SNAPSHOT nar org.apache.nifi.minifi minifi-persistent-provenance-repository - 0.1.0-SNAPSHOT org.apache.nifi diff --git a/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml b/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml index d2f89954b..35f27fa97 100644 --- a/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml +++ b/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml @@ -16,13 +16,12 @@ --> 4.0.0 - org.apache.nifi - nifi-ssl-context-bundle - 1.0.0 + org.apache.nifi.minifi + minifi-nar-bundles + 0.1.0-SNAPSHOT org.apache.nifi.minifi minifi-ssl-context-service-nar - 0.1.0-SNAPSHOT nar true @@ -32,7 +31,6 @@ org.apache.nifi.minifi minifi-standard-services-api-nar - 0.1.0-SNAPSHOT nar diff --git a/minifi-nar-bundles/minifi-standard-nar/pom.xml b/minifi-nar-bundles/minifi-standard-nar/pom.xml index bfafc2b11..473ec788d 100644 --- a/minifi-nar-bundles/minifi-standard-nar/pom.xml +++ b/minifi-nar-bundles/minifi-standard-nar/pom.xml @@ -35,7 +35,6 @@ limitations under the License. org.apache.nifi.minifi minifi-standard-services-api-nar nar - 0.1.0-SNAPSHOT org.apache.nifi @@ -57,18 +56,15 @@ limitations under the License. com.sun.jersey jersey-server provided - 1.19 com.google.guava guava - 18.0 provided org.bouncycastle bcprov-jdk15on - 1.54 provided @@ -80,31 +76,26 @@ limitations under the License. org.bouncycastle bcpkix-jdk15on - 1.54 provided org.codehaus.jackson jackson-mapper-asl - 1.9.13 provided com.fasterxml.jackson.core jackson-databind - 2.6.1 provided org.apache.httpcomponents httpclient - 4.4.1 provided org.apache.commons commons-lang3 - 3.4 provided diff --git a/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml index e3bb0e3c6..ea7bcd828 100644 --- a/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml +++ b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml @@ -19,10 +19,10 @@ org.apache.nifi nifi-standard-services 1.0.0 + org.apache.nifi.minifi minifi-standard-services-api-nar - 0.1.0-SNAPSHOT nar true @@ -63,19 +63,16 @@ org.bouncycastle bcprov-jdk15on - 1.54 provided org.bouncycastle bcpg-jdk15on - 1.54 provided org.bouncycastle bcpkix-jdk15on - 1.54 provided diff --git a/pom.xml b/pom.xml index be771dd01..d167191de 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ limitations under the License. scm:git:https://git-wip-us.apache.org/repos/asf/nifi-minifi.git https://git-wip-us.apache.org/repos/asf?p=nifi-minifi.git HEAD - + JIRA https://issues.apache.org/jira/browse/MINIFI @@ -95,9 +95,9 @@ limitations under the License. UTF-8 2016 1.7.12 + 1.1.3 1.0.0 9.2.11.v20150529 - 1.7.12 9.3.9.v20160517 4.10.4 4.2.4.RELEASE @@ -193,18 +193,45 @@ limitations under the License. minifi-runtime 0.1.0-SNAPSHOT + + org.apache.nifi.minifi + minifi-persistent-provenance-repository + 0.1.0-SNAPSHOT + + + org.apache.nifi.minifi + minifi-framework-bundle + 0.1.0-SNAPSHOT + + + org.apache.nifi.minifi + minifi-framework-nar + 0.1.0-SNAPSHOT + org.apache.nifi.minifi minifi-standard-nar 0.1.0-SNAPSHOT nar + + org.apache.nifi.minifi + minifi-standard-services-api-nar + 0.1.0-SNAPSHOT + nar + org.apache.nifi.minifi minifi-update-attribute-nar 0.1.0-SNAPSHOT nar + + org.apache.nifi.minifi + minifi-ssl-context-service-nar + 0.1.0-SNAPSHOT + nar + org.apache.nifi.minifi minifi-docs @@ -217,12 +244,12 @@ limitations under the License. ch.qos.logback logback-classic - 1.1.3 + ${logback.version} ch.qos.logback jcl-over-slf4j - 1.1.3 + ${logback.version} provided @@ -255,6 +282,12 @@ limitations under the License. ${org.slf4j.version} provided + + org.eclipse.jetty + jetty-util + ${jetty.version} + provided + org.eclipse.jetty jetty-server @@ -512,6 +545,11 @@ limitations under the License. nifi-persistent-provenance-repository ${org.apache.nifi.version} + + org.apache.nifi + nifi-ssl-context-service + ${org.apache.nifi.version} + commons-io commons-io @@ -566,68 +604,7 @@ limitations under the License. okhttp 3.4.1 - - ch.qos.logback - logback-classic - 1.1.3 - - - ch.qos.logback - jcl-over-slf4j - 1.1.3 - provided - - - org.slf4j - slf4j-api - - - - - org.slf4j - jcl-over-slf4j - ${org.slf4j.version} - provided - - - org.slf4j - log4j-over-slf4j - ${org.slf4j.version} - provided - - - org.slf4j - jul-to-slf4j - ${org.slf4j.version} - provided - - - org.slf4j - slf4j-api - ${org.slf4j.version} - provided - - - junit - junit - 4.12 - - - org.mockito - mockito-core - 1.10.19 - - - org.mockito - mockito-all - 1.10.19 - test - - - org.slf4j - slf4j-simple - ${org.slf4j.version} - + org.apache.commons commons-compress @@ -683,28 +660,6 @@ limitations under the License. commons-net 3.3 - - commons-io - commons-io - 2.5 - - - org.bouncycastle - bcprov-jdk15on - 1.54 - provided - - - org.bouncycastle - bcpg-jdk15on - 1.54 - - - org.bouncycastle - bcpkix-jdk15on - 1.54 - provided - com.jcraft jsch @@ -816,8 +771,6 @@ limitations under the License. spring-core ${spring.version} - commons-logging commons-logging @@ -1104,54 +1057,18 @@ limitations under the License. jersey-client ${jersey.version} - - org.eclipse.jetty - jetty-server - ${jetty.version} - provided - - - org.eclipse.jetty - jetty-servlet - ${jetty.version} - provided - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - provided - - - org.eclipse.jetty - jetty-servlets - ${jetty.version} - provided - org.eclipse.jetty jetty-annotations ${jetty.version} provided - - org.eclipse.jetty - apache-jsp - ${jetty.version} - provided - org.eclipse.jetty apache-jstl ${jetty.version} provided - - javax.servlet - javax.servlet-api - 3.1.0 - provided - com.google.guava guava @@ -1224,532 +1141,22 @@ limitations under the License. jsoup 1.8.3 - - - org.apache.nifi - nifi-api - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-framework-api - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-site-to-site-client - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-web-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-expression-language - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-custom-ui-utilities - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-ui-extension - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-flowfile-packager - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-socket-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-data-provenance-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-runtime - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-bootstrap - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-toolkit-tls - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-toolkit-encrypt-config - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-toolkit-s2s - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-resources - ${org.apache.nifi.version} - resources - runtime - zip - + + - org.apache.nifi - nifi-docs - ${org.apache.nifi.version} - resources - runtime - zip + org.codehaus.groovy + groovy-all + 2.4.5 - org.apache.nifi - nifi-framework-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-provenance-repository-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-standard-services-api-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-ssl-context-service-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-ssl-context-service-api - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-distributed-cache-services-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-standard-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-jetty-bundle - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-update-attribute-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-hadoop-libraries-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-hadoop-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-html-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-kite-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-mongodb-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-solr-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-kafka-0-8-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-kafka-0-9-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-kafka-0-10-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-ignite-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-http-context-map-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-social-media-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-hl7-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-language-translation-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-enrich-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-aws-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-flume-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-dbcp-service-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-ambari-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-avro-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-media-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-couchbase-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-riemann-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-amqp-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-jms-cf-service-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-jms-processors-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-spring-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-mqtt-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-snmp-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-hbase_1_1_2-client-service-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-hbase-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-azure-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-ldap-iaa-providers-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-kerberos-iaa-providers-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-scripting-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-slack-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-elasticsearch-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-lumberjack-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-email-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-splunk-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-cassandra-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-hive-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-site-to-site-reporting-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-evtx-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-windows-event-log-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-datadog-nar - ${org.apache.nifi.version} - nar - - - org.apache.nifi - nifi-properties - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-properties-loader - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-security-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-logging-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-nar-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-processor-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-hadoop-utils - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-mock - ${org.apache.nifi.version} - test - - - org.apache.nifi - nifi-write-ahead-log - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-dbcp-service - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-dbcp-service-api - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-hbase-client-service-api - ${org.apache.nifi.version} - - - org.apache.nifi - nifi-assembly - ${org.apache.nifi.version} - - - com.jayway.jsonpath - json-path - 2.0.0 - - - org.apache.derby - derby - 10.11.1.1 - - - com.amazonaws - aws-java-sdk - 1.11.8 - - - - - org.codehaus.groovy - groovy-all - 2.4.5 - test - - - org.spockframework - spock-core - 1.0-groovy-2.4 - test + org.spockframework + spock-core + 1.0-groovy-2.4 org.hamcrest hamcrest-all 1.3 - test From 303c7ad651f87486348391696151f91dac917b08 Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Tue, 15 Nov 2016 15:16:18 -0500 Subject: [PATCH 29/33] MINIFI-115 - Changing YAML to use UUIDs --- .../test/resources/config-multiple-RPGs.yml | 4 +-- .../resources/config-multiple-input-ports.yml | 4 +-- .../src/test/resources/config.yml | 6 ++-- .../commons/schema/ProcessGroupSchema.java | 8 +++++ .../schema/common/BaseSchemaWithId.java | 18 ++++++---- .../commons/schema/v1/ConfigSchemaV1.java | 36 ++++++++----------- .../commons/schema/ConfigSchemaTest.java | 13 ++++--- .../commons/schema/ConnectionSchemaTest.java | 4 ++- .../commons/schema/v1/ConfigSchemaV1Test.java | 29 ++++++++------- .../schema/v1/ConnectionSchemaV1Test.java | 10 +++--- .../src/test/resources/config-minimal-v2.yml | 10 +++--- .../minifi-standard-services-api-nar/pom.xml | 1 + minifi-nar-bundles/pom.xml | 2 +- .../configuration/dto/BaseSchemaTester.java | 3 +- .../dto/ConnectionSchemaTest.java | 4 ++- .../dto/PortSchemaFunctionTest.java | 5 ++- .../dto/ProcessorSchemaTest.java | 4 ++- .../dto/RemoteInputPortSchemaTest.java | 4 ++- .../src/test/resources/config.yml | 6 ++-- 19 files changed, 99 insertions(+), 72 deletions(-) diff --git a/minifi-bootstrap/src/test/resources/config-multiple-RPGs.yml b/minifi-bootstrap/src/test/resources/config-multiple-RPGs.yml index a779a42ed..a79909756 100644 --- a/minifi-bootstrap/src/test/resources/config-multiple-RPGs.yml +++ b/minifi-bootstrap/src/test/resources/config-multiple-RPGs.yml @@ -77,7 +77,7 @@ Connections: - name: TailToS2S source name: TailAppLog source relationship name: success - destination name: dgs809qe-9qw7-q985-jl1u-y789ohi87g78 + destination name: 37b4b78a-a5c9-3eac-bfb7-56fc4875aace max work queue size: 0 max work queue data size: 1 MB flowfile expiration: 60 sec @@ -109,7 +109,7 @@ Remote Processing Groups: timeout: 30 secs yield period: 10 sec Input Ports: - - id: dgs809qe-9qw7-q985-jl1u-y789ohi87g78 + - id: 37b4b78a-a5c9-3eac-bfb7-56fc4875aace name: tailed log comments: max concurrent tasks: 1 diff --git a/minifi-bootstrap/src/test/resources/config-multiple-input-ports.yml b/minifi-bootstrap/src/test/resources/config-multiple-input-ports.yml index 9654edf34..158116391 100644 --- a/minifi-bootstrap/src/test/resources/config-multiple-input-ports.yml +++ b/minifi-bootstrap/src/test/resources/config-multiple-input-ports.yml @@ -77,7 +77,7 @@ Connections: - name: TailToS2S source name: TailAppLog source relationship name: success - destination name: dgs809qe-9qw7-q985-jl1u-y789ohi87g78 + destination name: 37b4b78a-a5c9-3eac-bfb7-56fc4875aace max work queue size: 0 max work queue data size: 1 MB flowfile expiration: 60 sec @@ -103,7 +103,7 @@ Remote Processing Groups: comments: max concurrent tasks: 1 use compression: false - - id: dgs809qe-9qw7-q985-jl1u-y789ohi87g78 + - id: 37b4b78a-a5c9-3eac-bfb7-56fc4875aace name: tailed log2 comments: max concurrent tasks: 1 diff --git a/minifi-bootstrap/src/test/resources/config.yml b/minifi-bootstrap/src/test/resources/config.yml index 64a28fe6d..dab1e8e96 100644 --- a/minifi-bootstrap/src/test/resources/config.yml +++ b/minifi-bootstrap/src/test/resources/config.yml @@ -56,7 +56,7 @@ Security Properties: algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL provider: BC Processors: -- id: TailFile +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 name: TailFile class: org.apache.nifi.processors.standard.TailFile max concurrent tasks: 1 @@ -71,9 +71,9 @@ Processors: Initial Start Position: Beginning of File Rolling Filename Pattern: minifi-app* Connections: -- id: TailToS2S +- id: f6cef9b0-8982-391c-8d70-76f33917ac12 name: TailToS2S - source id: TailFile + source id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 source relationship names: - success destination id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ProcessGroupSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ProcessGroupSchema.java index 7be69c032..be7f9fc12 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ProcessGroupSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ProcessGroupSchema.java @@ -164,4 +164,12 @@ public List getOutputPortSchemas() { public List getInputPortSchemas() { return inputPortSchemas; } + + @Override + protected boolean isValidId(String value) { + if (ID_DEFAULT.equals(value)) { + return true; + } + return super.isValidId(value); + } } diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java index 96b47a71a..96dc4d757 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java @@ -21,14 +21,11 @@ import java.util.List; import java.util.Map; -import java.util.regex.Pattern; +import java.util.UUID; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; public class BaseSchemaWithId extends BaseSchema implements WritableSchema { - public static final Pattern VALID_ID_PATTERN = Pattern.compile("[A-Za-z0-9_-]+"); - public static final String ID_DOES_NOT_MATCH_VALID_ID_PATTERN = "Id does not match valid pattern (" + VALID_ID_PATTERN + "): "; - private final String wrapperName; private String id; @@ -65,9 +62,18 @@ public List getValidationIssues() { List validationIssues = super.getValidationIssues(); if (StringUtil.isNullOrEmpty(id)) { validationIssues.add(getIssueText(CommonPropertyKeys.ID_KEY, getWrapperName(), IT_WAS_NOT_FOUND_AND_IT_IS_REQUIRED)); - } else if (!VALID_ID_PATTERN.matcher(id).matches()) { - validationIssues.add(getIssueText(CommonPropertyKeys.ID_KEY, getWrapperName(), ID_DOES_NOT_MATCH_VALID_ID_PATTERN)); + } else if (!isValidId(id)) { + validationIssues.add(getIssueText(CommonPropertyKeys.ID_KEY, getWrapperName(), "Id value of " + id + " is not a valid UUID")); } return validationIssues; } + + protected boolean isValidId(String value) { + try { + UUID.fromString(value); + return true; + } catch (Exception e) { + return false; + } + } } \ No newline at end of file diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java index 2ebf28a16..49dc087c4 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java @@ -36,12 +36,14 @@ import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static org.apache.nifi.minifi.commons.schema.ConfigSchema.TOP_LEVEL_NAME; @@ -133,12 +135,12 @@ public ConfigSchemaV1(Map map) { } protected List getProcessorSchemas() { - Map idMap = new HashMap<>(); + Set ids = new HashSet<>(); List processorSchemas = new ArrayList<>(processors.size()); for (ProcessorSchemaV1 processor : processors) { ProcessorSchema processorSchema = processor.convert(); - processorSchema.setId(getUniqueId(idMap, processorSchema.getName())); + processorSchema.setId(getUniqueId(ids, processorSchema.getName())); processorSchemas.add(processorSchema); } @@ -146,7 +148,7 @@ protected List getProcessorSchemas() { } protected List getConnectionSchemas(List processors, List validationIssues) { - Map idMap = new HashMap<>(); + Set ids = new HashSet<>(); Map processorNameToIdMap = new HashMap<>(); @@ -175,7 +177,7 @@ protected List getConnectionSchemas(List proc List connectionSchemas = new ArrayList<>(connections.size()); for (ConnectionSchemaV1 connection : connections) { ConnectionSchema convert = connection.convert(); - convert.setId(getUniqueId(idMap, convert.getName())); + convert.setId(getUniqueId(ids, convert.getName())); String sourceName = connection.getSourceName(); if (remoteInputPortIds.contains(sourceName)) { @@ -233,27 +235,19 @@ public ConfigSchema convert() { } /** - * Will replace all characters not in [A-Za-z0-9_] with _ - *

- * This has potential for collisions so it will also append numbers as necessary to prevent that + * Will deterministically (per config file in the case of collisions) map the name to a uuid. * - * @param ids id map of already incremented numbers + * @param ids the set of UUIDs already assigned * @param name the name - * @return a unique filesystem-friendly id + * @return a UUID string */ - public static String getUniqueId(Map ids, String name) { - String baseId = StringUtil.isNullOrEmpty(name) ? EMPTY_NAME : ID_REPLACE_PATTERN.matcher(name).replaceAll("_"); - String id = baseId; - Integer idNum = ids.get(baseId); - while (ids.containsKey(id)) { - id = baseId + "_" + idNum++; + public static String getUniqueId(Set ids, String name) { + UUID id = UUID.nameUUIDFromBytes(name == null ? EMPTY_NAME.getBytes(StandardCharsets.UTF_8) : name.getBytes(StandardCharsets.UTF_8)); + while (ids.contains(id)) { + id = new UUID(id.getMostSignificantBits(), id.getLeastSignificantBits() + 1); } - // Using != on a string comparison here is intentional. The two will be reference equal iff the body of the while loop was never executed. - if (id != baseId) { - ids.put(baseId, idNum); - } - ids.put(id, 2); - return id; + ids.add(id); + return id.toString(); } @Override diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java index afca6e05c..83b5525ef 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java @@ -23,10 +23,13 @@ import org.junit.Test; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; import static org.junit.Assert.assertEquals; @@ -93,8 +96,8 @@ public void testInvalidSourceAndDestinationIds() throws IOException, SchemaLoade List> connections = (List>) yamlAsMap.get(CommonPropertyKeys.CONNECTIONS_KEY); assertEquals(1, connections.size()); - String fakeSource = "fakeSource"; - String fakeDestination = "fakeDestination"; + String fakeSource = UUID.nameUUIDFromBytes("fakeSource".getBytes(StandardCharsets.UTF_8)).toString(); + String fakeDestination = UUID.nameUUIDFromBytes("fakeDestination".getBytes(StandardCharsets.UTF_8)).toString(); Map connection = connections.get(0); connection.put(ConnectionSchema.SOURCE_ID_KEY, fakeSource); @@ -102,9 +105,9 @@ public void testInvalidSourceAndDestinationIds() throws IOException, SchemaLoade ConfigSchema configSchema = new ConfigSchema(yamlAsMap); List validationIssues = configSchema.getValidationIssues(); - assertEquals(2, validationIssues.size()); - assertEquals(ConfigSchema.CONNECTION_WITH_ID + connection.get(ID_KEY) + ConfigSchema.HAS_INVALID_DESTINATION_ID + fakeDestination, validationIssues.get(0)); - assertEquals(ConfigSchema.CONNECTION_WITH_ID + connection.get(ID_KEY) + ConfigSchema.HAS_INVALID_SOURCE_ID + fakeSource, validationIssues.get(1)); + assertEquals(new ArrayList<>( + Arrays.asList(ConfigSchema.CONNECTION_WITH_ID + connection.get(ID_KEY) + ConfigSchema.HAS_INVALID_DESTINATION_ID + fakeDestination, + ConfigSchema.CONNECTION_WITH_ID + connection.get(ID_KEY) + ConfigSchema.HAS_INVALID_SOURCE_ID + fakeSource)), validationIssues); } public static List> getListWithNames(String... names) { diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConnectionSchemaTest.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConnectionSchemaTest.java index 61f9e9615..860bf41fe 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConnectionSchemaTest.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConnectionSchemaTest.java @@ -23,11 +23,13 @@ import org.junit.Before; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.junit.Assert.assertEquals; @@ -46,7 +48,7 @@ public class ConnectionSchemaTest { @Before public void setup() { - testId = "testId"; + testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); testName = "testName"; testSourceId = "testSourceId"; testSourceRelationShip1 = "testSourceRelationShip1"; diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1Test.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1Test.java index 7ce0587d1..b72133a3a 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1Test.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1Test.java @@ -29,9 +29,12 @@ import org.junit.Test; import java.io.IOException; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.UUID; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; import static org.junit.Assert.assertEquals; @@ -43,7 +46,7 @@ public void testValid() throws IOException, SchemaLoaderException { Map yamlAsMap = SchemaLoader.loadYamlAsMap(ConfigSchemaTest.class.getClassLoader().getResourceAsStream("config-minimal.yml")); ConfigSchema configSchema = new ConfigSchemaV1(yamlAsMap).convert(); List validationIssues = configSchema.getValidationIssues(); - assertEquals(0, validationIssues.size()); + assertEquals(new ArrayList<>(), validationIssues); } @Test public void testValidationIssuesFromNewer() throws IOException, SchemaLoaderException { @@ -69,9 +72,9 @@ public void testInvalidSourceAndDestinationNames() throws IOException, SchemaLoa ConfigSchema configSchema = new ConfigSchemaV1(yamlAsMap).convert(); List validationIssues = configSchema.getValidationIssues(); assertEquals(4, validationIssues.size()); - assertEquals(BaseSchema.getIssueText(ConnectionSchema.DESTINATION_ID_KEY, "Connection(id: TailToSplit, name: TailToSplit)", BaseSchema.IT_WAS_NOT_FOUND_AND_IT_IS_REQUIRED), - validationIssues.get(0)); - assertEquals(BaseSchema.getIssueText(ConnectionSchema.SOURCE_ID_KEY, "Connection(id: TailToSplit, name: TailToSplit)", BaseSchema.IT_WAS_NOT_FOUND_AND_IT_IS_REQUIRED), + assertEquals(BaseSchema.getIssueText(ConnectionSchema.DESTINATION_ID_KEY, "Connection(id: 0401b747-1dca-31c7-ab4b-cdacf7e6c44b, name: TailToSplit)", + BaseSchema.IT_WAS_NOT_FOUND_AND_IT_IS_REQUIRED), validationIssues.get(0)); + assertEquals(BaseSchema.getIssueText(ConnectionSchema.SOURCE_ID_KEY, "Connection(id: 0401b747-1dca-31c7-ab4b-cdacf7e6c44b, name: TailToSplit)", BaseSchema.IT_WAS_NOT_FOUND_AND_IT_IS_REQUIRED), validationIssues.get(1)); assertEquals(ConfigSchemaV1.CONNECTION_WITH_NAME + connection.get(NAME_KEY) + ConfigSchemaV1.HAS_INVALID_DESTINATION_NAME + fakeDestination, validationIssues.get(2)); assertEquals(ConfigSchemaV1.CONNECTION_WITH_NAME + connection.get(NAME_KEY) + ConfigSchemaV1.HAS_INVALID_SOURCE_NAME + fakeSource, validationIssues.get(3)); @@ -79,18 +82,18 @@ public void testInvalidSourceAndDestinationNames() throws IOException, SchemaLoa @Test public void testGetUniqueIdConflicts() { - Map ids = new HashMap<>(); - assertEquals("test_id", ConfigSchemaV1.getUniqueId(ids, "test/id")); - assertEquals("test_id_2", ConfigSchemaV1.getUniqueId(ids, "test$id")); - assertEquals("test_id_3", ConfigSchemaV1.getUniqueId(ids, "test$id")); - assertEquals("test_id_4", ConfigSchemaV1.getUniqueId(ids, "test$id")); - assertEquals("test_id_5", ConfigSchemaV1.getUniqueId(ids, "test$id")); - assertEquals("test_id_2_2", ConfigSchemaV1.getUniqueId(ids, "test_id_2")); + Set ids = new HashSet<>(); + assertEquals("56e7ae5d-aae1-351e-bca9-cdc4446c6386", ConfigSchemaV1.getUniqueId(ids, "test/id")); + assertEquals("348c4f93-cd15-3d91-82f2-bfe5d43834d8", ConfigSchemaV1.getUniqueId(ids, "test$id")); + assertEquals("348c4f93-cd15-3d91-82f2-bfe5d43834d9", ConfigSchemaV1.getUniqueId(ids, "test$id")); + assertEquals("348c4f93-cd15-3d91-82f2-bfe5d43834da", ConfigSchemaV1.getUniqueId(ids, "test$id")); + assertEquals("348c4f93-cd15-3d91-82f2-bfe5d43834db", ConfigSchemaV1.getUniqueId(ids, "test$id")); + assertEquals("697487ff-979c-342d-9aab-5c710ae43a28", ConfigSchemaV1.getUniqueId(ids, "test_id_2")); } @Test public void testGetUniqueIdEmptySet() { String testId = "testId"; - assertEquals(testId + "___", ConfigSchemaV1.getUniqueId(new HashMap<>(), testId + "/ $")); + assertEquals("17841b04-ce22-34a3-9603-d95ec31d02dc", ConfigSchemaV1.getUniqueId(new HashSet<>(), testId + "/ $")); } } diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConnectionSchemaV1Test.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConnectionSchemaV1Test.java index 28b000c27..50180ddf9 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConnectionSchemaV1Test.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v1/ConnectionSchemaV1Test.java @@ -183,10 +183,10 @@ public void testConnectionGeneratedIds() { assertEquals(5, connections.size()); // Generated unique ids - assertEquals("test", connections.get(0).getId()); - assertEquals("test_2", connections.get(1).getId()); - assertEquals("test_2_2", connections.get(2).getId()); - assertEquals("test_3", connections.get(3).getId()); - assertEquals("test_2_3", connections.get(4).getId()); + assertEquals("098f6bcd-4621-3373-8ade-4e832627b4f6", connections.get(0).getId()); + assertEquals("098f6bcd-4621-3373-8ade-4e832627b4f7", connections.get(1).getId()); + assertEquals("b9dacbc6-7962-39a9-9e45-394e2cf0dbf6", connections.get(2).getId()); + assertEquals("098f6bcd-4621-3373-8ade-4e832627b4f8", connections.get(3).getId()); + assertEquals("b9dacbc6-7962-39a9-9e45-394e2cf0dbf7", connections.get(4).getId()); } } diff --git a/minifi-commons/minifi-commons-schema/src/test/resources/config-minimal-v2.yml b/minifi-commons/minifi-commons-schema/src/test/resources/config-minimal-v2.yml index 2472a328f..8047741fd 100644 --- a/minifi-commons/minifi-commons-schema/src/test/resources/config-minimal-v2.yml +++ b/minifi-commons/minifi-commons-schema/src/test/resources/config-minimal-v2.yml @@ -20,19 +20,19 @@ Flow Controller: # When running the Flow (not just doing the transform) these processors will be invalid due to not having the necesary properties/auto-terminated relationships Processors: - - id: bcd + - id: d4b7c284-882c-39e2-88bb-65e8abd5f4c8 class: org.apache.nifi.processors.standard.TailFile scheduling strategy: TIMER_DRIVEN scheduling period: 10 sec - - id: cde + - id: a256e6b3-36af-3c38-8564-789c399b516c class: org.apache.nifi.processors.standard.PutFile max concurrent tasks: 1 scheduling strategy: TIMER_DRIVEN scheduling period: 0 sec Connections: - - id: abc - source id: bcd + - id: 90015098-3cd2-3fb0-9696-3f7d28e17f72 + source id: d4b7c284-882c-39e2-88bb-65e8abd5f4c8 source relationship names: - success - destination id: cde + destination id: a256e6b3-36af-3c38-8564-789c399b516c diff --git a/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml index ea7bcd828..cb4793cb0 100644 --- a/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml +++ b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml @@ -23,6 +23,7 @@ org.apache.nifi.minifi minifi-standard-services-api-nar + 0.1.0-SNAPSHOT nar true diff --git a/minifi-nar-bundles/pom.xml b/minifi-nar-bundles/pom.xml index a02b75b4a..e60b8d7a9 100644 --- a/minifi-nar-bundles/pom.xml +++ b/minifi-nar-bundles/pom.xml @@ -27,8 +27,8 @@ minifi-framework-bundle minifi-provenance-repository-bundle minifi-update-attribute-nar - minifi-standard-nar minifi-standard-services-api-nar + minifi-standard-nar minifi-ssl-context-service-nar diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java index 6ad7e0387..96b0e3e28 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -43,7 +44,7 @@ protected void assertDtoAndMapConstructorAreSame(int validationErrors) { assertEquals(dtoSchema.getValidationIssues(), mapSchema.getValidationIssues()); assertSchemaEquals(dtoSchema, mapSchemaFunction.apply(dtoSchema.toMap())); assertSchemaEquals(mapSchema, mapSchemaFunction.apply(mapSchema.toMap())); - assertEquals(validationErrors, dtoSchema.getValidationIssues().size()); + assertEquals(dtoSchema.getValidationIssues().stream().collect(Collectors.joining(", ")), validationErrors, dtoSchema.getValidationIssues().size()); } public abstract void assertSchemaEquals(Schema one, Schema two); diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java index 6aa77fb5b..051c4a7b3 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java @@ -25,17 +25,19 @@ import org.junit.Before; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; public class ConnectionSchemaTest extends BaseSchemaTester { - private final String testId = "testId"; + private final String testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); private final String testName = "testName"; private final String testSourceId = "testSourceId"; private final String testSelectedRelationship = "testSelectedRelationship"; diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java index 05c83cb81..12a715191 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java @@ -24,6 +24,9 @@ import org.junit.Before; import org.junit.Test; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -37,7 +40,7 @@ public class PortSchemaFunctionTest { @Before public void setup() { - testId = "testId"; + testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); testName = "testName"; testWrapperName = "testWrapperName"; portDTO = new PortDTO(); diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java index f98afe331..719c89abf 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java @@ -26,15 +26,17 @@ import org.junit.Before; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.UUID; import static org.junit.Assert.assertEquals; public class ProcessorSchemaTest extends BaseSchemaTester { private final String testName = "testName"; - private final String testId = "testId"; + private final String testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); private final String testProcessorClass = "testProcessorClass"; private final String testSchedulingStrategy = SchedulingStrategy.PRIMARY_NODE_ONLY.name(); private final String testSchedulingPeriod = "10 s"; diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java index 00f334833..94b5bebdc 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java @@ -23,13 +23,15 @@ import org.junit.Before; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.UUID; import static org.junit.Assert.assertEquals; public class RemoteInputPortSchemaTest extends BaseSchemaTester { - private String testId = "testId"; + private String testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); private String testName = "testName"; private String testComment = "testComment"; private int testMaxConcurrentTasks = 111; diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml index 64a28fe6d..dab1e8e96 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml @@ -56,7 +56,7 @@ Security Properties: algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL provider: BC Processors: -- id: TailFile +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 name: TailFile class: org.apache.nifi.processors.standard.TailFile max concurrent tasks: 1 @@ -71,9 +71,9 @@ Processors: Initial Start Position: Beginning of File Rolling Filename Pattern: minifi-app* Connections: -- id: TailToS2S +- id: f6cef9b0-8982-391c-8d70-76f33917ac12 name: TailToS2S - source id: TailFile + source id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 source relationship names: - success destination id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 From ea29a855ffbcc054d7ab31965526cdb891a6742b Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Fri, 18 Nov 2016 14:02:31 -0500 Subject: [PATCH 30/33] MINIFI-115: Providing handling of ID for Remote Process Groups to use the generated approach created for other components and introducing a RemoteProcessingGroupSchemaV1. Corrected typos, adjusted ProvenanceReportingTask to be the bundled NiFi class, and updating specified NiFi version for the ConfigTransformer. --- .../bootstrap/util/ConfigTransformer.java | 7 +- .../bootstrap/util/ParentGroupIdResolver.java | 2 +- .../bootstrap/util/ConfigTransformerTest.java | 2 +- .../util/ParentGroupIdResolverTest.java | 7 +- .../src/test/resources/config.yml | 3 +- .../schema/RemoteProcessingGroupSchema.java | 20 +--- .../schema/common/BaseSchemaWithId.java | 2 +- .../common/BaseSchemaWithIdAndName.java | 2 +- .../commons/schema/v1/ConfigSchemaV1.java | 22 +++- .../v1/RemoteProcessingGroupSchemaV1.java | 107 ++++++++++++++++++ .../toolkit/configuration/ConfigMain.java | 3 - .../RemoteProcessingGroupSchemaFunction.java | 1 + .../toolkit/configuration/ConfigMainTest.java | 2 +- .../dto/RemoteProcessingGroupSchemaTest.java | 13 ++- .../InvokeHttpMiNiFiTemplateTest-v1.yml | 2 +- .../InvokeHttpMiNiFiTemplateTest.xml | 6 +- .../InvokeHttpMiNiFiTemplateTest.yml | 3 +- .../ProcessGroupsAndRemoteProcessGroups.yml | 6 +- .../src/test/resources/config.yml | 3 +- 19 files changed, 171 insertions(+), 42 deletions(-) create mode 100644 minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/RemoteProcessingGroupSchemaV1.java diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index da707b9b4..a5b51e0a1 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -78,8 +78,9 @@ public final class ConfigTransformer { // Underlying version of NIFI will be using - public static final String NIFI_VERSION = "0.6.1"; + public static final String NIFI_VERSION = "1.0.0"; public static final String ROOT_GROUP = "Root-Group"; + public static final String DEFAULT_PROV_REPORTING_TASK_CLASS = "org.apache.nifi.SiteToSiteProvenanceReportingTask"; // Final util classes should have private constructor private ConfigTransformer() { @@ -471,7 +472,7 @@ protected static void addProvenanceReportingTask(final Element element, ConfigSc addTextElement(taskElement, "id", "Provenance-Reporting"); addTextElement(taskElement, "name", "Site-To-Site-Provenance-Reporting"); addTextElement(taskElement, "comment", provenanceProperties.getComment()); - addTextElement(taskElement, "class", "org.apache.nifi.minifi.provenance.reporting.ProvenanceReportingTask"); + addTextElement(taskElement, "class", DEFAULT_PROV_REPORTING_TASK_CLASS); addTextElement(taskElement, "schedulingPeriod", provenanceProperties.getSchedulingPeriod()); addTextElement(taskElement, "scheduledState", "RUNNING"); addTextElement(taskElement, "schedulingStrategy", provenanceProperties.getSchedulingStrategy()); @@ -524,7 +525,7 @@ protected static void addRemoteProcessGroup(final Element parentElement, RemoteP final Document doc = parentElement.getOwnerDocument(); final Element element = doc.createElement("remoteProcessGroup"); parentElement.appendChild(element); - addTextElement(element, "id", remoteProcessingGroupProperties.getName()); + addTextElement(element, "id", remoteProcessingGroupProperties.getId()); addTextElement(element, "name", remoteProcessingGroupProperties.getName()); addPosition(element); addTextElement(element, "comment", remoteProcessingGroupProperties.getComment()); diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java index 05aa27d16..0d7582359 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java @@ -65,7 +65,7 @@ protected static Map getRemoteInputPortParentIdMap(ProcessGroupS protected static void getRemoteInputPortParentIdMap(ProcessGroupSchema processGroupSchema, Map output) { for (RemoteProcessingGroupSchema remoteProcessingGroupSchema : processGroupSchema.getRemoteProcessingGroups()) { for (RemoteInputPortSchema remoteInputPortSchema : remoteProcessingGroupSchema.getInputPorts()) { - output.put(remoteInputPortSchema.getId(), remoteProcessingGroupSchema.getName()); + output.put(remoteInputPortSchema.getId(), remoteProcessingGroupSchema.getId()); } } processGroupSchema.getProcessGroupSchemas().forEach(p -> getRemoteInputPortParentIdMap(p, output)); diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index 6c2ed7811..4aff4889b 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -193,7 +193,7 @@ private void testProcessor(Element element, ProcessorSchema processorSchema) thr } private void testRemoteProcessGroups(Element element, RemoteProcessingGroupSchema remoteProcessingGroupSchema) throws XPathExpressionException { - assertEquals(remoteProcessingGroupSchema.getName(), getText(element, "id")); + assertEquals(remoteProcessingGroupSchema.getId(), getText(element, "id")); assertEquals(remoteProcessingGroupSchema.getName(), getText(element, "name")); assertEquals(remoteProcessingGroupSchema.getComment(), getText(element, "comment")); assertEquals(remoteProcessingGroupSchema.getUrl(), getText(element, "url")); diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java index c67d250dc..424c0fc02 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java @@ -41,16 +41,18 @@ public void testRemoteInputPortParentId() throws IOException, SchemaLoaderExcept configLines.add("MiNiFi Config Version: 2"); configLines.add("Remote Processing Groups:"); configLines.add("- name: rpgOne"); + configLines.add(" id: rpgOneId"); configLines.add(" Input Ports:"); configLines.add(" - id: one"); configLines.add("Process Groups:"); configLines.add("- Remote Processing Groups:"); configLines.add(" - name: rpgTwo"); + configLines.add(" id: rpgTwoId"); configLines.add(" Input Ports:"); configLines.add(" - id: two"); ParentGroupIdResolver parentGroupIdResolver = createParentGroupIdResolver(configLines); - assertEquals("rpgOne", parentGroupIdResolver.getRemoteInputPortParentId("one")); - assertEquals("rpgTwo", parentGroupIdResolver.getRemoteInputPortParentId("two")); + assertEquals("rpgOneId", parentGroupIdResolver.getRemoteInputPortParentId("one")); + assertEquals("rpgTwoId", parentGroupIdResolver.getRemoteInputPortParentId("two")); assertNull(parentGroupIdResolver.getInputPortParentId("one")); assertNull(parentGroupIdResolver.getInputPortParentId("two")); assertNull(parentGroupIdResolver.getOutputPortParentId("one")); @@ -132,6 +134,7 @@ public void testProcessorParentId() throws IOException, SchemaLoaderException { @Test public void testFunnelParentId() throws IOException, SchemaLoaderException { + StringBuilder sb = new StringBuilder(); List configLines = new ArrayList<>(); configLines.add("MiNiFi Config Version: 2"); configLines.add("Funnels:"); diff --git a/minifi-bootstrap/src/test/resources/config.yml b/minifi-bootstrap/src/test/resources/config.yml index dab1e8e96..46b3674ef 100644 --- a/minifi-bootstrap/src/test/resources/config.yml +++ b/minifi-bootstrap/src/test/resources/config.yml @@ -82,7 +82,8 @@ Connections: flowfile expiration: 60 sec queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer Remote Processing Groups: -- name: NiFi Flow +- id: 8644cbcc-a45c-40e0-964d-4d243afcbbb2 + name: NiFi Flow url: https://localhost:8090/nifi comment: '' timeout: 30 secs diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessingGroupSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessingGroupSchema.java index 6d2bb204f..44109ded5 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessingGroupSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessingGroupSchema.java @@ -17,20 +17,16 @@ package org.apache.nifi.minifi.commons.schema; -import org.apache.nifi.minifi.commons.schema.common.BaseSchema; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.minifi.commons.schema.common.WritableSchema; +import org.apache.nifi.minifi.commons.schema.common.BaseSchemaWithIdAndName; import java.util.List; import java.util.Map; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.INPUT_PORTS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.REMOTE_PROCESSING_GROUPS_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.YIELD_PERIOD_KEY; -public class RemoteProcessingGroupSchema extends BaseSchema implements WritableSchema { +public class RemoteProcessingGroupSchema extends BaseSchemaWithIdAndName { public static final String URL_KEY = "url"; public static final String TIMEOUT_KEY = "timeout"; @@ -38,7 +34,6 @@ public class RemoteProcessingGroupSchema extends BaseSchema implements WritableS public static final String DEFAULT_TIMEOUT = "30 secs"; public static final String DEFAULT_YIELD_PERIOD = "10 sec"; - private String name; private String url; private List inputPorts; @@ -47,8 +42,8 @@ public class RemoteProcessingGroupSchema extends BaseSchema implements WritableS private String yieldPeriod = DEFAULT_YIELD_PERIOD; public RemoteProcessingGroupSchema(Map map) { - name = getRequiredKeyAsType(map, NAME_KEY, String.class, REMOTE_PROCESSING_GROUPS_KEY); - String wrapperName = new StringBuilder("RemoteProcessingGroup(name: ").append(StringUtil.isNullOrEmpty(name) ? "unknown" : name).append(")").toString(); + super(map, "RemoteProcessingGroup(id: {id}, name: {name})"); + String wrapperName = getWrapperName(); url = getRequiredKeyAsType(map, URL_KEY, String.class, wrapperName); inputPorts = convertListToType(getRequiredKeyAsType(map, INPUT_PORTS_KEY, List.class, wrapperName), "input port", RemoteInputPortSchema.class, INPUT_PORTS_KEY); if (inputPorts != null) { @@ -64,8 +59,7 @@ public RemoteProcessingGroupSchema(Map map) { @Override public Map toMap() { - Map result = mapSupplier.get(); - result.put(NAME_KEY, name); + Map result = super.toMap(); result.put(URL_KEY, url); result.put(COMMENT_KEY, comment); result.put(TIMEOUT_KEY, timeout); @@ -74,10 +68,6 @@ public Map toMap() { return result; } - public String getName() { - return name; - } - public String getComment() { return comment; } diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java index 96dc4d757..bc6693e0f 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithId.java @@ -47,7 +47,7 @@ public String getId() { } public String getWrapperName() { - return wrapperName.replace("{id}", StringUtil.isNullOrEmpty(id) ? "unkown" : id); + return wrapperName.replace("{id}", StringUtil.isNullOrEmpty(id) ? "unknown" : id); } @Override diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java index 71a4f4161..5c9f8c7de 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java @@ -40,7 +40,7 @@ public void setName(String name) { } public String getWrapperName() { - return super.getWrapperName().replace("{name}", StringUtil.isNullOrEmpty(name) ? "unkown" : name); + return super.getWrapperName().replace("{name}", StringUtil.isNullOrEmpty(name) ? "unknown" : name); } @Override diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java index 49dc087c4..9c538bd20 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java @@ -77,7 +77,7 @@ public class ConfigSchemaV1 extends BaseSchema implements ConvertableSchema processors; private List connections; - private List remoteProcessingGroups; + private List remoteProcessingGroups; private ProvenanceReportingSchema provenanceReportingProperties; private ProvenanceRepositorySchema provenanceRepositorySchema; @@ -95,7 +95,7 @@ public ConfigSchemaV1(Map map) { processors = convertListToType(getOptionalKeyAsType(map, PROCESSORS_KEY, List.class, TOP_LEVEL_NAME, new ArrayList<>()), PROCESSORS_KEY, ProcessorSchemaV1.class, TOP_LEVEL_NAME); remoteProcessingGroups = convertListToType(getOptionalKeyAsType(map, REMOTE_PROCESSING_GROUPS_KEY, List.class, TOP_LEVEL_NAME, new ArrayList<>()), "remote processing group", - RemoteProcessingGroupSchema.class, REMOTE_PROCESSING_GROUPS_KEY); + RemoteProcessingGroupSchemaV1.class, REMOTE_PROCESSING_GROUPS_KEY); connections = convertListToType(getOptionalKeyAsType(map, CONNECTIONS_KEY, List.class, TOP_LEVEL_NAME, new ArrayList<>()), CONNECTIONS_KEY, ConnectionSchemaV1.class, TOP_LEVEL_NAME); @@ -117,7 +117,7 @@ public ConfigSchemaV1(Map map) { checkForDuplicates(this::addValidationIssue, FOUND_THE_FOLLOWING_DUPLICATE_PROCESSOR_NAMES, processorNames); checkForDuplicates(this::addValidationIssue, FOUND_THE_FOLLOWING_DUPLICATE_CONNECTION_NAMES, connections.stream().map(ConnectionSchemaV1::getName).collect(Collectors.toList())); - checkForDuplicates(this::addValidationIssue, FOUND_THE_FOLLOWING_DUPLICATE_REMOTE_PROCESSING_GROUP_NAMES, remoteProcessingGroups.stream().map(RemoteProcessingGroupSchema::getName) + checkForDuplicates(this::addValidationIssue, FOUND_THE_FOLLOWING_DUPLICATE_REMOTE_PROCESSING_GROUP_NAMES, remoteProcessingGroups.stream().map(RemoteProcessingGroupSchemaV1::getName) .collect(Collectors.toList())); Set connectableNames = new HashSet<>(processorNames); @@ -214,6 +214,19 @@ protected List getConnectionSchemas(List proc return connectionSchemas; } + protected List getRemoteProcessingGroupSchemas() { + Set ids = new HashSet<>(); + List rpgSchemas= new ArrayList<>(remoteProcessingGroups.size()); + + for (RemoteProcessingGroupSchemaV1 rpg : remoteProcessingGroups) { + RemoteProcessingGroupSchema rpgSchema = rpg.convert(); + rpgSchema.setId(getUniqueId(ids, rpgSchema.getName())); + rpgSchemas.add(rpgSchema); + } + + return rpgSchemas; + + } @Override public ConfigSchema convert() { Map map = new HashMap<>(); @@ -229,7 +242,7 @@ public ConfigSchema convert() { putListIfNotNull(map, PROCESSORS_KEY, processorSchemas); List validationIssues = getValidationIssues(); putListIfNotNull(map, CONNECTIONS_KEY, getConnectionSchemas(processorSchemas, validationIssues)); - putListIfNotNull(map, REMOTE_PROCESSING_GROUPS_KEY, remoteProcessingGroups); + putListIfNotNull(map, REMOTE_PROCESSING_GROUPS_KEY, getRemoteProcessingGroupSchemas()); putIfNotNull(map, PROVENANCE_REPORTING_KEY, provenanceReportingProperties); return new ConfigSchema(map, validationIssues); } @@ -242,6 +255,7 @@ public ConfigSchema convert() { * @return a UUID string */ public static String getUniqueId(Set ids, String name) { + System.out.println("Using name " + name); UUID id = UUID.nameUUIDFromBytes(name == null ? EMPTY_NAME.getBytes(StandardCharsets.UTF_8) : name.getBytes(StandardCharsets.UTF_8)); while (ids.contains(id)) { id = new UUID(id.getMostSignificantBits(), id.getLeastSignificantBits() + 1); diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/RemoteProcessingGroupSchemaV1.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/RemoteProcessingGroupSchemaV1.java new file mode 100644 index 000000000..a6d2d6266 --- /dev/null +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/RemoteProcessingGroupSchemaV1.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.schema.v1; + +import org.apache.nifi.minifi.commons.schema.RemoteInputPortSchema; +import org.apache.nifi.minifi.commons.schema.RemoteProcessingGroupSchema; +import org.apache.nifi.minifi.commons.schema.common.BaseSchema; +import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; +import org.apache.nifi.minifi.commons.schema.common.StringUtil; + +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.INPUT_PORTS_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.REMOTE_PROCESSING_GROUPS_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.YIELD_PERIOD_KEY; + +public class RemoteProcessingGroupSchemaV1 extends BaseSchema implements ConvertableSchema { + public static final String URL_KEY = "url"; + public static final String TIMEOUT_KEY = "timeout"; + + public static final String DEFAULT_COMMENT = ""; + public static final String DEFAULT_TIMEOUT = "30 secs"; + public static final String DEFAULT_YIELD_PERIOD = "10 sec"; + + private String name; + private String url; + private List inputPorts; + + private String comment = DEFAULT_COMMENT; + private String timeout = DEFAULT_TIMEOUT; + private String yieldPeriod = DEFAULT_YIELD_PERIOD; + + public RemoteProcessingGroupSchemaV1(Map map) { + name = getRequiredKeyAsType(map, NAME_KEY, String.class, REMOTE_PROCESSING_GROUPS_KEY); + String wrapperName = new StringBuilder("RemoteProcessingGroup(name: ").append(StringUtil.isNullOrEmpty(name) ? "unknown" : name).append(")").toString(); + url = getRequiredKeyAsType(map, URL_KEY, String.class, wrapperName); + inputPorts = convertListToType(getRequiredKeyAsType(map, INPUT_PORTS_KEY, List.class, wrapperName), "input port", RemoteInputPortSchema.class, INPUT_PORTS_KEY); + if (inputPorts != null) { + for (RemoteInputPortSchema remoteInputPortSchema : inputPorts) { + addIssuesIfNotNull(remoteInputPortSchema); + } + } + + comment = getOptionalKeyAsType(map, COMMENT_KEY, String.class, wrapperName, DEFAULT_COMMENT); + timeout = getOptionalKeyAsType(map, TIMEOUT_KEY, String.class, wrapperName, DEFAULT_TIMEOUT); + yieldPeriod = getOptionalKeyAsType(map, YIELD_PERIOD_KEY, String.class, wrapperName, DEFAULT_YIELD_PERIOD); + } + + @Override + public RemoteProcessingGroupSchema convert() { + Map result = mapSupplier.get(); + result.put(NAME_KEY, name); + result.put(URL_KEY, url); + result.put(COMMENT_KEY, comment); + result.put(TIMEOUT_KEY, timeout); + result.put(YIELD_PERIOD_KEY, yieldPeriod); + putListIfNotNull(result, INPUT_PORTS_KEY, inputPorts); + return new RemoteProcessingGroupSchema(result); + } + + public String getName() { + return name; + } + + public String getComment() { + return comment; + } + + public String getUrl() { + return url; + } + + public String getTimeout() { + return timeout; + } + + public String getYieldPeriod() { + return yieldPeriod; + } + + public List getInputPorts() { + return inputPorts; + } + + @Override + public int getVersion() { + return ConfigSchemaV1.CONFIG_VERSION; + } +} diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java index 76d1a5fef..2077f0b37 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java @@ -145,9 +145,6 @@ private static void enrichFlowSnippetDTO(FlowSnippetDTO flowSnippetDTO) { Set remoteProcessGroups = getAll(allFlowSnippets, FlowSnippetDTO::getRemoteProcessGroups).collect(Collectors.toSet()); - // RPGs with no name get Target URI as name - remoteProcessGroups.stream().filter(r -> StringUtil.isNullOrEmpty(r.getName())).forEach(r -> r.setName(r.getTargetUri())); - Map connectableNameMap = getAll(allFlowSnippets, FlowSnippetDTO::getProcessors).collect(Collectors.toMap(ComponentDTO::getId, ProcessorDTO::getName)); for (RemoteProcessGroupDTO remoteProcessGroupDTO : remoteProcessGroups) { diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaFunction.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaFunction.java index 2ad2e675f..95b9ff9b9 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaFunction.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaFunction.java @@ -40,6 +40,7 @@ public RemoteProcessingGroupSchemaFunction(RemoteInputPortSchemaFunction remoteI @Override public RemoteProcessingGroupSchema apply(RemoteProcessGroupDTO remoteProcessGroupDTO) { Map map = new HashMap<>(); + map.put(CommonPropertyKeys.ID_KEY, remoteProcessGroupDTO.getId()); map.put(CommonPropertyKeys.NAME_KEY, remoteProcessGroupDTO.getName()); map.put(RemoteProcessingGroupSchema.URL_KEY, remoteProcessGroupDTO.getTargetUri()); diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java index 9f7559ec3..10f263ef8 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java @@ -346,7 +346,7 @@ private void getMapDifferences(String path, List differences, Map extraKeys = new HashSet<>(actual.keySet()); extraKeys.removeAll(expected.keySet()); for (String extraKey : extraKeys) { - differences.add("Extra key: " + path + extraKey); + differences.add("Extra key: " + path + " " + extraKey); } } diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaTest.java index b12e16c45..bcd0a7717 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessingGroupSchemaTest.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -37,6 +38,7 @@ public class RemoteProcessingGroupSchemaTest extends BaseSchemaTester { private final RemoteInputPortSchemaTest remoteInputPortSchemaTest; + private String testId = UUID.randomUUID().toString(); private String testName = "testName"; private String testUrl = "testUrl"; private String testComment = "testComment"; @@ -53,6 +55,7 @@ public void setup() { remoteInputPortSchemaTest.setup(); dto = new RemoteProcessGroupDTO(); + dto.setId(testId); dto.setName(testName); dto.setTargetUri(testUrl); @@ -66,6 +69,7 @@ public void setup() { map = new HashMap<>(); + map.put(CommonPropertyKeys.ID_KEY, testId); map.put(CommonPropertyKeys.NAME_KEY, testName); map.put(RemoteProcessingGroupSchema.URL_KEY, testUrl); map.put(CommonPropertyKeys.INPUT_PORTS_KEY, new ArrayList<>(Arrays.asList(remoteInputPortSchemaTest.map))); @@ -74,11 +78,18 @@ public void setup() { map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, testYieldPeriod); } + @Test + public void testNoId() { + dto.setId(null); + map.remove(CommonPropertyKeys.ID_KEY); + assertDtoAndMapConstructorAreSame(1); + } + @Test public void testNoName() { dto.setName(null); map.remove(CommonPropertyKeys.NAME_KEY); - assertDtoAndMapConstructorAreSame(1); + assertDtoAndMapConstructorAreSame(0); } @Test diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml index 6a07ae440..3cc4ca773 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml @@ -249,7 +249,7 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer Remote Processing Groups: -- name: http://nifi:8080/nifi +- name: '' url: http://nifi:8080/nifi comment: '' timeout: 30 sec diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml index 38e5be277..99358baf8 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml @@ -24,7 +24,7 @@ limitations under the License. 1 GB 10000 - 5b0a5c18-a02e-4a85-8080-0187ee679763 + d41d8cd9-8f00-3204-a980-0998ecf8427e b23a4621-cf19-42e6-967c-ffd3716e6a24 REMOTE_INPUT_PORT @@ -216,7 +216,7 @@ limitations under the License. 1567.7883365880348 - 5b0a5c18-a02e-4a85-8080-0187ee679763 + d41d8cd9-8f00-3204-a980-0998ecf8427e b23a4621-cf19-42e6-967c-ffd3716e6a24 REMOTE_INPUT_PORT @@ -620,7 +620,7 @@ limitations under the License. org.apache.nifi.processors.standard.InvokeHTTP - 5b0a5c18-a02e-4a85-8080-0187ee679763 + d41d8cd9-8f00-3204-a980-0998ecf8427e ee27ad42-d125-4d09-a9f5-c42da132ab6b 636.0 diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml index b518f2895..30e33f41d 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml @@ -282,7 +282,8 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer Remote Processing Groups: -- name: http://nifi:8080/nifi +- id: d41d8cd9-8f00-3204-a980-0998ecf8427e + name: '' url: http://nifi:8080/nifi comment: '' timeout: 30 sec diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml index 7783c531a..064cd44a5 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml @@ -174,7 +174,8 @@ Process Groups: flowfile expiration: 0 sec queue prioritizer class: '' Remote Processing Groups: - - name: http://localhost:9091/nifi + - id: 21a651bb-0158-1000-0000-000000000000 + name: '' url: http://localhost:9091/nifi comment: '' timeout: 30 sec @@ -232,7 +233,8 @@ Process Groups: flowfile expiration: 0 sec queue prioritizer class: '' Remote Processing Groups: - - name: http://localhost:9090/nifi + - id: 21a4e2f0-0158-1000-0000-000000000000 + name: '' url: http://localhost:9090/nifi comment: '' timeout: 30 sec diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml index dab1e8e96..b6a490239 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml @@ -82,7 +82,8 @@ Connections: flowfile expiration: 60 sec queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer Remote Processing Groups: -- name: NiFi Flow +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e3 + name: NiFi Flow url: https://localhost:8090/nifi comment: '' timeout: 30 secs From 1871e5ad65b520e3199de9f67181991cf15ea53c Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 23 Nov 2016 12:15:06 -0500 Subject: [PATCH 31/33] Treat bcpkix as provided dependency like bcprov. --- minifi-assembly/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 89ed9e4cb..875c91819 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -188,6 +188,11 @@ limitations under the License. bcprov-jdk15on compile + + org.bouncycastle + bcpkix-jdk15on + compile + org.eclipse.jetty jetty-util From b2d40e78dd01d12a59bc326f961ebf900d5bcea8 Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 23 Nov 2016 16:33:22 -0500 Subject: [PATCH 32/33] Correct handling of provenance reporting task. --- minifi-assembly/pom.xml | 12 +++ .../bootstrap/util/ConfigTransformer.java | 4 +- .../minifi-provenance-reporting-nar/pom.xml | 84 +++++++++++++++++++ .../pom.xml | 31 +++++++ minifi-nar-bundles/pom.xml | 1 + pom.xml | 20 +++++ 6 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml create mode 100644 minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml diff --git a/minifi-assembly/pom.xml b/minifi-assembly/pom.xml index 875c91819..68271102f 100644 --- a/minifi-assembly/pom.xml +++ b/minifi-assembly/pom.xml @@ -135,6 +135,12 @@ limitations under the License. minifi-update-attribute-nar nar + + org.apache.nifi.minifi + minifi-provenance-reporting-nar + nar + 0.1.0-SNAPSHOT + @@ -181,6 +187,12 @@ limitations under the License. org.apache.nifi nifi-persistent-provenance-repository + + org.apache.nifi + nifi-site-to-site-reporting-task + 1.0.0 + compile + diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index a5b51e0a1..98fccb559 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -80,7 +80,7 @@ public final class ConfigTransformer { // Underlying version of NIFI will be using public static final String NIFI_VERSION = "1.0.0"; public static final String ROOT_GROUP = "Root-Group"; - public static final String DEFAULT_PROV_REPORTING_TASK_CLASS = "org.apache.nifi.SiteToSiteProvenanceReportingTask"; + public static final String DEFAULT_PROV_REPORTING_TASK_CLASS = "org.apache.nifi.reporting.SiteToSiteProvenanceReportingTask"; // Final util classes should have private constructor private ConfigTransformer() { @@ -480,7 +480,7 @@ protected static void addProvenanceReportingTask(final Element element, ConfigSc Map attributes = new HashMap<>(); attributes.put("Destination URL", provenanceProperties.getDestinationUrl()); attributes.put("Input Port Name", provenanceProperties.getPortName()); - attributes.put("MiNiFi URL", provenanceProperties.getOriginatingUrl()); + attributes.put("Instance URL", provenanceProperties.getOriginatingUrl()); attributes.put("Compress Events", provenanceProperties.getUseCompression()); attributes.put("Batch Size", provenanceProperties.getBatchSize()); attributes.put("Communications Timeout", provenanceProperties.getTimeout()); diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml new file mode 100644 index 000000000..d353b91d3 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/minifi-provenance-reporting-nar/pom.xml @@ -0,0 +1,84 @@ + + + 4.0.0 + + org.apache.nifi.minifi + minifi-provenance-reporting-bundle + 0.1.0-SNAPSHOT + + + minifi-provenance-reporting-nar + nar + + true + true + + + + org.apache.nifi + nifi-site-to-site-reporting-task + 1.0.0 + + + org.apache.nifi.minifi + minifi-standard-services-api-nar + nar + + + + + org.apache.commons + commons-lang3 + provided + + + org.codehaus.jackson + jackson-mapper-asl + provided + + + org.apache.httpcomponents + httpclient + provided + + + org.apache.httpcomponents + httpcore + provided + + + org.apache.httpcomponents + httpcore-nio + provided + + + org.apache.nifi + nifi-client-dto + provided + + + commons-io + commons-io + provided + + + org.apache.httpcomponents + httpasyncclient + provided + + + + \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml new file mode 100644 index 000000000..7863a1507 --- /dev/null +++ b/minifi-nar-bundles/minifi-provenance-reporting-bundle/pom.xml @@ -0,0 +1,31 @@ + + + + 4.0.0 + + org.apache.nifi.minifi + minifi-nar-bundles + 0.1.0-SNAPSHOT + + + minifi-provenance-reporting-bundle + pom + + + minifi-provenance-reporting-nar + + + diff --git a/minifi-nar-bundles/pom.xml b/minifi-nar-bundles/pom.xml index e60b8d7a9..f2a100623 100644 --- a/minifi-nar-bundles/pom.xml +++ b/minifi-nar-bundles/pom.xml @@ -26,6 +26,7 @@ minifi-framework-bundle minifi-provenance-repository-bundle + minifi-provenance-reporting-bundle minifi-update-attribute-nar minifi-standard-services-api-nar minifi-standard-nar diff --git a/pom.xml b/pom.xml index d167191de..44c36fe92 100644 --- a/pom.xml +++ b/pom.xml @@ -581,6 +581,26 @@ limitations under the License. 1.54 provided + + org.apache.httpcomponents + httpcore + 4.4.1 + + + org.apache.httpcomponents + httpcore-nio + 4.4.5 + + + org.apache.nifi + nifi-client-dto + ${org.apache.nifi.version} + + + org.apache.httpcomponents + httpasyncclient + 4.1.1 + From 715a211ed599cb860286391d34b5b7d64782a5db Mon Sep 17 00:00:00 2001 From: Aldrin Piri Date: Wed, 23 Nov 2016 18:01:18 -0500 Subject: [PATCH 33/33] Adjust provided dependencies. --- .../minifi-framework-nar/pom.xml | 20 +++++++++++++++++++ .../minifi-ssl-context-service-nar/pom.xml | 20 +++++++++++++++++++ .../minifi-standard-services-api-nar/pom.xml | 15 ++++++++++++++ pom.xml | 5 +++++ 4 files changed, 60 insertions(+) diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml index 38159525e..fcfde2404 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework-nar/pom.xml @@ -98,5 +98,25 @@ limitations under the License. compile + + + ch.qos.logback + logback-classic + provided + + + ch.qos.logback + logback-core + provided + + + org.apache.commons + commons-lang3 + + + commons-io + commons-io + + \ No newline at end of file diff --git a/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml b/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml index 35f27fa97..91ee6af0c 100644 --- a/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml +++ b/minifi-nar-bundles/minifi-ssl-context-service-nar/pom.xml @@ -57,5 +57,25 @@ + + + + ch.qos.logback + logback-classic + provided + + + ch.qos.logback + logback-core + provided + + + org.apache.commons + commons-lang3 + + + commons-io + commons-io + diff --git a/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml index cb4793cb0..06b10c750 100644 --- a/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml +++ b/minifi-nar-bundles/minifi-standard-services-api-nar/pom.xml @@ -60,6 +60,7 @@ nifi-hbase-client-service-api compile + org.bouncycastle @@ -76,5 +77,19 @@ bcpkix-jdk15on provided + + ch.qos.logback + logback-classic + provided + + + org.apache.commons + commons-lang3 + + + commons-io + commons-io + + diff --git a/pom.xml b/pom.xml index 44c36fe92..02adbb89d 100644 --- a/pom.xml +++ b/pom.xml @@ -246,6 +246,11 @@ limitations under the License. logback-classic ${logback.version} + + ch.qos.logback + logback-core + ${logback.version} + ch.qos.logback jcl-over-slf4j