Skip to content

Commit

Permalink
NIFI-9901 Added nifi-xml-processing to nifi-commons
Browse files Browse the repository at this point in the history 
- Refactored XML parsing to use providers from nifi-xml-processing
- Configured spotbugs-maven-plugin with findsecbugs-plugin in nifi-xml-processing
- Disabled Validate DTD in default configuration for EvaluateXPath and EvaluateXQuery
- Replaced configuration of DocumentBuilder and streaming XML Readers with shared components
- Removed XML utilities from nifi-security-utils
- Moved Commons Configuration classes to nifi-lookup-services

This closes #5962
Signed-off-by: Paul Grey <greyp@apache.org>
  • Loading branch information
@exceptionfactory @joewitt
exceptionfactory authored and joewitt committed Apr 14, 2022
1 parent fa82764 commit 5d94e7f
Show file tree
Hide file tree
Showing 120 changed files with 2,082 additions and 1,273 deletions.
4 changes: 4 additions & 0 deletions minifi/minifi-bootstrap/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ limitations under the License.
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
Expand Down
19 changes: 8 additions & 11 deletions ...nifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,14 @@
import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
Expand Down Expand Up @@ -148,7 +147,7 @@ protected static void writeNiFiPropertiesFile(ByteArrayOutputStream nifiProperti
}
}

protected static void writeFlowXmlFile(ConfigSchema configSchema, OutputStream outputStream) throws TransformerException, ConfigTransformerException, ConfigurationChangeException, IOException {
protected static void writeFlowXmlFile(ConfigSchema configSchema, OutputStream outputStream) throws TransformerException, ConfigTransformerException {
final StreamResult streamResult = new StreamResult(outputStream);

// configure the transformer and convert the DOM
Expand Down Expand Up @@ -307,14 +306,12 @@ protected static void writeNiFiProperties(ConfigSchema configSchema, OutputStrea
}
}

protected static DOMSource createFlowXml(ConfigSchema configSchema) throws IOException, ConfigurationChangeException, ConfigTransformerException {
protected static DOMSource createFlowXml(ConfigSchema configSchema) throws ConfigTransformerException {
try {
// create a new, empty document
final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setNamespaceAware(true);

final DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
final Document doc = docBuilder.newDocument();
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
final Document doc = documentProvider.newDocument();

// populate document with controller state
final Element rootNode = doc.createElement("flowController");
Expand Down Expand Up @@ -365,7 +362,7 @@ protected static DOMSource createFlowXml(ConfigSchema configSchema) throws IOExc
}

return new DOMSource(doc);
} catch (final ParserConfigurationException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) {
} catch (final ProcessingException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) {
throw new ConfigTransformerException(e);
} catch (Exception e) {
throw new ConfigTransformerException("Failed to parse the config YAML while writing the top level of the flow xml", e);
Expand Down
39 changes: 12 additions & 27 deletions ...-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
import org.apache.nifi.util.StringUtils;
import org.apache.nifi.xml.processing.parsers.DocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
Expand All @@ -42,9 +44,6 @@
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
Expand All @@ -55,7 +54,6 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringBufferInputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -80,15 +78,14 @@ public class ConfigTransformerTest {
Arrays.asList("processor", "inputPort", "outputPort", "funnel", "processGroup", "remoteProcessGroup", "connection"));
private XPathFactory xPathFactory;
private Element config;
private DocumentBuilder documentBuilder;

@Rule
final public TemporaryFolder tempOutputFolder = new TemporaryFolder();

@Before
public void setup() throws ParserConfigurationException {
documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
final Document document = documentBuilder.newDocument();
public void setup() {
final DocumentProvider documentProvider = new StandardDocumentProvider();
final Document document = documentProvider.newDocument();
config = document.createElement("config");
xPathFactory = XPathFactory.newInstance();
}
Expand Down Expand Up @@ -484,11 +481,11 @@ public void checkSSLOverrides() throws Exception {
assertTrue(flowXml.exists());
assertTrue(flowXml.canRead());

String flow = loadFlowXML(new FileInputStream(flowXml));

DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
Document xml = db.parse(new StringBufferInputStream(flow));
final DocumentProvider documentProvider = new StandardDocumentProvider();
final Document xml;
try (final InputStream inputStream = new GZIPInputStream(new FileInputStream(flowXml))) {
xml = documentProvider.parse(inputStream);
}

XPath xPath = XPathFactory.newInstance().newXPath();
String result = xPath.evaluate("/flowController/rootGroup/processor/property[name = \"SSL Context Service\"]/value/text()", xml);
Expand All @@ -504,7 +501,8 @@ public void testConfigFileTransform(String configFile) throws Exception {

ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ConfigTransformer.writeFlowXmlFile(configSchema, outputStream);
Document document = documentBuilder.parse(new ByteArrayInputStream(outputStream.toByteArray()));
final DocumentProvider documentProvider = new StandardDocumentProvider();
Document document = documentProvider.parse(new ByteArrayInputStream(outputStream.toByteArray()));

testProcessGroup((Element) xPathFactory.newXPath().evaluate("flowController/rootGroup", document, XPathConstants.NODE), configSchema.getProcessGroupSchema());
testReportingTasks((Element) xPathFactory.newXPath().evaluate("flowController/reportingTasks", document, XPathConstants.NODE), configSchema.getReportingTasksSchema());
Expand Down Expand Up @@ -786,17 +784,4 @@ public static Properties getTestBootstrapProperties(final String fileName) throw
}
return bootstrapProperties;
}

public static String loadFlowXML(InputStream compressedData) throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
GZIPInputStream gzipInputStream = new GZIPInputStream(compressedData);

byte[] buffer = new byte[1024];
int len;
while ((len = gzipInputStream.read(buffer)) != -1) {
byteArrayOutputStream.write(buffer, 0, len);
}

return byteArrayOutputStream.toString();
}
}
5 changes: 5 additions & 0 deletions minifi/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,11 @@ limitations under the License.
<artifactId>nifi-security-utils</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-logging-utils</artifactId>
Expand Down
5 changes: 5 additions & 0 deletions nifi-bootstrap/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ language governing permissions and limitations under the License. -->
<artifactId>nifi-security-utils</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-flow-encryptor</artifactId>
Expand Down
36 changes: 4 additions & 32 deletions nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,14 @@
import org.apache.nifi.components.resource.StandardResourceReferenceFactory;
import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.registry.VariableRegistry;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
Expand Down Expand Up @@ -91,27 +87,6 @@ public void setMaxNotificationAttempts(final int maxAttempts) {
this.maxAttempts = maxAttempts;
}

private static DocumentBuilder createSafeDocumentBuilder() throws ParserConfigurationException {
final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setNamespaceAware(false);

// These features are used to disable processing external entities in the DocumentBuilderFactory to protect against XXE attacks
final String DISALLOW_DOCTYPES = "http://apache.org/xml/features/disallow-doctype-decl";
final String ALLOW_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities";
final String ALLOW_EXTERNAL_PARAM_ENTITIES = "http://xml.org/sax/features/external-parameter-entities";
final String ALLOW_EXTERNAL_DTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd";

// Disable DTDs and external entities to protect against XXE
docFactory.setAttribute(DISALLOW_DOCTYPES, true);
docFactory.setAttribute(ALLOW_EXTERNAL_DTD, false);
docFactory.setAttribute(ALLOW_EXTERNAL_GENERAL_ENTITIES, false);
docFactory.setAttribute(ALLOW_EXTERNAL_PARAM_ENTITIES, false);
docFactory.setXIncludeAware(false);
docFactory.setExpandEntityReferences(false);

return docFactory.newDocumentBuilder();
}

/**
* Loads the Notification Services from the given XML configuration file.
*
Expand Down Expand Up @@ -143,17 +118,14 @@ private static DocumentBuilder createSafeDocumentBuilder() throws ParserConfigur
*
* @param servicesFile the XML file to load services from.
* @throws IOException if unable to read from the given file
* @throws ParserConfigurationException if unable to parse the given file as XML properly
* @throws SAXException if unable to parse the given file properly
*/
public void loadNotificationServices(final File servicesFile) throws IOException, ParserConfigurationException, SAXException {
final DocumentBuilder docBuilder = createSafeDocumentBuilder();

public void loadNotificationServices(final File servicesFile) throws IOException {
final Map<String, ConfiguredNotificationService> serviceMap = new HashMap<>();
try (final InputStream fis = new FileInputStream(servicesFile);
final InputStream in = new BufferedInputStream(fis)) {

final Document doc = docBuilder.parse(new InputSource(in));
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
final Document doc = documentProvider.parse(in);
final List<Element> serviceElements = getChildElementsByTagName(doc.getDocumentElement(), "service");
logger.debug("Found {} service elements", serviceElements.size());

Expand Down
66 changes: 0 additions & 66 deletions nifi-commons/nifi-security-utils/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,6 @@
<artifactId>nifi-security-utils-api</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-kms</artifactId>
<version>1.16.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
Expand All @@ -73,67 +68,6 @@
<artifactId>bcrypt</artifactId>
<version>0.9.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
<version>2.7</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes combine.children="append">
<exclude>src/test/resources/xxe_template.xml</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>

<profiles>
<profile>
<!-- This profile, activating when compiling on Java versions above 1.8, provides configuration changes to
allow NiFi to be compiled on those JDKs. -->
<id>jigsaw</id>
<activation>
<jdk>(1.8,)</jdk>
</activation>
<dependencies>
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.sun.activation</groupId>
<artifactId>javax.activation</artifactId>
</dependency>
</dependencies>
</profile>
</profiles>


<scm>
<tag>nifi-1.16.0-RC3</tag>
</scm>
</project>

Loading

0 comments on commit 5d94e7f

Please sign in to comment.