WIP on doctype restriction

daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala

@@ -28,7 +28,6 @@ import java.nio.channels.Channels
 import java.nio.file.Paths
 import java.util.Scanner
 import java.util.concurrent.Executors
-
 import com.typesafe.config.ConfigFactory
 
 import scala.concurrent.Await
@@ -37,7 +36,6 @@ import scala.concurrent.Future
 import scala.concurrent.duration.Duration
 import scala.xml.Node
 import scala.xml.SAXParseException
-
 import javax.xml.parsers.DocumentBuilderFactory
 import javax.xml.parsers.SAXParserFactory
 import javax.xml.transform.TransformerFactory
@@ -106,8 +104,10 @@ import org.rogach.scallop
 import org.rogach.scallop.ArgType
 import org.rogach.scallop.ScallopOption
 import org.rogach.scallop.ValueConverter
+import org.xml.sax.XMLReader
 
 import scala.util.matching.Regex
+import scala.xml.SAXParser
 
 class CommandLineSAXErrorHandler() extends org.xml.sax.ErrorHandler with Logging {
 
@@ -807,7 +807,18 @@ object Main extends Logging {
           case Left(bytes) => new ByteArrayInputStream(bytes)
           case Right(is) => is
         }
-        scala.xml.XML.load(is)
+        val parser: SAXParser = {
+          val f = SAXParserFactory.newInstance()
+          f.setNamespaceAware(false)
+          val p = f.newSAXParser()
+          p
+        }
+        //
+        // FIXME: This needs to use a Daffodil-created loader, so that we can
+        // insure that the XMLReader used from it has the setSecureDefaults treatment.
+        //
+        System.err.println("FIXME: Needs to use a Daffodil-created loader.")
+        scala.xml.XML.withSAXParser(parser).load(is)
       }
       case InfosetType.JDOM => {
         val is = data match {
@@ -1488,7 +1499,9 @@ object Main extends Logging {
   private def unparseWithSAX(
     is: InputStream,
     contentHandler: DFDL.DaffodilUnparseContentHandler): UnparseResult = {
-    val xmlReader = SAXParserFactory.newInstance.newSAXParser.getXMLReader
+    val xmlReader: XMLReader = SAXParserFactory.newInstance.newSAXParser.getXMLReader
+    // standard secure XML parsing features
+    XMLUtils.setSecureDefaults(xmlReader)
     xmlReader.setContentHandler(contentHandler)
     xmlReader.setFeature(XMLUtils.SAX_NAMESPACES_FEATURE, true)
     xmlReader.setFeature(XMLUtils.SAX_NAMESPACE_PREFIXES_FEATURE, true)
@@ -1512,6 +1525,7 @@ object Main extends Logging {
     saxXmlRdr.setErrorHandler(errorHandler)
     saxXmlRdr.setProperty(XMLUtils.DAFFODIL_SAX_URN_BLOBDIRECTORY, blobDir)
     saxXmlRdr.setProperty(XMLUtils.DAFFODIL_SAX_URN_BLOBSUFFIX, blobSuffix)
+    XMLUtils.setSecureDefaults(saxXmlRdr)
     saxXmlRdr.parse(data)
     val pr = saxXmlRdr.getProperty(XMLUtils.DAFFODIL_SAX_URN_PARSERESULT).asInstanceOf[ParseResult]
     pr

daffodil-core/src/main/scala/org/apache/daffodil/dsom/DFDLSchemaFile.scala

@@ -145,7 +145,6 @@ final class DFDLSchemaFile(
       val ldr = new DaffodilXMLLoader(this)
       ldr.setValidation(true)
       try {
-        ldr.load(schemaSource) // validate as XML file with XML Schema for DFDL Schemas
         ldr.validateSchema(schemaSource) // validate as XSD (catches UPA errors for example)
       } catch {
         // ok to absorb SAX Parse Exception as we've captured those errors in error handling

daffodil-core/src/test/scala/org/apache/daffodil/processor/TestSAXParseUnparseAPI.scala

@@ -51,6 +51,7 @@ class TestSAXParseUnparseAPI {
 
     val unparseXMLReader = SAXParserFactory.newInstance().newSAXParser().getXMLReader
     unparseXMLReader.setFeature(XMLUtils.SAX_NAMESPACES_FEATURE, true)
+    XMLUtils.setSecureDefaults(unparseXMLReader)
     val baosUnparse = new ByteArrayOutputStream()
     val wbcUnparse = java.nio.channels.Channels.newChannel(baosUnparse)
     val unparseContentHandler = dp.newContentHandlerInstance(wbcUnparse)
@@ -78,6 +79,7 @@ class TestSAXParseUnparseAPI {
 
     val unparseXMLReader = SAXParserFactory.newInstance().newSAXParser().getXMLReader
     unparseXMLReader.setFeature(XMLUtils.SAX_NAMESPACES_FEATURE, true)
+    XMLUtils.setSecureDefaults(unparseXMLReader)
     val baosUnparse = new ByteArrayOutputStream()
     val wbcUnparse = java.nio.channels.Channels.newChannel(baosUnparse)
     val unparseContentHandler = dp.newContentHandlerInstance(wbcUnparse)
@@ -101,6 +103,7 @@ class TestSAXParseUnparseAPI {
     val wbcUnparse = java.nio.channels.Channels.newChannel(baosUnparse)
     val unparseContentHandler = dp.newContentHandlerInstance(wbcUnparse)
     unparseXMLReader.setContentHandler(unparseContentHandler)
+    XMLUtils.setSecureDefaults(unparseXMLReader)
     val baisUnparse = new ByteArrayInputStream(testInfosetString.getBytes)
     val inputSourceUnparse = new InputSource(baisUnparse)
     unparseXMLReader.parse(inputSourceUnparse)
@@ -130,6 +133,7 @@ class TestSAXParseUnparseAPI {
     val wbcUnparse = java.nio.channels.Channels.newChannel(baosUnparse)
     val unparseContentHandler = dp.newContentHandlerInstance(wbcUnparse)
     unparseXMLReader.setContentHandler(unparseContentHandler)
+    XMLUtils.setSecureDefaults(unparseXMLReader)
     val baisUnparse = new ByteArrayInputStream(testInfosetString.getBytes)
     val inputSourceUnparse = new InputSource(baisUnparse)
     unparseXMLReader.parse(inputSourceUnparse)

daffodil-core/src/test/scala/org/apache/daffodil/processor/TestSAXUnparseAPI.scala

@@ -19,14 +19,14 @@ package org.apache.daffodil.processor
 
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
-
 import javax.xml.parsers.SAXParserFactory
 import org.apache.daffodil.Implicits.intercept
 import org.apache.daffodil.xml.XMLUtils
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertTrue
 import org.junit.Test
 import org.xml.sax.InputSource
+import org.xml.sax.SAXParseException
 import org.xml.sax.XMLReader
 
 class TestSAXUnparseAPI {
@@ -38,6 +38,7 @@ class TestSAXUnparseAPI {
    */
   @Test def testUnparseContentHandler_unparse(): Unit = {
     val xmlReader: XMLReader = SAXParserFactory.newInstance.newSAXParser.getXMLReader
+    XMLUtils.setSecureDefaults(xmlReader)
     val bao = new ByteArrayOutputStream()
     val wbc = java.nio.channels.Channels.newChannel(bao)
     val unparseContentHandler = dp.newContentHandlerInstance(wbc)
@@ -69,6 +70,7 @@ class TestSAXUnparseAPI {
    */
   @Test def testUnparseContentHandler_unparse_namespace_feature(): Unit = {
     val xmlReader: XMLReader = SAXParserFactory.newInstance.newSAXParser.getXMLReader
+    XMLUtils.setSecureDefaults(xmlReader)
     val bao = new ByteArrayOutputStream()
     val wbc = java.nio.channels.Channels.newChannel(bao)
     val unparseContentHandler = dp.newContentHandlerInstance(wbc)
@@ -87,6 +89,7 @@ class TestSAXUnparseAPI {
    */
   @Test def testUnparseContentHandler_unparse_namespace_prefix_feature(): Unit = {
     val xmlReader: XMLReader = SAXParserFactory.newInstance.newSAXParser.getXMLReader
+    XMLUtils.setSecureDefaults(xmlReader)
     val bao = new ByteArrayOutputStream()
     val wbc = java.nio.channels.Channels.newChannel(bao)
     val unparseContentHandler = dp.newContentHandlerInstance(wbc)
@@ -99,4 +102,40 @@ class TestSAXUnparseAPI {
     assertTrue(!ur.isError)
     assertEquals(testData, bao.toString)
   }
+
+  /**
+   * Verifies if the XML reader has secure defaults, then unparsing does
+   * detect and refuse the use of DOCTYPE decls.
+   *
+   * This is part of fixing DAFFODIL-1422, DAFFODIL-1659 - disallow doctype decls.
+   */
+  @Test def testUnparse_NoDocType_feature(): Unit = {
+    val xmlReader: XMLReader = SAXParserFactory.newInstance.newSAXParser.getXMLReader
+    //
+    // In case of unparsing, we can't really insist that setSecureDefaults is called, because
+    // there is no Daffodil API call that is passed the xmlReader where we can enforce that this
+    // setSecureDefaults is called.
+    //
+    XMLUtils.setSecureDefaults(xmlReader)
+    val bao = new ByteArrayOutputStream()
+    val wbc = java.nio.channels.Channels.newChannel(bao)
+    val unparseContentHandler = dp.newContentHandlerInstance(wbc)
+    xmlReader.setContentHandler(unparseContentHandler)
+    xmlReader.setFeature(XMLUtils.SAX_NAMESPACES_FEATURE, false)
+    xmlReader.setFeature(XMLUtils.SAX_NAMESPACE_PREFIXES_FEATURE, true)
+    val xmlWithDocType = """<?xml version="1.0" ?>
+      <!DOCTYPE root_element [
+        Document Type Definition (DTD):
+        elements/attributes/entities/notations/
+          processing instructions/comments/PE references
+    ]>
+    <list xmlns="http://example.com"><w>9</w><w>1</w><w>0</w></list>
+    """
+    val bai = new ByteArrayInputStream(xmlWithDocType.getBytes)
+    val e = intercept[SAXParseException] {
+      xmlReader.parse(new InputSource(bai))
+    }
+    val m = e.getMessage()
+    assertTrue(m.contains("DOCTYPE is disallowed"))
+  }
 }

daffodil-core/src/test/scala/org/apache/daffodil/util/TestUtils.scala

@@ -85,6 +85,17 @@ object TestUtils {
     runSchemaOnRBC(testSchema, Misc.stringToReadableByteChannel(data), areTracing)
   }
 
+  /**
+   * Exposes the data processor object so that you can test its API conveniently.
+   */
+  def dataProcessorForTestString(testSchema: Node, data: String): (DataProcessor, InputStream) = {
+    val rbc = Misc.stringToReadableByteChannel(data)
+    val is = Channels.newInputStream(rbc)
+    val p = compileSchema(testSchema)
+    (p, is)
+    // runDataProcessorOnInputStream(p, is, areTracing)
+  }
+
   def testBinary(testSchema: Node, hexData: String, areTracing: Boolean = false): (DFDL.ParseResult, Node) = {
     val b = Misc.hex2Bytes(hexData)
     testBinary(testSchema, b, areTracing)

daffodil-japi/src/test/java/org/apache/daffodil/example/TestJavaAPI.java

@@ -40,6 +40,7 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.daffodil.japi.*;
 import org.apache.daffodil.japi.infoset.XMLTextInfosetOutputter;
+import org.apache.daffodil.xml.XMLUtils;
 import org.jdom2.output.Format;
 import org.junit.Test;
 
@@ -1008,6 +1009,7 @@ public void testJavaAPI20() throws IOException, ClassNotFoundException {
         try {
             org.xml.sax.XMLReader unparseXMLReader = javax.xml.parsers.SAXParserFactory.newInstance()
                     .newSAXParser().getXMLReader();
+            XMLUtils.setSecureDefaults(unparseXMLReader);
             unparseXMLReader.setContentHandler(unparseContentHandler);
             unparseXMLReader.setErrorHandler(errorHandler);
             unparseXMLReader.setFeature(SAX_NAMESPACES_FEATURE, true);
@@ -1143,6 +1145,7 @@ public void testJavaAPI23() throws IOException, ClassNotFoundException {
         try {
             org.xml.sax.XMLReader unparseXMLReader = javax.xml.parsers.SAXParserFactory.newInstance()
                     .newSAXParser().getXMLReader();
+            XMLUtils.setSecureDefaults(unparseXMLReader);
             unparseXMLReader.setContentHandler(unparseContentHandler);
             unparseXMLReader.setFeature(SAX_NAMESPACES_FEATURE, true);
             unparseXMLReader.setFeature(SAX_NAMESPACE_PREFIXES_FEATURE, true);

daffodil-lib/src/main/scala/org/apache/daffodil/api/DaffodilSchemaSource.scala

@@ -17,15 +17,19 @@
 
 package org.apache.daffodil.api
 import org.xml.sax.InputSource
+
 import java.net.URI
 import scala.xml.Node
 import java.io.FileInputStream
 import org.apache.daffodil.xml.XMLUtils
 import org.apache.commons.io.input.XmlStreamReader
+
 import java.io.File
 import java.nio.file.Paths
 import org.apache.daffodil.exceptions.Assert
 import org.apache.daffodil.equality._
+
+import java.io.ByteArrayInputStream
 import java.nio.file.FileSystemNotFoundException
 
 /**
@@ -109,6 +113,27 @@ class URISchemaSource protected (val fileOrResource: URI) extends DaffodilSchema
   }
 }
 
+/**
+ * Convenient for testing. Allows creating XML strings for loading that contain things
+ * not supported by scala XML syntax like DOCTYPE decls.
+ *
+ * @param str - string that is loaded as XML.
+ */
+case class StringSchemaSource(str: String)
+extends DaffodilSchemaSource {
+
+  private val delegate = InputStreamSchemaSource(new ByteArrayInputStream(str.getBytes()), None, "string", "")
+  /**
+   * Use to get a org.xml.sax.InputSource for use by Xerces
+   */
+  override def newInputSource() = delegate.newInputSource()
+
+  /**
+   * Use to get the URI that can be used to load the xml.
+   */
+  override def uriForLoading = delegate.uriForLoading
+}
+
 /**
  * For stdin, or other anonymous pipe-like source of schema.
  */

daffodil-lib/src/main/scala/org/apache/daffodil/externalvars/ExternalVariablesXMLValidator.scala

@@ -17,22 +17,31 @@
 
 package org.apache.daffodil.externalvars
 
+import org.apache.daffodil.util.Misc
+import org.apache.daffodil.xml.DFDLCatalogResolver
+import org.apache.daffodil.xml.XMLUtils
+import org.apache.xerces.jaxp.validation.XMLSchemaFactory
+
 import javax.xml.transform.stream.StreamSource
 import org.xml.sax.SAXException
+
 import java.io.File
 
 object ExternalVariablesValidator {
 
   final val extVarXsd = {
-    val stream = this.getClass().getResourceAsStream("/xsd/dafext.xsd")
+    val uri = Misc.getRequiredResource("org/apache/daffodil/xsd/dafext.xsd")
+    val stream = uri.toURL.openStream()
     stream
   }
 
   def validate(xmlFile: File): Either[java.lang.Throwable, _] = {
     try {
-      val factory = new org.apache.xerces.jaxp.validation.XMLSchemaFactory()
+      val factory = new XMLSchemaFactory()
+      factory.setResourceResolver(DFDLCatalogResolver.get)
       val schema = factory.newSchema(new StreamSource(extVarXsd))
       val validator = schema.newValidator()
+      validator.setFeature(XMLUtils.XML_DISALLOW_DOCTYPE_FEATURE, true)
       validator.validate(new StreamSource(xmlFile))
     } catch {
       case ex: SAXException => Left(ex)

daffodil-lib/src/main/scala/org/apache/daffodil/xml/DaffodilConstructingLoader.scala

@@ -86,6 +86,14 @@ class DaffodilConstructingLoader(uri: URI, errorHandler: org.xml.sax.ErrorHandle
     Source.fromInputStream(is, enc)
   }, true) {
 
+  /**
+   * Ensures that DOCTYPES aka DTDs, if encountered, are rejected.
+   */
+  override def parseDTD(): Unit = {
+    val e = makeSAXParseException(pos, "DOCTYPE is disallowed.")
+    throw e
+  }
+
   // This one line is a bit of a hack to get consistent line numbers. The
   // scala-xml libary reads XML from a scala.io.Source which maintains private
   // line/col information about where in the Source we are reading from (i.e.