Skip to content

Commit

Permalink
Merge pull request xmlcalabash#4 from ndw/file-content-types
Browse files Browse the repository at this point in the history 
Cleanup File inputs and DefaultDocumentManager
  • Loading branch information
@ndw
ndw committed Oct 23, 2021
2 parents 4b52705 + c14607e commit eccd853
Show file tree
Hide file tree
Showing 8 changed files with 104 additions and 110 deletions.
2 changes: 1 addition & 1 deletion build.sbt
View file
Expand Up @@ -2,7 +2,7 @@ import java.io.{BufferedReader, InputStreamReader}

enablePlugins(JavaAppPackaging)

lazy val xmlCalabashVersion = "2.99.6"
lazy val xmlCalabashVersion = "2.99.7"
lazy val jafplVersion = "0.3.75"
lazy val saxonVersion = "10.6"
lazy val useSaxonEE = Option(System.getProperty("saxonEdition")).getOrElse("HE") == "EE"
Expand Down
133 changes: 67 additions & 66 deletions src/main/scala/com/xmlcalabash/XMLCalabash.scala
View file
Expand Up @@ -22,7 +22,7 @@ import net.sf.saxon.s9api.{ItemType, Processor, QName, XdmAtomicValue, XdmNode,
import org.slf4j.{Logger, LoggerFactory}
import org.xml.sax.{EntityResolver, InputSource}

import java.io.{ByteArrayInputStream, FileInputStream}
import java.io.{BufferedReader, ByteArrayInputStream, FileInputStream, FileReader}
import java.net.URI
import java.nio.charset.StandardCharsets
import javax.xml.transform.URIResolver
Expand Down Expand Up @@ -84,12 +84,10 @@ class XMLCalabash private(userProcessor: Option[Processor], val configurer: XPro
private var _errorExplanation: ErrorExplanation = _
private var _documentManager: DocumentManager = _
private var _htmlSerializer = false
private var _trim_inline_whitespace = false
private var _staticBaseURI = URIUtils.cwdAsURI
private var _locale = defaultLocale
private var _episode = computeEpisode
private var _showErrors = false
private var _builtinSteps = ListBuffer.empty[Library]
private val _builtinSteps = ListBuffer.empty[Library]
private val _importedURIs = mutable.HashMap.empty[URI, DeclContainer]
// Do not allow the order to be random
private val _imports = ListBuffer.empty[URI]
Expand Down Expand Up @@ -377,17 +375,20 @@ class XMLCalabash private(userProcessor: Option[Processor], val configurer: XPro
case v: PipelineUriOption => new XdmAtomicValue(v.value)
case v: PipelineUntypedOption => new XdmAtomicValue(v.value, ItemType.UNTYPED_ATOMIC)
case v: PipelineDocumentOption =>
val uri = v.value match {
v.value match {
case doc: PipelineFilenameDocument =>
URIUtils.cwdAsURI.resolve(doc.value)
val req = new DocumentRequest(URIUtils.cwdAsURI.resolve(doc.value))
val resp = documentManager.parse(req)
resp.value
case doc: PipelineURIDocument =>
URIUtils.cwdAsURI.resolve(doc.value)
val req = new DocumentRequest(URIUtils.cwdAsURI.resolve(doc.value))
val resp = documentManager.parse(req)
resp.value
case doc: PipelineFileDocument =>
doc.value.toURI
val req = new DocumentRequest(doc.value.toURI, doc.contentType)
val resp = documentManager.parse(req, new FileInputStream(doc.value))
resp.value
}
val req = new DocumentRequest(uri)
val doc = documentManager.parse(req)
doc.value
case v: PipelineExpressionOption =>
val bindings = mutable.HashMap.empty[String,Message]
for ((name, value) <- options) {
Expand All @@ -401,7 +402,7 @@ class XMLCalabash private(userProcessor: Option[Processor], val configurer: XPro
case v: PipelineXdmValueOption =>
v.value
case _ =>
throw new RuntimeException("Unexpected expression value")
throw XProcException.xiThisCantHappen(s"Unexpected pipeline option type: ${opt}")
}
if (options.contains(name)) {
val newvalue = options(name).value.append(value)
Expand Down Expand Up @@ -434,69 +435,69 @@ class XMLCalabash private(userProcessor: Option[Processor], val configurer: XPro
}

def run(): Unit = {
loadPipeline()

if (!debugOptions.run) {
return
}

val context = new StaticContext(this)

for (port <- _inputs.keySet) {
for (opt <- _inputs(port)) {
val doc = opt match {
case in: PipelineInputFilename =>
loadDocument(context, URIUtils.cwdAsURI.resolve(in.value))
case in: PipelineInputURI =>
loadDocument(context, URIUtils.cwdAsURI.resolve(in.value))
case in: PipelineInputFile =>
val request = new DocumentRequest(URIUtils.cwdAsURI.resolve(in.value.getAbsolutePath))
val response = documentManager.parse(request, new FileInputStream(in.value))
val meta = new XProcMetadata(response.contentType, response.props)
new XdmValueItemMessage(response.value, meta, context)
case in: PipelineInputXdm =>
val meta = new XProcMetadata(MediaType.XML)
new XdmValueItemMessage(in.value, meta, context)
case in: PipelineInputText =>
val bais = new ByteArrayInputStream(in.text.getBytes(StandardCharsets.UTF_8))
val request = new DocumentRequest(URIUtils.cwdAsURI, in.contentType)
val response = documentManager.parse(request, bais)
val meta = new XProcMetadata(response.contentType, response.props)
new XdmValueItemMessage(response.value, meta, context)
try {
loadPipeline()

if (!debugOptions.run) {
return
}

val context = new StaticContext(this)

for (port <- _inputs.keySet) {
for (opt <- _inputs(port)) {
val doc = opt match {
case in: PipelineInputFilename =>
loadDocument(context, URIUtils.cwdAsURI.resolve(in.value))
case in: PipelineInputURI =>
loadDocument(context, URIUtils.cwdAsURI.resolve(in.value))
case in: PipelineInputFile =>
val request = new DocumentRequest(URIUtils.cwdAsURI.resolve(in.value.getAbsolutePath), in.contentType)
val response = documentManager.parse(request, new FileInputStream(in.value))
val meta = new XProcMetadata(response.contentType, response.props)
new XdmValueItemMessage(response.value, meta, context)
case in: PipelineInputXdm =>
val meta = new XProcMetadata(MediaType.XML)
new XdmValueItemMessage(in.value, meta, context)
case in: PipelineInputText =>
val bais = new ByteArrayInputStream(in.text.getBytes(StandardCharsets.UTF_8))
val request = new DocumentRequest(URIUtils.cwdAsURI, in.contentType)
val response = documentManager.parse(request, bais)
val meta = new XProcMetadata(response.contentType, response.props)
new XdmValueItemMessage(response.value, meta, context)
}
_runtime.input(port, doc.item, doc.metadata)
}
_runtime.input(port, doc.item, doc.metadata)
}
}

for (port <- runtime.outputs) {
val output = runtime.decl.output(port)
val pc = if (_outputs.contains(port)) {
val opts = _outputs(port)
if (opts.nonEmpty && opts.head.isInstanceOf[PipelineOutputConsumer]) {
opts.head.asInstanceOf[PipelineOutputConsumer].value
} else {
val files = ListBuffer.empty[String]
for (opt <- opts.toList) {
opt match {
case out: PipelineOutputFilename =>
files += out.value
case out: PipelineOutputURI =>
files += URIUtils.cwdAsURI.resolve(out.value).getPath
for (port <- runtime.outputs) {
val output = runtime.decl.output(port)
val pc = if (_outputs.contains(port)) {
val opts = _outputs(port)
if (opts.nonEmpty && opts.head.isInstanceOf[PipelineOutputConsumer]) {
opts.head.asInstanceOf[PipelineOutputConsumer].value
} else {
val files = ListBuffer.empty[String]
for (opt <- opts.toList) {
opt match {
case out: PipelineOutputFilename =>
files += out.value
case out: PipelineOutputURI =>
files += URIUtils.cwdAsURI.resolve(out.value).getPath
}
}
new PrintingConsumer(runtime, output, files.toList)
}
new PrintingConsumer(runtime, output, files.toList)
} else {
new PrintingConsumer(runtime, output)
}
} else {
new PrintingConsumer(runtime, output)
runtime.output(port, pc)
}
runtime.output(port, pc)
}

for ((name, value) <- _options) {
runtime.option(name, value.value, value.context)
}
for ((name, value) <- _options) {
runtime.option(name, value.value, value.context)
}

try {
runtime.run()
} catch {
case ex: Exception =>
Expand Down
3 changes: 3 additions & 0 deletions src/main/scala/com/xmlcalabash/runtime/PrintingConsumer.scala
View file
Expand Up @@ -10,8 +10,10 @@ import com.xmlcalabash.model.util.UniqueId
import com.xmlcalabash.model.xml.DeclareOutput
import com.xmlcalabash.util.S9Api
import net.sf.saxon.s9api.{QName, Serializer}
import org.slf4j.{Logger, LoggerFactory}

class PrintingConsumer private(config: XMLCalabashRuntime, output: DeclareOutput, outputs: Option[List[String]]) extends DataConsumer {
protected val logger: Logger = LoggerFactory.getLogger(this.getClass)
private val _id = UniqueId.nextId.toString
private var index = 0

Expand All @@ -24,6 +26,7 @@ class PrintingConsumer private(config: XMLCalabashRuntime, output: DeclareOutput
}

override def consume(port: String, message: Message): Unit = {
logger.debug(s"PrintingConsumer received message on ${port}")
message match {
case msg: XProcItemMessage =>
// Check that the message content type is allowed on the output port
Expand Down
2 changes: 1 addition & 1 deletion src/main/scala/com/xmlcalabash/runtime/XMLCalabashRuntime.scala
View file
Expand Up @@ -182,7 +182,7 @@ class XMLCalabashRuntime protected[xmlcalabash] (val decl: DeclareStep) extends
try {
runtime.runSync()
} catch {
case ex: Throwable =>
case ex: Exception =>
ex match {
case j: JafplException =>
throw JafplExceptionMapper.remap(j)
Expand Down
18 changes: 13 additions & 5 deletions src/main/scala/com/xmlcalabash/util/ArgBundle.scala
View file
Expand Up @@ -56,7 +56,7 @@ class ArgBundle() {
}

def pipeline(xpl: File): Unit = {
_pipeline = Some(new PipelineFileDocument(xpl))
_pipeline = Some(new PipelineFileDocument(xpl, MediaType.XML))
}

def pipeline(xpl: XdmNode): Unit = {
Expand All @@ -75,8 +75,12 @@ class ArgBundle() {
_parameters += new PipelineInputURI(port, document)
}

def input(port: String, document: File): Unit = {
_parameters += new PipelineInputFile(port, document)
def input(port: String, document: File, mediaType: MediaType): Unit = {
_parameters += new PipelineInputFile(port, document, mediaType)
}

def input(port: String, document: File, contentType: String): Unit = {
input(port, document, MediaType.parse(contentType))
}

def input(port: String, document: XdmNode): Unit = {
Expand Down Expand Up @@ -131,8 +135,12 @@ class ArgBundle() {
_parameters += new PipelineDocumentOption(eqname, new PipelineURIDocument(document))
}

def optionDocument(eqname: String, document: File): Unit = {
_parameters += new PipelineDocumentOption(eqname, new PipelineFileDocument(document))
def optionDocument(eqname: String, document: File, mediaType: MediaType): Unit = {
_parameters += new PipelineDocumentOption(eqname, new PipelineFileDocument(document, mediaType))
}

def optionDocument(eqname: String, document: File, contentType: String): Unit = {
optionDocument(eqname, document, MediaType.parse(contentType))
}

def optionExpression(eqname: String, value: String): Unit = {
Expand Down
41 changes: 10 additions & 31 deletions src/main/scala/com/xmlcalabash/util/DefaultDocumentManager.scala
View file
Expand Up @@ -11,7 +11,7 @@ import com.xmlcalabash.model.util.{SaxonTreeBuilder, XProcConstants}
import com.xmlcalabash.runtime.{BinaryNode, StaticContext, XProcMetadata, XProcXPathExpression}
import com.xmlcalabash.util.xc.Errors
import net.sf.saxon.Configuration
import net.sf.saxon.lib.ErrorReporter
import net.sf.saxon.lib.{AugmentedSource, ErrorReporter, ParseOptions}
import net.sf.saxon.s9api.{QName, SaxonApiException, XdmAtomicValue, XdmNode, XdmValue}
import net.sf.saxon.trans.XPathException
import nu.validator.htmlparser.common.XmlViolationPolicy
Expand Down Expand Up @@ -240,7 +240,15 @@ class DefaultDocumentManager(xmlCalabash: XMLCalabash) extends DocumentManager {
val value = parseHtml(request, new InputSource(stream)).value
new DocumentResponse(value, contentType, props)
} else if (contentType.xmlContentType) {
val source = new SAXSource(new InputSource(stream))
val errors = new Errors(xmlCalabash)
val listener = new CachingErrorListener(errors)

val parseOptions = new ParseOptions()
parseOptions.setEntityResolver(xmlCalabash.entityResolver)
parseOptions.setErrorReporter(listener)
parseOptions.setLineNumbering(true)

val source = new AugmentedSource(new SAXSource(new InputSource(stream)), parseOptions)

if (request.href.isDefined) {
source.setSystemId(request.href.get.toASCIIString)
Expand All @@ -260,28 +268,6 @@ class DefaultDocumentManager(xmlCalabash: XMLCalabash) extends DocumentManager {
}
}

// Is this necessary? I'm carefully synchronizing calls that mess with the
// configuration's error listener.

val errors = new Errors(xmlCalabash)
val listener = new CachingErrorListener(errors)
val saxonConfig = xmlCalabash.processor.getUnderlyingConfiguration

// FIXME: Saxon10
/*
saxonConfig.synchronized {
listener.chainedReporter = saxonConfig.makeErrorReporter()
saxonConfig.setErrorReporterFactory((config: Configuration) => {
def foo(config: Configuration): ErrorReporter = {
val reporter = listener
reporter
}
foo(config)
})
}
*/

val builder = xmlCalabash.processor.newDocumentBuilder
builder.setDTDValidation(request.dtdValidate)
builder.setLineNumbering(true)
Expand Down Expand Up @@ -321,13 +307,6 @@ class DefaultDocumentManager(xmlCalabash: XMLCalabash) extends DocumentManager {
throw XProcException.xdNotWFXML(href, msg, request.location)
}
}
} finally {
// FIXME: Saxon10
/*
saxonConfig.synchronized {
saxonConfig.setErrorListener(listener.chainedListener.get)
}
*/
}

new DocumentResponse(node, contentType, props)
Expand Down
7 changes: 5 additions & 2 deletions src/main/scala/com/xmlcalabash/util/PipelineParameter.scala
View file
Expand Up @@ -20,6 +20,7 @@ trait PipelineParameterURIValue {

trait PipelineParameterFileValue {
val value: File
val contentType: MediaType
}

trait PipelineParameterXdmNodeValue {
Expand All @@ -41,7 +42,7 @@ abstract class PipelineDocument() extends PipelineParameter { }

class PipelineFilenameDocument(val value: String) extends PipelineDocument with PipelineParameterStringValue {}
class PipelineURIDocument(val value: URI) extends PipelineDocument with PipelineParameterURIValue {}
class PipelineFileDocument(val value: File) extends PipelineDocument with PipelineParameterFileValue {}
class PipelineFileDocument(val value: File, val contentType: MediaType) extends PipelineDocument with PipelineParameterFileValue {}
class PipelineXdmDocument(val value: XdmNode) extends PipelineDocument with PipelineParameterXdmNodeValue {}
class PipelineTextDocument(val text: String, val contentType: MediaType) extends PipelineDocument with PipelineParameterTextValue {
def this(text: String, contentType: String) = this(text, MediaType.parse(contentType))
Expand All @@ -51,7 +52,9 @@ abstract class PipelineInputDocument(val port: String) extends PipelineDocument

class PipelineInputFilename(port: String, val value: String) extends PipelineInputDocument(port) with PipelineParameterStringValue {}
class PipelineInputURI(port: String, val value: URI) extends PipelineInputDocument(port) with PipelineParameterURIValue {}
class PipelineInputFile(port: String, val value: File) extends PipelineInputDocument(port) with PipelineParameterFileValue {}
class PipelineInputFile(port: String, val value: File, val contentType: MediaType) extends PipelineInputDocument(port) with PipelineParameterFileValue {
def this(port: String, value: File, contentType: String) = this(port, value, MediaType.parse(contentType))
}
class PipelineInputXdm(port: String, val value: XdmNode) extends PipelineInputDocument(port) with PipelineParameterXdmNodeValue {}
class PipelineInputText(port: String, val text: String, val contentType: MediaType) extends PipelineInputDocument(port) with PipelineParameterTextValue {
def this(port: String, text: String, contentType: String) = this(port, text, MediaType.parse(contentType))
Expand Down
8 changes: 4 additions & 4 deletions src/test/scala/com/xmlcalabash/test/XmlCalabashSpec.scala
View file
Expand Up @@ -25,7 +25,7 @@ class XmlCalabashSpec extends AnyFlatSpec {
val proc = XMLCalabash.newInstance()
proc.args.pipeline(identity_xpl)
proc.args.input("source", "src/test/resources/config.xml")
proc.args.input("source", new File("src/test/resources/config.xml"))
proc.args.input("source", new File("src/test/resources/config.xml"), MediaType.XML)
proc.args.input("source", URI.create(s"${System.getProperty("user.dir")}/src/test/resources/config.xml"))
proc.resolve()
}
Expand All @@ -34,7 +34,7 @@ class XmlCalabashSpec extends AnyFlatSpec {
val proc = XMLCalabash.newInstance()
proc.args.pipeline(identity_xpl)
proc.args.input("result1", "/tmp/one")
proc.args.input("result2", new File("/tmp/two"))
proc.args.input("result2", new File("/tmp/two"), MediaType.OCTET_STREAM)
proc.args.input("result3", URI.create("file:///tmp/three"))
proc.resolve()
}
Expand All @@ -43,7 +43,7 @@ class XmlCalabashSpec extends AnyFlatSpec {
val proc = XMLCalabash.newInstance()
proc.args.pipeline(identity_xpl)
proc.args.input("result", "/tmp/one")
proc.args.input("result", new File("/tmp/two"))
proc.args.input("result", new File("/tmp/two"), MediaType.XML)
proc.args.input("result", URI.create("file:///tmp/three"))
proc.resolve()
}
Expand Down Expand Up @@ -107,7 +107,7 @@ class XmlCalabashSpec extends AnyFlatSpec {
proc.args.pipeline(identity_xpl)
proc.args.optionDocument("json", URIUtils.cwdAsURI.resolve("src/test/resources/doc.json"))
proc.args.optionDocument("xmlstring", "src/test/resources/config.xml")
proc.args.optionDocument("xmlfile", new File("src/test/resources/config.xml"))
proc.args.optionDocument("xmlfile", new File("src/test/resources/config.xml"), MediaType.XML)
proc.resolve()
}

Expand Down

0 comments on commit eccd853

Please sign in to comment.