add inline document set and FSA manifest processing

arelle/CntlrWinMain.py

@@ -663,7 +663,7 @@ def showLoadedXbrl(self, modelXbrl, attach, selectTopView=False):
                 if modelXbrl.hasTableRendering:  # show rendering grid even without any facts
                     ViewWinRenderedGrid.viewRenderedGrid(modelXbrl, self.tabWinTopRt, lang=self.labelLang)
                     if topView is None: topView = modelXbrl.views[-1]
-                if modelXbrl.modelDocument.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
+                if modelXbrl.modelDocument.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET):
                     currentAction = "table view of facts"
                     if not modelXbrl.hasTableRendering: # table view only if not grid rendered view
                         ViewWinFactTable.viewFacts(modelXbrl, self.tabWinTopRt, lang=self.labelLang)

arelle/FunctionXs.py

@@ -8,6 +8,8 @@
 from arelle import (XPathContext, ModelValue)
 from arelle.FunctionUtil import (anytypeArg, atomicArg, stringArg, numericArg, qnameArg, nodeArg)
 from arelle.XPathParser import ProgHeader
+from math import isnan, fabs, isinf
+from decimal import Decimal
 
 class FORG0001(Exception):
     def __init__(self):
@@ -255,7 +257,6 @@ def xsString(xc, p, source):
     if isinstance(source,bool):
         return 'true' if source else 'false'
     elif isinstance(source,float):
-        from math import (isnan, fabs, isinf)
         if isnan(source):
             return "NaN"
         elif isinf(source):
@@ -270,6 +271,12 @@ def xsString(xc, p, source):
         if s.endswith(".0"):
             s = s[:-2]
         return s
+    elif isinstance(source,Decimal):
+        if isnan(source):
+            return "NaN"
+        elif isinf(source):
+            return "INF"
+        return str(source)
     elif isinstance(source,ModelValue.DateTime):
         return ('{0:%Y-%m-%d}' if source.dateOnly else '{0:%Y-%m-%dT%H:%M:%S}').format(source)
     return str(source)

arelle/ModelDocument.py

@@ -143,7 +143,7 @@ def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDisc
             modelXbrl.error("xmlSchema:syntax",
                     _("Unrecoverable error: %(error)s, %(fileName)s, %(sourceAction)s source element"),
                     modelObject=referringElement, fileName=os.path.basename(uri), 
-                    error=str(err), sourceAction=("including" if isIncluded else "importing"))
+                    error=str(err), sourceAction=("including" if isIncluded else "importing"), exc_info=True)
             modelXbrl.urlUnloadableDocs.add(mappedUri)
             return None
     except Exception as err:
@@ -166,6 +166,8 @@ def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDisc
         ns = rootNode.namespaceURI
 
         # type classification
+        _type = None
+        _class = ModelDocument
         if ns == XbrlConst.xsd and ln == "schema":
             _type = Type.SCHEMA
             if not isEntry and not isIncluded:
@@ -194,6 +196,8 @@ def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDisc
                 _type = Type.INLINEXBRL
         elif ln == "report" and ns == XbrlConst.ver:
             _type = Type.VERSIONINGREPORT
+            from arelle.ModelVersReport import ModelVersReport
+            _class = ModelVersReport
         elif ln in ("testcases", "documentation", "testSuite"):
             _type = Type.TESTCASESINDEX
         elif ln in ("testcase", "testSet"):
@@ -204,34 +208,35 @@ def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDisc
             _type = Type.XPATHTESTSUITE
         elif ln == "rss":
             _type = Type.RSSFEED
+            from arelle.ModelRssObject import ModelRssObject 
+            _class = ModelRssObject
         elif ln == "ptvl":
             _type = Type.ARCSINFOSET
         elif ln == "facts":
             _type = Type.FACTDIMSINFOSET
         else:
-            _type = Type.UnknownXML
-            nestedInline = None
-            for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}html"):
-                nestedInline = htmlElt
-                break
-            if nestedInline is None:
-                for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}xhtml"):
+            for pluginMethod in pluginClassMethods("ModelDocument.IdentifyType"):
+                _identifiedType = pluginMethod(modelXbrl, rootNode, filepath)
+                if _identifiedType is not None:
+                    _type, _class, rootNode = _identifiedType
+                    break
+            if _type is None:
+                _type = Type.UnknownXML
+
+                nestedInline = None
+                for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}html"):
                     nestedInline = htmlElt
                     break
-            if nestedInline is not None:
-                if XbrlConst.ixbrl in nestedInline.nsmap.values():
-                    _type = Type.INLINEXBRL
-                    rootNode = nestedInline
+                if nestedInline is None:
+                    for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}xhtml"):
+                        nestedInline = htmlElt
+                        break
+                if nestedInline is not None:
+                    if XbrlConst.ixbrl in nestedInline.nsmap.values():
+                        _type = Type.INLINEXBRL
+                        rootNode = nestedInline
 
-        #create modelDocument object or subtype as identified
-        if _type == Type.VERSIONINGREPORT:
-            from arelle.ModelVersReport import ModelVersReport
-            modelDocument = ModelVersReport(modelXbrl, _type, mappedUri, filepath, xmlDocument)
-        elif _type == Type.RSSFEED:
-            from arelle.ModelRssObject import ModelRssObject 
-            modelDocument = ModelRssObject(modelXbrl, _type, mappedUri, filepath, xmlDocument)
-        else:
-            modelDocument = ModelDocument(modelXbrl, _type, mappedUri, filepath, xmlDocument)
+        modelDocument = _class(modelXbrl, _type, mappedUri, filepath, xmlDocument)
         rootNode.init(modelDocument)
         modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
         modelDocument.parserLookupName = _parserLookupName
@@ -244,6 +249,11 @@ def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDisc
             modelDocument.inDTS = True
 
         # discovery (parsing)
+        for pluginMethod in pluginClassMethods("ModelDocument.Discover"):
+            if pluginMethod(modelDocument):
+                # discovery was performed by plug-in, we're done
+                return modelDocument
+
         if _type == Type.SCHEMA:
             modelDocument.schemaDiscover(rootNode, isIncluded, namespace)
         elif _type == Type.LINKBASE:
@@ -375,15 +385,16 @@ class Type:
     INLINEXBRL=5
     lastXBRLtype=5  # first filetype that is XBRL and can hold a linkbase, etc inside it
     DTSENTRIES=6  # multiple schema/linkbase Refs composing a DTS but not from an instance document
-    VERSIONINGREPORT=7
-    TESTCASESINDEX=8
-    TESTCASE=9
-    REGISTRY=10
-    REGISTRYTESTCASE=11
-    XPATHTESTSUITE=12
-    RSSFEED=13
-    ARCSINFOSET=14
-    FACTDIMSINFOSET=15
+    INLINEXBRLDOCUMENTSET=7
+    VERSIONINGREPORT=8
+    TESTCASESINDEX=9
+    TESTCASE=10
+    REGISTRY=11
+    REGISTRYTESTCASE=12
+    XPATHTESTSUITE=13
+    RSSFEED=14
+    ARCSINFOSET=15
+    FACTDIMSINFOSET=16
 
     TESTCASETYPES = (TESTCASESINDEX, TESTCASE, REGISTRY, REGISTRYTESTCASE, XPATHTESTSUITE)
 
@@ -394,6 +405,7 @@ class Type:
                 "instance", 
                 "inline XBRL instance",
                 "entry point set",
+                "inline XBRL document set",
                 "versioning report",
                 "testcases index", 
                 "testcase",

arelle/ModelInstanceObject.py

@@ -511,7 +511,19 @@ def transformedValue(self, value):
         num = 0
         negate = -1 if self.sign else 1
         try:
-            num = float(value)
+            if self.concept is not None:
+                baseXsdType = self.concept.baseXsdType
+                if baseXsdType in {"integer",
+                                   "nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
+                                   "long","unsignedLong",
+                                   "int","unsignedInt",
+                                   "short","unsignedShort",
+                                   "byte","unsignedByte"}:
+                    num = _INT(value)
+                else:
+                    num = float(value)
+            else:
+                num = float(value)
             scale = self.scale
             if scale is not None:
                 num *= 10 ** _INT(self.scale)
@@ -526,7 +538,7 @@ def value(self):
         try:
             return self._ixValue
         except AttributeError:
-            v = XmlUtil.innerText(self, ixExclude=True, strip=True) # transforms are whitespace-collapse
+            v = XmlUtil.innerText(self, ixExclude=True, ixEscape=(self.get("escape") == "true"), strip=True) # transforms are whitespace-collapse
             f = self.format
             if f is not None:
                 if (f.namespaceURI in FunctionIxt.ixtNamespaceURIs and
@@ -555,7 +567,9 @@ def propertyView(self):
                 ("html value", XmlUtil.innerText(self)))
         else:
             numProperties = ()
-        return super(ModelInlineFact,self).propertyView + \
+        return (("file", self.modelDocument.basename),
+                ("line", self.sourceline)) + \
+               super(ModelInlineFact,self).propertyView + \
                numProperties
 
     def __repr__(self):

arelle/ModelValue.py

@@ -5,6 +5,7 @@
 (c) Copyright 2011 Mark V Systems Limited, All rights reserved.
 '''
 import re, copy, datetime
+XmlUtil = None
 
 def qname(value, name=None, noPrefixIsNoNamespace=False, castException=None, prefixException=None):
     # either value can be an etree ModelObject element: if no name then qname is element tag quanem
@@ -52,7 +53,7 @@ def qname(value, name=None, noPrefixIsNoNamespace=False, castException=None, pre
         else:
             namespaceURI = None
             namespaceDict = None
-        prefix,sep,localName = value.partition(":")
+        prefix,sep,localName = value.strip().partition(":")  # must be whitespace collapsed
         if len(localName) == 0:
             #default namespace
             localName = prefix
@@ -64,7 +65,9 @@ def qname(value, name=None, noPrefixIsNoNamespace=False, castException=None, pre
     elif namespaceDict and prefix in namespaceDict:
         return QName(prefix, namespaceDict[prefix], localName)
     elif element is not None:
-        from arelle import (XmlUtil)
+        global XmlUtil
+        if XmlUtil is None:
+            from arelle import XmlUtil
         namespaceURI = XmlUtil.xmlns(element, prefix)
     if not namespaceURI:
         if prefix: 

arelle/ModelXbrl.py

@@ -51,7 +51,7 @@ def load(modelManager, url, nextaction=None, base=None, useFileSource=None, erro
         modelXbrl.closeFileSource= True
         url = modelXbrl.fileSource.url
     else:
-        modelXbrl.fileSource = FileSource.FileSource(url)
+        modelXbrl.fileSource = FileSource.FileSource(url, modelManager.cntlr)
         modelXbrl.closeFileSource= True
     modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True)
     del modelXbrl.entryLoadingUrl
@@ -78,7 +78,7 @@ def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, create
     modelXbrl = ModelXbrl(modelManager, errorCaptureLevel=errorCaptureLevel)
     modelXbrl.locale = modelManager.locale
     if newDocumentType:
-        modelXbrl.fileSource = FileSource.FileSource(url) # url may be an open file handle, use str(url) below
+        modelXbrl.fileSource = FileSource.FileSource(url, modelManager.cntlr) # url may be an open file handle, use str(url) below
         modelXbrl.closeFileSource= True
         if createModelDocument:
             modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, str(url), schemaRefs=schemaRefs, isEntry=isEntry)
@@ -399,7 +399,7 @@ def createInstance(self, url=None):
         if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
             return self.modelDocument # use existing instance entry point
         priorFileSource = self.fileSource
-        self.fileSource = FileSource.FileSource(url)
+        self.fileSource = FileSource.FileSource(url, self.modelManager.cntlr)
         if self.uri.startswith("http://"):
             schemaRefUri = self.uri
         else:   # relativize local paths
@@ -469,7 +469,7 @@ def matchContext(self, entityIdentScheme, entityIdentValue, periodType, periodSt
         return None
 
     def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs,
-                      afterSibling=None, beforeSibling=None):
+                      afterSibling=None, beforeSibling=None, id=None):
         """Creates a new ModelContext and validates (integrates into modelDocument object model).
         
         :param entityIdentScheme: Scheme to match
@@ -492,12 +492,14 @@ def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodS
         :type beforeSibling: ModelObject
         :param afterSibling: lxml element in instance to insert new concept after
         :type afterSibling: ModelObject
+        :param id: id to assign to new context, if absent an id will be generated
+        :type id: str
         :returns: ModelContext -- New model context object
         """
         xbrlElt = self.modelDocument.xmlRootElement
         if afterSibling == AUTO_LOCATE_ELEMENT:
             afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context"))
-        cntxId = 'c-{0:02n}'.format( len(self.contexts) + 1)
+        cntxId = id if id else 'c-{0:02n}'.format( len(self.contexts) + 1)
         newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId),
                                       afterSibling=afterSibling, beforeSibling=beforeSibling)
         entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
@@ -523,18 +525,21 @@ def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodS
                 context element, but for shortcut will see if each dimension is already reported in an
                 unambiguous valid contextElement
             '''
-            dims[2] = priItem # Aspect.CONCEPT: prototype needs primary item as an aspect
-            fp = FactPrototype(self, dims)
-            del dims[2] # Aspect.CONCEPT
-            # force trying a valid prototype's context Elements
-            if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True):
-                self.info("arelleLinfo",
-                    _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"), 
-                    modelObject=self, priItem=priItem)
-            fpDims = fp.context.qnameDims
+            if priItem is not None: # creating concept for a specific fact
+                dims[2] = priItem # Aspect.CONCEPT: prototype needs primary item as an aspect
+                fp = FactPrototype(self, dims)
+                del dims[2] # Aspect.CONCEPT
+                # force trying a valid prototype's context Elements
+                if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True):
+                    self.info("arelle:info",
+                        _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"), 
+                        modelObject=self, priItem=priItem)
+                fpDims = fp.context.qnameDims
+            else:
+                fpDims = dims # dims known to be valid (such as for inline extraction) 
             for dimQname in sorted(fpDims.keys()):
                 dimValue = fpDims[dimQname]
-                if isinstance(dimValue, DimValuePrototype):
+                if isinstance(dimValue, (DimValuePrototype,ModelDimensionValue)):
                     dimMemberQname = dimValue.memberQname  # None if typed dimension
                     contextEltName = dimValue.contextElement
                 else: # qname for explicit or node for typed
@@ -594,7 +599,7 @@ def matchUnit(self, multiplyBy, divideBy):
                 return u
         return None
 
-    def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None):
+    def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None, id=None):
         """Creates new unit, by measures, as in formula usage, if any
         
         :param multiplyBy: List of multiply-by measure QNames (or top level measures if no divideBy)
@@ -605,12 +610,14 @@ def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None
         :type beforeSibling: ModelObject
         :param afterSibling: lxml element in instance to insert new concept after
         :type afterSibling: ModelObject
+        :param id: id to assign to new unit, if absent an id will be generated
+        :type id: str
         :returns: ModelUnit -- New unit object
         """
         xbrlElt = self.modelDocument.xmlRootElement
         if afterSibling == AUTO_LOCATE_ELEMENT:
             afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit"))
-        unitId = 'u-{0:02n}'.format( len(self.units) + 1)
+        unitId = id if id else 'u-{0:02n}'.format( len(self.units) + 1)
         newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
                                       afterSibling=afterSibling, beforeSibling=beforeSibling)
         if len(divideBy) == 0:
@@ -750,8 +757,8 @@ def createFact(self, conceptQname, attributes=None, text=None, parent=None, afte
         :param conceptQname: QNames of concept
         :type conceptQname: QName
         :param attributes: Tuple of name, value, or tuples of name, value tuples (name,value) or ((name,value)[,(name,value...)]), where name is either QName or clark-notation name string
-        :param text: Text content of fact
-        :type text: str
+        :param text: Text content of fact (will be converted to xpath compatible str by FunctionXS.xsString)
+        :type text: object
         :param parent: lxml element in instance to append as child of
         :type parent: ModelObject
         :param beforeSibling: lxml element in instance to insert new concept before

arelle/PythonUtil.py

@@ -4,6 +4,7 @@
 do not convert 3 to 2
 '''
 import sys
+from decimal import Decimal
 
 if sys.version[0] >= '3':
     import builtins
@@ -12,8 +13,8 @@
     builtins.__dict__['_STR_UNICODE'] = str
     builtins.__dict__['_INT'] = int
     builtins.__dict__['_INT_TYPES'] = int
-    builtins.__dict__['_NUM_TYPES'] = (int,float)
-    builtins.__dict__['_STR_NUM_TYPES'] = (str,int,float)
+    builtins.__dict__['_NUM_TYPES'] = (int,float,Decimal)
+    builtins.__dict__['_STR_NUM_TYPES'] = (str,int,float,Decimal)
     builtins.__dict__['_RANGE'] = range
     def noop(x): return x
     builtins.__dict__['_DICT_SET'] = noop
@@ -23,8 +24,8 @@ def noop(x): return x
     __builtins__['_STR_UNICODE'] = unicode
     __builtins__['_INT'] = long
     __builtins__['_INT_TYPES'] = (int,long)
-    __builtins__['_NUM_TYPES'] = (int,long,float)
-    __builtins__['_STR_NUM_TYPES'] = (basestring,int,long,float)
+    __builtins__['_NUM_TYPES'] = (int,long,float,Decimal)
+    __builtins__['_STR_NUM_TYPES'] = (basestring,int,long,float,Decimal)
     __builtins__['_RANGE'] = xrange
     __builtins__['_DICT_SET'] = set
 

arelle/Version.py

@@ -5,4 +5,4 @@
 (c) Copyright 2013 Mark V Systems Limited, All rights reserved.
 
 '''
-version = '2013-05-09 21:50 UTC'
+version = '2013-05-15 01:36 UTC'