Merge pull request #835 from hermfischer-wf/edgar23.3-preview

arelle/CntlrWinMain.py

@@ -158,11 +158,8 @@ def __init__(self, parent):
         self.modelManager.validateCalcs = self.config.setdefault("validateCalcsEnum", CalcsMode.NONE)
         self.calcChoiceEnumVar = IntVar(self.parent, value=self.modelManager.validateCalcs)
         self.calcChoiceEnumVar.trace("w", self.setCalcChoiceEnumVar)
-        calcMenu.add_radiobutton(label=_('No calculation checks'), underline=0, var=self.calcChoiceEnumVar, value=CalcsMode.NONE)
-        calcMenu.add_radiobutton(label=_('Calc 1.0 calculations'), underline=0, var=self.calcChoiceEnumVar, value=CalcsMode.XBRL_v2_1)
-        calcMenu.add_radiobutton(label=_('Calc 1.0 with de-duplication'), underline=0, var=self.calcChoiceEnumVar, value=CalcsMode.XBRL_v2_1_DEDUPLICATE)
-        calcMenu.add_radiobutton(label=_('Calc 1.1 round-to-nearest mode'), underline=0, var=self.calcChoiceEnumVar, value=CalcsMode.ROUND_TO_NEAREST)
-        calcMenu.add_radiobutton(label=_('Calc 1.1 truncation mode'), underline=0, var=self.calcChoiceEnumVar, value=CalcsMode.TRUNCATION)
+        for calcChoiceMenuLabel, calcChoiceEnumValue in CalcsMode.menu().items():
+            calcMenu.add_radiobutton(label=calcChoiceMenuLabel, underline=0, var=self.calcChoiceEnumVar, value=calcChoiceEnumValue)
         toolsMenu.add_cascade(label=_("Calc linkbase"), menu=calcMenu, underline=0)
         self.modelManager.validateUtr = self.config.setdefault("validateUtr",True)
         self.validateUtr = BooleanVar(value=self.modelManager.validateUtr)
@@ -858,7 +855,7 @@ def backgroundLoadXbrl(self, filesource, importToDTS, selectTopView):
                                         (action, time.time() - startedAt)))
             self.showStatus(_("Loading terminated"), 15000)
 
-    def showLoadedXbrl(self, modelXbrl, attach, selectTopView=False):
+    def showLoadedXbrl(self, modelXbrl, attach, selectTopView=False, isSupplementalModelXbrl=False):
         startedAt = time.time()
         currentAction = "setting title"
         topView = None
@@ -948,20 +945,25 @@ def showLoadedXbrl(self, modelXbrl, attach, selectTopView=False):
             modelXbrl.profileStat("view", viewTime)
             self.addToLog(format_string(self.modelManager.locale,
                                         _("views %.2f secs"), viewTime))
-            if selectTopView and topView:
-                topView.select()
-            self.currentView = topView
-            currentAction = "plugin method CntlrWinMain.Xbrl.Loaded"
-            for xbrlLoadedMethod in pluginClassMethods("CntlrWinMain.Xbrl.Loaded"):
-                xbrlLoadedMethod(self, modelXbrl, attach) # runs in GUI thread
+            if hasattr(modelXbrl, "supplementalModelXbrls"):
+                for supplementalModelXbrl in modelXbrl.supplementalModelXbrls:
+                    self.showLoadedXbrl(supplementalModelXbrl, False, isSupplementalModelXbrl=True)
+            if not isSupplementalModelXbrl:
+                if selectTopView and topView:
+                    topView.select()
+                self.currentView = topView
+                currentAction = "plugin method CntlrWinMain.Xbrl.Loaded"
+                for xbrlLoadedMethod in pluginClassMethods("CntlrWinMain.Xbrl.Loaded"):
+                    xbrlLoadedMethod(self, modelXbrl, attach) # runs in GUI thread
         except Exception as err:
             msg = _("Exception preparing {0}: {1}, at {2}").format(
                      currentAction,
                      err,
                      traceback.format_tb(sys.exc_info()[2]))
             tkinter.messagebox.showwarning(_("Exception preparing view"),msg, parent=self.parent)
             self.addToLog(msg);
-        self.showStatus(_("Ready..."), 2000)
+        if not isSupplementalModelXbrl:
+            self.showStatus(_("Ready..."), 2000)
 
     def showFormulaOutputInstance(self, priorOutputInstance, currentOutputInstance):
         currentAction = "closing prior formula output instance"
@@ -1280,7 +1282,7 @@ def setValidateTooltipText(self):
             if valType == ModelDocument.Type.VERSIONINGREPORT:
                 v = _("Validate versioning report")
             else:
-                c = "\n" + CalcsMode.label[self.modelManager.validateCalcs]
+                c = "\n" + CalcsMode.label(self.modelManager.validateCalcs)
                 if self.modelManager.validateUtr:
                     u = _("\nCheck unit type registry")
                 else:

arelle/DialogRssWatch.py

@@ -15,6 +15,7 @@
 from arelle.CntlrWinTooltip import ToolTip
 from arelle.PluginManager import pluginClassMethods
 from arelle.UrlUtil import isValidAbsolute
+from arelle.ValidateXbrlCalcs import ValidateCalcsMode as CalcsMode
 
 '''
 caller checks accepted, if True, caller retrieves url
@@ -115,16 +116,23 @@ def __init__(self, mainWin, options):
                     "validateXbrlRules"),
            checkbox(frame, 2, row+1,
                     "Selected disclosure system rules",
-                    "validateDisclosureSystemRules"),
-           checkbox(frame, 2, row+2,
-                    "Calculation linkbase roll-up",
-                    "validateCalcLinkbase"),
-           checkbox(frame, 2, row+3,
+                    "validateDisclosureSystemRules")
+        )
+        row += 2
+        self.calcModeMenu = CalcsMode.menu() # dynamic, may change after initialization if language changes
+        vals = list(self.calcModeMenu.keys())
+        keys = dict((v,k) for k,v in self.calcModeMenu.items())
+        self.validateCalcs = gridCombobox(frame, 2, row, keys.get(options.get("validateCalcs",0)), values=vals)
+        self.validateCalcs.grid(pady=2,padx=(44,0))
+        ToolTip(self.validateCalcs, text=_("Select calculations validation"), wraplength=240)
+        row += 1
+        self.checkboxes += (
+           checkbox(frame, 2, row,
                     "Formula assertions",
                     "validateFormulaAssertions"),
            # Note: if adding to this list keep ModelFormulaObject.FormulaOptions in sync
         )
-        row += 4
+        row += 1
         for pluginXbrlMethod in pluginClassMethods("DialogRssWatch.ValidateChoices"):
             pluginXbrlMethod(self, frame, row, options, mainWin)
             row += 1
@@ -241,6 +249,7 @@ def setOptions(self):
             self.options["latestPubDate"] = None
         for checkbox in self.checkboxes:
             self.options[checkbox.attr] = checkbox.value
+        self.options["validateCalcs"] = self.calcModeMenu[self.validateCalcs.value]
 
     def ok(self, event=None):
         if not self.checkEntries():

arelle/FileSource.py

@@ -28,8 +28,6 @@
 POST_UPLOADED_ZIP = os.sep + "POSTupload.zip"
 SERVER_WEB_CACHE = os.sep + "_HTTP_CACHE"
 
-XMLdeclaration = re.compile(r"<\?xml[^><\?]*\?>", re.DOTALL)
-
 TAXONOMY_PACKAGE_FILE_NAMES = ('.taxonomyPackage.xml', 'catalog.xml') # pre-PWD packages
 
 def openFileSource(
@@ -824,9 +822,11 @@ def openXmlFileStream(
     else:
         if stripDeclaration:
             # strip XML declaration
-            xmlDeclarationMatch = XMLdeclaration.search(text)
+            xmlDeclarationMatch = XmlUtil.xmlDeclarationPattern.match(text)
             if xmlDeclarationMatch: # remove it for lxml
-                start,end = xmlDeclarationMatch.span()
+                if xmlDeclarationMatch.group(1) is not None:
+                    raise XmlUtil.XmlDeclarationLocationException
+                start,end = xmlDeclarationMatch.span(2)
                 text = text[0:start] + text[end:]
         return (FileNamedStringIO(filepath, initial_value=text), encoding)
 

arelle/ModelDocument.py

@@ -204,6 +204,11 @@ def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDisc
                     error=str(err), exc_info=True)
             modelXbrl.urlUnloadableDocs[normalizedUri] = True  # not loadable due to parser issues
             return None
+    except XmlUtil.XmlDeclarationLocationException as err:
+        modelXbrl.error("xmlSyntax:xmlDeclarationError",
+                _("XML file syntax error: %(error)s, %(fileName)s"),
+                modelObject=(referringElement, os.path.basename(uri)), fileName=os.path.basename(uri), error=str(err))
+        return None
     except Exception as err:
         modelXbrl.error(type(err).__name__,
                 _("Unrecoverable error: %(error)s, %(fileName)s"),
@@ -1452,9 +1457,10 @@ def addDocumentReference(self, doc, referenceType, referringModelObject=None):
 
 # inline document set level compilation
 # modelIxdsDocument is an inlineDocumentSet or entry inline document (if not a document set)
+#   note that multi-target and multi-instance facts may have html elements belonging to primary ixds instead of this instance ixds
 def inlineIxdsDiscover(modelXbrl, modelIxdsDocument):
     for pluginMethod in pluginClassMethods("ModelDocument.SelectIxdsTarget"):
-        pluginMethod(modelXbrl)
+        pluginMethod(modelXbrl, modelIxdsDocument)
     # extract for a single target document
     ixdsTarget = getattr(modelXbrl, "ixdsTarget", None)
     # compile inline result set
@@ -1616,24 +1622,24 @@ def inlineIxdsDiscover(modelXbrl, modelIxdsDocument):
             for elt in inlineElement.iterchildren("{http://www.xbrl.org/2003/instance}context"):
                 id = elt.get("id")
                 if id in contextRefs or (not ixdsTarget and id not in allContextRefs):
-                    mdlDoc.contextDiscover(elt)
+                    modelIxdsDocument.contextDiscover(elt)
             for elt in inlineElement.iterchildren("{http://www.xbrl.org/2003/instance}unit"):
                 id = elt.get("id")
                 if id in unitRefs or (not ixdsTarget and id not in allUnitRefs):
-                    mdlDoc.unitDiscover(elt)
+                    modelIxdsDocument.unitDiscover(elt)
             for refElement in inlineElement.iterchildren("{http://www.xbrl.org/2003/linkbase}roleRef"):
                 r = refElement.get("roleURI")
                 if r in targetRoleUris[ixdsTarget]:
                     modelXbrl.targetRoleRefs[r] = refElement
-                    if mdlDoc.discoverHref(refElement) is None: # discover role-defining schema file
+                    if modelIxdsDocument.discoverHref(refElement) is None: # discover role-defining schema file
                         modelXbrl.error("xmlSchema:requiredAttribute",
                                 _("Reference for roleURI href attribute missing or malformed"),
                                 modelObject=refElement)
             for refElement in inlineElement.iterchildren("{http://www.xbrl.org/2003/linkbase}arcroleRef"):
                 r = refElement.get("arcroleURI")
                 if r in targetArcroleUris[ixdsTarget]:
                     modelXbrl.targetArcroleRefs[r] = refElement
-                    if mdlDoc.discoverHref(refElement) is None: # discover arcrole-defining schema file
+                    if modelIxdsDocument.discoverHref(refElement) is None: # discover arcrole-defining schema file
                         modelXbrl.error("xmlSchema:requiredAttribute",
                                 _("Reference for arcroleURI href attribute missing or malformed"),
                                 modelObject=refElement)
@@ -1755,7 +1761,7 @@ def addItemFactToTarget(modelInlineFact):
                             modelObject=modelInlineFact, qname=modelInlineFact.qname, localName=modelInlineFact.elementQname,
                             type=modelInlineFact.concept.baseXsdType)
         else:
-            mdlDoc.modelXbrl.factsInInstance.add( modelInlineFact )
+            modelIxdsDocument.modelXbrl.factsInInstance.add( modelInlineFact )
 
     _customTransforms = modelXbrl.modelManager.customTransforms or {}
     for htmlElement in modelXbrl.ixdsHtmlElements:
@@ -1932,7 +1938,7 @@ def checkForTupleCycle(parentTuple, tupleNesting):
                 if linkrole in footnoteLinkPrototypes:
                     linkPrototype = footnoteLinkPrototypes[linkrole]
                 else:
-                    linkPrototype = LinkPrototype(mdlDoc, mdlDoc.xmlRootElement, XbrlConst.qnLinkFootnoteLink, linkrole)
+                    linkPrototype = LinkPrototype(modelIxdsDocument, mdlDoc.xmlRootElement, XbrlConst.qnLinkFootnoteLink, linkrole)
                     footnoteLinkPrototypes[linkrole] = linkPrototype
                     for baseSetKey in (("XBRL-footnotes",None,None,None),
                                        ("XBRL-footnotes",linkrole,None,None),
@@ -1942,7 +1948,7 @@ def checkForTupleCycle(parentTuple, tupleNesting):
                         modelXbrl.baseSets[baseSetKey].append(linkPrototype)
                 # locs
                 for modelFact in footnoteRefs[footnoteID]:
-                    locPrototype = LocPrototype(mdlDoc, linkPrototype, footnoteLocLabel, modelFact)
+                    locPrototype = LocPrototype(modelIxdsDocument, linkPrototype, footnoteLocLabel, modelFact)
                     linkPrototype.childElements.append(locPrototype)
                     linkPrototype.labeledResources[footnoteLocLabel].append(locPrototype)
                 # resource
@@ -1957,7 +1963,7 @@ def checkForTupleCycle(parentTuple, tupleNesting):
             if isinstance(modelInlineRel,ModelObject):
                 linkrole = modelInlineRel.get("linkRole", XbrlConst.defaultLinkRole)
                 if linkrole not in linkPrototypes:
-                    linkPrototypes[linkrole] = LinkPrototype(mdlDoc, mdlDoc.xmlRootElement, XbrlConst.qnLinkFootnoteLink, linkrole, sourceElement=modelInlineRel)
+                    linkPrototypes[linkrole] = LinkPrototype(modelIxdsDocument, mdlDoc.xmlRootElement, XbrlConst.qnLinkFootnoteLink, linkrole, sourceElement=modelInlineRel)
 
 
     for htmlElement in modelXbrl.ixdsHtmlElements:
@@ -2038,7 +2044,7 @@ def checkForTupleCycle(parentTuple, tupleNesting):
                                     modelObject=modelInlineRel, fromToMatchedIds=', '.join(sorted(fromToMatchedIds)))
                 for fromLabel in fromLabels:
                     for toLabel in toLabels: # toLabels is empty if no to fact or footnote is in target
-                        linkPrototype.childElements.append(ArcPrototype(mdlDoc, linkPrototype, XbrlConst.qnLinkFootnoteArc,
+                        linkPrototype.childElements.append(ArcPrototype(modelIxdsDocument, linkPrototype, XbrlConst.qnLinkFootnoteArc,
                                                                         fromLabel, toLabel,
                                                                         linkrole, arcrole,
                                                                         modelInlineRel.get("order", "1"), sourceElement=modelInlineRel))
@@ -2074,6 +2080,9 @@ def checkForTupleCycle(parentTuple, tupleNesting):
         modelIxdsDocument.targetXbrlRootElement = modelXbrl.ixTargetRootElements[ixdsTarget]
         modelIxdsDocument.targetXbrlElementTree = PrototypeElementTree(modelIxdsDocument.targetXbrlRootElement)
 
+    for pluginMethod in pluginClassMethods("ModelDocument.IxdsTargetDiscovered"):
+        pluginMethod(modelXbrl, modelIxdsDocument)
+
 class LoadingException(Exception):
     pass
 

arelle/ModelXbrl.py

@@ -1213,7 +1213,7 @@ def log(self, level: str, codes: Any, msg: str, **args: Any) -> None:
             return
         if self.logHasRelevelerPlugin:
             for pluginXbrlMethod in pluginClassMethods("Logging.Severity.Releveler"):
-                level = pluginXbrlMethod(self, level, messageCode, args) # args must be passed as dict because it may contain modelXbrl or messageCode key value
+                level, messageCode = pluginXbrlMethod(self, level, messageCode, args) # args must be passed as dict because it may contain modelXbrl or messageCode key value
         if (messageCode and
               (not logger.messageCodeFilter or logger.messageCodeFilter.match(messageCode)) and
               (not logger.messageLevelFilter or logger.messageLevelFilter.match(level.lower()))):

arelle/ValidateFilingText.py

@@ -8,11 +8,10 @@
 import os, io, base64
 import regex as re
 from arelle.XbrlConst import ixbrlAll, xhtml
-from arelle.XmlUtil import setXmlns, xmlstring
+from arelle.XmlUtil import setXmlns, xmlstring, xmlDeclarationPattern, XmlDeclarationLocationException
 from arelle.ModelObject import ModelObject
 from arelle.UrlUtil import decodeBase64DataImage, isHttpUrl, scheme
 
-XMLdeclaration = re.compile(r"<\?xml.*\?>", re.DOTALL)
 XMLpattern = re.compile(r".*(<|&lt;|&#x3C;|&#60;)[A-Za-z_]+[A-Za-z0-9_:]*[^>]*(/>|>|&gt;|/&gt;).*", re.DOTALL)
 CDATApattern = re.compile(r"<!\[CDATA\[(.+)\]\]")
 #EFM table 5-1 and all &xxx; patterns
@@ -469,9 +468,11 @@ def close(self): pass
                             _("Disallowed character '%(text)s' in file %(file)s at line %(line)s col %(column)s"),
                             modelDocument=filepath, text=text, file=os.path.basename(filepath), line=lineNum, column=match.start())
             if lineNum == 1:
-                xmlDeclarationMatch = XMLdeclaration.search(line)
+                xmlDeclarationMatch = xmlDeclarationPattern.match(line)
                 if xmlDeclarationMatch: # remove it for lxml
-                    start,end = xmlDeclarationMatch.span()
+                    if xmlDeclarationMatch.group(1) is not None:
+                        raise XmlDeclarationLocationException
+                    start,end = xmlDeclarationMatch.span(2)
                     line = line[0:start] + line[end:]
                     foundXmlDeclaration = True
             if _parser: # feed line after removal of xml declaration
@@ -497,9 +498,11 @@ def close(self): pass
             lineNum += 1
     result = ''.join(result)
     if not foundXmlDeclaration: # may be multiline, try again
-        xmlDeclarationMatch = XMLdeclaration.search(result)
+        xmlDeclarationMatch = xmlDeclarationPattern.match(result)
         if xmlDeclarationMatch: # remove it for lxml
-            start,end = xmlDeclarationMatch.span()
+            if xmlDeclarationMatch.group(1) is not None:
+                raise XmlDeclarationLocationException
+            start,end = xmlDeclarationMatch.span(2)
             result = result[0:start] + result[end:]
             foundXmlDeclaration = True