Merge pull request #1024 from hermfischer-wf/edgar24.0.1-preview

Edgar 24.0.1 Preview
Arelle · Jan 28, 2024 · 07c4597 · 07c4597
2 parents 03fdaf5 + b1be907
commit 07c4597
Show file tree

Hide file tree

Showing 13 changed files with 3,220 additions and 401 deletions.
diff --git a/arelle/Cntlr.py b/arelle/Cntlr.py
@@ -631,11 +631,11 @@ def workingOnlineOrInCache(self, url: str) -> bool:
 
 
 def logRefsFileLines(refs: list[dict[str, Any]]) -> str:
-    fileLines: defaultdict[Any, set[str]] = defaultdict(set)
+    fileLines = defaultdict(set)
     for ref in refs:
         href = ref.get("href")
         if href:
-            fileLines[href.partition("#")[0]].add(ref.get("sourceLine", 0))
+            fileLines[href.partition("#")[0]].add(ref.get("sourceLine") or 0)
     return ", ".join(file + " " + ', '.join(str(line)
                                             for line in sorted(lines, key=lambda l: l)
                                             if line)
@@ -731,7 +731,7 @@ def propElts(properties: list[tuple[Any, Any, Any]], indent: str, truncateAt: in
         msg = self.format(logRec)
         if logRec.args and isinstance(logRec.args, Mapping):
             args = "".join([' {0}="{1}"'.format(ncNameEncode(n), entityEncode(v,
-                                                truncateAt=(65535 if n in ("json",) else 128)))
+                                                truncateAt=(4096000 if n in ("json",) else 128)))
                             for n, v in logRec.args.items()])
         else:
             args = ""
@@ -930,11 +930,12 @@ def getHtml(self, clearLogBuffer: bool = True) -> str:
         ]
         if self.logRecordBuffer:
             for logRec in self.logRecordBuffer:
-                html.append(self.recordToHtml(logRec))
+                if all(p(logRec) for p in PluginManager.pluginClassMethods("Cntlr.Log.RecFilter.Html")):
+                    html.append(self.recordToHtml(logRec))
             if clearLogBuffer:
                 self.clearLogBuffer()
             html.append("</tbody>\n</table>\n</body>\n</html>\n")
-        else:
+        if len(html) < 3: # no entries were added to log. Display no log errors message
             html = ["""<!doctype html>
             <html>
             <head>

diff --git a/arelle/ModelDocument.py b/arelle/ModelDocument.py
@@ -1476,7 +1476,7 @@ def inlineIxdsDiscover(modelXbrl, modelIxdsDocument, setTargetModelXbrl=False):
     # extract for a single target document
     ixdsTarget = getattr(modelXbrl, "ixdsTarget", None)
     # compile inline result set
-    ixdsEltById = defaultdict(list)
+    ixdsEltById = modelXbrl.ixdsEltById = defaultdict(list)
     for htmlElement in modelXbrl.ixdsHtmlElements:
         for elt in htmlElement.iterfind(".//*[@id]"):
             if isinstance(elt,ModelObject) and elt.id:

diff --git a/arelle/plugin/validate/EFM/Consts.py b/arelle/plugin/validate/EFM/Consts.py
@@ -66,7 +66,7 @@
     (None, "dei-validations.json")
     )
 supplementalAttachmentDocumentTypesPattern = re.compile(r"EX-FILING FEES.*|EX-26.*")
-exhibitTypesStrippingOnErrorPattern = re.compile(r"EX-FILING FEES.*|EX-26.*")
+exhibitTypesStrippingOnErrorPattern = re.compile(r"EX-26.*")
 
 standardNamespacesPattern = re.compile(
     # non-IFRS groups 1 - authority, 2 - taxonomy (e.g. us-gaap, us-types), 3 - year

diff --git a/arelle/plugin/validate/EFM/Filing.py b/arelle/plugin/validate/EFM/Filing.py
@@ -27,7 +27,7 @@
 from arelle.PythonUtil import pyNamedObject, strTruncate, normalizeSpace, lcStr, flattenSequence, flattenToSet, OrderedSet
 from arelle.UrlUtil import isHttpUrl
 from arelle.ValidateXbrlCalcs import inferredDecimals, rangeValue, roundValue, ONE
-from arelle.XmlValidate import VALID
+from arelle.XmlValidate import VALID, INVALID
 from .DTS import checkFilingDTS
 from .Consts import submissionTypesAllowingSeriesClasses, \
                     submissionTypesRequiringOefClasses, invCompanyTypesRequiringOefClasses, \
@@ -1570,6 +1570,14 @@ def getNumberofDaysLate(fiscalYearEnd, lateAfter=90):
 
                 return max((datetimeNowAtSEC - dueDate).days, 0)
 
+            def find_fact_in_context(contextID, name=None):
+                for fact in modelXbrl.facts:
+                    if fact.contextID == contextID:
+                        if not name:
+                            return fact
+                        if name == ftName(fact):
+                            return fact
+
             unexpectedDeiNameEfmSects = defaultdict(set) # name and sev(s)
             expectedDeiNames = defaultdict(set)
             coverVisibleQNames = {}  # true if error, false if warning when not visible
@@ -1598,6 +1606,7 @@ def getNumberofDaysLate(fiscalYearEnd, lateAfter=90):
                 storeDbObject = sev.get("store-db-object")
                 storeDbAction = sev.get("store-db-action")
                 storeDbInnerTextTruncate = sev.get("store-db-inner-text-truncate")
+                storeDbInnerTextOnly = sev.get("store-db-inner-text-truncate")
                 efmSection = sev.get("efm")
                 validation = sev.get("validation")
                 checkAfter = sev.get("check-after")
@@ -1921,6 +1930,9 @@ def getNumberofDaysLate(fiscalYearEnd, lateAfter=90):
                                             else (not value.inRange(f.xValue)) if isinstance(value, ValueRange)
                                             else (value is not None and f.xValue != value)):
                                 sevMessage(sev, subType=submissionType, modelObject=f, efmSection=efmSection, tag=ftName(name), label=ftLabel(name), value=("(none)" if f is None else f.xValue), expectedValue=value, ftContext=ftContext(axisKey,f))
+                                if validation.startswith("ov") and (sev.get("value-numeric-range") or sev.get("value-pattern") or sev.get("value-date-range")):
+                                    # avoid writing to store-db-object since this is an invalid value
+                                    f.xValid = INVALID
                             if f is None and name in eloValueFactNames:
                                 missingReqInlineTag = True
                 elif validation  == "not-in-future":
@@ -1982,11 +1994,7 @@ def getNumberofDaysLate(fiscalYearEnd, lateAfter=90):
                         sevMessage(sev, subType=submissionType, modelObject=val.requiredContext, tag="Required Context Period Duration",
                                    value=f"{monthsDuration:.1f} months", expectedValue=f"{value} months", contextID=val.requiredContext.id)
                 # fee tagging
-                elif validation in ("fe", "fw","fo"): # note where clauses are incompatible with this validation
-                    def find_fact_in_context(contextID):
-                        for fact in modelXbrl.facts:
-                            if fact.contextID == contextID:
-                                return fact
+                elif validation in ("fe", "fw","fo"):
                     instDurNames = defaultdict(list)
                     for name in names:
                         concept = modelXbrl.qnameConcepts.get(qname(name, deiDefaultPrefixedNamespaces))
@@ -2236,6 +2244,12 @@ def find_fact_in_context(contextID):
                     comparison = sev.get("comparison")
                     for name1 in names:
                         for f in sevFacts(sev, name1, deduplicate=True, whereKey="where"): # these all are sum facts
+                            if sev.get("comparison-element-match"):
+                                mainFactElementMatch = {}
+                                for elemName in sev.get("comparison-element-match"):
+                                    mainFact = find_fact_in_context(f.contextID, name=elemName)
+                                    if mainFact is not None:
+                                        mainFactElementMatch[elemName] = mainFact.xValue
                             for name2 in referenceTag:
                                 for g in sevFacts(sev, name2, f, axisKey=sev.get("references-axes"), deduplicate=True, whereKey="references-where"):
                                     if "references-date-format" in sev:
@@ -2247,12 +2261,20 @@ def find_fact_in_context(contextID):
                                         if "%d" not in sev.get("references-date-format").lower():
                                             referenceDate = referenceDate.replace(year=f.xValue.day)
                                         g.xValue = ModelValue.dateTime(referenceDate.date().isoformat(), type=ModelValue.DATE)
+                                    if sev.get("comparison-element-match"):
+                                        otherFactElementMatch = {}
+                                        for elemName in sev.get("comparison-element-match"):
+                                            otherFact = find_fact_in_context(g.contextID, name=elemName)
+                                            if otherFact is not None:
+                                                otherFactElementMatch[elemName] = otherFact.xValue
+                                        if mainFactElementMatch != otherFactElementMatch:
+                                            continue
                                     if ((comparison == "equal" and f.xValue != g.xValue) or
                                         (comparison == "not equal" and f.xValue == g.xValue) or
                                         (comparison in ("less than or equal", "not greater") and f.xValue > g.xValue)):
                                         comparisonText = sev.get("comparisonText", deiValidations["validations"][sev["validation"]].get("comparisonText", comparison)).format(comparison=comparison)
                                         sevMessage(sev, subType=submissionType, modelObject=(f,g), ftContext=ftContext(axisKey,f), comparison=comparisonText,
-                                                   tag=ftName(name1), label=ftLabel(name1), otherTag=ftName(name2), otherLabel=ftLabel(name2), value=f.xValue, otherValue=g.xValue)
+                                                   tag=ftName(name1), label=ftLabel(name1), otherTag=ftName(name2), otherLabel=ftLabel(name2), value=f.xValue, otherValue=g.xValue, otherftContext=ftContext(axisKey,g))
                 elif validation == "calculation":
                     comparison = sev.get("comparison")
                     operators = sev.get("references-operators")
@@ -2362,8 +2384,9 @@ def getEvalFunctionStringAndArgs(sev, functionName, argumentsKey="function-argum
                                 if qname(_axis, deiDefaultPrefixedNamespaces) == dim.dimensionQname
                                 )
                             if storeDbName:
-                                storeDbObjectFacts.setdefault(storeDbObject,{}).setdefault(_axisKey,{})[
-                                    _storeDbName] = getStoreDBValue(ftName(f), eloValueOfFact(names[0], f.xValue))
+                                if not (storeDbInnerTextOnly and storeDbInnerTextTruncate): # only write truncated inner text to output file
+                                    storeDbObjectFacts.setdefault(storeDbObject,{}).setdefault(_axisKey,{})[
+                                        _storeDbName] = getStoreDBValue(ftName(f), eloValueOfFact(names[0], f.xValue))
                                 if storeDbInnerTextTruncate:
                                     storeDbObjectFacts.setdefault(storeDbObject,{}).setdefault(_axisKey,{})[
                                         f"{_storeDbName}InnerText"] = strTruncate(normalizeSpace(XmlUtil.innerText(f,
@@ -3262,14 +3285,14 @@ def elementsReferencingTxClass(txClass):
                                     for _rel in parentChildRels.fromModelObject(relTo))):
                                         modelXbrl.warning("EFM.6.12.08",
                                             _("In \"%(linkrole)s\" axis %(axis)s has no domain element children, which effectively filters out every fact."),
-                                            modelObject=(relFrom,relTo), axis=relFrom.qname,
+                                            modelObject=(relFrom,relTo), axis=relTo.qname,
                                             linkrole=ELR, linkroleDefinition=modelXbrl.roleTypeDefinition(ELR), linkroleName=modelXbrl.roleTypeName(ELR))
                                 if (relFrom.isExplicitDimension and not any(
                                     isinstance(_rel.toModelObject, ModelConcept) and _rel.toModelObject.type is not None and _rel.toModelObject.type.isDomainItemType
                                     for _rel in siblingRels)):
                                         modelXbrl.warning("EFM.6.12.08",
                                             _("In \"%(linkrole)s\" axis %(axis)s has no domain element children, which effectively filters out every fact."),
-                                            modelObject=relFrom, axis=relFrom.qname,
+                                            modelObject=relFrom, axis=relTo.qname,
                                             linkrole=ELR, linkroleDefinition=modelXbrl.roleTypeDefinition(ELR), linkroleName=modelXbrl.roleTypeName(ELR))
                         targetConceptPreferredLabels.clear()
                         orderRels.clear()

diff --git a/arelle/plugin/validate/EFM/MessageNumericId.py b/arelle/plugin/validate/EFM/MessageNumericId.py
@@ -88,7 +88,8 @@
     "r457uSecType": 12, "rqdFileNb": 13, "ttlFeeAmt": 14, "ttlOfAmt": 15, "ttlOfstAmt": 16, "ttlOfstLeTotFee": 17,
     "ttlRqdFlds": 18, "ttlTxValtn": 19, "txtFldChars": 20, "usrSty27": 21, "usrSty30": 22, "usrSty32": 23,
     "usrSty3334MAOP": 24, "usrSty3334MAOPsums": 25, "usrSty3334valSecRcvd": 26, "usrSty3334RqdFlds": 87, "usrSty42": 30, "usrSty43": 31, "usrSty50": 32, "usrSty6a": 33, "usrSty6b": 34, "usrSty6c": 35,
-    "usrSty9": 36, "uusFileNb": 37, "uusNAflds": 38, "usrSty2": 41, "usrSty10": 40
+    "usrSty9": 36, "uusFileNb": 37, "uusNAflds": 38, "usrSty2": 41, "usrSty10": 40, "uusRqdChild": 42, "footNoteLen": 27,
+    "subTpMismatch": 1, "numRate": 66, "r415a6FileNb": 51
 }
 
 def messageNumericId(modelXbrl, level, messageCode, args):

diff --git a/arelle/plugin/validate/EFM/Util.py b/arelle/plugin/validate/EFM/Util.py
@@ -382,9 +382,8 @@ def loadUgtRelQnames(modelXbrl, dqcRules):
     _file = openFileStream(modelXbrl.modelManager.cntlr, _ugtRelsFileName, 'rt', encoding='utf-8')
     ugtRels = json.load(_file) # {localName: date, ...}
     _file.close()
-    prefixedNamespaces = modelXbrl.prefixedNamespaces
     def qn(nsPrefix, localName):
-        return qname(nsPrefix + ":" + localName, prefixedNamespaces)
+        return qname(nsPrefix + ":" + localName, modelXbrl.prefixedNamespaces)
     ugtCalcsByQnames = defaultdict(dict) # store as concept indices to avoid using memory for repetitive strings
     for wgt, fromNSes in ugtRels["calcs"].items():
         calcWgtObj = ugtCalcsByQnames.setdefault(float(wgt), {}) # json weight object needs to be float

diff --git a/arelle/plugin/validate/EFM/__init__.py b/arelle/plugin/validate/EFM/__init__.py
@@ -375,7 +375,7 @@ def filingStart(cntlr, options, filesource, entrypointFiles, sourceZipStream=Non
 
 def guiTestcasesStart(cntlr, modelXbrl, *args, **kwargs):
     modelManager = cntlr.modelManager
-    if cntlr.hasGui and modelManager.validateDisclosureSystem and getattr(modelManager.disclosureSystem, "EFMplugin", False):
+    if cntlr.hasGui: # enable EdgarRenderer to initiate ixviewer irregardless of whether an EFM disclosure system is active
         for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Gui.Run"):
             pluginXbrlMethod(cntlr, modelXbrl, *args,
                              # pass plugin items to GUI mode of EdgarRenderer
@@ -820,7 +820,7 @@ def json(self): # stringify un-jsonable attributes
 __pluginInfo__ = {
     # Do not use _( ) in pluginInfo itself (it is applied later, after loading
     'name': 'Validate EFM',
-    'version': '1.23.4', # SEC EDGAR release 23.4
+    'version': '1.24.0.1', # SEC EDGAR release 24.0.1
     'description': '''EFM Validation.''',
     'license': 'Apache-2',
     'import': ('transforms/SEC',), # SEC inline can use SEC transformations