Skip to content

Commit

Permalink
Merge pull request #869 from aaroncameron-wk/package-manager-offline
Browse files Browse the repository at this point in the history
  • Loading branch information
@austinmatherne-wk
austinmatherne-wk committed Sep 25, 2023
2 parents 8a6bd38 + 560ba28 commit 651fe12
Show file tree
Hide file tree
Showing 4 changed files with 108 additions and 63 deletions.
17 changes: 17 additions & 0 deletions arelle/Cntlr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,23 @@ def memoryUsed(self) -> float | int:
pass
return 0

def workingOnlineOrInCache(self, url: str) -> bool:
"""
Determines if the given URL should be requested based on the web cache's internet connectivity status
and whether the URL already exists in the cache.
:param url: Web URL
:return: True if the URL should be requested, False if not
"""
if not self.webCache.workOffline:
# Working online, can proceed regardless of presence in cache
return True
cacheFilepath = self.webCache.urlToCacheFilepath(url)
if os.path.exists(cacheFilepath):
# The file exists in cache, we can proceed despite working offline
return True
return False


def logRefsFileLines(refs: list[dict[str, Any]]) -> str:
fileLines: defaultdict[Any, set[str]] = defaultdict(set)
for ref in refs:
Expand Down
120 changes: 66 additions & 54 deletions arelle/CntlrCmdLine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,13 @@ def monitorParentProcess():

setDisableRTL(options.disableRtl) # not saved to config

# Some options below (e.g. `packages`) may trigger web requests,
# so the `workOffline` flag needs to be set early on.
if options.internetConnectivity == "offline":
self.webCache.workOffline = True
elif options.internetConnectivity == "online":
self.webCache.workOffline = False

if options.proxy:
if options.proxy != "show":
proxySettings = proxyTuple(options.proxy)
Expand Down Expand Up @@ -709,57 +716,9 @@ def monitorParentProcess():
moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
messageCode="info", file=moduleInfo.get("moduleURL"))

if options.packages:
from arelle import PackageManager
savePackagesChanges = True
showPackages = False
for packageCmd in options.packages.split('|'):
cmd = packageCmd.strip()
if cmd == "show":
showPackages = True
elif cmd == "temp":
savePackagesChanges = False
elif cmd.startswith("+"):
packageInfo = PackageManager.addPackage(self, cmd[1:], options.packageManifestName)
if packageInfo:
self.addToLog(_("Addition of package {0} successful.").format(packageInfo.get("name")),
messageCode="info", file=packageInfo.get("URL"))
else:
self.addToLog(_("Unable to load package."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PackageManager.reloadPackageModule(self, cmd[1:]):
self.addToLog(_("Reload of package successful."), messageCode="info", file=cmd[1:])
else:
self.addToLog(_("Unable to reload package."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PackageManager.removePackageModule(self, cmd[1:]):
self.addToLog(_("Deletion of package successful."), messageCode="info", file=cmd[1:])
else:
self.addToLog(_("Unable to delete package."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePackagesChanges = False
packageInfo = PackageManager.addPackage(self, cmd, options.packageManifestName)
if packageInfo:
self.addToLog(_("Activation of package {0} successful.").format(packageInfo.get("name")),
messageCode="info", file=packageInfo.get("URL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load package \"%(name)s\". "),
messageCode="arelle:packageLoadingError",
messageArgs={"name": cmd, "file": cmd}, level=logging.ERROR)
if PackageManager.packagesConfigChanged:
PackageManager.rebuildRemappings(self)
if savePackagesChanges:
PackageManager.save(self)
else:
PackageManager.packagesConfigChanged = False
if showPackages:
self.addToLog(_("Taxonomy packages:"), messageCode="info")
for packageInfo in PackageManager.orderedPackagesConfig()["packages"]:
self.addToLog(_("Package: {0}; version: {1}; status: {2}; date: {3}; description: {4}.").format(
packageInfo.get("name"), packageInfo.get("version"), packageInfo.get("status"),
packageInfo.get("fileDate"), packageInfo.get("description")),
messageCode="info", file=packageInfo.get("URL"))
self.loadPackages(options.packages, options.packageManifestName)

if options.showEnvironment:
self.addToLog(_("Config directory: {0}").format(self.configDir))
Expand Down Expand Up @@ -843,10 +802,6 @@ def monitorParentProcess():
if options.outputAttribution:
self.modelManager.outputAttribution = options.outputAttribution
self.modelManager.validateTestcaseSchema = options.validateTestcaseSchema
if options.internetConnectivity == "offline":
self.webCache.workOffline = True
elif options.internetConnectivity == "online":
self.webCache.workOffline = False
if options.internetTimeout is not None:
self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout
if options.internetLogDownloads:
Expand Down Expand Up @@ -1184,6 +1139,63 @@ def showStatusOnPipe(self, message, clearAfter=None):
# fh.write("Status pipe exception {} {}\n".format(type(ex), ex))
system.exit()

def loadPackages(self, packages: str, packageManifestName: str):
"""
Loads specified packages.
:param packages: Pipe-separated list of options. See CLI documentation for 'packages'.
:param packageManifestName: Unix shell style pattern used to find package manifest.
"""
from arelle import PackageManager
savePackagesChanges = True
showPackages = False
for packageCmd in packages.split('|'):
cmd = packageCmd.strip()
if cmd == "show":
showPackages = True
elif cmd == "temp":
savePackagesChanges = False
elif cmd.startswith("+"):
packageInfo = PackageManager.addPackage(self, cmd[1:], packageManifestName)
if packageInfo:
self.addToLog(_("Addition of package {0} successful.").format(packageInfo.get("name")),
messageCode="info", file=packageInfo.get("URL"))
else:
self.addToLog(_("Unable to load package."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PackageManager.reloadPackageModule(self, cmd[1:]):
self.addToLog(_("Reload of package successful."), messageCode="info", file=cmd[1:])
else:
self.addToLog(_("Unable to reload package."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PackageManager.removePackageModule(self, cmd[1:]):
self.addToLog(_("Deletion of package successful."), messageCode="info", file=cmd[1:])
else:
self.addToLog(_("Unable to delete package."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePackagesChanges = False
packageInfo = PackageManager.addPackage(self, cmd, packageManifestName)
if packageInfo:
self.addToLog(_("Activation of package {0} successful.").format(packageInfo.get("name")),
messageCode="info", file=packageInfo.get("URL"))
else:
self.addToLog(_("Unable to load package \"%(name)s\". "),
messageCode="arelle:packageLoadingError",
messageArgs={"name": cmd, "file": cmd}, level=logging.ERROR)
if PackageManager.packagesConfigChanged:
PackageManager.rebuildRemappings(self)
if savePackagesChanges:
PackageManager.save(self)
else:
PackageManager.packagesConfigChanged = False
if showPackages:
self.addToLog(_("Taxonomy packages:"), messageCode="info")
for packageInfo in PackageManager.orderedPackagesConfig()["packages"]:
self.addToLog(_("Package: {0}; version: {1}; status: {2}; date: {3}; description: {4}.").format(
packageInfo.get("name"), packageInfo.get("version"), packageInfo.get("status"),
packageInfo.get("fileDate"), packageInfo.get("description")),
messageCode="info", file=packageInfo.get("URL"))

if __name__ == "__main__":
'''
if '--COMserver' in sys.argv:
Expand Down
32 changes: 24 additions & 8 deletions arelle/PackageManager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,27 @@ def langCloseness(l1, l2):
return i
return _len


def _parseFile(cntlr, parser, filepath, file, schemaUrl):
"""
Returns tree from `file`, parsed with `parser`, and validated against the provided schema at `schemaUrl`.
:return: Tree if parsed and validated, None if `schemaUrl` could not be loaded.
"""
tree = etree.parse(file, parser=parser)
# schema validate tp xml
if cntlr.workingOnlineOrInCache(schemaUrl):
xsdTree = etree.parse(schemaUrl, parser=parser)
etree.XMLSchema(xsdTree).assertValid(tree)
else:
cntlr.addToLog(_("File could not be validated against the schema (%(schemaUrl)s) because the schema was not "
"found in the cache and Arelle is configured to work offline."),
messageArgs={"schemaUrl": schemaUrl},
messageCode="tpe:workingOffline",
file=filepath,
level=logging.ERROR)
return tree


def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
global ArchiveFileIOError
if ArchiveFileIOError is None:
Expand All @@ -72,10 +93,7 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
_file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web
parser = lxmlResolvingParser(cntlr)
try:
tree = etree.parse(_file,parser=parser)
# schema validate tp xml
xsdTree = etree.parse(TP_XSD,parser=parser)
etree.XMLSchema(xsdTree).assertValid(tree)
tree = _parseFile(cntlr, parser, metadataFile, _file, TP_XSD)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
cntlr.addToLog(_("Taxonomy package file syntax error %(error)s"),
messageArgs={"error": str(err)},
Expand Down Expand Up @@ -166,10 +184,8 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
"http://xbrl.org/REC/2016-04-19/taxonomy-package"):
catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0],parser=parser)
# schema validate tp xml
xsdTree = etree.parse(CAT_XSD,parser=parser)
etree.XMLSchema(xsdTree).assertValid(rewriteTree)
_file = filesource.file(catalogFile)[0]
rewriteTree = _parseFile(cntlr, parser, catalogFile, _file, CAT_XSD)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
cntlr.addToLog(_("Catalog file syntax error %(error)s"),
messageArgs={"error": str(err)},
Expand Down
2 changes: 1 addition & 1 deletion arelle/WebCache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ def normalizeUrl(self, url: Optional[str], base: Optional[str] = None) -> Any:
def encodeForFilename(self, pathpart):
return self.encodeFileChars.sub(lambda m: '^{0:03}'.format(ord(m.group(0))), pathpart)

def urlToCacheFilepath(self, url):
def urlToCacheFilepath(self, url: str) -> str:
scheme, sep, path = url.partition("://")
filepath = [self.cacheDir, scheme]
pathparts = path.split('/')
Expand Down

0 comments on commit 651fe12

Please sign in to comment.