Skip to content

Commit

Permalink
Use memoize decorators to cache results of expensive functions withou…
Browse files Browse the repository at this point in the history 
…t nasty module globals
  • Loading branch information
@gregneagle
gregneagle committed Dec 13, 2016
1 parent 74aed49 commit ac003c8
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 112 deletions.
144 changes: 78 additions & 66 deletions code/client/munkilib/munkicommon.py
View file
Expand Up @@ -118,6 +118,20 @@ class TimeoutError(Error):
"""Timeout limit exceeded since last I/O."""


class memoize(dict):
'''Class to cache the return values of an expensive function.
This version supports only functions with non-keyword arguments'''
def __init__(self, func):
self.func = func

def __call__(self, *args):
return self[args]

def __missing__(self, key):
result = self[key] = self.func(*key)
return result


def getOsVersion(only_major_minor=True, as_tuple=False):
"""Returns an OS version.
Expand Down Expand Up @@ -2406,78 +2420,76 @@ def getIntel64Support():
else:
return False

MACHINE = {}
@memoize
def getMachineFacts():
"""Gets some facts about this machine we use to determine if a given
installer is applicable to this OS or hardware"""
if not MACHINE:
MACHINE['hostname'] = os.uname()[1]
MACHINE['arch'] = os.uname()[4]
MACHINE['os_vers'] = getOsVersion(only_major_minor=False)
hardware_info = get_hardware_info()
MACHINE['machine_model'] = hardware_info.get('machine_model', 'UNKNOWN')
MACHINE['munki_version'] = get_version()
MACHINE['ipv4_address'] = get_ip_addresses('IPv4')
MACHINE['ipv6_address'] = get_ip_addresses('IPv6')
MACHINE['serial_number'] = hardware_info.get('serial_number', 'UNKNOWN')

if MACHINE['arch'] == 'x86_64':
MACHINE['x86_64_capable'] = True
elif MACHINE['arch'] == 'i386':
MACHINE['x86_64_capable'] = getIntel64Support()
return MACHINE


CONDITIONS = {}
machine = dict()
machine['hostname'] = os.uname()[1]
machine['arch'] = os.uname()[4]
machine['os_vers'] = getOsVersion(only_major_minor=False)
hardware_info = get_hardware_info()
machine['machine_model'] = hardware_info.get('machine_model', 'UNKNOWN')
machine['munki_version'] = get_version()
machine['ipv4_address'] = get_ip_addresses('IPv4')
machine['ipv6_address'] = get_ip_addresses('IPv6')
machine['serial_number'] = hardware_info.get('serial_number', 'UNKNOWN')

if machine['arch'] == 'x86_64':
machine['x86_64_capable'] = True
elif machine['arch'] == 'i386':
machine['x86_64_capable'] = getIntel64Support()
return machine


@memoize
def getConditions():
"""Fetches key/value pairs from condition scripts
which can be placed into /usr/local/munki/conditions"""
global CONDITIONS
if not CONDITIONS:
# define path to conditions directory which would contain
# admin created scripts
scriptdir = os.path.realpath(os.path.dirname(sys.argv[0]))
conditionalscriptdir = os.path.join(scriptdir, "conditions")
# define path to ConditionalItems.plist
conditionalitemspath = os.path.join(
pref('ManagedInstallDir'), 'ConditionalItems.plist')
try:
# delete CondtionalItems.plist so that we're starting fresh
os.unlink(conditionalitemspath)
except (OSError, IOError):
pass
if os.path.exists(conditionalscriptdir):
from munkilib import utils
for conditionalscript in listdir(conditionalscriptdir):
if conditionalscript.startswith('.'):
# skip files that start with a period
continue
conditionalscriptpath = os.path.join(
conditionalscriptdir, conditionalscript)
if os.path.isdir(conditionalscriptpath):
# skip directories in conditions directory
continue
try:
# attempt to execute condition script
dummy_result, dummy_stdout, dummy_stderr = (
utils.runExternalScript(conditionalscriptpath))
except utils.ScriptNotFoundError:
pass # script is not required, so pass
except utils.RunExternalScriptError, err:
print >> sys.stderr, unicode(err)
else:
# /usr/local/munki/conditions does not exist
pass
if (os.path.exists(conditionalitemspath) and
validPlist(conditionalitemspath)):
# import conditions into CONDITIONS dict
CONDITIONS = FoundationPlist.readPlist(conditionalitemspath)
os.unlink(conditionalitemspath)
else:
# either ConditionalItems.plist does not exist
# or does not pass validation
CONDITIONS = {}
return CONDITIONS
# define path to conditions directory which would contain
# admin created scripts
scriptdir = os.path.realpath(os.path.dirname(sys.argv[0]))
conditionalscriptdir = os.path.join(scriptdir, "conditions")
# define path to ConditionalItems.plist
conditionalitemspath = os.path.join(
pref('ManagedInstallDir'), 'ConditionalItems.plist')
try:
# delete CondtionalItems.plist so that we're starting fresh
os.unlink(conditionalitemspath)
except (OSError, IOError):
pass
if os.path.exists(conditionalscriptdir):
from munkilib import utils
for conditionalscript in listdir(conditionalscriptdir):
if conditionalscript.startswith('.'):
# skip files that start with a period
continue
conditionalscriptpath = os.path.join(
conditionalscriptdir, conditionalscript)
if os.path.isdir(conditionalscriptpath):
# skip directories in conditions directory
continue
try:
# attempt to execute condition script
dummy_result, dummy_stdout, dummy_stderr = (
utils.runExternalScript(conditionalscriptpath))
except utils.ScriptNotFoundError:
pass # script is not required, so pass
except utils.RunExternalScriptError, err:
print >> sys.stderr, unicode(err)
else:
# /usr/local/munki/conditions does not exist
pass
if (os.path.exists(conditionalitemspath) and
validPlist(conditionalitemspath)):
# import conditions into conditions dict
conditions = FoundationPlist.readPlist(conditionalitemspath)
os.unlink(conditionalitemspath)
else:
# either ConditionalItems.plist does not exist
# or does not pass validation
conditions = {}
return conditions


def isAppRunning(appname):
Expand Down
89 changes: 43 additions & 46 deletions code/client/munkilib/updatecheck.py
View file
Expand Up @@ -54,6 +54,20 @@
FORCE_INSTALL_WARNING_HOURS = 4


class memoize(dict):
'''Class to cache the return values of an expensive function.
This version supports only functions with non-keyword arguments'''
def __init__(self, func):
self.func = func

def __call__(self, *args):
return self[args]

def __missing__(self, key):
result = self[key] = self.func(*key)
return result


def makeCatalogDB(catalogitems):
"""Takes an array of catalog items and builds some indexes so we can
get our common data faster. Returns a dict we can use like a database"""
Expand Down Expand Up @@ -152,10 +166,10 @@ def addPackageids(catalogitems, itemname_to_pkgid, pkgid_to_itemname):
pkgid_to_itemname[pkgid][name].append(vers)


INSTALLEDPKGS = {}
@memoize
def getInstalledPackages():
"""Builds a dictionary of installed receipts and their version number"""
#global INSTALLEDPKGS
installedpkgs = {}

# we use the --regexp option to pkgutil to get it to return receipt
# info for all installed packages. Huge speed up.
Expand All @@ -168,7 +182,7 @@ def getInstalledPackages():
if pliststr:
plist = FoundationPlist.readPlistFromString(pliststr)
if 'pkg-version' in plist and 'pkgid' in plist:
INSTALLEDPKGS[plist['pkgid']] = (
installedpkgs[plist['pkgid']] = (
plist['pkg-version'] or '0.0.0.0.0')
else:
break
Expand All @@ -184,27 +198,20 @@ def getInstalledPackages():
pkgid = pkginfo.get('packageid')
thisversion = pkginfo.get('version')
if pkgid:
if not pkgid in INSTALLEDPKGS:
INSTALLEDPKGS[pkgid] = thisversion
if not pkgid in installedpkgs:
installedpkgs[pkgid] = thisversion
else:
# pkgid is already in our list. There must be
# multiple receipts with the same pkgid.
# in this case, we want the highest version
# number, since that's the one that's
# installed, since presumably
# the newer package replaced the older one
storedversion = INSTALLEDPKGS[pkgid]
storedversion = installedpkgs[pkgid]
if (munkicommon.MunkiLooseVersion(thisversion) >
munkicommon.MunkiLooseVersion(storedversion)):
INSTALLEDPKGS[pkgid] = thisversion

# debug code. left here for future debug use
#ManagedInstallDir = munkicommon.pref('ManagedInstallDir')
#receiptsdatapath = os.path.join(ManagedInstallDir, 'FoundReceipts.plist')
#try:
# FoundationPlist.writePlist(INSTALLEDPKGS, receiptsdatapath)
#except FoundationPlist.NSPropertyListWriteException:
# pass
installedpkgs[pkgid] = thisversion
return installedpkgs


def bestVersionMatch(vers_num, item_dict):
Expand All @@ -226,16 +233,11 @@ def bestVersionMatch(vers_num, item_dict):
return None


# global pkgdata cache
PKGDATA = None
@memoize
def analyzeInstalledPkgs():
"""Analyzed installed packages in an attempt to determine what is
installed."""
global PKGDATA
# if we've populated the cache, just return it
if PKGDATA is not None:
return PKGDATA
PKGDATA = {}
pkgdata = {}
itemname_to_pkgid = {}
pkgid_to_itemname = {}
for catalogname in CATALOG.keys():
Expand All @@ -244,9 +246,7 @@ def analyzeInstalledPkgs():
# itemname_to_pkgid now contains all receipts (pkgids) we know about
# from items in all available catalogs

if not INSTALLEDPKGS:
getInstalledPackages()
# INSTALLEDPKGS now contains all receipts found on this machine
installedpkgs = getInstalledPackages()

installed = []
partiallyinstalled = []
Expand All @@ -256,14 +256,14 @@ def analyzeInstalledPkgs():
somepkgsfound = False
allpkgsfound = True
for pkgid in itemname_to_pkgid[name].keys():
if pkgid in INSTALLEDPKGS.keys():
if pkgid in installedpkgs.keys():
somepkgsfound = True
if not name in installedpkgsmatchedtoname:
installedpkgsmatchedtoname[name] = []
# record this pkgid for Munki install item name
installedpkgsmatchedtoname[name].append(pkgid)
else:
# didn't find pkgid in INSTALLEDPKGS
# didn't find pkgid in installedpkgs
allpkgsfound = False
if allpkgsfound:
# we found all receipts by pkgid on disk
Expand Down Expand Up @@ -306,16 +306,16 @@ def analyzeInstalledPkgs():
references[pkgid] = []
references[pkgid].append(name)

# look through all our INSTALLEDPKGS, looking for ones that have not been
# look through all our installedpkgs, looking for ones that have not been
# attached to any Munki names yet
orphans = [pkgid for pkgid in INSTALLEDPKGS.keys()
orphans = [pkgid for pkgid in installedpkgs.keys()
if pkgid not in references]

# attempt to match orphans to Munki item names
matched_orphans = []
for pkgid in orphans:
if pkgid in pkgid_to_itemname:
installed_pkgid_version = INSTALLEDPKGS[pkgid]
installed_pkgid_version = installedpkgs[pkgid]
possible_match_items = pkgid_to_itemname[pkgid]
best_match = bestVersionMatch(
installed_pkgid_version, possible_match_items)
Expand All @@ -334,28 +334,28 @@ def analyzeInstalledPkgs():
if not name in references[pkgid]:
references[pkgid].append(name)

PKGDATA['receipts_for_name'] = installedpkgsmatchedtoname
PKGDATA['installed_names'] = installed
PKGDATA['pkg_references'] = references
pkgdata['receipts_for_name'] = installedpkgsmatchedtoname
pkgdata['installed_names'] = installed
pkgdata['pkg_references'] = references

# left here for future debugging/testing use....
#PKGDATA['itemname_to_pkgid'] = itemname_to_pkgid
#PKGDATA['pkgid_to_itemname'] = pkgid_to_itemname
#PKGDATA['partiallyinstalled_names'] = partiallyinstalled
#PKGDATA['orphans'] = orphans
#PKGDATA['matched_orphans'] = matched_orphans
#pkgdata['itemname_to_pkgid'] = itemname_to_pkgid
#pkgdata['pkgid_to_itemname'] = pkgid_to_itemname
#pkgdata['partiallyinstalled_names'] = partiallyinstalled
#pkgdata['orphans'] = orphans
#pkgdata['matched_orphans'] = matched_orphans
#ManagedInstallDir = munkicommon.pref('ManagedInstallDir')
#pkgdatapath = os.path.join(ManagedInstallDir, 'PackageData.plist')
#try:
# FoundationPlist.writePlist(PKGDATA, pkgdatapath)
# FoundationPlist.writePlist(pkgdata, pkgdatapath)
#except FoundationPlist.NSPropertyListWriteException:
# pass
#catalogdbpath = os.path.join(ManagedInstallDir, 'CatalogDB.plist')
#try:
# FoundationPlist.writePlist(CATALOG, catalogdbpath)
#except FoundationPlist.NSPropertyListWriteException:
# pass
return PKGDATA
return pkgdata


def getAppBundleID(path):
Expand Down Expand Up @@ -691,8 +691,7 @@ def compareReceiptVersion(item):
'Skipping %s because it is marked as optional',
item.get('packageid', item.get('name')))
return 1
if not INSTALLEDPKGS:
getInstalledPackages()
installedpkgs = getInstalledPackages()
if 'packageid' in item and 'version' in item:
pkgid = item['packageid']
vers = item['version']
Expand All @@ -701,7 +700,7 @@ def compareReceiptVersion(item):

munkicommon.display_debug1('Looking for package %s, version %s',
pkgid, vers)
installedvers = INSTALLEDPKGS.get(pkgid)
installedvers = installedpkgs.get(pkgid)
if installedvers:
return compareVersions(installedvers, vers)
else:
Expand Down Expand Up @@ -2400,7 +2399,7 @@ def processRemoval(manifestitem, cataloglist, installinfo):
packagesToReallyRemove = []
for pkg in packagesToRemove:
munkicommon.display_debug1('Considering %s for removal...', pkg)
# find pkg in PKGDATA['pkg_references'] and remove the reference
# find pkg in pkgdata['pkg_references'] and remove the reference
# so we only remove packages if we're the last reference to it
pkgdata = analyzeInstalledPkgs()
if pkg in pkgdata['pkg_references']:
Expand Down Expand Up @@ -2970,12 +2969,10 @@ def check(client_id='', localmanifestpath=None):
0 if there are no available updates, and -1 if there were errors."""

global MACHINE
munkicommon.getMachineFacts()
MACHINE = munkicommon.getMachineFacts()
munkicommon.report['MachineInfo'] = MACHINE

global CONDITIONS
munkicommon.getConditions()
CONDITIONS = munkicommon.getConditions()

keychain_obj = keychain.MunkiKeychain()
Expand Down

0 comments on commit ac003c8

Please sign in to comment.