Skip to content
This repository has been archived by the owner on Jan 11, 2018. It is now read-only.

Commit

Permalink
Merge pull request #37 from d3ming/compare2
Browse files Browse the repository at this point in the history 
Redo the outputs for compare and compare testing
  • Loading branch information
@d3ming
d3ming committed Dec 17, 2015
2 parents 3828ab5 + 098b7fe commit 3c57390
Show file tree
Hide file tree
Showing 153 changed files with 1,067 additions and 238 deletions.
17 changes: 9 additions & 8 deletions tagcompare/capture.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
import logger


MAX_REMOTE_JOBS = 6
LOGGER = logger.Logger(name="capture", writefile=True).get()


def __capture_tags_for_configs(cids, pathbuilder,
comparisons=settings.DEFAULT.comparisons,
configs=settings.DEFAULT.configs,
tagsizes=settings.DEFAULT.tagsizes,
tagtypes=settings.DEFAULT.tagtypes,
Expand All @@ -24,15 +24,15 @@ def __capture_tags_for_configs(cids, pathbuilder,
LOGGER.warn("No tags found to capture!")
return

LOGGER.info("Capturing tags for %s campaigns", len(cids))
LOGGER.info("Capturing tags for %s campaigns over %s configs", len(cids),
len(configs))
# TODO: Implement progress bar
errors = []
all_configs = configs
config_names = settings.get_unique_configs_from_comparisons(comparisons)
pool = ThreadPool(processes=10)
pool = ThreadPool(processes=MAX_REMOTE_JOBS)
results = {}

for config_name in config_names:
config_data = all_configs[config_name]
for config_name in configs:
config_data = configs[config_name]
if not config_data['enabled']:
LOGGER.debug("Skipping disabled config %s" % config_name)
continue
Expand Down Expand Up @@ -113,7 +113,7 @@ def __capture_tags(capabilities, tags, pathbuilder,
for cid in tags:
pathbuilder.cid = cid
tags_per_campaign = tags[cid]
LOGGER.debug("tags_per_campaign: %s", str(tags_per_campaign))
# LOGGER.debug("tags_per_campaign: %s", str(tags_per_campaign))
# TODO: Refactor better with __capture_tag
# It's weird that we pass in a pathbuilder object and do two nested loops here
for tagsize in tagsizes:
Expand All @@ -131,6 +131,7 @@ def __capture_tags(capabilities, tags, pathbuilder,
else:
browser_errors += r
num_captured += 1
LOGGER.debug("Captured tags for campaign %s on %s", cid, capabilities)
except KeyboardInterrupt:
driver.quit()
driver = None
Expand Down
292 changes: 139 additions & 153 deletions tagcompare/compare.py
View file
Original file line number Diff line number Diff line change
@@ -1,168 +1,155 @@
"""
We want to make a set of comparisons factoring in popular browser/OS variations
Comparisons are determined from usage stats:
- OS usage stats:
http://www.w3schools.com/browsers/browsers_os.asp
- Browser usage stats:
http://www.w3schools.com/browsers/browsers_stats.asp
Configs are made based on supported capabilities:
- saucelabs:
https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/
- browserstack:
https://www.browserstack.com/list-of-browsers-and-platforms?product=automate
"""
import itertools
import os
import itertools
from multiprocessing.pool import ThreadPool

import settings
import placelocal
import logger
import output
import settings
import image
import placelocal


LOGGER = logger.Logger("compare", writefile=True).get()
NUM_COMPARE_PROCESSES = 8


class CompareResult:
def __init__(self):
self.result = {
settings.ImageErrorLevel.INVALID: 0,
settings.ImageErrorLevel.NONE: 0,
settings.ImageErrorLevel.SLIGHT: 0,
settings.ImageErrorLevel.MODERATE: 0,
settings.ImageErrorLevel.SEVERE: 0,
}

self.total = 0

def __str__(self):
return "CompareResult (total={}): \n {}".format(self.total, self.result)

def increment(self, key):
if key not in self.result:
raise ValueError("Invalid key for results: %s", key)

self.result[key] += 1
self.total += 1
return self.result


def __write_result_image(pathbuilder, result_image,
outputdir=None, info=None, prefix="merged"):
if not outputdir:
outputdir = pathbuilder.create(allow_partial=True)
elif not os.path.exists(outputdir):
os.makedirs(outputdir)

filename = os.path.join(outputdir,
prefix + pathbuilder.tagname + ".png")
LOGGER.debug("Saving merged image for %s to %s", pathbuilder.cid, filename)
if info:
result_image = image.add_info(result_image, info)
result_image.save(open(filename, 'wb'))


def _merge_images(pathbuilder, configs):
merged_image = None
compare_pb = pathbuilder.clone(build=output.DEFAULT_BUILD_NAME)
for c in configs:
pb = compare_pb.clone(config=c)
tagimage = image.normalize_img(pb.tagimage)
image.add_label(image=tagimage, label=c)
if not merged_image:
merged_image = tagimage
else:
merged_image = image.merge_images(merged_image, tagimage)
return merged_image


def _compare_configs(pathbuilder, configs, comparison, result):
diff = _compare_configs_internal(pathbuilder=pathbuilder, configs=configs)
if diff == settings.ImageErrorLevel.INVALID:
result.increment(key=diff)
return False

# TODO: a class should be made to track a compare job


def compare_configs(pathbuilder, configs):
# TODO: Should we check if configs are enabled before comparing?
assert configs, "No configs!"
result_image = _merge_images(pathbuilder=pathbuilder, configs=configs)
prefix_label = comparison + "_"
r = __handle_output(pathbuilder=pathbuilder, result_image=result_image,
diff=diff, prefix=prefix_label)
result.increment(key=r)

# Always compare against default job path
cid = pathbuilder.cid
build = pathbuilder.build
compare_build = output.DEFAULT_BUILD_NAME
sizes = settings.DEFAULT.tagsizes
types = settings.DEFAULT.tagtypes
count = 0
errorcount = 0
skipcount = 0

# Compare all combinations of configs
for a, b in itertools.combinations(configs, 2):
for s in sizes:
for t in types:
pba = pathbuilder.clone(build=compare_build, config=a,
tagsize=s,
tagtype=t, cid=cid)
pbb = pba.clone(config=b)
pba_img = pba.tagimage
pbb_img = pbb.tagimage
count += 1

result_pb = pba.clone(build=build)
compare_result = compare_images(pba_img, pbb_img,
pathbuilder=result_pb)
if compare_result is None:
skipcount += 1
elif compare_result is False:
errorcount += 1

LOGGER.debug("Compared %s images: %s errors, %s skipped", count,
errorcount,
skipcount)
return errorcount, count, skipcount


def compare_images(file1, file2, pathbuilder):
"""Compares two image files.
Returns True if compare took place, False otherwise
:param file1:
:param file2:
:return:
def _compare_configs_internal(pathbuilder, configs):
"""
if not os.path.exists(file1):
LOGGER.warn("SKIPPING compare - %s not found!", file1)
return None
if not os.path.exists(file2):
LOGGER.warn("SKIPPING compare - %s not found!", file2)
return None

diff = image.compare(file1, file2)
if diff is False:
# Unable to produce diff due to errors
return False

# Write likely errors to the cidpath, otherwise write to the tagpath
if diff > settings.ImageErrorThreshold.BAD:
p = __write_merged_image(file1, file2, diff,
outputpath=pathbuilder.buildpath)
LOGGER.warn("IMAGE_DIFF: %s, see %s", diff, p)
return False
if diff > settings.ImageErrorThreshold.MODERATE:
p = __write_merged_image(file1, file2, diff,
outputpath=pathbuilder.cidpath)
LOGGER.warn("IMAGE_DIFF: %s, see %s", diff, p)
return False
if diff > settings.ImageErrorThreshold.SLIGHT:
__write_merged_image(file1, file2, diff,
outputpath=pathbuilder.tagpath)
return True


def __get_compare_name(file1, file2):
filename1 = os.path.basename(file1)
filename2 = os.path.basename(file2)
compare_name = str.format("{}__vs__{}".format(
os.path.splitext(filename1)[0], os.path.splitext(filename2)[0]))
return compare_name


def __write_merged_image(file1, file2, diff, outputpath):
assert os.path.exists(file1), "file1 doesn't exist at path {}".format(
file1)
assert os.path.exists(file2), "file2 doesn't exist at path {}".format(
file2)
compare_name = __get_compare_name(file1, file2)

# Generate additional info in output
mergedimg = image.merge_images(file1, file2)
info = {"name": compare_name, "diff": diff}
mergedimg2 = image.add_info(mergedimg, info)

# comparisons are always done for the same campaign, so output to the cidpath
if not os.path.exists(outputpath):
os.makedirs(outputpath)
merged_path = os.path.join(outputpath, compare_name + ".png")
if not settings.TEST_MODE:
mergedimg2.save(open(merged_path, 'wb'))
LOGGER.debug("IMAGE_DIFF=%s (%s). See %s", diff, compare_name, merged_path)
return merged_path


def do_all_comparisons(cids=settings.DEFAULT.campaigns,
pids=settings.DEFAULT.publishers, pathbuilder=None):
cids = placelocal.get_cids(cids=cids, pids=pids)

total_compares = 0
total_errors = 0
total_skipped = 0

if not pathbuilder:
pathbuilder = output.create(
build=output.generate_build_string())
Compares a given tag for all combinations of the specified list of configs
:param pathbuilder: the pathbuilder pointing to the tag to compare
:param configs: the list of configs to compare
:return: the average diff from image comparison
"""
total_diff = 0
compare_pb = pathbuilder.clone(build=output.DEFAULT_BUILD_NAME)
combo_count = 0
for a, b in itertools.combinations(configs, 2):
pba = compare_pb.clone(config=a)
pbb = compare_pb.clone(config=b)
pba_image = pba.tagimage
pbb_image = pbb.tagimage
if not os.path.exists(pba_image):
LOGGER.error("File not found: %s", pba_image)
return settings.ImageErrorLevel.INVALID
if not os.path.exists(pbb_image):
LOGGER.error("File not found: %s", pbb_image)
return settings.ImageErrorLevel.INVALID
total_diff += image.compare(pba.tagimage, pbb.tagimage)
combo_count += 1
return total_diff / combo_count


def compare(pb, cids=None, sizes=settings.DEFAULT.tagsizes,
types=settings.DEFAULT.tagtypes,
comparison="latest",
configs=None):
if not cids:
cids = placelocal.get_cids(cids=settings.DEFAULT.campaigns,
pids=settings.DEFAULT.publishers)
if not configs:
configs = settings.DEFAULT.comparisons[comparison]

pool = ThreadPool(processes=NUM_COMPARE_PROCESSES)
compare_threads = []
result = CompareResult()
LOGGER.info("Starting compare for %s campaigns over %s configs...",
len(cids), len(configs))

for cid in cids:
pathbuilder.cid = cid
comparisons = settings.DEFAULT.comparisons
for name in comparisons:
LOGGER.debug("*** Comparing set: %s...", name)
configs_to_compare = comparisons[name]
errors, count, skipped = compare_configs(
pathbuilder=pathbuilder,
configs=configs_to_compare)
total_errors += errors
total_compares += count
total_skipped += skipped

LOGGER.info("*** RESULTS ***\nCompared %s images: %s errors, %s skipped",
total_compares, total_errors,
total_skipped)
LOGGER.info("See additional logs at: %s", pathbuilder.buildpath)
for s in sizes:
for t in types:
newpb = pb.clone(cid=cid, tagsize=s, tagtype=t)
compare_threads.append(pool.apply_async(func=_compare_configs,
args=(newpb, configs,
comparison, result)))
for ct in compare_threads:
ct.get()
LOGGER.info("Compare over configs=%s, result=%s", configs, result)
return result


def __handle_output(pathbuilder, result_image, diff, prefix=""):
info = {"name": pathbuilder.tagname, "diff": diff}
result = settings.ImageErrorLevel.NONE

if diff > settings.ImageErrorLevel.SEVERE:
prefix += "severe_"
result = settings.ImageErrorLevel.SEVERE
elif diff > settings.ImageErrorLevel.MODERATE:
prefix += "moderate_"
result = settings.ImageErrorLevel.MODERATE
elif diff > settings.ImageErrorLevel.SLIGHT:
prefix += "slight_"
result = settings.ImageErrorLevel.SLIGHT
__write_result_image(pathbuilder=pathbuilder, result_image=result_image,
info=info, prefix=prefix)
return result


def main(build=None):
Expand All @@ -172,8 +159,7 @@ def main(build=None):
build = output.generate_build_string()
jobname = "compare_" + build
pb = output.create(build=jobname)
do_all_comparisons(cids=settings.DEFAULT.campaigns,
pids=settings.DEFAULT.publishers, pathbuilder=pb)
compare(pb)
return pb


Expand Down
Loading

0 comments on commit 3c57390

Please sign in to comment.