Skip to content
Permalink
Branch: master
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
2075 lines (1613 sloc) 62.9 KB
#
# Collective Knowledge: CK-powered AI/SW/HW co-design for ReQuEST tournaments
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, cTuning foundation/dividiti
#
cfg={} # Will be updated by CK (meta description of this module)
work={} # Will be updated by CK (temporal data)
ck=None # Will be updated by CK (initialized CK kernel)
repo_with_validated_results='ck-request-asplos18-results'
# Local settings
line='================================================================'
form_name='request_web_form'
onchange='document.'+form_name+'.submit();'
hextra='<center>\n'
hextra+='<p>\n'
hextra+=' Beta scoreboard for <a href="http://cknowledge.org/request-cfp-asplos2018.html">ReQuEST@ASPLOS\'18 AI/SW/HW co-design competition</a>\n'
hextra+=' (see <a href="https://doi.org/10.1145/3229762">ACM proceedings</a> and <a href="https://portalparts.acm.org/3230000/3229762/fm/frontmatter.pdf">results report</a>).\n'
#hextra+=' (your <a href="https://github.com/ctuning/ck-request/issues">feedback</a>)\n'
hextra+='</center>\n'
hextra+='<p>\n'
selector=[
{'name':'Results', 'key':'results', 'skip_update':'yes', 'new_line_after':'yes', 'skip_from_reset':'yes'}, # need to skip from reset to be able to get values from other repos
{'name':'Algorithm species', 'key':'algorithm_species', 'module_uoa':'1702c3e426ca54c5'},
# {'name':'Competition', 'key':'scenario_module_uoa', 'module_uoa':'032630d041b4fd8a'},
{'name':'Model species', 'key':'model_species', 'module_uoa':'38e7de41acb41d3b'},
{'name':'Precision', 'key':'model_precision'},
{'name':'Dataset species', 'key':'dataset_species', 'new_line':'yes', 'keep_empty':'yes'},
{'name':'Dataset size', 'key':'dataset_size', 'type':'int'},
{'name':'Cloud/farm', 'key':'farm', 'new_line':'yes', 'keep_empty':'yes'},
{'name':'Platform species', 'key':'platform_species'},
{'name':'Platform', 'key':'plat_name'},
{'name':'OS name', 'key':'os_name', 'new_line':'yes'},
{'name':'CPU name', 'key':'cpu_name', 'keep_empty':'yes'},
{'name':'GPGPU name', 'key':'gpgpu_name', 'keep_empty':'yes'}
]
selector2=[
{'name':'Algorithm implementation (CK program)', 'key':'##choices#data_uoa#min'},
{'name':'Model design', 'key':'##meta#model_design_name'},
{'name':'Compiler', 'key':'##meta#compiler_name','new_line':'yes', 'keep_empty':'yes'},
{'name':'Library/framework', 'key':'##meta#library_name', 'keep_empty':'yes'},
{'name':'OpenCL driver', 'key':'##features#gpgpu@0#gpgpu_misc#opencl c version#min',
'extra_key':'##features#gpgpu@0#gpgpu_misc#opencl_c_version#min', 'new_line':'yes', 'keep_empty':'yes'},
{'name':'Batch size', 'key':'##features#batch_size#min','type':'int', 'new_line':'yes'},
{'name':'CPU freq (MHz)', 'key':'##features#cpu_freq#min','type':'int', 'keep_empty':'yes'},
{'name':'GPU freq (MHz)', 'key':'##features#gpu_freq#min','type':'int', 'keep_empty':'yes'},
{'name':'Freq (MHz)', 'key':'##features#freq#min','type':'int', 'keep_empty':'yes'}
]
selector3=[
{'name':'Plot dimension 1 (X)', 'key':'plot_dimension1'},
{'name':'Show variation', 'key':'plot_variation_dimension1', 'new_line_after':'yes'},
{'name':'Plot dimension 2 (Y)', 'key':'plot_dimension2'},
{'name':'Show variation', 'key':'plot_variation_dimension2', 'new_line_after':'yes'}
]
k_hi_uid='highlight_behavior_uid'
k_hi_user='highlight_by_user'
k_view_all='all'
hidden_keys=[k_hi_uid, k_hi_user, k_view_all]
dimensions=[
{"key":"experiment", "name":"Experiment number", "skip_from_cache":"yes", "view_key":"__number"},
{"key":"##characteristics#run#prediction_time_avg_s", "name":"Prediction time per 1 image (min, sec.)"},
{"key":"##characteristics#run#inference_latency", "name":"Inference latency for 1 image (min, sec.)"},
{"key":"##characteristics#run#inference_throughput", "name":"Inference throughput (max, images per sec.)", "reverse":"yes"},
{"key":"##characteristics#run#accuracy_top1", "name":"Accuracy on all images (Top1)"},
{"key":"##characteristics#run#accuracy_top5", "name":"Accuracy on all images (Top5)"},
{"key":"##features#model_size", "name":"Model size (B)"},
{"key":"##meta#platform_peak_power", "name":"Platform peak power (W)", "from_meta":"yes"},
{"key":"##meta#platform_price", "name":"Platform price ($)", "from_meta":"yes"},
{"key":"##characteristics#run#usage_cost", "name":"Usage cost ($)"},
{"key":"##meta#platform_species", "name":"Platform species", "from_meta":"yes"},
{"key":"##meta#model_species", "name":"Model species", "from_meta":"yes", 'module_uoa':'38e7de41acb41d3b'},
{"key":"##meta#model_precision", "name":"Model precision", "from_meta":"yes"},
{"key":"##meta#dataset_species", "name":"Dataset species", "from_meta":"yes"},
{"key":"##features#freq", "name":"Device frequency (MHz)"},
{"key":"##features#cpu_freq", "name":"CPU frequency (MHz)"},
{"key":"##features#gpu_freq", "name":"GPU frequency (MHz)"},
{"key":"##features#batch_size", "name":"Batch size"}
]
# Only from points (not from entry meta!)
view_cache=[
"##choices#data_uoa#min",
"##choices#env#*#min",
"##pipeline_state#fail_bool#min",
"##pipeline_state#fail_reason#min",
"##characteristics#compile#compilation_success_bool#min",
"##characteristics#run#run_success_bool#min",
"##characteristics#run#output_check_failed_bool#min",
"##characteristics#run#execution_time#min",
"##characteristics#run#execution_time#max",
"##features#cpu_freq#min",
"##features#gpu_freq#min",
"##features#freq#min",
"##features#batch_size#min",
"##features#gpgpu@0#gpgpu_misc#opencl c version#min"
]
table_view=[
{"key":"##meta#algorithm_species", "name":"Algorithm species", 'module_uoa':'1702c3e426ca54c5', "skip_if_key_in_input":"algorithm_species"},
{"key":"##choices#data_uoa#min", "name":"Workload (program,model,library)", "skip_if_the_same_key_in_input":"yes"},
{"key":"##meta#model_species", "name":"Model species", 'module_uoa':'38e7de41acb41d3b', "skip_if_key_in_input":"model_species"},
{"key":"##meta#model_precision", "name":"Precision", "skip_if_key_in_input":"model_precision"},
{"key":"##meta#dataset_species", "name":"Dataset species", "skip_if_key_in_input":"dataset_species"},
{"key":"##meta#dataset_size", "name":"Dataset size", "skip_if_key_in_input":"dataset_size", "type":"int"},
{"key":"##meta#farm", "name":"Farm", "skip_if_key_in_input":"farm"},
{"key":"##meta#platform_species", "name":"Platform species", "skip_if_key_in_input":"platform_species"},
{"key":"##meta#plat_name", "name":"Platform name", "skip_if_key_in_input":"plat_name"},
{'key':'##meta#cpu_name', 'name':'CPU name', "skip_if_key_in_input":"cpu_name"},
{"key":"##features#cpu_freq#min", "name":"CPU freq (MHz)", "skip_if_the_same_key_in_input":"yes"},
{'key':'##meta#gpgpu_name', 'name':'GPGPU name', "skip_if_key_in_input":"gpgpu_name"},
{"key":"##features#gpu_freq#min", "name":"GPU freq (MHz)", "skip_if_the_same_key_in_input":"yes"},
{'key':'##meta#os_name', 'name':'OS name', "skip_if_key_in_input":"os_name"},
{"key":"##meta#versions", "name":"SW deps and versions", "json_and_pre":"yes", "align":"left"},
{"key":"##meta#model_design_name", "name":"Model design", "skip_if_the_same_key_in_input":"yes"},
{"key":"##meta#compiler_name", "name":"Compiler", "skip_if_the_same_key_in_input":"yes"},
{"key":"##meta#library_name", "name":"Library", "skip_if_the_same_key_in_input":"yes"},
{"key":"##choices#env#", "name":"Environment", "starts_with":"yes", "align":"left"},
{"key":"##characteristics#run#prediction_time_avg_s#min", "name":"Classification time per 1 image (sec. min/max)", "check_extra_key":"max", "format":"%.4f"},
{"key":"##characteristics#run#inference_latency#min", "name":"Inference latency for 1 image (min, sec.)", "check_extra_key":"max", "format":"%.4f"},
{"key":"##characteristics#run#inference_throughput#max", "name":"Inference throughput (max, images per sec.)", "check_extra_key":"min", "format":"%.1f"},
{"key":"##extra#accuracy_sum", "name":"Accuracy (Top1&nbsp;/&nbsp;Top5)"},
{"key":"##features#batch_size#min", "name":"Batch size"},
{"key":"##features#model_size#min", "name":"Model size (B)"},
{"key":"##features#memory_usage#min", "name":"Memory usage (B)"},
{"key":"##meta#platform_peak_power", "name":"Platform peak power (W)", "check_extra_key":"max", "format":"%.3f"},
{"key":"##meta#platform_price_str", "name":"Platform price ($)"},
{"key":"##meta#platform_price_date", "name":"Platform price date"},
{"key":"##characteristics#run#usage_cost#min", "name":"Usage cost per image ($)", "type":"float", "format":"%.2e"},
{"key":"##extra#html_reproducibility", "name":"Reproducibility", "align":"left"}
]
artifacts={}
artifacts_cache={}
prune_first_level=100
prune_second_level=400
##############################################################################
# Initialize module
def init(i):
"""
Input: {}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
return {'return':0}
##############################################################################
# show results
def show(i):
"""
Input: {
(crowd_module_uoa) - if rendered from experiment crowdsourcing
(crowd_key) - add extra name to Web keys to avoid overlapping with original crowdsourcing HTML
(crowd_on_change) - reuse onchange doc from original crowdsourcing HTML
(highlight_behavior_uid) - highlight specific result (behavior)!
(highlight_by_user) - highlight all results from a given user
(refresh_cache) - if 'yes', refresh view cache
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import os
import copy
import time
import json
# Preparing various parameters to render HTML dashboard
st=''
view_all=i.get(k_view_all,'')
cmuoa=i.get('crowd_module_uoa','')
ckey=i.get('crowd_key','')
wchoices3={
ckey+'plot_dimension1':[],
ckey+'plot_variation_dimension1':[{'name':'no', 'value':'no'}, {'name':'yes', 'value':'yes'}],
ckey+'plot_dimension2':[],
ckey+'plot_variation_dimension2':[{'name':'no', 'value':'no'}, {'name':'yes', 'value':'yes'}]
}
if i.get(ckey+'plot_dimension1','')=='': i[ckey+'plot_dimension1']=dimensions[3]['key']
if i.get(ckey+'plot_dimension2','')=='': i[ckey+'plot_dimension2']=dimensions[4]['key']
if 'reset_'+form_name in i: reset=True
else: reset=False
if 'all_choices_'+form_name in i: all_choices=True
else: all_choices=False
debug=(i.get('debug','')=='yes')
# debug=True
conc=i.get('crowd_on_change','')
if conc=='':
conc=onchange
hi_uid=i.get(k_hi_uid,'')
hi_user=i.get(k_hi_user,'')
refresh_cache=i.get('refresh_cache','')
if 'refresh_cache_'+form_name in i:
refresh_cache='yes'
bd='<div style="background-color:#bfffbf;margin:5px;">'
# h='<hr>\n'
h='<center>\n'
h+='\n\n<script language="JavaScript">function copyToClipboard (text) {window.prompt ("Copy to clipboard: Ctrl+C, Enter", text);}</script>\n\n'
h+=hextra
# h+='<hr>\n'
# h+='<br>\n'
# Check host URL prefix and default module/action *********************************************
rx=ck.access({'action':'form_url_prefix',
'module_uoa':'wfe',
'host':i.get('host',''),
'port':i.get('port',''),
'template':i.get('template','')})
if rx['return']>0: return rx
url0=rx['url']
template=rx['template']
url=url0
action=i.get('action','')
muoa=i.get('module_uoa','')
url+='action=index&module_uoa=wfe&native_action='+action+'&'+'native_module_uoa='+muoa
url1=url
# Check and add hidden keys ***************************************************
h+='\n\n'
for k in hidden_keys:
if i.get(k,'')!='':
h+='<input type="hidden" name="'+k+'" value="'+i[k]+'">\n'
h+='\n\n'
# Check repos
experiment_tags='request-asplos18'
experiment_repos=['ck-request', repo_with_validated_results]
# only selected repo (to simplify analysis)
# experiment_repos=[]
x=i.get(ckey+'results','')
if x=='all':
experiment_repos=[]
elif x=='local':
experiment_repos.append('local')
elif x!='':
experiment_repos.append(x)
# Prepare first level of selection with pruning ***********************************************
r=ck.access({'action':'prepare_selector',
'module_uoa':cfg['module_deps']['experiment'],
'original_input':i,
'tags':experiment_tags,
'search_repos':experiment_repos,
'debug': debug,
'selector':selector,
'crowd_key':ckey,
'crowd_on_change':conc,
'url1':url1,
'form_name':form_name,
'background_div':bd,
'skip_html_selector':'yes'})
if r['return']>0: return r
olst=r['lst'] # original list (if all_choices)
plst=r['pruned_lst']
# Compose extra meta (such as deps, versions, etc)
plst1=[]
for q in plst:
duid=q['data_uid']
meta=q['meta']['meta']
if meta.get('processed','')!='yes':
continue
ds=meta.get('deps_summary',{})
xds={}
ver=json.dumps(ds, indent=2, sort_keys=True)
ver1=''
for k in sorted(ds):
x=ds[k]
r=make_deps_full_name({'deps':x})
if r['return']>0: return r
y=r['full_name']
x['full_name']=y
ver1+='<b>'+k+'</b>: '+str(y)+'<br>\n'
# dver=meta.get('engine_meta',{}).get(cpu_abi,{})
# ver+='main: '+str(dver.get('program_version',''))+'\n'
# dps=dver.get('deps_versions',{})
# for dx in dps:
# ver+=dx+': '+str(dps[dx].get('version',''))+'\n'
ver=ver.replace("\'","'").replace("'","\\'").replace('\"','"').replace('"',"\\'").replace('\n','\\n')
if ver!='':
ver='<center><input type="button" class="ck_small_button" onClick="alert(\''+ver+'\');" value="View json"></center>\n'
ver+='<br>'+ver1
meta['versions']=ver
x=meta.get('platform_price','')
if x!='':
x=str(x)
y=meta.get('platform_price_date','')
if y!='':
x+=' <i>('+y+')</i>'
meta['platform_price_str']=x
# Artifact info
auoa=meta.get('artifact','')
if auoa!='' and artifacts.get(duid,'')=='':
x=artifacts_cache.get(auoa,'')
if x=='':
amuoa=cfg['module_deps']['artifact']
r=ck.access({'action':'load',
'module_uoa':amuoa,
'data_uoa':auoa})
if r['return']==0:
dx=r['dict']
ap=r['path']
# Check repos
repos=dx.get('repos',[])
if len(repos)>0:
x='<b>Repositories</b><br>\n'
x+='<div style="margin:7px;">\n'
for a in repos:
n=a['name']
u=a['url']
x+='<a href="'+u+'">'+n+'</a><br>\n'
x+='</div>\n'
# Check interactive report
ir=dx.get('interactive_report','')
if ir!='':
u=url0+'wcid='+cfg['module_deps']['report']+':'+ir
x+='<p><b>Report</b><br>\n'
x+='<div style="margin:7px;">\n'
x+='<a href="'+u+'" target="_blank">Interactive</a>\n'
x+='</div>\n'
# Check DOIs
dois=dx.get('dois',[])
if len(dois)>0:
x+='<p><b>DOIs</b><br>\n'
x+='<div style="margin:7px;">\n'
for a in dois:
n=a['name']
u=a['url']
x+='<a href="'+u+'">'+n+'</a><br>\n'
x+='</div>\n'
# Check ACM badges
y=''
acm=dx.get('acm_badges',{})
if acm.get('available','')=='yes':
y='available'
if acm.get('reusable','')=='yes':
if y!='':y+=', '
y+='reusable'
elif acm.get('functional','')=='yes':
if y!='':y+=', '
y='functional'
if acm.get('replicated','')=='yes':
if y!='':y+=', '
y+='replicated'
elif acm.get('replicated','')=='yes':
if y!='':y+=', '
y+='reproduced'
if y!='':
x+='<p><b>ACM badges</b><br>\n'
x+='<div style="margin:7px;">\n'
x+=y+'<br>\n'
x+='</div>\n'
# Check review
apf='review.html'
apr=os.path.join(ap,apf)
if os.path.isfile(apr):
u=url0+'action=pull&common_action=yes&cid='+amuoa+':'+auoa+'&filename='+apf
x+='<p><b>Review</b><br>\n'
x+='<div style="margin:7px;">\n'
x+=' See <a href="'+u+'" target="_blank">notes</a>\n'
x+='</div>\n'
x+='</div>\n'
# Check extra HTML
y=dx.get('html','')
if y!='':
x+='<p>'+y+'<br>\n'
x+='<p>\n'
artifacts[duid]=x
plst1.append(q)
plst=plst1
# Sort list ***********************************************************************************
dt=time.time()
splst=plst
# sorted(plst, key=lambda x: (
# x.get('meta',{}).get('meta',{}).get('prog_uoa',''),
# x.get('meta',{}).get('meta',{}).get('dataset_uoa',''),
# x.get('meta',{}).get('meta',{}).get('plat_name',''),
# x.get('meta',{}).get('meta',{}).get('timestamp','')
# ))
#
# if debug: h+='\n<p>Debug time (sorting table): '+str(time.time()-dt)+' sec.<p>\n'
# Prune list **********************************************************************************
len_splst=len(splst)
if len_splst>prune_first_level:
splst=splst[:prune_first_level]
h+='\n<i>Showing '+str(prune_first_level)+' of '+str(len_splst)+' entries ...</i><br>\n'
# Prepare and cache results for the table
for dim in dimensions:
k=dim['key']
if dim.get('skip_from_cache','')!='yes':
if dim.get('from_meta','')=='yes':
if k not in view_cache:
view_cache.append(k)
else:
k1=k+'#min'
if k1 not in view_cache:
view_cache.append(k1)
k2=k+'#max'
view_cache.append(k2)
r=ck.access({'action':'get_and_cache_results',
'module_uoa':cfg['module_deps']['experiment'],
'lst':splst,
'cache_uid':work['self_module_uid'],
'refresh_cache':refresh_cache,
'view_cache':view_cache,
'table_view':table_view})
if r['return']>0: return
table=r['table']
# Prepare second level of selection with pruning ***********************************************
r=ck.access({'action':'prepare_selector',
'module_uoa':cfg['module_deps']['experiment'],
'original_input':i,
'tags':experiment_tags,
'search_repos':experiment_repos,
'lst':table,
'skip_meta_key':'yes',
'debug': debug,
'selector':selector2,
'crowd_key':ckey,
'crowd_on_change':conc,
'url1':url1,
'form_name':form_name,
'skip_form_init':'yes',
'background_div':bd})
if r['return']>0: return r
h2=r['html']
table=r['pruned_lst']
choices2=r['choices']
wchoices2=r['wchoices']
# Extra fields (customized for this module) *****************************************************************************
for row in table:
duoa=row.get('##data_uid','')
dpoint=row.get('##point_uid','')
# Replay
x=''
if duoa!='' and dpoint!='':
x='ck replay experiment:'+duoa+' --point='+str(dpoint)
y=ck.cfg.get('add_extra_to_replay','')
if y!='':x+=' '+y
xh='<p><input type="button" class="ck_small_button" onClick="copyToClipboard(\''+x+'\');" value="CK replay">\n'
# Move to validated
x=''
if duoa!='' and dpoint!='':
x='ck validate request.asplos18 --experiment='+duoa+' --point='+str(dpoint)
xh+='<p><input type="button" class="ck_small_button" onClick="copyToClipboard(\''+x+'\');" value="Validate">\n'
xh=artifacts.get(duoa,'')+xh
row['##extra#html_reproducibility']=xh
# Accuracy sum (Top1/Top5)
x=''
x1=row.get('##characteristics#run#accuracy_top1#min','')
x2=row.get('##characteristics#run#accuracy_top5#min','')
if x1!='':
x='%.3f' % x1
if x2!='':
x+='&nbsp;/&nbsp;'
x+='%.3f' % x2
row['##extra#accuracy_sum']=x
# Images per second
t=row.get('##characteristics#run#prediction_time_avg_s#min','')
if t!=None and t!='':
ips=1/t
row['images_per_second']=ips
# Prune first list based on second selection*****************************************************************************
if all_choices:
nsplst=olst
elif reset:
nsplst=splst
else:
all_uid=[]
for row in table:
duid=row.get('##data_uid','')
if duid!='' and duid not in all_uid:
all_uid.append(duid)
nsplst=[]
for q in splst:
if q['data_uid'] in all_uid:
nsplst.append(q)
# Check if too many *****************************************************************************************************
ltable=len(table)
min_view=False
hx=''
if ltable==0:
hx='\n<b>No results found!</b>'
# return {'return':0, 'html':h, 'style':st}
elif ltable>prune_second_level and view_all!='yes':
table=table[:prune_second_level]
hx='\n<i>Showing '+str(prune_second_level)+' of '+str(ltable)+' entries ...</i><br>\n'
# Get unique values and create html selector 1 (after selector 2)
r=ck.access({'action':'get_unique_keys_from_list',
'module_uoa':cfg['module_deps']['experiment'],
'lst':nsplst,
'selector':selector,
'crowd_key':ckey,
'original_input':i})
if r['return']>0: return
choices1=r['choices']
wchoices1=r['wchoices']
# Check results from papers
x=[
{"name":"only validated","value":""},
# {"name":"all","value":"all"},
{"name":"local","value":"local"},
{"name":"for ReQuEST@ASPLOS'18 workflow \"mobilenets-armcl-opencl\"","value":"ck-request-asplos18-results-mobilenets-armcl-opencl"},
{"name":"for ReQuEST@ASPLOS'18 worklfow \"caffe-intel\"","value":"ck-request-asplos18-results-caffe-intel"},
{"name":"for ReQuEST@ASPLOS'18 workflow \"mobilenets-tvm-arm\"","value":"ck-request-asplos18-results-mobilenets-tvm-arm"},
{"name":"for ReQuEST@ASPLOS'18 workflow \"iot-farm\"","value":"ck-request-asplos18-results-iot-farm"},
{"name":"for ReQuEST@ASPLOS'18 workflow \"resnet-tvm-fpga\"","value":"ck-request-asplos18-results-resnet-tvm-fpga"}
]
wchoices1[ckey+'results']=x
# Prepare selector 1 (based on choices from selector 2)
r=ck.access({'action':'prepare_html_selector',
'module_uoa':cfg['module_deps']['experiment'],
'start_form':'yes',
'url1':url1,
'form_name':form_name,
'background_div':bd,
'selector':selector,
'crowd_key':ckey,
'crowd_on_change':conc,
'wchoices':wchoices1,
'original_input':i})
if r['return']>0: return r
h1=r['html']
h+=h1+'\n'+h2
ltable=len(table)
min_view=False
if ltable==0:
h+=''
# h+='<b>No results found!</b>'
# return {'return':0, 'html':h, 'style':st}
elif ltable>prune_second_level and view_all!='yes':
table=table[:prune_second_level]
h+='\n<i>Showing '+str(prune_second_level)+' of '+str(ltable)+' entries ...</i><br>\n'
# Prepare selector 3 (without pruning - about tables and graphs)
arc='yes'
if ck.cfg.get('request_skip_refresh_cache_button','')=='yes': arc=''
# Prepare dimensions
for dim in dimensions:
k=dim['key']
n=dim['name']
wchoices3[ckey+'plot_dimension1'].append({'name':n, 'value':k})
wchoices3[ckey+'plot_dimension2'].append({'name':n, 'value':k})
r=ck.access({'action':'prepare_html_selector',
'module_uoa':cfg['module_deps']['experiment'],
'start_form':'no',
'url1':url1,
'form_name':form_name,
'background_div':bd,
'selector':selector3,
'crowd_key':ckey,
'crowd_on_change':conc,
'wchoices':wchoices3,
'original_input':i,
'add_refresh_cache':arc,
'add_reset':'yes'})
if r['return']>0: return r
h+='\n'+r['html']+'\n'
h+='\n'+hx+'\n'
if ltable==0:
return {'return':0, 'html':h, 'style':st}
# Prepare graph *********************************************************************************************************
bgraph={'0':[], '1':[]}
igraph={'0':[], '1':[]}
# stable=sorted(table, key=lambda row: (
# ck.safe_float(row.get('##characteristics#run#execution_time#min',None),0.0)
# ))
ix=0
kdim1=i.get(ckey+'plot_dimension1','')
kvdim1=i.get(ckey+'plot_variation_dimension1','')
kdim2=i.get(ckey+'plot_dimension2','')
kvdim2=i.get(ckey+'plot_variation_dimension2','')
# Find X/Y names
ndim1=''
ndim2=''
dim1_from_meta=False
dim2_from_meta=False
dim1_module=''
dim2_module=''
dim1_reverse=''
dim2_reverse=''
for k in dimensions:
kk=k['key']
kn=k['name']
if kk==kdim1:
ndim1=kn
if k.get('from_meta','')=='yes':
dim1_from_meta=True
dim1_module=k.get('module_uoa','')
dim1_reverse=k.get('reverse','')
if kk==kdim2:
ndim2=kn
if k.get('from_meta','')=='yes':
dim2_from_meta=True
dim2_module=k.get('module_uoa','')
dim2_reverse=k.get('reverse','')
kdim1min=kdim1
kdim2min=kdim2
kdim1max=kdim1
kdim2max=kdim2
max1=0
# Labels if not int and not float
ldim1={}
ldim2={}
if not dim1_from_meta:
if dim1_reverse!='yes':
kdim1min+='#min'
kdim1max+='#max'
else:
kdim1min+='#max'
kdim1max+='#min'
if not dim2_from_meta:
if dim2_reverse!='yes':
kdim2min+='#min'
kdim2max+='#max'
else:
kdim2min+='#max'
kdim2max+='#min'
for row in table:
ix+=1
six=str(ix)
if kdim1=='experiment':
dim1=ix
else:
v=row.get(kdim1min,None)
if v==None or v=='': continue
v=check_label(v, ldim1)
dim1=v
if dim1>max1: max1=dim1
point=[dim1]
if kdim1!='experiment' and kvdim1=='yes':
v=row.get(kdim1max,None)
if v==None or v=='': continue
v=check_label(v, ldim1)
dim1max=v
delta=0.0
if dim1!=0.0 and dim1max!=0.0:
delta=abs(dim1max-dim1)
if dim1_reverse!='yes':
dm=dim1+delta
else:
dm=dim1-delta
point.append(dm)
if kdim2=='experiment':
dim2=ix
else:
v=row.get(kdim2min,None)
if v==None or v=='': continue
v=check_label(v, ldim2)
dim2=v
point.append(dim2)
if kdim2!='experiment' and kvdim2=='yes':
v=row.get(kdim2max,None)
if v==None or v=='': continue
v=check_label(v, ldim2)
dim2max=v
delta=0.0
if dim2!=0.0 and dim2max!=0.0:
delta=abs(dim2max-dim2)
if dim2_reverse!='yes':
dm=dim2+delta
else:
dm=dim2-delta
point.append(dm)
raw_data_url=url0#+'wcid='+x+':'+duid
ind='0'
ruoa=row.get('##repo_uoa','')
if ruoa==repo_with_validated_results: ind='1'
bgraph[ind].append(point)
igraph[ind].append({'size':4, 'features':row, 'anchor':'id'+six})
#Old
# igraph['0'].append({'size':sizem, 'color':xcol, 'features':row, 'url':'', 'url_ext':raw_data_url})
# igraph['0'].append({'size':4, 'features':row, 'anchor':'id'+six}) #, 'url':'', 'url_ext':''})
if len(bgraph['0'])>0 or len(bgraph['1'])>0:
dt=time.time()
ii={'action':'plot',
'module_uoa':cfg['module_deps']['graph'],
"table":bgraph,
"table_info":igraph,
"xmin":-(float(max1)/20),
"ymin":0,
"ignore_point_if_none":"yes",
"plot_type":"d3_2d_scatter",
"display_x_error_bar2":kvdim1,
"display_y_error_bar2":kvdim2,
"title":"Powered by Collective Knowledge",
"x_ticks_period":10,
"axis_x_desc": ndim1,
"axis_y_desc": ndim2,
"plot_grid":"yes",
"d3_div":"ck_interactive",
"point_style":{"1":{"color":"#0198E1", "connect_lines":"no"},
"0":{"color":"#dc3912", "connect_lines":"no"}},
"image_width":"900",
"image_height":"400",
"wfe_url":url0}
r=ck.access(ii)
if r['return']==0:
x=r.get('html','')
if x!='':
st+=r.get('style','')
h+='<center>\n'
h+='<div id="ck_box_with_shadow" style="width:940px;">\n'
h+=' <div id="ck_interactive" style="text-align:center;font-size:11px;">\n'
h+=x+'\n'
h+=' </div>\n'
h+='</div>\n'
h+='</center>\n'
h+='<br>\n'
if len(ldim1)>0:
r=html_labels({'labels':ldim1, 'axis':'X', 'module_uoa':dim1_module})
if r['return']>0: return r
h+=r['html']
if len(ldim2)>0:
r=html_labels({'labels':ldim2, 'axis':'Y', 'module_uoa':dim2_module})
if r['return']>0: return r
h+=r['html']
h+='<br>\n'
# In the future, we may want to use Django + numpy here
# Prepare table header ******************************************************************
bgc='dfffdf'
bg=' style="background-color:#'+bgc+';"'
bg1=' style="background-color:#bfffbf;"'
bg2=' style="background-color:#afffaf;"'
h+='<small><table border="1" cellpadding="7" cellspacing="0">\n'
ha='align="$#align#$" valign="top"'
# Prepare table header *****************************************************************
h+=' <tr style="background-color:#dddddd">\n'
h+=' <td '+ha.replace('$#align#$','center')+'><b>#</b></td>\n'
for tv in table_view:
k=tv['key']
align=tv.get('align','')
if align=='': align='center'
skip=False
kk=tv.get('skip_if_key_in_input','')
if kk!='' and i.get(ckey+kk,'')!='':
skip=True
if not skip and tv.get('skip_if_the_same_key_in_input','')=='yes' and i.get(ckey+k,'')!='':
skip=True
if not skip:
n=tv.get('name','')
if n=='': n=k
h+=' <td '+ha.replace('$#align#$',align)+'><b>'+n+'</b></td>\n'
h+=' </tr>\n'
# Draw table ***************************************************************************
dt=time.time()
ix=0
for q in table:
ix+=1
six=str(ix)
# Check colors
bgx=bg
bgx1=bg1
bgx2=bg2
if (hi_uid!='' and duid==hi_uid) or (hi_user!='' and hi_user==user):
bgx=' style="background-color:#ffcf7f"'
bgx1=' style="background-color:#ffbf5f"'
bgx2=' style="background-color:#ffaf2f"'
# Starting raw ***************************************
h+=' <tr'+bgx+'>\n'
h+=' <td '+ha.replace('$#align#$','center')+'><a name="id'+six+'" id="id'+six+'">'+six+'</a></td>\n'
for tv in table_view:
k=tv['key']
align=tv.get('align','')
if align=='': align='center'
skip=False
kk=tv.get('skip_if_key_in_input','')
if kk!='' and i.get(ckey+kk,'')!='':
skip=True
if not skip and tv.get('skip_if_the_same_key_in_input','')=='yes' and i.get(ckey+k,'')!='':
skip=True
if not skip:
v=q.get(k,'')
format=tv.get('format','')
if format!='' and v!='' and v!=None and v!="None":
v=format % float(v)
if tv.get('json_and_pre','')=='yes' and v!='' and type(v)==dict:
v1=''
for kx in v:
v1+=kx+'='+str(v[kx])+'<br>'
v=v1
if tv.get('starts_with','')=='yes':
v=''
for kx in sorted(q):
if kx!=k and kx.startswith(k):
v+=kx[len(k):-4]+'='+str(q.get(kx,''))+'<br>'
v=str(v)
cek=tv.get('check_extra_key','')
if cek!='':
j=k.rfind('#')
if j>0:
k1=k[:j+1]+cek
v1=q.get(k1,'')
if format!='' and v1!='' and v1!=None:
v1=format % float(v1)
v1=str(v1)
if v1!='':
v+=' .. '+v1
h+=' <td '+ha.replace('$#align#$',align)+'>'+v+'</td>\n'
h+=' <tr>\n'
h+='</table></small>\n'
h+='</center>\n'
if debug: h+='\n<p>Debug time (preparing html of a table): '+str(time.time()-dt)+' sec.<p>\n'
if cmuoa=='':
h+='</form>\n'
# Add <br> to be able to select anchor on top
for j in range(0,30):
h+='<br>\n'
return {'return':0, 'html':h, 'style':st}
##############################################################################
# show info for all layers
def html_viewer(i):
"""
Input: {
data_uoa - CK entry UOA to view
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import os
duoa=i.get('data_uoa','')
# Load entry
r=ck.access({'action':'load',
'module_uoa':work['self_module_uid'],
'data_uoa':duoa})
if r['return']>0: return r
p=r['path']
d=r['dict']
dchars=d.get('characteristics',{})
dchoices=d.get('choices',{})
dmeta=d.get('meta',{})
# Load stats
dstat={}
fstat=os.path.join(p,'ck-stat-flat-characteristics.json')
if os.path.isfile(fstat):
r=ck.load_json_file({'json_file':fstat, 'dict':dstat})
if r['return']>0: return r
dstat=r['dict']
# Prepare table
h=''
# h+='<hr>\n'
h+='<br>\n'
h+='<center>\n'
h+='<h2>DNN engine and model evaluation statistics per layer (crowd-tuning)</h2><br>\n'
h+='</center>\n'
xdeps=dmeta.get('xdeps',{})
lcaffe=xdeps.get('lib-caffe',{})
lmodel=xdeps.get('caffemodel',{})
# Prepare extra info
h+='<p>\n'
h+='<table border="1" cellpadding="8" cellspacing="0">\n'
h+=' <tr>\n'
h+=' <td><b>DNN engine name:</b></td>\n'
h+=' <td>'+lcaffe.get('data_name','')+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>DNN engine version:</b></td>\n'
h+=' <td>'+lcaffe.get('ver','')+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>DNN engine type:</b></td>\n'
h+=' <td>'+dmeta.get('dnn_type','')+'</td>\n'
h+=' </tr>\n'
x=''
dx=dmeta.get('xversions',{})
for k in sorted(dx):
v=dx[k]
if v!='':
if x!='': x+='<br>\n'
x+=k+'='+str(v)+'\n'
h+=' <tr>\n'
h+=' <td><b>DNN engine dependencies:</b></td>\n'
h+=' <td>'+x+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>DNN model name:</b></td>\n'
h+=' <td>'+lmodel.get('data_name','')+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>DNN model version:</b></td>\n'
h+=' <td>'+lmodel.get('ver','')+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>Batch size:</b></td>\n'
h+=' <td>'+dchars.get('run',{}).get('REAL_ENV_CK_CAFFE_BATCH_SIZE','')+'</td>\n'
h+=' </tr>\n'
# TBD: Need to show min,exp,max!
# h+=' <tr>\n'
# h+=' <td><b>FWBW time (ms.):</b></td>\n'
# h+=' <td>'+str(dchars.get('run',{}).get('time_bw_ms',''))+'</td>\n'
# h+=' </tr>\n'
# h+=' <tr>\n'
# h+=' <td><b>FW time (ms.):</b></td>\n'
# h+=' <td>'+str(dchars.get('run',{}).get('time_fw_ms',''))+'</td>\n'
# h+=' </tr>\n'
# h+=' <tr>\n'
# h+=' <td><b>BW time (ms.):</b></td>\n'
# h+=' <td>'+str(dchars.get('run',{}).get('time_bw_ms',''))+'</td>\n'
# h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>Platform:</b></td>\n'
h+=' <td>'+dmeta.get('plat_name','')+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>OS:</b></td>\n'
h+=' <td>'+dmeta.get('os_name','')+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>CPU:</b></td>\n'
h+=' <td>'+dmeta.get('cpu_name','')+'</td>\n'
h+=' </tr>\n'
h+=' <tr>\n'
h+=' <td><b>GPU:</b></td>\n'
h+=' <td>'+dmeta.get('gpu_name','')+'</td>\n'
h+=' </tr>\n'
h+=' </tr>\n'
h+='</table>\n'
h+='<center>\n'
h+='<p>\n'
h+='<table border="0" cellpadding="10" cellspacing="0">\n'
h+=' <tr>\n'
h+=' <td><b>Name</b></td>\n'
h+=' <td><b>Direction</b></td>\n'
h+=' <td align="right"><b>Min time (ms.):</b></td>\n'
h+=' <td align="right"><b>Expected time (ms.):</b></td>\n'
h+=' <td align="right"><b>Max time (ms.):</b></td>\n'
h+=' <td align="right"><b>Repetitions:</b></td>\n'
h+=' </tr>\n'
# Detecting number of layers
jj={}
for j in range(0,1000):
k3='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#min'
v3=dstat.get(k3,'')
if v3=='': break
jj[j]=v3
# Sorting by min time
if i.get('all_params',{}).get('skip_sort','')!='yes':
jj=sorted(jj, key=lambda x: jj[x], reverse=True)
# Also layers
for j in jj:
k1='##characteristics#run#per_layer_info@'+str(j)+'#direction#min'
k2='##characteristics#run#per_layer_info@'+str(j)+'#label#min'
k3='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#min'
k4='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#max'
k5='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#exp_allx'
k7='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#repeats'
v1=dstat.get(k1,'')
v2=dstat.get(k2,'')
v3=dstat.get(k3,'')
v4=dstat.get(k4,'')
v5=dstat.get(k5,[])
v7=dstat.get(k7,'')
if v1!='' and v2!='' and v3!='' and v4!='':
v6=0
if len(v5)>0:
v6=v5[0]
xv3=''
xv4=''
xv6=''
if v3!='':
if v3<0.1: xv3='0'
else: xv3='<b>'+('%.1f'%v3)+'</b>'
if v4!='':
if v4<0.1: xv4='0'
else: xv4='<b>'+('%.1f'%v4)+'</b>'
if v6!='':
if v6<0.1: xv6='0'
else: xv6='<b>'+('%.1f'%v6)+'</b>'
h+=' <tr>\n'
h+=' <td>'+v2+'</td>\n'
h+=' <td>'+v1+'</td>\n'
h+=' <td align="right">'+xv3+'</td>\n'
h+=' <td align="right">'+xv6+'</td>\n'
h+=' <td align="right">'+xv4+'</td>\n'
h+=' <td align="right">'+str(v7)+'</td>\n'
h+=' </tr>\n'
h+='</table>\n'
h+='</center>\n'
return {'return':0, 'html':h, 'show_top':'yes'}
##############################################################################
# replay experiment (TBD)
def replay(i):
"""
Input: {
(data_uoa)
(remote)
(host_os)
(target_os)
(device_id)
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import copy
import os
# Setting output
o=i.get('out','')
oo=''
if o=='con': oo='con'
duoa=i.get('data_uoa','')
remote=i.get('remote','')
er=''
esr=''
if remote=='yes':
er=i.get('exchange_repo','')
if er=='': er=ck.cfg['default_exchange_repo_uoa']
esr=i.get('exchange_subrepo','')
if esr=='': esr=ck.cfg['default_exchange_subrepo_uoa']
# Try to load info
if o=='con':
ck.out('Loading experiment entry ...')
ck.out('')
r=ck.access({'action':'load',
'module_uoa':work['self_module_uid'],
'data_uoa':duoa,
'repo_uoa':er,
'remote_repo_uoa':esr})
if r['return']>0: return r
d=r['dict']
hos=i.get('host_os','')
tos=i.get('target_os','')
tdid=i.get('device_id','')
# Check two main deps (engine and model)
meta=d.get('meta',{})
xdeps=meta.get('xdeps',{})
# TBD: rebuild env by tags!
#
# dnn=xdeps.get('lib-caffe',{})
# model=xdeps.get('caffemodel',{})
#
# pdnn=dnn.get('package_uoa','')
# pmodel=model.get('package_uoa','')
#
# preset_env={}
# penv=[pdnn,pmodel]
#
# for j in range(0, len(penv)):
# px=''
# py=penv[j]
#
# if py!='':
# # Search by package
# r=ck.access({'action':'search',
# 'module_uoa':cfg['module_deps']['env'],
# 'search_dict':{'package_uoa':py}})
# if r['return']>0: return r
#
# l=r['lst']
#
# if j==0: preset_env['lib-caffe']=px
# elif j==1: preset_env['caffemodel']=px
# Run pipeline
choices=d.get('choices',{})
# Clean various vars
for k in replay_clean_vars:
if k in choices:
del(choices[k])
if i.get('target_os','')!='' and not i['target_os'].startswith('android'):
del(i['target_os'])
env=choices.get('env',{})
for k in replay_clean_env_vars:
if k in env:
del(env[k])
choices['env']=env
if hos!='': choices['host_os']=hos
if tos!='': choices['target_os']=tos
if tdid!='': choices['device_id']=tdid
pipeline_data_uoa=choices['module_uoa']
# Prepare pipeline
ii={'action':'pipeline',
'module_uoa':cfg['module_deps']['program'],
'prepare':'yes',
'choices':choices,
'out':o}
rr=ck.access(ii)
if rr['return']>0: return rr
fail=rr.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+rr.get('fail_reason','')+')'}
ready=rr.get('ready','')
if ready!='yes':
return {'return':11, 'error':'couldn\'t prepare universal CK program workflow'}
# Run pipeline
ii={'action':'run',
'module_uoa':cfg['module_deps']['pipeline'],
'data_uoa':pipeline_data_uoa,
'pipeline':rr,
'out':o}
rr=ck.access(ii)
if rr['return']>0: return rr
fail=rr.get('fail','')
if fail=='yes':
return {'return':10, 'error':'pipeline failed ('+rr.get('fail_reason','')+')'}
if o=='con':
ck.out('')
ck.out('Your results:')
ck.out('')
dstat=rr.get('last_stat_analysis',{}).get('dict_flat',{})
x0=dstat.get("##characteristics#run#time_fwbw_ms#min",None)
x0e=dstat.get("##characteristics#run#time_fwbw_ms#exp",None)
if x0!=None:
ck.out('* FWBW min: '+('%.0f'%x0)+' ms.')
if x0e!=None:
ck.out('* FWBW exp: '+('%.0f'%x0e)+' ms.')
x1=dstat.get("##characteristics#run#time_fw_ms#min",None)
x1e=dstat.get("##characteristics#run#time_fw_ms#exp",None)
if x1!=None:
ck.out('* FW min: '+('%.0f'%x1)+' ms.')
if x1e!=None:
ck.out('* FW exp: '+('%.0f'%x1e)+' ms.')
x2=dstat.get("##characteristics#run#time_bw_ms#min",None)
x2e=dstat.get("##characteristics#run#time_bw_ms#exp",None)
if x2!=None:
ck.out('* BW min: '+('%.0f'%x2)+' ms.')
if x2e!=None:
ck.out('* BW exp: '+('%.0f'%x2e)+' ms.')
return {'return':0}
##############################################################################
# see ReQuEST results (CK Dashboard)
def dashboard(i):
"""
Input: {
(host) - Internal web server host
(port) - Internal web server port
(wfe_host) - External web server host
(wfe_port) - External web server port
(extra_url) - extra URL
(results) - repo with experiments to visualize (will be added as &results=... to extra_url)
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
# Old style
# i['action']='browser'
# i['cid']=''
# i['module_uoa']=''
# i['template']='nntest'
i['action']='start'
i['module_uoa']='web'
i['browser']='yes'
i['template']='request.asplos18'
i['cid']=''
if i.get('results','')!='':
x=i.get('extra_url','')
x+='&results='+i['results']
i['extra_url']=x
return ck.access(i)
##############################################################################
# prepare common meta for ReQuEST @ ASPLOS"18
def prepare_common_meta(i):
"""
Input: {
platform_dict - output from platform detection
request_dict - info about ReQuEST submission
deps - resolved deps to summarize deps (package, tags, version)
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
tags - tags for experiment entry
meta - meta for experiment entry
record_dict - dict to update records
}
"""
from time import strftime, gmtime
import copy
# Get current timestamp
r=ck.get_current_date_time({})
if r['return']>0: return r
timestamp=r['iso_datetime']
#striped timestamp
j=timestamp.find('.')
if j>0: timestamp=timestamp[:j]
stimestamp=timestamp.replace('-','').replace(':','').replace('T','')
pi=i['platform_dict']
rd=i['request_dict']
deps=i.get('deps',{})
# Prepare tags
tags=[
'request',
'request-asplos18',
timestamp,
stimestamp
]
tags.append(rd['algorithm_species'])
# Prepare meta
meta=copy.deepcopy(rd)
meta['timestamp']=timestamp
meta['stimestamp']=stimestamp
meta['request_version']=1
# Check platform
hos=pi['host_os_uoa']
hos_uid=pi['host_os_uid']
hosd=pi['host_os_dict']
tos=pi['os_uoa']
tos_uid=pi['os_uid']
tosd=pi['os_dict']
tbits=tosd.get('bits','')
remote=tosd.get('remote','')
tdid=pi['device_id']
features=pi.get('features',{})
fplat=features.get('platform',{})
fos=features.get('os',{})
fcpu=features.get('cpu',{})
plat_name=fplat.get('name','')
plat_uid=features.get('platform_uid','')
os_name=fos.get('name','')
os_uid=features.get('os_uid','')
cpu_name=fcpu.get('name','')
cpu_abi=fcpu.get('cpu_abi','')
if cpu_name=='' and cpu_abi!='': cpu_name='unknown-'+cpu_abi
cpu_uid=features.get('cpu_uid','')
sn=fos.get('serial_number','')
# Check if CPU clusters (TBD: need to know which one is actually used!)
if cpu_name=='':
ufcpu=features.get('cpu_unique',[])
if len(ufcpu)>1:
for q in ufcpu:
if cpu_name!='': cpu_name+=' ; '
x=q.get('ck_cpu_name','')
if x!='': cpu_name+=x
# GPU
fgpu=features.get('gpu',{})
gpu_name=fgpu.get('name','')
gpgpu0=features.get('gpgpu',{})
if type(gpgpu0)==list:
if len(gpgpu0)>0:
gpgpu0=gpgpu0[0]
fgpgpu=gpgpu0.get('gpgpu',{})
gpgpu_name=fgpgpu.get('name','')
gpgpu_vendor=fgpgpu.get('vendor','')
gpgpu_type=fgpgpu.get('type','')
gpgpu_name2=gpgpu_name
if gpgpu_vendor!='': gpgpu_name2=gpgpu_vendor+' '+gpgpu_name
fgpgpu_misc=gpgpu0.get('gpgpu_misc',{})
opencl=fgpgpu_misc.get('opencl c version','')
# Assembling meta for platform
meta.update({
'scenario_module_uoa': work['self_module_uid'],
'host_os_uid':hos_uid,
'target_os_uid':tos_uid,
'target_device_id':tdid,
'cpu_name':cpu_name,
'cpu_abi':cpu_abi,
'cpu_uid':cpu_uid,
'os_name':os_name,
'os_uid':os_uid,
'plat_name':plat_name,
'plat_uid':plat_uid,
'gpu_name':gpu_name,
'gpgpu_name':gpgpu_name2,
'gpgpu_vendor':gpgpu_vendor,
'opencl':opencl
})
# Process dependencies
r=ck.access({'action':'deps_summary',
'module_uoa':cfg['module_deps']['env'],
'deps':deps})
if r['return']>0: return r
deps_summary=r['deps_summary']
meta['deps_summary']=deps_summary
# Misc info
rd={
'subview_uoa':'f84ca49f79a1446a' # ReQuEST default table view from ck-autotuning repo
# ck-autotuning:experiment.view:request-default
}
return {'return':0, 'tags':tags, 'meta':meta, 'record_dict':rd}
##############################################################################
# move validated experiment to an official repo
def validate(i):
"""
Input: {
(experiment) - experiment UOA
(point) - point to move
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import os
import shutil
experiment=i.get('experiment','')
if experiment=='':
return {'return':1, 'error':'experiment is not defined'}
# Find experiment
r=ck.access({'action':'load',
'module_uoa':cfg['module_deps']['experiment'],
'data_uoa':experiment})
if r['return']>0: return r
duid=r['data_uid']
duoa=r['data_uoa']
d=r['dict']
p=r['path']
# Check if entry is already there
r=ck.access({'action':'update',
'module_uoa':cfg['module_deps']['experiment'],
'repo_uoa':repo_with_validated_results,
'data_uoa':'validated-'+duoa,
'dict':d,
'common_func':'yes',
'substitute':'yes',
'sort_keys':'yes'})
if r['return']>0: return r
nduid=r['data_uid']
np=r['path']
# Check point
point=i.get('point','')
if point!='':
p1=os.listdir(p)
for d in p1:
if d.startswith('ckp-'+point+'.'):
pf1=os.path.join(p,d)
pf2=os.path.join(np,d)
shutil.copyfile(pf1,pf2)
ck.out('Copied to '+np)
return {'return':0}
##############################################################################
# make full name for dependency
def make_deps_full_name(i):
"""
Input: {
deps - dict with deps
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
full_name - full name of deps
}
"""
x=i['deps']
y=str(x.get('data_name',''))
y1=str(x.get('version',''))
if y1!='': y+=' '+y1
y1=str(x.get('git_revision',''))
if y1!='': y+=' ('+y1+')'
return {'return':0, 'full_name':y}
##############################################################################
# check label in value
def check_label(v, ldim):
if type(v)!=float and type(v)!=int:
v=str(v)
label=0
found=False
for k in ldim:
if k==v:
v=ldim[k]
found=True
break
else:
if ldim[k]>label:
label=ldim[k]
if not found:
ldim[v]=label+1
v=label+1
return v
##############################################################################
# check label in value
def html_labels(i):
ldim=i['labels']
axis=i['axis']
muoa=i.get('module_uoa','')
h=''
h1='<small><b>Labels '+axis+': </b>\n'
for j in range(1,10000):
l=None
for k in ldim:
v=ldim[k]
if v==j:
l=k
break
if l==None:
break
if muoa!='':
r=ck.access({'action':'load', 'module_uoa':muoa, 'data_uoa':l})
if r['return']==0:
l=r['data_name']
if l!='' and l!=None:
h+=str(j)+') '+l+'; '
if h!='':
h=h1+h+'<br>\n'
return {'return':0, 'html':h}
##############################################################################
# start scoreboard
def scoreboard(i):
return dashboard(i)
##############################################################################
# get raw data for repo-widget
def get_raw_data(i):
"""
Input: {
(crowd_module_uoa) - if rendered from experiment crowdsourcing
(crowd_key) - add extra name to Web keys to avoid overlapping with original crowdsourcing HTML
(crowd_on_change) - reuse onchange doc from original crowdsourcing HTML
(highlight_behavior_uid) - highlight specific result (behavior)!
(highlight_by_user) - highlight all results from a given user
(refresh_cache) - if 'yes', refresh view cache
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import os
import copy
import time
import json
# Preparing various parameters to render HTML dashboard
st=''
view_all=i.get(k_view_all,'')
cmuoa=i.get('crowd_module_uoa','')
ckey=i.get('crowd_key','')
if i.get(ckey+'plot_dimension1','')=='': i[ckey+'plot_dimension1']=dimensions[3]['key']
if i.get(ckey+'plot_dimension2','')=='': i[ckey+'plot_dimension2']=dimensions[4]['key']
if 'reset_'+form_name in i: reset=True
else: reset=False
if 'all_choices_'+form_name in i: all_choices=True
else: all_choices=False
debug=(i.get('debug','')=='yes')
conc=i.get('crowd_on_change','')
if conc=='':
conc=onchange
hi_uid=i.get(k_hi_uid,'')
hi_user=i.get(k_hi_user,'')
refresh_cache=i.get('refresh_cache','')
if 'refresh_cache_'+form_name in i:
refresh_cache='yes'
# Check host URL prefix and default module/action *********************************************
rx=ck.access({'action':'form_url_prefix',
'module_uoa':'wfe',
'host':i.get('host',''),
'port':i.get('port',''),
'template':i.get('template','')})
if rx['return']>0: return rx
url0=rx['url']
template=rx['template']
url=url0
action=i.get('action','')
muoa=i.get('module_uoa','')
url+='action=index&module_uoa=wfe&native_action='+action+'&'+'native_module_uoa='+muoa
url1=url
# Check and add hidden keys ***************************************************
# Check repos
experiment_tags='request-asplos18'
experiment_repos=['ck-request', repo_with_validated_results]
# only selected repo (to simplify analysis)
# experiment_repos=[]
x=i.get(ckey+'results','')
if x=='all':
experiment_repos=[]
elif x=='local':
experiment_repos.append('local')
elif x!='':
experiment_repos.append(x)
# Prepare first level of selection with pruning ***********************************************
r=ck.access({'action':'prepare_selector',
'module_uoa':cfg['module_deps']['experiment'],
'original_input':i,
'tags':experiment_tags,
'search_repos':experiment_repos,
'debug': debug,
'selector':selector,
'crowd_key':ckey,
'crowd_on_change':conc,
'url1':url1,
'form_name':form_name,
'skip_html_selector':'yes'})
if r['return']>0: return r
olst=r['lst'] # original list (if all_choices)
plst=r['pruned_lst']
# Compose extra meta (such as deps, versions, etc)
plst1=[]
for q in plst:
duid=q['data_uid']
meta=q['meta']['meta']
if meta.get('processed','')!='yes':
continue
ds=meta.get('deps_summary',{})
meta['versions']=ds
x=meta.get('platform_price','')
if x!='':
meta['platform_price_str']=str(x)
plst1.append(q)
plst=plst1
# Sort list ***********************************************************************************
dt=time.time()
splst=plst
len_splst=len(splst)
if len_splst>prune_first_level:
splst=splst[:prune_first_level]
# Prepare and cache results for the table
for dim in dimensions:
k=dim['key']
if dim.get('skip_from_cache','')!='yes':
if dim.get('from_meta','')=='yes':
if k not in view_cache:
view_cache.append(k)
else:
k1=k+'#min'
if k1 not in view_cache:
view_cache.append(k1)
k2=k+'#max'
view_cache.append(k2)
r=ck.access({'action':'get_and_cache_results',
'module_uoa':cfg['module_deps']['experiment'],
'lst':splst,
'cache_uid':work['self_module_uid'],
'refresh_cache':refresh_cache,
'view_cache':view_cache,
'table_view':table_view})
if r['return']>0: return
table=r['table']
# Prepare second level of selection with pruning ***********************************************
r=ck.access({'action':'prepare_selector',
'module_uoa':cfg['module_deps']['experiment'],
'original_input':i,
'tags':experiment_tags,
'search_repos':experiment_repos,
'lst':table,
'skip_meta_key':'yes',
'debug': debug,
'selector':selector2,
'crowd_key':ckey,
'crowd_on_change':conc,
'url1':url1,
'form_name':form_name,
'skip_form_init':'yes'})
if r['return']>0: return r
h2=r['html']
table=r['pruned_lst']
choices2=r['choices']
wchoices2=r['wchoices']
# Extra fields (customized for this module) *****************************************************************************
for row in table:
duoa=row.get('##data_uid','')
dpoint=row.get('##point_uid','')
xhj=[]
# Replay
x=''
if duoa!='' and dpoint!='':
x='ck replay experiment:'+duoa+' --point='+str(dpoint)
y=ck.cfg.get('add_extra_to_replay','')
if y!='':x+=' '+y
xhj.append({
'title': 'CK replay',
'cmd': x
})
# Move to validated
x=''
if duoa!='' and dpoint!='':
x='ck validate request.asplos18 --experiment='+duoa+' --point='+str(dpoint)
xhj.append({
'title': 'Validate',
'cmd': x
})
row['##extra#html_reproducibility']=xhj
# Accuracy sum (Top1/Top5)
x=''
x1=row.get('##characteristics#run#accuracy_top1#min','')
x2=row.get('##characteristics#run#accuracy_top5#min','')
if x1!='':
x='%.3f' % x1
if x2!='':
x+=' / '
x+='%.3f' % x2
row['##extra#accuracy_sum']=x
# Images per second
t=row.get('##characteristics#run#prediction_time_avg_s#min','')
if t!=None and t!='':
ips=1/t
row['images_per_second']=ips
for view in table_view:
if view.get('module_uoa', None)!=None:
r=ck.access({'action':'load','module_uoa':view['module_uoa'],'data_uoa':row[view['key']]})
if r['return']==0:
mk='##data_name'
rx=ck.get_by_flat_key({'dict':r, 'key':mk})
if rx['return']==0:
row[view['key']] = rx['value']
# Check if too many *****************************************************************************************************
ltable=len(table)
if ltable>prune_second_level and view_all!='yes' and ltable!=0:
table=table[:prune_second_level]
return {'return':0, 'table':table, 'view_cache':view_cache}
##############################################################################
# get raw config for repo widget
def get_raw_config(i):
"""
Input: {
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
return {
'return':0,
'selector':selector,
'selector2':selector2,
'selector3':selector3,
'dimensions':dimensions,
'view_cache':view_cache,
'table_view':table_view
}
You can’t perform that action at this time.