Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
917 lines (737 sloc) 27.3 KB
#
# Collective Knowledge (compiler choices)
#
# See CK LICENSE.txt for licensing details
# See CK COPYRIGHT.txt for copyright details
#
# Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
#
cfg={} # Will be updated by CK (meta description of this module)
work={} # Will be updated by CK (temporal data)
ck=None # Will be updated by CK (initialized CK kernel)
# Local settings
##############################################################################
# Initialize module
def init(i):
"""
Input: {}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
return {'return':0}
##############################################################################
# extract choices to universal pipeline tuning
def extract_to_pipeline(i):
"""
Input: {
(data_uoa) - compiler description
(file_in) - JSON pipeline template
(file_out) - output prepared pipeline to this file
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
pipeline - prepared pipeline
}
"""
fo=i.get('file_out','')
fi=i.get('file_in','')
duoa=i.get('data_uoa','')
# Prepare pipeline template
ck.out('Preparing pipeline template ...')
if fi=='':
fi=os.path.join(work['path'],cfg['pipeline_template'])
r=ck.load_json_file({'json_file':fi})
if r['return']>0: return r
pipeline=r['dict']
# Load or select compiler description
ck.out('')
ck.out('Selecting compiler and description ...')
if duoa=='':
rx=ck.access({'action':'search',
'module_uoa':work['self_module_uid']})
if rx['return']>0: return rx
lst=rx['lst']
if len(lst)==0:
duoa=''
elif len(lst)==1:
duid=lst[0]['data_uid']
duoa=lst[0]['data_uoa']
else:
# SELECTOR *************************************
ck.out('')
ck.out('Multiple choices available:')
ck.out('')
r=ck.select_uoa({'choices':lst})
if r['return']>0: return r
duoa=r['choice']
if duoa=='':
return {'return':1, 'error':'no compiler description selected'}
# Load compiler desc
rx=ck.access({'action':'load',
'module_uoa':work['self_module_uid'],
'data_uoa':duoa})
if rx['return']>0: return rx
d=rx['dict']
dsc=rx.get('desc',{})
dx=dsc.get('all_compiler_flags_desc',{})
# Update pipeline
ck.out('')
ck.out('Updating pipeline choices with compiler flags ...')
if 'pipeline' not in pipeline: pipeline['pipeline']={}
px=pipeline['pipeline']
px['choices_desc']={}
for q in dx:
qq=dx[q]
q1=q
if q1.startswith('##'): q1=q1[1:]
q1='##compiler_flags'+q1
px['choices_desc'][q1]=qq
# Saving file
if fo!='':
ck.out('')
ck.out('Writing pipeline ...')
rx=ck.save_json_to_file({'json_file':fo, 'dict':pipeline})
if rx['return']>0: return rx
return {'return':0, 'pipeline':pipeline}
##############################################################################
# extract optimization flags and parameters from a compiler for autotuning (currently GCC)
# Note: GCC sources are needed - this script searches for GCC opts in all sub-directories
# in a current directory. Therefore, just untar all GCC sources in current directory.
#
# (partially based on scripts from Abdul Wahid Memon and Yuriy Kashnikov)
def extract_opts(i):
"""
Input: {
(record) - if 'yes, record to repo
(repo_uoa) - repo UOA to record compiler description
(data_uoa) - data UOA to record compiler description (if empty, autogenerate)
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import os
import sys
import datetime
o=i.get('out','')
p=os.getcwd()
f1a=cfg['gcc_src_opt_file1']
f1b=cfg['gcc_src_opt_file2']
f2=cfg['gcc_src_params']
results={}
# Checking if GCC
try:
dirList=os.listdir(p)
except Exception as e:
None
else:
for fn in dirList:
pp=os.path.join(p, fn)
if os.path.isdir(pp) and fn.startswith('gcc-'):
found=False
p1a=os.path.join(pp,f1a)
p1b=os.path.join(pp,f1b)
p2=os.path.join(pp,f2)
if os.path.isfile(p1a) or os.path.isfile(p1b) or os.path.isfile(p2):
# Found GCC source directory with needed files
ck.out('*************************************')
ck.out('Processing GCC directory: '+fn)
ck.out('')
ver=fn[4:]
lver=ver.split('.')
dd={}
iopt=0
iparam=0
p1=''
if os.path.isfile(p1a):
p1=p1a
elif os.path.isfile(p1b):
p1=p1b
if p1!='':
# Load opt file
rx=ck.load_text_file({'text_file':p1,
'split_to_list':'yes',
'encoding':sys.stdin.encoding})
if rx['return']>0: return rx
lopts=rx['lst']
found=False
for q in range(0, len(lopts)):
qq=lopts[q].strip()
if not found:
if qq=='@item Optimization Options':
ck.out('Found optimization flags')
found=True
else:
if qq.startswith('@end ') or qq=='':
break
if qq.startswith('@gccoptlist{'):
qq=qq[11:]
elif qq.startswith('@'):
continue
if qq.endswith('@gol'):
qq=qq[:-5]
jj=qq.split(' ')
# Check if base flag
if len(jj)>0 and jj[0].startswith('-O'):
choice=[]
if '-O3' in jj: choice.append('-O3')
for j in jj:
if j!='' and j!='-O3' and j!='-O':
if j.endswith('}'): j=j[:-1].strip()
choice.append(j)
dd["##base_opt"]={
"choice": choice,
"default": "",
"desc": "base compiler flag",
"sort": 10000,
"tags": [
"base",
"basic",
"optimization"
],
"type": "text"
}
else:
for j in jj:
if j!='' and not j.startswith('--param') and not j.startswith('@') and j.startswith('-f') and \
j.find('profile')==-1 and j.find('coverage')==-1:
if '@' in j:
iparam+=1
ij=j.find('@')
opt=j[:ij]
if opt.endswith('@var{n}'): opt=opt[:-7]
ck.out('Adding param '+str(iparam)+' '+opt)
dd['##param-'+opt]={
"can_omit": "yes",
"default": "",
"desc": "compiler flag: "+j,
"sort": iparam*10+30000,
"explore_prefix": j,
"explore_start": 0,
"explore_step": 1,
"explore_stop": 0,
"tags": [
"basic",
"parametric",
"optimization"
],
"type":"integer"
}
else:
iopt+=1
opt=j[2:]
if opt.startswith('no-'):
opt=opt[3:]
ck.out('Adding opt '+str(iopt)+' '+j)
dd['##bool-'+opt]={
"can_omit": "yes",
"choice": [
"-f"+opt,
"-fno-"+opt
],
"default": "",
"desc": "compiler flag: "+j,
"sort": iopt*10+10000,
"tags": [
"basic",
"boolean",
"optimization"
],
"type":"text"
}
# Checking params
if os.path.isfile(p2):
# Load opt file
rx=ck.load_text_file({'text_file':p2,
'split_to_list':'yes',
'encoding':sys.stdin.encoding})
if rx['return']>0: return rx
lparams=rx['lst']
# Parsing params
opt=''
opt1=''
desc=''
desc1=''
add=False
finish=False
for q in range(0, len(lparams)):
qq=lparams[q].strip()
if qq.startswith('DEFPARAM'):
iparam+=1
q+=1
opt=lparams[q].strip()[1:-2]
if opt.find('profile')==-1 and opt.find('coverage')==-1:
q+=1
desc=lparams[q].strip()[1:-2]
line='x'
while True:
q+=1
line=lparams[q].strip()
if line[-1]==')': break
desc+=line[1:-2]
e1=0
e2=0
exp=line[:-1].split(',')
skip=False
for j in range(0, len(exp)):
jj=exp[j].strip()
if jj.find('*')>0 or jj.find('_')>0:
skip=True
elif jj.rfind('\"') != -1:
continue
else:
jj=int(exp[j].strip())
exp[j]=jj
if not skip:
if len(exp)>1 and exp[2]>exp[1]:
e1=exp[1]
e2=exp[2]
else:
e1=0
e2=exp[0]*2
ck.out('Adding param '+str(iparam)+' "'+opt+'" - '+desc)
dd['##param-'+opt]={
"can_omit": "yes",
"default": "",
"desc": "compiler flag: --param "+opt+"= ("+desc+")",
"sort": iparam*10+30000,
"explore_prefix": "--param "+opt+"=",
"explore_start": e1,
"explore_step": 1,
"explore_stop": e2,
"tags": [
"basic",
"parametric",
"optimization"
],
"type":"integer"
}
# Prepare CK entry
if i.get('record','')=='yes':
duoa=i.get('data_uoa','')
if duoa=='':
duoa=fn+'-auto'
if o=='con':
ck.out('')
ck.out('Recording to '+duoa+' ...')
ii={'action':'add',
'module_uoa':work['self_module_uid'],
'repo_uoa':i.get('repo_uoa',''),
'data_uoa':duoa,
'desc':{'all_compiler_flags_desc':dd},
'dict':{
"tags": [
"compiler",
"gcc",
"v"+lver[0],
"v"+lver[0]+"."+lver[1],
"auto"
]
}
}
r=ck.access(ii)
if r['return']>0: return r
p9=os.path.join(pp, 'ChangeLog')
year=''
if os.path.isfile(p9):
t = os.path.getmtime(p9)
t1=str(datetime.datetime.fromtimestamp(t))
year=t1[:4]
results[fn]={'boolean_opts':iopt, 'parametric_opts':iparam, 'year':year}
# Final info
if o=='con':
ck.out('')
ck.out('Compiler: '+str(fn))
ck.out(' Year: '+year)
ck.out(' Number of boolean opts: '+str(iopt))
ck.out(' Number of parameteric opts: '+str(iparam))
# Summary
if len(results)>0:
ck.out('*********************************************************')
for q in sorted(list(results.keys())):
qq=results[q]
x=qq['year']+', '
ib=str(qq['boolean_opts'])
x+=' '*(6-len(ib))+ib+', '
ip=str(qq['parametric_opts'])
x+=' '*(6-len(ip))+ip+', '
x+=q+' '*(10-len(q))
ck.out(x)
return {'return':0}
##############################################################################
# extract optimization flags and parameters from a compiler for autotuning (currently GCC)
def extract_opts_new(i):
"""
Input: {
(host_os) - host OS (detect, if omitted)
(target_os) - OS module to check (if omitted, analyze host)
(device_id) - device id if remote (such as adb)
(compiler_id) - currently support only "gcc" (default)
(repo_uoa) - repo UOA to record compiler description
(data_uoa) - data UOA to record compiler description (if empty, autogenerate)
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
"""
import os
import sys
o=i.get('out','')
comp_id=i.get('compiler_id','').lower()
if comp_id=='': comp_id='gcc'
# Check OS
hos=i.get('host_os','')
tos=i.get('target_os','')
tdid=i.get('device_id','')
# Get some info about platforms
ii={'action':'detect',
'module_uoa':cfg['module_deps']['platform.os'],
'host_os':hos,
'target_os':tos,
'device_id':tdid}
r=ck.access(ii)
if r['return']>0: return r
hos=r['host_os_uid']
hosx=r['host_os_uoa']
hosd=r['host_os_dict']
tos=r['os_uid']
tosx=r['os_uoa']
tosd=r['os_dict']
tdid=r['device_id']
sext=hosd.get('script_ext','')
scall=hosd.get('env_call','')
sbp=hosd.get('bin_prefix','')
tags='compiler'
if comp_id=='gcc': tags+=',gcc'
# Get compiler env
ii={'action':'set',
'module_uoa':cfg['module_deps']['env'],
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'tags':tags,
'out':o}
r=ck.access(ii)
if r['return']>0: return r
env_uoa=r['env_uoa']
bat=r['bat']
# Detect version
ii={'action':'internal_detect',
'module_uoa':cfg['module_deps']['soft'],
'host_os':hos,
'target_os':tos,
'device_id':tdid,
'tags':tags,
'env':bat}
r=ck.access(ii)
if r['return']>0: return r
vstr=r['version_str']
if o=='con':
ck.out('Detected GCC version: '+vstr)
# Prepare batch file
rx=ck.gen_tmp_file({'prefix':'tmp-', 'suffix':sext, 'remove_dir':'yes'})
if rx['return']>0: return rx
fbat=rx['file_name']
# Prepare 2 tmp files
rx=ck.gen_tmp_file({'prefix':'tmp-', 'suffix':'.tmp', 'remove_dir':'yes'})
if rx['return']>0: return rx
fout1=rx['file_name']
# Prepare GCC help
bat+='\n'
bat+='gcc --help=optimizer > '+fout1+'\n'
# Save file
rx=ck.save_text_file({'text_file':fbat, 'string':bat})
if rx['return']>0: return rx
# Execute
y=scall+' '+sbp+fbat
if o=='con':
ck.out('')
ck.out('Executing prepared batch file '+fbat+' ...')
ck.out('')
sys.stdout.flush()
rx=os.system(y)
os.remove(fbat)
# Load opt file
rx=ck.load_text_file({'text_file':fout1,
'split_to_list':'yes',
'encoding':sys.stdin.encoding,
'delete_after_read':'yes'})
if rx['return']>0: return rx
lopts=rx['lst']
# Check if want params
ck.out('')
r=ck.inp({'text':'Enter full path to params.def file if you have GCC sources (or Enter to skip it): '})
if r['return']>0: return r
fpar=r['string'].strip()
lparams=[]
if fpar!='':
rx=ck.load_text_file({'text_file':fpar,
'split_to_list':'yes',
'encoding':sys.stdin.encoding})
if rx['return']>0: return rx
lparams=rx['lst']
# Parsing opts
dd={"##base_opt": {
"choice": [
"-O3",
"-Ofast",
"-O0",
"-O1",
"-O2",
"-Os"
],
"default": "",
"desc": "base compiler flag",
"sort": 10000,
"tags": [
"base",
"basic",
"optimization"
],
"type": "text"
}
}
iopt=0
iparam=0
opt=''
opt1=''
desc=''
desc1=''
add=False
finish=False
for q in range(1, len(lopts)):
qq=lopts[q]
if len(qq)>2:
if qq[2]=='-':
qq=qq.strip()
j=qq.find(' ')
desc1=''
if j>=0:
desc1=qq[j:].strip()
else:
j=len(qq)
opt1=qq[1:j]
if not opt1.startswith('O'):
if opt=='':
opt=opt1
desc=desc1
else:
add=True
else:
qq=qq.strip()
if len(qq)>0:
desc+=' '+qq
else:
add=True
finish=True
if add and opt!='':
iopt+=1
opt=opt[1:]
if opt.startswith('fno-'):
opt=opt[4:]
ck.out('Adding opt '+str(iopt)+' "'+opt+'" - '+desc)
dd['##'+opt]={
"can_omit": "yes",
"choice": [
"-f"+opt,
"-fno-"+opt
],
"default": "",
"desc": "compiler flag: -f"+opt+"("+desc+")",
"sort": iopt*100,
"tags": [
"basic",
"optimization"
],
"type":"text"
}
opt=opt1
desc=desc1
add=False
if finish:
break
# Parsing params
opt=''
opt1=''
desc=''
desc1=''
add=False
finish=False
for q in range(0, len(lparams)):
qq=lparams[q].strip()
if qq.startswith('DEFPARAM'):
iparam+=1
q+=1
opt=lparams[q].strip()[1:-2]
q+=1
desc=lparams[q].strip()[1:-2]
line='x'
while True:
q+=1
line=lparams[q].strip()
if line[-1]==')': break
desc+=line[1:-2]
e1=0
e2=0
exp=line[:-1].split(',')
skip=False
for j in range(0, len(exp)):
jj=exp[j].strip()
if jj.find('*')>0 or jj.find('_')>0:
skip=True
else:
jj=int(exp[j].strip())
exp[j]=jj
if not skip:
if exp[2]>exp[1]:
e1=exp[1]
e2=exp[2]
else:
e1=0
e2=exp[0]*2
ck.out('Adding param '+str(iparam)+' "'+opt+'" - '+desc)
dd['##param_'+opt]={
"can_omit": "yes",
"default": "",
"desc": "compiler flag: --param "+opt+"= ("+desc+")",
"sort": iparam*100+30000,
"explore_prefix": "--param "+opt+"=",
"explore_start": e1,
"explore_step": 1,
"explore_stop": e2,
"tags": [
"basic",
"optimization"
],
"type":"integer"
}
# Prepare CK entry
v=vstr.split('.')
duoa=i.get('data_uoa','')
if duoa=='':
duoa='gcc-'+v[0]+v[1]+'x'
if o=='con':
ck.out('')
ck.out('Recording to '+duoa+' ...')
ii={'action':'add',
'module_uoa':work['self_module_uid'],
'repo_uoa':i.get('repo_uoa',''),
'data_uoa':duoa,
'desc':{'all_compiler_flags_desc':dd},
'dict':{
"tags": [
"compiler",
"gcc",
"v"+v[0],
"v"+v[0]+"."+v[1]
]
}
}
r=ck.access(ii)
if r['return']>0: return r
# Final info
if o=='con':
ck.out('')
ck.out('Compiler version: '+vstr)
ck.out('Number of boolean opts: '+str(iopt))
ck.out('Number of parameteric opts: '+str(iparam))
return {'return':0}
##############################################################################
# viewing compiler description as HTML
def html_viewer(i):
"""
Input: {
data_uoa
url_base
url_pull
url_pull_tmp
tmp_data_uoa
url_wiki
html_share
form_name - current form name
(all_params)
}
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
html - generated HTML
raw - if 'yes', output in CK raw format
show_top - if 'yes', include CK top header with QR-code
}
"""
h=''
raw='no'
top='yes'
duoa=i['data_uoa']
if duoa!='':
# Load entry
rx=ck.access({'action':'load',
'module_uoa':work['self_module_uid'],
'data_uoa':duoa})
if rx['return']>0: return rx
desc=rx['desc']
h+='<hr>\n'
# Output compiler flags description
acfd=desc.get('all_compiler_flags_desc',{})
if len(acfd)>0:
h+='<center>\n'
h+='<b>Compiler optimization flag description (boolean and parametric)</b><br><br>\n'
h+='<table border="1" cellpadding="5" cellspacing="0">\n'
h+=' <tr>\n'
h+=' <td><b>JSON key</b></td>\n'
h+=' <td><b>Description</b></td>\n'
h+=' <td><b>Type</b></td>\n'
h+=' <td><b>Order</b></td>\n'
h+=' <td><b>Choices</b></td>\n'
h+=' <td><b>Parameter prefix</b></td>\n'
h+=' <td><b>Start</b></td>\n'
h+=' <td><b>Stop</b></td>\n'
h+=' <td><b>Step</b></td>\n'
h+=' <td><b>Tags</b></td>\n'
h+=' </tr>\n'
ll=list(acfd.keys())
for k in sorted(ll, key=lambda j: (int(acfd[j].get('sort',0)))):
v=acfd[k]
tags=v.get('tags',[])
stags=''
for q in tags:
if stags!='': stags+=', '
stags+=q
desc=v.get('desc','')
sort=v.get('sort','')
tp=v.get('type','')
choice=v.get('choice',[])
schoice=''
for q in choice:
if schoice!='': schoice+=', '
schoice+=q
ep=v.get('explore_prefix','')
estart=str(v.get('explore_start',''))
estep=str(v.get('explore_step',''))
estop=str(v.get('explore_stop',''))
h+=' <tr>\n'
h+=' <td><i>'+k+'</i></td>\n'
h+=' <td>'+desc+'</td>\n'
h+=' <td>'+tp+'</td>\n'
h+=' <td>'+str(sort)+'</td>\n'
h+=' <td>'+schoice+'</td>\n'
h+=' <td>'+ep+'</td>\n'
h+=' <td>'+estart+'</td>\n'
h+=' <td>'+estop+'</td>\n'
h+=' <td>'+estep+'</td>\n'
h+=' <td>'+stags+'</td>\n'
h+=' </tr>\n'
h+='</table>\n'
h+='</center>\n'
h+='<hr>\n'
return {'return':0, 'raw':raw, 'show_top':top, 'html':h}