Permalink
Browse files

add linter flake8 for tensorboardX

  • Loading branch information...
leVirve committed Dec 31, 2017
1 parent 28e9565 commit 86932bc3efab8b36bb734a95ca6661f5f508eb36
View
@@ -0,0 +1,4 @@
[flake8]
max-line-length = 120
ignore = E305,E402,E721,E741,F401,F403,F405,F821,F841,F999
exclude = tensorboardX/src
View
@@ -22,14 +22,15 @@ install:
- conda info -a
# Replace dep1 dep2 ... with your dependencies
- conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION chainer pytorch torchvision -c pytorch
- conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION chainer pytorch torchvision -c pytorch
- source activate test-environment
- which python
- conda list
- pip install --upgrade pytest
- pip install --upgrade pytest flake8
- python setup.py install
script:
- flake8 tensorboardX
- pytest
# Your test script goes here
- pip uninstall -y tensorboardX
View
@@ -66,7 +66,7 @@
0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e,
0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,
)
)
CRC_INIT = 0
View
@@ -1,5 +1,6 @@
import os
def make_tsv(metadata, save_path):
metadata = [str(x) for x in metadata]
with open(os.path.join(save_path, 'metadata.tsv'), 'w') as f:
@@ -13,34 +14,36 @@ def make_sprite(label_img, save_path):
import torch
import torchvision
from .x2num import makenp
# this ensures the sprite image has correct dimension as described in
# this ensures the sprite image has correct dimension as described in
# https://www.tensorflow.org/get_started/embedding_viz
nrow = int(math.ceil((label_img.size(0)) ** 0.5))
label_img = torch.from_numpy(makenp(label_img)) #for other framework
label_img = torch.from_numpy(makenp(label_img)) # for other framework
# augment images so that #images equals nrow*nrow
label_img = torch.cat((label_img, torch.randn(nrow ** 2 - label_img.size(0), *label_img.size()[1:]) * 255), 0)
torchvision.utils.save_image(label_img, os.path.join(save_path, 'sprite.png'), nrow=nrow, padding=0)
def append_pbtxt(metadata, label_img, save_path, global_step, tag):
with open(os.path.join(save_path, 'projector_config.pbtxt'), 'a') as f:
#step = os.path.split(save_path)[-1]
# step = os.path.split(save_path)[-1]
f.write('embeddings {\n')
f.write('tensor_name: "{}:{}"\n'.format(tag, global_step))
f.write('tensor_path: "{}"\n'.format(os.path.join(global_step,"tensors.tsv")))
f.write('tensor_path: "{}"\n'.format(os.path.join(global_step, 'tensors.tsv')))
if metadata is not None:
f.write('metadata_path: "{}"\n'.format(os.path.join(global_step,"metadata.tsv")))
f.write('metadata_path: "{}"\n'.format(os.path.join(global_step, 'metadata.tsv')))
if label_img is not None:
f.write('sprite {\n')
f.write('image_path: "{}"\n'.format(os.path.join(global_step,"sprite.png")))
f.write('image_path: "{}"\n'.format(os.path.join(global_step, 'sprite.png')))
f.write('single_image_dim: {}\n'.format(label_img.size(3)))
f.write('single_image_dim: {}\n'.format(label_img.size(2)))
f.write('}\n')
f.write('}\n')
def make_mat(matlist, save_path):
with open(os.path.join(save_path, 'tensors.tsv'), 'w') as f:
for x in matlist:
x = [str(i) for i in x]
f.write('\t'.join(x) + '\n')
f.write('\t'.join(x) + '\n')
@@ -45,8 +45,7 @@ def __init__(self, file_prefix):
Events files have a name of the form
'/some/file/path/events.out.tfevents.[timestamp].[hostname]'
'''
self._file_prefix = file_prefix + ".out.tfevents." \
+ str(time.time())[:10] + "." + socket.gethostname()
self._file_prefix = file_prefix + ".out.tfevents." + str(time.time())[:10] + "." + socket.gethostname()
# Open(Create) the log file with the particular form of name.
logging.basicConfig(filename=self._file_prefix)
View
@@ -6,39 +6,44 @@
global id2name
global list_of_nodes
def make_name(obj):
if hasattr(obj, 'variable'):#weight/bias in module
if hasattr(obj, 'variable'): # weight/bias in module
if obj.variable is not None:
return id2name[id(obj.variable)]+'_'+str(id(obj.variable))
return id2name[id(obj.variable)] + '_' + str(id(obj.variable))
else:
return 'inputTensor_'+str(id(obj))
return 'inputTensor_' + str(id(obj))
else:
return type(obj).__name__.replace('Backward','_')+str(id(obj))
return type(obj).__name__.replace('Backward', '_') + str(id(obj))
def make_list_of_nodes(fn):
if fn is None:
return
return
inputs = []
for next_fn, _ in fn.next_functions:
inputs.append(make_name(next_fn))
make_list_of_nodes(next_fn)
attrshape = []
if hasattr(fn, 'variable'):#weight/bias in module
attrshape = []
if hasattr(fn, 'variable'): # weight/bias in module
if fn.variable is not None:
attrshape = list(fn.variable.size())
list_of_nodes.append({'name':make_name(fn), 'op':type(fn).__name__, 'inputs':inputs, 'attr.shape':attrshape})
list_of_nodes.append({'name': make_name(fn), 'op': type(fn).__name__, 'inputs': inputs, 'attr.shape': attrshape})
def graph(model, lastVar):
global id2name
global list_of_nodes
id2name = {id(m):n.replace('.', '/')+'(parameters)' for n, m in model.named_parameters()}
id2name = {id(m): n.replace('.', '/') + '(parameters)' for n, m in model.named_parameters()}
nodes = []
list_of_nodes = []
make_list_of_nodes(lastVar.grad_fn)
for node in list_of_nodes:
#shape = TensorShapeProto(dim=[TensorShapeProto.Dim(size=i) for i in node['attr.shape']]) ugly...
# shape = TensorShapeProto(dim=[TensorShapeProto.Dim(size=i) for i in node['attr.shape']]) ugly...
shape_str = str(node['attr.shape']).encode(encoding='utf_8')
nodes.append(NodeDef(name=node['name'], op=node['op'], input=node['inputs'], attr={'shape':AttrValue(s=shape_str)}))#, 'T':AttrValue(type="DT_FLOAT")}))
nodes.append(
NodeDef(name=node['name'], op=node['op'], input=node['inputs'],
attr={'shape': AttrValue(s=shape_str)})) # , 'T':AttrValue(type="DT_FLOAT")}))
return GraphDef(node=nodes, versions=VersionDef(producer=22))
View
@@ -3,28 +3,31 @@
from .src.versions_pb2 import VersionDef
from .src.attr_value_pb2 import AttrValue
from .src.tensor_shape_pb2 import TensorShapeProto
#from .src.onnx_pb2 import ModelProto
# from .src.onnx_pb2 import ModelProto
def gg(fname):
import onnx # 0.2.1
import onnx # 0.2.1
m = onnx.load(fname)
nodes_proto = []
nodes = []
nodes = []
g = m.graph
import itertools
for node in itertools.chain(g.input, g.output):
nodes_proto.append(node)
for node in nodes_proto:
shapeproto = TensorShapeProto(dim=[TensorShapeProto.Dim(size=d.dim_value) for d in node.type.tensor_type.shape.dim])
shapeproto = TensorShapeProto(
dim=[TensorShapeProto.Dim(size=d.dim_value) for d in node.type.tensor_type.shape.dim])
nodes.append(NodeDef(
name=node.name,
op='Variable',
name=node.name,
op='Variable',
input=[],
attr={
'dtype':AttrValue(type=node.type.tensor_type.elem_type),
'shape':AttrValue(shape=shapeproto),
},
))
'dtype': AttrValue(type=node.type.tensor_type.elem_type),
'shape': AttrValue(shape=shapeproto),
})
)
for node in g.node:
attr = []
@@ -33,24 +36,23 @@ def gg(fname):
attr = ', '.join(attr).encode(encoding='utf_8')
nodes.append(NodeDef(
name=node.output[0],
op=node.op_type,
name=node.output[0],
op=node.op_type,
input=node.input,
attr={
'parameters':AttrValue(s=attr),
},
))
attr={'parameters': AttrValue(s=attr)},
))
# two pass token replacement, appends opname to object id
mapping = {}
for node in nodes:
mapping[node.name] = node.op+'_'+node.name
for node in nodes:
mapping[node.name] = node.op + '_' + node.name
nodes, mapping = updatenodes(nodes, mapping)
mapping = smartGrouping(nodes, mapping)
nodes, mapping = updatenodes(nodes, mapping)
return GraphDef(node=nodes, versions=VersionDef(producer=22))
def updatenodes(nodes, mapping):
for node in nodes:
newname = mapping[node.name]
@@ -65,6 +67,7 @@ def updatenodes(nodes, mapping):
newmap[v] = v
return nodes, newmap
def findnode(nodes, name):
""" input: node name
returns: node object
@@ -73,83 +76,83 @@ def findnode(nodes, name):
if n.name == name:
return n
def parser(s, nodes, node):
print(s)
if len(s)==0:
if len(s) == 0:
return
if len(s)>0:
if s[0]==node.op:
if len(s) > 0:
if s[0] == node.op:
print(s[0], node.name, s[1], node.input)
for n in node.input:
print(n, s[1])
parser(s[1], nodes, findnode(nodes, n))
else:
return False
# TODO: use recursive parse
def smartGrouping(nodes, mapping):
# a Fully Conv is: (TODO: check var1.size(0)==var2.size(0))
# GEMM <-- Variable (c1)
# ^-- Transpose (c2) <-- Variable (c3)
# ^-- Transpose (c2) <-- Variable (c3)
# a Conv with bias is: (TODO: check var1.size(0)==var2.size(0))
# Add <-- Conv (c2) <-- Variable (c3)
# ^-- Variable (c1)
#
#gemm = ('Gemm', ('Variable', ('Transpose', ('Variable'))))
#
# gemm = ('Gemm', ('Variable', ('Transpose', ('Variable'))))
FCcounter = 1
Convcounter = 1
for node in nodes:
if node.op=='Gemm':
if node.op == 'Gemm':
c1 = c2 = c3 = False
for name_in in node.input:
n = findnode(nodes, name_in)
if n.op=='Variable':
if n.op == 'Variable':
c1 = True
c1name = n.name
if n.op=='Transpose':
if n.op == 'Transpose':
c2 = True
c2name = n.name
if len(n.input)==1:
if len(n.input) == 1:
nn = findnode(nodes, n.input[0])
if nn.op=='Variable':
if nn.op == 'Variable':
c3 = True
c3name = nn.name
#print(n.op, n.name, c1, c2, c3)
# print(n.op, n.name, c1, c2, c3)
if c1 and c2 and c3:
#print(c1name, c2name, c3name)
# print(c1name, c2name, c3name)
mapping[c1name] = 'FC{}/{}'.format(FCcounter, c1name)
mapping[c2name] = 'FC{}/{}'.format(FCcounter, c2name)
mapping[c3name] = 'FC{}/{}'.format(FCcounter, c3name)
mapping[node.name] = 'FC{}/{}'.format(FCcounter, node.name)
FCcounter+=1
FCcounter += 1
continue
if node.op=='Add':
if node.op == 'Add':
c1 = c2 = c3 = False
for name_in in node.input:
n = findnode(nodes, name_in)
if n.op=='Variable':
if n.op == 'Variable':
c1 = True
c1name = n.name
if n.op=='Conv':
if n.op == 'Conv':
c2 = True
c2name = n.name
if len(n.input)>=1:
if len(n.input) >= 1:
for nn_name in n.input:
nn = findnode(nodes, nn_name)
if nn.op=='Variable':
if nn.op == 'Variable':
c3 = True
c3name = nn.name
if c1 and c2 and c3:
#print(c1name, c2name, c3name)
# print(c1name, c2name, c3name)
mapping[c1name] = 'Conv{}/{}'.format(Convcounter, c1name)
mapping[c2name] = 'Conv{}/{}'.format(Convcounter, c2name)
mapping[c3name] = 'Conv{}/{}'.format(Convcounter, c3name)
mapping[node.name] = 'Conv{}/{}'.format(Convcounter, node.name)
Convcounter+=1
Convcounter += 1
return mapping
@@ -45,4 +45,3 @@ def make_valid_tf_name(name):
# Must make it valid somehow, but don't want to remove stuff
name = '.' + name
return '_'.join(_VALID_OP_NAME_PART.findall(name))
Oops, something went wrong.

0 comments on commit 86932bc

Please sign in to comment.