diff --git a/tokenizer/python/python_tokenizer.py b/tokenizer/python/python_tokenizer.py index ce4c586..b1211dc 100644 --- a/tokenizer/python/python_tokenizer.py +++ b/tokenizer/python/python_tokenizer.py @@ -1,8 +1,7 @@ import parso from parso.python.tokenize import tokenize -from parso.python.token import tok_name import json -import sys +import sys file = open(sys.argv[1],'r') file_content = file.read() @@ -10,12 +9,12 @@ tokens = [] for token in tokenize(file_content, version_info=(3,6)): - tmp = dict() - tmp["line"]=(token.start_pos)[0]; - tmp["char"]=((token.start_pos)[1])+1; - tmp["type"]=str(tok_name[token.type]) - tmp["value"]=str(token.string) - tokens.append(tmp) + if (str(token.string) != ""): + tmp = dict() + tmp["line"]=(token.start_pos)[0]; + tmp["char"]=((token.start_pos)[1])+1; + tmp["type"]=((str(token.type))[10:]).strip(")") + tmp["value"]=str(token.string) + tokens.append(tmp) print ( json.dumps(tokens, indent=4, sort_keys=True) ) -