Skip to content

Commit

Permalink
Expand idl codegen (#1475)
Browse files Browse the repository at this point in the history 
* fix and expand XBR IDL code generator
* fix wstest workflow (maybe;)
  • Loading branch information
@oberstet
oberstet committed Mar 17, 2021
1 parent aa0e268 commit 126e180
Show file tree
Hide file tree
Showing 7 changed files with 336 additions and 78 deletions.
25 changes: 7 additions & 18 deletions .github/workflows/wstest.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,26 +35,20 @@ jobs:
# Checkout sources
- uses: actions/checkout@v2

# Install OS packages, as we install Python packages from source:
#
# libenchant-dev needed for pyenchant, needed for sphinx-spellcheck
# libbz2-dev, libsnappy-dev needed for compression
# libunwind-dev needed for vmprof
#
- name: Install OS package dependencies
run: |
sudo apt update
sudo apt install build-essential libssl-dev libffi-dev libunwind-dev \
libreadline-dev zlib1g-dev libbz2-dev libsqlite3-dev libncurses5-dev \
libsnappy-dev
# Use this Python
# https://github.com/actions/setup-python/blob/main/README.md
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}

- name: Install Python package dependencies
run: |
curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python3 get-pip.py "pip<20"
pip3 show pip
pip3 install -r requirements-dev.txt
- name: Set environment
run: |
echo AUTOBAHN_BUILD_DATE=`date -u +"%Y-%m-%d"` >> $GITHUB_ENV
Expand All @@ -76,11 +70,6 @@ jobs:
echo " AWS_S3_BUCKET_NAME = ${AWS_S3_BUCKET_NAME}"
echo ""
- name: Install Python package dependencies
run: |
python -m pip install -U pip
pip install -U -r requirements-dev.txt
- name: Install this package
run: |
pip install .[all]
Expand Down
79 changes: 63 additions & 16 deletions autobahn/xbr/_cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@

from jinja2 import Environment, FileSystemLoader

# https://github.com/google/yapf#example-as-a-module
from yapf.yapflib.yapf_api import FormatCode

from autobahn import xbr
from autobahn import __version__
from autobahn.xbr import FbsType
Expand Down Expand Up @@ -824,6 +827,12 @@ def _main():
type=str,
help='FlatBuffers binary schema file to read (.bfbs)')

parser.add_argument('-b',
'--basemodule',
dest='basemodule',
type=str,
help='Render to this base module')

_LANGUAGES = ['python', 'json']
parser.add_argument('-l',
'--language',
Expand Down Expand Up @@ -984,7 +993,7 @@ def _main():
elif args.command == 'codegen-schema':

# load repository from flatbuffers schema files
repo = FbsRepository()
repo = FbsRepository(render_to_basemodule=args.basemodule)
repo.load(args.schema)

# print repository summary
Expand Down Expand Up @@ -1044,11 +1053,23 @@ def _main():
if args.language == 'python':
# render obj|enum|service.py.jinja2 template
tmpl = env.get_template('{}.py.jinja2'.format(category))
code = tmpl.render(repo=repo, metadata=metadata, FbsType=FbsType, render_imports=is_first, is_first_by_category=is_first_by_category)
code = tmpl.render(repo=repo, metadata=metadata, FbsType=FbsType,
render_imports=is_first,
is_first_by_category=is_first_by_category,
render_to_basemodule=args.basemodule)
code = FormatCode(code)[0]

# render test_obj|enum|service.py.jinja2 template
test_tmpl = env.get_template('test_{}.py.jinja2'.format(category))
test_code = test_tmpl.render(repo=repo, metadata=metadata, FbsType=FbsType, render_imports=is_first, is_first_by_category=is_first_by_category)
test_code = test_tmpl.render(repo=repo, metadata=metadata, FbsType=FbsType,
render_imports=is_first,
is_first_by_category=is_first_by_category,
render_to_basemodule=args.basemodule)
try:
test_code = FormatCode(test_code)[0]
except Exception as e:
print('error during formatting code:\n{}\n{}'.format(test_code, e))

elif args.language == 'json':
code = json.dumps(metadata.marshal(),
separators=(', ', ': '),
Expand All @@ -1069,6 +1090,20 @@ def _main():
else:
test_code_modules[modulename].append(None)

# ['', 'com.example.bla.blub', 'com.example.doo']
namespaces = {}
for code_file in code_modules.keys():
name_parts = code_file.split('.')
for i in range(len(name_parts)):
pn = name_parts[i]
ns = '.'.join(name_parts[:i])
if ns not in namespaces:
namespaces[ns] = []
if pn and pn not in namespaces[ns]:
namespaces[ns].append(pn)

print('Namespaces:\n{}\n'.format(pformat(namespaces)))

# write out code modules
#
i = 0
Expand All @@ -1080,19 +1115,29 @@ def _main():
else:
code_file_dir = ['']

# FIXME: cleanup this mess
for i in range(len(code_file_dir)):
d = os.path.join(args.output, *(code_file_dir[:i + 1]))
if not os.path.isdir(d):
os.mkdir(d)
if args.language == 'python':
fn = os.path.join(d, '__init__.py')
if not os.path.exists(fn):
_modulename = '.'.join(code_file_dir[:i + 1])[1:]
with open(fn, 'wb') as f:
tmpl = env.get_template('module.py.jinja2')
init_code = tmpl.render(repo=repo, modulename=_modulename)
f.write(init_code.encode('utf8'))
initialized.add(fn)
if args.language == 'python':
fn = os.path.join(d, '__init__.py')

_modulename = '.'.join(code_file_dir[:i + 1])[1:]
_imports = namespaces[_modulename]
tmpl = env.get_template('module.py.jinja2')
init_code = tmpl.render(repo=repo, modulename=_modulename, imports=_imports,
render_to_basemodule=args.basemodule)
data = init_code.encode('utf8')

if not os.path.exists(fn):
with open(fn, 'wb') as f:
f.write(data)
print('Ok, rendered "module.py.jinja2" in {} bytes to "{}"'.format(len(data), fn))
initialized.add(fn)
else:
with open(fn, 'ab') as f:
f.write(data)

if args.language == 'python':
if code_file:
Expand All @@ -1119,11 +1164,13 @@ def _main():
fn = os.path.join(*(code_file_dir + [code_file_name]))
fn = os.path.join(args.output, fn)

# FIXME
if fn not in initialized and os.path.exists(fn):
os.remove(fn)
with open(fn, 'wb') as fd:
fd.write('# Generated by Autobahn v{}\n'.format(__version__).encode('utf8'))
initialized.add(fn)
print('D' * 100, fn)
# os.remove(fn)
# with open(fn, 'wb') as fd:
# fd.write('# Generated by Autobahn v{}\n'.format(__version__).encode('utf8'))
# initialized.add(fn)

with open(fn, 'ab') as fd:
fd.write(data)
Expand Down

0 comments on commit 126e180

Please sign in to comment.