Skip to content
Permalink
Browse files

Create script to generate library

Summary:
Automate generating a dagster library. Uses a template found in bin/assets and a user-provided name to set up a library correctly, since it's easy to miss all the places you have to include the library name.

The generated library includes a simple "helloworld" resource and solid

Test Plan:
To use:

```
python bin/create_library.py --name="sftp"
```

To test:
```
cd python_modules/libraries/dagster-sftp
pytest
```

Reviewers: #ft, max, alangenfeld, schrockn

Reviewed By: #ft, schrockn

Subscribers: schrockn, alangenfeld, max, natekupp

Differential Revision: https://dagster.phacility.com/D759
  • Loading branch information...
helloworld committed Aug 3, 2019
1 parent 23e79f7 commit 6c8085ab85af5971536af7ca778ac1b218097f87
@@ -0,0 +1,2 @@
[run]
branch = True
@@ -0,0 +1 @@
# dagster-{{LIBRARY_NAME}}
@@ -0,0 +1 @@
from .version import __version__
@@ -0,0 +1,3 @@
__version__ = '{{VERSION}}'

__nightly__ = 'nightly-2019.07.29'
@@ -0,0 +1,5 @@
from dagster_{{LIBRARY_NAME}}.version import __version__


def test_version():
assert __version__
No changes.
No changes.
@@ -0,0 +1,52 @@
import argparse
import sys

from setuptools import find_packages, setup


def get_version(name):
version = {}
with open('dagster_{{LIBRARY_NAME}}/version.py') as fp:
exec(fp.read(), version) # pylint: disable=W0122

if name == 'dagster-{{LIBRARY_NAME}}':
return version['__version__']
elif name == 'dagster-{{LIBRARY_NAME}}-nightly':
return version['__nightly__']
else:
raise Exception('Shouldn\'t be here: bad package name {name}'.format(name=name))


parser = argparse.ArgumentParser()
parser.add_argument('--nightly', action='store_true')


def _do_setup(name='dagster-{{LIBRARY_NAME}}'):
setup(
name='dagster_{{LIBRARY_NAME}}',
version=get_version(name),
author='Elementl',
license='Apache-2.0',
description='A Dagster integration for {{LIBRARY_NAME}}',
url='https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-{{LIBRARY_NAME}}',
classifiers=[
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'License :: OSI Approved :: Apache Software License',
'Operating System :: OS Independent',
],
packages=find_packages(exclude=['test']),
install_requires=['dagster'],
tests_require=[],
zip_safe=False,
)


if __name__ == '__main__':
parsed, unparsed = parser.parse_known_args()
sys.argv = [sys.argv[0]] + unparsed
if parsed.nightly:
_do_setup('dagster-{{LIBRARY_NAME}}-nightly')
else:
_do_setup('dagster-{{LIBRARY_NAME}}')
@@ -0,0 +1,15 @@
[tox]
envlist = py37,py36,py35,py27

[testenv]
passenv = CI_* COVERALLS_REPO_TOKEN
deps =
-e ../../dagster
-r ../../dagster/dev-requirements.txt
-e .
commands =
coverage erase
pytest -vv --junitxml=test_results.xml --cov=dagster_{{LIBRARY_NAME}} --cov-append --cov-report=
coverage report --omit='.tox/*,**/test_*.py' --skip-covered
coverage html --omit='.tox/*,**/test_*.py'
coverage xml --omit='.tox/*,**/test_*.py'
@@ -0,0 +1,64 @@
#!/usr/bin/env python3

"""
To use, run
python bin/create_library.py --name library_name
"""
from __future__ import absolute_import

import click
import os
import shutil

from git_tag import get_most_recent_git_tag


def copy_directory(src, dest):
try:
shutil.copytree(src, dest, ignore=shutil.ignore_patterns('.DS_Store'))
# Directories are the same
except shutil.Error as e:
print('Directory not copied. Error: %s' % e)
# Any error saying that the directory doesn't exist
except OSError as e:
print('Directory not copied. Error: %s' % e)


@click.command()
@click.option('--name', prompt='Name of library', help='Name of library')
def main(name):
template_library_path = os.path.abspath('bin/assets/dagster-library-tmpl')
new_template_library_path = os.path.abspath(
'python_modules/libraries/dagster-{name}'.format(name=name)
)

if os.path.exists(new_template_library_path):
raise click.UsageError('Library with name {name} already exists'.format(name=name))

copy_directory(template_library_path, new_template_library_path)

version = get_most_recent_git_tag()

for dname, _, files in os.walk(new_template_library_path):
for fname in files:
fpath = os.path.join(dname, fname)
with open(fpath) as f:
s = f.read()
s = s.replace('{{LIBRARY_NAME}}', name)
s = s.replace('{{VERSION}}', version)
with open(fpath, 'w') as f:
f.write(s)

new_fname = fname.replace('.tmpl', '')
new_fpath = os.path.join(dname, new_fname)
shutil.move(fpath, new_fpath)
print('Created {path}'.format(path=new_fpath))

new_dname = dname.replace('library-tmpl', name)
shutil.move(dname, new_dname)

print('Library created at {path}'.format(path=new_template_library_path))


if __name__ == "__main__":
main() # pylint: disable=no-value-for-parameter
@@ -0,0 +1,63 @@
import re
import subprocess


def get_git_tag():
try:
git_tag = str(
subprocess.check_output(
['git', 'describe', '--exact-match', '--abbrev=0'], stderr=subprocess.STDOUT
)
).strip('\'b\\n')
except subprocess.CalledProcessError as exc_info:
match = re.search(
'fatal: no tag exactly matches \'(?P<commit>[a-z0-9]+)\'', str(exc_info.output)
)
if match:
raise Exception(
'Bailing: there is no git tag for the current commit, {commit}'.format(
commit=match.group('commit')
)
)
raise Exception(str(exc_info.output))

return git_tag


def get_most_recent_git_tag():
try:
git_tag = str(
subprocess.check_output(['git', 'describe', '--abbrev=0'], stderr=subprocess.STDOUT)
).strip('\'b\\n')
except subprocess.CalledProcessError as exc_info:
raise Exception(str(exc_info.output))
return git_tag


def set_git_tag(tag, signed=False):
try:
if signed:
subprocess.check_output(['git', 'tag', '-s', '-m', tag, tag], stderr=subprocess.STDOUT)
else:
subprocess.check_output(['git', 'tag', '-a', '-m', tag, tag], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc_info:
match = re.search('error: gpg failed to sign the data', str(exc_info.output))
if match:
raise Exception(
'Bailing: cannot sign tag. You may find '
'https://stackoverflow.com/q/39494631/324449 helpful. Original error '
'output:\n{output}'.format(output=str(exc_info.output))
)

match = re.search(
r'fatal: tag \'(?P<tag>[\.a-z0-9]+)\' already exists', str(exc_info.output)
)
if match:
raise Exception(
'Bailing: cannot release version tag {tag}: already exists'.format(
tag=match.group('tag')
)
)
raise Exception(str(exc_info.output))

return tag
@@ -12,7 +12,6 @@
import fnmatch
import inspect
import os
import re

import subprocess
import tempfile
@@ -27,6 +26,7 @@
import virtualenv

from pypirc import ConfigFileError, RCParser
from git_tag import get_git_tag, set_git_tag, get_most_recent_git_tag

assert os.getenv('SLACK_RELEASE_BOT_TOKEN'), 'No SLACK_RELEASE_BOT_TOKEN env variable found.'
slack_client = slackclient.SlackClient(os.environ['SLACK_RELEASE_BOT_TOKEN'])
@@ -164,67 +164,6 @@ def publish_all(nightly):
publish_module(module, nightly, library=True)


def get_most_recent_git_tag():
try:
git_tag = str(
subprocess.check_output(['git', 'describe', '--abbrev=0'], stderr=subprocess.STDOUT)
).strip('\'b\\n')
except subprocess.CalledProcessError as exc_info:
raise Exception(str(exc_info.output))
return git_tag


def get_git_tag():
try:
git_tag = str(
subprocess.check_output(
['git', 'describe', '--exact-match', '--abbrev=0'], stderr=subprocess.STDOUT
)
).strip('\'b\\n')
except subprocess.CalledProcessError as exc_info:
match = re.search(
'fatal: no tag exactly matches \'(?P<commit>[a-z0-9]+)\'', str(exc_info.output)
)
if match:
raise Exception(
'Bailing: there is no git tag for the current commit, {commit}'.format(
commit=match.group('commit')
)
)
raise Exception(str(exc_info.output))

return git_tag


def set_git_tag(tag, signed=False):
try:
if signed:
subprocess.check_output(['git', 'tag', '-s', '-m', tag, tag], stderr=subprocess.STDOUT)
else:
subprocess.check_output(['git', 'tag', '-a', '-m', tag, tag], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc_info:
match = re.search('error: gpg failed to sign the data', str(exc_info.output))
if match:
raise Exception(
'Bailing: cannot sign tag. You may find '
'https://stackoverflow.com/q/39494631/324449 helpful. Original error '
'output:\n{output}'.format(output=str(exc_info.output))
)

match = re.search(
r'fatal: tag \'(?P<tag>[\.a-z0-9]+)\' already exists', str(exc_info.output)
)
if match:
raise Exception(
'Bailing: cannot release version tag {tag}: already exists'.format(
tag=match.group('tag')
)
)
raise Exception(str(exc_info.output))

return tag


def format_module_versions(module_versions):
return '\n'.join(
[

0 comments on commit 6c8085a

Please sign in to comment.
You can’t perform that action at this time.