Skip to content
Browse files

update lint setup

* move config in to pylintrc so arc and CI stay aligned
* stop using pylint targets and just lint all python files

Test Plan: edit a snapshot file, run `arc lint --trace` verify it skips it.

Reviewers: max, natekupp

Reviewed By: max, natekupp

Differential Revision:
  • Loading branch information...
alangenfeld committed May 24, 2019
1 parent 8194c3b commit e79252248666743bec230a488cb9b3221e226df0
Showing with 34 additions and 77 deletions.
  1. +0 −32 .pylint_targets
  2. +9 −16 .pylintrc
  3. +5 −11 Makefile
  4. +20 −17 bin/
  5. +0 −1 js_modules/dagit/

This file was deleted.

@@ -9,24 +9,17 @@
# can either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once).#
# C0102 blacklisted variable name (foo)
# C0103 invalid variable name (e.g. two chars)
# R0100 duplicate code (false positive in test cases)
# R0201 method could be a function
# R0903 too few public methods
# R0913 too many arguments
# R - refactoring related checks
# C - convention related checks
# W0511 disable TODO warning
# W1201, W1202 disable log format warning. False positives (I think)
# R1705 Unnecessary else after return
# R0902 Too many instance attributes. Sometimes big config objects are what you want
# R1710 All returns should return an expression
# C0302 too many lines in single module
# R0911 Too many returns
# R0901 Too many ancestors
# R0205 py3-only inherit from object warning
# C0330 bad-continuation (we let black take care of this)


# See:
ignored-classes= responses
ignored-classes= responses

@@ -1,15 +1,5 @@
# This is a hack because we are getting timeouts on CircleCI running pylint on all the targets
# at once
set -e;
for target in `cat .pylint_targets` ; do \
echo $$target; \
pylint -j 0 $$target --rcfile=.pylintrc --disable=R,C || exit 1;\
set +e;

pylint -j 0 `cat .pylint_targets` --rcfile=.pylintrc --disable=R,C
pylint -j 0 `find . -name '*.py'` --rcfile=.pylintrc

pytest docs --snapshot-update
@@ -40,8 +30,12 @@ install_dev_python_modules:
pip install -e python_modules/libraries/dagster-snowflake
pip install -e python_modules/libraries/dagster-spark
pip install -e python_modules/libraries/dagster-pyspark
pip install -e python_modules/libraries/dagster-pagerduty
pip install -e python_modules/libraries/dagster-slack
pip install -e python_modules/libraries/dagster-datadog
pip install -e python_modules/automation
pip install -e examples[full]
pip install -r bin/requirements.txt

cd js_modules/dagit/; make generate-types
@@ -14,15 +14,12 @@
import os
import re

# import shlex
import subprocess
import sys
import tempfile

from distutils import spawn # pylint: disable=no-name-in-module
from itertools import groupby
from threading import Thread

import click
import packaging.version
@@ -76,7 +73,7 @@ def construct_publish_comands(additional_steps=None, nightly=False):
return publish_commands

'''For dagit, we need to build the JS assets.'''
# For dagit, we need to build the JS assets.
'pushd ../../js_modules/dagit; yarn install && yarn build-for-python; popd'
@@ -199,7 +196,7 @@ def set_git_tag(tag, signed=False):

match =
'fatal: tag \'(?P<tag>[\.a-z0-9]+)\' already exists', str(exc_info.output)
r'fatal: tag \'(?P<tag>[\.a-z0-9]+)\' already exists', str(exc_info.output)
if match:
raise Exception(
@@ -212,7 +209,7 @@ def set_git_tag(tag, signed=False):
return tag

def format_module_versions(module_versions, nightly=False):
def format_module_versions(module_versions):
return '\n'.join(
' {module_name}: {version} {nightly}'.format(
@@ -371,7 +368,9 @@ def check_new_version(new_version):
parsed_version = packaging.version.parse(new_version)
module_versions = check_existing_version()
errors = {}
last_version = None
for module_name, module_version in module_versions.items():
last_version = module_version
if packaging.version.parse(module_version['__version__']) >= parsed_version:
errors[module_name] = module_version['__version__']
if errors:
@@ -387,13 +386,13 @@ def check_new_version(new_version):
or parsed_version.is_postrelease
or parsed_version.is_devrelease
parsed_previous_version = packaging.version.parse(module_version['__version__'])
parsed_previous_version = packaging.version.parse(last_version['__version__'])
if not (parsed_previous_version.release == parsed_version.release):
should_continue = input(
'You appear to be releasing a new version, {new_version}, without having '
'previously run a prerelease.\n(Last version found was {previous_version})\n'
'Are you sure you know what you\'re doing? (Y/n)'.format(
new_version=new_version, previous_version=module_version['__version__']
new_version=new_version, previous_version=last_version['__version__']
if not should_continue == 'Y':
@@ -687,20 +686,20 @@ def publish(nightly, autoclean):

def release(version):
def release(ver):
"""Tags all submodules for a new release.
Ensures that git tags, as well as the files in each submodule, agree and that the
new version is strictly greater than the current version. Will fail if the new version
is not an increment (following PEP 440). Creates a new git tag and commit.
'Successfully set new version and created git tag {version}. You may continue with the '
'release checklist.'.format(version=version)
'release checklist.'.format(version=ver)

@@ -722,12 +721,16 @@ def version():
print('All modules in lockstep with most recent tagged version: {git_tag}'.format(git_tag))
'All modules in lockstep with most recent tagged version: {git_tag}'.format(

def audit(version):
def audit(ver):
"""Checks that the given version is installable from PyPI in a new virtualenv."""

bootstrap_text = '''
@@ -746,7 +749,7 @@ def after_install(options, home_dir):

bootstrap_script = virtualenv.create_bootstrap_script(bootstrap_text)
@@ -1,5 +1,4 @@
import glob
import json
import os

if __name__ == '__main__':

0 comments on commit e792522

Please sign in to comment.
You can’t perform that action at this time.