diff --git a/.env b/.env new file mode 100644 index 00000000..ad82af13 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +LC_ALL=C.UTF-8 +LANG=C.UTF-8 diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml index 4099cb95..b4d9482d 100644 --- a/.github/workflows/style_check.yml +++ b/.github/workflows/style_check.yml @@ -14,7 +14,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.7] + python-version: [3.8] steps: - uses: actions/checkout@v2 @@ -24,25 +24,30 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install python dependencies run: | - python -m pip install --upgrade pip - pip install black + python -m pip install --upgrade pip pipenv + pipenv install --dev --skip-lock - name: Black pep8 style run: | - black ./ --check - + make check-style-python + cpp-style-check: runs-on: ubuntu-latest strategy: max-parallel: 4 + matrix: + python-version: [3.8] steps: - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install python dependencies + run: | + python -m pip install --upgrade pip pipenv + pipenv install --dev --skip-lock - name: Lint with clang-format run: | - # lint files (all .cpp and .h files) inplace - find ./src/bindings/ -iname *.hpp -o -iname *.cpp -o -iname *.h -o -iname *.cc | xargs clang-format -i -style='file' - # print changes - git diff src - # already well formated if 'git diff' doesn't output - ! ( git diff src | grep -q ^ ) || exit + make check-style-cpp diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b97854b..45675898 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,32 +22,17 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install coverage - pip install wheel - - name: Set up submodules - run: | - git submodule update --init --recursive - rm -rf third_party/differential-privacy/java - rm -rf third_party/differential-privacy/examples/java + - name: Set environment variables + uses: allenevans/set-env@v1.0.0 + with: + PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }} - name: Install pre-requisites run: | sudo apt-get update bash ./prereqs_linux.sh - export PATH="$PATH:$HOME/bin" - name: Build pydp lib run: | - bazel build src/python:bindings_test --verbose_failures - rm -f pydp.so - cp -f ./bazel-bin/src/bindings/pydp.so ./pydp + bash ./build_PyDP.sh - name: Run tests run: | - pip install pycodestyle - pip install pytest - python setup.py test - - name: Check test coverage - run: | - coverage run --source pydp setup.py test - coverage report --ignore-errors --fail-under 95 -m + make run-tests-only diff --git a/.gitignore b/.gitignore index aa7d9b28..58e227dc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ __pycache__/ *$py.class # Distribution / packaging +Pipfile.lock .Python env/ build/ @@ -37,7 +38,6 @@ wheels/ # Unit test / coverage reports htmlcov/ -.tox/ .coverage .coverage.* .cache @@ -46,4 +46,4 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ -.mypy_cache/ \ No newline at end of file +.mypy_cache/ diff --git a/Makefile b/Makefile index ff4c1b53..077bd7f0 100644 --- a/Makefile +++ b/Makefile @@ -13,16 +13,6 @@ webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) endef export BROWSER_PYSCRIPT -# check for python version -python_version_full := $(wordlist 2,4,$(subst ., ,$(shell python --version 2>&1))) -python_version_major := $(word 1,${python_version_full}) - -ifeq ($(python_version_major), 2) - python_var := python3 -else - python_var := python -endif - define PRINT_HELP_PYSCRIPT import re, sys @@ -34,10 +24,13 @@ for line in sys.stdin: endef export PRINT_HELP_PYSCRIPT -BROWSER := python -c "$$BROWSER_PYSCRIPT" +BROWSER := pipenv run python -c "$$BROWSER_PYSCRIPT" help: - @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + @pipenv run python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +build: ## compile bindings and genearte Python module + ./build_PyDP.sh clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts @@ -55,30 +48,41 @@ clean-pyc: ## remove Python file artifacts find . -name '__pycache__' -exec rm -fr {} + clean-test: ## remove test and coverage artifacts - rm -fr .tox/ rm -f .coverage rm -fr htmlcov/ rm -fr .pytest_cache -test: ## run tests quickly with the default Python - ${python_var} setup.py test +format-style-python: ## format Python files code style in-place + @ pipenv run black ./ + +format-style-cpp: ## format C++ files code style in-place + @ find ./src/bindings/ -iname *.hpp -o -iname *.cpp -o -iname *.h -o -iname *.cc | \ + xargs clang-format -i -style='file' + +check-style-python: + @ echo "\e[36mChecking Python code style.\e[0m" && \ + pipenv run black ./ --check --diff || \ + ( echo "\e[33mRun \e[36mmake format-style-python\e[33m to fix style errors.\e[0m"; \ + exit 1 ) + +check-style-cpp: + @ echo "\e[36mChecking C++ code style.\e[0m" && \ + pipenv run ./run-clang-format.py -r src/bindings/ || \ + ( echo "\e[33mRun \e[34mmake format-style-cpp\e[33m to fix style errors.\e[0m"; \ + exit 1 ) -test-all: ## run tests on every Python version with tox - tox +run-tests-only: install + pipenv run pytest tests -coverage: ## check code coverage quickly with the default Python - coverage run --source pydp setup.py test - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html +test: check-style-python check-style-cpp run-tests-only ## check style and run tests release: dist ## package and upload a release twine upload dist/* dist: clean ## builds source and wheel package - ${python_var} setup.py sdist - ${python_var} setup.py bdist_wheel + pipenv run python setup.py sdist + pipenv run python setup.py bdist_wheel ls -l dist install: clean ## install the package to the active Python's site-packages - ${python_var} setup.py install + pipenv run python setup.py install diff --git a/Pipfile b/Pipfile index 77a970d1..19137135 100644 --- a/Pipfile +++ b/Pipfile @@ -6,10 +6,8 @@ verify_ssl = true [dev-packages] setuptools = "*" bumpversion = "==0.5.3" -tox = "==3.5.2" pytest = "*" -pycodestyle = "*" +black = "*" twine = "*" -coverage = "*" [packages] diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index d84b52a6..00000000 --- a/Pipfile.lock +++ /dev/null @@ -1,351 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "943465b7bb7b3f5558dda3b7d35ffd5d71b27fe19f1bf38c66d7131cc62f2f5b" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": {}, - "develop": { - "appdirs": { - "hashes": [ - "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", - "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" - ], - "version": "==1.4.4" - }, - "attrs": { - "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" - ], - "version": "==19.3.0" - }, - "bleach": { - "hashes": [ - "sha256:2bce3d8fab545a6528c8fa5d9f9ae8ebc85a56da365c7f85180bfe96a35ef22f", - "sha256:3c4c520fdb9db59ef139915a5db79f8b51bc2a7257ea0389f30c846883430a4b" - ], - "version": "==3.1.5" - }, - "bumpversion": { - "hashes": [ - "sha256:6744c873dd7aafc24453d8b6a1a0d6d109faf63cd0cd19cb78fd46e74932c77e", - "sha256:6753d9ff3552013e2130f7bc03c1007e24473b4835952679653fb132367bdd57" - ], - "index": "pypi", - "version": "==0.5.3" - }, - "certifi": { - "hashes": [ - "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", - "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" - ], - "version": "==2020.4.5.1" - }, - "cffi": { - "hashes": [ - "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", - "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", - "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", - "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", - "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", - "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", - "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", - "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", - "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", - "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", - "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", - "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", - "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", - "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", - "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", - "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", - "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", - "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", - "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", - "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", - "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", - "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", - "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", - "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", - "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", - "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", - "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", - "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" - ], - "version": "==1.14.0" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "cryptography": { - "hashes": [ - "sha256:091d31c42f444c6f519485ed528d8b451d1a0c7bf30e8ca583a0cac44b8a0df6", - "sha256:18452582a3c85b96014b45686af264563e3e5d99d226589f057ace56196ec78b", - "sha256:1dfa985f62b137909496e7fc182dac687206d8d089dd03eaeb28ae16eec8e7d5", - "sha256:1e4014639d3d73fbc5ceff206049c5a9a849cefd106a49fa7aaaa25cc0ce35cf", - "sha256:22e91636a51170df0ae4dcbd250d318fd28c9f491c4e50b625a49964b24fe46e", - "sha256:3b3eba865ea2754738616f87292b7f29448aec342a7c720956f8083d252bf28b", - "sha256:651448cd2e3a6bc2bb76c3663785133c40d5e1a8c1a9c5429e4354201c6024ae", - "sha256:726086c17f94747cedbee6efa77e99ae170caebeb1116353c6cf0ab67ea6829b", - "sha256:844a76bc04472e5135b909da6aed84360f522ff5dfa47f93e3dd2a0b84a89fa0", - "sha256:88c881dd5a147e08d1bdcf2315c04972381d026cdb803325c03fe2b4a8ed858b", - "sha256:96c080ae7118c10fcbe6229ab43eb8b090fccd31a09ef55f83f690d1ef619a1d", - "sha256:a0c30272fb4ddda5f5ffc1089d7405b7a71b0b0f51993cb4e5dbb4590b2fc229", - "sha256:bb1f0281887d89617b4c68e8db9a2c42b9efebf2702a3c5bf70599421a8623e3", - "sha256:c447cf087cf2dbddc1add6987bbe2f767ed5317adb2d08af940db517dd704365", - "sha256:c4fd17d92e9d55b84707f4fd09992081ba872d1a0c610c109c18e062e06a2e55", - "sha256:d0d5aeaedd29be304848f1c5059074a740fa9f6f26b84c5b63e8b29e73dfc270", - "sha256:daf54a4b07d67ad437ff239c8a4080cfd1cc7213df57d33c97de7b4738048d5e", - "sha256:e993468c859d084d5579e2ebee101de8f5a27ce8e2159959b6673b418fd8c785", - "sha256:f118a95c7480f5be0df8afeb9a11bd199aa20afab7a96bcf20409b411a3a85f0" - ], - "version": "==2.9.2" - }, - "distlib": { - "hashes": [ - "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21" - ], - "version": "==0.3.0" - }, - "docutils": { - "hashes": [ - "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", - "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" - ], - "version": "==0.16" - }, - "filelock": { - "hashes": [ - "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", - "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" - ], - "version": "==3.0.12" - }, - "idna": { - "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" - ], - "version": "==2.9" - }, - "importlib-metadata": { - "hashes": [ - "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", - "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e" - ], - "markers": "python_version < '3.8'", - "version": "==1.6.0" - }, - "importlib-resources": { - "hashes": [ - "sha256:6f87df66833e1942667108628ec48900e02a4ab4ad850e25fbf07cb17cf734ca", - "sha256:85dc0b9b325ff78c8bef2e4ff42616094e16b98ebd5e3b50fe7e2f0bbcdcde49" - ], - "markers": "python_version < '3.7'", - "version": "==1.5.0" - }, - "jeepney": { - "hashes": [ - "sha256:3479b861cc2b6407de5188695fa1a8d57e5072d7059322469b62628869b8e36e", - "sha256:d6c6b49683446d2407d2fe3acb7a368a77ff063f9182fe427da15d622adc24cf" - ], - "markers": "sys_platform == 'linux'", - "version": "==0.4.3" - }, - "keyring": { - "hashes": [ - "sha256:3401234209015144a5d75701e71cb47239e552b0882313e9f51e8976f9e27843", - "sha256:c53e0e5ccde3ad34284a40ce7976b5b3a3d6de70344c3f8ee44364cc340976ec" - ], - "version": "==21.2.1" - }, - "more-itertools": { - "hashes": [ - "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", - "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" - ], - "version": "==8.3.0" - }, - "packaging": { - "hashes": [ - "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", - "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" - ], - "version": "==20.4" - }, - "pkginfo": { - "hashes": [ - "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", - "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" - ], - "version": "==1.5.0.1" - }, - "pluggy": { - "hashes": [ - "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", - "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" - ], - "version": "==0.13.1" - }, - "py": { - "hashes": [ - "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", - "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" - ], - "version": "==1.8.1" - }, - "pycodestyle": { - "hashes": [ - "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", - "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" - ], - "index": "pypi", - "version": "==2.6.0" - }, - "pycparser": { - "hashes": [ - "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", - "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" - ], - "version": "==2.20" - }, - "pygments": { - "hashes": [ - "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", - "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" - ], - "version": "==2.6.1" - }, - "pyparsing": { - "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" - ], - "version": "==2.4.7" - }, - "pytest": { - "hashes": [ - "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3", - "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698" - ], - "index": "pypi", - "version": "==5.4.2" - }, - "readme-renderer": { - "hashes": [ - "sha256:cbe9db71defedd2428a1589cdc545f9bd98e59297449f69d721ef8f1cfced68d", - "sha256:cc4957a803106e820d05d14f71033092537a22daa4f406dfbdd61177e0936376" - ], - "version": "==26.0" - }, - "requests": { - "hashes": [ - "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", - "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" - ], - "version": "==2.23.0" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", - "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" - ], - "version": "==0.9.1" - }, - "secretstorage": { - "hashes": [ - "sha256:15da8a989b65498e29be338b3b279965f1b8f09b9668bd8010da183024c8bff6", - "sha256:b5ec909dde94d4ae2fa26af7c089036997030f0cf0a5cb372b4cccabd81c143b" - ], - "markers": "sys_platform == 'linux'", - "version": "==3.1.2" - }, - "six": { - "hashes": [ - "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", - "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" - ], - "version": "==1.15.0" - }, - "toml": { - "hashes": [ - "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", - "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" - ], - "version": "==0.10.1" - }, - "tox": { - "hashes": [ - "sha256:217fb84aecf9792a98f93f07cfcaf014205a76c64e52bd7c2b4135458e6ad2a1", - "sha256:4baeb3d8ebdcd9f43afce38aa67d06f1165a87d221d5bb21e8b39a0d4880c134" - ], - "index": "pypi", - "version": "==3.5.2" - }, - "tqdm": { - "hashes": [ - "sha256:4733c4a10d0f2a4d098d801464bdaf5240c7dadd2a7fde4ee93b0a0efd9fb25e", - "sha256:acdafb20f51637ca3954150d0405ff1a7edde0ff19e38fb99a80a66210d2a28f" - ], - "version": "==4.46.0" - }, - "twine": { - "hashes": [ - "sha256:c1af8ca391e43b0a06bbc155f7f67db0bf0d19d284bfc88d1675da497a946124", - "sha256:d561a5e511f70275e5a485a6275ff61851c16ffcb3a95a602189161112d9f160" - ], - "index": "pypi", - "version": "==3.1.1" - }, - "urllib3": { - "hashes": [ - "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", - "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" - ], - "version": "==1.25.9" - }, - "virtualenv": { - "hashes": [ - "sha256:a116629d4e7f4d03433b8afa27f43deba09d48bc48f5ecefa4f015a178efb6cf", - "sha256:a730548b27366c5e6cbdf6f97406d861cccece2e22275e8e1a757aeff5e00c70" - ], - "version": "==20.0.21" - }, - "wcwidth": { - "hashes": [ - "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1", - "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1" - ], - "version": "==0.1.9" - }, - "webencodings": { - "hashes": [ - "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", - "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" - ], - "version": "==0.5.1" - }, - "zipp": { - "hashes": [ - "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", - "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96" - ], - "markers": "python_version < '3.8'", - "version": "==3.1.0" - } - } -} diff --git a/build_PyDP.sh b/build_PyDP.sh index 3ace6ac6..fee7b2ca 100755 --- a/build_PyDP.sh +++ b/build_PyDP.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/bin/bash -pipenv run pipenv install --dev --skip-lock -pipenv run bazel build src/python:bindings_test --verbose_failures +pipenv install --dev --skip-lock +bazel build src/python:bindings_test --verbose_failures find ./ -name pydp.so -print0 | xargs -0 -I {} rm {} cp -f ./bazel-bin/src/bindings/pydp.so ./pydp diff --git a/contributing.md b/contributing.md index e5cde8fd..5646e60b 100644 --- a/contributing.md +++ b/contributing.md @@ -9,7 +9,7 @@ All our development is done using Git and Github. If you're not too familiar wit ### Slack A great first place to join the Community is the Slack workspace . -- For support in using this library, please join the **#lib_pydp** Slack channel. +- For support in using this library, please join the **#lib_pydp** Slack channel. - If you’d like to follow along with any code changes to the library, please join the **#code_dp_python** Slack channel. ### Issues @@ -76,7 +76,7 @@ If you want to use a specific version of python3 please refer to the Build the python library: ``` -$ ./build_PyDP.sh +$ make build ``` Run the test example: @@ -119,11 +119,11 @@ To run the image: $ docker run --rm -it pydp:test ``` -Docker Run will allow to interactively with the files; to check if your code works +Docker Run will allow to interact with the files; to check if your code works perfectly, you can type: ``` -$ make test-all +$ make test ``` To run the code, you can open python interpreter inside the container by typing @@ -146,7 +146,7 @@ If you are new to the project and want to get into the code, we recommend pickin ### Issue Allocation -Each issue someone is currently working on should have an assignee. If you want to contribute to an issue someone else is already working on please make sure to get in touch with that person via slack or GitHub and organize the workflow collaboratively. +Each issue someone is currently working on should have an assignee. If you want to contribute to an issue someone else is already working on please make sure to get in touch with that person via slack or GitHub and organize the workflow collaboratively. If you want to work on an open issue, please post a comment telling that you will work on that issue, we will assign you as the assignee then. diff --git a/prereqs_linux.sh b/prereqs_linux.sh index 543c0151..9267e9df 100755 --- a/prereqs_linux.sh +++ b/prereqs_linux.sh @@ -27,7 +27,7 @@ echo "Checking for pipenv" if python3 -c "import pipenv" &> /dev/null; then echo "pipenv is already installed" else - echo "installing pipenv" + echo "Installing pipenv" pip3 install pipenv fi @@ -57,4 +57,4 @@ cd - mv third_party/differential-privacy/cc/WORKSPACE.bazel third_party/differential-privacy/cc/WORKSPACE # Removing the java part -rm -rf third_party/differential-privacy/java third_party/differential-privacy/examples/java \ No newline at end of file +rm -rf third_party/differential-privacy/java third_party/differential-privacy/examples/java diff --git a/run-clang-format.py b/run-clang-format.py new file mode 100755 index 00000000..e10f6d08 --- /dev/null +++ b/run-clang-format.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python +"""A wrapper script around clang-format, suitable for linting multiple files +and to use for continuous integration. + +This is an alternative API for the clang-format command line. +It runs over multiple files and directories in parallel. +A diff output is produced and a sensible exit code is returned. + +MIT License + +Copyright (c) 2017 Guillaume Papin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from __future__ import print_function, unicode_literals + +import argparse +import codecs +import difflib +import fnmatch +import io +import errno +import multiprocessing +import os +import signal +import subprocess +import sys +import traceback + +from functools import partial + +try: + from subprocess import DEVNULL # py3k +except ImportError: + DEVNULL = open(os.devnull, "wb") + + +DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" +DEFAULT_CLANG_FORMAT_IGNORE = ".clang-format-ignore" + + +class ExitStatus: + SUCCESS = 0 + DIFF = 1 + TROUBLE = 2 + + +def excludes_from_file(ignore_file): + excludes = [] + try: + with io.open(ignore_file, "r", encoding="utf-8") as f: + for line in f: + if line.startswith("#"): + # ignore comments + continue + pattern = line.rstrip() + if not pattern: + # allow empty lines + continue + excludes.append(pattern) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + return excludes + + +def list_files(files, recursive=False, extensions=None, exclude=None): + if extensions is None: + extensions = [] + if exclude is None: + exclude = [] + + out = [] + for file in files: + if recursive and os.path.isdir(file): + for dirpath, dnames, fnames in os.walk(file): + fpaths = [os.path.join(dirpath, fname) for fname in fnames] + for pattern in exclude: + # os.walk() supports trimming down the dnames list + # by modifying it in-place, + # to avoid unnecessary directory listings. + dnames[:] = [ + x + for x in dnames + if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) + ] + fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)] + for f in fpaths: + ext = os.path.splitext(f)[1][1:] + if ext in extensions: + out.append(f) + else: + out.append(file) + return out + + +def make_diff(file, original, reformatted): + return list( + difflib.unified_diff( + original, + reformatted, + fromfile="{}\t(original)".format(file), + tofile="{}\t(reformatted)".format(file), + n=3, + ) + ) + + +class DiffError(Exception): + def __init__(self, message, errs=None): + super(DiffError, self).__init__(message) + self.errs = errs or [] + + +class UnexpectedError(Exception): + def __init__(self, message, exc=None): + super(UnexpectedError, self).__init__(message) + self.formatted_traceback = traceback.format_exc() + self.exc = exc + + +def run_clang_format_diff_wrapper(args, file): + try: + ret = run_clang_format_diff(args, file) + return ret + except DiffError: + raise + except Exception as e: + raise UnexpectedError("{}: {}: {}".format(file, e.__class__.__name__, e), e) + + +def run_clang_format_diff(args, file): + try: + with io.open(file, "r", encoding="utf-8") as f: + original = f.readlines() + except IOError as exc: + raise DiffError(str(exc)) + invocation = [args.clang_format_executable, file] + + # Use of utf-8 to decode the process output. + # + # Hopefully, this is the correct thing to do. + # + # It's done due to the following assumptions (which may be incorrect): + # - clang-format will returns the bytes read from the files as-is, + # without conversion, and it is already assumed that the files use utf-8. + # - if the diagnostics were internationalized, they would use utf-8: + # > Adding Translations to Clang + # > + # > Not possible yet! + # > Diagnostic strings should be written in UTF-8, + # > the client can translate to the relevant code page if needed. + # > Each translation completely replaces the format string + # > for the diagnostic. + # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation + # + # It's not pretty, due to Python 2 & 3 compatibility. + encoding_py3 = {} + if sys.version_info[0] >= 3: + encoding_py3["encoding"] = "utf-8" + + try: + proc = subprocess.Popen( + invocation, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + **encoding_py3 + ) + except OSError as exc: + raise DiffError( + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(invocation), exc + ) + ) + proc_stdout = proc.stdout + proc_stderr = proc.stderr + if sys.version_info[0] < 3: + # make the pipes compatible with Python 3, + # reading lines should output unicode + encoding = "utf-8" + proc_stdout = codecs.getreader(encoding)(proc_stdout) + proc_stderr = codecs.getreader(encoding)(proc_stderr) + # hopefully the stderr pipe won't get full and block the process + outs = list(proc_stdout.readlines()) + errs = list(proc_stderr.readlines()) + proc.wait() + if proc.returncode: + raise DiffError( + "Command '{}' returned non-zero exit status {}".format( + subprocess.list2cmdline(invocation), proc.returncode + ), + errs, + ) + return make_diff(file, original, outs), errs + + +def bold_red(s): + return "\x1b[1m\x1b[31m" + s + "\x1b[0m" + + +def colorize(diff_lines): + def bold(s): + return "\x1b[1m" + s + "\x1b[0m" + + def cyan(s): + return "\x1b[36m" + s + "\x1b[0m" + + def green(s): + return "\x1b[32m" + s + "\x1b[0m" + + def red(s): + return "\x1b[31m" + s + "\x1b[0m" + + for line in diff_lines: + if line[:4] in ["--- ", "+++ "]: + yield bold(line) + elif line.startswith("@@ "): + yield cyan(line) + elif line.startswith("+"): + yield green(line) + elif line.startswith("-"): + yield red(line) + else: + yield line + + +def print_diff(diff_lines, use_color): + if use_color: + diff_lines = colorize(diff_lines) + if sys.version_info[0] < 3: + sys.stdout.writelines((l.encode("utf-8") for l in diff_lines)) + else: + sys.stdout.writelines(diff_lines) + + +def print_trouble(prog, message, use_colors): + error_text = "error:" + if use_colors: + error_text = bold_red(error_text) + print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--clang-format-executable", + metavar="EXECUTABLE", + help="path to the clang-format executable", + default="clang-format", + ) + parser.add_argument( + "--extensions", + help="comma separated list of file extensions (default: {})".format( + DEFAULT_EXTENSIONS + ), + default=DEFAULT_EXTENSIONS, + ) + parser.add_argument( + "-r", + "--recursive", + action="store_true", + help="run recursively over directories", + ) + parser.add_argument("files", metavar="file", nargs="+") + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="disable output, useful for the exit code", + ) + parser.add_argument( + "-j", + metavar="N", + type=int, + default=0, + help="run N clang-format jobs in parallel" " (default number of cpus + 1)", + ) + parser.add_argument( + "--color", + default="auto", + choices=["auto", "always", "never"], + help="show colored diff (default: auto)", + ) + parser.add_argument( + "-e", + "--exclude", + metavar="PATTERN", + action="append", + default=[], + help="exclude paths matching the given glob-like pattern(s)" + " from recursive search", + ) + + args = parser.parse_args() + + # use default signal handling, like diff return SIGINT value on ^C + # https://bugs.python.org/issue14229#msg156446 + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + signal.SIGPIPE + except AttributeError: + # compatibility, SIGPIPE does not exist on Windows + pass + else: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + colored_stdout = False + colored_stderr = False + if args.color == "always": + colored_stdout = True + colored_stderr = True + elif args.color == "auto": + colored_stdout = sys.stdout.isatty() + colored_stderr = sys.stderr.isatty() + + version_invocation = [args.clang_format_executable, str("--version")] + try: + subprocess.check_call(version_invocation, stdout=DEVNULL) + except subprocess.CalledProcessError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + return ExitStatus.TROUBLE + except OSError as e: + print_trouble( + parser.prog, + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(version_invocation), e + ), + use_colors=colored_stderr, + ) + return ExitStatus.TROUBLE + + retcode = ExitStatus.SUCCESS + + excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) + excludes.extend(args.exclude) + + files = list_files( + args.files, + recursive=args.recursive, + exclude=excludes, + extensions=args.extensions.split(","), + ) + + if not files: + return + + njobs = args.j + if njobs == 0: + njobs = multiprocessing.cpu_count() + 1 + njobs = min(len(files), njobs) + + if njobs == 1: + # execute directly instead of in a pool, + # less overhead, simpler stacktraces + it = (run_clang_format_diff_wrapper(args, file) for file in files) + pool = None + else: + pool = multiprocessing.Pool(njobs) + it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files) + while True: + try: + outs, errs = next(it) + except StopIteration: + break + except DiffError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + retcode = ExitStatus.TROUBLE + sys.stderr.writelines(e.errs) + except UnexpectedError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + sys.stderr.write(e.formatted_traceback) + retcode = ExitStatus.TROUBLE + # stop at the first unexpected error, + # something could be very wrong, + # don't process all files unnecessarily + if pool: + pool.terminate() + break + else: + sys.stderr.writelines(errs) + if outs == []: + continue + if not args.quiet: + print_diff(outs, use_color=colored_stdout) + if retcode == ExitStatus.SUCCESS: + retcode = ExitStatus.DIFF + return retcode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.cfg b/setup.cfg index 69d77c87..4267ed48 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,10 +11,5 @@ replace = version="{new_version}" search = __version__ = "{current_version}" replace = __version__ = "{new_version}" -[pycodestyle] -max-line-length = 88 -exclude = build,.tox -ignore = E231 - [aliases] # Define setup.py command aliases here diff --git a/tests/__init__.py b/tests/__init__.py index 61f4ab4c..e69de29b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,13 +0,0 @@ -import unittest -import pycodestyle - - -class TestCodeFormat(unittest.TestCase): - def test_conformance(self): - """Test that we conform to PEP-8.""" - style = pycodestyle.StyleGuide(quiet=True) - # style. - style.input_dir("algorithms") - style.input_dir("base") - result = style.check_files() - assert result.total_errors == 0 diff --git a/third_party/differential-privacy b/third_party/differential-privacy index e819e03a..b7f4c39d 160000 --- a/third_party/differential-privacy +++ b/third_party/differential-privacy @@ -1 +1 @@ -Subproject commit e819e03a20f9d7b0a30f2547c00ba74065b3f549 +Subproject commit b7f4c39d9f73d67b34cdbd1b8483e5f72072fc73 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index c2fc88d7..00000000 --- a/tox.ini +++ /dev/null @@ -1,11 +0,0 @@ -[tox] -envlist = py35, py36, py37, py38 - -[testenv] -changedir = tests -deps = pytest -# change pytest tempdir and add posargs from command line -; commands = pytest - -commands = python setup.py test -