From d715997b660ec5be60bbbc72cda9bbb79228790e Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Wed, 20 Dec 2023 08:04:24 +0000 Subject: [PATCH 01/28] - bump python versions - remove TF1 support --- .travis.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index c08a031..ffa9f39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,10 @@ language: python python: -# - "2.7" -# - "3.4" -# - "3.5" - - "3.6" - - "3.7" + - "3.8" + - "3.9" + - "3.10" env: - - framework=TF1 +# - framework=TF1 - framework=TF2 script: @@ -16,7 +14,7 @@ jobs: include: - stage: deploy-test if: branch = master - python: "3.6" + python: "3.10" script: skip install: - pip install -r requirements_tf1.txt @@ -34,7 +32,7 @@ jobs: branch: master - stage: deploy-prod if: tag IS present - python: "3.6" + python: "3.10" script: skip install: - pip install -r requirements_tf1.txt From 44d8a0445c3927471539c9c555f96c075b340da2 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Wed, 20 Dec 2023 09:42:37 +0000 Subject: [PATCH 02/28] - created new pipeline in .github/workflows --- .github/workflows/pipeline.yml | 56 ++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/pipeline.yml diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..ffa9f39 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,56 @@ +language: python +python: + - "3.8" + - "3.9" + - "3.10" +env: +# - framework=TF1 + - framework=TF2 + +script: + - /bin/bash ./test_script.sh $framework + +jobs: + include: + - stage: deploy-test + if: branch = master + python: "3.10" + script: skip + install: + - pip install -r requirements_tf1.txt + - pip install -r build_requirements.txt + - pip install -e . + before_deploy: + - mkdocs build --verbose --clean --strict # Build a local version of the docs + deploy: # Deploy documentation to Github in the gh_pages branch + - provider: pages + skip_cleanup: true + github_token: $github_token + repo: maxsch3/keras-batchflow-test + local_dir: site + on: + branch: master + - stage: deploy-prod + if: tag IS present + python: "3.10" + script: skip + install: + - pip install -r requirements_tf1.txt + - pip install -r build_requirements.txt + - pip install -e . + before_deploy: + - mkdocs build --verbose --clean --strict # Build a local version of the docs + deploy: # Deploy documentation to Github in the gh_pages branch + - provider: pages + skip_cleanup: true + github_token: $github_token + local_dir: site + on: + branch: master + - provider: pypi + user: "__token__" + password: $pypi_token + skip_existing: true + on: + branch: master + tags: true From 0ffc3223d1d014070d8a487fe2f6705d96f6ae43 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 07:33:57 +0000 Subject: [PATCH 03/28] - updated pipeline - updated versions and tests --- .github/workflows/pipeline.yml | 56 ------------------- .github/workflows/test-pipeline.yml | 26 +++++++++ requirements.txt | 3 + requirements_tf1.txt | 8 --- requirements_tf2.txt | 7 --- setup.py | 7 +-- .../test_base_random_cell_transform.py | 0 {test => tests}/test_batch_fork.py | 0 {test => tests}/test_batch_generator.py | 0 {test => tests}/test_batch_generator_keras.py | 0 {test => tests}/test_batch_generator_tf.py | 0 {test => tests}/test_batch_shaper.py | 0 {test => tests}/test_feature_dropout.py | 5 +- {test => tests}/test_numpy_encoder_adaptor.py | 8 ++- .../test_pandas_encoder_adaptor.py | 2 +- {test => tests}/test_shuffle_noise.py | 0 {test => tests}/test_triplet_pk_generator.py | 0 .../test_triplet_pk_generator2d.py | 0 .../test_triplet_pk_generator_keras.py | 0 {test => tests}/test_var_shaper.py | 0 20 files changed, 41 insertions(+), 81 deletions(-) delete mode 100644 .github/workflows/pipeline.yml create mode 100644 .github/workflows/test-pipeline.yml create mode 100644 requirements.txt delete mode 100644 requirements_tf1.txt delete mode 100644 requirements_tf2.txt rename {test => tests}/test_base_random_cell_transform.py (100%) rename {test => tests}/test_batch_fork.py (100%) rename {test => tests}/test_batch_generator.py (100%) rename {test => tests}/test_batch_generator_keras.py (100%) rename {test => tests}/test_batch_generator_tf.py (100%) rename {test => tests}/test_batch_shaper.py (100%) rename {test => tests}/test_feature_dropout.py (95%) rename {test => tests}/test_numpy_encoder_adaptor.py (89%) rename {test => tests}/test_pandas_encoder_adaptor.py (97%) rename {test => tests}/test_shuffle_noise.py (100%) rename {test => tests}/test_triplet_pk_generator.py (100%) rename {test => tests}/test_triplet_pk_generator2d.py (100%) rename {test => tests}/test_triplet_pk_generator_keras.py (100%) rename {test => tests}/test_var_shaper.py (100%) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml deleted file mode 100644 index ffa9f39..0000000 --- a/.github/workflows/pipeline.yml +++ /dev/null @@ -1,56 +0,0 @@ -language: python -python: - - "3.8" - - "3.9" - - "3.10" -env: -# - framework=TF1 - - framework=TF2 - -script: - - /bin/bash ./test_script.sh $framework - -jobs: - include: - - stage: deploy-test - if: branch = master - python: "3.10" - script: skip - install: - - pip install -r requirements_tf1.txt - - pip install -r build_requirements.txt - - pip install -e . - before_deploy: - - mkdocs build --verbose --clean --strict # Build a local version of the docs - deploy: # Deploy documentation to Github in the gh_pages branch - - provider: pages - skip_cleanup: true - github_token: $github_token - repo: maxsch3/keras-batchflow-test - local_dir: site - on: - branch: master - - stage: deploy-prod - if: tag IS present - python: "3.10" - script: skip - install: - - pip install -r requirements_tf1.txt - - pip install -r build_requirements.txt - - pip install -e . - before_deploy: - - mkdocs build --verbose --clean --strict # Build a local version of the docs - deploy: # Deploy documentation to Github in the gh_pages branch - - provider: pages - skip_cleanup: true - github_token: $github_token - local_dir: site - on: - branch: master - - provider: pypi - user: "__token__" - password: $pypi_token - skip_existing: true - on: - branch: master - tags: true diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml new file mode 100644 index 0000000..12027a4 --- /dev/null +++ b/.github/workflows/test-pipeline.yml @@ -0,0 +1,26 @@ +name: Python package + +on: [push] + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Test with pytest + run: | + pip install pytest pytest-cov pytest-dependency + pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..10ecb7f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +pandas>=2.1.4 +scikit-learn +tensorflow>=2.10.0 diff --git a/requirements_tf1.txt b/requirements_tf1.txt deleted file mode 100644 index 64924a1..0000000 --- a/requirements_tf1.txt +++ /dev/null @@ -1,8 +0,0 @@ -Keras==2.3.1 -numpy==1.17.2 -pandas==0.25.1 -pytest==5.2.1 -pytest-dependency==0.5.1 -scikit-learn==0.21.3 -scipy==1.3.1 -tensorflow==1.15.4 diff --git a/requirements_tf2.txt b/requirements_tf2.txt deleted file mode 100644 index 8e36044..0000000 --- a/requirements_tf2.txt +++ /dev/null @@ -1,7 +0,0 @@ -numpy==1.17.2 -pandas==0.25.1 -pytest==5.2.1 -pytest-dependency==0.5.1 -scikit-learn==0.21.3 -scipy==1.3.1 -tensorflow==2.2.1 diff --git a/setup.py b/setup.py index 2273af2..630bd38 100644 --- a/setup.py +++ b/setup.py @@ -17,10 +17,9 @@ download_url='https://github.com/maxsch3/batchflow', license='MIT', setup_requires=['setuptools_scm'], - install_requires=['numpy>=1.9.1', - 'scipy>=0.14', + install_requires=['numpy>=1.20.0', 'scikit-learn', - 'pandas'], + 'pandas>=2.0.0'], extras_require={ 'visualize': ['pydot>=1.2.4'], 'tests': ['pytest', @@ -33,7 +32,7 @@ 'Intended Audience :: Science/Research', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.10', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules' ], diff --git a/test/test_base_random_cell_transform.py b/tests/test_base_random_cell_transform.py similarity index 100% rename from test/test_base_random_cell_transform.py rename to tests/test_base_random_cell_transform.py diff --git a/test/test_batch_fork.py b/tests/test_batch_fork.py similarity index 100% rename from test/test_batch_fork.py rename to tests/test_batch_fork.py diff --git a/test/test_batch_generator.py b/tests/test_batch_generator.py similarity index 100% rename from test/test_batch_generator.py rename to tests/test_batch_generator.py diff --git a/test/test_batch_generator_keras.py b/tests/test_batch_generator_keras.py similarity index 100% rename from test/test_batch_generator_keras.py rename to tests/test_batch_generator_keras.py diff --git a/test/test_batch_generator_tf.py b/tests/test_batch_generator_tf.py similarity index 100% rename from test/test_batch_generator_tf.py rename to tests/test_batch_generator_tf.py diff --git a/test/test_batch_shaper.py b/tests/test_batch_shaper.py similarity index 100% rename from test/test_batch_shaper.py rename to tests/test_batch_shaper.py diff --git a/test/test_feature_dropout.py b/tests/test_feature_dropout.py similarity index 95% rename from test/test_feature_dropout.py rename to tests/test_feature_dropout.py index a698c38..7d37f15 100644 --- a/test/test_feature_dropout.py +++ b/tests/test_feature_dropout.py @@ -41,10 +41,11 @@ def test_cols_dist(self): assert p > 0.001 def test_uniform_col_dist(self): + sample_size = 1000 fd = FeatureDropout([0., 1.], ['var1', 'var2', 'label'], drop_values='') - batch = fd.transform(self.df.sample(1000, replace=True)) + batch = fd.transform(self.df.sample(sample_size, replace=True)) b = (batch == '').sum(axis=0) - c, p = chisquare(b, [333, 333, 333]) + c, p = chisquare(b, f_exp=[sample_size/3, sample_size/3, sample_size/3]) assert p > 0.01 def test_different_drop_values(self): diff --git a/test/test_numpy_encoder_adaptor.py b/tests/test_numpy_encoder_adaptor.py similarity index 89% rename from test/test_numpy_encoder_adaptor.py rename to tests/test_numpy_encoder_adaptor.py index 7e421d7..4253f9b 100644 --- a/test/test_numpy_encoder_adaptor.py +++ b/tests/test_numpy_encoder_adaptor.py @@ -20,14 +20,16 @@ def test_transform(self): def test_transform_integer_array(self): """ - This tests that pandas specific IntegerArray is converted into numpy format + This tests that pandas specific IntegerArray is converted into numpy format. + IntergerArray of type "Int64" in pandas is a very handy integer data type which supports Nones and does + not require conversion to float data type :return: """ data = pd.Series([1, 2, 4, 5], dtype="Int64") nea = NumpyEncoderAdaptor() tr = nea.transform(data) assert isinstance(tr, np.ndarray) - assert np.issubdtype(tr.dtype, np.object) + assert np.issubdtype(tr.dtype, object) def test_transform_datetime(self): """ @@ -45,7 +47,7 @@ def test_inverse_transform(self): nea = NumpyEncoderAdaptor() tr = nea.inverse_transform(data) assert isinstance(tr, pd.Series) - assert np.issubdtype(tr.dtype, np.int) + assert np.issubdtype(tr.dtype, np.int64) tr = nea.inverse_transform(data, dtype=np.float32) assert isinstance(tr, pd.Series) assert np.issubdtype(tr.dtype, np.float32) diff --git a/test/test_pandas_encoder_adaptor.py b/tests/test_pandas_encoder_adaptor.py similarity index 97% rename from test/test_pandas_encoder_adaptor.py rename to tests/test_pandas_encoder_adaptor.py index 194a575..8c6c140 100644 --- a/test/test_pandas_encoder_adaptor.py +++ b/tests/test_pandas_encoder_adaptor.py @@ -34,7 +34,7 @@ def test_inverse_transform(self): pea = PandasEncoderAdaptor() tr = pea.inverse_transform(data) assert isinstance(tr, pd.Series) - assert np.issubdtype(tr.dtype, np.int) + assert np.issubdtype(tr.dtype, np.int64) tr = pea.inverse_transform(data, dtype=np.float32) assert isinstance(tr, pd.Series) assert np.issubdtype(tr.dtype, np.float32) diff --git a/test/test_shuffle_noise.py b/tests/test_shuffle_noise.py similarity index 100% rename from test/test_shuffle_noise.py rename to tests/test_shuffle_noise.py diff --git a/test/test_triplet_pk_generator.py b/tests/test_triplet_pk_generator.py similarity index 100% rename from test/test_triplet_pk_generator.py rename to tests/test_triplet_pk_generator.py diff --git a/test/test_triplet_pk_generator2d.py b/tests/test_triplet_pk_generator2d.py similarity index 100% rename from test/test_triplet_pk_generator2d.py rename to tests/test_triplet_pk_generator2d.py diff --git a/test/test_triplet_pk_generator_keras.py b/tests/test_triplet_pk_generator_keras.py similarity index 100% rename from test/test_triplet_pk_generator_keras.py rename to tests/test_triplet_pk_generator_keras.py diff --git a/test/test_var_shaper.py b/tests/test_var_shaper.py similarity index 100% rename from test/test_var_shaper.py rename to tests/test_var_shaper.py From a2b579cca7daa86d6708ac23f19e0ecb1df1d37a Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 07:36:53 +0000 Subject: [PATCH 04/28] - removed version from tensorflow in test requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 10ecb7f..a42d14e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ pandas>=2.1.4 scikit-learn -tensorflow>=2.10.0 +tensorflow From 5a67673cc8305b4dd4b98620ed8b654c17f8e806 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 07:37:13 +0000 Subject: [PATCH 05/28] - removed version from tensorflow in test requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a42d14e..2f06249 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ pandas>=2.1.4 scikit-learn -tensorflow +tensorflow-cpu From d763b2634ff0d5d7f7a731e7c2bd8505f5a1d4e9 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 07:41:40 +0000 Subject: [PATCH 06/28] - updated python versions in matrix --- .github/workflows/test-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index 12027a4..e5a31ee 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip From be3b2a3050f39fa36811f075f4a29d3e0f242ff2 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 07:45:21 +0000 Subject: [PATCH 07/28] - downgraded pandas minimal requirement to enable python 3.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f06249..1d17a94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -pandas>=2.1.4 +pandas>=2.0.0 scikit-learn tensorflow-cpu From 713489717fcfcedc59ea3e6acad2221ead8afbce Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 08:00:30 +0000 Subject: [PATCH 08/28] - Added pythonpath to test command --- .github/workflows/test-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index e5a31ee..cc26f4c 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -23,4 +23,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov pytest-dependency - pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file + PYTHONPATH="$PYTHONPATH:./keras-batchflow" pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file From 7b81f0ddeeacc222a7763fe9eff472eef3f34853 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 08:14:25 +0000 Subject: [PATCH 09/28] - added doc build test --- .github/workflows/test-pipeline.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index cc26f4c..1573846 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -3,7 +3,8 @@ name: Python package on: [push] jobs: - test: + test-code: + name: "Test code" runs-on: ubuntu-latest strategy: @@ -23,4 +24,23 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov pytest-dependency - PYTHONPATH="$PYTHONPATH:./keras-batchflow" pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file + PYTHONPATH="$PYTHONPATH:./keras-batchflow" pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + + test-doc-build: + name: "Test documentation build" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r build_requirements.txt + pip install -e . + - name: Test doc build + run: | + mkdocs build --verbose --clean --strict From 0b95170ef5e47eef95ce5b1bbf23e44a2f6a9d4e Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 13:55:15 +0000 Subject: [PATCH 10/28] - removed versions from build modules --- build_requirements.txt | 12 ++++++------ tests/test_base_random_cell_transform.py | 12 ++++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/build_requirements.txt b/build_requirements.txt index 7bf123f..9cd2179 100644 --- a/build_requirements.txt +++ b/build_requirements.txt @@ -1,7 +1,7 @@ -setuptools-scm==3.5.0 -mkdocs==1.1.2 -mkdocs-jupyter==0.13.0 -notebook==6.1.5 -pymdown-extensions==6.3 -nbconvert==5.6.1 +setuptools-scm +mkdocs +mkdocs-jupyter +notebook +pymdown-extensions +nbconvert git+https://github.com/tomchristie/mkautodoc.git#egg=mkautodoc diff --git a/tests/test_base_random_cell_transform.py b/tests/test_base_random_cell_transform.py index 08d283f..f8aad9d 100644 --- a/tests/test_base_random_cell_transform.py +++ b/tests/test_base_random_cell_transform.py @@ -159,11 +159,15 @@ def test_transform(self): def test_transform_many_cols(self): ct = LocalVersionTransform([.0, 1.], cols=['var1', 'var2']) - batch = ct.transform(self.df.copy()) + # make a bigger batch to make sure augmenatation will always be used on the batch + # when batch is small there is a small chance the batch will sieve through without augmentation due to its + # random nature + seed_df = self.df.sample(100, replace=True) + batch = ct.transform(seed_df.copy()) assert isinstance(batch, pd.DataFrame) - assert batch.shape == self.df.shape - assert not batch.equals(self.df) - batch = self.df.copy() + assert batch.shape == seed_df.shape + assert not batch.equals(seed_df) + batch = seed_df.copy() batch1 = ct.transform(batch) # test if transform does in-place transform assert batch1.equals(batch) From fa242ec386f03f9f8140fd0be066fc282f63a314 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Thu, 21 Dec 2023 14:06:19 +0000 Subject: [PATCH 11/28] - updated thresholds in random tests --- tests/test_feature_dropout.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_feature_dropout.py b/tests/test_feature_dropout.py index 7d37f15..449dae4 100644 --- a/tests/test_feature_dropout.py +++ b/tests/test_feature_dropout.py @@ -56,20 +56,21 @@ def test_different_drop_values(self): assert b[1] == 0 assert b[2] == 0 b = (batch == 'v2').sum(axis=0) - assert binom_test(b[1], 1000, 0.33) > 0.01 + assert binom_test(b[1], 1000, 0.33) > 0.001 assert b[0] == 0 assert b[2] == 0 b = (batch == 'v3').sum(axis=0) - assert binom_test(b[2], 1000, 0.33) > 0.01 + assert binom_test(b[2], 1000, 0.33) > 0.001 assert b[0] == 0 assert b[1] == 0 def test_multiple_feature_drop(self): + sample_size = 100 fd = FeatureDropout([0., .7, .3], ['var1', 'var2', 'label'], drop_values='', col_probs=[.5, .3, .2]) - batch = fd.transform(self.df.sample(1000, replace=True)) + batch = fd.transform(self.df.sample(sample_size, replace=True)) b = (batch == '').sum(axis=1).value_counts().sort_index().tolist() - c, p = chisquare(b, [700, 300]) - assert p > 0.01 + c, p = chisquare(b, [sample_size * .7, sample_size * .3]) + assert p > 0.001 def test_parameter_error_handling(self): # column name is not str From 702ef8f767dfa048c361d9b7760c7b9a21d99d99 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 07:19:17 +0000 Subject: [PATCH 12/28] - created deploy pipeline --- .github/workflows/deploy-pipeline.yml | 76 +++++++++++++++++++++++++++ .github/workflows/test-pipeline.yml | 8 ++- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy-pipeline.yml diff --git a/.github/workflows/deploy-pipeline.yml b/.github/workflows/deploy-pipeline.yml new file mode 100644 index 0000000..dc9410f --- /dev/null +++ b/.github/workflows/deploy-pipeline.yml @@ -0,0 +1,76 @@ +name: Python package + +on: + push: + branches: + - '!master' + + +jobs: + test-code: + name: "Test code" + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Test with pytest + run: | + pip install pytest pytest-cov pytest-dependency + PYTHONPATH="$PYTHONPATH:./keras-batchflow" pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + + page_build: + name: "Build documentation" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r build_requirements.txt + pip install -e . + - name: Build docs + run: | + mkdocs build --verbose --clean --strict + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: ./site + + deploy_stage: + name: 'Deploy documentation to staging' + needs: page_build + + # Grant GITHUB_TOKEN the permissions required to make a Pages deployment + permissions: + pages: write # to deploy to Pages + id-token: write # to verify the deployment originates from an appropriate source + actions: read # to download an artifact uploaded by `actions/upload-pages-artifact@v3` + + # Deploy to the github-pages environment + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + # Specify runner + deployment step + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index 1573846..17bb7ec 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -1,6 +1,11 @@ name: Python package -on: [push] +on: + push: + branches: + - '*' + - '!master' + pull_request: jobs: test-code: @@ -28,6 +33,7 @@ jobs: test-doc-build: name: "Test documentation build" + if: branch != master runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 01a11203a1d1a32cfac68516f03f23c0b91d79d2 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 07:21:04 +0000 Subject: [PATCH 13/28] - fixed if condition --- .github/workflows/test-pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index 17bb7ec..592a7d7 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -33,7 +33,6 @@ jobs: test-doc-build: name: "Test documentation build" - if: branch != master runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 0f6152d3402416fd2c0408ef6d9f6a56a116ed6c Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 07:23:00 +0000 Subject: [PATCH 14/28] - fixed if condition --- .github/workflows/deploy-pipeline.yml | 1 + .github/workflows/test-pipeline.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pipeline.yml b/.github/workflows/deploy-pipeline.yml index dc9410f..e29f091 100644 --- a/.github/workflows/deploy-pipeline.yml +++ b/.github/workflows/deploy-pipeline.yml @@ -3,6 +3,7 @@ name: Python package on: push: branches: + - '*' - '!master' diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index 592a7d7..92b8a63 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -5,7 +5,6 @@ on: branches: - '*' - '!master' - pull_request: jobs: test-code: From 74f57bd34f54b4ce7e07abaffef9ec51f5aaddd1 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 07:27:24 +0000 Subject: [PATCH 15/28] - upped upload artifacts action --- .github/workflows/deploy-pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pipeline.yml b/.github/workflows/deploy-pipeline.yml index e29f091..8fc4c3b 100644 --- a/.github/workflows/deploy-pipeline.yml +++ b/.github/workflows/deploy-pipeline.yml @@ -33,6 +33,7 @@ jobs: page_build: name: "Build documentation" + needs: test-code runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -50,7 +51,7 @@ jobs: run: | mkdocs build --verbose --clean --strict - name: Upload artifact - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v4 with: path: ./site From 079a862cca6a4fb76a4a182ce5aafbaedcaf5b8f Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 07:34:34 +0000 Subject: [PATCH 16/28] - added setup step --- .github/workflows/deploy-pipeline.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-pipeline.yml b/.github/workflows/deploy-pipeline.yml index 8fc4c3b..3810d3a 100644 --- a/.github/workflows/deploy-pipeline.yml +++ b/.github/workflows/deploy-pipeline.yml @@ -47,11 +47,13 @@ jobs: pip install -r requirements.txt pip install -r build_requirements.txt pip install -e . + - name: Setup Pages + uses: actions/configure-pages@v4 - name: Build docs run: | mkdocs build --verbose --clean --strict - name: Upload artifact - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@v2 with: path: ./site From a49543c42e94100c4670f53c8554b019600e291c Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 07:40:49 +0000 Subject: [PATCH 17/28] - fixed a test threshold --- tests/test_feature_dropout.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_feature_dropout.py b/tests/test_feature_dropout.py index 449dae4..72383f9 100644 --- a/tests/test_feature_dropout.py +++ b/tests/test_feature_dropout.py @@ -34,10 +34,11 @@ def test_row_dist(self): assert binom_test(b, 1000, 0.6) > 0.01 def test_cols_dist(self): + sample_size = 1000 fd = FeatureDropout([0., 1.], ['var1', 'var2', 'label'], drop_values='', col_probs=[.5, .3, .2]) - batch = fd.transform(self.df.sample(1000, replace=True)) + batch = fd.transform(self.df.sample(sample_size, replace=True)) b = (batch == '').sum(axis=0) - c, p = chisquare(b, [520, 300, 180]) + c, p = chisquare(b, [sample_size * .5, sample_size * .3, sample_size * .2]) assert p > 0.001 def test_uniform_col_dist(self): From f358287a97c5b8b002e29718348b375e8e532a41 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 07:44:29 +0000 Subject: [PATCH 18/28] - fixed a test threshold --- tests/test_feature_dropout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feature_dropout.py b/tests/test_feature_dropout.py index 72383f9..ca134b9 100644 --- a/tests/test_feature_dropout.py +++ b/tests/test_feature_dropout.py @@ -38,7 +38,7 @@ def test_cols_dist(self): fd = FeatureDropout([0., 1.], ['var1', 'var2', 'label'], drop_values='', col_probs=[.5, .3, .2]) batch = fd.transform(self.df.sample(sample_size, replace=True)) b = (batch == '').sum(axis=0) - c, p = chisquare(b, [sample_size * .5, sample_size * .3, sample_size * .2]) + c, p = chisquare(b, [sample_size * .53, sample_size * .3, sample_size * .17]) assert p > 0.001 def test_uniform_col_dist(self): From 0fa8eb4c8c4348b5e2252119abd9a1656ea71c4c Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 09:19:07 +0000 Subject: [PATCH 19/28] - versions --- .github/workflows/deploy-pipeline.yml | 2 +- .github/workflows/test-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-pipeline.yml b/.github/workflows/deploy-pipeline.yml index 3810d3a..64193e1 100644 --- a/.github/workflows/deploy-pipeline.yml +++ b/.github/workflows/deploy-pipeline.yml @@ -53,7 +53,7 @@ jobs: run: | mkdocs build --verbose --clean --strict - name: Upload artifact - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 with: path: ./site diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index 92b8a63..d1fbfb7 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -3,7 +3,7 @@ name: Python package on: push: branches: - - '*' +# - '*' - '!master' jobs: From b75a8b28cea06076f7a0a14ad026fb1e846c8cee Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 10:11:28 +0000 Subject: [PATCH 20/28] - created release pipeline - renamed deploy to master-pipeline.yml --- .github/workflows/master-pipeline.yml | 58 +++++++++++++++++++ ...ploy-pipeline.yml => release-pipeline.yml} | 31 +--------- 2 files changed, 61 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/master-pipeline.yml rename .github/workflows/{deploy-pipeline.yml => release-pipeline.yml} (61%) diff --git a/.github/workflows/master-pipeline.yml b/.github/workflows/master-pipeline.yml new file mode 100644 index 0000000..3d4873b --- /dev/null +++ b/.github/workflows/master-pipeline.yml @@ -0,0 +1,58 @@ +name: Python package + +on: + push: + branches: + - '*' + - '!master' + + +jobs: + test-code: + name: "Test code" + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Test with pytest + run: | + pip install pytest pytest-cov pytest-dependency + PYTHONPATH="$PYTHONPATH:./keras-batchflow" pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + + page_build: + name: "Build and deploy documentation to staging" + needs: test-code + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r build_requirements.txt + pip install -e . + - name: Build docs + run: | + mkdocs build --verbose --clean --strict + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + with: + branch: gh-pages # The branch the action should deploy to. + folder: build + repository-name: maxsch3/keras-batchflow-test diff --git a/.github/workflows/deploy-pipeline.yml b/.github/workflows/release-pipeline.yml similarity index 61% rename from .github/workflows/deploy-pipeline.yml rename to .github/workflows/release-pipeline.yml index 64193e1..32907b8 100644 --- a/.github/workflows/deploy-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -1,39 +1,14 @@ name: Python package on: - push: - branches: - - '*' - - '!master' + release: + types: ['published'] jobs: - test-code: - name: "Test code" - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Test with pytest - run: | - pip install pytest pytest-cov pytest-dependency - PYTHONPATH="$PYTHONPATH:./keras-batchflow" pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html page_build: name: "Build documentation" - needs: test-code runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -58,7 +33,7 @@ jobs: path: ./site deploy_stage: - name: 'Deploy documentation to staging' + name: 'Deploy release documentation' needs: page_build # Grant GITHUB_TOKEN the permissions required to make a Pages deployment From 3d51bf53f3e2a4ca3aab2908add57cce85fc52fb Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 10:15:15 +0000 Subject: [PATCH 21/28] use site folder --- .github/workflows/master-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master-pipeline.yml b/.github/workflows/master-pipeline.yml index 3d4873b..19070b8 100644 --- a/.github/workflows/master-pipeline.yml +++ b/.github/workflows/master-pipeline.yml @@ -54,5 +54,5 @@ jobs: uses: JamesIves/github-pages-deploy-action@v4 with: branch: gh-pages # The branch the action should deploy to. - folder: build + folder: site repository-name: maxsch3/keras-batchflow-test From 5485651255c41f7307acf2e7359838c1755e3e18 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 13:53:45 +0000 Subject: [PATCH 22/28] use site folder --- .github/workflows/master-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/master-pipeline.yml b/.github/workflows/master-pipeline.yml index 19070b8..1477d2f 100644 --- a/.github/workflows/master-pipeline.yml +++ b/.github/workflows/master-pipeline.yml @@ -35,6 +35,7 @@ jobs: name: "Build and deploy documentation to staging" needs: test-code runs-on: ubuntu-latest + environment: github-pages-staging steps: - uses: actions/checkout@v4 - name: Set up Python @@ -56,3 +57,4 @@ jobs: branch: gh-pages # The branch the action should deploy to. folder: site repository-name: maxsch3/keras-batchflow-test + token: ${{ secrets.GH_DEPLOY_SECRET }} From 18c16099c7fddfce6b83db2dc4e2fce1ba05fbe4 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Fri, 22 Dec 2023 17:10:20 +0000 Subject: [PATCH 23/28] changed token to ssh-key --- .github/workflows/master-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master-pipeline.yml b/.github/workflows/master-pipeline.yml index 1477d2f..9f25416 100644 --- a/.github/workflows/master-pipeline.yml +++ b/.github/workflows/master-pipeline.yml @@ -57,4 +57,4 @@ jobs: branch: gh-pages # The branch the action should deploy to. folder: site repository-name: maxsch3/keras-batchflow-test - token: ${{ secrets.GH_DEPLOY_SECRET }} + ssh-key: ${{ secrets.GH_DEPLOY_SECRET }} From bb295185389bbf02c21116c7b408273746282e08 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Sat, 23 Dec 2023 09:29:31 +0000 Subject: [PATCH 24/28] added build pypi --- .github/workflows/master-pipeline.yml | 3 +-- .github/workflows/release-pipeline.yml | 37 +++++++++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/master-pipeline.yml b/.github/workflows/master-pipeline.yml index 9f25416..a95b88f 100644 --- a/.github/workflows/master-pipeline.yml +++ b/.github/workflows/master-pipeline.yml @@ -3,8 +3,7 @@ name: Python package on: push: branches: - - '*' - - '!master' + - 'master' jobs: diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 32907b8..2201e0f 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -1,9 +1,12 @@ name: Python package on: - release: - types: ['published'] - +# release: +# types: ['published'] + push: + branches: + - '*' + - '!master' jobs: @@ -19,6 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + python -m pip install --upgrade build pip install -r requirements.txt pip install -r build_requirements.txt pip install -e . @@ -27,14 +31,35 @@ jobs: - name: Build docs run: | mkdocs build --verbose --clean --strict - - name: Upload artifact + - name: Upload page artifact uses: actions/upload-pages-artifact@v3 with: path: ./site - deploy_stage: + pypi-build: + name: Build pypi package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade build + pip install -r requirements.txt + pip install -r build_requirements.txt + pip install -e . + - name: Build pypi package + run: | + python -m build + ls -l ./dist + + deploy_page: name: 'Deploy release documentation' - needs: page_build + needs: [page_build, pypi_build] # Grant GITHUB_TOKEN the permissions required to make a Pages deployment permissions: From a02bdf801ee6d09b7f919c65837aa6a609ad5769 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Sat, 23 Dec 2023 09:31:19 +0000 Subject: [PATCH 25/28] fixed typo in job name --- .github/workflows/release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 2201e0f..a609f6b 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -36,7 +36,7 @@ jobs: with: path: ./site - pypi-build: + pypi_build: name: Build pypi package runs-on: ubuntu-latest steps: From db6f3116c0901237bcce4395b80dc89537d4fa21 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Sun, 24 Dec 2023 06:34:12 +0000 Subject: [PATCH 26/28] Added pypi deploy --- .github/workflows/release-pipeline.yml | 27 +++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index a609f6b..cb38466 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -55,7 +55,11 @@ jobs: - name: Build pypi package run: | python -m build - ls -l ./dist + - name: Upload build package as artifact + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ deploy_page: name: 'Deploy release documentation' @@ -78,3 +82,24 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 + + deploy_pypi: + name: Publish to pypi + needs: [page_build, pypi_build] + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/keras-batchflow + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} \ No newline at end of file From 1b0e88b450e9f78770a1f1a7b10f2901e28036ca Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Sun, 24 Dec 2023 06:38:34 +0000 Subject: [PATCH 27/28] fixed name for PYPI_TOKEN --- .github/workflows/release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index cb38466..9db86d8 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -102,4 +102,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_TOKEN }} \ No newline at end of file From 7c36ebbb3d53369cd37f8878562b2b8d117acad0 Mon Sep 17 00:00:00 2001 From: maxsch3 Date: Sun, 24 Dec 2023 06:46:53 +0000 Subject: [PATCH 28/28] set triggers for all three pipleines --- .github/workflows/release-pipeline.yml | 9 +++------ .github/workflows/test-pipeline.yml | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 9db86d8..5d5ecce 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -1,12 +1,9 @@ name: Python package on: -# release: -# types: ['published'] - push: - branches: - - '*' - - '!master' + release: + types: ['published'] + jobs: diff --git a/.github/workflows/test-pipeline.yml b/.github/workflows/test-pipeline.yml index d1fbfb7..92b8a63 100644 --- a/.github/workflows/test-pipeline.yml +++ b/.github/workflows/test-pipeline.yml @@ -3,7 +3,7 @@ name: Python package on: push: branches: -# - '*' + - '*' - '!master' jobs: