ci.yml

name: ci
on:
  pull_request:
    paths:
      - .github/**
      - asv_bench/**
      - modin/**
      - requirements/**
      - scripts/**
      - environment-dev.yml
      - requirements-dev.txt
      - setup.cfg
      - setup.py
      - versioneer.py
jobs:
  lint-black:
    name: lint (black)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.x"
          architecture: "x64"
      - run: pip install black
      - run: black --check --diff modin/ asv_bench/benchmarks scripts/doc_checker.py

  build-docs:
    name: build docs
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.x"
          architecture: "x64"
      - run: pip install -r docs/requirements-doc.txt
      - run: cd docs && sphinx-build -T -E -W -b html . build

  lint-pydocstyle:
    name: lint (pydocstyle)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.x"
          architecture: "x64"
      # The `numpydoc` version here MUST match the versions in the dev requirements files.
      - run: pip install pytest pytest-cov pydocstyle numpydoc==1.1.0 xgboost
      - run: pytest scripts/test
      - run: pip install -e .[all]
      - run: |
          python scripts/doc_checker.py --add-ignore=D101,D102,D103,D105 --disable-numpydoc \
            modin/pandas/dataframe.py modin/pandas/series.py \
            modin/pandas/groupby.py \
            modin/pandas/series_utils.py modin/pandas/general.py \
            modin/pandas/plotting.py modin/pandas/utils.py \
            modin/pandas/iterator.py modin/pandas/indexing.py \
      - run: python scripts/doc_checker.py modin/core/dataframe
      - run: python scripts/doc_checker.py modin/core/execution/dask
      - run: |
          python scripts/doc_checker.py \
            modin/pandas/accessor.py modin/pandas/general.py \
            modin/pandas/groupby.py modin/pandas/indexing.py \
            modin/pandas/iterator.py modin/pandas/plotting.py \
            modin/pandas/series_utils.py modin/pandas/utils.py \
            modin/pandas/base.py \
            modin/pandas/io.py \
            asv_bench/benchmarks/utils \
            asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \
            asv_bench/benchmarks/scalability/__init__.py \
            modin/core/io \
            modin/experimental/core/execution/ray/implementations/pandas_on_ray \
            modin/experimental/core/execution/ray/implementations/pyarrow_on_ray \
            modin/pandas/series.py \
            modin/core/execution/python \
            modin/pandas/dataframe.py \
            modin/config/__init__.py \
            modin/config/__main__.py \
            modin/config/envvars.py \
            modin/config/pubsub.py
      - run: python scripts/doc_checker.py modin/distributed
      - run: python scripts/doc_checker.py modin/utils.py
      - run: python scripts/doc_checker.py modin/experimental/sklearn
      - run: |
          python scripts/doc_checker.py modin/experimental/xgboost/__init__.py \
            modin/experimental/xgboost/utils.py modin/experimental/xgboost/xgboost.py \
            modin/experimental/xgboost/xgboost_ray.py
      - run: python scripts/doc_checker.py modin/core/execution/ray
      - run: |
          python scripts/doc_checker.py modin/core/execution/dispatching/factories/factories.py \
            modin/core/execution/dispatching/factories/dispatcher.py                            \
      - run: python scripts/doc_checker.py scripts/doc_checker.py
      - run: |
          python scripts/doc_checker.py modin/experimental/pandas/io.py \
            modin/experimental/pandas/numpy_wrap.py modin/experimental/pandas/__init__.py
      - run: python scripts/doc_checker.py modin/core/storage_formats/base
      - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/pyarrow
      - run: python scripts/doc_checker.py modin/core/storage_formats/pandas
      - run: |
          python scripts/doc_checker.py \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/io \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_algebra.py \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_builder.py \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_serializer.py \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/df_algebra.py \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/expr.py \
            modin/experimental/core/execution/native/implementations/omnisci_on_native/omnisci_worker.py \
      - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/omnisci
      - run: python scripts/doc_checker.py modin/experimental/core/execution/native/implementations/omnisci_on_native/exchange/dataframe_protocol
      - run: python scripts/doc_checker.py modin/experimental/batch/pipeline.py
      - run: python scripts/doc_checker.py modin/logging

  lint-flake8:
    name: lint (flake8)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.x"
          architecture: "x64"
      - run: pip install flake8 flake8-print flake8-no-implicit-concat
      - run: flake8 modin/ asv_bench/benchmarks scripts/doc_checker.py

  test-api:
    runs-on: ubuntu-latest
    name: test api
    defaults:
      run:
        # `shell: bash -l {0}` - special way to activate modin environment
        shell: bash -l {0}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: sudo apt update && sudo apt install -y libhdf5-dev
      - name: Api tests
        run: python -m pytest modin/pandas/test/test_api.py
      - name: Executions Api tests
        run: python -m pytest modin/test/test_executions_api.py

  test-headers:
    runs-on: ubuntu-latest
    name: test-headers
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Headers tests
        run: python -m pytest modin/test/test_headers.py

  test-clean-install-ubuntu:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    name: test-clean-install-ubuntu
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.x"
          architecture: "x64"
      - name: Clean install and run
        run: |
          python -m pip install -e .[all]
          MODIN_ENGINE=dask python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
          MODIN_ENGINE=ray python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"

  test-clean-install-windows:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: windows-latest
    defaults:
      run:
        shell: bash -l {0}
    name: test-clean-install-windows
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: actions/setup-python@v2
        with:
          python-version: "3.8.x"
          architecture: "x64"
      - name: Clean install and run
        run: |
          python -m pip install -e .[all]
          MODIN_ENGINE=dask python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
          MODIN_ENGINE=ray python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"

  test-internals:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    name: test-internals
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Internals tests
        run: python -m pytest modin/core/execution/dispatching/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py
      - run: python -m pytest modin/config/test
      - run: python -m pytest modin/test/test_envvar_catcher.py
      - run: python -m pytest modin/test/storage_formats/base/test_internals.py
      - run: python -m pytest modin/test/storage_formats/pandas/test_internals.py
      - run: python -m pytest modin/test/test_envvar_npartitions.py
      - run: python -m pytest -n 2 modin/test/test_partition_api.py
      - run: python -m pytest modin/test/test_utils.py
      - run: python -m pytest asv_bench/test/test_utils.py
      - run: python -m pytest modin/test/exchange/dataframe_protocol/base
      - uses: codecov/codecov-action@v2

  test-defaults:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        execution: [BaseOnPython]
    env:
      MODIN_MEMORY: 1000000000
      MODIN_TEST_DATASET_SIZE: "small"
    name: Test ${{ matrix.execution }} execution, Python 3.8
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Install HDF5
        run: sudo apt update && sudo apt install -y libhdf5-dev
      - run: pytest modin/experimental/xgboost/test/test_default.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/test/storage_formats/base/test_internals.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_binary.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_default.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_indexing.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_iter.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_join_sort.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_map_metadata.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_reduce.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_udf.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_window.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_pickle.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_series.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_rolling.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_concat.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_groupby.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_reshape.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_general.py --execution=${{ matrix.execution }}
      - uses: codecov/codecov-action@v2

  test-omnisci:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    env:
      MODIN_STORAGE_FORMAT: "omnisci"
    name: Test OmniSci storage format, Python 3.8
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - name: Setting up Modin environment
        uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin_on_omnisci
          environment-file: requirements/env_omnisci.yml
          python-version: 3.8
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Install HDF5
        run: sudo apt update && sudo apt install -y libhdf5-dev
      - run: pytest modin/test/storage_formats/omnisci/test_internals.py
      - run: MODIN_BENCHMARK_MODE=True pytest modin/pandas/test/internals/test_benchmark_mode.py
      - run: pytest modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py
      - run: pytest modin/pandas/test/test_io.py::TestCsv --verbose
      - run: pytest modin/test/exchange/dataframe_protocol/test_general.py
      - run: pytest modin/test/exchange/dataframe_protocol/omnisci
      - uses: codecov/codecov-action@v2

  test-asv-benchmarks:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    env:
      MODIN_ENGINE: ray
      MODIN_MEMORY: 1000000000
      MODIN_TEST_DATASET_SIZE: small
    name: test-asv-benchmarks
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          auto-activate-base: true
          activate-environment: ""
      - name: ASV installation
        run: |
          # FIXME: use the tag or release version of ASV as soon as it appears;
          # The ability to build a conda environment by specifying yml file has not
          # yet appeared in the release versions;
          pip install git+https://github.com/airspeed-velocity/asv.git@ef016e233cb9a0b19d517135104f49e0a3c380e9
      - name: Running benchmarks
        run: |
          # ASV correctly creates environments for testing only from the branch
          # with `master` name
          git checkout -b master
          cd asv_bench
          asv check -v
          git remote add upstream https://github.com/modin-project/modin.git
          git fetch upstream
          if git diff upstream/master --name-only | grep -q "^asv_bench/"; then
              asv machine --yes

              # check Modin on Ray
              asv run --quick --strict --show-stderr --launch-method=spawn \
                -b ^benchmarks -b ^io -b ^scalability | tee benchmarks.log

              # check pure pandas
              MODIN_ASV_USE_IMPL=pandas asv run --quick --strict --show-stderr --launch-method=spawn \
                -b ^benchmarks -b ^io -b ^scalability | tee benchmarks.log

              # OmniSci: ERR_OUT_OF_CPU_MEM: Not enough host memory to execute the query (MODIN#4270)
              # just disable test for testing - it works well in a machine with more memory
              sed -i 's/def time_groupby_agg_nunique(self, \*args, \*\*kwargs):/# def time_groupby_agg_nunique(self, *args, **kwargs):/g' benchmarks/omnisci/benchmarks.py
              sed -i 's/execute(self.df.groupby(by=self.groupby_columns).agg("nunique"))/# execute(self.df.groupby(by=self.groupby_columns).agg("nunique"))/g' benchmarks/omnisci/benchmarks.py

              # check Modin on OmniSci
              MODIN_ENGINE=native MODIN_STORAGE_FORMAT=omnisci MODIN_EXPERIMENTAL=true asv run --quick --strict --show-stderr \
                --launch-method=forkserver --config asv.conf.omnisci.json \
                -b ^omnisci | tee benchmarks.log
          else
              echo "Benchmarks did not run, no changes detected"
          fi
        if: always()

      - name: Publish benchmarks artifact
        uses: actions/upload-artifact@master
        with:
          name: Benchmarks log
          path: asv_bench/benchmarks.log
        if: failure()

  test-all:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8"]
        engine: ["python", "ray", "dask"]
    env:
      MODIN_ENGINE: ${{matrix.engine}}
      MODIN_MEMORY: 1000000000
      # Only test reading from SQL server and postgres on ubuntu for now.
      # Eventually, we should test on Windows, too, but we will have to set up
      # the servers differently.
      MODIN_TEST_READ_FROM_SQL_SERVER: true
      MODIN_TEST_READ_FROM_POSTGRES: true
    name: test-ubuntu (engine ${{matrix.engine}}, python ${{matrix.python-version}})
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Install HDF5
        run: sudo apt update && sudo apt install -y libhdf5-dev
      - name: Set up postgres
        # Locally, specifying port 2345:5432 works, but 2345:2345 and 5432:5432 do not. This solution is from
        # https://stackoverflow.com/questions/36415654/cant-connect-docker-postgresql-9-3
        run: |
          sudo docker pull postgres
          sudo docker run --name some-postgres -e POSTGRES_USER=sa -e POSTGRES_PASSWORD=Strong.Pwd-123 -e POSTGRES_DB=postgres -d -p 2345:5432 postgres
      - run: MODIN_BENCHMARK_MODE=True pytest modin/pandas/test/internals/test_benchmark_mode.py
      - run: pytest -n 2 modin/experimental/xgboost/test/test_default.py
      - run: pytest -n 2 modin/experimental/xgboost/test/test_xgboost.py
        if: matrix.engine == 'ray'
      - run: pytest -n 2 modin/experimental/xgboost/test/test_dmatrix.py
        if: matrix.engine == 'ray'
      - run: pytest -n 2 modin/experimental/batch/test/test_pipeline.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_binary.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_default.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_indexing.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_iter.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_join_sort.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_map_metadata.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_reduce.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_udf.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_window.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_pickle.py
      - run: python -m pytest -n 2 modin/pandas/test/test_series.py
      - run: python -m pytest -n 2 modin/pandas/test/test_rolling.py
      - run: python -m pytest -n 2 modin/pandas/test/test_concat.py
        if: matrix.engine == 'python'
      - run: python -m pytest modin/pandas/test/test_concat.py # Ray and Dask versions fails with -n 2
        if: matrix.engine != 'python'
      - run: python -m pytest -n 2 modin/pandas/test/test_groupby.py
      - run: python -m pytest -n 2 modin/pandas/test/test_reshape.py
      - run: python -m pytest -n 2 modin/pandas/test/test_general.py
      - run: chmod +x ./.github/workflows/sql_server/set_up_sql_server.sh
      - run: ./.github/workflows/sql_server/set_up_sql_server.sh
      - run: python -m pytest modin/pandas/test/test_io.py --verbose
      - run: python -m pytest modin/experimental/pandas/test/test_io_exp.py
      - run: pip install "dfsql>=0.4.2" "pyparsing<=2.4.7" && pytest modin/experimental/sql/test/test_sql.py
      - run: pytest modin/test/exchange/dataframe_protocol/test_general.py
      - run: pytest modin/test/exchange/dataframe_protocol/pandas/test_protocol.py
      - uses: codecov/codecov-action@v2

  test-experimental:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    env:
      MODIN_ENGINE: "python"
      MODIN_EXPERIMENTAL: "True"
      MODIN_MEMORY: 1000000000
    name: test experimental
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: python -m pytest -n 2 modin/pandas/test/dataframe/test_map_metadata.py
      - run: python -m pytest -n 2 modin/pandas/test/test_series.py
      - run: python -m pytest modin/pandas/test/test_io.py --verbose
      - uses: codecov/codecov-action@v2

  test-cloud:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    env:
      MODIN_ENGINE: "python"
      MODIN_EXPERIMENTAL: "True"
      MODIN_MEMORY: 1000000000
    name: test cloud
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
        # TODO(https://github.com/modin-project/modin/issues/4004): Re-add
        # "python -m pytest --simulate-cloud=normal modin/pandas/test/test_io.py --verbose"
        # once that test stops crashing.
      - run: python -m pytest --simulate-cloud=normal modin/pandas/test/dataframe/test_default.py::test_kurt_kurtosis --verbose
      - # When running without parameters, some of the tests fail
        run: python -m pytest --simulate-cloud=normal modin/pandas/test/dataframe/test_binary.py::test_math_functions[add-rows-scalar]
      - uses: codecov/codecov-action@v2

  test-windows:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: windows-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8"]
        engine: ["ray", "dask"]
        test-task:
          - modin/pandas/test/dataframe/test_binary.py
          - modin/pandas/test/dataframe/test_default.py
          - modin/pandas/test/dataframe/test_indexing.py
          - modin/pandas/test/dataframe/test_iter.py
          - modin/pandas/test/dataframe/test_join_sort.py
          - modin/pandas/test/dataframe/test_map_metadata.py
          - modin/pandas/test/dataframe/test_reduce.py
          - modin/pandas/test/dataframe/test_udf.py
          - modin/pandas/test/dataframe/test_window.py
          - modin/pandas/test/dataframe/test_pickle.py
          - modin/pandas/test/test_series.py
          - modin/pandas/test/test_rolling.py
          - modin/pandas/test/test_concat.py
          - modin/pandas/test/test_groupby.py
          - modin/pandas/test/test_reshape.py
          - modin/pandas/test/test_general.py
          - modin/pandas/test/test_io.py
    env:
      MODIN_ENGINE: ${{matrix.engine}}
      MODIN_MEMORY: 1000000000
    name: test-windows
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
          auto-update-conda: true # this enable `use-only-tar-bz2` feature on Windows
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: python -m pytest ${{matrix.test-task}}
        if: matrix.test-task != 'modin/pandas/test/test_io.py'
      - timeout-minutes: 30
        run: python -m pytest modin/pandas/test/test_io.py --verbose
        if: matrix.test-task == 'modin/pandas/test/test_io.py'
      - uses: codecov/codecov-action@v2

  test-pyarrow:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8"]
    env:
      MODIN_STORAGE_FORMAT: pyarrow
      MODIN_EXPERIMENTAL: "True"
    name: test (pyarrow, python ${{matrix.python-version}})
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: sudo apt update && sudo apt install -y libhdf5-dev
      - run: python -m pytest modin/pandas/test/test_io.py::TestCsv --verbose

  test-spreadsheet:
    needs: [lint-flake8, lint-black, test-api, test-headers]
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8" ]
        engine: ["ray", "dask"]
    env:
      MODIN_EXPERIMENTAL: "True"
      MODIN_ENGINE: ${{matrix.engine}}
    name: test-spreadsheet (engine ${{matrix.engine}}, python ${{matrix.python-version}})
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: python -m pytest modin/experimental/spreadsheet/test/test_general.py