Feature/GitHub actions (#22)

* testing github actions
* trying coveralls.io integration
* dropping most tests for faster tweaks of github actions workflow
* testing faster UCSC setting
* testing without UCSC SQL server
* true offline fetch_genes()
* testing matrix; circleci now faster
* macos-latest fails
* speeding up; not testing windows or mac
* actions badge and docs update
* v1.0.1 ready

.circleci/config.yml

@@ -11,10 +11,13 @@ jobs:
           # key: <projectname>-{{ .Branch }}-{{ checksum "Pipfile.lock" }}  projectname can be anything, so long as it matches save_cache below.
           key: deps-0-{{ .Branch }}-{{ checksum "requirements.txt" }}
       - run:
+          name: Installing pipenv packages
           command: |
             pip install pipenv
             pipenv install --dev --skip-lock
             pipenv install -e . --skip-lock # pytest recommends this: install your package in "editable" mode by running this from the same directory
+            pipenv install pytest-cov --skip-lock
+            pipenv install coveralls --skip-lock # for coveralls.io -- this can be in Pipfile [dev] instead.
             # pipenv will use requirements.txt with these params.
       - save_cache:
           key: deps-0-{{ .Branch }}-{{ checksum "requirements.txt" }}
@@ -25,19 +28,14 @@ jobs:
       - run:
           name: Running tests and test coverage
           command: |
-            pipenv install coveralls --skip-lock # for coveralls.io -- this can be in Pipfile [dev] instead.
-            pipenv run coverage run setup.py test
-            pipenv run pytest tests --junit-xml=htmlcov/junit.xml # rerun tests to collect test-results
+            pipenv run coverage run -m pytest tests --junit-xml=htmlcov/junit.xml
             pipenv run coverage report
             pipenv run coverage html
             pipenv run coveralls
             # set COVERALLS_REPO_TOKEN=<...> in ENVIRONMENT variables in circleci
-            # pipenv run coverage run --omit '*tests*','*/home/*' setup.py test
-            # moved omit to setup.cfg; broke on circleci - no uploads
 
       - store_test_results:
           path: htmlcov
-
       - store_artifacts:
           path: htmlcov
           destination: methylize

.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: tests
+on:
+  push:
+    branches:
+      - master
+      - 'feature/**'
+  pull_request:
+    branches:
+      - master
+
+env:
+  GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+  COVERALLS_SERVICE_NAME: 'github'
+  COVERALLS_REPO_TOKEN : ${{ secrets.COVERALLS_REPO_TOKEN }}
+
+jobs:
+  test:
+    name: Test ${{ matrix.os }}, Python ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      max-parallel: 2
+      matrix:
+        os: [ubuntu-latest] # macOS-latest wouldn't run, so skipping. 'windows-latest' takes 50% longer than ubuntu
+        python-version: [3.8] # [3.6, 3.7, 3.8]
+    steps:
+      - name: Dump GitHub context
+        id: github_context_step
+        run: echo '${{ toJSON(github) }}'
+      #- name: Dump runner context
+      #  run: echo '${{ toJSON(runner) }}'
+      - name: Dump strategy context
+        run: echo '${{ toJSON(strategy) }}'
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Install Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+            pip install pipenv pytest_cov
+            # pipenv install pytest_cov
+            pipenv install --dev --skip-lock
+            pipenv install -e . --skip-lock
+            # pipenv uses requirements.txt
+
+      - name: Testing and coverage report
+        run: |
+          pipenv run coverage run setup.py test
+          pipenv run coverage xml
+
+      - name: Publish to codecov
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          #files: ./coverage.xml #./coverage1.xml,./coverage2.xml # optional
+          directory: ./
+          flags: unittests # optional
+          name: codecov-umbrella # optional
+          fail_ci_if_error: true # optional (default = false)
+          verbose: false # optional (default = false)
+
+      - name: Publish to GitHub # cobertura-report format publishes coverage.xml
+        uses: 5monkeys/cobertura-action@v12
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          minimum_coverage: 40
+          fail_below_threshold: 40
+
+# GHA workflow: diffs are limited to 300 files. If there are files changed that aren't matched in the first 300 files returned by the
+# filter, the workflow will not run. You may need to create more specific filters so that the workflow will run automatically.

LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Life Epigenetics, Inc.
+Copyright (c) 2022 FOXO Technologies, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Pipfile

@@ -16,13 +16,14 @@ pylint = "*"
 pytest = "*"
 pytest-runner = "*"
 pytest-mock = "*"
-coverage = "*"
 nbsphinx = "*"
 sphinx = "*"
 # xlrd = ">=1.0.0" (after Dec 2020 xlsx not supported; use openpyxl)
 sphinxcontrib-apidoc = "*"
 m2r = "*"
 ipykernel = "*"
+coverage = "*"
+coveralls = "*"
 
 [packages]
 tqdm = "*"
@@ -36,6 +37,10 @@ methylcheck = "*"
 pymysql = "*"
 toolshed = "*"
 interlap = "*"
+pytest-cov = "*"
 
 [requires]
 python_version = "3.8"
+
+[scripts]
+test="pytest"

README.md

@@ -1,6 +1,6 @@
 `methylize` is a python package for analyzing output from Illumina methylation arrays. It complements `methylprep` and `methylcheck` and provides methods for computing differentially methylated probes and regions, and annotating these regions with the UCSC Genome Browser.  View on [ReadTheDocs.](https://life-epigenetics-methylize.readthedocs-hosted.com/en/latest/)
 
-[![Readthedocs](https://readthedocs.com/projects/life-epigenetics-methylize/badge/?version=latest)](https://life-epigenetics-methylize.readthedocs-hosted.com/en/latest/) [![image](https://img.shields.io/pypi/l/pipenv.svg)](https://python.org/pypi/pipenv) [![CircleCI](https://circleci.com/gh/FoxoTech/methylize/tree/master.svg?style=shield)](https://circleci.com/gh/FoxoTech/methylize/tree/master) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/099d26465bd64c2387afa063810a13e6)](https://www.codacy.com/gh/FoxoTech/methylize/dashboard?utm_source=github.com&utm_medium=referral&utm_content=FOXOBioScience/methylize&utm_campaign=Badge_Grade) [![Coverage Status](https://coveralls.io/repos/github/FoxoTech/methylize/badge.svg?branch=master)](https://coveralls.io/github/FoxoTech/methylize?branch=master) ![PYPI-Downloads](https://img.shields.io/pypi/dm/methylize.svg?label=pypi%20downloads&logo=PyPI&logoColor=white)
+[![tests](https://github.com/FoxoTech/methylize/workflows/tests/badge.svg)](https://github.com/FoxoTech/methylize/actions/workflows/ci.yml) [![Readthedocs](https://readthedocs.com/projects/life-epigenetics-methylize/badge/?version=latest)](https://life-epigenetics-methylize.readthedocs-hosted.com/en/latest/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![CircleCI](https://circleci.com/gh/FoxoTech/methylize/tree/master.svg?style=shield)](https://circleci.com/gh/FoxoTech/methylize/tree/master) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/099d26465bd64c2387afa063810a13e6)](https://www.codacy.com/gh/FoxoTech/methylize/dashboard?utm_source=github.com&utm_medium=referral&utm_content=FOXOBioScience/methylize&utm_campaign=Badge_Grade) [![Coverage Status](https://coveralls.io/repos/github/FoxoTech/methylize/badge.svg?branch=master)](https://coveralls.io/github/FoxoTech/methylize?branch=master) ![PYPI-Downloads](https://img.shields.io/pypi/dm/methylize.svg?label=pypi%20downloads&logo=PyPI&logoColor=white) [![codecov](https://codecov.io/gh/FoxoTech/methylize/branch/master/graph/badge.svg?token=KDV3HHDCTU)](https://codecov.io/gh/FoxoTech/methylize)
 
 ## methylize is part of the methylsuite
 

docs/release-history.md

@@ -1,5 +1,10 @@
 # Release History
 
+## v1.0.1
+- Differentially methylated regions (DMR) will no annotate via cached UCSC database (via fetch_genes) without using the internet, if user wants. (previously, it would still contact the internet database even if user told it not to)
+- Added testing via github actions, and increased speed
+- updated documentation
+
 ## v1.0.0
   - fixed bug in fetch_genes() from UCSC browser; function will now accept either the filepath or the DMR dataframe output.
 

index.rst

@@ -1,4 +1,4 @@
-`methylize` documentation
+methylize documentation
 =========================
 
 ---------------

methylize/diff_meth_pos.py

@@ -329,7 +329,7 @@ def diff_meth_pos(
         ##Format binary data as 0's and 1's if it was given as a list of strings with
             ##2 different string values
         if zeroes_ones:
-            pheno_data_binary = np.array(pheno_data,dtype=np.int)
+            pheno_data_binary = np.array(pheno_data,dtype=int)
         else:
             pheno_data_binary = np.array(pheno_data)
             ##Turn the first phenotype into zeroes wherever it occurs in the array

methylize/genome_browser.py

@@ -194,15 +194,34 @@ def _tidy(desc):
 
     # add column(s) for gene tissue expression
     if tissue != None:
-        # tissue == 'all'
-        tissues = fetch_genes(sql="select * from hgFixed.gtexTissueV8;")
+        tissue_cache = False
+        if no_sync == True: # avoid hitting UCSC server here and use the cached copy
+            tissue_names_file = Path(package_path, 'data', f"gtexTissueV8_names.pkl")
+            tissues_file = Path(package_path, 'data', f"gtexTissueV8.pkl")
+            tissue_names_available = tissue_names_file.exists()
+            tissues_available = tissues_file.exists()
+            if not tissues_available and tissue_names_available:
+                LOGGER.warning("No cache gtex Tissues data available; downloading from UCSC.")
+            else:
+                tissues = pd.read_pickle(tissue_names_file)
+                tissue_cache = True
+        else:
+            # tissue == 'all'
+            tissues = fetch_genes(sql="select * from hgFixed.gtexTissueV8;")
         sorted_tissues = [i['name'] for i in tissues]
         gene_names = [i.split(',') for i in list(regions['genes']) if i != '']
         N_regions_with_multiple_genes = len([i for i in gene_names if len(i) > 1])
         if N_regions_with_multiple_genes > 0:
             LOGGER.warning(f"{N_regions_with_multiple_genes} of the {len(gene_names)} regions have multiple genes matching in the same region, and output won't show tissue expression levels.")
         gene_names = tuple([item for sublist in gene_names for item in sublist])
-        gtex = fetch_genes(sql=f"select name, expScores from gtexGeneV8 WHERE name in {gene_names} and score > 0;")
+
+        # next, filter the full database table by gene_names with scores
+        if tissue_cache:
+            annot = pd.read_pickle(tissues_file) # pulled with "where score > 0"
+            # annot is a list of dicts; i['score'] is a stringified list of numbers, matching order of sorted tissues
+            gtex = [i for i in annot if i['name'] in gene_names]
+        else: # SLOWER; use live database by default
+            gtex = fetch_genes(sql=f"select name, expScores from gtexGeneV8 WHERE name in {gene_names} and score > 0;")
         if len(gtex) > 0:
             # convert to a lookup dict of gene name: list of tissue scores
             gtex = {item['name']: [float(i) for i in item['expScores'].decode().split(',') if i != ''] for item in gtex}

requirements.txt

@@ -6,8 +6,6 @@ scipy #>=1.2.3
 matplotlib
 statsmodels
 joblib
-pytest
-coverage
 sphinx>=1.4
 sphinxcontrib-apidoc
 m2r

setup.cfg

@@ -29,3 +29,7 @@ load-plugins = pylint.extensions.bad_builtin,pylint.extensions.check_elif,pylint
 
 [coverage:run]
 omit=*/site-packages/*,*/tests/*,*/.eggs/*,/home/circleci/.local/share/virtualenvs/*
+relative_files = True
+
+[scripts]
+test = "pytest"

setup.py

@@ -2,6 +2,20 @@
 from setuptools import setup, find_packages
 exec(open('methylize/version.py').read())
 
+requirements = [
+    'pandas',
+    'numpy',
+    'scipy',
+    'statsmodels',
+    'matplotlib',
+    'methylprep',
+    'methylcheck',
+    'pymysql',
+    'toolshed',
+    'interlap',
+    #'cpv', #'cpv @ git+https://github.com/brentp/combined-pvalues.git@v0.50.6#egg=cpv',
+]
+
 setup(
     name='methylize',
     version=__version__,
@@ -35,21 +49,18 @@
     author_email='info@FOXOBioScience.com',
     packages=find_packages(),
     include_package_data=True,
-    install_requires=[
-        'pandas',
-        'numpy',
-        'scipy',
-        'statsmodels',
-        'matplotlib',
-        'methylprep',
-        'methylcheck',
-        #'cpv', #'cpv @ git+https://github.com/brentp/combined-pvalues.git@v0.50.6#egg=cpv',
-        'pymysql',
-        'toolshed',
-        'interlap',
-    ],
+    install_requires=requirements,
+    extras_require={
+        'dev': [
+            'pytest',
+            'coverage'
+        ]
+    },
     setup_requires=['pytest-runner'],
-    tests_require=['pytest'],
+    tests_require=[
+        'pytest',
+        'coverage'
+        ],
     entry_points={
         'console_scripts': [
             'methylize = methylize:main',