Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change testing for direct mode and V6 to exclusion lists #1789

Merged
merged 9 commits into from
Sep 5, 2017
Merged

Change testing for direct mode and V6 to exclusion lists #1789

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1d9cc38
NF: Provide decorators to skip tests in v6 and direct mode
bpoldrack Sep 4, 2017
7331b03
ENH: Exclude not yet passing tests from direct mode build
bpoldrack Sep 4, 2017
7f01f02
ENH: Exclude not yet passing tests from v6 build
bpoldrack Sep 4, 2017
9f82c7d
Add FIXME label to tests failing in direct mode
bpoldrack Sep 4, 2017
d34a742
Add v6 build to Travis' matrix
bpoldrack Sep 4, 2017
2d22030
BF: Don't test direct mode switch in v6 build
bpoldrack Sep 4, 2017
f8da9b4
BF: Disable a missed test for v6
bpoldrack Sep 4, 2017
857c7ec
NF: Introducing datalad.tests.skipknownfailures (default: yes), which…
bpoldrack Sep 5, 2017
81d4f5b
BF: Use of config.obtain() led to interactively asking for datalad.re…
bpoldrack Sep 5, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ env:
matrix:
- DATALAD_REPO_DIRECT=1
- DATALAD_REPO_DIRECT=
- DATALAD_REPO_VERSION=6
- DATALAD_REPO_VERSION=

# Disabled since old folks don't want to change workflows of submitting through central authority
# - secure: "k2rHdTBjUU3pUUASqfRr7kHaaSmNKLLAR2f66A0fFSulih4CXxwLrR3g8/HP9m+jMve8mAYEiPSI7dT7cCm3WMA/piyLh2wKCGgzDD9oLjtvPAioR8dgLpzbgjxV/Vq6fwwPMlvbqqa+MmAImnAoSufEmI7zVQHCq11Hd5nd6Es="
Expand Down Expand Up @@ -117,20 +119,13 @@ matrix:
env:
- TMPDIR="/tmp/nfsmount"
- _DATALAD_NONPR_ONLY=1
- python: 2.7
# test modules that should not fail in direct mode now:
env:
- TESTS_TO_PERFORM="datalad/tests datalad/support"
- DATALAD_REPO_DIRECT=1

allow_failures:
# Test under NFS mount (full, only in master)
- python: 2.7
env:
- TMPDIR="/tmp/nfsmount"
- _DATALAD_NONPR_ONLY=1
# For now, don't fail entirely when direct mode fails
- env: DATALAD_REPO_DIRECT=1

# Causes complete laptop or travis instance crash atm, but survives in a docker
# need to figure it out (looks like some PID explosion)
Expand Down
4 changes: 4 additions & 0 deletions datalad/cmdline/tests/test_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Test functioning of the datalad main cmdline utility """

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
import re
import sys
from six.moves import StringIO
Expand Down Expand Up @@ -150,6 +152,8 @@ def test_incorrect_options():
yield check_incorrect_option, ('--dbg',), err_insufficient
yield check_incorrect_option, tuple(), err_insufficient

@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_script_shims():
runner = Runner()
for script in [
Expand Down
8 changes: 8 additions & 0 deletions datalad/crawler/nodes/tests/test_annex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
from os import listdir
from os.path import join as opj, exists, lexists, basename
from collections import OrderedDict
Expand Down Expand Up @@ -49,6 +51,7 @@ def test_annexificator_no_git_if_dirty(outdir):

@with_tempfile(mkdir=True)
@with_tempfile()
@skip_direct_mode #FIXME
def test_initiate_dataset(path, path2):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yikes... Even initiation of a data doesn't work in direct mode?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you mean by "even"? ;-)
Come on - it's currently 91 out of 764. Admittedly it's not single digit yet, but it's not THAT bad, given that we didn't monitor anything but datalad/tests and datalad/support.

dataset_path = opj(path, 'test')
datas = list(initiate_dataset('template', 'testdataset', path=dataset_path)())
Expand Down Expand Up @@ -166,6 +169,8 @@ def _test_annex_file(mode, topdir, topurl, outdir):
assert_equal(output[0]['datalad_stats'], ActivityStats(files=1, add_git=1))


@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_annex_file():
for mode in ('full', 'fast', 'relaxed',):
yield _test_annex_file, mode
Expand Down Expand Up @@ -213,6 +218,8 @@ def _test_add_archive_content_tar(direct, repo_path):
assert_false(annex.repo.dirty)


@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_add_archive_content_tar():
for direct in (True, False):
yield _test_add_archive_content_tar, direct
Expand All @@ -222,6 +229,7 @@ def test_add_archive_content_tar():
@with_tempfile(mkdir=True)
@with_tree(tree={'file': 'load'})
@serve_path_via_http
@skip_direct_mode #FIXME
def test_add_dir_file(repo_path, p, topurl):
# test whenever file becomes a directory and then back a file. Should all work!
annex = Annexificator(path=repo_path, auto_finalize=False)
Expand Down
8 changes: 8 additions & 0 deletions datalad/crawler/pipelines/tests/test_balsa.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
from datalad.crawler.pipelines.tests.utils import _test_smoke_pipelines
from ..balsa import pipeline as ofpipeline, superdataset_pipeline
import os
Expand Down Expand Up @@ -124,6 +126,8 @@ def test_smoke_pipelines():
@serve_path_via_http
@with_tempfile
@with_tempfile
@skip_direct_mode #FIXME
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note that if the entire module of tests failes (probably for issues not to be fixed in that module per se), might be worth just skip it at the module level, e.g. how it is done already for scrapy

try:
    import scrapy
except ImportError:
    raise SkipTest("Needs scrapy")

Copy link
Member Author

@bpoldrack bpoldrack Sep 4, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The entire point is to do it on a per-test level, so we are able to easily ensure we don't worsen the status quo and we can fix that stuff in less complex steps. Have a look at #1562.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, please keep in mind that these are steps towards not having anything fail. We need a quick way to say if there is anything worth fixing at the level of and when touching individual tests. Please keep individual test labeling.

@skip_v6 #FIXME
def test_balsa_extract_meta(ind, topurl, outd, clonedir):
list(initiate_dataset(
template="balsa",
Expand Down Expand Up @@ -197,6 +201,8 @@ def test_balsa_extract_meta(ind, topurl, outd, clonedir):
@serve_path_via_http
@with_tempfile
@with_tempfile
@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_balsa_pipeline1(ind, topurl, outd, clonedir):
list(initiate_dataset(
template="balsa",
Expand Down Expand Up @@ -304,6 +310,8 @@ def test_balsa_pipeline1(ind, topurl, outd, clonedir):
@serve_path_via_http
@with_tempfile
@with_tempfile
@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_balsa_pipeline2(ind, topurl, outd, clonedir):
list(initiate_dataset(
template="balsa",
Expand Down
4 changes: 4 additions & 0 deletions datalad/crawler/pipelines/tests/test_crcns.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
from .utils import _test_smoke_pipelines
from ..crcns import pipeline, superdataset_pipeline
from ..crcns import get_metadata
Expand All @@ -15,6 +17,8 @@
from datalad.tests.utils import ok_startswith


@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_smoke_pipelines():
yield _test_smoke_pipelines, pipeline, ['bogus', "bogusgroup"]
yield _test_smoke_pipelines, superdataset_pipeline, []
Expand Down
4 changes: 4 additions & 0 deletions datalad/crawler/pipelines/tests/test_fcptable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
from os.path import exists
from requests.exceptions import InvalidURL

Expand All @@ -29,6 +31,8 @@

TOPURL = "http://fcon_1000.projects.nitrc.org/fcpClassic/FcpTable.html"

@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_smoke_pipelines():
yield _test_smoke_pipelines, pipeline, ['bogus']
yield _test_smoke_pipelines, superdataset_pipeline, []
Expand Down
6 changes: 6 additions & 0 deletions datalad/crawler/pipelines/tests/test_openfmri.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
import os
from glob import glob
from os.path import join as opj
Expand Down Expand Up @@ -204,6 +206,8 @@ def __test_basic_openfmri_dataset_pipeline_with_annex(path):
@serve_path_via_http
@with_tempfile
@with_tempfile
@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_openfmri_pipeline1(ind, topurl, outd, clonedir):
index_html = opj(ind, 'ds666', 'index.html')

Expand Down Expand Up @@ -433,6 +437,8 @@ def _pipeline(*args, **kwargs):
)
@serve_path_via_http
@with_tempfile
@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_openfmri_pipeline2(ind, topurl, outd):
# no versioned files -- should still work! ;)

Expand Down
2 changes: 2 additions & 0 deletions datalad/crawler/pipelines/tests/test_openfmri_collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

from datalad.tests.utils import skip_direct_mode
import os
from glob import glob
from os.path import join as opj, exists
Expand Down Expand Up @@ -53,6 +54,7 @@
)
@serve_path_via_http
@with_tempfile
@skip_direct_mode #FIXME
def test_openfmri_superdataset_pipeline1(ind, topurl, outd):

list(initiate_dataset(
Expand Down
8 changes: 8 additions & 0 deletions datalad/crawler/pipelines/tests/test_simple_with_archives.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
from os.path import join as opj

from datalad.crawler.pipelines.tests.utils import _test_smoke_pipelines
Expand All @@ -29,6 +31,8 @@
lgr = getLogger('datalad.crawl.tests')


@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_smoke_pipelines():
yield _test_smoke_pipelines, pipeline, ["random_url"]

Expand All @@ -39,6 +43,8 @@ def test_smoke_pipelines():
@with_tree(tree=TEST_TREE1, archives_leading_dir=False)
@serve_path_via_http
@with_tempfile
@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_simple1(ind, topurl, outd):

list(initiate_dataset(
Expand Down Expand Up @@ -74,6 +80,8 @@ def test_simple1(ind, topurl, outd):
}, archives_leading_dir=False)
@serve_path_via_http
@with_tempfile
@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_crawl_autoaddtext(ind, topurl, outd):
ds = create(outd, text_no_annex=True)
with chpwd(outd): # TODO -- dataset argument
Expand Down
6 changes: 6 additions & 0 deletions datalad/customremotes/tests/test_archives.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Tests for customremotes archives providing dl+archive URLs handling"""

from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
from ..archives import ArchiveAnnexCustomRemote
from ..base import AnnexExchangeProtocol
from ...support.annexrepo import AnnexRepo
Expand Down Expand Up @@ -119,6 +121,8 @@ def check_basic_scenario(fn_archive, fn_extracted, direct, d, d2):
@with_tree(
tree={'a.tar.gz': {'d': {fn_inarchive_obscure: '123'}}}
)
@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_annex_get_from_subdir(topdir):
from datalad.api import add_archive_content
annex = AnnexRepo(topdir, init=True)
Expand Down Expand Up @@ -151,6 +155,8 @@ def test_get_git_environ_adjusted():
assert_equal(sys_env["PWD"], os.environ.get("PWD"))


@skip_direct_mode #FIXME
@skip_v6 #FIXME
def test_basic_scenario():
yield check_basic_scenario, 'a.tar.gz', 'simple.txt', False
if not on_windows:
Expand Down
2 changes: 2 additions & 0 deletions datalad/customremotes/tests/test_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Tests for the base of our custom remotes"""

from datalad.tests.utils import skip_direct_mode
from os.path import isabs

from datalad.tests.utils import with_tree
Expand All @@ -17,6 +18,7 @@


@with_tree(tree={'file.dat': ''})
@skip_direct_mode #FIXME
def test_get_contentlocation(tdir):
repo = AnnexRepo(tdir, create=True, init=True)
repo.add('file.dat')
Expand Down
5 changes: 5 additions & 0 deletions datalad/distribution/tests/test_add.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

"""

from datalad.tests.utils import skip_direct_mode
import logging
from os.path import join as opj

Expand Down Expand Up @@ -116,6 +117,7 @@ def test_add_files(path):


@with_tempfile(mkdir=True)
@skip_direct_mode #FIXME
def test_add_recursive(path):
# make simple hierarchy
parent = Dataset(path).create()
Expand Down Expand Up @@ -147,6 +149,7 @@ def test_add_recursive(path):


@with_tree(**tree_arg)
@skip_direct_mode #FIXME
def test_add_dirty_tree(path):
ds = Dataset(path)
ds.create(force=True, save=False)
Expand Down Expand Up @@ -303,6 +306,7 @@ def test_add_source(path, url, ds_dir):

@with_tree(**tree_arg)
@with_tempfile(mkdir=True)
@skip_direct_mode #FIXME
def test_add_subdataset(path, other):
subds = create(opj(path, 'dir'), force=True)
ds = create(path, force=True)
Expand Down Expand Up @@ -339,6 +343,7 @@ def test_add_subdataset(path, other):
'file2.txt': 'some text to go to annex',
'.gitattributes': '* annex.largefiles=(not(mimetype=text/*))'}
)
@skip_direct_mode #FIXME
def test_add_mimetypes(path):
# XXX apparently there is symlinks dereferencing going on while deducing repo
# type there!!!! so can't use following invocation -- TODO separately
Expand Down
5 changes: 5 additions & 0 deletions datalad/distribution/tests/test_clone.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
"""


from datalad.tests.utils import skip_v6
from datalad.tests.utils import skip_direct_mode
from os.path import join as opj
from os.path import isdir
from os.path import exists
Expand Down Expand Up @@ -120,6 +122,7 @@ def test_clone_datasets_root(tdir):

@with_testrepos('.*basic.*', flavors=['local-url', 'network', 'local'])
@with_tempfile(mkdir=True)
@skip_v6 #FIXME
def test_clone_simple_local(src, path):
origin = Dataset(path)

Expand Down Expand Up @@ -162,6 +165,7 @@ def test_clone_simple_local(src, path):

@with_testrepos(flavors=['local-url', 'network', 'local'])
@with_tempfile
@skip_v6 #FIXME
def test_clone_dataset_from_just_source(url, path):
with chpwd(path, mkdir=True):
ds = clone(url, result_xfm='datasets', return_type='item-or-list')
Expand Down Expand Up @@ -213,6 +217,7 @@ def test_clone_isnot_recursive(src, path_nr, path_r):
# .git/config show a submodule url "file:///aaa/bbb%20b/..."
# this is delivered by with_testrepos as the url to clone
@with_tempfile
@skip_direct_mode #FIXME
def test_clone_into_dataset(source, top_path):

ds = create(top_path)
Expand Down
Loading