Skip to content

Commit

Permalink
[TRAVIS] run flaky test when label is defined (#9509)
Browse files Browse the repository at this point in the history
to declare a test as flaky:

* for cpp, use

```
if ( !QgsTest::runFlakyTests() )
    QSKIP( "This test is disabled on Travis CI environment" );
```

* for Python, you can use `RUN_FLAKY_TEST` environment variable
  • Loading branch information
3nids authored Mar 27, 2019
1 parent b9fdd02 commit 7fb752e
Show file tree
Hide file tree
Showing 6 changed files with 46 additions and 3 deletions.
3 changes: 2 additions & 1 deletion .ci/travis/linux/docker-variables.env
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

# Env variables for Docker
# These without assignment are taken from the host env variabbles
# These without assignment are taken from the host env variables

# TRAVIS variables
TRAVIS_AVAILABLE_TIME
Expand All @@ -11,6 +11,7 @@ TRAVIS_PULL_REQUEST
TRAVIS_OS_NAME
TRAVIS_CONFIG
TRAVIS
RUN_FLAKY_TESTS

# CTEST
LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
Expand Down
2 changes: 2 additions & 0 deletions .ci/travis/linux/script.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

set -e

echo "Running flaky test: ${RUN_FLAKY_TESTS}"

# build QGIS in docker
echo "travis_fold:start:docker_build_qgis"
echo "${bold}Docker build QGIS${endbold}"
Expand Down
33 changes: 33 additions & 0 deletions .ci/travis/scripts/pr_has_label.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env python3

import sys
import json
from urllib.request import urlopen # using urllib since it is a standard module (vs. requests)
from urllib.error import URLError
import argparse

parser = argparse.ArgumentParser(description='Determines if a pull request has a defined label')
parser.add_argument('pull_request', type=int,
help='pull request id')
parser.add_argument('label', type=int,
help='label ID')

args = parser.parse_args()

url = "https://api.github.com/repos/qgis/QGIS/pulls/{}".format(args.pull_request)

try:
data = urlopen(url).read().decode('utf-8')
except URLError as err:
print("URLError: ".format(err.reason))
sys.exit(1)

obj = json.loads(data)

for label in obj['labels']:
if label["id"] == args.label:
print("true")
sys.exit(0)

print("label not found")
sys.exit(1)
4 changes: 3 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ cache:
- ${HOME}/.ccache_docker_build_cosmic
- ${HOME}/.ccache_docker_build_bionic
timeout: 1000
if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/ AND NOT branch =~ /-patch-\d+$/
if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/ AND NOT branch =~ /-patch-\d+$/

env:
global:
Expand Down Expand Up @@ -46,6 +46,8 @@ matrix:
- CCACHE_DIR=${HOME}/.ccache_testing
- DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile
# Label ID can be found here https://api.github.com/repos/qgis/QGIS/labels
- RUN_FLAKY_TESTS=$(.ci/travis/scripts/pr_has_label.py $TRAVIS_PULL_REQUEST 1271248184)

##########################################################
# CODE LAYOUT
Expand Down
5 changes: 5 additions & 0 deletions src/test/qgstest.h
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ namespace QgsTest
{
return qgetenv( "TRAVIS" ) == QStringLiteral( "true" );
}

bool runFlakyTests()
{
return qgetenv( "RUN_FLAKY_TESTS" ) == QStringLiteral( "true" );
}
}

#endif // QGSTEST_H
2 changes: 1 addition & 1 deletion tests/src/providers/testqgsogrprovider.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ void TestQgsOgrProvider::testThread()
// Disabled by @m-kuhn
// This test is flaky
// See https://travis-ci.org/qgis/QGIS/jobs/505008602#L6464-L7108
if ( QgsTest::isTravis() )
if ( !QgsTest::runFlakyTests() )
QSKIP( "This test is disabled on Travis CI environment" );

// After reading a QgsVectorLayer (getFeatures) from another thread the QgsOgrConnPoolGroup starts
Expand Down

0 comments on commit 7fb752e

Please sign in to comment.