implement skip flag for tests, closes #146

CHANGELOG.md

@@ -6,6 +6,8 @@
 
 - Remove executions from report dashboard
 
+- Skip flag to test [#146](https://github.com/golemhq/golem/issues/146)
+
 ### Fixed
 
 - [#169](https://github.com/golemhq/golem/issues/169)

docs/source/tests.md

@@ -21,21 +21,34 @@ tags = []
 
 pages = []
 
+skip = False
+
+
 def setup(data):
     pass
 
+
 def test(data):
     pass
 
+
 def teardown(data):
     pass
-
 ```
 
-A test must implement at least a 'test' function that receives a data object as argument.
+A test must implement at least a **test** function that receives a data object as argument.
+
 
+## Test Data
 
-## Infile Test Data
+Test data can be defined inside the file or in a separate CSV file.
+For detailed info about see: [Test Data](test-data.html) 
+
+### CSV Data
+
+It should be defined in a CSV file with the same name and in the same folder as the test.
+
+### Infile Test Data
 
 A test can have data defined as a list of dictionaries.
 
@@ -59,12 +72,23 @@ def test(data):
 
 Note: when saving a test using the Test Module, if the *test_data* setting is not 'infile', any data stored in the test will be moved to a CSV file.
 
+## Skip flag
+
+A flag variable to indicate that this test should be skipped.
+It should be a boolean or a string to use as skip message.
+Note: tests will only be skipped when running from a suite.
+
+## Tags
+
+A list of tags (strings).
+Tags can be used to filter tests when running a suite.
+See [Filter Tests by Tags](running-tests.html#filter-tests-by-tags).
 
 ## Implicit vs Explicit Imports
 
 By default the test runner imports the golem.actions module and any page module implicitly during the execution.
 Pages are saved as a list of strings.
-The GUI test builder complies with this format and generates code as the following:
+The GUI test builder complies with this format and generates code like the following:
 
 ```python
 pages = ['page1']

golem/core/test.py

@@ -84,7 +84,7 @@ def duplicate_test(project, name, new_name):
     return errors
 
 
-def edit_test(project, test_name, description, pages, steps, test_data, tags):
+def edit_test(project, test_name, description, pages, steps, test_data, tags, skip=False):
     """Save test contents to file"""
 
     def _format_description(description):
@@ -167,6 +167,10 @@ def _format_steps(steps):
                 test_data_module.remove_csv_if_exists(project, test_name)
         else:
             test_data_module.save_external_test_data_file(project, test_name, test_data)
+        if skip:
+            if type(skip) is str:
+                skip = "'{}'".format(skip)
+            f.write('skip = {}\n\n'.format(skip))
         f.write('\n')
         f.write('def setup(data):\n')
         if steps['setup']:
@@ -271,6 +275,10 @@ def steps(self):
             steps['teardown'] = []
         return steps
 
+    @property
+    def skip(self):
+        return getattr(self.get_module(), 'skip', False)
+
     @property
     def components(self):
         """Parse and return the components of a Test in
@@ -287,6 +295,7 @@ def components(self):
             'description': self.description,
             'pages': self.pages,
             'tags': self.tags,
-            'steps': self.steps
+            'steps': self.steps,
+            'skip': self.skip
         }
         return components

golem/gui/api.py

@@ -644,9 +644,10 @@ def test_save():
     test_data_content = request.json['testData']
     test_steps = request.json['steps']
     tags = request.json['tags']
+    skip = request.json['skip']
     _verify_permissions(Permissions.STANDARD, project)
     test_module.edit_test(project, test_name, description, pages, test_steps,
-                          test_data_content, tags)
+                          test_data_content, tags, skip)
     return jsonify('test-saved')
 
 

golem/gui/static/js/main.js

@@ -655,10 +655,6 @@ const Main = new function(){
         'skipped': {
             code: 'skipped',
             color: '#ced4da'
-        },
-        'not run': {
-            code: 'skipped',
-            color: '#868e96'
         }
     }
 

golem/gui/static/js/test_case.js

@@ -281,6 +281,14 @@ var Test = new function(){
                 }
             });
         }
+        let skip = ($("#skipCheckbox").prop('checked'));
+        if(skip){
+            let reason = $("#skipReason").val().trim()
+            if(reason.length){
+                skip = reason
+            }
+        }
+
         let data = {
             'description': description,
             'pages': pageObjects,
@@ -289,7 +297,9 @@ var Test = new function(){
             'project': Test.project,
             'testName': Test.fullName,
             'tags': tags,
+            'skip': skip
         }
+
         $.ajax({
             url: "/api/test/save",
             data: JSON.stringify(data),
@@ -643,5 +653,13 @@ var Test = new function(){
             if(section == 'test') return $("#testSteps .steps")
             if(section == 'teardown') return $("#teardownSteps .steps")
         }
+
+        this.onSkipCheckboxChange = function(){
+            if($("#skipCheckbox").prop('checked')) {
+                $("#skipReason").show();
+            } else {
+                $("#skipReason").hide();
+            }
+        }
     }
 }

golem/gui/templates/test_builder/test_case.html

@@ -39,13 +39,27 @@ <h3 id="testNameContainer">
         <div class="col-sm-6 same-heigth-col gray-1-background" id="descriptionContainer" style="border: 1px solid #e0e0e0;  border-top-left-radius: 3px; ">
             <h4>Description</h4>
             <textarea id="description" class="form-control" rows="3">{{test_components.description }}</textarea>
-            <br>
+            <div class="clearfix" style="margin-bottom: 5px"></div>
             <div class="form-horizontal">
                 <div class="col-xs-2 control-label no-padding-left" style="text-align: left; font-weight: bold">Tags</div>
                 <div class="col-xs-10" style="padding-right: 0px; padding-left: 0px;">
                     <input type="text" class="form-control" id="tags" value="{{ ', '.join(test_components.tags) + ', ' if test_components.tags else '' }}">
                 </div>
             </div>
+            <div class="clearfix" style="margin-bottom: 5px"></div>
+            <div class="form-horizontal" style="height: 34px">
+                <div class="col-xs-2 control-label no-padding-left" style="text-align: left; font-weight: bold">Skip</div>
+                <div class="col-xs-1" style="padding-right: 0px; padding-left: 0px;">
+                    <div class="checkbox">
+                        <input type="checkbox" id="skipCheckbox" style="margin-left: 0px" onchange="Test.Utils.onSkipCheckboxChange()" {% if test_components.skip %}checked{% endif %}>
+                    </div>
+                </div>
+                <div class="col-xs-9" style="padding-right: 0px; padding-left: 0px;">
+                    <input type="text" class="form-control" id="skipReason" {% if not test_components.skip %}style="display: none"{% endif %}
+                           value="{% if test_components.skip is string %}{{test_components.skip}}{% endif %}" placeholder="Skip message (optional)">
+                </div>
+            </div>
+
         </div>
         <div class="col-sm-6 same-heigth-col gray-1-background" id="pageObjectsContainerContainer" style="border-top: 1px solid #e0e0e0; border-right: 1px solid #e0e0e0; border-bottom: 1px solid #e0e0e0; border-top-right-radius: 3px;">
             <h4>Pages</h4>

golem/test_runner/conf.py

@@ -9,3 +9,4 @@ class ResultsEnum:
     SUCCESS = 'success'
     STOPPED = 'stopped'
     NOT_RUN = 'not run'
+    SKIPPED = 'skipped'

golem/test_runner/execution_runner.py

@@ -394,12 +394,14 @@ def _execute(self):
                     run_test(session.testdir, self.project, test.name,
                              test.data_set, test.secrets, test.browser,
                              session.settings, test.reportdir,
-                             self.execution.has_failed_tests, self.execution.tags)
+                             self.execution.has_failed_tests,
+                             self.execution.tags, self.is_suite)
             else:
                 # run tests using multiprocessing
                 multiprocess_executor(self.project, self.execution.tests,
                                       self.execution.has_failed_tests,
-                                      self.execution.processes, self.execution.tags)
+                                      self.execution.processes, self.execution.tags,
+                                      self.is_suite)
 
         # run suite `after` function
         if self.suite.after:

golem/test_runner/multiprocess_executor.py

@@ -9,7 +9,7 @@
 
 
 def multiprocess_executor(project, execution_list, has_failed_tests, processes=1,
-                          tags=None):
+                          tags=None, is_suite=False):
     """Runs a list of tests in parallel using multiprocessing"""
     pool = Pool(processes=processes, maxtasksperchild=1)
     results = []
@@ -23,7 +23,8 @@ def multiprocess_executor(project, execution_list, has_failed_tests, processes=1
                 session.settings,
                 test.reportdir,
                 has_failed_tests,
-                tags)
+                tags,
+                is_suite)
         apply_async = pool.apply_async(run_test, args=args)
         results.append(apply_async)
     map(ApplyResult.wait, results)

golem/test_runner/test_runner.py

@@ -50,21 +50,25 @@ def _get_set_name(test_data):
 
 
 def run_test(testdir, project, test_name, test_data, secrets, browser,
-             settings, report_directory, execution_has_failed_tests=None, tags=None):
+             settings, report_directory, execution_has_failed_tests=None,
+             tags=None, from_suite=False):
     """Run a single test"""
     session.testdir = testdir
     runner = TestRunner(testdir, project, test_name, test_data, secrets, browser,
-                        settings, report_directory, execution_has_failed_tests, tags)
+                        settings, report_directory, execution_has_failed_tests,
+                        tags, from_suite)
     runner.prepare()
 
 
 class TestRunner:
+
     __test__ = False  # ignore this class from Pytest
 
     def __init__(self, testdir, project, test_name, test_data, secrets, browser,
-                 settings, report_directory, execution_has_failed_tests=None, tags=None):
+                 settings, report_directory, execution_has_failed_tests=None,
+                 tags=None, from_suite=False):
         self.result = {
-            'result': '',
+            'result': None,
             'errors': [],
             'description': '',
             'steps': [],
@@ -88,6 +92,7 @@ def __init__(self, testdir, project, test_name, test_data, secrets, browser,
         self.logger = None
         self.execution_has_failed_tests = execution_has_failed_tests
         self.execution_tags = tags or []
+        self.from_suite = from_suite
 
     def prepare(self):
         self.result['set_name'] = _get_set_name(self.test_data)
@@ -145,7 +150,18 @@ def import_modules(self):
                 trcbk = traceback.format_exc()
                 actions._add_error(message=message, description=trcbk)
                 self.result['result'] = ResultsEnum.CODE_ERROR
-        if self.result['result'] == ResultsEnum.CODE_ERROR:
+
+            # check for skip flag
+            # test is skipped only when run from a suite
+            skip = getattr(self.test_module, 'skip', False)
+            if skip and self.from_suite:
+                self.result['result'] = ResultsEnum.SKIPPED
+                msg = 'Skip: {}'.format(skip) if type(skip) is str else 'Skip'
+                execution.logger.info(msg)
+
+
+
+        if self.result['result'] in [ResultsEnum.CODE_ERROR, ResultsEnum.SKIPPED]:
             self.finalize()
         else:
             self.run_setup()
@@ -222,8 +238,9 @@ def finalize(self):
         if self.result['result'] not in [ResultsEnum.CODE_ERROR, ResultsEnum.FAILURE]:
             if execution.errors:
                 self.result['result'] = ResultsEnum.ERROR
-            else:
-                self.result['result'] = ResultsEnum.SUCCESS
+
+        if self.result['result'] is None:
+            self.result['result'] = ResultsEnum.SUCCESS
         execution.logger.info('Test Result: {}'.format(self.result['result'].upper()))
 
         self.result['description'] = execution.description

tests/conftest.py

@@ -242,9 +242,11 @@ def create_random_suite(project):
         return suite_name
 
     @staticmethod
-    def create_random_page(project):
+    def create_random_page(project, code=None):
         page_name = TestUtils.random_string(10)
         page.create_page(project, page_name)
+        if code is not None:
+            page.edit_page_code(project, page_name, code)
         return page_name