Cleaning running atests. Not yet ready but getting there... #2091

atest/resources/TestCheckerLibrary.py

@@ -24,19 +24,24 @@ class NoSlotsTestSuite(TestSuite):
 
 class TestCheckerLibrary:
 
-    def process_output(self, path):
+    def process_output(self, path, process=True):
+        set_suite_variable = BuiltIn().set_suite_variable
+        if not utils.is_truthy(process):
+            set_suite_variable('$SUITE', None)
+            print "Not processing output."
+            return
         path = path.replace('/', os.sep)
         try:
             print "Processing output '%s'" % path
             result = Result(root_suite=NoSlotsTestSuite())
             ExecutionResultBuilder(path).build(result)
         except:
+            set_suite_variable('$SUITE', None)
             raise RuntimeError('Processing output failed: %s'
                                % utils.get_error_message())
-        setter = BuiltIn().set_suite_variable
-        setter('$SUITE', process_suite(result.suite))
-        setter('$STATISTICS', result.statistics)
-        setter('$ERRORS', process_errors(result.errors))
+        set_suite_variable('$SUITE', process_suite(result.suite))
+        set_suite_variable('$STATISTICS', result.statistics)
+        set_suite_variable('$ERRORS', process_errors(result.errors))
 
     def get_test_from_suite(self, suite, name):
         tests = self.get_tests_from_suite(suite, name)

atest/resources/TestHelper.py

@@ -12,16 +12,6 @@ def set_read_only(self, path):
     def set_read_write(self, path):
         os.chmod(path, S_IREAD | S_IWRITE)
 
-    def get_output_name(self, *datasources):
-        if not datasources:
-            raise RuntimeError('One or more data sources must be given!')
-        if len(datasources) == 1:
-            return self._get_name(datasources[0])
-        return '_'.join(self._get_name(source) for source in datasources)
-
-    def _get_name(self, path):
-        return os.path.splitext(os.path.basename(path))[0]
-
     def file_should_have_correct_line_separators(self, output, sep=os.linesep):
         if os.path.isfile(output):
             with open(output, 'rb') as infile:

atest/resources/atest_resource.robot

@@ -11,21 +11,21 @@ Library         read_interpreter.py
 Variables       atest_variables.py
 
 *** Variables ***
-# FIXME: Variables below are currently only used as defaults and overridden in
-# "Set Variables". Should rather use same values always but should also decide
-# what are good values.
-${OUTDIR}       %{TEMPDIR}
-${OUTFILE}      %{TEMPDIR}${/}output.xml
-${SYSLOG FILE}  %{TEMPDIR}${/}syslog.txt
-${STDERR FILE}  %{TEMPDIR}${/}stdout.txt
-${STDOUT FILE}  %{TEMPDIR}${/}stderr.txt
-
+${OUTDIR}       %{TEMPDIR}/output
+${OUTFILE}      ${OUTDIR}${/}output.xml
+${SYSLOG FILE}  ${OUTDIR}${/}syslog.txt
+${STDERR FILE}  ${OUTDIR}${/}stdout.txt
+${STDOUT FILE}  ${OUTDIR}${/}stderr.txt
+${OUTFILE COPY}  %{TEMPDIR}/output-copy.xml
 ${SUITE}        Set in Run Helper
 ${ERRORS}       -- ;; --
-${LIBPATH1}     ${CURDIR}${/}..${/}testresources${/}testlibs
-${LIBPATH2}     ${CURDIR}${/}..${/}testresources${/}listeners
 ${USAGE_TIP}    \n\nTry --help for usage information.
 ${TESTNAME}     ${EMPTY}    # Used when not running test
+${RUNNER DEFAULTS}
+...             --ConsoleMarkers OFF
+...             --PYTHONPATH "${CURDIR}${/}..${/}testresources${/}testlibs"
+...             --PYTHONPATH "${CURDIR}${/}..${/}testresources${/}listeners"
+${REBOT DEFAULTS}    ${EMPTY}
 
 *** Keywords ***
 Run Robot Directly
@@ -45,80 +45,61 @@ Run Rebot Directly
     [Return]  ${result}
 
 Run Tests
-    [Arguments]  ${options}  @{data list}
-    ${rc} =  Run Tests Helper  ${options}  @{data list}
-    Process Output  ${OUTFILE}
-    [Return]  ${rc}
+    [Arguments]  ${options}  @{datasources}  &{config}
+    [Documentation]    *OUTDIR:* file://${OUTDIR} (regenerated for every run)
+    @{arguments} =    Get Execution Arguments    ${options}    ${config}
+    ...    ${RUNNER DEFAULTS}    @{datasources}
+    ${result} =  Run Process  @{INTERPRETER.runner}    @{arguments}
+    ...    stdout=${STDOUTFILE}  stderr=${STDERRFILE}    timeout=5min    on_timeout=terminate
+    Process Output    ${OUTFILE}    ${config.pop('process_output', True)}
+    Log    ${result.stdout}
+    Log    ${result.stderr}
+    [Return]  ${result.rc}   # TODO: Return result, not only rc. Also elsewhere.
+
+Get Execution Arguments
+    [Arguments]    ${options}     ${config}    ${defaults}    @{sources}
+    ${defaults} =    Catenate
+    ...  --consolecolors  OFF
+    ...  --outputdir  ${OUTDIR}
+    ...  --output  ${OUTFILE}
+    ...  --report  NONE
+    ...  --log  NONE
+    ...  ${defaults}
+    ${defaults} =    Pop From Dictionary    ${config}    default_options    ${defaults}
+    @{options} =   Command line to list    ${defaults} ${options}
+    @{sources} =  Join Paths  ${DATADIR}  @{sources}
+    Setup Execution Environment
+    [Return]    @{options}    @{sources}
+
+Setup Execution Environment
+    Remove Directory    ${OUTDIR}    recursive
+    Create Directory    ${OUTDIR}
+    Set Environment Variable    ROBOT_SYSLOG_FILE    ${SYSLOG_FILE}
 
 Run Tests Without Processing Output
-    [Arguments]  ${options}  @{data list}
-    ${rc} =  Run Tests Helper  ${options}  @{data list}
+    [Arguments]  ${options}  @{datasources}
+    ${rc} =    Run Tests    ${options}    @{datasources}    process_output=False
     [Return]  ${rc}
 
-Run Tests Helper
-    [Arguments]  ${user options}  @{data list}
-    @{data list} =  Set Variables And Get Datasources  @{data list}
-    @{user options} =   Command line to list  ${user options}
-    ${result} =  Run Helper  ${INTERPRETER.runner}
-    ...    --ConsoleMarkers    OFF    # AUTO (default) doesn't work with IronPython
-    ...    @{user options}
-    ...    --pythonpath    ${LIBPATH1}
-    ...    --pythonpath    ${LIBPATH2}
-    ...    @{data list}
-    [Return]  ${result.rc}
-
 Run Rebot
-    [Arguments]  ${options}  @{data list}
-    @{data list} =  Set Variables And Get Datasources  @{data list}
-    @{options} =   Command line to list  ${options}
-    ${result} =  Run Helper  ${INTERPRETER.rebot}  @{options}  @{data list}
-    Process Output  ${OUTFILE}
-    [Return]  ${result.rc}
+    [Arguments]  ${options}  @{datasources}  &{config}
+    [Documentation]    *OUTDIR:* file://${OUTDIR} (regenerated for every run)
+    @{arguments} =    Get Execution Arguments    ${options}    ${config}
+    ...    ${REBOT DEFAULTS}    @{datasources}
+    ${result} =  Run Process  @{INTERPRETER.rebot}    @{arguments}
+    ...    stdout=${STDOUTFILE}  stderr=${STDERRFILE}    timeout=5min    on_timeout=terminate
+    Process Output    ${OUTFILE}    ${config.pop('process_output', True)}
+    Log    ${result.stdout}
+    Log    ${result.stderr}
+    [Return]  ${result.rc}   # TODO: Return result, not only rc. Also elsewhere.
 
 Run Rebot Without Processing Output
-    [Arguments]  ${options}  @{data list}
-    @{data list} =  Set Variables And Get Datasources  @{data list}
-    @{options} =   Command line to list  ${options}
-    ${result} =  Run Helper  ${INTERPRETER.rebot}  @{options}  @{data list}
-    [Return]  ${result.rc}
-
-Run Helper
-    [Arguments]  ${runner}  @{arguments}
-    Remove Files  ${OUTFILE}  ${OUTDIR}/*.xml  ${OUTDIR}/*.html
-    ${cmd} =  Create list  @{runner}
-    ...  --consolecolors  OFF
-    ...  --outputdir  ${OUTDIR}
-    ...  --output  ${OUTFILE}
-    ...  --report  NONE
-    ...  --log  NONE
-    ...  @{arguments}
-    ${result} =  Run Process  @{cmd}   stdout=${STDOUTFILE}  stderr=${STDERRFILE}
-    ...    timeout=5min    on_timeout=terminate
-    Log  <a href="file://${OUTDIR}">${OUTDIR}</a>  HTML
-    Log  <a href="file://${OUTFILE}">${OUTFILE}</a>  HTML
-    Log  <a href="file://${STDOUTFILE}">${STDOUTFILE}</a>  HTML
-    Log  <a href="file://${STDERRFILE}">${STDERRFILE}</a>  HTML
-    Log  <a href="file://${SYSLOGFILE}">${SYSLOGFILE}</a>  HTML
-    [Return]  ${result}
-
-Set Variables And Get Datasources
-    [Arguments]  @{data list}
-    Set Suite Variable  $SUITE  ${NONE}
-    ${name} =  Get Output Name  @{data list}
-    Set Variables  ${name}
-    @{data list} =  Join Paths  ${DATADIR}  @{data list}
-    [Return]  @{data list}
+    [Arguments]  ${options}  @{datasources}
+    ${rc} =    Run Rebot    ${options}    @{datasources}    process_output=False
+    [Return]  ${rc}
 
-Set Variables
-    [Arguments]  ${name}
-    ${OUTDIR} =  Join Path  ${OUTPUTDIR}  output  ${name}
-    Set Global Variable  $OUTDIR  ${OUTDIR.encode('ascii', 'ignore').replace('?', '_') .replace('*', '_')}
-    Create Directory  ${OUTDIR}
-    Set Suite Variable  $OUTFILE  ${OUTDIR}${/}output.xml
-    Set Suite Variable  $STDOUT_FILE  ${OUTDIR}${/}stdout.txt
-    Set Suite Variable  $STDERR_FILE  ${OUTDIR}${/}stderr.txt
-    Set Suite Variable  $SYSLOG_FILE  ${OUTDIR}${/}syslog.txt
-    Set Environment Variable  ROBOT_SYSLOG_FILE  ${SYSLOG_FILE}
+Copy Previous Outfile
+    Copy File    ${OUTFILE}    ${OUTFILE COPY}
 
 Check Test Case
     [Arguments]  ${name}=${TESTNAME}  ${status}=${NONE}  ${message}=${NONE}

atest/robot/cli/runner/argumentfile.robot

@@ -53,7 +53,7 @@ Arguments From Stdin
 Option And Argument File Together
     Create Argument File  ${ARGFILE}  --name My name
     Create Argument File  ${ARGFILE2}  --doc My docu
-    Run Tests  --argumentfile=${ARGFILE} -A${ARGFILE2}  ${TESTFILE}
+    Run Tests  --argumentfile\=${ARGFILE} -A${ARGFILE2}  ${TESTFILE}
     Should Be Equal  ${SUITE.name}  My name
     Should Be Equal  ${SUITE.doc}  My docu
 

atest/robot/core/suite_setup_and_teardown.robot

@@ -148,8 +148,9 @@ Failed teardown is noticed when generating only report with Rebot
     ${rc} =    Run Tests Without Processing Output
     ...    --report report.html    core/failing_suite_teardown.robot
     Should Be Equal As Integers    ${rc}    2
+    Copy Previous Outfile
     ${rc} =    Run Rebot Without Processing Output
-    ...    --report report.html --output NONE    ${OUTFILE}
+    ...    --report report.html --output NONE    ${OUTFILE COPY}
     Should Be Equal As Integers    ${rc}    2
 
 Long Error Messages

atest/robot/external/unit_tests.robot

@@ -1,5 +1,4 @@
 *** Settings ***
-Suite Setup     Set Variables    unit
 Force Tags      no-standalone
 Resource        atest_resource.robot
 

atest/robot/output/flatten_keyword.robot

@@ -56,7 +56,7 @@ Flattened in log after execution
     Should Contain    ${LOG}    *<p>Logs the given message with the given level.\\x3c/p>\\n${FLAT HTML}
 
 Flatten for loops
-    Run Rebot    --flatten For    ${ORIG OUTFILE}
+    Run Rebot    --flatten For    ${OUTFILE COPY}
     ${tc} =    Check Test Case    For loop
     Should Be Equal    ${tc.kws[0].type}    for
     Should Be Equal    ${tc.kws[0].doc}    ${FLAT TEXT}
@@ -71,7 +71,7 @@ Flatten for loops
     \    Check Log Message    ${tc.kws[0].msgs[${index * 6 + 5}]}    1
 
 Flatten for loop items
-    Run Rebot    --flatten ForItem    ${ORIG OUTFILE}
+    Run Rebot    --flatten ForItem    ${OUTFILE COPY}
     ${tc} =    Check Test Case    For loop
     Should Be Equal    ${tc.kws[0].type}    for
     Should Be Empty    ${tc.kws[0].doc}
@@ -90,7 +90,7 @@ Flatten for loop items
     \    Check Log Message    ${tc.kws[0].kws[${index}].msgs[5]}    1
 
 Invalid usage
-    Run Rebot Without Processing Output    ${FLATTEN} --FlattenKeywords invalid   ${ORIG OUTFILE}
+    Run Rebot Without Processing Output    ${FLATTEN} --FlattenKeywords invalid   ${OUTFILE COPY}
     Stderr Should Be Equal To    ${ERROR}
     Run Tests Without Processing Output    ${FLATTEN} --FlattenKeywords invalid   output/flatten_keywords.robot
     Stderr Should Be Equal To    ${ERROR}
@@ -100,7 +100,7 @@ Run And Rebot Flattened
     Run Tests Without Processing Output     ${FLATTEN}    output/flatten_keywords.robot
     ${LOG} =    Get File    ${OUTDIR}/log.html
     Set Suite Variable    $LOG
-    Set Suite Variable    $ORIG OUTFILE    ${OUTFILE}
-    Run Rebot    ${FLATTEN}    ${OUTFILE}
+    Copy Previous Outfile
+    Run Rebot    ${FLATTEN}    ${OUTFILE COPY}
     ${TC} =    Check Test Case    Flatten stuff
     Set Suite Variable    $TC

atest/robot/output/processing_output.robot

@@ -45,7 +45,8 @@ Minimal hand-created output
 My Run Robot And Rebot
     [Arguments]  ${params}  @{paths}
     Run Tests Without Processing Output  ${params}  @{paths}
-    Run Rebot  ${EMPTY}  ${OUTFILE}
+    Copy Previous Outfile
+    Run Rebot  ${EMPTY}  ${OUTFILE COPY}
 
 Check Normal Suite Defaults
     [Arguments]  ${mysuite}  ${message}=  ${tests}=[]  ${setup}=${None}  ${teardown}=${None}
@@ -98,4 +99,3 @@ Check Suite Got From Misc/suites/ Directory
     Check Normal Suite Defaults  ${SUITE.suites[2].suites[0]}
     Check Normal Suite Defaults  ${SUITE.suites[3]}
     Check Normal Suite Defaults  ${SUITE.suites[4]}
-

atest/robot/output/statistics_with_rebot.robot

@@ -47,15 +47,15 @@ Suite statistics should be Correct
 *** Keywords ***
 My Setup
     Run Tests    ${EMPTY}    misc/suites
-    Move File    ${OUTFILE}    ${OUTDIR}/statistics.xml
+    Copy Previous Outfile
     ${options} =    Catenate
     ...    --critical t1
     ...    --tagstatcombine d1_AND_d2:Custom_title_AND-OR-NOT
     ...    --suitestatlevel 2
     ...    --tagstatexclude t2
     ...    --TagStatComb F1NOTT_1
     ...    --SetTag XxX
-    Run Rebot    ${options}    ${OUTDIR}/statistics.xml
+    Run Rebot    ${options}    ${OUTFILE COPY}
 
 Node Should Be Correct
     [Arguments]    ${node}    ${name}    ${pass}    ${fail}

atest/robot/output/suite_and_test_id_in_output.robot

@@ -7,7 +7,8 @@ Ids in output after test run
     Suite And Test Ids Should Be Correct
 
 Ids in output after rebot
-    Run Rebot    ${EMPTY}    ${OUTFILE}
+    Copy Previous Outfile
+    Run Rebot    ${EMPTY}    ${OUTFILE COPY}
     Suite And Test Ids Should Be Correct
 
 *** Keywords ***

atest/robot/rebot/filter_by_names.robot

@@ -96,7 +96,8 @@ Elapsed Time
     Check Times  ${SUITE}  20061227 11:59:59.000  20061227 12:00:08.999  9999  # Suite
     Should Be Equal As Integers  ${SUITE.test_count}  6
     Comment  2) Filter output created in earlier step and check  that times are set accordingly.
-    Run Rebot  --test ExcSTAR --test Incl-1 ${ESCAPES}  ${OUTFILE}
+    Copy Previous Outfile
+    Run Rebot  --test ExcSTAR --test Incl-1 ${ESCAPES}  ${OUTFILE COPY}
     Check Times  ${SUITE.tests[0]}  20061227 12:00:00.000  20061227 12:00:01.000  1000  # Incl-1
     Check Times  ${SUITE.tests[1]}  20061227 12:00:07.000  20061227 12:00:07.001  0001  # Excl-1
     Check Times  ${SUITE.tests[2]}  20061227 12:00:07.001  20061227 12:00:07.003  0002  # Excl-12

atest/robot/rebot/output_file.robot

@@ -13,7 +13,8 @@ Generate output with Robot
     Run keyword and return    Parse output file
 
 Generate output with Rebot
-    Run rebot    -c fail    ${OUTFILE}
+    Copy Previous Outfile
+    Run rebot    -c fail    ${OUTFILE COPY}
     Run keyword and return    Parse output file
 
 Parse output file

atest/robot/rebot/start_and_endtime_from_cli.robot

@@ -6,6 +6,7 @@ Resource        rebot_resource.robot
 *** Variables ***
 ${INPUT1}       %{TEMPDIR}${/}rebot-test-a.xml
 ${INPUT2}       %{TEMPDIR}${/}rebot-test-b.xml
+${COMBINED}     %{TEMPDIR}${/}combined.xml
 
 *** Test Cases ***
 Combine With Both Starttime and endtime should Set Correct Elapsed Time
@@ -28,8 +29,8 @@ Combine With Only Endtime Should Only Affect Endtime
     Should Be Equal  ${SUITE.elapsedtime}  ${ORIG_ELAPSED}
 
 Recombining Should Work
-    Run Rebot Without Processing Output  --starttime 2007:09:25:21:51 --endtime 2007:09:26:01:12:30:200 --output once_combined.xml  ${INPUT1}  ${INPUT2}
-    Run Rebot  ${EMPTY}  ${INPUT1}  ${INPUT2}  ${OUTDIR}${/}once_combined.xml
+    Run Rebot Without Processing Output  --starttime 2007:09:25:21:51 --endtime 2007:09:26:01:12:30:200 --output ${COMBINED}  ${INPUT1}  ${INPUT2}
+    Run Rebot  ${EMPTY}  ${INPUT1}  ${INPUT2}  ${COMBINED}
     Should Be True  '${SUITE.elapsedtime}' > '03:21:30.200'
 
 It should Be possible to Omit Time Altogether

atest/robot/running/nolog.py

@@ -1,20 +1,23 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*- 
+import os
 
-def difference_between_stuff(file1, file2):
-    with open(file1) as f1:
+
+def reports_should_be_equal(file1, file2):
+    with open(file1.replace('/', os.sep)) as f1:
         content1 = f1.readlines()
-        with open(file2) as f2:
+        with open(file2.replace('/', os.sep)) as f2:
             content2 = f2.readlines()
-            for l1,l2 in zip(content1, content2):
-                if 'generatedTimestamp' in l1:
-                    continue
-                if 'generatedMillis' in l1:
-                    continue
-                if l1 != l2:
+            for l1, l2 in zip(content1, content2):
+                if not _lines_are_equal(l1, l2):
                     raise AssertionError('%r\n is not same as\n%r' % (l1, l2))
             if len(content1) != len(content2):
                 raise AssertionError("file %r len %d is different "
                                      "than file %r len %d" %
                                      (file1, len(content1),
                                       file2, len(content2)))
+
+
+def _lines_are_equal(line1, line2):
+    for changing in ('generatedTimestamp', 'generatedMillis'):
+        if changing in line1 and changing in line2:
+            return True
+    return line1 == line2