diff --git a/.gitignore b/.gitignore index eacc860..f17625c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +__pycache__/ +.cache/ +bin/env_vars.sh .tox *.pyc *.swp diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b1dba3a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Dockerfile to build container with Codebender selenium tests. + +FROM ubuntu:14.04 +# TODO: add MAINTAINER + +# Install requirements and their dependencies +RUN apt-get update -y +RUN apt-get install -y \ + gcc \ + libffi-dev \ + python \ + python-dev \ + python-setuptools \ + openssl \ + libssl-dev \ + mailutils \ + ssmtp \ + sharutils + +RUN easy_install pip +RUN pip install --upgrade pip +#RUN pip install -U setuptools + +# Add source code and install dependencies +RUN mkdir -p /opt/codebender +ADD . /opt/codebender/seleniumTests + +WORKDIR /opt/codebender/seleniumTests + +RUN pip install -r requirements.txt + +COPY ssmtp.conf /etc/ssmtp/ + +# Specify a default command for the container. +# Right now we simply run bash. TODO: add ENTRYPOINT for running tests. + +#CMD ["/bin/bash"] diff --git a/README.md b/README.md index 0866f75..4cc8b3b 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,144 @@ # Codebender Selenium Tests -This repo contains Selenium tests for the codebender website. -The tests are written in Python 3. +This repo contains Selenium tests for the codebender website. The tests are +written in Python 2, and utilize pytest as a testing framework and Selenium +for browser automation. ## Running Tests -To run tests locally, you'll need to be running a selenium server. See -[here](https://selenium-python.readthedocs.org/installation.html#downloading-selenium-server) -for instructions. +Tests are run by invoking `tox`. Run `tox --help` to see all the +Codebender-specific arguments that can be passed to `py.test`. -Once you've got a Selenium server running, simply run `$ tox` from within the -repo. If you don't have tox, run `$ sudo pip3 install -r requirements-dev.txt` -from within the repo to install it. +In addition to these arguments, there are certain environment variables that +should be set when running tests: + +- `CODEBENDER_SELENIUM_HUB_URL`: the URL of the Selenium Hub. If you are using + SauceLabs, the URL has the following format: + `http://{USERNAME}:{ACCESS_KEY}@ondemand.saucelabs.com:80/wd/hub`. You can + also use a [docker-selenium](https://github.com/SeleniumHQ/docker-selenium) + hub. In that case, it is necessary to link the docker-selenium instance to the + Docker instance from which tests are running. +- `CODEBENDER_TEST_USER`: username that the webdriver will use to log into the + site in order to perform tests. +- `CODEBENDER_TEST_PASS`: password for `CODEBENDER_TEST_USER`. + +Rather than invoking `tox` directly, the easiest way to run tests is with +Docker. If you are not familiar with Docker, please consult the +[documentation](http://docs.docker.com/) for an introduction. + +First, build the image with `$ docker build . -t codebender/selenium`. + +Then invoke `tox` via `docker run`. Here is a sample command to run all tests, +where the Codebender server is running at `http://192.168.1.2:8080`: + +``` +$ docker run -e CODEBENDER_SELENIUM_HUB_URL=http://johndoe:12345678-1234-1234-1234-12345678910@ondemand.saucelabs.com:80/wd/hub \ + -e CODEBENDER_TEST_USER=tester \ + -e CODEBENDER_TEST_PASS=1234 \ + -it codebender/selenium \ + tox -- --url http://192.168.1.2:8080 --source bachelor +``` + +### Running Tests Manually + +The recommended way of running tests is with Docker. If you would like to +manually provision your machine to be able to run tests, you can use the +Dockerfile as a step-by-step guide for provisioning. Then invoke `tox` to run +tests. + +#### Specifying a URL for Tests + +Tests can either be run for the +[bachelor](https://github.com/codebendercc/bachelor) version of the site, +running locally, or for the live site. The version of the site that is running +is inferred from the `--url` parameter. You can run `$ tox --url +http://localhost` to run the tests for a locally running bachelor site (this is +the default url), or `$ tox --url http://codebender.cc` to run the tests for the +live site. + +Certain tests are specially written for one site or the other. This is +implemented with a custom `pytest` marker. Tests that require a certain `--url` +are decorated with `@pytest.mark.requires_url()`. + +### Changing Test Configuration + +Various global configuration parameters are specified in +`codebender_testing/config.py`. Such parameters include URLs and site endpoints +which are subject to change. This is also where the webdrivers (Firefox and +Chrome) are specified. + +## Compilation Logs + +Certain tests exist to iterate through groups of sketches and compile them +one-by-one. Since these tests take a long time, they are not run in full by +default. You can run them by specifying the `--full` option; for example: `$ tox +tests/cb_compile_tester --full`. + +The following test cases are compile tests that generate such logs: +- `tests/libraries/test_libraries.py::TestLibraryExamples` +- `tests/compile_tester/test_compile_tester_projects.py::TestCompileTester` + +The generated logs are placed in the `logs` directory. They give detailed output +in JSON format containing the codebender site URL that was used to run the +tests, along with the URLs of the individual sketches that were compiled, and +whether they succeeded or failed to compile. + +## Framework Overview + +The following outlines the structure of the repository as well as important +framework components. + +### Directory Structure + +#### `tests/` + +The `tests/` directory contains all of the actual unit tests for the codebender +site. That is, all of the tests discovered by `py.test` should come from this +directory. + +**`tests/conftest.py`** contains the global configuration for pytest, +including specifying the webdriver fixtures as well as the available command +line arguments. + +#### `codebender_testing/` + +This is where all major components of the testing framework live. All of the +unit tests rely on the files in this directory. + +**`codebender_testing/config.py`** specifies global configuration parameters for +testing (see "Changing Test Configuration" above). + +**`codebender_testing/utils.py`** defines codebender-specific utilities used to +test the site. These mostly consist of abstractions to the Selenium framework. +The most important class is `SeleniumTestCase`, which all of the unit test cases +inherit from. This grants them access (via `self`) to a number of methods and +attributes that are useful for performing codebender-specific actions. + +**`codebender_testing/capabilities.yaml`** defines a list of `capabilities` to +be passed as arguments when instantiating remote webdrivers. In particular, it +specifies the web browsers that we would like to use. Consult this file for more +information. + +#### `batch/` + +The `batch/` directory contains any executable scripts not directly used to +perform tests. For example, it contains a script `fetch_projects.py` which can +be used to download all of the public projects of a particular codebender user. + +#### `extensions/` + +The `extensions/` directory contains the codebender browser extensions to be +used by the Selenium webdrivers. + +#### `test_data/` + +The `test_data/` directory contains any data used for testing. For example, it +contains example projects that we should successfully be able to upload and +compile. + +#### `logs/` + +The `logs/` directory contains the results of running certain tests, e.g. +whether certain sets of sketches have compiled successfully (see "Compilation +Logs"). -When running tox, you might get a `pkg_resources.DistributionNotFound` error -with reference to `virtualenv`. This is likely due to an out of date setuptools. -To fix this issue, run `$ sudo pip3 install -U setuptools`. diff --git a/tests/home/__init__.py b/batch/__init__.py similarity index 100% rename from tests/home/__init__.py rename to batch/__init__.py diff --git a/batch/fetch_projects.py b/batch/fetch_projects.py new file mode 100755 index 0000000..a2c1df1 --- /dev/null +++ b/batch/fetch_projects.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""A script to downloads all projects for a particular user.""" + +# This is necessary in order to run the script; otherwise the script needs to +# be run as a python module (which is inconvenient). +if __name__ == '__main__' and __package__ is None: + from os import sys, path + sys.path.append(path.join(path.dirname(path.abspath(__file__)), '..')) + + +from urllib.request import urlopen +from urllib.request import urlretrieve +import argparse +import os + +from codebender_testing.config import LIVE_SITE_URL + +from lxml import html + + +def download_projects(url, user, path): + connection = urlopen('/'.join([url, 'user', user])) + dom = html.fromstring(connection.read().decode('utf8')) + os.chdir(path) + for link in dom.xpath('//table[@id="user_projects"]//a'): + project_name = link.xpath('text()')[0] + sketch_num = link.xpath('@href')[0].split(':')[-1] + print("Downloading %s (sketch %s)" % (project_name, sketch_num)) + urlretrieve('%s/utilities/download/%s' % (url, sketch_num), + os.path.join(path, '%s.zip' % project_name)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("user", help="the user whose projects we want to download") + parser.add_argument("-u", "--url", help="url of the codebender site to use", + default=LIVE_SITE_URL) + parser.add_argument("-d", "--directory", help="output directory of the downloaded projects", + default=".") + args = parser.parse_args() + download_projects(args.url, args.user, args.directory) diff --git a/batch/requirements-batch.txt b/batch/requirements-batch.txt new file mode 100644 index 0000000..58d87b8 --- /dev/null +++ b/batch/requirements-batch.txt @@ -0,0 +1,6 @@ +# Requirements for running batch scripts. + +# If lxml fails to install in Ubuntu, install the following dependencies with +# apt-get: libxml2-dev libxslt1-dev python3-dev + +lxml diff --git a/bin/env_vars.sh.template b/bin/env_vars.sh.template new file mode 100755 index 0000000..374b5b0 --- /dev/null +++ b/bin/env_vars.sh.template @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +SAUCELABS_USER="" +SAUCELABS_KEY="" +export CODEBENDER_SELENIUM_HUB_URL=http://${SAUCELABS_USER}:${SAUCELABS_KEY}@ondemand.saucelabs.com:80/wd/hub +export CODEBENDER_TEST_USER="" +export CODEBENDER_TEST_PASS="" +export DISQUS_ACCESS_TOKEN="" +export DISQUS_API_SECRET="" +export DISQUS_API_PUBLIC="" +export DISQUS_SSO_ID="" +export DISQUS_SSO_USERNAME="" +export DISQUS_SSO_EMAIL="" +export EMAIL="" diff --git a/bin/test_common.sh b/bin/test_common.sh new file mode 100755 index 0000000..9aca9e3 --- /dev/null +++ b/bin/test_common.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +source ./env_vars.sh +cd .. +time tox tests/common -- --url=https://codebender.cc --source=codebender_cc +RETVAL=$? +cd - +echo "tests return value: ${RETVAL}" +exit ${RETVAL} diff --git a/bin/test_examples.sh b/bin/test_examples.sh new file mode 100755 index 0000000..3676e63 --- /dev/null +++ b/bin/test_examples.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source ./env_vars.sh +export CAPABILITIES='capabilities_firefox.yaml' +export CODEBENDER_SELENIUM_HUB_URL="http://127.0.0.1:4444/wd/hub" +cd .. +time tox tests/libraries -- --url=https://codebender.cc --source=codebender_cc -F +RETVAL=$? +cd - +echo "tests return value: ${RETVAL}" +exit ${RETVAL} diff --git a/bin/test_sketches.sh b/bin/test_sketches.sh new file mode 100755 index 0000000..f33acfd --- /dev/null +++ b/bin/test_sketches.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + + +source ./env_vars.sh +export CAPABILITIES='capabilities_firefox.yaml' +cd .. +time tox tests/compile_tester -- --url=https://codebender.cc --source=codebender_cc -F +RETVAL=$? +cd - +echo "tests return value: ${RETVAL}" +exit ${RETVAL} diff --git a/codebender_testing/capabilities.yaml b/codebender_testing/capabilities.yaml new file mode 100644 index 0000000..46468b8 --- /dev/null +++ b/codebender_testing/capabilities.yaml @@ -0,0 +1,18 @@ +# This file contains a list of capabilities which will be used to instantiate +# the remote selenium webdrivers. +# Each list entry will cause the entire test suite to be run for a remote +# webdriver with the capabilities specified in the entry. + +# See here for more on test configuration: +# https://docs.saucelabs.com/reference/test-configuration + +- browserName: "firefox" + version: 42 + public: "public restricted" + seleniumVersion: "2.48.0" + maxDuration: 10800 +- browserName: "chrome" + version: 41 + public: "public restricted" + seleniumVersion: "2.48.0" + maxDuration: 10800 diff --git a/codebender_testing/capabilities_firefox.yaml b/codebender_testing/capabilities_firefox.yaml new file mode 100644 index 0000000..12197f7 --- /dev/null +++ b/codebender_testing/capabilities_firefox.yaml @@ -0,0 +1,13 @@ +# This file contains a list of capabilities which will be used to instantiate +# the remote selenium webdrivers. +# Each list entry will cause the entire test suite to be run for a remote +# webdriver with the capabilities specified in the entry. + +# See here for more on test configuration: +# https://docs.saucelabs.com/reference/test-configuration + +- browserName: "firefox" + version: 42 + public: "public restricted" + seleniumVersion: "2.48.0" + maxDuration: 10800 diff --git a/codebender_testing/config.py b/codebender_testing/config.py index 9dfd241..f6ab267 100644 --- a/codebender_testing/config.py +++ b/codebender_testing/config.py @@ -1,17 +1,82 @@ import os from selenium import webdriver +from selenium.webdriver import chrome +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +import yaml +import simplejson -# URL of the site to be used for testing +def _rel_path(*args): + """Forms a path relative to this file's directory.""" + return os.path.join(os.path.dirname(__file__), *args) + +def get_path(directory, filename=None): + path = os.path.join(os.path.dirname( __file__ ), '..', directory) + if filename: + path = os.path.join(os.path.dirname( __file__ ), '..', directory, filename) + return os.path.abspath(path) + +def jsondump(data): + return simplejson.dumps(data, sort_keys=True, indent=4 * ' ') + +# URL of the default site to be used for testing BASE_URL = "http://localhost" +# URL of the actual Codebender website +LIVE_SITE_URL = "https://codebender.cc" + +# Names of sources (i.e. repositories) used to generate the codebender site. +SOURCE_BACHELOR = 'bachelor' +SOURCE_CODEBENDER_CC = 'codebender_cc' + +# User whose projects we'd like to compile in our compile_tester +# test case(s). +COMPILE_TESTER_URL = "/user/cb_compile_tester" +# The prefix for all filenames of log files. +# Note that it is given as a time format string, which will +# be formatted appropriately. +LOGFILE_PREFIX = _rel_path("..", "logs", "%Y-%m-%d_%H-%M-%S-{log_name}.json") -_EXTENSIONS_DIR = 'extensions' +# Logfile for COMPILE_TESTER compilation results +COMPILE_TESTER_LOGFILE = LOGFILE_PREFIX.format(log_name="cb_compile_tester") + +# Logfile for /libraries compilation results +LIBRARIES_TEST_LOGFILE = LOGFILE_PREFIX.format(log_name="libraries_test") + +_EXTENSIONS_DIR = _rel_path('..', 'extensions') _FIREFOX_EXTENSION_FNAME = 'codebender.xpi' +_CHROME_EXTENSION_FNAME = 'codebendercc-extension.crx' -# Set up Selenium Webdrivers to be used for selenium tests +# Maximum version number that we can use the Chrome extension with. +# For versions higher than this, we need to use the newer Codebender app +CHROME_EXT_MAX_CHROME_VERSION = 41 + +# Path to YAML file specifying capability list +DEFAULT_CAPABILITIES_FILE = os.getenv('CAPABILITIES', 'capabilities.yaml') +DEFAULT_CAPABILITIES_FILE_PATH = _rel_path(DEFAULT_CAPABILITIES_FILE) + +# Files used for testing +TEST_DATA_DIR = _rel_path('..', 'test_data') +TEST_DATA_INO = os.path.join(TEST_DATA_DIR, 'upload_ino.ino') +TEST_DATA_ZIP = os.path.join(TEST_DATA_DIR, 'upload_zip.zip') + +# Directory in which the local compile tester files are stored. +COMPILE_TESTER_DIR = os.path.join(TEST_DATA_DIR, 'cb_compile_tester') + +# Credentials to use when logging into the bachelor site +TEST_CREDENTIALS = { + "username": "tester", + "password": "testerPASS" +} + +TEST_PROJECT_NAME = "test_project" + +TIMEOUT = { + 'LOCATE_ELEMENT': 30 +} +# Set up Selenium Webdrivers to be used for selenium tests def _get_firefox_profile(): """Returns the Firefox profile to be used for the FF webdriver. Specifically, we're equipping the webdriver with the Codebender @@ -23,17 +88,66 @@ def _get_firefox_profile(): ) return firefox_profile -WEBDRIVERS = { - "firefox": webdriver.Firefox(firefox_profile=_get_firefox_profile()) -} +def get_browsers(capabilities_file_path=None): + """Returns a list of capabilities. Each item in the list will cause + the entire suite of tests to be re-run for a browser with those + particular capabilities. -# Credentials to use when logging into the site via selenium -TEST_CREDENTIALS = { - "username": "tester", - "password": "testerPASS" -} + `capabilities_file_path` is a path to a YAML file specifying a list of + capabilities for each browser. "Capabilities" are the dictionaries + passed as the `desired_capabilities` argument to the webdriver constructor. + """ + if capabilities_file_path is None: + capabilities_file_path = DEFAULT_CAPABILITIES_FILE_PATH + stream = file(capabilities_file_path, 'rb') + return yaml.load(stream) -TEST_PROJECT_NAME = "test_project" -# How long we wait until giving up on trying to locate an element -ELEMENT_FIND_TIMEOUT = 5 +def create_webdriver(command_executor, desired_capabilities): + """Creates a new remote webdriver with the following properties: + - The remote URL of the webdriver is defined by `command_executor`. + - desired_capabilities is a dict with the same interpretation as + it is used elsewhere in selenium. If no browserName key is present, + we default to firefox. + """ + if 'browserName' not in desired_capabilities: + desired_capabilities['browserName'] = 'firefox' + browser_name = desired_capabilities['browserName'] + # Fill in defaults from DesiredCapabilities.{CHROME,FIREFOX} if they are + # missing from the desired_capabilities dict above. + _capabilities = desired_capabilities + browser_profile = None + + if browser_name == "chrome": + desired_capabilities = DesiredCapabilities.CHROME.copy() + desired_capabilities.update(_capabilities) + + # NOTE: the following logic is disabled since the remote webdriver is + # not properly installing the codebender extension. It is kept for + # reference until we can figure out how to properly add the Chrome + # extension. + + # # Add chrome extension to capabilities + # options = chrome.options.Options() + # options.add_extension(os.path.join(_EXTENSIONS_DIR, _CHROME_EXTENSION_FNAME)) + # desired_capabilities.update(options.to_capabilities()) + # # Right now we only support up to v41 for this testing suite. + # if "version" in desired_capabilities: + # if desired_capabilities["version"] > CHROME_EXT_MAX_CHROME_VERSION: + # raise ValueError("The testing suite only supports Chrome versions up to v%d, " + # "but v%d was specified. Please specify a lower version " + # "number." % (CHROME_EXT_MAX_CHROME_VERSION, desired_capabilities["version"])) + # else: + # desired_capabilities["version"] = CHROME_EXT_MAX_CHROME_VERSION + + elif browser_name == "firefox": + desired_capabilities = DesiredCapabilities.FIREFOX.copy() + desired_capabilities.update(_capabilities) + browser_profile = _get_firefox_profile() + else: + raise ValueError("Invalid webdriver %s (only chrome and firefox are supported)" % browser_name) + return webdriver.Remote( + command_executor=command_executor, + desired_capabilities=desired_capabilities, + browser_profile=browser_profile, + ) diff --git a/codebender_testing/disqus.py b/codebender_testing/disqus.py new file mode 100644 index 0000000..ed54dba --- /dev/null +++ b/codebender_testing/disqus.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from codebender_testing.config import get_path +import disqusapi +import simplejson +import base64 +import hashlib +import hmac +import time +import os +import re + + +FORUM = 'codebender-cc' +AUTHOR_NAME = 'codebender' +AUTHOR_URL = 'https://codebender.cc/user/codebender' +DISQUS_REQUESTS_PER_HOUR = 1000 +DISQUS_WAIT = (DISQUS_REQUESTS_PER_HOUR / 60) / 60 +CHANGE_LOG = 'examples_compile_log.json' +DISQUS_COMMENTS = 'disqus_comments.json' +EXAMPLES_WITHOUT_LIBRARY_DB = 'examples_without_library.json' + + +class DisqusWrapper: + def __init__(self, log_time): + self.log_time = log_time + self.DISQUS_API_SECRET = os.getenv('DISQUS_API_SECRET', None) + self.DISQUS_API_PUBLIC = os.getenv('DISQUS_API_PUBLIC', None) + self.DISQUS_ACCESS_TOKEN = os.getenv('DISQUS_ACCESS_TOKEN', None) + self.user = { + 'id': os.getenv('DISQUS_SSO_ID', None), + 'username': os.getenv('DISQUS_SSO_USERNAME', None), + 'email': os.getenv('DISQUS_SSO_EMAIL', None), + } + self.SSO_KEY = self.get_disqus_sso(self.user) + self.disqus = disqusapi.DisqusAPI(api_secret=self.DISQUS_API_SECRET, public_key=self.DISQUS_API_PUBLIC, remote_auth=self.SSO_KEY) + self.change_log = {} + self.last_post = None + self.last_library = None + + with open(get_path('data', DISQUS_COMMENTS)) as f: + self.messages = simplejson.loads(f.read()) + + with open(get_path('data', EXAMPLES_WITHOUT_LIBRARY_DB)) as f: + self.examples_without_library = simplejson.loads(f.read()) + + def get_disqus_sso(self, user): + # create a JSON packet of our data attributes + data = simplejson.dumps(user) + # encode the data to base64 + message = base64.b64encode(data) + # generate a timestamp for signing the message + timestamp = int(time.time()) + # generate our hmac signature + sig = hmac.HMAC(self.DISQUS_API_SECRET, '%s %s' % (message, timestamp), hashlib.sha1).hexdigest() + return "{0} {1} {2}".format(message, sig, timestamp) + + def update_comment(self, sketch, results, current_date, log_entry, openFailFlag, counter, total_sketches): + # Comment examples + if not openFailFlag: + log_entry = self.handle_example_comment(sketch, results, current_date, log_entry) + + # Comment libraries when finished with the examples + library_match = re.match(r'.+\/example\/(.+)\/.+', sketch) + library = None + if library_match: + library = library_match.group(1) + if not self.last_library: + self.last_library = library + if library and library != self.last_library and (library not in self.examples_without_library or counter >= total_sketches-1): + log_entry = self.handle_library_comment(library, current_date, log_entry) + self.last_library = library + + return log_entry + + def handle_library_comment(self, library, current_date, log): + url = '/library/' + library + identifier = 'ident:' + url + paginator = disqusapi.Paginator(self.disqus.api.threads.list, forum=FORUM, thread=identifier, method='GET') + if paginator: + for page in paginator: + post_id, existing_message = self.get_posts(page['id']) + if post_id and existing_message: + new_message = self.messages['library'].replace('TEST_DATE', current_date) + if url not in log: + log[url] = {} + log[url]['comment'] = self.update_post(post_id, new_message) + else: + log[url]['comment'] = False + return log + + def handle_example_comment(self, url, results, current_date, log): + identifier = url.replace('https://codebender.cc', '') + identifier = 'ident:' + identifier + paginator = disqusapi.Paginator(self.disqus.api.threads.list, forum=FORUM, thread=identifier, method='GET') + if paginator: + for page in paginator: + post_id, existing_message = self.get_posts(page['id']) + if post_id and existing_message: + boards = [] + unsupportedFlag = False + for result in results: + if result['status'] == 'success': + board = result['board'] + if re.match(r'Arduino Mega.+', board): + board = 'Arduino Mega' + boards.append(board) + elif result['status'] == 'unsupported': + unsupportedFlag = True + + new_message = self.messages['example_fail'].replace('TEST_DATE', current_date) + if len(boards) > 0: + new_message = self.messages['example_success'].replace('TEST_DATE', current_date).replace('BOARDS_LIST', ', '.join(boards)) + elif unsupportedFlag: + new_message = self.messages['example_unsupported'].replace('TEST_DATE', current_date) + log[url]['comment'] = self.update_post(post_id, new_message) + break + else: + log[url]['comment'] = False + return log + + def get_posts(self, thread_id): + post_id = None + raw_message = None + paginator = disqusapi.Paginator(self.disqus.api.posts.list, forum=FORUM, thread=thread_id, order='asc', method='GET') + if paginator: + for result in paginator: + if result['author']['name'] == AUTHOR_NAME and result['author']['url'] == AUTHOR_URL: + post_id = result['id'] + raw_message = result['raw_message'] + break + return post_id, raw_message + + def update_post(self, post_id, message): + if not self.last_post: + self.last_post = message + elif re.match(r'^.+\.$', self.last_post): + message = message[:-1] + self.last_post = message + try: + response = self.disqus.posts.update(api_secret=self.DISQUS_API_SECRET, api_key=self.DISQUS_API_PUBLIC, remote_auth=self.SSO_KEY, access_token=self.DISQUS_ACCESS_TOKEN, post=post_id, message=message, method='POST') + if response['raw_message'] == message: + return True + return False + except Exception as error: + print 'Error:', error + return False diff --git a/codebender_testing/utils.py b/codebender_testing/utils.py index ea3d31d..c5f23ae 100644 --- a/codebender_testing/utils.py +++ b/codebender_testing/utils.py @@ -1,35 +1,270 @@ +from contextlib import contextmanager +from time import gmtime +from time import strftime +from time import strptime +from urlparse import urlparse +import time +import random +import os import re +import sys +import shutil +import tempfile +import simplejson +import pytest -from selenium import webdriver from selenium.common.exceptions import NoSuchElementException +from selenium.common.exceptions import StaleElementReferenceException +from selenium.common.exceptions import WebDriverException +from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions from selenium.webdriver.support.ui import WebDriverWait -import pytest from codebender_testing.config import BASE_URL -from codebender_testing.config import ELEMENT_FIND_TIMEOUT +from codebender_testing.config import TIMEOUT from codebender_testing.config import TEST_CREDENTIALS from codebender_testing.config import TEST_PROJECT_NAME -from codebender_testing.config import WEBDRIVERS +from codebender_testing.config import get_path +from codebender_testing.config import jsondump +from codebender_testing.disqus import DisqusWrapper -class SeleniumTestCase(object): - """Base class for all Selenium tests.""" +# Time to wait until we give up on a DOM property becoming available. +DOM_PROPERTY_DEFINED_TIMEOUT = 30 - @classmethod - @pytest.fixture(scope="class", autouse=True) - def _testcase_attrs(cls, webdriver): - """Sets up any class attributes to be used by any SeleniumTestCase. - Here, we just store fixtures as class attributes. This allows us to avoid - the pytest boilerplate of getting a fixture value, and instead just - refer to the fixture as `self.`. +# JavaScript snippet to extract all the links to sketches on the current page. +# `selector` is a CSS selector selecting these links. +_GET_SKETCHES_SCRIPT = "return $('{selector}').map(function() {{ return this.href; }}).toArray();" + +def SELECT_BOARD_SCRIPT(board): + return """ + $('#cb_cf_boards').val('{}').trigger('change'); + """.format(board) + +# JavaScript snippet to verify the code on the current page. +_VERIFY_SCRIPT = """ +compilerflasher.verify(); +""" + +_TEST_INPUT_ID = "_cb_test_input" + +# Creates an input into which we can upload files using Selenium. +_CREATE_INPUT_SCRIPT = """ +var input = $(''); +$('body').append(input); +""".format(input_id=_TEST_INPUT_ID) + +# After the file is chosen via Selenium, this script moves the file object +# (in the DOM) to the Dropzone. +def _move_file_to_dropzone_script(dropzone_selector): + return """ + $(function () {{ + var fileInput = document.getElementById('{input_id}'); + var file = fileInput.files[0]; + var dropzone = Dropzone.forElement('{selector}'); + dropzone.drop({{ dataTransfer: {{ files: [file] }} }}); + }}) + """.format(input_id=_TEST_INPUT_ID, selector=dropzone_selector) + +# How long (in seconds) to wait before assuming that an example +# has failed to compile +VERIFY_TIMEOUT = 30 + +# Messages displayed to the user after verifying a sketch. +VERIFICATION_SUCCESSFUL_MESSAGE = "Verification Successful" +VERIFICATION_FAILED_MESSAGE = "Verification failed." + +# Max test runtime into saucelabs +# 2.5 hours (3 hours max) +SAUCELABS_TIMEOUT_SECONDS = 10800 - 1800 + +# Throttle between compiles +COMPILES_PER_MINUTE = 10 +def throttle_compile(): + min = 60 / COMPILES_PER_MINUTE + max = min + 1 + time.sleep(random.uniform(min, max)) + +BOARDS_FILE = 'boards_db.json' +BOARDS_PATH = get_path('data', BOARDS_FILE) +with open(BOARDS_PATH) as f: + BOARDS_DB = simplejson.loads(f.read()) + +def read_last_log(compile_type): + logs = os.listdir(get_path('logs')) + logs_re = re.compile(r'.+cb_compile_tester.+') + if compile_type == 'library': + logs_re = re.compile(r'.+libraries_test.+') + logs = sorted([x for x in logs if x != '.gitignore' and logs_re.match(x)]) + + log_timestamp_re = re.compile(r'(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})-.+\.json') + log = None + timestamp = None + if len(logs) > 0: + log = logs[-1] + timestamp = log_timestamp_re.match(log).group(1) + + last_log = None + if log: + with open(get_path('logs', log)) as f: + last_log = simplejson.loads(f.read()) + + return { + 'log': last_log, + 'timestamp': timestamp + } + + +# Creates a report json after each compile test +def report_creator(compile_type, log_entry, log_file): + logs = os.listdir(get_path('logs')) + logs_re = re.compile(r'.+cb_compile_tester.+') + if compile_type == 'library': + logs_re = re.compile(r'.+libraries_test.+') + + logs = sorted([x for x in logs if x != '.gitignore' and logs_re.match(x)]) + tail = logs[-2:] + logs_to_examine = [] + for log in tail: + try: + with open(get_path('logs', log)) as f: + logs_to_examine.append(simplejson.loads(f.read())) + except: + print 'Log:', log, 'not found' + + diff = {} + changes = 0 + if len(logs_to_examine) >= 2: + old_log = logs_to_examine[0] + new_log = logs_to_examine[1] + + for url in new_log.keys(): + if url not in old_log: + diff[url] = new_log[url] + changes += 1 + continue + + for result in new_log[url].keys(): + if result not in old_log[url]: + if not url in diff: + diff[url] = {} + diff[url][result] = new_log[url][result] + changes += 1 + continue + + if result == 'success' or result == 'fail': + if result not in old_log[url]: + if not url in diff: + diff[url] = {} + diff[url][result] = new_log[url][result] + changes += 1 + continue + + for board in new_log[url][result]: + oposite = 'success' + if result == 'success': + oposite = 'fail' + if oposite in old_log[url] and board in old_log[url][oposite]: + if not url in diff: + diff[url] = {} + if result not in diff[url]: + diff[url][result] = [] + diff[url][result].append(board) + changes += 1 + elif result == 'open_fail' or result == 'error' or result == 'comment': + if result not in old_log[url]: + if not url in diff: + diff[url] = {} + diff[url][result] = new_log[url][result] + changes += 1 + continue + if old_log[url][result] != new_log[url][result]: + if not url in diff: + diff[url] = {} + diff[url][result] = new_log[url][result] + changes += 1 + elif len(logs_to_examine) == 1: + diff = logs_to_examine[0] + changes += 1 + else: + diff = log_entry + changes += 1 + + filename_tokens = os.path.basename(log_file).split('.') + filename = '.'.join(filename_tokens[0:-1]) + extension = filename_tokens[-1] + filename = 'report_' + filename + '_' + str(changes) + '.' + extension + path = get_path('reports', filename) + with open(path, 'w') as f: + f.write(jsondump(diff)) + + +@contextmanager +def temp_copy(fname): + """Creates a temporary copy of the file `fname`. + This is useful for testing features that derive certain properties + from the filename, and we want a unique filename each time we run the + test (in case, for example, there is leftover garbage from previous + tests with the same name). + """ + extension = fname.split('.')[-1] + with tempfile.NamedTemporaryFile(mode='w+b', suffix='.%s' % extension) as copy: + with open(fname, 'rb') as original: + shutil.copyfileobj(original, copy) + copy.flush() + yield copy + + +class CodebenderSeleniumBot(object): + """Contains various utilities for navigating the Codebender website.""" + + # This can be configured on a per-test case basis to use a different + # URL for testing; e.g., http://localhost, or http://codebender.cc. + # It is set via command line option in _testcase_attrs (below) + site_url = None + + def init(self, url=None, webdriver=None): + """Create a bot with the given selenium webdriver, operating on `url`. + We can't do this in an __init__ method, otherwise py.test complains, + presumably because it does something special with __init__ for test + cases. """ - cls.driver = webdriver + self.driver = webdriver - @pytest.fixture(scope="class") - def tester_login(self): - self.login() + if url is None: + url = BASE_URL + self.site_url = url + + @classmethod + @contextmanager + def session(cls, **kwargs): + """Start a new session with a new webdriver. Regardless of whether an + exception is raised, the webdriver is guaranteed to quit. + The keyword arguments should be interpreted as in `start`. + + Sample usage: + + ``` + with CodebenderSeleniumBot.session(url="localhost", + webdriver="firefox") as bot: + # The browser is now open + bot.open("/") + assert "Codebender" in bot.driver.title + # The browser is now closed + ``` + + Test cases shouldn't need to use this method; it's mostly useful for + scripts, automation, etc. + """ + try: + bot = cls() + bot.start(**kwargs) + yield bot + bot.driver.quit() + except: + bot.driver.quit() + raise def open(self, url=None): """Open the resource specified by `url`. @@ -39,12 +274,12 @@ def open(self, url=None): """ if url is None: url = '' - if re.match(".+?://^", url): + if re.match(".+?://", url): # url specifies an absolute path. return self.driver.get(url) else: url = url.lstrip('/') - return self.driver.get("%s/%s" % (BASE_URL, url)) + return self.driver.get("%s/%s" % (self.site_url, url)) def open_project(self, project_name=None): """Opens the project specified by `name`, bringing the driver to the @@ -56,27 +291,456 @@ def open_project(self, project_name=None): project_link = self.driver.find_element_by_link_text(project_name) project_link.send_keys(Keys.ENTER) - def login(self): - """Performs a login.""" + def login(self, credentials=None): + """Performs a login. Note that the current URL may change to an + unspecified location when calling this function. + `credentials` should be a dict with keys 'username' and 'password', + mapped to the appropriate values.""" + if credentials is None: + credentials = TEST_CREDENTIALS try: self.open() login_button = self.driver.find_element_by_id('login_btn') login_button.send_keys(Keys.ENTER) # Enter credentials and log in user_field = self.driver.find_element_by_id('username') - user_field.send_keys(TEST_CREDENTIALS['username']) + user_field.send_keys(credentials['username']) pass_field = self.driver.find_element_by_id('password') - pass_field.send_keys(TEST_CREDENTIALS['password']) + pass_field.send_keys(credentials['password']) do_login = self.driver.find_element_by_id('_submit') do_login.send_keys(Keys.ENTER) except NoSuchElementException: # 'Log In' is not displayed, so we're already logged in. pass + def logout(self): + """Logs out of the site.""" + try: + logout_button = self.driver.find_element_by_id("logout") + logout_button.send_keys(Keys.ENTER) + except NoSuchElementException: + # 'Log out' is not displayed, so we're already logged out. + pass + def get_element(self, *locator): """Waits for an element specified by *locator (a tuple of (By., str)), then returns it if it is found.""" - WebDriverWait(self.driver, ELEMENT_FIND_TIMEOUT).until( - expected_conditions.presence_of_element_located(locator)) + WebDriverWait(self.driver, TIMEOUT['LOCATE_ELEMENT']).until( + expected_conditions.visibility_of_element_located(locator)) return self.driver.find_element(*locator) - + + def get_elements(self, *locator): + """Like `get_element`, but returns a list of all elements matching + the selector.""" + WebDriverWait(self.driver, TIMEOUT['LOCATE_ELEMENT']).until( + expected_conditions.visibility_of_all_elements_located_by(locator)) + return self.driver.find_elements(*locator) + + def find(self, selector): + """Alias for `self.get_element(By.CSS_SELECTOR, selector)`.""" + return self.get_element(By.CSS_SELECTOR, selector) + + def find_all(self, selector): + """Alias for `self.get_elements(By.CSS_SELECTOR, selector)`.""" + return self.get_elements(By.CSS_SELECTOR, selector) + + def dropzone_upload(self, selector, fname): + """Uploads a file specified by `fname` via the Dropzone within the + element specified by `selector`. (Dropzone refers to Dropzone.js) + """ + # Create an artificial file input. + self.execute_script(_CREATE_INPUT_SCRIPT, '$') + test_input = self.get_element(By.ID, _TEST_INPUT_ID) + test_input.send_keys(fname) + self.execute_script(_move_file_to_dropzone_script(selector), '$', 'Dropzone') + + def upload_project(self, dropzone_selector, test_fname, sketch_name=None): + """Tests that we can successfully upload `test_fname`. + `project_name` is the expected name of the project; by + default it is inferred from the file name. + Returns a pair of (the name of the project, the url of the project sketch) + """ + # A tempfile is used here since we want the name to be + # unique; if the file has already been successfully uploaded + # then the test might give a false-positive. + with temp_copy(test_fname) as test_file: + self.dropzone_upload(dropzone_selector, test_file.name) + + if sketch_name: + return sketch_name + return '.'.join(os.path.basename(test_file.name).split('.')[0:-1]) + + def delete_project(self, project_name): + """Deletes the project specified by `project_name`. Note that this will + navigate to the user's homepage.""" + self.open('/') + try: + created_project = self.get_element(By.LINK_TEXT, project_name) + delete_button_li = created_project.find_element_by_xpath('..') + delete_button = delete_button_li.find_element_by_css_selector('.delete-sketch') + delete_button.click() + popup_delete_button = self.get_element(By.ID, 'deleteProjectButton') + popup_delete_button.click() + except: + pass + + def compile_sketch(self, url, boards, iframe=False): + """Compiles the sketch located at `url`, or an iframe within the page + referred to by `url`. Raises an exception if it does not compile. + """ + self.open(url) + # When example does not load + if 'Sorry! The example could not be fetched.' in self.driver.page_source: + compilation_results = [] + result = {} + result['status'] = 'open_fail' + compilation_results.append(result) + return compilation_results + # Switch into iframe if needed + if iframe: + self.switch_into_iframe(url) + # Compile the target for each provided board + compilation_results = [] + for board in boards: + result = { + 'board': board + } + try: + self.execute_script(SELECT_BOARD_SCRIPT(board), '$', 'compilerflasher.pluginHandler.plugin_found') + self.execute_script(_VERIFY_SCRIPT, 'compilerflasher') + # In the BACHELOR site the id is 'operation_output', but in the live + # site the id is 'cb_cf_operation_output'. The [id$=operation_output] + # here selects an id that _ends_ with 'operation_output'. + compile_result = WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + any_text_to_be_present_in_element( + (By.CSS_SELECTOR, "[id$=operation_output]"), + VERIFICATION_SUCCESSFUL_MESSAGE, VERIFICATION_FAILED_MESSAGE + ) + ) + except WebDriverException as error: + compile_result = "%s; %s" % (type(error).__name__, str(error)) + result['status'] = 'error' + result['message'] = compile_result + + if compile_result == VERIFICATION_SUCCESSFUL_MESSAGE: + result['status'] = 'success' + else: + result['status'] = 'fail' + + compilation_results.append(result) + + throttle_compile() + + self.driver.switch_to_default_content() + + return compilation_results + + def compile_all_sketches(self, url, selector, **kwargs): + """Compiles all sketches on the page at `url`. `selector` is a CSS selector + that should select all relevant tags containing links to sketches. + See `compile_sketches` for the possible keyword arguments that can be specified. + """ + self.open(url) + sketches = self.execute_script(_GET_SKETCHES_SCRIPT.format(selector=selector), '$') + assert len(sketches) > 0 + self.compile_sketches(sketches, **kwargs) + + def compile_sketches(self, sketches, iframe=False, logfile=None, compile_type='sketch', create_report=False, comment=False): + """Compiles the sketches with URLs given by the `sketches` list. + `logfile` specifies a path to a file to which test results will be + logged. If it is not `None`, compile errors will not cause the test + to halt, but rather be logged to the given file. `logfile` may be a time + format string, which will be formatted appropriately. + `iframe` specifies whether the urls pointed to by `selector` are contained + within an iframe. + If the `--full` argument is provided (and hence + `self.run_full_compile_tests` is `True`, we do not log, and limit the + number of sketches compiled to 1. + """ + + # Log filename + log_time = gmtime() + # Keeps the logs of each compile + log_entry = {} + + urls_visited = {} + last_log = read_last_log(compile_type) + if last_log['log']: + # resume previous compile + log_time = strptime(last_log['timestamp'], '%Y-%m-%d_%H-%M-%S') + log_entry = last_log['log'] + for url in last_log['log']: + urls_visited[url] = True + + urls_to_visit = [] + for url in sketches: + if url not in urls_visited: + urls_to_visit.append(url) + + if len(urls_to_visit) == 0: + urls_to_visit = sketches + log_entry = {} + log_time = gmtime() + + current_date = strftime('%Y-%m-%d', log_time) + # Initialize DisqusWrapper + disqus_wrapper = DisqusWrapper(log_time) + if logfile: + log_file = strftime(logfile, log_time) + + print '\nCompiling:', len(urls_to_visit), 'sketches' + total_sketches = len(urls_to_visit) + tic = time.time() + + for counter, sketch in enumerate(urls_to_visit): + # Read the boards map in case current sketch/example requires a special board configuration + boards = BOARDS_DB['default_boards'] + url_fragments = urlparse(sketch) + if url_fragments.path in BOARDS_DB['special_boards']: + boards = BOARDS_DB['special_boards'][url_fragments.path] + + if len(boards) > 0: + # Run Verify + results = self.compile_sketch(sketch, boards, iframe=iframe) + else: + results = [ + { + 'status': 'unsupported' + } + ] + + # Used when not funning in Full mode + if logfile is None or not self.run_full_compile_tests: + continue + + # Register current URL into log + if sketch not in log_entry: + log_entry[sketch] = {} + + test_status = '.' + # Log the compilation results + openFailFlag = False + for result in results: + if result['status'] in ['success', 'fail', 'error'] and result['status'] not in log_entry[sketch]: + log_entry[sketch][result['status']] = [] + + if result['status'] == 'success': + log_entry[sketch]['success'].append(result['board']) + elif result['status'] == 'fail': + log_entry[sketch]['fail'].append(result['board']) + test_status = 'F' + elif result['status'] == 'open_fail': + log_entry[sketch]['open_fail'] = True + openFailFlag = True + test_status = 'O' + elif result['status'] == 'error': + log_entry[sketch]['error'].append({ + 'board': result['board'], + 'error': result['message'] + }) + test_status = 'E' + elif result['status'] == 'unsupported': + log_entry[sketch]['unsupported'] = True + test_status = 'U' + + # Update Disqus comments + if compile_type == 'library' and comment: + log_entry = disqus_wrapper.update_comment(sketch, results, current_date, log_entry, openFailFlag, counter, total_sketches) + + # Dump the test results to `logfile`. + with open(log_file, 'w') as f: + f.write(jsondump(log_entry)) + + # Display progress + sys.stdout.write(test_status) + sys.stdout.flush() + + toc = time.time() + if toc - tic >= SAUCELABS_TIMEOUT_SECONDS: + print '\nStopping tests to avoid saucelabs timeout' + print 'Test duration:', int(toc - tic), 'sec' + return + + # Generate a report if requested + if create_report: + report_creator(compile_type, log_entry, log_file) + print '\nTest duration:', int(toc - tic), 'sec' + + def execute_script(self, script, *deps): + """Waits for all JavaScript variables in `deps` to be defined, then + executes the given script.""" + if len(deps) > 0: + WebDriverWait(self.driver, DOM_PROPERTY_DEFINED_TIMEOUT).until( + dom_properties_defined(*deps) + ) + return self.driver.execute_script(script) + + def create_sketch(self, name): + """Creates a sketch with a given name""" + createSketchBtn = self.driver.find_element_by_id('create_sketch_btn') + createSketchBtn.click() + sketchHeading = self.get_element(By.ID, 'editor_heading_project_name') + sketchHeading.click() + renameInput = '#editor_heading_project_name input' + headingInput = self.get_element(By.CSS_SELECTOR, renameInput) + headingInput.clear() + headingInput.send_keys(name) + headingInput.send_keys(Keys.ENTER) + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.invisibility_of_element_located( + (By.CSS_SELECTOR, "#editor_heading_project_name i") + ) + ) + + def check_iframe(self): + """Returns the contents of an iframe [project_name, user_name, sketch_contents]""" + self.driver.switch_to_frame(self.driver.find_element_by_tag_name('iframe')) + project_name = self.driver.find_element_by_class_name('projectName').text + user_name = self.driver.find_element_by_class_name('userName').text + sketch_contents = self.execute_script('return editor.aceEditor.getValue();', 'editor') + self.driver.switch_to_default_content() + return [project_name, user_name, sketch_contents] + + def switch_into_iframe(self, url): + if 'embed' not in url: + url = url.replace('https://codebender.cc/', 'https://codebender.cc/embed/') + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.visibility_of( + self.get_element(By.CSS_SELECTOR, 'iframe[src="' + url + '"]') + ) + ) + script = """ + var iframes = document.body.getElementsByTagName('iframe'); + var iframe_index = 0; + for (var i=0; i -1) {{ + iframe_index = i; + }} + }} + return iframe_index; + """.format(url=url) + index = self.execute_script(script) + iframe = self.driver.find_elements_by_tag_name('iframe')[index] + self.driver.switch_to_frame(iframe) + + +class SeleniumTestCase(CodebenderSeleniumBot): + """Base class for all Selenium tests.""" + + @classmethod + @pytest.fixture(scope="class", autouse=True) + def _testcase_attrs(cls, webdriver, testing_url, testing_full): + """Sets up any class attributes to be used by any SeleniumTestCase. + Here, we just store fixtures as class attributes. This allows us to avoid + the pytest boilerplate of getting a fixture value, and instead just + refer to the fixture as `self.`. + """ + cls.driver = webdriver + cls.site_url = testing_url + cls.run_full_compile_tests = testing_full + + @pytest.fixture(scope="class") + def tester_login(self, testing_credentials): + """A fixture to perform a login with the credentials provided by the + `testing_credentials` fixture. + """ + self.login(credentials=testing_credentials) + + @pytest.fixture(scope="class") + def tester_logout(self): + """A fixture to guarantee that we are logged out before running a test.""" + self.logout() + +class CodebenderEmbeddedTestCase(SeleniumTestCase): + """base class for testing embedded views""" + + @pytest.fixture(scope="class") + def test_embedded_sketch(self, selector): + self.switch_into_iframe(selector) + + project_name = self.driver.find_element_by_class_name('projectName').text + assert len(project_name) > 0 + + user_name = self.driver.find_element_by_class_name('userName').text + assert len(user_name) > 0 + + edit_button = self.driver.find_element_by_id('edit-button').text + assert edit_button == 'Edit' + + clone_link = self.driver.find_element_by_class_name('clone-link').text + assert clone_link == 'Clone & Edit' + + download_link = self.driver.find_element_by_class_name('download-link').text + assert download_link == 'Download' + + editor_contents = self.execute_script('return editor.aceEditor.getValue();', 'editor') + assert len(editor_contents) > 0 + + assert self.check_element_exists('#cb_cf_flash_btn') == True + assert self.check_element_exists('#cb_cf_boards') == True + assert self.check_element_exists('#cb_cf_ports') == True + + boards_list = self.driver.find_element_by_id('cb_cf_boards').text + assert len(boards_list) > 0 + + self.driver.switch_to_default_content() + + @pytest.fixture(scope="class") + def test_serial_monitor(self, selector): + self.switch_into_iframe(selector) + + title = self.driver.find_element_by_css_selector('.well > h4').text.strip() + assert title == 'Serial Monitor:' + + ports_label = self.driver.find_element_by_css_selector('.well > span').text.strip() + assert ports_label == 'Port:' + + assert self.check_element_exists('#cb_cf_ports') == True + assert self.check_element_exists('#cb_cf_baud_rates') == True + assert self.check_element_exists('#cb_cf_serial_monitor_connect') == True + + def check_element_exists(self, css_path): + try: + self.driver.find_element_by_css_selector(css_path) + return True + except NoSuchElementException: + return False + +class VerificationError(Exception): + """An exception representing a failed verification of a sketch.""" + pass + + +class dom_properties_defined(object): + """An expectation for the given DOM properties to be defined. + See selenium.webdriver.support.expected_conditions for more on how this + type of class works. + """ + + def __init__(self, *properties): + self._properties = properties + + def __call__(self, driver): + return all( + driver.execute_script("return window.%s !== undefined" % prop) + for prop in self._properties) + + +class any_text_to_be_present_in_element(object): + """An expectation for checking if any of the given strings are present in + the specified element. Returns the string that was present. + """ + def __init__(self, locator, *texts): + self.locator = locator + self.texts = texts + + def __call__(self, driver): + try : + element_text = expected_conditions._find_element(driver, self.locator).text + for text in self.texts: + if text in element_text: + return text + return False + except StaleElementReferenceException: + return False diff --git a/data/boards_db.json b/data/boards_db.json new file mode 100644 index 0000000..3973d32 --- /dev/null +++ b/data/boards_db.json @@ -0,0 +1,9 @@ +{ + "default_boards": [ + "Arduino Uno", + "Arduino Leonardo", + "Arduino Mega 2560 or Mega ADK" + ], + "special_boards": { + } +} \ No newline at end of file diff --git a/data/disqus_comments.json b/data/disqus_comments.json new file mode 100644 index 0000000..1414721 --- /dev/null +++ b/data/disqus_comments.json @@ -0,0 +1,6 @@ +{ + "library": "This library and its examples where tested on TEST_DATE with common Arduino boards. For more detailed information about the test results, please look at each example's comments.", + "example_success": "This example was tested on TEST_DATE and it compiles on BOARDS_LIST.", + "example_fail": "This example was tested on TEST_DATE and it failed to compile on common Arduino boards.", + "example_unsupported": "This example is not known to compile with any of the codebender supported boards at least until TEST_DATE." +} \ No newline at end of file diff --git a/data/examples_without_library.json b/data/examples_without_library.json new file mode 100644 index 0000000..00b6423 --- /dev/null +++ b/data/examples_without_library.json @@ -0,0 +1,13 @@ +[ + "01.Basics", + "02.Digital", + "03.Analog", + "04.Communication", + "05.Control", + "06.Sensors", + "07.Display", + "07.Display", + "09.USB", + "10.StarterKit", + "ArduinoISP" +] diff --git a/extensions/codebender-app.crx b/extensions/codebender-app.crx new file mode 100644 index 0000000..7ef1fb5 Binary files /dev/null and b/extensions/codebender-app.crx differ diff --git a/extensions/codebender.xpi b/extensions/codebender.xpi index c518bcf..a06eebb 100644 Binary files a/extensions/codebender.xpi and b/extensions/codebender.xpi differ diff --git a/extensions/codebendercc-extension.crx b/extensions/codebendercc-extension.crx new file mode 100644 index 0000000..40a0b29 Binary files /dev/null and b/extensions/codebendercc-extension.crx differ diff --git a/logs/.gitignore b/logs/.gitignore new file mode 100644 index 0000000..9b18ab2 --- /dev/null +++ b/logs/.gitignore @@ -0,0 +1,8 @@ +# Ignore everything in this directory +* + +# Except this file +!.gitignore + +# We need this gitignore since the logs directory is required for storing logs, +# but we don't want to commit the logs themselves. diff --git a/reports/.gitignore b/reports/.gitignore new file mode 100644 index 0000000..9b18ab2 --- /dev/null +++ b/reports/.gitignore @@ -0,0 +1,8 @@ +# Ignore everything in this directory +* + +# Except this file +!.gitignore + +# We need this gitignore since the logs directory is required for storing logs, +# but we don't want to commit the logs themselves. diff --git a/requirements-dev.txt b/requirements-dev.txt index b79c1e6..85be7c2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ flake8 pytest +setuptools>=12.1 tox -virtualenv +virtualenv \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index acad28e..221bb1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,11 @@ -pytest==2.6.4 -selenium==2.44.0 +pyopenssl +ndg-httpsclient +pyasn1 +cryptography +tox +virtualenv +pyyaml +pytest +selenium +simplejson +disqus-python \ No newline at end of file diff --git a/setup.py b/setup.py index 9b91ffe..2d89f09 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name='codebender_selenium', - version='0.0.0', + version='1.0.0', description='Selenium tests for codebender.cc', url='http://github.com/codebendercc/seleniumTests', packages=[], diff --git a/test_data/cb_compile_tester/07-02 Morse.zip b/test_data/cb_compile_tester/07-02 Morse.zip new file mode 100644 index 0000000..39dd52f Binary files /dev/null and b/test_data/cb_compile_tester/07-02 Morse.zip differ diff --git a/test_data/cb_compile_tester/ACController.zip b/test_data/cb_compile_tester/ACController.zip new file mode 100644 index 0000000..7ff861b Binary files /dev/null and b/test_data/cb_compile_tester/ACController.zip differ diff --git a/test_data/cb_compile_tester/DHCP Chat Server.zip b/test_data/cb_compile_tester/DHCP Chat Server.zip new file mode 100644 index 0000000..ffbbe5b Binary files /dev/null and b/test_data/cb_compile_tester/DHCP Chat Server.zip differ diff --git a/test_data/cb_compile_tester/Gyrometer.zip b/test_data/cb_compile_tester/Gyrometer.zip new file mode 100644 index 0000000..bbee0de Binary files /dev/null and b/test_data/cb_compile_tester/Gyrometer.zip differ diff --git a/test_data/cb_compile_tester/LED Packman Adafruit.zip b/test_data/cb_compile_tester/LED Packman Adafruit.zip new file mode 100644 index 0000000..93e2d6d Binary files /dev/null and b/test_data/cb_compile_tester/LED Packman Adafruit.zip differ diff --git a/test_data/cb_compile_tester/LightSchedulerV2.zip b/test_data/cb_compile_tester/LightSchedulerV2.zip new file mode 100644 index 0000000..7f376fb Binary files /dev/null and b/test_data/cb_compile_tester/LightSchedulerV2.zip differ diff --git a/test_data/cb_compile_tester/P-Space Jarvis attiny85.zip b/test_data/cb_compile_tester/P-Space Jarvis attiny85.zip new file mode 100644 index 0000000..874224a Binary files /dev/null and b/test_data/cb_compile_tester/P-Space Jarvis attiny85.zip differ diff --git a/test_data/cb_compile_tester/PCF8574Test.zip b/test_data/cb_compile_tester/PCF8574Test.zip new file mode 100644 index 0000000..aca9784 Binary files /dev/null and b/test_data/cb_compile_tester/PCF8574Test.zip differ diff --git a/test_data/cb_compile_tester/SaswatEthernet.zip b/test_data/cb_compile_tester/SaswatEthernet.zip new file mode 100644 index 0000000..9bda2a1 Binary files /dev/null and b/test_data/cb_compile_tester/SaswatEthernet.zip differ diff --git a/test_data/cb_compile_tester/TFT First Example.zip b/test_data/cb_compile_tester/TFT First Example.zip new file mode 100644 index 0000000..1530a36 Binary files /dev/null and b/test_data/cb_compile_tester/TFT First Example.zip differ diff --git a/test_data/cb_compile_tester/Test Adafruit Motor Shield.zip b/test_data/cb_compile_tester/Test Adafruit Motor Shield.zip new file mode 100644 index 0000000..13c6c1e Binary files /dev/null and b/test_data/cb_compile_tester/Test Adafruit Motor Shield.zip differ diff --git a/test_data/cb_compile_tester/Twitter.zip b/test_data/cb_compile_tester/Twitter.zip new file mode 100644 index 0000000..a9ef436 Binary files /dev/null and b/test_data/cb_compile_tester/Twitter.zip differ diff --git a/test_data/cb_compile_tester/Ultimate RC Car.zip b/test_data/cb_compile_tester/Ultimate RC Car.zip new file mode 100644 index 0000000..e8f2e30 Binary files /dev/null and b/test_data/cb_compile_tester/Ultimate RC Car.zip differ diff --git a/test_data/cb_compile_tester/Window Vent.zip b/test_data/cb_compile_tester/Window Vent.zip new file mode 100644 index 0000000..da1f49c Binary files /dev/null and b/test_data/cb_compile_tester/Window Vent.zip differ diff --git a/test_data/cb_compile_tester/XBeeRadio_Default_Rx.zip b/test_data/cb_compile_tester/XBeeRadio_Default_Rx.zip new file mode 100644 index 0000000..f8100e0 Binary files /dev/null and b/test_data/cb_compile_tester/XBeeRadio_Default_Rx.zip differ diff --git a/test_data/cb_compile_tester/XBee_Rx.zip b/test_data/cb_compile_tester/XBee_Rx.zip new file mode 100644 index 0000000..aaa8255 Binary files /dev/null and b/test_data/cb_compile_tester/XBee_Rx.zip differ diff --git a/test_data/cb_compile_tester/dog-needs-water.zip b/test_data/cb_compile_tester/dog-needs-water.zip new file mode 100644 index 0000000..ed4a679 Binary files /dev/null and b/test_data/cb_compile_tester/dog-needs-water.zip differ diff --git a/test_data/cb_compile_tester/ds18b20_webserver copy.zip b/test_data/cb_compile_tester/ds18b20_webserver copy.zip new file mode 100644 index 0000000..2f4fae4 Binary files /dev/null and b/test_data/cb_compile_tester/ds18b20_webserver copy.zip differ diff --git a/test_data/cb_compile_tester/ds18b20_webserver.zip b/test_data/cb_compile_tester/ds18b20_webserver.zip new file mode 100644 index 0000000..8a4e11a Binary files /dev/null and b/test_data/cb_compile_tester/ds18b20_webserver.zip differ diff --git a/test_data/cb_compile_tester/ds18b20_webserver_01.zip b/test_data/cb_compile_tester/ds18b20_webserver_01.zip new file mode 100644 index 0000000..4355f27 Binary files /dev/null and b/test_data/cb_compile_tester/ds18b20_webserver_01.zip differ diff --git a/test_data/cb_compile_tester/fotellos_test_proj.zip b/test_data/cb_compile_tester/fotellos_test_proj.zip new file mode 100644 index 0000000..e44a6c9 Binary files /dev/null and b/test_data/cb_compile_tester/fotellos_test_proj.zip differ diff --git a/test_data/cb_compile_tester/helmet interrupt.zip b/test_data/cb_compile_tester/helmet interrupt.zip new file mode 100644 index 0000000..4a89e0a Binary files /dev/null and b/test_data/cb_compile_tester/helmet interrupt.zip differ diff --git a/test_data/cb_compile_tester/lcd_example.zip b/test_data/cb_compile_tester/lcd_example.zip new file mode 100644 index 0000000..2752abe Binary files /dev/null and b/test_data/cb_compile_tester/lcd_example.zip differ diff --git a/test_data/cb_compile_tester/logic_analyzer_inline_2mhz.zip b/test_data/cb_compile_tester/logic_analyzer_inline_2mhz.zip new file mode 100644 index 0000000..cf635bf Binary files /dev/null and b/test_data/cb_compile_tester/logic_analyzer_inline_2mhz.zip differ diff --git a/test_data/cb_compile_tester/pedalboard.zip b/test_data/cb_compile_tester/pedalboard.zip new file mode 100644 index 0000000..61287e3 Binary files /dev/null and b/test_data/cb_compile_tester/pedalboard.zip differ diff --git a/test_data/cb_compile_tester/servo_test.zip b/test_data/cb_compile_tester/servo_test.zip new file mode 100644 index 0000000..ed641c5 Binary files /dev/null and b/test_data/cb_compile_tester/servo_test.zip differ diff --git a/test_data/cb_compile_tester/strandtest.zip b/test_data/cb_compile_tester/strandtest.zip new file mode 100644 index 0000000..91bc0c3 Binary files /dev/null and b/test_data/cb_compile_tester/strandtest.zip differ diff --git a/test_data/cb_compile_tester/temphumidlcdcosm.zip b/test_data/cb_compile_tester/temphumidlcdcosm.zip new file mode 100644 index 0000000..97a9823 Binary files /dev/null and b/test_data/cb_compile_tester/temphumidlcdcosm.zip differ diff --git a/test_data/cb_compile_tester/twitter.zip b/test_data/cb_compile_tester/twitter.zip new file mode 100644 index 0000000..9402d4d Binary files /dev/null and b/test_data/cb_compile_tester/twitter.zip differ diff --git a/test_data/upload_ino.ino b/test_data/upload_ino.ino new file mode 100644 index 0000000..78fb5d9 --- /dev/null +++ b/test_data/upload_ino.ino @@ -0,0 +1,10 @@ +/** + * A blank Arduino project. + * This should compile with no issues. + */ + +void setup() { +} + +void loop() { +} diff --git a/test_data/upload_zip.zip b/test_data/upload_zip.zip new file mode 100644 index 0000000..5ae342a Binary files /dev/null and b/test_data/upload_zip.zip differ diff --git a/tests/sketch/__init__.py b/tests/common/embedded_views/__init__.py similarity index 100% rename from tests/sketch/__init__.py rename to tests/common/embedded_views/__init__.py diff --git a/tests/common/embedded_views/test_embedded.py b/tests/common/embedded_views/test_embedded.py new file mode 100644 index 0000000..1705ea4 --- /dev/null +++ b/tests/common/embedded_views/test_embedded.py @@ -0,0 +1,29 @@ +from codebender_testing.utils import CodebenderEmbeddedTestCase +import re + + +EMBEDDED_VIEWS = [ + 'http://blog.codebender.cc/2014/03/07/lesson-1-inputs-and-outputs/', + 'https://www.sparkfun.com/news/1803', + 'http://1sheeld.com/blog/announcing-4-new-shields-tasker-integration-partnership-with-codebender/', + 'http://edu.olympiacircuits.com/codebender.html', + 'http://redbearlab.com/quick-start-codebender/', + 'https://tiny-circuits.com/tinyscreen', + 'http://lowpowerlab.com/programming/', + 'http://lowpowerlab.com/blog/2014/12/03/moteino-now-on-codebender-cc/', + 'http://microview.io/intro/getting-started.html', + 'http://www.hummingbirdkit.com/learning/arduino-programming' +] + + +class TestHome(CodebenderEmbeddedTestCase): + + def test_embedded_views(self): + embed_sketch_re = re.compile('^https:\/\/codebender\.cc\/embed\/sketch:\d+$') + for embedded in EMBEDDED_VIEWS: + self.open(embedded) + iframes = self.driver.find_elements_by_tag_name('iframe') + for iframe in iframes: + iframe_src = iframe.get_attribute('src') + if embed_sketch_re.match(iframe_src): + self.test_embedded_sketch(iframe_src) diff --git a/tests/common/home/__init__.py b/tests/common/home/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/common/home/test_home.py b/tests/common/home/test_home.py new file mode 100644 index 0000000..a0c3e36 --- /dev/null +++ b/tests/common/home/test_home.py @@ -0,0 +1,87 @@ +from codebender_testing.utils import SeleniumTestCase +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.by import By +from codebender_testing import config +import os + +class TestHome(SeleniumTestCase): + + def test_navigate_home(self, tester_logout): + """ opens browser to codebender bachelor """ + self.open("/") + assert "codebender" in self.driver.title + + def test_login(self, tester_logout): + credentials = { + 'username': os.environ.get('CODEBENDER_TEST_USER', config.TEST_CREDENTIALS['username']), + 'password': os.environ.get('CODEBENDER_TEST_PASS', config.TEST_CREDENTIALS['password']), + } + driver = self.driver + self.open("/") + + """ tests to ensure login div appears """ + login_elem = self.get_element(By.ID, "login_btn") #finds login button + login_elem.send_keys(Keys.RETURN) #clicks login button + logbox_elem = self.get_element(By.ID, "login_box") #finds login div + assert logbox_elem.is_displayed() #checks to see if div is visible + + """ tests login with invalid username """ + # define elements in login form + username_elem = self.get_element(By.ID, "username") + password_elem = self.get_element(By.ID, "password") + submit_elem = self.get_element(By.ID, "_submit") + + # enter invalid username with correct password + username_elem.send_keys("asdfghjklpoiuytrewqzxcvbnm") + password_elem.send_keys('1234567890') + submit_elem.click() + + # check for error message + error_elem = self.get_element(By.CLASS_NAME, 'text-error') + assert error_elem.is_displayed() + assert error_elem.text.strip() == 'Invalid username or password' + + """ tests login with invalid password """ + # refresh page so error message no longer visible + driver.refresh() + + # re-click on login button + login_elem = self.get_element(By.ID, "login_btn") + login_elem.send_keys(Keys.RETURN) + + # re-define elements in login form + username_elem = self.get_element(By.ID, "username") + password_elem = self.get_element(By.ID, "password") + submit_elem = self.get_element(By.ID, "_submit") + + # enter correct username with invalid password + username_elem.clear() + username_elem.send_keys(credentials['username']) + password_elem.send_keys(1234567890) + submit_elem.click() + + # re-define error message element and test + error_elem = self.get_element(By.CLASS_NAME, 'text-error') + assert error_elem.is_displayed() + assert error_elem.text.strip() == 'Invalid username or password' + + """ tests that login takes you to user's home """ + # refresh page so error message no longer visible + driver.refresh() + + # re-click on login button + login_elem = self.get_element(By.ID, "login_btn") + login_elem.send_keys(Keys.RETURN) + + # re-define elements in login form + username_elem = self.get_element(By.ID, "username") + password_elem = self.get_element(By.ID, "password") + submit_elem = self.get_element(By.ID, "_submit") + + # log in to site using correct credentials + username_elem.clear() + password_elem.clear() + username_elem.send_keys(credentials['username']) + password_elem.send_keys(credentials['password']) + submit_elem.click() + assert "Logged in as" in driver.page_source diff --git a/tests/common/sketch/__init__.py b/tests/common/sketch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/common/sketch/test_sketch.py b/tests/common/sketch/test_sketch.py new file mode 100644 index 0000000..3744969 --- /dev/null +++ b/tests/common/sketch/test_sketch.py @@ -0,0 +1,158 @@ +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions +from selenium.webdriver.support.select import Select +from selenium.webdriver.support.ui import WebDriverWait +import pytest + +from codebender_testing.config import TEST_PROJECT_NAME +from codebender_testing.utils import SeleniumTestCase +from codebender_testing.utils import SELECT_BOARD_SCRIPT +from codebender_testing.utils import throttle_compile + + +# How long to wait before we give up on trying to assess the result of commands +VERIFY_TIMEOUT = 30 +FLASH_TIMEOUT = 30 + +# Board to test for the dropdown selector. +TEST_BOARD = "Arduino Fio" + + +class TestSketch(SeleniumTestCase): + """Tests various functions of the /sketch view.""" + + @pytest.fixture(scope="class", autouse=True) + def create_test_project(self, tester_login): + """Makes sure we are logged in and have a project open before + performing any of these tests.""" + self.create_sketch(TEST_PROJECT_NAME) + + def test_verify_code(self): + """Ensures that we can compile code and see the success message.""" + boards = ['Arduino Uno', 'Arduino Leonardo', 'Arduino Mega 2560 or Mega ADK'] + for board in boards: + self.execute_script(SELECT_BOARD_SCRIPT(board)) + compile_button = self.get_element(By.ID, "cb_cf_verify_btn") + compile_button.click() + + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.invisibility_of_element_located( + (By.ID, "progress")) + ) + + operation_output = self.driver.find_element_by_id('operation_output') + assert operation_output.text.strip() == 'Verification successful!' + throttle_compile() + + def test_boards_dropdown(self): + """Tests that the boards dropdown is present, and that we can change + the board successfully.""" + boards_dropdown = Select(self.get_element(By.ID, "cb_cf_boards")) + + # Click something other than the first option + boards_dropdown.select_by_visible_text(TEST_BOARD) + + assert boards_dropdown.first_selected_option.text == TEST_BOARD + + @pytest.mark.requires_extension + def test_ports_dropdown(self): + """Tests that the ports dropdown exists.""" + ports = self.get_element(By.ID, "cb_cf_ports") + assert ports.text == 'No ports detected' + + @pytest.mark.requires_extension + def test_run_with_no_port(self): + """Makes sure that there is an error when we attempt to run with no + port selected.""" + flash_button = self.get_element(By.ID, "cb_cf_flash_btn") + flash_button.click() + WebDriverWait(self.driver, FLASH_TIMEOUT).until( + expected_conditions.text_to_be_present_in_element( + (By.ID, "operation_output"), "Please select a valid port!" + ) + ) + + @pytest.mark.requires_extension + def test_speeds_dropdown(self): + """Tests that the speeds dropdown exists.""" + self.get_element(By.ID, "cb_cf_baud_rates") + + @pytest.mark.requires_extension + def test_serial_monitor_disables_fields(self): + """Tests that opening the serial monitor disables the port and baudrate + fields.""" + open_serial_monitor_button = self.get_element(By.ID, 'cb_cf_serial_monitor_connect') + open_serial_monitor_button.click() + + WebDriverWait(self.driver, FLASH_TIMEOUT).until( + expected_conditions.text_to_be_present_in_element( + (By.ID, "operation_output"), 'Please select a valid port!' + ) + ) + + def test_clone_project(self): + """Tests that clicking the 'Clone Project' link brings us to a new + sketch with the title 'test_project clone'.""" + clone_link = self.get_element(By.ID, 'clone_btn') + clone_link.click() + project_name = self.get_element(By.ID, 'editor_heading_project_name') + # Here, I use `startswith` in case the user has a bunch of + # projects like "test_project copy copy copy" ... + assert project_name.text.startswith("%s copy" % TEST_PROJECT_NAME) + + # Cleanup: delete the project we just created. + self.delete_project("%s copy" % TEST_PROJECT_NAME) + + def test_add_projectfile_direct(self): + """ Tests that new file can be added to project using create-new-file + field """ + self.open_project() + + add_button = self.get_element(By.CLASS_NAME, 'add-file-button') + add_button.click() + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.visibility_of( + self.get_element(By.ID, "creationModal") + ) + ) + create_field = self.get_element(By.ID, 'createfield') + create_field.send_keys('test_file.txt') + create_button = self.get_element(By.ID, 'createbutton') + create_button.click() + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.invisibility_of_element_located( + (By.ID, "creationModal") + ) + ) + assert 'test_file.txt' in self.driver.page_source + + def test_delete_file(self): + """Tests file delete modal """ + delete_file_button = self.get_element(By.CLASS_NAME, 'delete-file-button') + delete_file_button.click() + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.visibility_of( + self.get_element(By.ID, "filedeleteModal") + ) + ) + assert self.get_element(By.ID, 'filedeleteModal').is_displayed() + + def test_verify_deletion(self): + """ Verifies that file has been deleted """ + confirm_delete_button = self.get_element(By.ID, 'filedeleteButton') + confirm_delete_button.click() + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.invisibility_of_element_located( + (By.ID, "filedeleteModal") + ) + ) + operation_output = self.get_element(By.ID, 'operation_output') + assert operation_output.text.strip() == 'File successfully deleted.' + WebDriverWait(self.driver, VERIFY_TIMEOUT).until( + expected_conditions.invisibility_of_element_located( + (By.CSS_SELECTOR, '#files_list a[data-name="test_file.txt"]') + ) + ) + + def test_remove_sketch(self): + self.delete_project(TEST_PROJECT_NAME) diff --git a/tests/common/user_home/__init__.py b/tests/common/user_home/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/common/user_home/test_user_home.py b/tests/common/user_home/test_user_home.py new file mode 100644 index 0000000..f2f9bb7 --- /dev/null +++ b/tests/common/user_home/test_user_home.py @@ -0,0 +1,83 @@ +from selenium.webdriver.common.by import By +import pytest + +from codebender_testing.config import TEST_DATA_INO +from codebender_testing.config import TEST_DATA_ZIP +from codebender_testing.config import SOURCE_BACHELOR +from codebender_testing.config import SOURCE_CODEBENDER_CC +from codebender_testing.utils import SeleniumTestCase + + +# Name to be used for the new project that is created. +NEW_PROJECT_NAME = 'selenium_TestUserHome' + +class TestUserHome(SeleniumTestCase): + + @pytest.fixture(scope="class", autouse=True) + def open_user_home(self, tester_login): + """Makes sure we are logged in and are at the user home page + performing any of these tests.""" + pass + + @pytest.mark.requires_source(SOURCE_BACHELOR) + def test_create_project_blank_name(self): + """Test that we get an error when creating a project with no name.""" + create_button = self.get_element(By.CSS_SELECTOR, '.form-search button') + create_button.click() + error_heading = self.get_element(By.CSS_SELECTOR, '.alert h4') + assert error_heading.text.startswith('Error') + + @pytest.mark.requires_source(SOURCE_BACHELOR) + def test_create_project_invalid_name(self): + """Test that we get an error when creating a project with an + invalid name (e.g., a name containing a backslash). + """ + project_name_input = self.get_element(By.CSS_SELECTOR, + '.form-search input[type=text]') + project_name_input.clear() + project_name_input.send_keys('foo\\bar') + create_button = self.get_element(By.CSS_SELECTOR, '.form-search button') + create_button.click() + + error_heading = self.get_element(By.CSS_SELECTOR, '.alert h4') + assert error_heading.text.startswith('Error') + + @pytest.mark.requires_source(SOURCE_BACHELOR) + def test_create_project_valid_name(self): + """Test that we can successfully create a project with a valid name.""" + project_name_input = self.get_element(By.CSS_SELECTOR, + '.form-search input[type=text]') + project_name_input.clear() + project_name_input.send_keys(NEW_PROJECT_NAME) + create_button = self.get_element(By.CSS_SELECTOR, '.form-search button') + create_button.click() + + project_heading = self.get_element(By.ID, 'editor_heading_project_name') + assert project_heading.text == NEW_PROJECT_NAME + + # Cleanup: delete the project we just created. + self.delete_project(NEW_PROJECT_NAME) + + def _upload_test(self, dropzone_selector, test_fname, sketch_name=None): + """Tests that we can successfully upload `test_fname`. + `project_name` is the expected name of the project; by + default it is inferred from the file name. + We delete the project if it is successfully uploaded. + """ + try: + upload_name = self.upload_project(dropzone_selector, test_fname, sketch_name) + assert upload_name in self.driver.page_source + finally: + self.delete_project(upload_name) + + @pytest.mark.requires_source(SOURCE_CODEBENDER_CC) + def test_upload_project_ino(self): + """Tests that we can upload a .ino file.""" + self._upload_test('#uploadInoModal form', TEST_DATA_INO) + + @pytest.mark.requires_source(SOURCE_CODEBENDER_CC) + def test_upload_project_zip(self): + """Tests that we can successfully upload a zipped project.""" + # TODO: how is the project name inferred from the zip file? + # Hardcoding the contents of the zip file feels weird here. + self._upload_test('#uploadFolderZip form', TEST_DATA_ZIP, sketch_name='upload_zip') diff --git a/tests/compile_tester/__init__.py b/tests/compile_tester/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/compile_tester/test_compile_tester_projects.py b/tests/compile_tester/test_compile_tester_projects.py new file mode 100644 index 0000000..fbbf516 --- /dev/null +++ b/tests/compile_tester/test_compile_tester_projects.py @@ -0,0 +1,39 @@ +from codebender_testing.config import COMPILE_TESTER_DIR +from codebender_testing.config import COMPILE_TESTER_LOGFILE +from codebender_testing.config import COMPILE_TESTER_URL +from codebender_testing.config import LIVE_SITE_URL +from codebender_testing.config import SOURCE_BACHELOR +from codebender_testing.utils import SeleniumTestCase +import os +import pytest + + +class TestCompileTester(SeleniumTestCase): + + # Here, we require the LIVE_SITE_URL since the compiler tester user + # does not exist anywhere else. + @pytest.mark.requires_url(LIVE_SITE_URL) + def test_compile_all_user_projects(self): + """Tests that all library examples compile successfully.""" + self.compile_all_sketches(COMPILE_TESTER_URL, '#user_projects tbody a', + iframe=True, logfile=COMPILE_TESTER_LOGFILE, + compile_type='sketch', create_report=True) + + # Here we require the bachelor site since cb_compile_tester's projects are + # already uploaded to the live site. + @pytest.mark.requires_source(SOURCE_BACHELOR) + def test_compile_local_files(self, tester_login): + """Tests that we can upload all of cb_compile_tester's projects + (stored locally in test_data/cb_compile_tester), compile them, + and finally delete them.""" + upload_limit = None if self.run_full_compile_tests else 1 + + test_files = [os.path.join(COMPILE_TESTER_DIR, name) + for name in next(os.walk(COMPILE_TESTER_DIR))[2]][:upload_limit] + projects = [self.upload_project(fname) for fname in test_files] + project_names, project_urls = zip(*projects) + + self.compile_sketches(project_urls, logfile=COMPILE_TESTER_LOGFILE, + compile_type='sketch', create_report=True) + for name in project_names: + self.delete_project(name) diff --git a/tests/conftest.py b/tests/conftest.py index 945f2f6..854b403 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,169 @@ +"""The configuration file for `py.test`. + +This file specifies global test fixtures, which include the selenium +webdrivers. + +This is also where command-line arguments and pytest markers are defined. +""" + +import os +import sys + import pytest -from codebender_testing.config import WEBDRIVERS +from codebender_testing import config + + +def pytest_addoption(parser): + """Adds command line options to the testing suite.""" + + parser.addoption("-U", "--url", action="store", default=config.BASE_URL, + help="URL to use for testing, e.g. http://localhost, http://codebender.cc") + + parser.addoption("-F", "--full", action="store_true", default=False, + help="Run the complete set of compile tests " + "(a minimal set of tests is run by default).") + + parser.addoption("-S", "--source", action="store", default=config.SOURCE_BACHELOR, + help="Indicate the source used to generate the repo. " + "By default, we assume `bachelor`. " + "You can instead use `codebender_cc` for the live site.") + + parser.addoption("-C", "--capabilities", action="store", + default=config.DEFAULT_CAPABILITIES_FILE_PATH, + help="Custom path to a YAML file containing a capability list.") + +def pytest_generate_tests(metafunc): + """Special function used by pytest to configure test generation.""" -@pytest.fixture(scope="session", params=WEBDRIVERS.keys()) -def webdriver(request): + # Paremetrize the desired_capabilities fixture on each of the capabilities + # objects in the YAML file. + if 'desired_capabilities' in metafunc.fixturenames: + capabilities_path = metafunc.config.option.capabilities + metafunc.parametrize('desired_capabilities', + config.get_browsers(capabilities_path), + scope="session") + + +@pytest.fixture(scope="session") +def webdriver(request, desired_capabilities): """Returns a webdriver that persists across the entire test session, and registers a finalizer to close the browser once the session is complete. The entire test session is repeated once per driver. """ - driver = WEBDRIVERS[request.param] - request.addfinalizer(lambda: driver.quit()) + + command_executor = os.environ['CODEBENDER_SELENIUM_HUB_URL'] + + driver = config.create_webdriver(command_executor, desired_capabilities) + + # TODO: update sauce status via SauceClient, but only if the command_executor + # is a sauce URL. + def finalizer(): + # print("Link to your job: https://saucelabs.com/jobs/%s" % driver.session_id) + try: + pass + # TODO: + # if sys.exc_info() == (None, None, None): + # sauce.jobs.update_job(driver.session_id, passed=True) + # else: + # sauce.jobs.update_job(driver.session_id, passed=False) + finally: + driver.quit() + + request.addfinalizer(finalizer) return driver + +@pytest.fixture(scope="class") +def testing_url(request): + """A fixture to get the --url parameter.""" + return request.config.getoption("--url") + + +@pytest.fixture(scope="class") +def source(request): + """A fixture to specify the source repository from which the site was + derived (e.g. bachelor or codebender_cc) + """ + return request.config.getoption("--source") + + +@pytest.fixture(scope="class") +def testing_credentials(request): + """A fixture to get testing credentials specified via the environment + variables CODEBENDER_TEST_USER and CODEBENDER_TEST_PASS. Defaults to the + credentials specified in config.TEST_CREDENTIALS. + """ + return { + 'username': os.environ.get('CODEBENDER_TEST_USER', config.TEST_CREDENTIALS['username']), + 'password': os.environ.get('CODEBENDER_TEST_PASS', config.TEST_CREDENTIALS['password']), + } + + +@pytest.fixture(scope="class") +def testing_full(request): + """A fixture to get the --full parameter.""" + return request.config.getoption("--full") + + +@pytest.fixture(autouse=True) +def requires_source(request, source): + """Skips tests that require a certain source version (e.g. bachelor or + codebender_cc) in order to run properly. + + This functionality should be invoked as a pytest marker, e.g.: + + ``` + @pytest.mark.requires_source("bachelor") + def test_some_feature(): + ... + ``` + """ + if request.node.get_marker('requires_source'): + required_source = request.node.get_marker('requires_source').args[0] + if required_source != source: + pytest.skip('skipped test that requires --source=' + source) + + +@pytest.fixture(autouse=True) +def requires_url(request, testing_url): + """Skips tests that require a certain site URL in order to run properly. + This is strictly more specific than requires_source; consider using that + marker instead. + + This functionality should be invoked as a pytest marker, e.g.: + + ``` + @pytest.mark.requires_url("http://codebender.cc") + def test_some_feature(): + ... + ``` + """ + if request.node.get_marker('requires_url'): + required_url = request.node.get_marker('requires_url').args[0] + if required_url.rstrip('/') != testing_url.rstrip('/'): + pytest.skip('skipped test that requires --url=%s' % required_url) + + +@pytest.fixture(autouse=True) +def requires_extension(request, webdriver): + """Mark that a test requires the codebender extension. + Ideally, this marker would not be necessary. However, it is used so that we + skip tests when running under chrome that require the extension (for now). + This is due to the fact that the chrome driver leaves open the + "confirm extension" dialogue without actually installing it. + + This functionality should be invoked as a pytest marker, e.g.: + + ``` + @pytest.mark.requires_extension + def test_some_feature(): + ... + ``` + """ + if request.node.get_marker('requires_extension'): + if webdriver.desired_capabilities["browserName"] == "chrome": + pytest.skip("skipped test that requires codebender extension. " + "The current webdriver is Chrome, and the ChromeDriver " + "does not properly install extensions.") \ No newline at end of file diff --git a/tests/home/test_home.py b/tests/home/test_home.py deleted file mode 100644 index 8ae7d72..0000000 --- a/tests/home/test_home.py +++ /dev/null @@ -1,90 +0,0 @@ -from codebender_testing.config import TEST_CREDENTIALS -from codebender_testing.utils import SeleniumTestCase -from selenium.webdriver.common.keys import Keys - -class TestHome(SeleniumTestCase): - - def test_navigate_home(self): - """ opens browser to codebender bachelor """ - self.open("/") - assert "Codebender" in self.driver.title - - def test_login(self): - driver = self.driver - self.open("/") - - """ tests to ensure login div appears """ - login_elem = driver.find_element_by_id("login_btn") #finds login button - login_elem.send_keys(Keys.RETURN) #clicks login button - logbox_elem = driver.find_element_by_id("login_box") #finds login div - assert logbox_elem.is_displayed() #checks to see if div is visible - - """ tests login with invalid username """ - # define elements in login form - username_elem = driver.find_element_by_id("username") - password_elem = driver.find_element_by_id("password") - submit_elem = driver.find_element_by_id("_submit") - - # enter invalid username with correct password - username_elem.send_keys("codebender") - password_elem.send_keys(TEST_CREDENTIALS['password']) - submit_elem.click() - - # check for error message - error_elem = driver.find_element_by_class_name('text-error') - assert error_elem.is_displayed() - - """ tests login with invalid password """ - # refresh page so error message no longer visible - driver.refresh() - - # re-click on login button - login_elem = driver.find_element_by_id("login_btn") - login_elem.send_keys(Keys.RETURN) - - # re-define elements in login form - username_elem = driver.find_element_by_id("username") - password_elem = driver.find_element_by_id("password") - submit_elem = driver.find_element_by_id("_submit") - - # enter correct username with invalid password - username_elem.clear() - username_elem.send_keys(TEST_CREDENTIALS['username']) - password_elem.send_keys("codebender") - submit_elem.click() - - # re-define error message element and test - error_elem = driver.find_element_by_class_name('text-error') - assert error_elem.is_displayed() - - """ tests that login takes you to user's home """ - # refresh page so error message no longer visible - driver.refresh() - - # re-click on login button - login_elem = driver.find_element_by_id("login_btn") - login_elem.send_keys(Keys.RETURN) - - # re-define elements in login form - username_elem = driver.find_element_by_id("username") - password_elem = driver.find_element_by_id("password") - submit_elem = driver.find_element_by_id("_submit") - - # log in to site using correct credentials - username_elem.clear() - password_elem.clear() - username_elem.send_keys(TEST_CREDENTIALS['username']) - password_elem.send_keys(TEST_CREDENTIALS['password']) - submit_elem.click() - assert "Logged in as" in driver.page_source - - - - - - - - - - - diff --git a/tests/libraries/__init__.py b/tests/libraries/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/libraries/test_libraries.py b/tests/libraries/test_libraries.py new file mode 100644 index 0000000..9d4925e --- /dev/null +++ b/tests/libraries/test_libraries.py @@ -0,0 +1,11 @@ +from codebender_testing.config import LIBRARIES_TEST_LOGFILE +from codebender_testing.utils import SeleniumTestCase + + +class TestLibraryExamples(SeleniumTestCase): + + def test_compile_all_libraries(self): + """Tests that all library examples compile successfully.""" + self.compile_all_sketches('/libraries', '.accordion li a', + logfile=LIBRARIES_TEST_LOGFILE, + compile_type='library', create_report=True, comment=True) diff --git a/tests/sketch/test_sketch.py b/tests/sketch/test_sketch.py deleted file mode 100644 index 57b636a..0000000 --- a/tests/sketch/test_sketch.py +++ /dev/null @@ -1,120 +0,0 @@ -import time - -from selenium.common.exceptions import StaleElementReferenceException -from selenium.webdriver.common.by import By -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.support import expected_conditions -from selenium.webdriver.support.select import Select -from selenium.webdriver.support.ui import WebDriverWait -import pytest - -from codebender_testing.config import TEST_PROJECT_NAME -from codebender_testing.utils import SeleniumTestCase - - -# How long to wait before we give up on trying to assess the result of commands -VERIFY_TIMEOUT = 10 -FLASH_TIMEOUT = 2 - -# Board to test for the dropdown selector. -TEST_BOARD = "Arduino Fio" - - -class TestSketch(SeleniumTestCase): - """Tests various functions of the /sketch view.""" - - @pytest.fixture(scope="class", autouse=True) - def open_test_project(self, tester_login): - """Makes sure we are logged in and have a project open before - performing any of these tests.""" - self.open_project() - # I get a StaleElementReferenceException without - # this wait. TODO: figure out how to get around this. - time.sleep(3) - - def test_verify_code(self): - """Ensures that we can compile code and see the success message.""" - compile_button = self.driver.find_element_by_id("compile") - compile_button.click() - - # test progress bar is visible - progress_bar = self.get_element(By.ID, 'progress') - assert progress_bar.is_displayed() - - WebDriverWait(self.driver, VERIFY_TIMEOUT).until( - expected_conditions.text_to_be_present_in_element( - (By.ID, "operation_output"), "Verification Successful!") - ) - - def test_boards_dropdown(self): - """Tests that the boards dropdown is present, and that we can change - the board successfully.""" - boards_dropdown = Select(self.get_element(By.ID, "boards")) - - # Click something other than the first option - boards_dropdown.select_by_visible_text(TEST_BOARD) - - assert boards_dropdown.first_selected_option.text == TEST_BOARD - - def test_ports_dropdown(self): - """Tests that the ports dropdown exists.""" - self.get_element(By.ID, "ports") - - def test_run_with_no_port(self): - """Makes sure that there is an error when we attempt to run with no - port selected.""" - flash_button = self.get_element(By.ID, "uploadusb") - flash_button.click() - WebDriverWait(self.driver, FLASH_TIMEOUT).until( - expected_conditions.text_to_be_present_in_element( - (By.ID, "operation_output"), "Please select a valid port or enable the plugin!!")) - - def test_speeds_dropdown(self): - """Tests that the speeds dropdown exists.""" - self.get_element(By.ID, "baudrates") - - def test_clone_project(self): - """Tests that clicking the 'Clone Project' link brings us to a new - sketch with the title 'test_project clone'.""" - clone_link = self.get_element(By.LINK_TEXT, 'Clone Project') - clone_link.click() - project_name = self.get_element(By.ID, 'editor_heading_project_name') - assert project_name.text.startswith("%s copy" % TEST_PROJECT_NAME) - - def test_add_projectfile_direct(self): - """ Tests that new file can be added to project using create-new-file field """ - add_button = self.get_element(By.CLASS_NAME, 'icon-plus') - add_button.click() - create_field = self.get_element(By.ID, 'createfield') - create_field.send_keys('test_file.txt') - create_button = self.get_element(By.CLASS_NAME, 'btn') - create_button.click() - self.driver.refresh() - assert 'test_file.txt' in self.driver.page_source - ''' - def test_add_projectfile_upload(self): - """ Tests that new file can be added to project using upload dialog """ - add_button = self.get_element(By.CLASS_NAME, 'icon-plus') - add_button.click() - drop_zone = self.get_element(By.CLASS_NAME, 'dz-clickable') - drop_zone.click() - self.driver.get("http://localhost/js/dropzone/min.js") - self.driver.execute_script("self.get_element(By.NAME,'uploadType').value = '/test.h'") - - #file_input_element = self.get_element(By.NAME, 'uploadType')''' - - def test_delete_file(self): - """Tests file delete modal """ - delete_file_button = self.get_element(By.CLASS_NAME, 'icon-remove') - delete_file_button.click() - delete_modal = self.get_element(By.ID, 'filedeleteModal') - assert delete_modal.is_displayed() - - def test_verify_deletion(self): - """ Verifies that file has been deleted """ - confirm_delete_button = self.get_element(By.ID, 'filedeleteButton') - confirm_delete_button.click() - self.driver.refresh() - assert 'test_file.txt' not in self.driver.page_source - - diff --git a/tox.ini b/tox.ini index 0578bf5..9287c8d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,9 @@ [tox] -envlist = py32 +envlist = py27 [testenv] -deps = -rrequirements.txt -commands = py.test +install_command = pip install -U {opts} {packages} +deps = -r{toxinidir}/requirements.txt +commands = py.test -s {posargs} # Pass all command line args from tox to py.test. + # e.g., `tox ` will invoke `py.test `. +passenv = *