From e614193b1ad11f9f92ff236ffbb84f94cd9adb02 Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Wed, 25 Aug 2021 17:15:07 -0400
Subject: [PATCH 01/12] Configuration: configuration for repository

---
 ecosystem/controllers/__init__.py |  0
 ecosystem/controllers/runner.py   | 68 +++++++++++++++++++++++++++++++
 ecosystem/entities.py             | 25 +++++++-----
 ecosystem/models/__init__.py      |  3 ++
 ecosystem/models/configuration.py | 62 ++++++++++++++++++++++++++++
 tests/common.py                   | 18 ++++++++
 tests/test_commands.py            | 13 +-----
 tests/test_configuration.py       | 26 ++++++++++++
 8 files changed, 193 insertions(+), 22 deletions(-)
 create mode 100644 ecosystem/controllers/__init__.py
 create mode 100644 ecosystem/controllers/runner.py
 create mode 100644 ecosystem/models/__init__.py
 create mode 100644 ecosystem/models/configuration.py
 create mode 100644 tests/common.py
 create mode 100644 tests/test_configuration.py

diff --git a/ecosystem/controllers/__init__.py b/ecosystem/controllers/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
new file mode 100644
index 0000000000..f21998b065
--- /dev/null
+++ b/ecosystem/controllers/runner.py
@@ -0,0 +1,68 @@
+"""Ecosystem test runner."""
+import os
+import shutil
+from abc import abstractmethod
+from logging import Logger
+from typing import Dict, Optional
+
+from ecosystem.entities import CommandExecutionSummary
+from ecosystem.logging import logger as ecosystem_logger
+
+
+class Runner:
+    """Runner for repository checks."""
+
+    def __init__(self,
+                 working_directory: Optional[str] = None,
+                 logger: Optional[Logger] = None):
+        self.working_directory = working_directory or "./"
+        self.logger = logger or ecosystem_logger
+        self.cloned_repos_directory = "{}/cloned_repos_directory" \
+                                      "".format(self.working_directory)
+
+    def set_up(self):
+        """Preparation step before running workload."""
+        if self.cloned_repos_directory and \
+                os.path.exists(self.cloned_repos_directory):
+            shutil.rmtree(self.cloned_repos_directory)
+        os.makedirs(self.cloned_repos_directory)
+
+    def tear_down(self):
+        """Execution after workload is finished."""
+        if self.cloned_repos_directory and \
+                os.path.exists(self.cloned_repos_directory):
+            shutil.rmtree(self.cloned_repos_directory)
+
+    @abstractmethod
+    def workload(self) -> Dict[str, CommandExecutionSummary]:
+        """Runs workload of commands to check repository."""
+
+    def run(self) -> Dict[str, CommandExecutionSummary]:
+        """Runs chain of commands to check repository."""
+        self.set_up()
+        result = self.workload()
+        self.tear_down()
+        return result
+
+
+class PythonRunner(Runner):
+    """Runners for Python repositories."""
+
+    def __init__(self,
+                 python_version: str = "py39"):
+        super().__init__()
+        self.template = ""
+        self.python_version = python_version
+
+    def workload(self) -> Dict[str, CommandExecutionSummary]:
+        """Runs checks for python repository.
+
+        Steps:
+        - check for configuration file
+        - optional: check for tox file
+        - optional: render tox file
+        - run tests
+        - form report
+
+        Returns: execution summary of steps
+        """
diff --git a/ecosystem/entities.py b/ecosystem/entities.py
index 28ae94d748..6615a01d38 100644
--- a/ecosystem/entities.py
+++ b/ecosystem/entities.py
@@ -32,7 +32,20 @@ def all(cls):
         return [cls.STANDARD, cls.DEV_COMPATIBLE, cls.STABLE_COMPATIBLE]
 
 
-class Repository(ABC):
+class JsonSerializable(ABC):
+    """Classes that can be serialized as json."""
+    def to_dict(self) -> dict:
+        """Converts repo to dict."""
+        result = dict()
+        for name, value in inspect.getmembers(self):
+            if not name.startswith('_') and \
+                    not inspect.ismethod(value) and not inspect.isfunction(value) and \
+                    hasattr(self, name):
+                result[name] = value
+        return result
+
+
+class Repository(JsonSerializable):
     """Main repository class."""
 
     tier: str
@@ -79,16 +92,6 @@ def __init__(self,
         if tier:
             self.tier = tier
 
-    def to_dict(self) -> dict:
-        """Converts repo to dict."""
-        result = dict()
-        for name, value in inspect.getmembers(self):
-            if not name.startswith('_') and \
-                    not inspect.ismethod(value) and not inspect.isfunction(value) and \
-                    hasattr(self, name):
-                result[name] = value
-        return result
-
     def __eq__(self, other: 'Repository'):
         return (self.tier == other.tier
                 and self.url == other.url
diff --git a/ecosystem/models/__init__.py b/ecosystem/models/__init__.py
new file mode 100644
index 0000000000..5143af05c2
--- /dev/null
+++ b/ecosystem/models/__init__.py
@@ -0,0 +1,3 @@
+"""Models for ecosystem."""
+
+from .configuration import RepositoryConfiguration
diff --git a/ecosystem/models/configuration.py b/ecosystem/models/configuration.py
new file mode 100644
index 0000000000..6a24a19a1b
--- /dev/null
+++ b/ecosystem/models/configuration.py
@@ -0,0 +1,62 @@
+"""Configuration for ecosystem repository."""
+import json
+import pprint
+from typing import Optional, List
+
+from ecosystem.entities import JsonSerializable
+
+
+class Languages:
+    """Supported configuration languages."""
+    PYTHON: str = "python"
+
+    def all(self) -> List[str]:
+        """Return all supported languages."""
+        return [self.PYTHON]
+
+    def __repr__(self):
+        return "Languages({})".format(",".join(self.all()))
+
+
+class RepositoryConfiguration(JsonSerializable):
+    """Configuration for ecosystem repository."""
+
+    def __init__(self,
+                 language: str = Languages.PYTHON,
+                 dependencies_files: Optional[List[str]] = None,
+                 extra_dependencies: Optional[List[str]] = None,
+                 tests_command: Optional[List[str]] = None,
+                 styles_check_command: Optional[List[str]] = None):
+        """Configuration for ecosystem repository.
+
+        Args:
+            language: repository language
+            dependencies_files: list of dependencies files paths relative to root of repo
+                ex: for python `requirements.txt`
+            extra_dependencies: list of extra dependencies for project to install during tests run
+                ex: for python it might be `qiskit==0.19`
+            tests_command: list of commands to run tests
+                ex: for python `python -m unittest -v`
+            styles_check_command: list of commands to run style checks
+                ex: for python `pylint -rn ecosystem tests`
+        """
+        self.language = language
+        self.dependencies_files = dependencies_files or []
+        self.extra_dependencies = extra_dependencies or []
+        self.tests_command = tests_command or []
+        self.styles_check_command = styles_check_command or []
+
+    def save(self, path: str):
+        """Saves configuration as json file."""
+        with open(path, "w") as json_file:
+            json.dump(self.to_dict(), json_file, indent=4)
+
+    @classmethod
+    def load(cls, path: str) -> 'RepositoryConfiguration':
+        """Loads json file into object."""
+        with open(path, "r") as json_file:
+            return json.load(json_file,
+                             object_hook=lambda d: RepositoryConfiguration(**d))
+
+    def __repr__(self):
+        return pprint.pformat(self.to_dict(), indent=4)
diff --git a/tests/common.py b/tests/common.py
new file mode 100644
index 0000000000..a5ab584c83
--- /dev/null
+++ b/tests/common.py
@@ -0,0 +1,18 @@
+"""Common test classes."""
+import os
+import shutil
+import unittest
+
+
+class TestCaseWithResources(unittest.TestCase):
+    """Test case with additional resources folder."""
+    path: str
+
+    def setUp(self) -> None:
+        self.path = "./resources"
+        if not os.path.exists(self.path):
+            os.makedirs(self.path)
+
+    def tearDown(self) -> None:
+        if os.path.exists(self.path):
+            shutil.rmtree(self.path)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 7816cba93a..083eec8c81 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -1,22 +1,13 @@
 """Tests for shell commands."""
 import os
-import shutil
-from unittest import TestCase
 
 from ecosystem.commands import _execute_command, _clone_repo
 from ecosystem.entities import CommandExecutionSummary
+from .common import TestCaseWithResources
 
 
-class TestCommands(TestCase):
+class TestCommands(TestCaseWithResources):
     """Tests shell commands."""
-    def setUp(self) -> None:
-        self.path = "./resources"
-        if not os.path.exists(self.path):
-            os.makedirs(self.path)
-
-    def tearDown(self) -> None:
-        if os.path.exists(self.path):
-            shutil.rmtree(self.path)
 
     def test_execute_command(self):
         """Tests command execution."""
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
new file mode 100644
index 0000000000..95987cd4a0
--- /dev/null
+++ b/tests/test_configuration.py
@@ -0,0 +1,26 @@
+"""Tests for configuration files."""
+
+from ecosystem.models import RepositoryConfiguration
+from tests.common import TestCaseWithResources
+
+
+class TestRepositoryConfiguration(TestCaseWithResources):
+    """Tests for RepositoryConfiguration file."""
+
+    def test_save_and_load(self):
+        """Tests saving and loading of configuration,"""
+        config = RepositoryConfiguration(dependencies_files=["requirements.txt",
+                                                             "requirements-dev.txt"],
+                                         extra_dependencies=["qiskit"],
+                                         tests_command=["python -m unittest -v"],
+                                         styles_check_command=["pylint -rn ecosystem tests"])
+        save_path = f"{self.path}/config.json"
+        config.save(save_path)
+
+        recovered_config = RepositoryConfiguration.load(save_path)
+
+        self.assertEqual(config.language, recovered_config.language)
+        self.assertEqual(config.tests_command, recovered_config.tests_command)
+        self.assertEqual(config.dependencies_files, recovered_config.dependencies_files)
+        self.assertEqual(config.extra_dependencies, recovered_config.extra_dependencies)
+        self.assertEqual(config.styles_check_command, recovered_config.styles_check_command)

From 8795372f155b55cf4d0a6271cec3aa88a99b9e59 Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Thu, 26 Aug 2021 16:04:41 -0400
Subject: [PATCH 02/12] Configurations: configurable runs

---
 ecosystem/commands.py                  |  2 +-
 ecosystem/controllers/runner.py        | 98 ++++++++++++++++++++------
 ecosystem/manager.py                   | 12 +++-
 ecosystem/models/__init__.py           |  2 +-
 ecosystem/models/configuration.py      | 40 ++++++++++-
 ecosystem/templates/configured_tox.ini | 28 ++++++++
 ecosystem/{logging.py => utils.py}     |  4 ++
 tests/test_configuration.py            | 14 +++-
 8 files changed, 173 insertions(+), 27 deletions(-)
 create mode 100644 ecosystem/templates/configured_tox.ini
 rename ecosystem/{logging.py => utils.py} (88%)

diff --git a/ecosystem/commands.py b/ecosystem/commands.py
index a0f4560280..ff0cf757d6 100644
--- a/ecosystem/commands.py
+++ b/ecosystem/commands.py
@@ -8,7 +8,7 @@
 from jinja2 import Template
 
 from ecosystem.entities import CommandExecutionSummary
-from ecosystem.logging import logger
+from ecosystem.utils import logger
 
 
 def _execute_command(command: List[str],
diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
index f21998b065..8fc9c9c11e 100644
--- a/ecosystem/controllers/runner.py
+++ b/ecosystem/controllers/runner.py
@@ -3,44 +3,56 @@
 import shutil
 from abc import abstractmethod
 from logging import Logger
-from typing import Dict, Optional
+from typing import Optional, Union, cast, List, Tuple
 
-from ecosystem.entities import CommandExecutionSummary
-from ecosystem.logging import logger as ecosystem_logger
+from ecosystem.commands import _clone_repo, _run_tox
+from ecosystem.entities import CommandExecutionSummary, Repository
+from ecosystem.utils import logger as ecosystem_logger
+from ecosystem.models import RepositoryConfiguration, PythonRepositoryConfiguration
+from ecosystem.utils import QiskitEcosystemException
 
 
 class Runner:
     """Runner for repository checks."""
 
     def __init__(self,
+                 repo: Union[str, Repository],
                  working_directory: Optional[str] = None,
                  logger: Optional[Logger] = None):
-        self.working_directory = working_directory or "./"
+        self.repo: str = repo.url if isinstance(repo, Repository) else repo
+        self.working_directory = f"{working_directory}/cloned_repo_directory" or "./"
         self.logger = logger or ecosystem_logger
-        self.cloned_repos_directory = "{}/cloned_repos_directory" \
-                                      "".format(self.working_directory)
+        name = self.repo.split("/")[-1]
+        self.cloned_repo_directory = f"{self.working_directory}/{name}"
 
     def set_up(self):
         """Preparation step before running workload."""
-        if self.cloned_repos_directory and \
-                os.path.exists(self.cloned_repos_directory):
-            shutil.rmtree(self.cloned_repos_directory)
-        os.makedirs(self.cloned_repos_directory)
+        if self.cloned_repo_directory and \
+                os.path.exists(self.cloned_repo_directory):
+            shutil.rmtree(self.cloned_repo_directory)
+        os.makedirs(self.cloned_repo_directory)
 
     def tear_down(self):
         """Execution after workload is finished."""
-        if self.cloned_repos_directory and \
-                os.path.exists(self.cloned_repos_directory):
-            shutil.rmtree(self.cloned_repos_directory)
+        if self.cloned_repo_directory and \
+                os.path.exists(self.cloned_repo_directory):
+            shutil.rmtree(self.cloned_repo_directory)
 
     @abstractmethod
-    def workload(self) -> Dict[str, CommandExecutionSummary]:
-        """Runs workload of commands to check repository."""
+    def workload(self) -> Tuple[str, CommandExecutionSummary]:
+        """Runs workload of commands to check repository.
 
-    def run(self) -> Dict[str, CommandExecutionSummary]:
+        Returns: tuple (qiskit_version, CommandExecutionSummary)
+        """
+
+    def run(self) -> Tuple[str, CommandExecutionSummary]:
         """Runs chain of commands to check repository."""
+        result = {}
         self.set_up()
-        result = self.workload()
+        try:
+            result = self.workload()
+        except Exception as exception:  # pylint: disable=broad-except)
+            self.logger.error(exception)
         self.tear_down()
         return result
 
@@ -49,12 +61,16 @@ class PythonRunner(Runner):
     """Runners for Python repositories."""
 
     def __init__(self,
+                 repo: Union[str, Repository],
+                 working_directory: Optional[str] = None,
+                 ecosystem_deps: Optional[List[str]] = None,
                  python_version: str = "py39"):
-        super().__init__()
-        self.template = ""
+        super().__init__(repo=repo,
+                         working_directory=working_directory)
         self.python_version = python_version
+        self.ecosystem_deps = ecosystem_deps or []
 
-    def workload(self) -> Dict[str, CommandExecutionSummary]:
+    def workload(self) -> Tuple[str, CommandExecutionSummary]:
         """Runs checks for python repository.
 
         Steps:
@@ -66,3 +82,45 @@ def workload(self) -> Dict[str, CommandExecutionSummary]:
 
         Returns: execution summary of steps
         """
+
+        # clone repository
+        self.logger.info("Cloning repository.")
+        clone_res = _clone_repo(self.repo, directory=self.working_directory)
+
+        if not clone_res.ok:
+            raise QiskitEcosystemException(
+                f"Something went wrong with cloning {self.repo} repository.")
+
+        # check for configuration file
+        if os.path.exists(f"{self.cloned_repo_directory}/qe_config.json"):
+            loaded_config = RepositoryConfiguration.load(
+                f"{self.cloned_repo_directory}/qe_config.json")
+            repo_config = cast(PythonRepositoryConfiguration, loaded_config)
+        else:
+            repo_config = PythonRepositoryConfiguration.default()
+
+        # check for existing tox file
+        self.logger.info(f"Creating tox file {self.cloned_repo_directory}/tox.ini")
+        if os.path.exists(f"{self.cloned_repo_directory}/tox.ini"):
+            self.logger.info("Tox file exists.")
+            os.rename(f"{self.cloned_repo_directory}/tox.ini",
+                      f"{self.cloned_repo_directory}/tox_default.ini")
+
+        # render new tox file for tests
+        with open(f"{self.cloned_repo_directory}/tox.ini", "w") as tox_file:
+            tox_file.write(repo_config.render_tox_file(
+                ecosystem_deps=self.ecosystem_deps))
+
+        # run tox
+        tox_tests_res = _run_tox(directory=self.cloned_repo_directory,
+                                 env=self.python_version)
+
+        # get terra version from file
+        if os.path.exists(f"{self.cloned_repo_directory}/terra_version.txt"):
+            with open(f"{self.cloned_repo_directory}/terra_version.txt", "r") as version_file:
+                terra_version = version_file.read()
+                self.logger.info(f"Terra version: {terra_version}")
+        else:
+            self.logger.warning("There in no terra version file...")
+
+        return terra_version, tox_tests_res
diff --git a/ecosystem/manager.py b/ecosystem/manager.py
index 6e28b2b456..c7e05de57c 100644
--- a/ecosystem/manager.py
+++ b/ecosystem/manager.py
@@ -5,9 +5,10 @@
 from jinja2 import Environment, PackageLoader, select_autoescape
 
 from .controller import Controller
+from .controllers.runner import PythonRunner
 from .entities import Tier, TestType
 from .commands import run_tests
-from .logging import logger
+from .utils import logger
 
 
 class Manager:
@@ -115,5 +116,10 @@ def dev_compatibility_tests(self,
                          tox_python=tox_python,
                          dependencies=["git+https://github.com/Qiskit/qiskit-terra.git@main"])
 
-    def __repr__(self):
-        return "Manager(CLI entrypoint)"
+    def stable_tests(self, repo_url: str):
+        """Runs tests against stable version of qiskit."""
+        runner = PythonRunner(repo_url,
+                              working_directory=self.resources_dir,
+                              ecosystem_deps=["qiskit"])
+        terra_version, _ = runner.run()
+        return terra_version
diff --git a/ecosystem/models/__init__.py b/ecosystem/models/__init__.py
index 5143af05c2..49e183ec57 100644
--- a/ecosystem/models/__init__.py
+++ b/ecosystem/models/__init__.py
@@ -1,3 +1,3 @@
 """Models for ecosystem."""
 
-from .configuration import RepositoryConfiguration
+from .configuration import RepositoryConfiguration, PythonRepositoryConfiguration
diff --git a/ecosystem/models/configuration.py b/ecosystem/models/configuration.py
index 6a24a19a1b..aceb18aecd 100644
--- a/ecosystem/models/configuration.py
+++ b/ecosystem/models/configuration.py
@@ -3,6 +3,8 @@
 import pprint
 from typing import Optional, List
 
+from jinja2 import Environment, PackageLoader, select_autoescape
+
 from ecosystem.entities import JsonSerializable
 
 
@@ -26,7 +28,7 @@ def __init__(self,
                  dependencies_files: Optional[List[str]] = None,
                  extra_dependencies: Optional[List[str]] = None,
                  tests_command: Optional[List[str]] = None,
-                 styles_check_command: Optional[List[str]] = None):
+                 styles_check_command: Optional[List[str]] = None,):
         """Configuration for ecosystem repository.
 
         Args:
@@ -60,3 +62,39 @@ def load(cls, path: str) -> 'RepositoryConfiguration':
 
     def __repr__(self):
         return pprint.pformat(self.to_dict(), indent=4)
+
+
+class PythonRepositoryConfiguration(RepositoryConfiguration):
+    """Repository configuration for python based projects."""
+    def __init__(self,
+                 dependencies_files: Optional[List[str]] = None,
+                 extra_dependencies: Optional[List[str]] = None,
+                 tests_command: Optional[List[str]] = None,
+                 styles_check_command: Optional[List[str]] = None):
+        super().__init__(language=Languages.PYTHON,
+                         dependencies_files=dependencies_files,
+                         extra_dependencies=extra_dependencies,
+                         tests_command=tests_command,
+                         styles_check_command=styles_check_command)
+        env = Environment(
+            loader=PackageLoader("ecosystem"),
+            autoescape=select_autoescape()
+        )
+        self.tox_template = env.get_template("configured_tox.ini")
+
+    @classmethod
+    def default(cls) -> 'PythonRepositoryConfiguration':
+        """Returns default python repository configuration."""
+        return PythonRepositoryConfiguration(dependencies_files=[
+                                                 "requirements.txt"
+                                             ],
+                                             tests_command=[
+                                                 "pip check",
+                                                 "pytest -W error::DeprecationWarning"
+                                             ])
+
+    def render_tox_file(self, ecosystem_deps: List[str] = None):
+        """Renders tox template from configuration."""
+        ecosystem_deps = ecosystem_deps or []
+        return self.tox_template.render({**self.to_dict(),
+                                         **{'ecosystem_deps': ecosystem_deps}})
diff --git a/ecosystem/templates/configured_tox.ini b/ecosystem/templates/configured_tox.ini
new file mode 100644
index 0000000000..111c6409a4
--- /dev/null
+++ b/ecosystem/templates/configured_tox.ini
@@ -0,0 +1,28 @@
+[tox]
+minversion = 3.6
+envlist = py36, py37, py38, py39
+skipsdist = True
+
+[testenv]
+usedevelop = true
+install_command = pip install -U {opts} {packages}
+setenv =
+  VIRTUAL_ENV={envdir}
+  LANGUAGE=en_US
+  LC_ALL=en_US.utf-8
+deps = pytest
+{% for dep_file in dependencies_files -%}
+       {{"-r"|indent(7, True)}} {{ dep_file }}
+{% endfor -%}
+{% for dep in extra_dependencies -%}
+       {{ dep|indent(7, True) }}
+{% endfor -%}
+{% for dep in ecosystem_deps -%}
+       {{ dep|indent(7, True) }}
+{% endfor -%}
+commands =
+  python -c 'import qiskit; f = open("./terra_version.txt", "w"); f.write(qiskit.__qiskit_version__["qiskit-terra"]); f.close();'
+{% for command in tests_command -%}
+  {{ command|indent(2, True) }}
+{% endfor -%}
+
diff --git a/ecosystem/logging.py b/ecosystem/utils.py
similarity index 88%
rename from ecosystem/logging.py
rename to ecosystem/utils.py
index 8636e9b96e..882867862a 100644
--- a/ecosystem/logging.py
+++ b/ecosystem/utils.py
@@ -3,6 +3,10 @@
 import os
 
 
+class QiskitEcosystemException(Exception):
+    """Exceptions for qiskit ecosystem."""
+
+
 class OneLineExceptionFormatter(logging.Formatter):
     """Exception formatter"""
     def formatException(self, ei):
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index 95987cd4a0..ee4fb2ad19 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -1,6 +1,6 @@
 """Tests for configuration files."""
 
-from ecosystem.models import RepositoryConfiguration
+from ecosystem.models import RepositoryConfiguration, PythonRepositoryConfiguration
 from tests.common import TestCaseWithResources
 
 
@@ -24,3 +24,15 @@ def test_save_and_load(self):
         self.assertEqual(config.dependencies_files, recovered_config.dependencies_files)
         self.assertEqual(config.extra_dependencies, recovered_config.extra_dependencies)
         self.assertEqual(config.styles_check_command, recovered_config.styles_check_command)
+
+    def test_python_configuration(self):
+        """Tests python configurations."""
+        config = PythonRepositoryConfiguration.default()
+        rendered_tox = config.render_tox_file()
+        self.assertTrue(config)
+        for command in config.tests_command:
+            self.assertTrue(command in rendered_tox)
+        for dep in config.extra_dependencies:
+            self.assertTrue(dep in rendered_tox)
+        for dep_file in config.dependencies_files:
+            self.assertTrue(dep_file in rendered_tox)

From 93f1b09566bccf23523a6859047d3e0693c38e5e Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Thu, 26 Aug 2021 16:41:17 -0400
Subject: [PATCH 03/12] Tests: simple repository test

---
 ecosystem/controllers/runner.py               | 20 ++++++-------
 tests/common.py                               |  2 +-
 .../python_repository/qlib/__init__.py        |  2 ++
 .../resources/python_repository/qlib/impl.py  | 18 +++++++++++
 .../python_repository/requirements.txt        |  1 +
 tests/resources/python_repository/setup.cfg   |  2 ++
 tests/resources/python_repository/setup.py    | 15 ++++++++++
 .../python_repository/tests/__init__.py       |  0
 .../python_repository/tests/test_impl.py      | 16 ++++++++++
 tests/test_runner.py                          | 30 +++++++++++++++++++
 10 files changed, 95 insertions(+), 11 deletions(-)
 create mode 100644 tests/resources/python_repository/qlib/__init__.py
 create mode 100644 tests/resources/python_repository/qlib/impl.py
 create mode 100644 tests/resources/python_repository/requirements.txt
 create mode 100644 tests/resources/python_repository/setup.cfg
 create mode 100644 tests/resources/python_repository/setup.py
 create mode 100644 tests/resources/python_repository/tests/__init__.py
 create mode 100644 tests/resources/python_repository/tests/test_impl.py
 create mode 100644 tests/test_runner.py

diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
index 8fc9c9c11e..751750df37 100644
--- a/ecosystem/controllers/runner.py
+++ b/ecosystem/controllers/runner.py
@@ -49,6 +49,15 @@ def run(self) -> Tuple[str, CommandExecutionSummary]:
         """Runs chain of commands to check repository."""
         result = {}
         self.set_up()
+
+        # clone repository
+        self.logger.info("Cloning repository.")
+        clone_res = _clone_repo(self.repo, directory=self.working_directory)
+
+        if not clone_res.ok:
+            raise QiskitEcosystemException(
+                f"Something went wrong with cloning {self.repo} repository.")
+
         try:
             result = self.workload()
         except Exception as exception:  # pylint: disable=broad-except)
@@ -82,15 +91,6 @@ def workload(self) -> Tuple[str, CommandExecutionSummary]:
 
         Returns: execution summary of steps
         """
-
-        # clone repository
-        self.logger.info("Cloning repository.")
-        clone_res = _clone_repo(self.repo, directory=self.working_directory)
-
-        if not clone_res.ok:
-            raise QiskitEcosystemException(
-                f"Something went wrong with cloning {self.repo} repository.")
-
         # check for configuration file
         if os.path.exists(f"{self.cloned_repo_directory}/qe_config.json"):
             loaded_config = RepositoryConfiguration.load(
@@ -119,7 +119,7 @@ def workload(self) -> Tuple[str, CommandExecutionSummary]:
         if os.path.exists(f"{self.cloned_repo_directory}/terra_version.txt"):
             with open(f"{self.cloned_repo_directory}/terra_version.txt", "r") as version_file:
                 terra_version = version_file.read()
-                self.logger.info(f"Terra version: {terra_version}")
+                self.logger.info("Terra version: %s", terra_version)
         else:
             self.logger.warning("There in no terra version file...")
 
diff --git a/tests/common.py b/tests/common.py
index a5ab584c83..63a2579e62 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -9,7 +9,7 @@ class TestCaseWithResources(unittest.TestCase):
     path: str
 
     def setUp(self) -> None:
-        self.path = "./resources"
+        self.path = "./resources/tests_tmp_data"
         if not os.path.exists(self.path):
             os.makedirs(self.path)
 
diff --git a/tests/resources/python_repository/qlib/__init__.py b/tests/resources/python_repository/qlib/__init__.py
new file mode 100644
index 0000000000..ceb3b68a52
--- /dev/null
+++ b/tests/resources/python_repository/qlib/__init__.py
@@ -0,0 +1,2 @@
+"""Docstring."""
+from .impl import Impl
diff --git a/tests/resources/python_repository/qlib/impl.py b/tests/resources/python_repository/qlib/impl.py
new file mode 100644
index 0000000000..11b9eaad49
--- /dev/null
+++ b/tests/resources/python_repository/qlib/impl.py
@@ -0,0 +1,18 @@
+"""Docstring."""
+import math
+from typing import Union
+
+
+class Impl:
+    """Demo impl."""
+    def __init__(self):
+        """Demo impl."""
+        self.pow = 2
+
+    def run(self, number: Union[int, float]) -> Union[int, float]:
+        """Run method."""
+        from collections import Hashable
+        return math.pow(number, self.pow)
+
+    def __repr__(self):
+        return f"Impl(pow: {self.pow})"
diff --git a/tests/resources/python_repository/requirements.txt b/tests/resources/python_repository/requirements.txt
new file mode 100644
index 0000000000..95ea1e6a02
--- /dev/null
+++ b/tests/resources/python_repository/requirements.txt
@@ -0,0 +1 @@
+pytest==6.2.4
diff --git a/tests/resources/python_repository/setup.cfg b/tests/resources/python_repository/setup.cfg
new file mode 100644
index 0000000000..b88034e414
--- /dev/null
+++ b/tests/resources/python_repository/setup.cfg
@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md
diff --git a/tests/resources/python_repository/setup.py b/tests/resources/python_repository/setup.py
new file mode 100644
index 0000000000..203a2d4621
--- /dev/null
+++ b/tests/resources/python_repository/setup.py
@@ -0,0 +1,15 @@
+"""Setup file for demo-impl."""
+
+import setuptools
+
+with open('requirements.txt') as fp:
+    install_requires = fp.read()
+
+setuptools.setup(
+    name="demo-impl",
+    description="demo-impl",
+    long_description="",
+    packages=setuptools.find_packages(),
+    install_requires=install_requires,
+    python_requires='>=3.6'
+)
diff --git a/tests/resources/python_repository/tests/__init__.py b/tests/resources/python_repository/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/resources/python_repository/tests/test_impl.py b/tests/resources/python_repository/tests/test_impl.py
new file mode 100644
index 0000000000..c753c27d8c
--- /dev/null
+++ b/tests/resources/python_repository/tests/test_impl.py
@@ -0,0 +1,16 @@
+"""Docstring."""
+import unittest
+import warnings
+
+from qlib import Impl
+
+
+class TestImpl(unittest.TestCase):
+    """Tests Impl class implementation."""
+
+    def test_run(self):
+        """Tests run method implementation."""
+        impl = Impl()
+
+        warnings.warn("test warning", DeprecationWarning)
+        self.assertEqual(impl.run(2), 4)
diff --git a/tests/test_runner.py b/tests/test_runner.py
new file mode 100644
index 0000000000..cd6accae8a
--- /dev/null
+++ b/tests/test_runner.py
@@ -0,0 +1,30 @@
+"""Tests for runners."""
+import os
+import unittest
+
+from ecosystem.controllers.runner import PythonRunner
+
+
+class TestPythonRunner(unittest.TestCase):
+    """Tests for Python runner."""
+    def setUp(self) -> None:
+        current_directory = os.path.dirname(os.path.abspath(__file__))
+        self.project_dir = f"{current_directory}/resources/python_repository"
+
+    def tearDown(self) -> None:
+        files_to_delete = ["tox.ini", "terra_version.txt"]
+        for file in files_to_delete:
+            if os.path.exists(f"{self.project_dir}/{file}"):
+                os.remove(f"{self.project_dir}/{file}")
+
+    def test_runner(self):
+        """Simple runner test."""
+        runner = PythonRunner("test",
+                              working_directory=self.project_dir,
+                              ecosystem_deps=["qiskit"])
+
+        runner.cloned_repo_directory = self.project_dir
+        terra_version, result = runner.workload()
+
+        self.assertFalse(result.ok)
+        self.assertTrue(terra_version)

From 64b694cfe87ebd7b8a9fa4507ea7ef9ef9ec71ba Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Thu, 26 Aug 2021 17:32:43 -0400
Subject: [PATCH 04/12] Tests: configured repository test

---
 ecosystem/controllers/runner.py               |  3 +-
 ecosystem/manager.py                          |  6 ++--
 ecosystem/models/configuration.py             | 29 +++++++++++------
 .../qe_config.json                            | 13 ++++++++
 .../qlib/__init__.py                          |  0
 .../qlib/impl.py                              |  0
 .../requirements-dev.txt}                     |  0
 .../requirements.txt                          |  1 +
 .../setup.cfg                                 |  0
 .../setup.py                                  |  0
 .../tests/__init__.py                         |  0
 .../tests/test_impl.py                        |  0
 .../simple_python_repository/qlib/__init__.py |  2 ++
 .../simple_python_repository/qlib/impl.py     | 18 +++++++++++
 .../simple_python_repository/requirements.txt |  1 +
 .../simple_python_repository/setup.cfg        |  2 ++
 .../simple_python_repository/setup.py         | 15 +++++++++
 .../tests/__init__.py                         |  0
 .../tests/test_impl.py                        | 16 ++++++++++
 tests/test_runner.py                          | 32 +++++++++++++++----
 20 files changed, 119 insertions(+), 19 deletions(-)
 create mode 100644 tests/resources/configured_python_repository/qe_config.json
 rename tests/resources/{python_repository => configured_python_repository}/qlib/__init__.py (100%)
 rename tests/resources/{python_repository => configured_python_repository}/qlib/impl.py (100%)
 rename tests/resources/{python_repository/requirements.txt => configured_python_repository/requirements-dev.txt} (100%)
 create mode 100644 tests/resources/configured_python_repository/requirements.txt
 rename tests/resources/{python_repository => configured_python_repository}/setup.cfg (100%)
 rename tests/resources/{python_repository => configured_python_repository}/setup.py (100%)
 rename tests/resources/{python_repository => configured_python_repository}/tests/__init__.py (100%)
 rename tests/resources/{python_repository => configured_python_repository}/tests/test_impl.py (100%)
 create mode 100644 tests/resources/simple_python_repository/qlib/__init__.py
 create mode 100644 tests/resources/simple_python_repository/qlib/impl.py
 create mode 100644 tests/resources/simple_python_repository/requirements.txt
 create mode 100644 tests/resources/simple_python_repository/setup.cfg
 create mode 100644 tests/resources/simple_python_repository/setup.py
 create mode 100644 tests/resources/simple_python_repository/tests/__init__.py
 create mode 100644 tests/resources/simple_python_repository/tests/test_impl.py

diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
index 751750df37..28c9b4d971 100644
--- a/ecosystem/controllers/runner.py
+++ b/ecosystem/controllers/runner.py
@@ -77,7 +77,7 @@ def __init__(self,
         super().__init__(repo=repo,
                          working_directory=working_directory)
         self.python_version = python_version
-        self.ecosystem_deps = ecosystem_deps or []
+        self.ecosystem_deps = ecosystem_deps or ["qiskit"]
 
     def workload(self) -> Tuple[str, CommandExecutionSummary]:
         """Runs checks for python repository.
@@ -93,6 +93,7 @@ def workload(self) -> Tuple[str, CommandExecutionSummary]:
         """
         # check for configuration file
         if os.path.exists(f"{self.cloned_repo_directory}/qe_config.json"):
+            self.logger.info("Configuration file exists.")
             loaded_config = RepositoryConfiguration.load(
                 f"{self.cloned_repo_directory}/qe_config.json")
             repo_config = cast(PythonRepositoryConfiguration, loaded_config)
diff --git a/ecosystem/manager.py b/ecosystem/manager.py
index c7e05de57c..8c87edea12 100644
--- a/ecosystem/manager.py
+++ b/ecosystem/manager.py
@@ -116,10 +116,12 @@ def dev_compatibility_tests(self,
                          tox_python=tox_python,
                          dependencies=["git+https://github.com/Qiskit/qiskit-terra.git@main"])
 
-    def stable_tests(self, repo_url: str):
+    def stable_tests(self, repo_url: str,
+                     python_version: str = "py39"):
         """Runs tests against stable version of qiskit."""
         runner = PythonRunner(repo_url,
                               working_directory=self.resources_dir,
-                              ecosystem_deps=["qiskit"])
+                              ecosystem_deps=["qiskit"],
+                              python_version=python_version)
         terra_version, _ = runner.run()
         return terra_version
diff --git a/ecosystem/models/configuration.py b/ecosystem/models/configuration.py
index aceb18aecd..97ac70807e 100644
--- a/ecosystem/models/configuration.py
+++ b/ecosystem/models/configuration.py
@@ -6,6 +6,7 @@
 from jinja2 import Environment, PackageLoader, select_autoescape
 
 from ecosystem.entities import JsonSerializable
+from ecosystem.utils import QiskitEcosystemException
 
 
 class Languages:
@@ -28,7 +29,7 @@ def __init__(self,
                  dependencies_files: Optional[List[str]] = None,
                  extra_dependencies: Optional[List[str]] = None,
                  tests_command: Optional[List[str]] = None,
-                 styles_check_command: Optional[List[str]] = None,):
+                 styles_check_command: Optional[List[str]] = None, ):
         """Configuration for ecosystem repository.
 
         Args:
@@ -57,8 +58,17 @@ def save(self, path: str):
     def load(cls, path: str) -> 'RepositoryConfiguration':
         """Loads json file into object."""
         with open(path, "r") as json_file:
-            return json.load(json_file,
-                             object_hook=lambda d: RepositoryConfiguration(**d))
+            config: RepositoryConfiguration = json.load(
+                json_file, object_hook=lambda d: RepositoryConfiguration(**d))
+            if config.language == Languages.PYTHON:
+                return PythonRepositoryConfiguration(
+                    dependencies_files=config.dependencies_files,
+                    extra_dependencies=config.extra_dependencies,
+                    tests_command=config.tests_command,
+                    styles_check_command=config.styles_check_command)
+            else:
+                raise QiskitEcosystemException("Unsupported language configuration type: %s",
+                                               config.language)
 
     def __repr__(self):
         return pprint.pformat(self.to_dict(), indent=4)
@@ -66,6 +76,7 @@ def __repr__(self):
 
 class PythonRepositoryConfiguration(RepositoryConfiguration):
     """Repository configuration for python based projects."""
+
     def __init__(self,
                  dependencies_files: Optional[List[str]] = None,
                  extra_dependencies: Optional[List[str]] = None,
@@ -86,12 +97,12 @@ def __init__(self,
     def default(cls) -> 'PythonRepositoryConfiguration':
         """Returns default python repository configuration."""
         return PythonRepositoryConfiguration(dependencies_files=[
-                                                 "requirements.txt"
-                                             ],
-                                             tests_command=[
-                                                 "pip check",
-                                                 "pytest -W error::DeprecationWarning"
-                                             ])
+            "requirements.txt"
+        ],
+            tests_command=[
+                "pip check",
+                "pytest -W error::DeprecationWarning"
+            ])
 
     def render_tox_file(self, ecosystem_deps: List[str] = None):
         """Renders tox template from configuration."""
diff --git a/tests/resources/configured_python_repository/qe_config.json b/tests/resources/configured_python_repository/qe_config.json
new file mode 100644
index 0000000000..e0b9a278fb
--- /dev/null
+++ b/tests/resources/configured_python_repository/qe_config.json
@@ -0,0 +1,13 @@
+{
+    "dependencies_files": [
+        "requirements.txt",
+        "requirements-dev.txt"
+    ],
+    "extra_dependencies": [
+        "pytest"
+    ],
+    "language": "python",
+    "tests_command": [
+        "pytest"
+    ]
+}
\ No newline at end of file
diff --git a/tests/resources/python_repository/qlib/__init__.py b/tests/resources/configured_python_repository/qlib/__init__.py
similarity index 100%
rename from tests/resources/python_repository/qlib/__init__.py
rename to tests/resources/configured_python_repository/qlib/__init__.py
diff --git a/tests/resources/python_repository/qlib/impl.py b/tests/resources/configured_python_repository/qlib/impl.py
similarity index 100%
rename from tests/resources/python_repository/qlib/impl.py
rename to tests/resources/configured_python_repository/qlib/impl.py
diff --git a/tests/resources/python_repository/requirements.txt b/tests/resources/configured_python_repository/requirements-dev.txt
similarity index 100%
rename from tests/resources/python_repository/requirements.txt
rename to tests/resources/configured_python_repository/requirements-dev.txt
diff --git a/tests/resources/configured_python_repository/requirements.txt b/tests/resources/configured_python_repository/requirements.txt
new file mode 100644
index 0000000000..95ea1e6a02
--- /dev/null
+++ b/tests/resources/configured_python_repository/requirements.txt
@@ -0,0 +1 @@
+pytest==6.2.4
diff --git a/tests/resources/python_repository/setup.cfg b/tests/resources/configured_python_repository/setup.cfg
similarity index 100%
rename from tests/resources/python_repository/setup.cfg
rename to tests/resources/configured_python_repository/setup.cfg
diff --git a/tests/resources/python_repository/setup.py b/tests/resources/configured_python_repository/setup.py
similarity index 100%
rename from tests/resources/python_repository/setup.py
rename to tests/resources/configured_python_repository/setup.py
diff --git a/tests/resources/python_repository/tests/__init__.py b/tests/resources/configured_python_repository/tests/__init__.py
similarity index 100%
rename from tests/resources/python_repository/tests/__init__.py
rename to tests/resources/configured_python_repository/tests/__init__.py
diff --git a/tests/resources/python_repository/tests/test_impl.py b/tests/resources/configured_python_repository/tests/test_impl.py
similarity index 100%
rename from tests/resources/python_repository/tests/test_impl.py
rename to tests/resources/configured_python_repository/tests/test_impl.py
diff --git a/tests/resources/simple_python_repository/qlib/__init__.py b/tests/resources/simple_python_repository/qlib/__init__.py
new file mode 100644
index 0000000000..ceb3b68a52
--- /dev/null
+++ b/tests/resources/simple_python_repository/qlib/__init__.py
@@ -0,0 +1,2 @@
+"""Docstring."""
+from .impl import Impl
diff --git a/tests/resources/simple_python_repository/qlib/impl.py b/tests/resources/simple_python_repository/qlib/impl.py
new file mode 100644
index 0000000000..11b9eaad49
--- /dev/null
+++ b/tests/resources/simple_python_repository/qlib/impl.py
@@ -0,0 +1,18 @@
+"""Docstring."""
+import math
+from typing import Union
+
+
+class Impl:
+    """Demo impl."""
+    def __init__(self):
+        """Demo impl."""
+        self.pow = 2
+
+    def run(self, number: Union[int, float]) -> Union[int, float]:
+        """Run method."""
+        from collections import Hashable
+        return math.pow(number, self.pow)
+
+    def __repr__(self):
+        return f"Impl(pow: {self.pow})"
diff --git a/tests/resources/simple_python_repository/requirements.txt b/tests/resources/simple_python_repository/requirements.txt
new file mode 100644
index 0000000000..95ea1e6a02
--- /dev/null
+++ b/tests/resources/simple_python_repository/requirements.txt
@@ -0,0 +1 @@
+pytest==6.2.4
diff --git a/tests/resources/simple_python_repository/setup.cfg b/tests/resources/simple_python_repository/setup.cfg
new file mode 100644
index 0000000000..b88034e414
--- /dev/null
+++ b/tests/resources/simple_python_repository/setup.cfg
@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md
diff --git a/tests/resources/simple_python_repository/setup.py b/tests/resources/simple_python_repository/setup.py
new file mode 100644
index 0000000000..203a2d4621
--- /dev/null
+++ b/tests/resources/simple_python_repository/setup.py
@@ -0,0 +1,15 @@
+"""Setup file for demo-impl."""
+
+import setuptools
+
+with open('requirements.txt') as fp:
+    install_requires = fp.read()
+
+setuptools.setup(
+    name="demo-impl",
+    description="demo-impl",
+    long_description="",
+    packages=setuptools.find_packages(),
+    install_requires=install_requires,
+    python_requires='>=3.6'
+)
diff --git a/tests/resources/simple_python_repository/tests/__init__.py b/tests/resources/simple_python_repository/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/resources/simple_python_repository/tests/test_impl.py b/tests/resources/simple_python_repository/tests/test_impl.py
new file mode 100644
index 0000000000..c753c27d8c
--- /dev/null
+++ b/tests/resources/simple_python_repository/tests/test_impl.py
@@ -0,0 +1,16 @@
+"""Docstring."""
+import unittest
+import warnings
+
+from qlib import Impl
+
+
+class TestImpl(unittest.TestCase):
+    """Tests Impl class implementation."""
+
+    def test_run(self):
+        """Tests run method implementation."""
+        impl = Impl()
+
+        warnings.warn("test warning", DeprecationWarning)
+        self.assertEqual(impl.run(2), 4)
diff --git a/tests/test_runner.py b/tests/test_runner.py
index cd6accae8a..5d64c5212f 100644
--- a/tests/test_runner.py
+++ b/tests/test_runner.py
@@ -7,24 +7,42 @@
 
 class TestPythonRunner(unittest.TestCase):
     """Tests for Python runner."""
+
     def setUp(self) -> None:
         current_directory = os.path.dirname(os.path.abspath(__file__))
-        self.project_dir = f"{current_directory}/resources/python_repository"
+        self.simple_project_dir = f"{current_directory}/" \
+                                  f"resources/simple_python_repository"
+        self.configured_project_dir = f"{current_directory}/" \
+                                      f"resources/configured_python_repository"
 
     def tearDown(self) -> None:
         files_to_delete = ["tox.ini", "terra_version.txt"]
-        for file in files_to_delete:
-            if os.path.exists(f"{self.project_dir}/{file}"):
-                os.remove(f"{self.project_dir}/{file}")
+        for directory in [self.simple_project_dir,
+                          self.configured_project_dir]:
+            for file in files_to_delete:
+                if os.path.exists(f"{directory}/{file}"):
+                    os.remove(f"{directory}/{file}")
 
-    def test_runner(self):
+    def test_runner_on_simple_repo(self):
         """Simple runner test."""
         runner = PythonRunner("test",
-                              working_directory=self.project_dir,
+                              working_directory=self.simple_project_dir,
                               ecosystem_deps=["qiskit"])
 
-        runner.cloned_repo_directory = self.project_dir
+        runner.cloned_repo_directory = self.simple_project_dir
         terra_version, result = runner.workload()
 
         self.assertFalse(result.ok)
         self.assertTrue(terra_version)
+
+    def test_runner_on_configured_repo(self):
+        """Configured repo runner test."""
+        runner = PythonRunner("test",
+                              working_directory=self.configured_project_dir,
+                              ecosystem_deps=["qiskit"])
+
+        runner.cloned_repo_directory = self.configured_project_dir
+        terra_version, result = runner.workload()
+
+        self.assertTrue(result.ok)
+        self.assertTrue(terra_version)

From 2004ab46df4a08fc743f2061bbbf875a4b16a88e Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Thu, 26 Aug 2021 17:39:29 -0400
Subject: [PATCH 05/12] Tests: fix linter

---
 ecosystem/controllers/runner.py   | 1 -
 ecosystem/models/configuration.py | 6 +++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
index 28c9b4d971..b06b83da16 100644
--- a/ecosystem/controllers/runner.py
+++ b/ecosystem/controllers/runner.py
@@ -101,7 +101,6 @@ def workload(self) -> Tuple[str, CommandExecutionSummary]:
             repo_config = PythonRepositoryConfiguration.default()
 
         # check for existing tox file
-        self.logger.info(f"Creating tox file {self.cloned_repo_directory}/tox.ini")
         if os.path.exists(f"{self.cloned_repo_directory}/tox.ini"):
             self.logger.info("Tox file exists.")
             os.rename(f"{self.cloned_repo_directory}/tox.ini",
diff --git a/ecosystem/models/configuration.py b/ecosystem/models/configuration.py
index 97ac70807e..46b572eb4e 100644
--- a/ecosystem/models/configuration.py
+++ b/ecosystem/models/configuration.py
@@ -60,15 +60,15 @@ def load(cls, path: str) -> 'RepositoryConfiguration':
         with open(path, "r") as json_file:
             config: RepositoryConfiguration = json.load(
                 json_file, object_hook=lambda d: RepositoryConfiguration(**d))
-            if config.language == Languages.PYTHON:
+            if config.language == Languages.PYTHON:  # pylint: disable=no-else-return
                 return PythonRepositoryConfiguration(
                     dependencies_files=config.dependencies_files,
                     extra_dependencies=config.extra_dependencies,
                     tests_command=config.tests_command,
                     styles_check_command=config.styles_check_command)
             else:
-                raise QiskitEcosystemException("Unsupported language configuration type: %s",
-                                               config.language)
+                raise QiskitEcosystemException(
+                    f"Unsupported language configuration type: {config.language}")
 
     def __repr__(self):
         return pprint.pformat(self.to_dict(), indent=4)

From 7f8153511dd6cb131806beef4631eee298d330af Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Mon, 30 Aug 2021 12:03:34 -0400
Subject: [PATCH 06/12] Configuration: python config + save results to db

---
 ecosystem/commands.py             |  10 ++-
 ecosystem/controller.py           |  61 +++++++----------
 ecosystem/controllers/runner.py   |  20 +++---
 ecosystem/entities.py             | 105 ++++++++++++++++++++++--------
 ecosystem/manager.py              | 100 ++++++----------------------
 ecosystem/models/configuration.py |   7 +-
 tests/test_controller.py          |  90 ++++++++++---------------
 tests/test_entities.py            |  24 +++++++
 tests/test_runner.py              |   4 +-
 9 files changed, 206 insertions(+), 215 deletions(-)
 create mode 100644 tests/test_entities.py

diff --git a/ecosystem/commands.py b/ecosystem/commands.py
index ff0cf757d6..c3530e0d6a 100644
--- a/ecosystem/commands.py
+++ b/ecosystem/commands.py
@@ -12,10 +12,12 @@
 
 
 def _execute_command(command: List[str],
-                     cwd: Optional[str] = None) -> CommandExecutionSummary:
+                     cwd: Optional[str] = None,
+                     name: Optional[str] = None) -> CommandExecutionSummary:
     """Executes specified command as subprocess in a directory."""
     with subprocess.Popen(command,
                           stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE,
                           cwd=cwd) as process:
         logs = []
         while True:
@@ -30,7 +32,8 @@ def _execute_command(command: List[str],
                     logs.append(str(output).strip())
 
                 return CommandExecutionSummary(code=return_code,
-                                               logs=logs)
+                                               logs=logs,
+                                               name=name)
 
 
 def _clone_repo(repo: str, directory: str) -> CommandExecutionSummary:
@@ -51,7 +54,8 @@ def _run_tox(directory: str, env: str) -> CommandExecutionSummary:
     """Run tox test."""
     return _execute_command(["tox", "-e{}".format(env),
                              "--workdir", directory],
-                            cwd=directory)
+                            cwd=directory,
+                            name="tox")
 
 
 def _cleanup(directory: Optional[str] = None):
diff --git a/ecosystem/controller.py b/ecosystem/controller.py
index 31b53cb8c1..6241dc157a 100644
--- a/ecosystem/controller.py
+++ b/ecosystem/controller.py
@@ -1,9 +1,9 @@
 """Entrypoint for CLI."""
 from typing import Optional, List
 
-from tinydb import TinyDB, Query, where
+from tinydb import TinyDB, Query
 
-from .entities import Repository, MainRepository, Tier
+from .entities import Repository, Tier, TestResult
 
 
 class Controller:
@@ -23,52 +23,35 @@ def insert(self, repo: Repository) -> int:
         table = self.database.table(repo.tier)
         return table.insert(repo.to_dict())
 
-    def get_all_main(self) -> List[MainRepository]:
+    def get_all_main(self) -> List[Repository]:
         """Returns all repositories from database."""
         table = self.database.table(Tier.MAIN)
-        return [MainRepository(**r) for r in table.all()]
+        return [Repository.from_dict(r) for r in table.all()]
 
     def get_by_url(self, url: str, tier: str) -> Optional[Repository]:
         """Returns repository by URL."""
         res = self.database.table(tier).get(Query().url == url)
-        return MainRepository(**res) if res else None
+        return Repository.from_dict(res) if res else None
 
-    def update_repo_tests_passed(self, repo: Repository,
-                                 tests_passed: List[str]) -> List[int]:
-        """Updates repository passed tests."""
-        table = self.database.table(repo.tier)
-        return table.update({"tests_passed": tests_passed},
-                            where('name') == repo.name)
-
-    def add_repo_test_passed(self,
-                             repo_url: str,
-                             test_passed: str,
-                             tier: str):
-        """Adds passed test if is not there yet."""
+    def add_repo_test_result(self, repo_url: str,
+                             tier: str,
+                             test_result: TestResult) -> Optional[List[int]]:
+        """Adds test result for repository."""
         table = self.database.table(tier)
-        repo = self.get_by_url(repo_url, tier)
-        if repo:
-            tests_passed = repo.tests_passed
-            if test_passed not in tests_passed:
-                tests_passed.append(test_passed)
-                return table.update({"tests_passed": tests_passed},
-                                    where('name') == repo.name)
-        return [0]
+        repository = Query()
 
-    def remove_repo_test_passed(self,
-                                repo_url: str,
-                                test_remove: str,
-                                tier: str):
-        """Remove passed tests."""
-        table = self.database.table(tier)
-        repo = self.get_by_url(repo_url, tier)
-        if repo:
-            tests_passed = repo.tests_passed
-            if test_remove in tests_passed:
-                tests_passed.remove(test_remove)
-                return table.update({"tests_passed": tests_passed},
-                                    where('name') == repo.name)
-        return [0]
+        fetched_repo_json = table.get(repository.url == repo_url)
+        if fetched_repo_json is not None:
+            fetched_repo = Repository.from_dict(fetched_repo_json)
+            fetched_test_results = fetched_repo.tests_results
+
+            new_test_results = [tr for tr in fetched_test_results
+                                if tr.test_type != test_result.test_type or
+                                tr.terra_version != test_result.terra_version] + [test_result]
+            fetched_repo.tests_results = new_test_results
+
+            return table.upsert(fetched_repo.to_dict(), repository.url == repo_url)
+        return None
 
     def delete(self, repo: Repository) -> List[int]:
         """Deletes entry."""
diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
index b06b83da16..676e648ce1 100644
--- a/ecosystem/controllers/runner.py
+++ b/ecosystem/controllers/runner.py
@@ -39,17 +39,15 @@ def tear_down(self):
             shutil.rmtree(self.cloned_repo_directory)
 
     @abstractmethod
-    def workload(self) -> Tuple[str, CommandExecutionSummary]:
+    def workload(self) -> Tuple[str, List[CommandExecutionSummary]]:
         """Runs workload of commands to check repository.
 
         Returns: tuple (qiskit_version, CommandExecutionSummary)
         """
 
-    def run(self) -> Tuple[str, CommandExecutionSummary]:
+    def run(self) -> Tuple[str, List[CommandExecutionSummary]]:
         """Runs chain of commands to check repository."""
-        result = {}
         self.set_up()
-
         # clone repository
         self.logger.info("Cloning repository.")
         clone_res = _clone_repo(self.repo, directory=self.working_directory)
@@ -61,6 +59,7 @@ def run(self) -> Tuple[str, CommandExecutionSummary]:
         try:
             result = self.workload()
         except Exception as exception:  # pylint: disable=broad-except)
+            result = ("-", CommandExecutionSummary(1, [], summary=str(exception)))
             self.logger.error(exception)
         self.tear_down()
         return result
@@ -73,13 +72,15 @@ def __init__(self,
                  repo: Union[str, Repository],
                  working_directory: Optional[str] = None,
                  ecosystem_deps: Optional[List[str]] = None,
-                 python_version: str = "py39"):
+                 python_version: str = "py39",
+                 repo_config: Optional[RepositoryConfiguration] = None):
         super().__init__(repo=repo,
                          working_directory=working_directory)
         self.python_version = python_version
         self.ecosystem_deps = ecosystem_deps or ["qiskit"]
+        self.repo_config = repo_config
 
-    def workload(self) -> Tuple[str, CommandExecutionSummary]:
+    def workload(self) -> Tuple[str, List[CommandExecutionSummary]]:
         """Runs checks for python repository.
 
         Steps:
@@ -92,7 +93,9 @@ def workload(self) -> Tuple[str, CommandExecutionSummary]:
         Returns: execution summary of steps
         """
         # check for configuration file
-        if os.path.exists(f"{self.cloned_repo_directory}/qe_config.json"):
+        if self.repo_config is not None:
+            repo_config = self.repo_config
+        elif os.path.exists(f"{self.cloned_repo_directory}/qe_config.json"):
             self.logger.info("Configuration file exists.")
             loaded_config = RepositoryConfiguration.load(
                 f"{self.cloned_repo_directory}/qe_config.json")
@@ -116,6 +119,7 @@ def workload(self) -> Tuple[str, CommandExecutionSummary]:
                                  env=self.python_version)
 
         # get terra version from file
+        terra_version = "-"
         if os.path.exists(f"{self.cloned_repo_directory}/terra_version.txt"):
             with open(f"{self.cloned_repo_directory}/terra_version.txt", "r") as version_file:
                 terra_version = version_file.read()
@@ -123,4 +127,4 @@ def workload(self) -> Tuple[str, CommandExecutionSummary]:
         else:
             self.logger.warning("There in no terra version file...")
 
-        return terra_version, tox_tests_res
+        return terra_version, [tox_tests_res]
diff --git a/ecosystem/entities.py b/ecosystem/entities.py
index 6615a01d38..95e4b4178c 100644
--- a/ecosystem/entities.py
+++ b/ecosystem/entities.py
@@ -1,8 +1,7 @@
 """Classes and controllers for json storage."""
-import inspect
+import pprint
 from abc import ABC
 from datetime import datetime
-import pprint
 from typing import Optional, List
 
 
@@ -34,22 +33,67 @@ def all(cls):
 
 class JsonSerializable(ABC):
     """Classes that can be serialized as json."""
+
+    @classmethod
+    def from_dict(cls, dictionary: dict):
+        """Converts dict to object."""
+
     def to_dict(self) -> dict:
         """Converts repo to dict."""
-        result = dict()
-        for name, value in inspect.getmembers(self):
-            if not name.startswith('_') and \
-                    not inspect.ismethod(value) and not inspect.isfunction(value) and \
-                    hasattr(self, name):
-                result[name] = value
+        result = {}
+        for key, val in self.__dict__.items():
+            if key.startswith("_"):
+                continue
+            element = []
+            if isinstance(val, list):
+                for item in val:
+                    if isinstance(item, JsonSerializable):
+                        element.append(item.to_dict())
+                    else:
+                        element.append(item)
+            elif isinstance(val, JsonSerializable):
+                element = val.to_dict()
+            else:
+                element = val
+            result[key] = element
         return result
 
 
+class TestResult(JsonSerializable):
+    """Tests status."""
+    _TEST_PASSED: str = "passed"
+    _TEST_FAILED: str = "failed"
+
+    def __init__(self,
+                 passed: bool,
+                 terra_version: str,
+                 test_type: str):
+        self.test_type = test_type
+        self.passed = passed
+        self.terra_version = terra_version
+
+    @classmethod
+    def from_dict(cls, dictionary: dict):
+        return TestResult(passed=dictionary.get("passed"),
+                          terra_version=dictionary.get("terra_version"),
+                          test_type=dictionary.get("test_type"))
+
+    def to_string(self) -> str:
+        """Test result as string."""
+        return self._TEST_PASSED if self.passed else self._TEST_FAILED
+
+    def __eq__(self, other: 'TestResult'):
+        return self.passed == other.passed \
+               and self.test_type == other.test_type \
+               and self.terra_version == other.terra_version
+
+    def __repr__(self):
+        return f"TestResult({self.passed}, {self.test_type}, {self.terra_version})"
+
+
 class Repository(JsonSerializable):
     """Main repository class."""
 
-    tier: str
-
     def __init__(self,
                  name: str,
                  url: str,
@@ -60,9 +104,8 @@ def __init__(self,
                  labels: Optional[List[str]] = None,
                  created_at: Optional[int] = None,
                  updated_at: Optional[int] = None,
-                 tier: Optional[str] = None,
-                 tests_to_run: List[str] = None,
-                 tests_passed: List[str] = None):
+                 tier: str = Tier.MAIN,
+                 tests_results: Optional[List[TestResult]] = None):
         """Repository controller.
 
         Args:
@@ -75,8 +118,7 @@ def __init__(self,
             labels: labels
             created_at: creation date
             updated_at: update date
-            tests_to_run: tests need to be executed against repo
-            tests_passed: tests passed by repo
+            tests_results: tests passed by repo
         """
         self.name = name
         self.url = url
@@ -87,10 +129,24 @@ def __init__(self,
         self.labels = labels if labels is not None else []
         self.created_at = created_at if created_at is not None else datetime.now().timestamp()
         self.updated_at = updated_at if updated_at is not None else datetime.now().timestamp()
-        self.tests_to_run = tests_to_run if tests_to_run else []
-        self.tests_passed = tests_passed if tests_passed else []
-        if tier:
-            self.tier = tier
+        self.tests_results = tests_results if tests_results else []
+        self.tier = tier
+
+    @classmethod
+    def from_dict(cls, dictionary: dict):
+        tests_results = []
+        if "tests_results" in dictionary:
+            tests_results = [TestResult.from_dict(r) for r in dictionary.get("tests_results", [])]
+
+        return Repository(name=dictionary.get("name"),
+                          url=dictionary.get("url"),
+                          description=dictionary.get("description"),
+                          licence=dictionary.get("licence"),
+                          contact_info=dictionary.get("contact_info"),
+                          alternatives=dictionary.get("alternatives"),
+                          labels=dictionary.get("labels"),
+                          tier=dictionary.get("tier"),
+                          tests_results=tests_results)
 
     def __eq__(self, other: 'Repository'):
         return (self.tier == other.tier
@@ -105,19 +161,16 @@ def __str__(self):
         return f"Repository({self.tier} | {self.name} | {self.url})"
 
 
-class MainRepository(Repository):
-    """Main tier repository."""
-    tier = Tier.MAIN
-
-
 class CommandExecutionSummary:
     """Utils for command execution results."""
 
     def __init__(self,
                  code: int,
                  logs: List[str],
-                 summary: Optional[str] = None):
+                 summary: Optional[str] = None,
+                 name: Optional[str] = None):
         """CommandExecutionSummary class."""
+        self.name = name or ""
         self.code = code
         self.logs = logs
         if summary:
@@ -142,4 +195,4 @@ def empty(cls) -> 'CommandExecutionSummary':
         return cls(0, [])
 
     def __repr__(self):
-        return f"CommandExecutionSummary(code: {self.code} | {self.summary})"
+        return f"CommandExecutionSummary({self.name} | code: {self.code} | {self.summary})"
diff --git a/ecosystem/manager.py b/ecosystem/manager.py
index 8c87edea12..4d625b6cd2 100644
--- a/ecosystem/manager.py
+++ b/ecosystem/manager.py
@@ -1,13 +1,12 @@
 """Manager class for controlling all CLI functions."""
 import os
-from typing import Optional, List
+from typing import Optional
 
 from jinja2 import Environment, PackageLoader, select_autoescape
 
 from .controller import Controller
 from .controllers.runner import PythonRunner
-from .entities import Tier, TestType
-from .commands import run_tests
+from .entities import Tier, TestType, TestResult
 from .utils import logger
 
 
@@ -28,6 +27,7 @@ def __init__(self):
         self.readme_template = self.env.get_template("readme.md")
         self.tox_template = self.env.get_template("tox.ini")
         self.controller = Controller(path=self.resources_dir)
+        self.logger = logger
 
     def generate_readme(self, path: Optional[str] = None):
         """Generates entire readme for ecosystem repository.
@@ -41,87 +41,27 @@ def generate_readme(self, path: Optional[str] = None):
         with open(f"{path}/README.md", "w") as file:
             file.write(readme_content)
 
-    def _run(self,
-             repo_url: str,
-             tier: str,
-             test_type: str,
-             tox_python: str,
-             dependencies: Optional[List[str]] = None):
-        """Run tests on repository.
-
-        Args:
-            repo_url: repository url
-            tier: tier of membership
-            tox_python: tox env to run tests on
-            dependencies: list of extra dependencies to install
-        """
-        try:
-            if dependencies is not None:
-                dev_tests_results = run_tests(
-                    repo_url,
-                    resources_dir=self.resources_dir,
-                    tox_python=tox_python,
-                    template_and_deps=(self.tox_template, dependencies))
-            else:
-                dev_tests_results = run_tests(
-                    repo_url,
-                    resources_dir=self.resources_dir,
-                    tox_python=tox_python)
-            # if all steps of test are successful
-            if all(c.ok for c in dev_tests_results.values()):
-                # update repo entry and assign successful tests
-                self.controller.add_repo_test_passed(repo_url=repo_url,
-                                                     test_passed=test_type,
-                                                     tier=tier)
-            else:
-                logger.warning("Some commands failed. Check logs.")
-                self.controller.remove_repo_test_passed(repo_url=repo_url,
-                                                        test_remove=test_type,
-                                                        tier=tier)
-        except Exception as exception:  # pylint: disable=broad-except)
-            logger.error("Exception: %s", exception)
-            # remove from passed tests if anything went wrong
-            self.controller.remove_repo_test_passed(repo_url=repo_url,
-                                                    test_remove=test_type,
-                                                    tier=tier)
-
-    def standard_tests(self, repo_url: str,
-                       tier: str = Tier.MAIN,
-                       tox_python: str = "py39"):
-        """Perform general checks for repository."""
-        return self._run(repo_url=repo_url,
-                         tier=tier,
-                         test_type=TestType.STANDARD,
-                         tox_python=tox_python)
-
-    def stable_compatibility_tests(self,
-                                   repo_url: str,
-                                   tier: str = Tier.MAIN,
-                                   tox_python: str = "py39"):
-        """Runs tests against stable version of Qiskit."""
-        return self._run(repo_url=repo_url,
-                         tier=tier,
-                         test_type=TestType.STABLE_COMPATIBLE,
-                         tox_python=tox_python,
-                         dependencies=["qiskit"])
-
-    def dev_compatibility_tests(self,
-                                repo_url: str,
-                                tier: str = Tier.MAIN,
-                                tox_python: str = "py39"):
-        """Runs tests against dev version of Qiskit (main branch)."""
-        return self._run(repo_url=repo_url,
-                         tier=tier,
-                         test_type=TestType.DEV_COMPATIBLE,
-                         tox_python=tox_python,
-                         dependencies=["git+https://github.com/Qiskit/qiskit-terra.git@main"])
-
     def stable_tests(self, repo_url: str,
-                     python_version: str = "py39"):
+                     tier: str = Tier.MAIN,
+                     python_version: str = "py39",):
         """Runs tests against stable version of qiskit."""
         runner = PythonRunner(repo_url,
                               working_directory=self.resources_dir,
                               ecosystem_deps=["qiskit"],
                               python_version=python_version)
-        terra_version, _ = runner.run()
+        terra_version, results = runner.run()
+
+        test_result = TestResult(passed=all(r.ok for r in results),
+                                 terra_version=terra_version,
+                                 test_type=TestType.STABLE_COMPATIBLE)
+        # save test res to db
+        result = self.controller.add_repo_test_result(repo_url=repo_url,
+                                                      tier=tier,
+                                                      test_result=test_result)
+        # print report
+        if result is None:
+            self.logger.warning("Test result was not saved."
+                                "There is not repo for url %s", repo_url)
+        self.logger.info("Test results: %s", test_result)
+
         return terra_version
diff --git a/ecosystem/models/configuration.py b/ecosystem/models/configuration.py
index 46b572eb4e..4e89a00f0b 100644
--- a/ecosystem/models/configuration.py
+++ b/ecosystem/models/configuration.py
@@ -96,9 +96,10 @@ def __init__(self,
     @classmethod
     def default(cls) -> 'PythonRepositoryConfiguration':
         """Returns default python repository configuration."""
-        return PythonRepositoryConfiguration(dependencies_files=[
-            "requirements.txt"
-        ],
+        return PythonRepositoryConfiguration(
+            dependencies_files=[
+                "requirements.txt"
+            ],
             tests_command=[
                 "pip check",
                 "pytest -W error::DeprecationWarning"
diff --git a/tests/test_controller.py b/tests/test_controller.py
index 6d00e1a407..4ca62f4417 100644
--- a/tests/test_controller.py
+++ b/tests/test_controller.py
@@ -1,10 +1,21 @@
 """Tests for entities."""
 import os
 from unittest import TestCase
-from ecosystem.entities import MainRepository, TestType
+from ecosystem.entities import Repository, TestResult, TestType
 from ecosystem.controller import Controller
 
 
+def get_main_repo() -> Repository:
+    """Return main mock repo."""
+    return Repository(name="mock-qiskit-terra-with-success-dev-test",
+                      url="https://github.com/MockQiskit/mock-qiskit-wsdt.terra",
+                      description="Mock description for repo. wsdt",
+                      licence="Apache 2.0",
+                      labels=["mock", "tests", "wsdt"],
+                      tests_results=[
+                          TestResult(True, "0.18.1", TestType.DEV_COMPATIBLE)])
+
+
 class TestController(TestCase):
     """Tests repository related functions."""
 
@@ -29,11 +40,7 @@ def test_repository_insert_and_delete(self):
         """Tests repository."""
         self._delete_members_json()
 
-        main_repo = MainRepository(name="mock-qiskit-terra",
-                                   url="https://github.com/MockQiskit/mock-qiskit.terra",
-                                   description="Mock description for repo.",
-                                   licence="Apache 2.0",
-                                   labels=["mock", "tests"])
+        main_repo = get_main_repo()
         controller = Controller(self.path)
 
         # insert entry
@@ -41,60 +48,35 @@ def test_repository_insert_and_delete(self):
         fetched_repo = controller.get_all_main()[0]
         self.assertEqual(main_repo, fetched_repo)
         self.assertEqual(main_repo.labels, fetched_repo.labels)
+        self.assertEqual(len(fetched_repo.tests_results), 1)
 
         # delete entry
         controller.delete(main_repo)
         self.assertEqual([], controller.get_all_main())
 
-    def test_add_and_remove_repo_test_passed(self):
-        """Tests addition of passed test to repo."""
+    def test_add_test_result(self):
+        """Tests adding result to repo."""
         self._delete_members_json()
-        main_repo = MainRepository(name="mock-qiskit-terra",
-                                   url="https://github.com/MockQiskit/mock-qiskit.terra",
-                                   description="Mock description for repo.",
-                                   licence="Apache 2.0",
-                                   labels=["mock", "tests"])
         controller = Controller(self.path)
-        controller.insert(main_repo)
 
-        controller.add_repo_test_passed(repo_url=main_repo.url,
-                                        test_passed=TestType.STANDARD,
-                                        tier=main_repo.tier)
-        fetched_repo = controller.get_by_url(main_repo.url, tier=main_repo.tier)
-        self.assertEqual(fetched_repo.tests_passed, [TestType.STANDARD])
-
-        controller.remove_repo_test_passed(repo_url=main_repo.url,
-                                           test_remove=TestType.STANDARD,
-                                           tier=main_repo.tier)
-        fetched_repo = controller.get_by_url(main_repo.url, tier=main_repo.tier)
-        self.assertEqual(fetched_repo.tests_passed, [])
-
-    def test_update_repo_tests_passed(self):
-        """Tests repository tests passed field."""
-        self._delete_members_json()
-
-        url = "https://github.com/MockQiskit/mock-qiskit.terra"
-        main_repo = MainRepository(name="mock-qiskit-terra",
-                                   url=url,
-                                   description="Mock description for repo.",
-                                   licence="Apache 2.0",
-                                   labels=["mock", "tests"])
-        controller = Controller(self.path)
+        main_repo = get_main_repo()
         controller.insert(main_repo)
-
-        controller.update_repo_tests_passed(main_repo, [TestType.STANDARD])
-        fetched_repo = controller.get_by_url(url, main_repo.tier)
-        self.assertEqual(len(fetched_repo.tests_passed), 1)
-        self.assertEqual(fetched_repo.tests_passed, [TestType.STANDARD])
-
-        controller.update_repo_tests_passed(main_repo, [TestType.STANDARD,
-                                                        TestType.DEV_COMPATIBLE])
-        fetched_repo = controller.get_by_url(url, main_repo.tier)
-        self.assertEqual(len(fetched_repo.tests_passed), 2)
-        self.assertEqual(fetched_repo.tests_passed, [TestType.STANDARD,
-                                                     TestType.DEV_COMPATIBLE])
-
-        controller.update_repo_tests_passed(main_repo, [])
-        fetched_repo = controller.get_by_url(url, main_repo.tier)
-        self.assertEqual(len(fetched_repo.tests_passed), 0)
-        self.assertEqual(fetched_repo.tests_passed, [])
+        res = controller.add_repo_test_result(main_repo.url,
+                                              main_repo.tier,
+                                              TestResult(False, "0.18.1",
+                                                         TestType.DEV_COMPATIBLE))
+        self.assertEqual(res, [1])
+        recovered_repo = controller.get_by_url(main_repo.url, tier=main_repo.tier)
+        self.assertEqual(recovered_repo.tests_results, [TestResult(False, "0.18.1",
+                                                                   TestType.DEV_COMPATIBLE)])
+
+        res = controller.add_repo_test_result(main_repo.url,
+                                              main_repo.tier,
+                                              TestResult(True, "0.18.2",
+                                                         TestType.DEV_COMPATIBLE))
+        self.assertEqual(res, [1])
+        recovered_repo = controller.get_by_url(main_repo.url, tier=main_repo.tier)
+        self.assertEqual(recovered_repo.tests_results, [TestResult(False, "0.18.1",
+                                                                   TestType.DEV_COMPATIBLE),
+                                                        TestResult(True, "0.18.2",
+                                                                   TestType.DEV_COMPATIBLE)])
diff --git a/tests/test_entities.py b/tests/test_entities.py
new file mode 100644
index 0000000000..4f0f551a00
--- /dev/null
+++ b/tests/test_entities.py
@@ -0,0 +1,24 @@
+"""Tests for entities."""
+
+import unittest
+
+from ecosystem.entities import Repository, TestResult, TestType
+
+
+class TestRepository(unittest.TestCase):
+    """Tests repository class."""
+
+    def test_serialization(self):
+        """Tests json serialization."""
+        main_repo = Repository(name="mock-qiskit-terra",
+                               url="https://github.com/MockQiskit/mock-qiskit.terra",
+                               description="Mock description for repo.",
+                               licence="Apache 2.0",
+                               labels=["mock", "tests"],
+                               tests_results=[
+                                   TestResult(True, '0.18.1', TestType.DEV_COMPATIBLE)
+                               ])
+        repo_dict = main_repo.to_dict()
+        recovered = Repository.from_dict(repo_dict)
+        self.assertEqual(main_repo, recovered)
+        self.assertEqual(main_repo.tests_results, recovered.tests_results)
diff --git a/tests/test_runner.py b/tests/test_runner.py
index 5d64c5212f..c1d8881a4f 100644
--- a/tests/test_runner.py
+++ b/tests/test_runner.py
@@ -32,7 +32,7 @@ def test_runner_on_simple_repo(self):
         runner.cloned_repo_directory = self.simple_project_dir
         terra_version, result = runner.workload()
 
-        self.assertFalse(result.ok)
+        self.assertFalse(all(r.ok for r in result))
         self.assertTrue(terra_version)
 
     def test_runner_on_configured_repo(self):
@@ -44,5 +44,5 @@ def test_runner_on_configured_repo(self):
         runner.cloned_repo_directory = self.configured_project_dir
         terra_version, result = runner.workload()
 
-        self.assertTrue(result.ok)
+        self.assertTrue(all(r.ok for r in result))
         self.assertTrue(terra_version)

From 0ef0c95ed7aaa92db83b1e98423512862e87b183 Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Mon, 30 Aug 2021 15:07:05 -0400
Subject: [PATCH 07/12] Configuration: skip tests if no setup file.

---
 CONTRIBUTING.md                  | 11 ++++-
 docs/dev/dev-doc.md              | 18 ++++++++
 ecosystem/controllers/runner.py  |  6 ++-
 ecosystem/manager.py             | 76 ++++++++++++++++++++++++--------
 ecosystem/resources/members.json | 36 ++++++++++-----
 5 files changed, 116 insertions(+), 31 deletions(-)
 create mode 100644 docs/dev/dev-doc.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6cf43fd43e..21c324a04b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,4 +5,13 @@ included in the qiskit documentation:
 
 https://qiskit.org/documentation/contributing_to_qiskit.html
 
-# Joining the Ecosystem
+## Joining the Ecosystem
+
+To join ecosystem you need to create 
+[submission issue](https://github.com/qiskit-community/ecosystem/issues/new?labels=&template=submission.yml&title=%5BSubmission%5D%3A+) 
+and fill in all required details. That's it!
+
+
+## Dev contributions
+
+[Refer to dev docs](./docs/dev/dev-doc.md)
\ No newline at end of file
diff --git a/docs/dev/dev-doc.md b/docs/dev/dev-doc.md
new file mode 100644
index 0000000000..7c38c0557a
--- /dev/null
+++ b/docs/dev/dev-doc.md
@@ -0,0 +1,18 @@
+Dev docs
+========
+
+As entire repository is designed to be run through GitHub Actions,
+we implemented ecosystem python package as runner of command line commands
+to be executed from steps in Actions. 
+
+Entrypoint is ``manager.py`` file in the root of repository.
+
+Example of commands:
+```shell
+python manager.py python_dev_tests https://github.com/IceKhan13/demo-implementation --python_version=py39
+python manager.py python_stable_tests https://github.com/IceKhan13/demo-implementation --python_version=py39
+```
+or in general
+```shell
+python manager.py <NAME_OF_FUNCTION_IN_MANAGER_FILE> <POSITIONAL_ARGUMENT> [FLAGS]
+```
diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
index 676e648ce1..fc335140ac 100644
--- a/ecosystem/controllers/runner.py
+++ b/ecosystem/controllers/runner.py
@@ -114,12 +114,16 @@ def workload(self) -> Tuple[str, List[CommandExecutionSummary]]:
             tox_file.write(repo_config.render_tox_file(
                 ecosystem_deps=self.ecosystem_deps))
 
+        terra_version = "-"
+        if not os.path.exists(f"{self.cloned_repo_directory}/setup.py"):
+            self.logger.error("No setup.py file for repository %s", self.repo)
+            return terra_version, []
+
         # run tox
         tox_tests_res = _run_tox(directory=self.cloned_repo_directory,
                                  env=self.python_version)
 
         # get terra version from file
-        terra_version = "-"
         if os.path.exists(f"{self.cloned_repo_directory}/terra_version.txt"):
             with open(f"{self.cloned_repo_directory}/terra_version.txt", "r") as version_file:
                 terra_version = version_file.read()
diff --git a/ecosystem/manager.py b/ecosystem/manager.py
index 4d625b6cd2..3ea0c89781 100644
--- a/ecosystem/manager.py
+++ b/ecosystem/manager.py
@@ -1,6 +1,6 @@
 """Manager class for controlling all CLI functions."""
 import os
-from typing import Optional
+from typing import Optional, List
 
 from jinja2 import Environment, PackageLoader, select_autoescape
 
@@ -41,27 +41,65 @@ def generate_readme(self, path: Optional[str] = None):
         with open(f"{path}/README.md", "w") as file:
             file.write(readme_content)
 
-    def stable_tests(self, repo_url: str,
-                     tier: str = Tier.MAIN,
-                     python_version: str = "py39",):
-        """Runs tests against stable version of qiskit."""
+    def _run_python_tests(self,
+                          repo_url: str,
+                          tier: str,
+                          python_version: str,
+                          test_type: str,
+                          ecosystem_deps: Optional[List[str]] = None):
+        """Runs tests using python runner.
+
+        Args:
+            repo_url: repository url
+            tier: tier of project
+            python_version: ex: py36, py37 etc
+            test_type: [dev, stable]
+            ecosystem_deps: extra dependencies to install for tests
+        """
+        ecosystem_deps = ecosystem_deps or []
         runner = PythonRunner(repo_url,
                               working_directory=self.resources_dir,
-                              ecosystem_deps=["qiskit"],
+                              ecosystem_deps=ecosystem_deps,
                               python_version=python_version)
         terra_version, results = runner.run()
-
-        test_result = TestResult(passed=all(r.ok for r in results),
-                                 terra_version=terra_version,
-                                 test_type=TestType.STABLE_COMPATIBLE)
-        # save test res to db
-        result = self.controller.add_repo_test_result(repo_url=repo_url,
-                                                      tier=tier,
-                                                      test_result=test_result)
-        # print report
-        if result is None:
-            self.logger.warning("Test result was not saved."
-                                "There is not repo for url %s", repo_url)
-        self.logger.info("Test results: %s", test_result)
+        if len(results) > 0:
+            test_result = TestResult(passed=all(r.ok for r in results),
+                                     terra_version=terra_version,
+                                     test_type=test_type)
+            # save test res to db
+            result = self.controller.add_repo_test_result(repo_url=repo_url,
+                                                          tier=tier,
+                                                          test_result=test_result)
+            # print report
+            if result is None:
+                self.logger.warning("Test result was not saved."
+                                    "There is not repo for url %s", repo_url)
+            self.logger.info("Test results: %s", test_result)
+        else:
+            self.logger.warning("Runner returned 0 results.")
 
         return terra_version
+
+    def python_dev_tests(self,
+                         repo_url: str,
+                         tier: str = Tier.MAIN,
+                         python_version: str = "py39"):
+        """Runs tests against dev version of qiskit."""
+        return self._run_python_tests(
+            repo_url=repo_url,
+            tier=tier,
+            python_version=python_version,
+            test_type=TestType.DEV_COMPATIBLE,
+            ecosystem_deps=["git+https://github.com/Qiskit/qiskit-terra.git@main"])
+
+    def python_stable_tests(self,
+                            repo_url: str,
+                            tier: str = Tier.MAIN,
+                            python_version: str = "py39"):
+        """Runs tests against stable version of qiskit."""
+        return self._run_python_tests(
+            repo_url=repo_url,
+            tier=tier,
+            python_version=python_version,
+            test_type=TestType.STABLE_COMPATIBLE,
+            ecosystem_deps=["qiskit"])
diff --git a/ecosystem/resources/members.json b/ecosystem/resources/members.json
index 4e707c5c5f..46d144e32f 100644
--- a/ecosystem/resources/members.json
+++ b/ecosystem/resources/members.json
@@ -13,7 +13,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.117589,
             "url": "https://github.com/Qiskit/qiskit",
-            "tests_passed": []
+            "tests_results": []
         },
         "2": {
             "alternatives": null,
@@ -29,7 +29,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.118061,
             "url": "https://github.com/Qiskit/qiskit-terra",
-            "tests_passed": []
+            "tests_results": []
         },
         "3": {
             "alternatives": null,
@@ -44,7 +44,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.118472,
             "url": "https://github.com/Qiskit/qiskit-aer",
-            "tests_passed": []
+            "tests_results": []
         },
         "4": {
             "alternatives": null,
@@ -60,7 +60,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.118821,
             "url": "https://github.com/Qiskit/qiskit-optimization",
-            "tests_passed": []
+            "tests_results": []
         },
         "5": {
             "alternatives": null,
@@ -76,7 +76,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.119205,
             "url": "https://github.com/Qiskit/qiskit-metal",
-            "tests_passed": []
+            "tests_results": []
         },
         "6": {
             "alternatives": null,
@@ -92,7 +92,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.119529,
             "url": "https://github.com/Qiskit/qiskit-machine-learning",
-            "tests_passed": []
+            "tests_results": []
         },
         "7": {
             "alternatives": null,
@@ -109,7 +109,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.119862,
             "url": "https://github.com/Qiskit/qiskit-nature",
-            "tests_passed": []
+            "tests_results": []
         },
         "8": {
             "alternatives": null,
@@ -125,7 +125,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.120268,
             "url": "https://github.com/Qiskit/qiskit-finance",
-            "tests_passed": []
+            "tests_results": []
         },
         "9": {
             "alternatives": null,
@@ -141,7 +141,7 @@
             "tier": "MAIN",
             "updated_at": 1628883441.120709,
             "url": "https://github.com/Qiskit/qiskit-tutorials",
-            "tests_passed": []
+            "tests_results": []
         },
         "10": {
             "alternatives": null,
@@ -157,7 +157,23 @@
             "tier": "MAIN",
             "updated_at": 1628883441.121111,
             "url": "https://github.com/Qiskit/qiskit.org",
-            "tests_passed": []
+            "tests_results": []
+        },
+        "11": {
+            "alternatives": null,
+            "contact_info": null,
+            "created_at": 1630349762.282323,
+            "description": "Qiskit's website",
+            "labels": [
+                "community",
+                "web"
+            ],
+            "licence": "Apache 2.0",
+            "name": "qiskit.org",
+            "tier": "MAIN",
+            "updated_at": 1630349762.282341,
+            "url": "https://github.com/IceKhan13/demo-implementation",
+            "tests_results": []
         }
     }
 }
\ No newline at end of file

From a2792137a2d06cdbaca58901471cd4bf02157576 Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Mon, 30 Aug 2021 15:10:29 -0400
Subject: [PATCH 08/12] Workflows: update command

---
 .github/workflows/check-main-repos.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/check-main-repos.yml b/.github/workflows/check-main-repos.yml
index 455f125495..560b5c9c50 100644
--- a/.github/workflows/check-main-repos.yml
+++ b/.github/workflows/check-main-repos.yml
@@ -41,15 +41,15 @@ jobs:
 
     # test runs with stable version of Qiskit
     - name: Stable version of Qiskit test for Qiskit-nature
-      run: python manager.py stable_compatibility_tests https://github.com/Qiskit/qiskit-nature --tox_python=py39
+      run: python manager.py python_stable_tests https://github.com/Qiskit/qiskit-nature --tox_python=py39
     - name: Stable version of Qiskit test for Qiskit-finance
-      run: python manager.py stable_compatibility_tests https://github.com/Qiskit/qiskit-finance --tox_python=py39
+      run: python manager.py python_stable_tests https://github.com/Qiskit/qiskit-finance --tox_python=py39
 
     # test runs with dev version of Qiskit
     - name: Dev version of Qiskit test for Qiskit-nature
-      run: python manager.py dev_compatibility_tests https://github.com/Qiskit/qiskit-nature --tox_python=py39
+      run: python manager.py python_dev_tests https://github.com/Qiskit/qiskit-nature --tox_python=py39
     - name: Dev version of Qiskit test for Qiskit-finance
-      run: python manager.py dev_compatibility_tests https://github.com/Qiskit/qiskit-finance --tox_python=py39
+      run: python manager.py python_dev_tests https://github.com/Qiskit/qiskit-finance --tox_python=py39
 
     - name: State of members.json file
       run: cat ecosystem/resources/members.json

From ea44a576c7e609b467971bd090f232743d9385ba Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Tue, 31 Aug 2021 09:35:41 -0400
Subject: [PATCH 09/12] Workflow: fix workflow

---
 .github/workflows/check-main-repos.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/check-main-repos.yml b/.github/workflows/check-main-repos.yml
index 560b5c9c50..99bfdfa611 100644
--- a/.github/workflows/check-main-repos.yml
+++ b/.github/workflows/check-main-repos.yml
@@ -15,7 +15,7 @@ jobs:
     steps:
     - name: Get current datetime
       id: date
-      run: echo "::set-output name=date::$(date +'%Y-%m-%d %H:%M')"
+      run: echo "::set-output name=date::$(date +'%Y_%m_%d_%H_%M')"
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
@@ -33,12 +33,6 @@ jobs:
         pip install -r requirements.txt
         pip install -r requirements-dev.txt
 
-    # test runs with standard tests
-    - name: Standard test for Qiskit-nature
-      run: python manager.py standard_tests https://github.com/Qiskit/qiskit-nature --tox_python=py39
-    - name: Standard test for Qiskit-finance
-      run: python manager.py standard_tests https://github.com/Qiskit/qiskit-finance --tox_python=py39
-
     # test runs with stable version of Qiskit
     - name: Stable version of Qiskit test for Qiskit-nature
       run: python manager.py python_stable_tests https://github.com/Qiskit/qiskit-nature --tox_python=py39

From 6d39539e33c70651721aa125bcd99a52f90661e7 Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Tue, 7 Sep 2021 10:26:05 -0400
Subject: [PATCH 10/12] Docs: dev docs

---
 docs/dev/dev-doc.md              | 30 +++++++++++++++++++++++++++++-
 ecosystem/controllers/runner.py  |  7 +++++--
 ecosystem/manager.py             |  7 ++++++-
 ecosystem/resources/members.json | 16 ----------------
 4 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/docs/dev/dev-doc.md b/docs/dev/dev-doc.md
index 7c38c0557a..c62b9332df 100644
--- a/docs/dev/dev-doc.md
+++ b/docs/dev/dev-doc.md
@@ -2,7 +2,7 @@ Dev docs
 ========
 
 As entire repository is designed to be run through GitHub Actions,
-we implemented ecosystem python package as runner of command line commands
+we implemented ecosystem python package as runner of CLI commands
 to be executed from steps in Actions. 
 
 Entrypoint is ``manager.py`` file in the root of repository.
@@ -16,3 +16,31 @@ or in general
 ```shell
 python manager.py <NAME_OF_FUNCTION_IN_MANAGER_FILE> <POSITIONAL_ARGUMENT> [FLAGS]
 ```
+
+#### Ecosystem workflows configuration
+
+In order to talk control of execution workflow of tests in ecosystem
+repository should have `qe_config.json` file in a root directory.
+
+Structure of config file:
+- dependencies_files: list[string] - files with package dependencies (ex: requirements.txt, packages.json)
+- extra_dependencies: list[string] - names of additional packages to install before tests execution
+- language: string - programming language for tests env. Only supported lang is Python at this moment.
+- tests_command: list[string] - list of commands to execute tests
+
+Example:
+```json
+{
+    "dependencies_files": [
+        "requirements.txt",
+        "requirements-dev.txt"
+    ],
+    "extra_dependencies": [
+        "pytest"
+    ],
+    "language": "python",
+    "tests_command": [
+        "pytest -p no:warnings --pyargs test"
+    ]
+}
+```
diff --git a/ecosystem/controllers/runner.py b/ecosystem/controllers/runner.py
index fc335140ac..b104a3d0b6 100644
--- a/ecosystem/controllers/runner.py
+++ b/ecosystem/controllers/runner.py
@@ -13,7 +13,10 @@
 
 
 class Runner:
-    """Runner for repository checks."""
+    """Runner for repository checks.
+
+    General class to run workflow for repository.
+    """
 
     def __init__(self,
                  repo: Union[str, Repository],
@@ -49,7 +52,7 @@ def run(self) -> Tuple[str, List[CommandExecutionSummary]]:
         """Runs chain of commands to check repository."""
         self.set_up()
         # clone repository
-        self.logger.info("Cloning repository.")
+        self.logger.info("Cloning repository: %s", self.repo)
         clone_res = _clone_repo(self.repo, directory=self.working_directory)
 
         if not clone_res.ok:
diff --git a/ecosystem/manager.py b/ecosystem/manager.py
index 3ea0c89781..d2ee67a428 100644
--- a/ecosystem/manager.py
+++ b/ecosystem/manager.py
@@ -13,6 +13,11 @@
 class Manager:
     """Manager class.
     Entrypoint for all CLI commands.
+
+    Each public method of this class is CLI command
+    and arguments for method are options/flags for this command.
+
+    Ex: `python manager.py generate_readme --path=<SOME_DIRECTORY>`
     """
 
     def __init__(self):
@@ -74,7 +79,7 @@ def _run_python_tests(self,
             if result is None:
                 self.logger.warning("Test result was not saved."
                                     "There is not repo for url %s", repo_url)
-            self.logger.info("Test results: %s", test_result)
+            self.logger.info("Test results for %s: %s", repo_url, test_result)
         else:
             self.logger.warning("Runner returned 0 results.")
 
diff --git a/ecosystem/resources/members.json b/ecosystem/resources/members.json
index 46d144e32f..17a79e7bdb 100644
--- a/ecosystem/resources/members.json
+++ b/ecosystem/resources/members.json
@@ -158,22 +158,6 @@
             "updated_at": 1628883441.121111,
             "url": "https://github.com/Qiskit/qiskit.org",
             "tests_results": []
-        },
-        "11": {
-            "alternatives": null,
-            "contact_info": null,
-            "created_at": 1630349762.282323,
-            "description": "Qiskit's website",
-            "labels": [
-                "community",
-                "web"
-            ],
-            "licence": "Apache 2.0",
-            "name": "qiskit.org",
-            "tier": "MAIN",
-            "updated_at": 1630349762.282341,
-            "url": "https://github.com/IceKhan13/demo-implementation",
-            "tests_results": []
         }
     }
 }
\ No newline at end of file

From 92d61ee287ad9e4da1f3170ecf11b5b2f4cbf0f7 Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Tue, 7 Sep 2021 10:27:27 -0400
Subject: [PATCH 11/12] Workflows: remove scheduled runs for main repos

---
 .github/workflows/check-main-repos.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/check-main-repos.yml b/.github/workflows/check-main-repos.yml
index 99bfdfa611..228e600500 100644
--- a/.github/workflows/check-main-repos.yml
+++ b/.github/workflows/check-main-repos.yml
@@ -1,8 +1,6 @@
 name: Test main tier of ecosystem
 
 on:
-  schedule:
-    - cron:  '5 8 * * 2' # each Tuesday at 8 05
   workflow_dispatch:
 
 jobs:

From d21b4a124fce73c27e5eb354f367a72d89e2dd3c Mon Sep 17 00:00:00 2001
From: Iskandar Sitdikov <iskandar.sitdikov@gmail.com>
Date: Sat, 18 Sep 2021 20:22:37 -0400
Subject: [PATCH 12/12] Workflow: change in parallel execution

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 174501976d..0d7fcd6a95 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -20,7 +20,7 @@ jobs:
   tests:
     runs-on: ubuntu-latest
     strategy:
-      max-parallel: 4
+      max-parallel: 2
       matrix:
         python-version: [3.9]
     steps: