Don't fetch the last commit every time

Introduces a caching mechanism to store the last commit, so we can fetch
it from the DB instead of through the GitHub API. When the webhook
receives a push event, we update the value stored.

It also adds back the statuses of the latest tests for a given sample.

mod_ci/controllers.py

@@ -15,6 +15,7 @@
 
 from mod_ci.models import Kvm
 from mod_deploy.controllers import request_from_github, is_valid_signature
+from mod_home.models import GeneralData
 from mod_regression.models import Category, RegressionTestOutput
 from mod_test.models import TestType, Test, TestStatus, TestProgress, Fork, \
     TestPlatform, TestResultFile, TestResult
@@ -314,6 +315,13 @@ def start_ci():
             gh_commit = gh.repos(g.github['repository_owner'])(
                 g.github['repository']).statuses(commit)
             queue_test(g.db, gh_commit, commit, TestType.commit)
+            # Update the db to the new last commit
+            ref = gh.repos(g.github['repository_owner'])(
+                g.github['repository']).git().refs('heads/master').get()
+            last_commit = GeneralData.query.filter(GeneralData.key ==
+                                                   'last_commit').first()
+            last_commit.value = ref['object']['sha']
+            g.db.commit()
 
         elif event == "pull_request":  # If it's a PR, run the tests
             commit = payload['after']

mod_home/controllers.py

@@ -2,7 +2,7 @@
 from github import GitHub
 
 from decorators import template_renderer
-from mod_home.models import CCExtractorVersion
+from mod_home.models import CCExtractorVersion, GeneralData
 
 mod_home = Blueprint('home', __name__)
 
@@ -19,18 +19,13 @@ def before_app_request():
 @mod_home.route('/', methods=['GET', 'POST'])
 @template_renderer()
 def index():
-    from run import app
-    # TODO: do not look this up on every request
-    g = GitHub(access_token=app.config.get('GITHUB_TOKEN', ''))
-    ref = g.repos(app.config.get('GITHUB_OWNER', ''))(
-        app.config.get('GITHUB_REPOSITORY', '')).git().refs(
-        'heads/master').get()
-
+    last_commit = GeneralData.query.filter(
+        GeneralData.key == 'last_commit').first().value
     last_release = CCExtractorVersion.query.order_by(
             CCExtractorVersion.released.desc()).first()
     return {
         'ccx_last_release': last_release,
-        'ccx_latest_commit': ref['object']['sha']
+        'ccx_latest_commit': last_commit
     }
 
 

mod_home/models.py

@@ -18,3 +18,18 @@ def __init__(self, version, released, commit):
 
     def __repr__(self):
         return '<Version %r>' % self.version
+
+
+class GeneralData(Base):
+    __tablename__ = 'general_data'
+    __table_args__ = {'mysql_engine': 'InnoDB'}
+    id = Column(Integer, primary_key=True)
+    key = Column(String(64), unique=True)
+    value = Column(Text(), nullable=False)
+
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+
+    def __repr__(self):
+        return '<GeneralData %r: %r>' % (self.key, self.value)

mod_regression/controllers.py

@@ -26,6 +26,12 @@ def index():
     }
 
 
+@mod_regression.route('/sample/<sample_id>')
+def by_sample(sample_id):
+    # Show all regression tests for sample
+    pass
+
+
 @mod_regression.route('/test/<regression_id>/view')
 def test_view(regression_id):
     pass

mod_sample/controllers.py

@@ -1,17 +1,20 @@
 import os
+from operator import and_
 
 from flask import Blueprint, make_response, request, redirect, url_for, g
 
 from decorators import template_renderer
 from mod_auth.controllers import login_required, check_access_rights
 from mod_auth.models import Role
-from mod_home.models import CCExtractorVersion
+from mod_home.models import CCExtractorVersion, GeneralData
+from mod_regression.models import RegressionTest
 from mod_sample.forms import EditSampleForm, DeleteSampleForm, \
     DeleteAdditionalSampleForm
 from mod_sample.media_info_parser import MediaInfoFetcher, \
     InvalidMediaInfoError
 
 from mod_sample.models import Sample, ExtraFile, ForbiddenExtension
+from mod_test.models import Test, TestResult, TestResultFile
 from mod_upload.models import Platform
 
 mod_sample = Blueprint('sample', __name__)
@@ -46,11 +49,69 @@ def display_sample_info(sample):
     except InvalidMediaInfoError as i:
         raise SampleNotFoundException(i.message)
 
+    latest_commit = GeneralData.query.filter(
+        GeneralData.key == 'last_commit').first().value
+    last_release = CCExtractorVersion.query.order_by(
+        CCExtractorVersion.released.desc()).first().commit
+
+    test_commit = Test.query.filter(Test.commit == latest_commit).first()
+    test_release = Test.query.filter(Test.commit == last_release).first()
+    regression_tests = RegressionTest.query.filter(
+        RegressionTest.sample_id == sample.id).all()
+    status = 'Unknown'
+    status_release = 'Unknown'
+    if len(regression_tests) > 0:
+        if test_commit is not None:
+            sq = g.db.query(RegressionTest.id).filter(
+                        RegressionTest.sample_id == sample.id).subquery()
+            exit_code = g.db.query(TestResult.exit_code).filter(and_(
+                TestResult.exit_code != 0,
+                and_(
+                    TestResult.test_id == test_commit.id,
+                    TestResult.regression_test_id.in_(sq)
+                )
+            )).first()
+            not_null = g.db.query(TestResultFile.got).filter(and_(
+                TestResultFile.got.isnot(None),
+                and_(
+                    TestResultFile.test_id == test_commit.id,
+                    TestResultFile.regression_test_id.in_(sq)
+                )
+            )).first()
+            if exit_code is None and not_null is None:
+                status = 'Pass'
+            else:
+                status = 'Fail'
+        if test_release is not None:
+            sq = g.db.query(RegressionTest.id).filter(
+                RegressionTest.sample_id == sample.id).subquery()
+            exit_code = g.db.query(TestResult.exit_code).filter(and_(
+                TestResult.exit_code != 0,
+                TestResult.test_id == test_release.id,
+                TestResult.regression_test_id.in_(sq)
+            )).first()
+            not_null = g.db.query(TestResultFile.got).filter(and_(
+                TestResultFile.got.isnot(None),
+                TestResultFile.test_id == test_release.id,
+                TestResultFile.regression_test_id.in_(sq)
+            )).first()
+            if exit_code is None and not_null is None:
+                status = 'Pass'
+            else:
+                status = 'Fail'
+    else:
+        status = 'Not present in regression tests'
+        status_release = 'Not present in regression tests'
+
     return {
         'sample': sample,
         'media': media_info,
         'additional_files': ExtraFile.query.filter(
-            ExtraFile.sample_id == sample.id).all()
+            ExtraFile.sample_id == sample.id).all(),
+        'latest_commit': status,
+        'latest_commit_test': test_commit,
+        'latest_release': status_release,
+        'latest_release_test': test_release
     }
 
 

mod_test/controllers.py

@@ -100,11 +100,6 @@ def by_commit(commit_hash):
     pass
 
 
-@mod_test.route('/sample/<sample_id>')
-def by_sample(sample_id):
-    pass
-
-
 @mod_test.route('/diff/<test_id>/<regression_test_id>/<output_id>')
 def generate_diff(test_id, regression_test_id, output_id):
     from run import config

mod_test/models.py

@@ -225,6 +225,7 @@ def __repr__(self):
         )
 
     def generate_html_diff(self, base_path):
+        # TODO: use difflib.SequenceMatcher and generate own HTML diff
         file_ok = os.path.join(
             base_path,
             self.expected + self.regression_test_output.correct_extension)

templates/home/index.html

@@ -15,8 +15,8 @@ <h1>{{ applicationName }}</h1>
                 <li>Run the regression tests</li>
             </ul>
             <h4>Development information</h4>
-            <p>Last official release: <a target="_blank" href="http://sourceforge.net/projects/ccextractor/files/ccextractor/{{ ccx_last_release.version }}/">{{ ccx_last_release.version }}</a> (release date: {{ ccx_last_release.released }}). Regression tests for this version are available here: <a href="{{ url_for("home.index", version=ccx_last_release.version) }}">regression tests</a></p>
-            <p>Latest GitHub commit: <a target="_blank" href="https://github.com/CCExtractor/ccextractor/commit/{{ ccx_latest_commit }}">{{ ccx_latest_commit }}</a>. View the <a href="{{ url_for("home.index", hash=ccx_latest_commit) }}">regression tests</a>.</p>
+            <p>Last official release: <a target="_blank" href="http://sourceforge.net/projects/ccextractor/files/ccextractor/{{ ccx_last_release.version }}/">{{ ccx_last_release.version }}</a> (release date: {{ ccx_last_release.released }}). Regression tests for this version are available here: <a href="{{ url_for("test.ccextractor_version", ccx_version=ccx_last_release.version) }}">regression tests</a></p>
+            <p>Latest GitHub commit: <a target="_blank" href="https://github.com/CCExtractor/ccextractor/commit/{{ ccx_latest_commit }}">{{ ccx_latest_commit }}</a>. View the <a href="{{ url_for("test.by_commit", commit_hash=ccx_latest_commit) }}">regression tests</a>.</p>
         </div>
         <div class="column medium-3">
             <h5>Helpful links</h5>

templates/sample/sample_info.html

@@ -55,11 +55,10 @@ <h2>Additional files</h2>
                         </tbody>
                     </table>
                 {% endif %}
-                <h2>Test status</h2>
-                <!--
-                <p>Repository: </p>
-                <p>Last release: </p> -->
-                <p><a href="{{ url_for("test.by_sample", sample_id=sample.id) }}">View all available tests for this sample</a></p>
+                <h2>Test status & regression tests</h2>
+                <p>Repository: {{ latest_commit }}</p>
+                <p>Last release: {{ latest_release }}</p>
+                <p><a href="{{ url_for("regression.by_sample", sample_id=sample.id) }}">See in which regression tests this sample has been used</a></p>
                 <h2>Media info</h2>
                 <p class="small">Full media info can be downloaded using the link at the right -->.</p>
                 <blockquote class="small">