Org: Adds working link check for internal links

Requests internal urls with ajax calls.

Link: SEA-331
Type: Improvement

src/onegov/org/app.py

@@ -512,6 +512,7 @@ def get_common_asset():
     yield 'common.js'
     yield '_blank.js'
     yield 'forms.js'
+    yield 'internal_link_check.js'
 
 
 @OrgApp.webasset('accordion')

src/onegov/org/assets/js/internal_link_check.js

@@ -0,0 +1,56 @@
+$('[data-link-check]').each(function () {
+    var check = $(this);
+    var totalCounter = check.data('check-total') ? $(check.data('check-total')) : undefined
+    var okCounter = check.data('check-ok') ? $(check.data('check-ok')) : undefined
+    var nokCounter = check.data('check-nok') ? $(check.data('check-nok')) : undefined
+    var errorCounter = check.data('check-error') ? $(check.data('check-error')) : undefined
+    var errorMsgSelector = check.data('check-error-msg');
+    var okClass = check.data('ok-class');
+    var nokClass = check.data('nok-class');
+    var statusSelector = check.data('status-selector');
+    var removeDelay = check.data('remove-delay') || '1000'
+
+    function addCount(obj, count) {
+        if (!obj) return;
+        var old = parseInt(obj.text());
+        obj.text(old + parseInt(count));
+    }
+
+    function toggleClass(obj, selector, name) {
+        if (selector && name) {
+            obj.find(selector).toggleClass(name)
+        }
+    }
+
+    function addErrorMsg(resp) {
+        if (errorMsgSelector) {
+            $(errorMsgSelector).text(resp.status || resp.statusText)
+        }
+    }
+
+    check.find('[data-check-url]').each(function () {
+        var self = $(this)
+        $.ajax({
+            type: "GET",
+            async: false,
+            url: self.data('check-url'),
+            statusCode : {
+                200: function () {
+                    addCount(totalCounter, 1);
+                    addCount(okCounter, 1);
+                    toggleClass(self, statusSelector, okClass);
+                    setTimeout(function () {
+                        self.remove()
+                    }, parseInt(removeDelay));
+                }
+            }
+        }).fail(function(jqXHR) {
+            if( jqXHR.status ) { addCount(nokCounter, 1); }
+            else { addCount(errorCounter, 1); }
+            toggleClass(self, statusSelector, nokClass);
+            addCount(totalCounter, 1);
+            addErrorMsg(jqXHR);
+          })
+    })
+})
+

src/onegov/org/management.py

@@ -205,8 +205,6 @@ def __init__(
         self.domain = self.request.domain
         self.extractor = URLExtract()
 
-        self.unhealthy_urls_stats = None
-        self._started = time.time()
         self.timeout = ClientTimeout(
             total=total_timout
         )
@@ -219,10 +217,6 @@ def internal_only(self):
     def external_only(self):
         return self.link_type == 'external'
 
-    @staticmethod
-    def stats_obj():
-        return
-
     def internal_link(self, url):
         return self.domain in url
 
@@ -252,19 +246,24 @@ def find_urls(self):
                         self.filter_urls(urls)
                     )
 
+    def url_list_generator(self):
+        for name, model_link, urls in self.find_urls():
+            for url in urls:
+                yield LinkCheck(name, model_link, url)
+
     def unhealthy_urls(self):
-        """ We check the urls in the backend, since in the frontend we run into
-        the content-security-policy. """
+        """ We check the urls in the backend, unless they are internal.
+         In that case, we can not do that since we do not have async support.
+         Otherwise returns the LinkChecks with empty statistics for use in
+         the frontend.
+         """
+        assert self.link_type, 'link_type must be set'
+        started = time.time()
 
         total_count = 0
         error_count = 0
         not_okay_status = 0
 
-        def url_list_generator():
-            for name, model_link, urls in self.find_urls():
-                for url in urls:
-                    yield LinkCheck(name, model_link, url)
-
         def handle_errors(check, exception):
             nonlocal total_count
             nonlocal error_count
@@ -283,20 +282,29 @@ def on_success(check, status):
                 not_okay_status += 1
             return check
 
-        urls = async_aiohttp_get_all(
-            urls=tuple(url_list_generator()),
-            response_attr='status',
-            callback=on_success,
-            handle_exceptions=handle_errors,
-            timeout=self.timeout
-        )
-
-        self.unhealthy_urls_stats = self.Statistic(
-            total=total_count,
-            error=error_count,
-            nok=not_okay_status,
-            ok=total_count - error_count - not_okay_status,
-            duration=time.time() - self._started
-        )
-
-        return tuple(check for check in urls if check.status != 200)
+        if self.link_type == 'external':
+            urls = async_aiohttp_get_all(
+                urls=tuple(self.url_list_generator()),
+                response_attr='status',
+                callback=on_success,
+                handle_exceptions=handle_errors,
+                timeout=self.timeout
+            )
+            stats = self.Statistic(
+                total=total_count,
+                error=error_count,
+                nok=not_okay_status,
+                ok=total_count - error_count - not_okay_status,
+                duration=time.time() - started
+            )
+        else:
+            urls = tuple(self.url_list_generator())
+            stats = self.Statistic(
+                total=0,
+                error=0,
+                nok=0,
+                ok=0,
+                duration=time.time() - started
+            )
+
+        return stats, tuple(check for check in urls if check.status != 200)

src/onegov/org/templates/linkcheck.pt

@@ -16,15 +16,28 @@
             </thead>
             <tbody>
                 <tr>
-                    <td>${stats.total}</td>
-                    <td>${stats.ok}</td>
-                    <td>${stats.nok}</td>
-                    <td>${stats.error}</td>
+                    <td id="total">${stats.total}</td>
+                    <td id="ok">${stats.ok}</td>
+                    <td id="nok">${stats.nok}</td>
+                    <td id="error-count">${stats.error}</td>
                     <td>${round(stats.duration, 1)}</td>
                 </tr>
             </tbody>
         </table>
-        <table class="link-check-results">
+        <table
+                tal:condition="check_responses"
+                class="link-check-results"
+                tal:attributes="data-link-check (True if healthcheck.internal_only else None)"
+                data-check-total="#total"
+                data-check-ok="#ok"
+                data-check-nok="#nok"
+                data-check-error="#error-count"
+                data-check-error-msg="[data-error]"
+                data-ok-class="ok"
+                data-nok-class="nok"
+                data-status-selector=".link-status"
+                data-remove-delay="1500"
+        >
           <thead>
             <tr>
               <th width="150" i18n:translate="">Path</th>
@@ -33,11 +46,11 @@
             </tr>
           </thead>
           <tbody>
-            <tr tal:repeat="result check_responses">
+            <tr tal:repeat="result check_responses" tal:attributes="data-check-url (result.url if healthcheck.internal_only else None)">
                 <td><a href="${result.link}" target="_blank">${result.name}</a></td>
-                <td>${result.message or ''}</td>
+                <td data-error>${result.message or ''}</td>
                 <td tal:define="url result.url">
-                    <a class="link-status nok ${internal_link(url) and 'internal'}" href="${url}" target="_blank">${truncate(url)}</a>
+                    <a class="link-status ${healthcheck.internal_link(url) and 'internal'} ${result.status and result.status != 200 and 'nok' or ''}" href="${url}" target="_blank">${truncate(url)}</a>
                 </td>
             </tr>
           </tbody>

src/onegov/org/views/settings.py

@@ -262,12 +262,14 @@ def handle_migrate_links(self, request, form, layout=None):
     permission=Secret, form=LinkHealthCheckForm)
 def handle_link_health_check(self, request, form, layout=None):
 
-    check_responses = None
     healthcheck = LinkHealthCheck(request)
+    check_responses = None
+    stats = None
 
     if form.submitted(request):
-        healthcheck.link_type = form.scope.data
-        check_responses = healthcheck.unhealthy_urls()
+        link_type = form.scope.data
+        healthcheck.link_type = link_type
+        stats, check_responses = healthcheck.unhealthy_urls()
 
     url_max_len = 80
 
@@ -281,7 +283,7 @@ def truncate(text):
         'form': form,
         'layout': layout or DefaultLayout(self, request),
         'check_responses': check_responses,
-        'internal_link': healthcheck.internal_link,
         'truncate': truncate,
-        'stats': healthcheck.unhealthy_urls_stats
+        'stats': stats,
+        'healthcheck': healthcheck
     }

src/onegov/town6/app.py

@@ -187,6 +187,7 @@ def get_common_asset():
     yield '_blank.js'
     yield 'animate.js'
     yield 'forms.js'
+    yield 'internal_link_check.js'
 
 
 @TownApp.webasset('editor')

tests/onegov/org/test_management.py

@@ -41,42 +41,35 @@ def get_request():
         "Invalid: $URL",
     ]
 
+    external_exp = [valid[0]] + not_found + invalid_domain
+    internal_exp = [valid[1]]
+
     links = valid + invalid_fmt + not_found + invalid_domain
     assert len(fragments) == len(links)
-    fetched_count = len(links) - len(invalid_fmt)
-    error_count = len(invalid_domain)
-    nok_count = len(not_found)
 
     text = "\n".join(
         f.replace('$URL', u) for f, u in zip(fragments, links))
 
     page = pages.query().first()
     page.lead = text
 
-    check = LinkHealthCheck(request)
+    check = LinkHealthCheck(request, 'external')
+
     found_urls = check.extractor.find_urls(text, only_unique=True)
     assert found_urls == valid + not_found + invalid_domain
 
     urls = tuple(check.find_urls())
-    assert urls == (('Topic', 'URL-Topic', found_urls),)
-
-    all_results = check.unhealthy_urls()
-    stats = check.unhealthy_urls_stats
+    assert urls == (('Topic', 'URL-Topic', tuple(external_exp)),)
 
-    print(all_results[0])
-    print(stats)
-    assert stats.total == fetched_count
-    assert stats.ok == len(valid)
-    assert stats.nok == nok_count
-    assert stats.error == error_count
-
-    # check filters
+    # check internal
     check.link_type = 'internal'
     urls = tuple(check.find_urls())
-    filtered = tuple([u for u in found_urls if test_domain in u])
-    assert urls == (('Topic', 'URL-Topic', filtered),)
-
-    check.link_type = 'external'
-    urls = tuple(check.find_urls())
-    filtered = tuple([u for u in found_urls if test_domain not in u])
-    assert urls == (('Topic', 'URL-Topic', filtered),)
+    assert urls == (('Topic', 'URL-Topic', tuple(internal_exp)),)
+    stats, all_results = check.unhealthy_urls()
+
+    # handled by js in the frontend, so the stats are basically empty
+    assert len(all_results) == len(internal_exp)
+    assert stats.total == 0
+    assert stats.ok == 0
+    assert stats.nok == 0
+    assert stats.error == 0