alex-moreno · iainp999 · Dec 16, 2020 · Dec 16, 2020 · Dec 16, 2020 · Dec 16, 2020
diff --git a/Vagrantfile b/Vagrantfile
@@ -76,7 +76,7 @@ Vagrant.configure("2") do |config|
     # Install dependencies
     add-apt-repository ppa:ondrej/php
     apt-get update
-    apt-get install -y apache2 git curl php7.4 php7.4-bcmath php7.4-bz2 php7.4-cli php7.4-curl php7.4-intl php7.4-json php7.4-mbstring php7.4-opcache php7.4-soap php7.4-sqlite3 php7.4-xml php7.4-xsl php7.4-zip libapache2-mod-php7.4 sqlite3 php-pear
+    apt-get install -y apache2 git curl php7.4 php7.4-bcmath php7.4-bz2 php7.4-cli php7.4-curl php7.4-intl php7.4-json php7.4-mbstring php7.4-opcache php7.4-soap php7.4-sqlite3 php7.4-xml php7.4-xsl php7.4-zip libapache2-mod-php7.4 sqlite3 php-pear composer
     pear install HTML_Table
   SHELL
 end
diff --git a/html/index.php b/html/index.php
@@ -1,55 +1,7 @@
-<?php
+<html>
 
-require_once __DIR__.'/../vendor/autoload.php';
+<a href="sites.php">Crawls results per dates</a>
+<br>
+<a href="status.php">List of status changes between crawls</a>
 
-$resultsStorage = new \ScraperBot\Storage\SqlLite3Storage('../railerdb.sqlite3');
-$crawls = $resultsStorage->getTimeStamps();
-
-$rows = [];
-$headers = [];
-
-$index = 0;
-
-// Iterate over the results, preparing columns and rows for the twig template.
-foreach ($crawls as $timestamp) {
-    // Get site crawl results for each timestamp.
-    $resultsByTimestamp = $resultsStorage->getResultsbyTimestamp($timestamp);
-    $headers[$index] = $timestamp;
-
-    // Get the list of results, per site, for a given timestamp and prepare
-    // array entries representing the rows.
-    foreach ($resultsByTimestamp as $listOfSites) {
-        foreach ($listOfSites as $site) {
-            $site_id = $site['site_id'];
-
-            // Initialise the row for the site if it's empty.
-            if (empty($rows[$site_id][$index])) {
-                $rows[$site_id][$index] = [];
-            }
-
-            array_push($rows[$site_id][$index], $site['size'], $site['statusCode']);
-        }
-    }
-
-    $index++;
-}
-
-// Populate missing data for the rows and sort by index to maintain
-// column order.
-foreach ($rows as $site_id => $row) {
-    for ($i = 0; $i < $index; $i++) {
-        if (empty($rows[$site_id][$i])) {
-            $rows[$site_id][$i] = ['', ''];
-        }
-    }
-    ksort($rows[$site_id]);
-}
-
-// Specify our Twig templates location
-$loader = new \Twig\Loader\FilesystemLoader(__DIR__.'/../src/templates');
-
-// Instantiate our Twig
-$twig = new \Twig\Environment($loader);
-
-$template = $twig->load('results.twig');
-echo $template->render(['headers' => $headers, 'rows' => $rows]);
+</html>
diff --git a/src/ScraperBot/Storage/SqlLite3Storage.php b/src/ScraperBot/Storage/SqlLite3Storage.php
@@ -53,7 +53,7 @@ public function getResults() {
      * @return mixed
      */
     public function getTimeStamps() {
-        $queryString = sprintf("SELECT * FROM sites group by timestamp ");
+        $queryString = sprintf("SELECT * FROM sites group by timestamp");
         $query = $this->pdo->query($queryString);
         while ($row = $query->fetchArray()) {
             $results[] = $row['timestamp'];
@@ -62,6 +62,21 @@ public function getTimeStamps() {
         return $results;
     }
 
+    /**
+     * Get different crawls
+     *
+     * @return mixed
+     */
+    public function getStatusCodes() {
+        $queryString = sprintf("SELECT statusCode FROM sites group by StatusCode");
+        $query = $this->pdo->query($queryString);
+        while ($row = $query->fetchArray()) {
+            $results[] = $row['statusCode'];
+        }
+
+        return $results;
+    }
+
     /**
      * Get results for a given date.
      *
@@ -78,4 +93,23 @@ public function getResultsbyTimestamp($timestamp) {
         return $results;
     }
 
+    /**
+     * Get different crawls
+     *
+     * @return mixed
+     */
+    public function getStatsByStatus($statusCode) {
+
+        $timeStamps = $this->getTimeStamps();
+        foreach ($timeStamps as $timeStamp) {
+            $queryString = sprintf("SELECT COUNT(*) as count FROM sites WHERE timestamp = '%s' AND statusCode = '%d'", $timeStamp, $statusCode);
+            $rows = $this->pdo->query($queryString);
+            $row = $rows->fetchArray();
+
+            $numRows[$timeStamp] = $row['count'];
+        }
+
+        return $numRows;
+    }
+
 }
diff --git a/src/templates/results.twig b/src/templates/results.twig
@@ -4,6 +4,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 </head>
 <body>
+<a href="index.php">Home</a>
     <div class="container-fluid">
         <h1>Results by date and time of crawl</h1>
         <table class="table table-striped" style="text-align:center">

diff --git a/src/templates/results_status.twig b/src/templates/results_status.twig
@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+</head>
+<body>
+<a href="index.php">Home</a>
+    <div class="container-fluid">
+        <h1>Status codes by date and time of crawl</h1>
+        <table class="table table-striped" style="text-align:center">
+            <thead class="thead-dark">
+                <tr>
+                    <th>Crawl</th>
+                    {% for header in headers %}
+                    <th style="text-align:center" colspan="1">{{ header }}</th>
+                    {%  endfor %}
+                </tr>
+            </thead>
+
+            {%  for key, row in rows %}
+            <tr>
+                <td style="text-align:center">
+                    {{ key|date }}
+                </td>
+
+                {%  for column in row %}
+                <td style="text-align:center">
+                    {{ column }}
+                </td>
+                {% endfor %}
+            </tr>
+            {% endfor %}
+        </table>
+    </div>
+
+    <script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+</body>
+</html>