Merge pull request #122 from waveform80/stored-procs-ftw

Stored Procedures!
piwheels · Mar 4, 2019 · c70288e · c70288e
2 parents 2a12432 + 491ed04
commit c70288e
Show file tree

Hide file tree

Showing 18 changed files with 1,121 additions and 320 deletions.
diff --git a/piwheels/initdb/sql/create_piwheels.sql b/piwheels/initdb/sql/create_piwheels.sql
diff --git a/piwheels/initdb/sql/update_piwheels_0.13_to_0.14.sql b/piwheels/initdb/sql/update_piwheels_0.13_to_0.14.sql
diff --git a/piwheels/initdb/sql/update_piwheels_0.5_to_0.6.sql b/piwheels/initdb/sql/update_piwheels_0.5_to_0.6.sql
@@ -26,17 +26,53 @@ GRANT SELECT ON build_abis TO {username};
 
 CREATE INDEX builds_pkgverid ON builds(build_id, package, version);
 
+DROP VIEW builds_pending;
+DROP VIEW statistics;
+
+CREATE TABLE files2 (
+    filename            VARCHAR(255) NOT NULL,
+    build_id            INTEGER NOT NULL,
+    filesize            INTEGER NOT NULL,
+    filehash            CHAR(64) NOT NULL,
+    package_tag         VARCHAR(200) NOT NULL,
+    package_version_tag VARCHAR(200) NOT NULL,
+    py_version_tag      VARCHAR(100) NOT NULL,
+    abi_tag             VARCHAR(100) NOT NULL,
+    platform_tag        VARCHAR(100) NOT NULL
+);
+INSERT INTO files2 (
+    filename,
+    build_id,
+    filesize,
+    filehash,
+    package_tag,
+    package_version_tag,
+    py_version_tag,
+    abi_tag,
+    platform_tag
+)
+SELECT
+    filename,
+    build_id,
+    filesize,
+    filehash,
+    regexp_replace(filename, '([^-]*)-.*$', E'\\1'),
+    package_version_tag,
+    py_version_tag,
+    abi_tag,
+    platform_tag
+FROM files;
+DROP TABLE files;
+ALTER TABLE files2 RENAME TO files;
 ALTER TABLE files
-    ADD COLUMN package_tag VARCHAR(200) NOT NULL DEFAULT '';
-UPDATE files
-    SET package_tag = regexp_replace(filename, '([^-]*)-.*$', E'\\1');
-ALTER TABLE files
-    ALTER COLUMN package_tag DROP DEFAULT;
-DROP INDEX files_size;
+    ADD CONSTRAINT files_pk PRIMARY KEY (filename),
+    ADD CONSTRAINT files_builds_fk FOREIGN KEY (build_id)
+        REFERENCES builds (build_id) ON DELETE CASCADE;
+CREATE INDEX files_builds ON files(build_id);
 CREATE INDEX files_size ON files(platform_tag, filesize) WHERE platform_tag <> 'linux_armv6l';
 CREATE INDEX files_abi ON files(build_id, abi_tag) WHERE abi_tag <> 'none';
+GRANT SELECT ON files TO {username};
 
-DROP VIEW builds_pending;
 CREATE VIEW builds_pending AS
     SELECT
         v.package,
@@ -73,7 +109,6 @@ CREATE VIEW builds_pending AS
 
 GRANT SELECT ON builds_pending TO {username};
 
-DROP VIEW statistics;
 CREATE VIEW statistics AS
     WITH package_stats AS (
         SELECT COUNT(*) AS packages_count

diff --git a/piwheels/master/big_brother.py b/piwheels/master/big_brother.py
@@ -60,9 +60,7 @@ class BigBrother(PauseableTask):
     def __init__(self, config):
         super().__init__(config)
         self.stats = {
-            'packages_count':        0,
             'packages_built':        0,
-            'versions_count':        0,
             'builds_count':          0,
             'builds_last_hour':      0,
             'builds_success':        0,
@@ -118,7 +116,6 @@ def loop(self):
             # Rename a couple of columns
             rec['builds_last_hour'] = rec.pop('builds_count_last_hour')
             rec['builds_success'] = rec.pop('builds_count_success')
-            rec.pop('versions_tried', None)
             self.stats.update(rec)
             self.web_queue.send_msg('HOME', self.stats)
             self.status_queue.send_msg('STATS', self.stats)

diff --git a/piwheels/master/cloud_gazer.py b/piwheels/master/cloud_gazer.py
@@ -81,7 +81,7 @@ def loop(self):
             if version is not None:
                 if self.db.add_new_package_version(
                         package, version, timestamp,
-                        skip=None if source else 'binary only'):
+                        skip='' if source else 'binary only'):
                     self.logger.info(
                         'added package %s version %s', package, version)
                     if not source:
@@ -91,7 +91,7 @@ def loop(self):
                     self.web_queue.send_msg('PKGPROJ', package)
                 elif source and self.db.get_version_skip(
                         package, version) == 'binary only':
-                    self.db.skip_package_version(package, version, None)
+                    self.db.skip_package_version(package, version, '')
                     self.logger.info(
                         'enabled package %s version %s', package, version)
                     self.web_queue.send_msg('PKGPROJ', package)