Merge pull request #124 from waveform80/architect-upgrade

Architect upgrade
piwheels · Mar 4, 2019 · bc4a3f7 · bc4a3f7
2 parents c70288e + 8b9d56e
commit bc4a3f7
Show file tree

Hide file tree

Showing 12 changed files with 294 additions and 170 deletions.
diff --git a/piwheels/initdb/sql/create_piwheels.sql b/piwheels/initdb/sql/create_piwheels.sql
@@ -291,53 +291,79 @@ GRANT SELECT ON versions_detail TO {username};
 -------------------------------------------------------------------------------
 
 CREATE VIEW builds_pending AS
+-- Finally, because I can't write this in order due to postgres' annoying
+-- materialization of CTEs, the same set as below but augmented with a per-ABI
+-- build queue position, based on version release date, primarily for the
+-- purposes of filtering
 SELECT
+    abi_tag,
+    ROW_NUMBER() OVER (PARTITION BY abi_tag ORDER BY released) AS position,
     package,
-    version,
-    MIN(abi_tag) AS abi_tag
-FROM (
-    SELECT
-        v.package,
-        v.version,
-        b.abi_tag
-    FROM
-        packages AS p
-        JOIN versions AS v ON v.package = p.package
-        CROSS JOIN build_abis AS b
-    WHERE
-        v.skip = ''
-        AND p.skip = ''
-
-    EXCEPT ALL
-
+    version
+FROM
     (
+        -- The set of package versions against each ABI for which they haven't
+        -- been attempted and for which no covering "none" ABI wheel exists
         SELECT
-            b.package,
-            b.version,
-            v.abi_tag
-        FROM
-            builds AS b
-            JOIN files AS f ON b.build_id = f.build_id
-            CROSS JOIN build_abis AS v
-        WHERE f.abi_tag = 'none'
-
-        UNION ALL
-
-        SELECT
-            b.package,
-            b.version,
-            COALESCE(f.abi_tag, b.abi_tag) AS abi_tag
+            q.package,
+            q.version,
+            v.released,
+            MIN(q.abi_tag) AS abi_tag
         FROM
-            builds AS b
-            LEFT JOIN files AS f ON b.build_id = f.build_id
-        WHERE
-            f.build_id IS NULL
-            OR f.abi_tag <> 'none'
-    )
-) AS t
-GROUP BY
-    package,
-    version;
+            (
+                -- The set of package versions X build ABIs that we want to
+                -- exist once the queue is complete
+                SELECT
+                    v.package,
+                    v.version,
+                    b.abi_tag
+                FROM
+                    packages AS p
+                    JOIN versions AS v ON v.package = p.package
+                    CROSS JOIN build_abis AS b
+                WHERE
+                    v.skip = ''
+                    AND p.skip = ''
+
+                EXCEPT ALL
+
+                (
+                    -- The set of package versions that successfully produced
+                    -- wheels with ABI "none", and which therefore count as
+                    -- all build ABIs
+                    SELECT
+                        b.package,
+                        b.version,
+                        v.abi_tag
+                    FROM
+                        builds AS b
+                        JOIN files AS f ON b.build_id = f.build_id
+                        CROSS JOIN build_abis AS v
+                    WHERE f.abi_tag = 'none'
+
+                    UNION ALL
+
+                    -- The set of package versions that successfully produced a
+                    -- wheel with a single ABI (abi_tag <> 'none') or which
+                    -- were attempted but failed (build_id IS NULL)
+                    SELECT
+                        b.package,
+                        b.version,
+                        COALESCE(f.abi_tag, b.abi_tag) AS abi_tag
+                    FROM
+                        builds AS b
+                        LEFT JOIN files AS f ON b.build_id = f.build_id
+                    WHERE
+                        f.build_id IS NULL
+                        OR f.abi_tag <> 'none'
+                )
+            ) AS q
+            JOIN versions v ON q.package = v.package AND q.version = v.version
+        GROUP BY
+            q.package,
+            q.version,
+            v.released
+    ) AS t;
 
 GRANT SELECT ON builds_pending TO {username};
 

diff --git a/piwheels/initdb/sql/update_piwheels_0.13_to_0.14.sql b/piwheels/initdb/sql/update_piwheels_0.13_to_0.14.sql
@@ -59,53 +59,79 @@ GRANT SELECT ON versions_detail TO {username};
 
 DROP VIEW builds_pending;
 CREATE VIEW builds_pending AS
+-- Finally, because I can't write this in order due to postgres' annoying
+-- materialization of CTEs, the same set as below but augmented with a per-ABI
+-- build queue position, based on version release date, primarily for the
+-- purposes of filtering
 SELECT
+    abi_tag,
+    ROW_NUMBER() OVER (PARTITION BY abi_tag ORDER BY released) AS position,
     package,
-    version,
-    MIN(abi_tag) AS abi_tag
-FROM (
-    SELECT
-        v.package,
-        v.version,
-        b.abi_tag
-    FROM
-        packages AS p
-        JOIN versions AS v ON v.package = p.package
-        CROSS JOIN build_abis AS b
-    WHERE
-        v.skip = ''
-        AND p.skip = ''
-
-    EXCEPT ALL
-
+    version
+FROM
     (
+        -- The set of package versions against each ABI for which they haven't
+        -- been attempted and for which no covering "none" ABI wheel exists
         SELECT
-            b.package,
-            b.version,
-            v.abi_tag
+            q.package,
+            q.version,
+            v.released,
+            MIN(q.abi_tag) AS abi_tag
         FROM
-            builds AS b
-            JOIN files AS f ON b.build_id = f.build_id
-            CROSS JOIN build_abis AS v
-        WHERE f.abi_tag = 'none'
+            (
+                -- The set of package versions X build ABIs that we want to
+                -- exist once the queue is complete
+                SELECT
+                    v.package,
+                    v.version,
+                    b.abi_tag
+                FROM
+                    packages AS p
+                    JOIN versions AS v ON v.package = p.package
+                    CROSS JOIN build_abis AS b
+                WHERE
+                    v.skip = ''
+                    AND p.skip = ''
 
-        UNION ALL
+                EXCEPT ALL
 
-        SELECT
-            b.package,
-            b.version,
-            COALESCE(f.abi_tag, b.abi_tag) AS abi_tag
-        FROM
-            builds AS b
-            LEFT JOIN files AS f ON b.build_id = f.build_id
-        WHERE
-            f.build_id IS NULL
-            OR f.abi_tag <> 'none'
-    )
-) AS t
-GROUP BY
-    package,
-    version;
+                (
+                    -- The set of package versions that successfully produced
+                    -- wheels with ABI "none", and which therefore count as
+                    -- all build ABIs
+                    SELECT
+                        b.package,
+                        b.version,
+                        v.abi_tag
+                    FROM
+                        builds AS b
+                        JOIN files AS f ON b.build_id = f.build_id
+                        CROSS JOIN build_abis AS v
+                    WHERE f.abi_tag = 'none'
+
+                    UNION ALL
+
+                    -- The set of package versions that successfully produced a
+                    -- wheel with a single ABI (abi_tag <> 'none') or which
+                    -- were attempted but failed (build_id IS NULL)
+                    SELECT
+                        b.package,
+                        b.version,
+                        COALESCE(f.abi_tag, b.abi_tag) AS abi_tag
+                    FROM
+                        builds AS b
+                        LEFT JOIN files AS f ON b.build_id = f.build_id
+                    WHERE
+                        f.build_id IS NULL
+                        OR f.abi_tag <> 'none'
+                )
+            ) AS q
+            JOIN versions v ON q.package = v.package AND q.version = v.version
+        GROUP BY
+            q.package,
+            q.version,
+            v.released
+    ) AS t;
 GRANT SELECT ON builds_pending TO {username};
 
 DROP VIEW statistics;

diff --git a/piwheels/master/db.py b/piwheels/master/db.py
@@ -39,7 +39,7 @@
 import warnings
 from datetime import datetime, timedelta, timezone
 from itertools import chain, groupby
-from operator import itemgetter
+from operator import itemgetter, attrgetter
 from collections import namedtuple
 
 from sqlalchemy import MetaData, Table, select, create_engine, func
@@ -310,17 +310,27 @@ def get_all_package_versions(self):
 
     def get_build_queue(self):
         """
-        Returns a generator covering the entire builds_pending view; streaming
-        results are activated for this query as it's more important to get the
-        first result quickly than it is to retrieve the entire set.
+        Returns a mapping of ABI tags to an ordered list of up to 1000 package
+        version tuples which currently need building for that ABI.
         """
         # NOTE: This method is not exposed on TheOracle as it is only used by
         # TheArchitect task
         with self._conn.begin():
-            for row in self._conn.\
-                    execution_options(stream_results=True).\
-                    execute(self._builds_pending.select()):
-                yield row
+            return {
+                abi_tag: [
+                    (row.package, row.version)
+                    for row in rows
+                ]
+                for abi_tag, rows in groupby(
+                    self._conn.execution_options(stream_results=True).\
+                    execute(
+                        self._builds_pending.select().
+                        where(self._builds_pending.c.position <= 1000).
+                        order_by(self._builds_pending.c.abi_tag,
+                                 self._builds_pending.c.position)
+                    ), key=attrgetter('abi_tag')
+                )
+            }
 
     def get_statistics(self):
         """

diff --git a/piwheels/master/file_juggler.py b/piwheels/master/file_juggler.py
@@ -104,7 +104,7 @@ def __init__(self, config):
         file_queue.bind(config.file_queue)
         fs_queue = self.ctx.socket(
             zmq.REP, protocol=protocols.file_juggler_fs)
-        fs_queue.hwm = 1
+        fs_queue.hwm = 10
         fs_queue.bind(config.fs_queue)
         self.stats_queue = self.ctx.socket(
             zmq.PUSH, protocol=reversed(protocols.big_brother))
@@ -339,7 +339,7 @@ def __init__(self, config):
         self.ctx = transport.Context.instance()
         self.fs_queue = self.ctx.socket(
             zmq.REQ, protocol=reversed(protocols.file_juggler_fs))
-        self.fs_queue.hwm = 1
+        self.fs_queue.hwm = 10
         self.fs_queue.connect(config.fs_queue)
 
     def close(self):