Merge pull request #248 from piwheels/241-project-page

New project page layout and JSON API

piwheels/__init__.py

@@ -67,7 +67,7 @@
 # pylint: disable=bad-whitespace
 
 __project__      = 'piwheels'
-__version__      = '0.18'
+__version__      = '0.19'
 __keywords__     = ['raspberrypi', 'pip', 'wheels']
 __author__       = 'Ben Nuttall'
 __author_email__ = 'ben@bennuttall.com'

piwheels/initdb/sql/create_piwheels.sql

@@ -20,7 +20,7 @@ CREATE TABLE configuration (
     CONSTRAINT config_pk PRIMARY KEY (id)
 );
 
-INSERT INTO configuration(id, version) VALUES (1, '0.18');
+INSERT INTO configuration(id, version) VALUES (1, '0.19');
 GRANT SELECT ON configuration TO {username};
 
 -- packages
@@ -1529,10 +1529,11 @@ GRANT EXECUTE ON FUNCTION get_version_files(TEXT, TEXT) TO {username};
 CREATE FUNCTION get_project_versions(pkg TEXT)
     RETURNS TABLE(
         version versions.version%TYPE,
-        skipped versions.skip%TYPE,
+        yanked BOOLEAN,
+        released TIMESTAMP WITH TIME ZONE,
+        skip versions.skip%TYPE,
         builds_succeeded TEXT,
-        builds_failed TEXT,
-        yanked BOOLEAN
+        builds_failed TEXT
     )
     LANGUAGE SQL
     RETURNS NULL ON NULL INPUT
@@ -1541,21 +1542,52 @@ CREATE FUNCTION get_project_versions(pkg TEXT)
 AS $sql$
     SELECT
         v.version,
-        COALESCE(NULLIF(v.skip, ''), p.skip) AS skipped,
-        COALESCE(STRING_AGG(DISTINCT b.abi_tag, ', ') FILTER (WHERE b.status), '') AS builds_succeeded,
-        COALESCE(STRING_AGG(DISTINCT b.abi_tag, ', ') FILTER (WHERE NOT b.status), '') AS builds_failed,
-        v.yanked
+        v.yanked,
+        v.released AT TIME ZONE 'UTC',
+        COALESCE(NULLIF(v.skip, ''), p.skip) AS skip_msg,
+        COALESCE(STRING_AGG(DISTINCT b.abi_tag, ',') FILTER (WHERE b.status), '') AS builds_succeeded,
+        COALESCE(STRING_AGG(DISTINCT b.abi_tag, ',') FILTER (WHERE NOT b.status), '') AS builds_failed
     FROM
         packages p
         JOIN versions v USING (package)
         LEFT JOIN builds b USING (package, version)
     WHERE v.package = pkg
-    GROUP BY version, skipped, yanked;
+    GROUP BY version, skip_msg, released, yanked;
 $sql$;
 
 REVOKE ALL ON FUNCTION get_project_versions(TEXT) FROM PUBLIC;
 GRANT EXECUTE ON FUNCTION get_project_versions(TEXT) TO {username};
 
+-- get_file_apt_dependencies(filename)
+-------------------------------------------------------------------------------
+-- Returns apt dependencies registered against the specified *filename*,
+-- excluding those listed in preinstalled_apt_packages with a matching ABI tag.
+-------------------------------------------------------------------------------
+
+CREATE FUNCTION get_file_apt_dependencies(fn VARCHAR)
+    RETURNS TABLE(
+        dependency dependencies.dependency%TYPE
+    )
+    LANGUAGE SQL
+    RETURNS NULL ON NULL INPUT
+    SECURITY DEFINER
+    SET search_path = public, pg_temp
+AS $sql$
+    SELECT dependency
+        FROM dependencies
+        WHERE filename = fn
+        AND tool = 'apt'
+    EXCEPT ALL
+    SELECT apt_package
+        FROM preinstalled_apt_packages p
+        JOIN files f
+        ON p.abi_tag = f.abi_tag
+        WHERE f.filename = fn;
+$sql$;
+
+REVOKE ALL ON FUNCTION get_file_apt_dependencies(VARCHAR) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION get_file_apt_dependencies(VARCHAR) TO {username};
+
 -- get_project_files(package)
 -------------------------------------------------------------------------------
 -- Return details about all files built for the given *package*.
@@ -1564,11 +1596,14 @@ GRANT EXECUTE ON FUNCTION get_project_versions(TEXT) TO {username};
 CREATE FUNCTION get_project_files(pkg TEXT)
     RETURNS TABLE(
         version builds.version%TYPE,
-        abi_tag files.abi_tag%TYPE,
+        platform_tag files.platform_tag%TYPE,
+        builder_abi builds.abi_tag%TYPE,
+        file_abi_tag files.abi_tag%TYPE,
         filename files.filename%TYPE,
         filesize files.filesize%TYPE,
         filehash files.filehash%TYPE,
-        yanked versions.yanked%TYPE
+        yanked versions.yanked%TYPE,
+        dependencies VARCHAR ARRAY
     )
     LANGUAGE SQL
     RETURNS NULL ON NULL INPUT
@@ -1577,52 +1612,34 @@ CREATE FUNCTION get_project_files(pkg TEXT)
 AS $sql$
     SELECT
         b.version,
-        f.abi_tag,
+        f.platform_tag,
+        b.abi_tag AS builder_abi,
+        f.abi_tag AS file_abi_tag,
         f.filename,
         f.filesize,
         f.filehash,
-        v.yanked
+        v.yanked,
+        ARRAY_AGG(d.dependency)
+            FILTER (WHERE d.dependency IS NOT NULL) AS dependencies
     FROM
         builds b
         JOIN files f USING (build_id)
         JOIN versions v USING (package, version)
+        LEFT JOIN LATERAL (
+            SELECT f.filename, d.dependency
+            FROM get_file_apt_dependencies(f.filename) AS d
+        ) d USING (filename)
     WHERE b.status
-    AND b.package = pkg;
+    AND b.package = pkg
+    GROUP BY (
+        version, platform_tag, builder_abi, file_abi_tag, filename, filesize,
+        filehash, yanked
+    );
 $sql$;
 
 REVOKE ALL ON FUNCTION get_project_files(TEXT) FROM PUBLIC;
 GRANT EXECUTE ON FUNCTION get_project_files(TEXT) TO {username};
 
--- get_file_apt_dependencies(filename)
--------------------------------------------------------------------------------
--- Returns apt dependencies registered against the specified *filename*,
--- excluding those listed in preinstalled_apt_packages with a matching ABI tag.
--------------------------------------------------------------------------------
-
-CREATE FUNCTION get_file_apt_dependencies(fn TEXT)
-    RETURNS TABLE(
-        dependency dependencies.dependency%TYPE
-    )
-    LANGUAGE SQL
-    RETURNS NULL ON NULL INPUT
-    SECURITY DEFINER
-    SET search_path = public, pg_temp
-AS $sql$
-    SELECT dependency
-        FROM dependencies
-        WHERE filename = fn
-        AND tool = 'apt'
-    EXCEPT ALL
-    SELECT apt_package
-        FROM preinstalled_apt_packages p
-        JOIN files f
-        ON p.abi_tag = f.abi_tag
-        WHERE f.filename = fn;
-$sql$;
-
-REVOKE ALL ON FUNCTION get_file_apt_dependencies(TEXT) FROM PUBLIC;
-GRANT EXECUTE ON FUNCTION get_file_apt_dependencies(TEXT) TO {username};
-
 -- save_rewrites_pending(...)
 -------------------------------------------------------------------------------
 -- Saves the state of the_secretary task's internal buffer in the

piwheels/initdb/sql/update_piwheels_0.18_to_0.19.sql

@@ -0,0 +1,107 @@
+UPDATE configuration SET version = '0.19';
+
+DROP FUNCTION get_project_versions(TEXT);
+CREATE FUNCTION get_project_versions(pkg TEXT)
+    RETURNS TABLE(
+        version versions.version%TYPE,
+        yanked BOOLEAN,
+        released TIMESTAMP WITH TIME ZONE,
+        skip versions.skip%TYPE,
+        builds_succeeded TEXT,
+        builds_failed TEXT
+    )
+    LANGUAGE SQL
+    RETURNS NULL ON NULL INPUT
+    SECURITY DEFINER
+    SET search_path = public, pg_temp
+AS $sql$
+    SELECT
+        v.version,
+        v.yanked,
+        v.released AT TIME ZONE 'UTC',
+        COALESCE(NULLIF(v.skip, ''), p.skip) AS skip_msg,
+        COALESCE(STRING_AGG(DISTINCT b.abi_tag, ',') FILTER (WHERE b.status), '') AS builds_succeeded,
+        COALESCE(STRING_AGG(DISTINCT b.abi_tag, ',') FILTER (WHERE NOT b.status), '') AS builds_failed
+    FROM
+        packages p
+        JOIN versions v USING (package)
+        LEFT JOIN builds b USING (package, version)
+    WHERE v.package = pkg
+    GROUP BY version, skip_msg, released, yanked;
+$sql$;
+
+REVOKE ALL ON FUNCTION get_project_versions(TEXT) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION get_project_versions(TEXT) TO {username};
+
+DROP FUNCTION get_file_apt_dependencies(fn TEXT);
+CREATE FUNCTION get_file_apt_dependencies(fn VARCHAR)
+    RETURNS TABLE(
+        dependency dependencies.dependency%TYPE
+    )
+    LANGUAGE SQL
+    RETURNS NULL ON NULL INPUT
+    SECURITY DEFINER
+    SET search_path = public, pg_temp
+AS $sql$
+    SELECT dependency
+        FROM dependencies
+        WHERE filename = fn
+        AND tool = 'apt'
+    EXCEPT ALL
+    SELECT apt_package
+        FROM preinstalled_apt_packages p
+        JOIN files f
+        ON p.abi_tag = f.abi_tag
+        WHERE f.filename = fn;
+$sql$;
+
+REVOKE ALL ON FUNCTION get_file_apt_dependencies(VARCHAR) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION get_file_apt_dependencies(VARCHAR) TO {username};
+
+DROP FUNCTION get_project_files(TEXT);
+CREATE FUNCTION get_project_files(pkg TEXT)
+    RETURNS TABLE(
+        version builds.version%TYPE,
+        platform_tag files.platform_tag%TYPE,
+        builder_abi builds.abi_tag%TYPE,
+        file_abi_tag files.abi_tag%TYPE,
+        filename files.filename%TYPE,
+        filesize files.filesize%TYPE,
+        filehash files.filehash%TYPE,
+        yanked versions.yanked%TYPE,
+        dependencies VARCHAR ARRAY
+    )
+    LANGUAGE SQL
+    RETURNS NULL ON NULL INPUT
+    SECURITY DEFINER
+    SET search_path = public, pg_temp
+AS $sql$
+    SELECT
+        b.version,
+        f.platform_tag,
+        b.abi_tag AS builder_abi,
+        f.abi_tag AS file_abi_tag,
+        f.filename,
+        f.filesize,
+        f.filehash,
+        v.yanked,
+        ARRAY_AGG(d.dependency)
+            FILTER (WHERE d.dependency IS NOT NULL) AS dependencies
+    FROM
+        builds b
+        JOIN files f USING (build_id)
+        JOIN versions v USING (package, version)
+        LEFT JOIN LATERAL (
+            SELECT f.filename, d.dependency
+            FROM get_file_apt_dependencies(f.filename) AS d
+        ) d USING (filename)
+    WHERE b.status
+    AND b.package = pkg
+    GROUP BY (
+        version, platform_tag, builder_abi, file_abi_tag, filename, filesize,
+        filehash, yanked
+    );
+$sql$;
+
+REVOKE ALL ON FUNCTION get_project_files(TEXT) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION get_project_files(TEXT) TO {username};

piwheels/master/db.py

@@ -52,9 +52,10 @@
 
 
 ProjectVersionsRow = namedtuple('ProjectVersionsRow', (
-    'version', 'skipped', 'builds_succeeded', 'builds_failed', 'yanked'))
+    'version', 'yanked', 'released', 'skip', 'builds_succeeded', 'builds_failed'))
 ProjectFilesRow = namedtuple('ProjectFilesRow', (
-    'version', 'abi_tag', 'filename', 'filesize', 'filehash', 'yanked'))
+    'version', 'platform_tag', 'builder_abi', 'file_abi_tag', 'filename',
+    'filesize', 'filehash', 'yanked', 'dependencies'))
 RewritePendingRow = namedtuple('RewritePendingRow', (
     'package', 'added_at', 'command'))
 
@@ -442,14 +443,17 @@ def log_build(self, build):
                     )).scalar()
             build.logged(build_id)
 
-    def get_build_abis(self):
+    def get_build_abis(self, exclude_skipped=False):
         """
-        Return the set of ABIs that the master should attempt to build.
+        Return a set of ABIs. If **exclude_skipped** is ``False``, return all
+        ABIs from the build_abis table, otherwise return only active ABIs (not
+        skipped).
         """
         with self._conn.begin():
             return {
                 rec.abi_tag
                 for rec in self._conn.execute(self._build_abis.select())
+                if rec.skip == '' or not exclude_skipped
             }
 
     def get_pypi_serial(self):
@@ -574,7 +578,8 @@ def get_project_versions(self, package):
             return [
                 ProjectVersionsRow(*row)
                 for row in self._conn.execute(
-                    "SELECT version, skipped, builds_succeeded, builds_failed, yanked "
+                    "SELECT version, yanked, released, skip, builds_succeeded, "
+                    "builds_failed "
                     "FROM get_project_versions(%s)", (package,)
                 )
             ]
@@ -588,7 +593,8 @@ def get_project_files(self, package):
             return [
                 ProjectFilesRow(*row)
                 for row in self._conn.execute(
-                    "SELECT version, abi_tag, filename, filesize, filehash, yanked "
+                    "SELECT version, platform_tag, builder_abi, file_abi_tag, "
+                    "filename, filesize, filehash, yanked, dependencies "
                     "FROM get_project_files(%s)", (package,)
                 )
             ]
@@ -604,19 +610,6 @@ def get_version_skip(self, package, version):
                 where(self._versions.c.version == version)
             )
 
-    def get_file_apt_dependencies(self, filename):
-        """
-        Returns a set of the apt dependencies for the specified *filename*.
-        """
-        with self._conn.begin():
-            return {
-                row['dependency']
-                for row in self._conn.execute(
-                    "SELECT dependency "
-                    "FROM get_file_apt_dependencies(%s)", (filename,)
-                )
-            }
-
     def delete_build(self, package, version):
         """
         Remove all builds for the specified package and version, along with