Merge pull request #254 from piwheels/208-requires-python

Implement Requires-Python

piwheels/importer/__init__.py

@@ -186,6 +186,7 @@ def print_state(state):
         logging.warning('  ABI tag:      %s', wheel.abi_tag)
         logging.warning('  Python tag:   %s', wheel.py_version_tag)
         logging.warning('  Platform tag: %s', wheel.platform_tag)
+        logging.warning('  Requires-Python: %s', wheel.requires_python)
 
 
 def do_import(config, packages, state):

piwheels/initdb/sql/create_piwheels.sql

@@ -190,6 +190,7 @@ CREATE TABLE files (
     py_version_tag      VARCHAR(100) NOT NULL,
     abi_tag             VARCHAR(100) NOT NULL,
     platform_tag        VARCHAR(100) NOT NULL,
+    requires_python     VARCHAR(100) NULL,
 
     CONSTRAINT files_pk PRIMARY KEY (filename),
     CONSTRAINT files_builds_fk FOREIGN KEY (build_id)
@@ -1115,7 +1116,8 @@ BEGIN
         package_version_tag,
         py_version_tag,
         abi_tag,
-        platform_tag
+        platform_tag,
+        requires_python
     )
         SELECT
             b.filename,
@@ -1126,7 +1128,8 @@ BEGIN
             b.package_version_tag,
             b.py_version_tag,
             b.abi_tag,
-            b.platform_tag
+            b.platform_tag,
+            b.requires_python
         FROM
             UNNEST(build_files) AS b;
     INSERT INTO dependencies (
@@ -1603,6 +1606,7 @@ CREATE FUNCTION get_project_files(pkg TEXT)
         filesize files.filesize%TYPE,
         filehash files.filehash%TYPE,
         yanked versions.yanked%TYPE,
+        requires_python files.requires_python%TYPE,
         dependencies VARCHAR ARRAY
     )
     LANGUAGE SQL
@@ -1619,6 +1623,7 @@ AS $sql$
         f.filesize,
         f.filehash,
         v.yanked,
+        f.requires_python,
         ARRAY_AGG(d.dependency)
             FILTER (WHERE d.dependency IS NOT NULL) AS dependencies
     FROM
@@ -1633,7 +1638,7 @@ AS $sql$
     AND b.package = pkg
     GROUP BY (
         version, platform_tag, builder_abi, file_abi_tag, filename, filesize,
-        filehash, yanked
+        filehash, yanked, requires_python
     );
 $sql$;
 

piwheels/initdb/sql/update_piwheels_0.18_to_0.19.sql

@@ -1,5 +1,8 @@
 UPDATE configuration SET version = '0.19';
 
+ALTER TABLE files
+    ADD COLUMN requires_python VARCHAR(100) NULL;
+
 DROP FUNCTION get_project_versions(TEXT);
 CREATE FUNCTION get_project_versions(pkg TEXT)
     RETURNS TABLE(
@@ -69,6 +72,7 @@ CREATE FUNCTION get_project_files(pkg TEXT)
         filesize files.filesize%TYPE,
         filehash files.filehash%TYPE,
         yanked versions.yanked%TYPE,
+        requires_python files.requires_python%TYPE,
         dependencies VARCHAR ARRAY
     )
     LANGUAGE SQL
@@ -85,6 +89,7 @@ AS $sql$
         f.filesize,
         f.filehash,
         v.yanked,
+        f.requires_python,
         ARRAY_AGG(d.dependency)
             FILTER (WHERE d.dependency IS NOT NULL) AS dependencies
     FROM
@@ -99,9 +104,109 @@ AS $sql$
     AND b.package = pkg
     GROUP BY (
         version, platform_tag, builder_abi, file_abi_tag, filename, filesize,
-        filehash, yanked
+        filehash, yanked, requires_python
     );
 $sql$;
 
 REVOKE ALL ON FUNCTION get_project_files(TEXT) FROM PUBLIC;
 GRANT EXECUTE ON FUNCTION get_project_files(TEXT) TO {username};
+
+DROP FUNCTION log_build_success(
+    TEXT, TEXT, INTEGER, INTERVAL, TEXT, TEXT, files ARRAY, dependencies ARRAY
+);
+
+CREATE FUNCTION log_build_success(
+    package TEXT,
+    version TEXT,
+    built_by INTEGER,
+    duration INTERVAL,
+    abi_tag TEXT,
+    output TEXT,
+    build_files files ARRAY,
+    build_deps dependencies ARRAY
+)
+    RETURNS INTEGER
+    LANGUAGE plpgsql
+    CALLED ON NULL INPUT
+    SECURITY DEFINER
+    SET search_path = public, pg_temp
+AS $sql$
+DECLARE
+    new_build_id INTEGER;
+BEGIN
+    IF ARRAY_LENGTH(build_files, 1) = 0 THEN
+        RAISE EXCEPTION integrity_constraint_violation
+            USING MESSAGE = 'Successful build must include at least one file';
+    END IF;
+    INSERT INTO builds (
+            package,
+            version,
+            built_by,
+            duration,
+            status,
+            abi_tag
+        )
+        VALUES (
+            package,
+            version,
+            built_by,
+            duration,
+            TRUE,
+            abi_tag
+        )
+        RETURNING build_id
+        INTO new_build_id;
+    INSERT INTO output (build_id, output) VALUES (new_build_id, output);
+    -- We delete the existing entries from files rather than using INSERT..ON
+    -- CONFLICT UPDATE because we need to delete dependencies associated with
+    -- those files too. This is considerably simpler than a multi-layered
+    -- upsert across tables.
+    DELETE FROM files f
+        USING UNNEST(build_files) AS b
+        WHERE f.filename = b.filename;
+    INSERT INTO files (
+        filename,
+        build_id,
+        filesize,
+        filehash,
+        package_tag,
+        package_version_tag,
+        py_version_tag,
+        abi_tag,
+        platform_tag,
+        requires_python
+    )
+        SELECT
+            b.filename,
+            new_build_id,
+            b.filesize,
+            b.filehash,
+            b.package_tag,
+            b.package_version_tag,
+            b.py_version_tag,
+            b.abi_tag,
+            b.platform_tag,
+            b.requires_python
+        FROM
+            UNNEST(build_files) AS b;
+    INSERT INTO dependencies (
+        filename,
+        tool,
+        dependency
+    )
+        SELECT
+            d.filename,
+            d.tool,
+            d.dependency
+        FROM
+            UNNEST(build_deps) AS d;
+    RETURN new_build_id;
+END;
+$sql$;
+
+REVOKE ALL ON FUNCTION log_build_success(
+    TEXT, TEXT, INTEGER, INTERVAL, TEXT, TEXT, files ARRAY, dependencies ARRAY
+    ) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION log_build_success(
+    TEXT, TEXT, INTEGER, INTERVAL, TEXT, TEXT, files ARRAY, dependencies ARRAY
+    ) TO {username};

piwheels/master/db.py

@@ -55,7 +55,7 @@
     'version', 'yanked', 'released', 'skip', 'builds_succeeded', 'builds_failed'))
 ProjectFilesRow = namedtuple('ProjectFilesRow', (
     'version', 'platform_tag', 'builder_abi', 'file_abi_tag', 'filename',
-    'filesize', 'filehash', 'yanked', 'dependencies'))
+    'filesize', 'filehash', 'yanked', 'requires_python', 'dependencies'))
 RewritePendingRow = namedtuple('RewritePendingRow', (
     'package', 'added_at', 'command'))
 
@@ -419,6 +419,7 @@ def log_build(self, build):
                             file.py_version_tag,
                             file.abi_tag,
                             file.platform_tag,
+                            file.requires_python,
                         )
                         for file in build.files.values()],
                         [(
@@ -594,7 +595,8 @@ def get_project_files(self, package):
                 ProjectFilesRow(*row)
                 for row in self._conn.execute(
                     "SELECT version, platform_tag, builder_abi, file_abi_tag, "
-                    "filename, filesize, filehash, yanked, dependencies "
+                    "filename, filesize, filehash, yanked, requires_python, "
+                    "dependencies "
                     "FROM get_project_files(%s)", (package,)
                 )
             ]

piwheels/master/mr_chase.py

@@ -34,7 +34,7 @@
 """
 
 from .. import const, protocols, transport, tasks
-from ..states import BuildState, FileState
+from ..states import BuildState
 from .the_oracle import DbClient
 from .file_juggler import FsClient
 from .slave_driver import build_armv6l_hack

piwheels/master/slave_driver.py

@@ -545,10 +545,11 @@ def build_armv6l_hack(build):
     """
     for file in list(build.files.values()):
         if file.platform_tag == 'linux_armv7l':
-            arm7_name = file.filename
-            arm6_name = arm7_name[:-16] + 'linux_armv6l.whl'
-            if arm6_name not in build.files:
-                build.files[arm6_name] = FileState(
-                    arm6_name, file.filesize, file.filehash, file.package_tag,
+            armv7_name = file.filename
+            armv6_name = armv7_name[:-16] + 'linux_armv6l.whl'
+            if armv6_name not in build.files:
+                build.files[armv6_name] = FileState(
+                    armv6_name, file.filesize, file.filehash, file.package_tag,
                     file.package_version_tag, file.py_version_tag,
-                    file.abi_tag, 'linux_armv6l', file.dependencies, True)
+                    file.abi_tag, 'linux_armv6l', file.requires_python,
+                    file.dependencies, True)

piwheels/master/templates/simple_package.pt

@@ -6,6 +6,6 @@
 <body>
 <h1>Links for ${package}</h1>
 
-<span tal:repeat="row files" tal:omit-tag="True"><a href="${row.filename}#sha256=${row.filehash}" tal:attributes="data-yanked '' if row.yanked else None">${row.filename}</a><br></span>
+<span tal:repeat="row files" tal:omit-tag="True"><a href="${row.filename}#sha256=${row.filehash}" tal:attributes="data-yanked '' if row.yanked else None; data-requires-python row.requires_python if row.requires_python else None">${row.filename}</a><br></span>
 </body>
 </html>

piwheels/master/the_scribe.py

@@ -160,9 +160,9 @@ def setup_output_path(self):
         for filename in pkg_resources.resource_listdir(__name__, 'templates'):
             if filename in startup_templates:
                 source = self.templates[filename](
-                        layout=self.templates['layout']['layout'],
-                        page=filename.replace('.pt', '')
-                    )
+                    layout=self.templates['layout']['layout'],
+                    page=filename.replace('.pt', '')
+                )
                 with AtomicReplaceFile(
                         (self.output_path / filename).with_suffix('.html'),
                         encoding='utf-8') as f:
@@ -570,6 +570,7 @@ def get_releases(self, versions, files):
                     'builder_abi': f.builder_abi,
                     'file_abi_tag': f.file_abi_tag,
                     'platform': f.platform_tag,
+                    'requires_python': f.requires_python,
                     'apt_dependencies': sorted(f.dependencies) if f.dependencies else [],
                 }
                 for f in files

piwheels/protocols.py

@@ -166,6 +166,7 @@ def __reversed__(self):
     str,            # py_version_tag
     str,            # abi_tag
     str,            # platform_tag
+    Any(str, None), # requires_python
     {str: [str]},   # dependencies
     # NOTE: the optional transferred field is never included. It is effectively
     # internal to whatever is tracking the file state

piwheels/slave/builder.py

@@ -97,7 +97,8 @@ def as_message(self):
             self.py_version_tag,
             self.abi_tag,
             self.platform_tag,
-            self.dependencies
+            self.requires_python,
+            self.dependencies,
         )
 
     @property
@@ -186,6 +187,14 @@ def open(self):
         """
         return self.wheel_file.open('rb')
 
+    @property
+    def requires_python(self):
+        """
+        Return the contents of the ``Requires-Python`` specification from the
+        wheel metadata.
+        """
+        return self.metadata['Requires-Python']
+
     @property
     def dependencies(self):
         """

piwheels/states.py

@@ -128,6 +128,9 @@ class FileState:
         The platform tag extracted from the filename (last "-" separated
         component).
 
+    :param str requires_python:
+        The ``Requires-Python`` specification for the file.
+
     :param set dependencies:
         The set of dependencies that are required to use this particular
         wheel.
@@ -138,7 +141,7 @@ class FileState:
     """
     def __init__(self, filename, filesize, filehash, package_tag,
                  package_version_tag, py_version_tag, abi_tag, platform_tag,
-                 dependencies, transferred=False):
+                 requires_python, dependencies, transferred=False):
         self._filename = filename
         self._filesize = filesize
         self._filehash = filehash
@@ -147,6 +150,7 @@ def __init__(self, filename, filesize, filehash, package_tag,
         self._py_version_tag = py_version_tag
         self._abi_tag = abi_tag
         self._platform_tag = platform_tag
+        self._requires_python = requires_python
         self._dependencies = dependencies
         self._transferred = transferred
 
@@ -166,7 +170,7 @@ def from_message(cls, value):
         return cls(*value)
 
     def __len__(self):
-        return 10
+        return 11
 
     def __getitem__(self, index):
         return (
@@ -178,6 +182,7 @@ def __getitem__(self, index):
             self._py_version_tag,
             self._abi_tag,
             self._platform_tag,
+            self._requires_python,
             self._dependencies,
             self._transferred,
         )[index]
@@ -231,6 +236,10 @@ def platform_tag(self):
     def dependencies(self):
         return self._dependencies
 
+    @property
+    def requires_python(self):
+        return self._requires_python
+
     @property
     def transferred(self):
         return self._transferred