Bump ruff from 0.15.22 to 0.16.0

scraperwiki/__init__.py

@@ -3,31 +3,26 @@
 https://scraperwiki.com/docs/python/python_help_documentation/
 """
 
-from .utils import scrape, pdftoxml, status, swimport
-from . import utils
-from . import sql
+from . import sql, utils
+from .utils import pdftoxml, scrape, status, swimport
 
 # Compatibility
 sqlite = sql
 
 __all__ = [
-    "scrape",
     "pdftoxml",
+    "scrape",
+    "sql",
     "status",
     "swimport",
     "utils",
-    "sql",
 ]
 
 
 class Error(Exception):
     """All ScraperWiki exceptions are instances of this class
     (usually via a subclass)."""
 
-    pass
-
 
 class CPUTimeExceededError(Error):
     """CPU time limit exceeded."""
-
-    pass

scraperwiki/sql.py

@@ -1,11 +1,10 @@
-from collections.abc import Iterable, Mapping
-
 import atexit
 import datetime
-import time
 import os
 import re
+import time
 import warnings
+from collections.abc import Iterable, Mapping
 
 import alembic.ddl
 import sqlalchemy
@@ -14,7 +13,7 @@
     "SCRAPERWIKI_DATABASE_NAME", "sqlite:///scraperwiki.sqlite"
 )
 
-DATABASE_TIMEOUT = float(os.environ.get("SCRAPERWIKI_DATABASE_TIMEOUT", 300))
+DATABASE_TIMEOUT = float(os.environ.get("SCRAPERWIKI_DATABASE_TIMEOUT", "300"))
 SECONDS_BETWEEN_COMMIT = 2
 
 # The scraperwiki.sqlite.SqliteError exception
@@ -26,8 +25,6 @@ class Blob(bytes):
     Represents a blob as a string.
     """
 
-    pass
-
 
 PYTHON_SQLITE_TYPE_MAP = {
     str: sqlalchemy.types.Text,
@@ -201,9 +198,7 @@ def save(unique_keys, data, table_name="swdata"):
     insert = sqlalchemy.insert(_State.table).prefix_with("OR REPLACE")
     for row in data:
         if not isinstance(row, Mapping):
-            raise TypeError(
-                "Elements of data must be mappings, got {}".format(type(row))
-            )
+            raise TypeError(f"Elements of data must be mappings, got {type(row)}")
         fit_row(connection, row, unique_keys)
         connection.execute(insert.values(row))
     _State.check_last_committed()
@@ -261,12 +256,12 @@ def save_var(name, value):
     else:
         value_blob = str(value).encode("utf-8")
 
-    values = dict(
-        name=name,
-        value_blob=value_blob,
+    values = {
+        "name": name,
+        "value_blob": value_blob,
         # value_blob=Blob(value),
-        type=column_type.__visit_name__.lower(),
-    )
+        "type": column_type.__visit_name__.lower(),
+    }
 
     stmt = sqlalchemy.insert(vars_table).prefix_with("OR REPLACE").values(**values)
     connection.execute(stmt)

scraperwiki/utils.py

@@ -5,10 +5,10 @@
 
 import os
 import shutil
-import warnings
 import tempfile
 import urllib.parse
 import urllib.request
+import warnings
 
 
 def scrape(url, params=None, user_agent=None):
@@ -43,23 +43,22 @@ def pdftoxml(pdfdata, options=""):
             "install pdftohtml."
         )
         return None
-    pdffout = tempfile.NamedTemporaryFile(suffix=".pdf")
-    pdffout.write(pdfdata)
-    pdffout.flush()
-
-    xmlin = tempfile.NamedTemporaryFile(mode="r", suffix=".xml", encoding="utf-8")
-    tmpxml = xmlin.name  # "temph.xml"
-    cmd = 'pdftohtml -xml -nodrm -zoom 1.5 -enc UTF-8 -noframes {} "{}" "{}"'.format(
-        options, pdffout.name, os.path.splitext(tmpxml)[0]
-    )
-    # can't turn off output, so throw away even stderr yeuch
-    cmd = cmd + " >/dev/null 2>&1"
-    os.system(cmd)
-
-    pdffout.close()
-    # xmlfin = open(tmpxml)
-    xmldata = xmlin.read()
-    xmlin.close()
+
+    with (
+        tempfile.NamedTemporaryFile(suffix=".pdf") as pdffout,
+        tempfile.NamedTemporaryFile(mode="r", suffix=".xml", encoding="utf-8") as xmlin,
+    ):
+        pdffout.write(pdfdata)
+        pdffout.flush()
+
+        tmpxml = xmlin.name  # "temph.xml"
+        cmd = f'pdftohtml -xml -nodrm -zoom 1.5 -enc UTF-8 -noframes {options} "{pdffout.name}" "{os.path.splitext(tmpxml)[0]}"'
+        # can't turn off output, so throw away even stderr yeuch
+        cmd = cmd + " >/dev/null 2>&1"
+        os.system(cmd)
+
+        xmldata = xmlin.read()
+
     return xmldata
 
 
@@ -70,7 +69,6 @@ def status(type, message=None):
         DeprecationWarning,
         stacklevel=2,
     )
-    return
 
 
 def swimport(scrapername):

tests/test_scraperwiki.py

@@ -2,17 +2,14 @@
 import json
 import os
 import sqlite3
+import sys
 import warnings
-
-from subprocess import Popen, PIPE
+from subprocess import PIPE, Popen
 from textwrap import dedent
-
 from unittest import TestCase
 
 import scraperwiki
 
-import sys
-
 # scraperwiki.sql._State.echo = True
 DB_NAME = "scraperwiki.sqlite"
 
@@ -57,7 +54,7 @@ def test_save_no_warn(self):
         with warnings.catch_warnings():
             warnings.simplefilter("error")
             scraperwiki.sql.save(
-                ["id"], dict(id=4, tumble="weed"), table_name="warning_test"
+                ["id"], {"id": 4, "tumble": "weed"}, table_name="warning_test"
             )
 
 
@@ -85,8 +82,8 @@ def test_bytes(self):
         self.savegetvar(b"asodpa\x00\x22")
 
     def test_date(self):
-        date1 = datetime.datetime.now()
-        date2 = datetime.date.today()
+        date1 = datetime.datetime.now(tz=datetime.timezone.utc)
+        date2 = datetime.datetime.now(tz=datetime.timezone.utc).date()
         scraperwiki.sql.save_var("weird\u1234", date1)
         self.assertEqual(scraperwiki.sql.get_var("weird\u1234"), str(date1))
         scraperwiki.sql.save_var("weird\u1234", date2)
@@ -107,7 +104,7 @@ def test_get(self):
 
 class TestSaveVar(DBTestCase):
     def setUp(self):
-        super(TestSaveVar, self).setUp()
+        super().setUp()
         scraperwiki.sql.save_var("birthday\xfe", "\u1234November 30, 1888")
         connection = sqlite3.connect(DB_NAME)
         self.cursor = connection.cursor()
@@ -142,13 +139,13 @@ def save_and_check(self, dataIn, tableIn, dataOut, tableOut=None, twice=True):
         # Observe with pysqlite
         connection = sqlite3.connect(DB_NAME)
         cursor = connection.cursor()
-        cursor.execute("SELECT * FROM %s" % tableOut)
+        cursor.execute(f"SELECT * FROM {tableOut}")
         observed1 = cursor.fetchall()
         connection.close()
 
         if twice:
             # Observe using this module
-            observed2 = scraperwiki.sql.select("* FROM %s" % tableOut)
+            observed2 = scraperwiki.sql.select(f"* FROM {tableOut}")
 
             # Check
             expected1 = dataOut
@@ -200,7 +197,7 @@ def test_two(self):
         }
         try:
             self.assertDictEqual(observed, expected1)
-        except Exception:
+        except AssertionError:
             self.assertDictEqual(observed, expected2)
 
         # Uniqueness
@@ -241,8 +238,8 @@ def test_add_column(self):
             )
         stdout, stderr = process.communicate()
         assert process.returncode == 0
-        self.assertEqual(stdout, "".encode("utf-8"))
-        self.assertEqual(stderr, "".encode("utf-8"))
+        self.assertEqual(stdout, b"")
+        self.assertEqual(stderr, b"")
 
 
 class TestSave(SaveAndCheck):
@@ -280,12 +277,12 @@ def test_save_table_name(self):
         subsequent .save without table_name= uses the `swdata`
         table again.
         """
-        scraperwiki.sql.save(["id"], dict(id=1, stuff=1), table_name="sticky\u1234")
-        scraperwiki.sql.save(["id"], dict(id=2, stuff=2))
+        scraperwiki.sql.save(["id"], {"id": 1, "stuff": 1}, table_name="sticky\u1234")
+        scraperwiki.sql.save(["id"], {"id": 2, "stuff": 2})
         results = scraperwiki.sql.select("* FROM sticky\u1234")
         self.assertEqual(1, len(results))
         (row,) = results
-        self.assertDictEqual(dict(id=1, stuff=1), row)
+        self.assertDictEqual({"id": 1, "stuff": 1}, row)
 
     def test_lxml_string(self):
         """Save lxml string."""
@@ -301,9 +298,9 @@ def test_lxml_string(self):
         self.save_and_check({"text": s}, "lxml", [(str(s),)])
 
     def test_save_and_drop(self):
-        scraperwiki.sql.save([], dict(foo=7), table_name="dropper\xaa")
+        scraperwiki.sql.save([], {"foo": 7}, table_name="dropper\xaa")
         scraperwiki.sql.execute("DROP TABLE dropper\xaa")
-        scraperwiki.sql.save([], dict(foo=9), table_name="dropper\xaa")
+        scraperwiki.sql.save([], {"foo": 9}, table_name="dropper\xaa")
 
 
 class TestQuestionMark(DBTestCase):
@@ -333,13 +330,17 @@ class TestDateTime(DBTestCase):
     def rawdate(self, table="swdata", column="datetime"):
         connection = sqlite3.connect(DB_NAME)
         cursor = connection.cursor()
-        cursor.execute("SELECT {} FROM {}".format(column, table))
+        cursor.execute(f"SELECT {column} FROM {table}")
         rawdate = cursor.fetchall()[0][0]
         connection.close()
         return rawdate
 
     def test_save_date(self):
-        d = datetime.datetime.strptime("1991-03-30", "%Y-%m-%d").date()
+        d = (
+            datetime.datetime.strptime("1991-03-30", "%Y-%m-%d")
+            .replace(tzinfo=datetime.timezone.utc)
+            .date()
+        )
         with scraperwiki.sql.Transaction():
             scraperwiki.sql.save([], {"birthday\xaa": d})
 
@@ -355,24 +356,16 @@ def test_save_date(self):
         self.assertEqual(str(d), self.rawdate(column="birthday\xaa"))
 
     def test_save_datetime(self):
-        d = datetime.datetime.strptime("1990-03-30", "%Y-%m-%d")
+        d = datetime.datetime.strptime("1990-03-30", "%Y-%m-%d").replace(
+            tzinfo=datetime.timezone.utc
+        )
         with scraperwiki.sql.Transaction():
             scraperwiki.sql.save([], {"birthday": d}, table_name="datetimetest")
 
-            exemplar = str(d)
-            # SQLAlchemy appears to convert with extended precision.
-            exemplar += ".000000"
-
-            self.assertEqual(
-                [{"birthday": exemplar}], scraperwiki.sql.select("* FROM datetimetest")
-            )
-            self.assertDictEqual(
-                {"keys": ["birthday"], "data": [(exemplar,)]},
-                scraperwiki.sql.execute("SELECT * FROM datetimetest"),
-            )
+        exemplar = d.strftime("%Y-%m-%d %H:%M:%S.000000")
 
         self.assertEqual(
-            exemplar, self.rawdate(table="datetimetest", column="birthday")
+            [{"birthday": exemplar}], scraperwiki.sql.select("* FROM datetimetest")
         )
 
 
@@ -401,19 +394,19 @@ def setUp(self):
         self.sw = __import__("scraperwiki")
 
     def test_import_scraperwiki_root(self):
-        self.sw.scrape
+        _ = self.sw.scrape
 
     def test_import_scraperwiki_sqlite(self):
-        self.sw.sqlite
+        _ = self.sw.sqlite
 
     def test_import_scraperwiki_sql(self):
-        self.sw.sql
+        _ = self.sw.sql
 
     def test_import_scraperwiki_status(self):
-        self.sw.status
+        _ = self.sw.status
 
     def test_import_scraperwiki_utils(self):
-        self.sw.utils
+        _ = self.sw.utils
 
     def test_import_scraperwiki_special_utils(self):
-        self.sw.pdftoxml
+        _ = self.sw.pdftoxml