Merge pull request LMFDB#6030 from roed314/psycodict_updates

Update backend code

lmfdb/backend/base.py

@@ -122,18 +122,14 @@ def jsonb_idx(cols, cols_type):
     "total",
     "important",
     "include_nones",
-    "table_description",
-    "col_description",
 )
 _meta_tables_cols_notrequired = (
     "count_cutoff",
     "stats_valid",
     "total",
     "important",
     "include_nones",
-    "table_description",
-    "col_description",
-)  # defaults: 1000, true, 0, false, false, "", {}
+)  # defaults: 1000, true, 0, false, false
 _meta_tables_types = dict(zip(_meta_tables_cols, (
     "text",
     "jsonb",
@@ -146,8 +142,6 @@ def jsonb_idx(cols, cols_type):
     "bigint",
     "boolean",
     "boolean",
-    "text",
-    "jsonb",
 )))
 _meta_tables_jsonb_idx = jsonb_idx(_meta_tables_cols, _meta_tables_types)
 
@@ -222,12 +216,13 @@ def __init__(self, loggername, db):
         self.slow_cutoff = logging_options["slowcutoff"]
         self.logger = l = logging.getLogger(loggername)
         l.propagate = False
+        # we only want 2 handlers
+        l.handlers = []
         l.setLevel(logging_options.get('loglevel', logging.INFO))
-        fhandler = logging.FileHandler(logging_options["slowlogfile"])
         formatter = logging.Formatter("%(asctime)s - %(message)s")
-        filt = QueryLogFilter()
+        fhandler = logging.FileHandler(logging_options["slowlogfile"])
         fhandler.setFormatter(formatter)
-        fhandler.addFilter(filt)
+        fhandler.addFilter(QueryLogFilter())
         l.addHandler(fhandler)
         shandler = logging.StreamHandler()
         shandler.setFormatter(formatter)

lmfdb/backend/database.py

@@ -44,11 +44,11 @@ def setup_connection(conn):
     register_json(conn, loads=Json.loads)
     try:
         from sage.all import Integer, RealNumber
+        from .encoding import RealEncoder, LmfdbRealLiteral
     except ImportError:
         pass
     else:
         register_adapter(Integer, AsIs)
-        from .encoding import RealEncoder, LmfdbRealLiteral
         register_adapter(RealNumber, RealEncoder)
         register_adapter(LmfdbRealLiteral, RealEncoder)
 
@@ -489,22 +489,22 @@ def _create_meta_tables_hist(self):
                 "(name text, sort jsonb, count_cutoff smallint DEFAULT 1000, "
                 "id_ordered boolean, out_of_order boolean, has_extras boolean, "
                 "stats_valid boolean DEFAULT true, label_col text, total bigint, "
-                "include_nones boolean, table_description text, col_description jsonb, version integer)"
+                "include_nones boolean, version integer)"
             ))
             version = 0
 
             # copy data from meta_tables
             rows = self._execute(SQL(
-                "SELECT name, sort, id_ordered, out_of_order, has_extras, label_col, total, include_nones, table_description, col_description FROM meta_tables "
+                "SELECT name, sort, id_ordered, out_of_order, has_extras, label_col, total, include_nones FROM meta_tables "
             ))
 
             for row in rows:
                 self._execute(
                     SQL(
                         "INSERT INTO meta_tables_hist "
                         "(name, sort, id_ordered, out_of_order, has_extras, label_col, "
-                        "total, include_nones, table_description, col_description, version) "
-                        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
+                        "total, include_nones, version) "
+                        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
                     ),
                     row + (version,),
                 )
@@ -778,8 +778,8 @@ def process_columns(coldict, colorder):
             # FIXME use global constants ?
             inserter = SQL(
                 "INSERT INTO meta_tables "
-                "(name, sort, id_ordered, out_of_order, has_extras, label_col, table_description, col_description) "
-                "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
+                "(name, sort, id_ordered, out_of_order, has_extras, label_col) "
+                "VALUES (%s, %s, %s, %s, %s, %s)"
             )
             self._execute(
                 inserter,
@@ -790,11 +790,9 @@ def process_columns(coldict, colorder):
                     not id_ordered,
                     extra_columns is not None,
                     label_col,
-                    table_description,
-                    Json(col_description),
                 ],
             )
-        self.__dict__[name] = self._search_table_class_(
+        new_table = self._search_table_class_(
             self,
             name,
             label_col,
@@ -804,6 +802,9 @@ def process_columns(coldict, colorder):
             has_extras=(extra_columns is not None),
             total=0,
         )
+        new_table.description(table_description)
+        new_table.column_description(description=col_description)
+        self.__dict__[name] = new_table
         self.tablenames.append(name)
         self.tablenames.sort()
         self.log_db_change(
@@ -1117,8 +1118,6 @@ def reload_all(
                     if len(rows) != 1:
                         raise RuntimeError("Expected only one row in {0}")
                     meta = dict(zip(_meta_tables_cols, rows[0]))
-                    import ast
-                    meta["col_description"] = ast.literal_eval(meta["col_description"])
                     assert meta["name"] == tablename
 
                     with search_table_file.open("r") as F:
@@ -1141,7 +1140,7 @@ def reload_all(
                                 extra_columns[typ].append(name)
                     # the rest of the meta arguments will be replaced on the reload_all
                     # We use force_description=False so that beta and prod can be out-of-sync with respect to columns and/or descriptions
-                    self.create_table(tablename, search_columns, None, table_description=meta["table_description"], col_description=meta["col_description"], extra_columns=extra_columns, force_description=False)
+                    self.create_table(tablename, search_columns, None, extra_columns=extra_columns, force_description=False)
 
             for tablename in self.tablenames:
                 included = []

lmfdb/backend/statstable.py

@@ -1581,6 +1581,7 @@ def refresh_stats(self, total=True, reset_None_to_1=False, suffix=""):
             if total:
                 # Refresh total in meta_tables
                 self.total = self._slow_count({}, suffix=suffix, extra=False)
+            self.refresh_null_counts(suffix=suffix)
             self.logger.info("Refreshed statistics in %.3f secs" % (time.time() - t0))
 
     def status(self, reset_None_to_1=False):

lmfdb/backend/table.py

@@ -1171,7 +1171,6 @@ def drop_tmp():
                         if not self._table_exists(table + "_tmp"):
                             self._clone(table, table + "_tmp")
                 self.stats.refresh_stats(suffix=suffix)
-                self.stats.refresh_null_counts(suffix=suffix)
             if not inplace:
                 swapped_tables = (
                     [self.search_table]
@@ -1786,7 +1785,7 @@ def reload(
                     if table not in tables:
                         tables.append(table)
 
-            if countsfile:
+            if self.stats.counts in tables:
                 # create index on counts table
                 self._create_counts_indexes(suffix=suffix)
 
@@ -2217,73 +2216,33 @@ def get_label(self):
 
     def description(self, table_description=None):
         """
-        Return or set the description string for this table in meta_tables
+        This stub defines the API for getting and setting the table description.
+        In the LMFDB, this is implemented using the knowl table, but we do nothing by default.
 
         INPUT:
 
-        - ``table_description`` -- if provided, set the description to this value.  If not, return the current description.
+        - ``table_description`` -- if provided, set the description to this value.
+          If not, return the current description.
         """
-        if table_description is None:
-            selecter = SQL("SELECT table_description FROM meta_tables WHERE name = %s")
-            desc = list(self._execute(selecter, [self.search_table]))
-            if desc and desc[0]:
-                return desc[0]
-            else:
-                return "(table description not yet updated on this server)"
-        else:
-            assert isinstance(table_description, str)
-            modifier = SQL("UPDATE meta_tables SET table_description = %s WHERE name = %s")
-            self._execute(modifier, [table_description, self.search_table])
+        pass
 
     def column_description(self, col=None, description=None, drop=False):
         """
-        Set the description for a column in meta_tables.
+        This stub defines the API for getting, setting and deleting column descriptions.
+        In the LMFDB, this is implemented using the knowl table, but we do nothing by default.
 
         INPUT:
 
-        - ``col`` -- the name of the column.  If None, ``description`` should be a dictionary with keys equal to the column names.
+        - ``col`` -- the name of the column.  If None, ``description`` should be a dictionary
+          with keys equal to the column names.
 
-        - ``description`` -- if provided, set the column description to this value.  If not, return the current description.
+        - ``description`` -- if provided, set the column description to this value.
+          If not, return the current description.
 
-        - ``drop`` -- if ``True``, delete the column from the description dictionary in preparation for dropping the column.
+        - ``drop`` -- if ``True``, delete the column from the description dictionary in
+          preparation for dropping the column.
         """
-        allcols = self.search_cols + self.extra_cols
-        # Get the current column description
-        selecter = SQL("SELECT col_description FROM meta_tables WHERE name = %s")
-        cur = self._execute(selecter, [self.search_table])
-        current = cur.fetchone()[0]
-
-        if not drop and description is None:
-            # We want to allow the set of columns to be out of date temporarily, on prod for example
-            if col is None:
-                for col in allcols:
-                    if col not in current:
-                        current[col] = "(description not yet updated on this server)"
-                return current
-            return current.get(col, "(description not yet updated on this server)")
-        else:
-            if not (drop or col is None or col in allcols):
-                raise ValueError("%s is not a column of this table" % col)
-            if drop:
-                if col is None:
-                    raise ValueError("Must specify column name to drop")
-                try:
-                    del current[col]
-                except KeyError:
-                    # column was already not present for some reason
-                    return
-            elif col is None:
-                assert isinstance(description, dict)
-                for col in description:
-                    if col not in allcols:
-                        raise ValueError("%s is not a column of this table" % col)
-                    assert isinstance(description[col], str)
-                    current[col] = description[col]
-            else:
-                assert isinstance(description, str)
-                current[col] = description
-            modifier = SQL("UPDATE meta_tables SET col_description = %s WHERE name = %s")
-            self._execute(modifier, [Json(current), self.search_table])
+        pass
 
     def add_column(self, name, datatype, description=None, extra=False, label=False, force_description=False):
         """

lmfdb/backend/utils.py

@@ -171,7 +171,7 @@ class QueryLogFilter():
     """
 
     def filter(self, record):
-        if record.pathname.startswith("db_backend.py"):
+        if record.pathname.endswith("base.py"):
             return 1
         else:
             return 0

lmfdb/knowledge/knowl.py

@@ -23,6 +23,7 @@
 top_knowl_re = re.compile(r"(.*)\.top$")
 comment_knowl_re = re.compile(r"(.*)\.(\d+)\.comment$")
 coldesc_knowl_re = re.compile(r"columns.([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)")
+tabledesc_knowl_re = re.compile(r"tables.([A-Za-z0-9_]+)")
 bottom_knowl_re = re.compile(r"(.*)\.bottom$")
 url_from_knowl = [
     (re.compile(r'g2c\.(\d+\.[a-z]+\.\d+\.\d+)'), 'Genus2Curve/Q/{0}', 'Genus 2 curve {0}'),
@@ -86,6 +87,9 @@ def extract_typ(kid):
     m = coldesc_knowl_re.match(kid)
     if m:
         return 2, m.group(1), m.group(2)
+    m = tabledesc_knowl_re.match(kid)
+    if m:
+        return 2, None, m.group(1)
     m = top_knowl_re.match(kid)
     if m:
         prelabel = m.group(1)
@@ -289,8 +293,8 @@ def save(self, knowl, who, most_recent=None, minor=False):
         else:
             typ, source, name = extract_typ(knowl.id)
         links = extract_links(knowl.content)
-        if typ == 2: # column description
-            defines = [knowl.id.split(".")[-1]]
+        if typ == 2: # column or table description
+            defines = [name]
         else:
             defines = extract_defines(knowl.content)
         # id, authors, cat, content, last_author, timestamp, title, status, type, links, defines, source, source_name
@@ -354,6 +358,31 @@ def drop_column(self, table, col):
         kwl = Knowl(kid, data=self.get_knowl(kid, beta=True))
         self.delete(kwl)
 
+    def get_table_description(self, table):
+        fields = ['id'] + self._default_fields
+        selecter = SQL("SELECT {0} FROM (SELECT DISTINCT ON (id) {0} FROM kwl_knowls WHERE id = %s AND type = %s AND status >= %s ORDER BY id, timestamp) knowls ORDER BY id").format(SQL(", ").join(map(Identifier, fields)))
+        rec = self._execute(selecter, [f"tables.{table}", 2, 0]).fetchone()
+        if rec:
+            return Knowl(rec[0], data=dict(zip(fields, rec)))
+
+    def set_table_description(self, table, description):
+        uid = db.login()
+        kid = f"tables.{table}"
+        data = {
+            'content': description,
+            'defines': table,
+        }
+        kwl = Knowl(kid, data=data)
+        old = self.get_knowl(kid, beta=True)
+        if old is None:
+            old = {'authors': []}
+        self.save(kwl, uid, most_recent=old)
+
+    def drop_table(self, table):
+        kid = f"tables.{table}"
+        kwl = Knowl(kid, data=self.get_knowl(kid, beta=True))
+        self.delete(kwl)
+
     def delete(self, knowl):
         """deletes this knowl from the db. This is effected by setting the status to -2 on all copies of the knowl"""
         updator = SQL("UPDATE kwl_knowls SET status=%s WHERE id=%s")
@@ -815,11 +844,19 @@ def __init__(self, ID, template_kwargs=None, data=None, editing=False, showing=F
         if self.type == 2:
             pieces = ID.split(".")
             # Ignore the title passed in
-            self.title = f"Column {pieces[2]} of table {pieces[1]}"
-            if pieces[1] in db.tablenames:
-                self.coltype = db[pieces[1]].col_type.get(pieces[2], "DEFUNCT")
-            else:
-                self.coltype = "DEFUNCT"
+            if len(pieces) == 3:
+                # Column
+                self.title = f"Column {pieces[2]} of table {pieces[1]}"
+                if pieces[1] in db.tablenames:
+                    self.coltype = db[pieces[1]].col_type.get(pieces[2], "DEFUNCT")
+                else:
+                    self.coltype = "DEFUNCT"
+            elif len(pieces) == 2:
+                # Table
+                self.title = f"Table {pieces[1]}"
+                self.coltype = None
+                if pieces[1] not in db.tablenames:
+                    self.title += " (DEFUNCT)"
         #self.reviewer = data.get('reviewer') # Not returned by get_knowl by default
         #self.review_timestamp = data.get('review_timestamp') # Not returned by get_knowl by default
 

lmfdb/knowledge/main.py

@@ -350,14 +350,10 @@ def edit(ID):
     elif knowl.type == 0:
         title = "Edit Knowl '%s'" % ID
     elif knowl.type == 2:
-        pieces = ID.split(".")
-        title = f"Edit column information for '{pieces[2]}' in '{pieces[1]}'"
-        knowl.title = f"Column {pieces[2]} of table {pieces[1]}"
-        from lmfdb import db
-        if pieces[1] in db.tablenames:
-            knowl.coltype = db[pieces[1]].col_type.get(pieces[2], "DEFUNCT")
+        if knowl.source:
+            title = f"Edit column information for '{knowl.source_name}' in '{knowl.source}'"
         else:
-            knowl.coltype = "DEFUNCT"
+            title = f"Edit description for '{knowl.source_name}'"
     else:
         ann_type = 'Top' if knowl.type == 1 else 'Bottom'
         title = 'Edit %s Knowl for <a href="/%s">%s</a>' % (ann_type, knowl.source, knowl.source_name)

lmfdb/knowledge/templates/knowl-show.html

@@ -4,7 +4,7 @@
 {% from "knowl-defs.html" import knowlbar with context %}
 {{ knowlbar(show_kid=True) }}
 
-{% if k.type == 2 %}
+{% if k.type == 2 and k.coltype %}
   <p>
     Postgres column type: {{ k.coltype }}
   </p>