Upgrade SQLFluff to latest

.github/workflows/linter.yml

@@ -31,7 +31,7 @@ jobs:
         run: |
           echo "VALIDATE_ALL_CODEBASE=false" >> $GITHUB_ENV
       - name: Lint Code Base
-        uses: github/super-linter@v4.10.1
+        uses: github/super-linter@v5.0.0
         #uses: docker://github/super-linter:v4.7.1
         env:
           DEFAULT_BRANCH: main

.github/workflows/test_website.yml

@@ -40,7 +40,7 @@ jobs:
     - name: Use more complete checks for generated HTML linting
       run: cp -f .github/linters/.htmlhintrc_morechecks .github/linters/.htmlhintrc
     - name: Lint Generated HTML
-      uses: github/super-linter@v4.10.1
+      uses: github/super-linter@v5.0.0
       env:
         DEFAULT_BRANCH: main
         FILTER_REGEX_INCLUDE: src/static/html/.*

sql/.sqlfluff

@@ -1,65 +1,30 @@
 [sqlfluff]
-# verbose is an integer (0-2) indicating the level of log output
-verbose = 0
-# Turn off color formatting of output
-nocolor = False
-# Supported dialects https://docs.sqlfluff.com/en/stable/dialects.html
-# Or run 'sqlfluff dialects'
 dialect = bigquery
-# One of [raw|jinja|python|placeholder]
-templater = jinja
-# Comma separated list of rules to check, default to all
-rules = all
 # Comma separated list of rules to exclude, or None
-exclude_rules = L011,L014,L015,L016,L020,L022,L026,L027,L028,L029,L030,L031,L032,L034,L035,L036,L037,L042,L043,L051,L060,L071
-# L011 - We don't always alias tables with AS ("FROM table1 AS tb1" instead of "FROM table1 tb1"). Do for columns but not for tables.
-# L014 - Unquoted identifiers (e.g. column names) will be mixed case so don't enforce case
-# L015 - Sometimes clearer to include brackets for complex COUNT(DISTINCT) cases
-# L016 - We allow longer lines as some of our queries are complex. Maybe should limit in future?
-# L020 - Asks for unique table aliases meaning it complains if selecting from two 2021_07_01 tables as implicit alias is table name (not fully qualified) so same.
-# L022 - CTEs may be chained and do not require a blank line separator, only the last one.
-# L026 - BigQuery uses STRUCTS which can look like incorrect table references
-# L027 - Asks for qualified columns for ambiguous ones, but we not qualify our columns, and they are not really ambiguous (or BigQuery would complain)
-# L028 - Insists on references in column names even if not ambiguous. Bit OTT.
-# L029 - Avoids keywords as identifiers but we use this a lot (e.g. AS count, AS max...etc.)
-# L030 - Function names will be mixed case so don't enforce case
-# L031 - Avoid aliases in from and join - why?
-# L032 - Uses joins instead of USING - why?
-# L034 - Insists on wildcards (*) in certain SELECT order - why?
-# L035 - Do not use ELSE NULL as redundant. But it's clearer!?
-# L036 - Select targets should be on new lines but sub clauses don't always obey this. Maybe revisit in future?
-# L037 - if using DESC in one ORDER BY column, then insist on ASC/DESC for all.
-# L042 - Use CTEs instead of subqueries. We don't use this consistently and big rewrite to do that.
-# L043 - Use coalesce instead of case if you can. But it's clearer!?
-# L051 - INNER JOIN must be fully qualified. Probably should use this but not our style.
-# L060 - Use COALESCE instead of IFNULL or NVL. We think ISNULL is clearer.
-# L071 - Turn off until https://github.com/sqlfluff/sqlfluff/issues/4188 is addressed
-# The depth to recursively parse to (0 for unlimited)
-recurse = 0
-# Below controls SQLFluff output, see max_line_length for SQL output
-output_line_length = 80
-# Number of passes to run before admitting defeat
-runaway_limit = 10
-# Ignore errors by category (one or more of the following, separated by commas: lexing,linting,parsing,templating)
-ignore = None
-# Ignore linting errors found within sections of code coming directly from
-# templated code (e.g. from within Jinja curly braces. Note that it does not
-# ignore errors from literal code found within template loops.
-ignore_templated_areas = True
-# can either be autodetect or a valid encoding e.g. utf-8, utf-8-sig
-encoding = autodetect
-# Ignore inline overrides (e.g. to test if still required)
-disable_noqa = False
-# Comma separated list of file extensions to lint
-# NB: This config will only apply in the root folder
-sql_file_exts = .sql,.sql.j2,.dml,.ddl
-# Allow fix to run on files, even if they contain parsing errors
-# Note altering this is NOT RECOMMENDED as can corrupt SQL
-fix_even_unparsable = False
-# Very large files can make the parser effectively hang.
-# This limit skips files over a certain character length
-# and warns the user what has happened.
-# Set this to 0 to disable.
+exclude_rules = AL01,AL04,AL07,AM03,AM05,CP02,CP03,CV02,LT05,LT08,LT09,RF01,RF02,RF03,RF04,ST01,ST02,ST05,ST06,ST07,ST08
+# AL01 - We don't always alias tables with AS ("FROM table1 AS tb1" instead of "FROM table1 tb1"). Do for columns but not for tables.
+# AL04 - Asks for unique table aliases meaning it complains if selecting from two 2021_07_01 tables as implicit alias is table name (not fully qualified) so same.
+# AL07 - Avoid aliases in from and join - why?
+# AM03 - if using DESC in one ORDER BY column, then insist on ASC/DESC for all.
+# AM05 - INNER JOIN must be fully qualified. Probably should use this but not our style.
+# CP02 - Unquoted identifiers (e.g. column names) will be mixed case so don't enforce case
+# CP03 - Function names will be mixed case so don't enforce case
+# CV02 - Use COALESCE instead of IFNULL or NVL. We think ISNULL is clearer.
+# LT05 - We allow longer lines as some of our queries are complex. Maybe should limit in future?
+# LT08 - CTEs may be chained and do not require a blank line separator, only the last one.
+# LT09 - Select targets should be on new lines but sub clauses don't always obey this. Maybe revisit in future?
+# RF01 - BigQuery uses STRUCTS which can look like incorrect table references
+# RF02 - Asks for qualified columns for ambiguous ones, but we not qualify our columns, and they are not really ambiguous (or BigQuery would complain)
+# RF03 - Insists on references in column names even if not ambiguous. Bit OTT.
+# RF04 - Avoids keywords as identifiers but we use this a lot (e.g. AS count, AS max...etc.)
+# ST01 - Do not use ELSE NULL as redundant. But it's clearer!?
+# ST02 - Use coalesce instead of case if you can. But it's clearer!?
+# ST05 - Use CTEs instead of subqueries. We don't use this consistently and big rewrite to do that.
+# ST06 - Insists on wildcards (*) in certain SELECT order - why?
+# ST07 - Uses joins instead of USING - why?
+# ST08 - Sometimes clearer to include brackets for complex COUNT(DISTINCT) cases
+
+
 large_file_skip_byte_limit = 40000
 # CPU processes to use while linting.
 # If positive, just implies number of processes.
@@ -68,202 +33,22 @@ large_file_skip_byte_limit = 40000
 processes = -1
 
 [sqlfluff:indentation]
-# See https://docs.sqlfluff.com/en/stable/indentation.html
-indented_joins = False
-indented_ctes = False
+# See https://docs.sqlfluff.com/en/stable/layout.html#configuring-indent-locations
 indented_using_on = False
-indented_on_contents = True
-template_blocks_indent = True
-
-# Layout configuration
-# See https://docs.sqlfluff.com/en/stable/layout.html#configuring-layout-and-spacing
-[sqlfluff:layout:type:comma]
-spacing_before = touch
-line_position = trailing
+tab_space_size = 2
 
 [sqlfluff:layout:type:binary_operator]
 line_position = trailing
 
-[sqlfluff:layout:type:statement_terminator]
-spacing_before = touch
-line_position = trailing
-
-[sqlfluff:layout:type:end_of_file]
-spacing_before = touch
-
-[sqlfluff:layout:type:set_operator]
-line_position = alone:strict
-
-[sqlfluff:layout:type:start_bracket]
-spacing_after = touch
-
-[sqlfluff:layout:type:end_bracket]
-spacing_before = touch
-
-[sqlfluff:layout:type:start_square_bracket]
-spacing_after = touch
-
-[sqlfluff:layout:type:end_square_bracket]
-spacing_before = touch
-
-[sqlfluff:layout:type:casting_operator]
-spacing_before = touch
-spacing_after = touch
-
-[sqlfluff:layout:type:comparison_operator]
-spacing_within = touch
-line_position = leading
-
-[sqlfluff:layout:type:object_reference]
-spacing_within = inline
-
-[sqlfluff:layout:type:comment]
-spacing_before = any
-spacing_after = any
-
-# Template loop tokens shouldn't dictate spacing around them.
-[sqlfluff:layout:type:template_loop]
-spacing_before = any
-spacing_after = any
-
-[sqlfluff:templater]
-unwrap_wrapped_queries = True
-
-[sqlfluff:templater:jinja]
-apply_dbt_builtins = True
-
-# Some rules can be configured directly from the config common to other rules
-[sqlfluff:rules]
-tab_space_size = 2
-max_line_length = 80
-indent_unit = space
-allow_scalar = True
-single_table_references = consistent
-unquoted_identifiers_policy = all
-
-# Some rules have their own specific config.
-[sqlfluff:rules:L003]
-hanging_indents = True
-
-[sqlfluff:rules:L010]
-# Keywords
+[sqlfluff:rules:capitalisation.keywords]
 capitalisation_policy = upper
-# Comma separated list of words to ignore for this rule
-ignore_words = None
-ignore_words_regex = None
-
-[sqlfluff:rules:L011]
-# Aliasing preference for tables
-aliasing = explicit
 
-[sqlfluff:rules:L012]
-# Aliasing preference for columns
-aliasing = explicit
-
-[sqlfluff:rules:L014]
-# Unquoted identifiers
-extended_capitalisation_policy = consistent
-# Comma separated list of words to ignore for this rule
-ignore_words = None
-ignore_words_regex = None
-
-[sqlfluff:rules:L016]
-# Line length
-ignore_comment_lines = False
-ignore_comment_clauses = False
-
-[sqlfluff:rules:L027]
-# Comma separated list of words to ignore for this rule
-ignore_words = None
-ignore_words_regex = None
-
-[sqlfluff:rules:L026]
-# References must be in FROM clause
-# Disabled for some dialects (e.g. bigquery)
-force_enable = False
-
-[sqlfluff:rules:L028]
-# References must be consistently used
-# Disabled for some dialects (e.g. bigquery)
-force_enable = False
-
-[sqlfluff:rules:L029]
-# Keywords should not be used as identifiers.
-unquoted_identifiers_policy = aliases
-quoted_identifiers_policy = none
-# Comma separated list of words to ignore for this rule
-ignore_words = None
-ignore_words_regex = None
-
-[sqlfluff:rules:L030]
-# Function names
-extended_capitalisation_policy = consistent
-# Comma separated list of words to ignore for this rule
-ignore_words = None
-ignore_words_regex = None
-
-[sqlfluff:rules:L031]
-# Avoid table aliases in from clauses and join conditions.
-# Disabled for some dialects (e.g. bigquery)
-force_enable = False
-
-[sqlfluff:rules:L036]
-wildcard_policy = single
-
-[sqlfluff:rules:L038]
-# Trailing commas
-select_clause_trailing_comma = forbid
-
-[sqlfluff:rules:L040]
-# Null & Boolean Literals
-capitalisation_policy = consistent
-# Comma separated list of words to ignore for this rule
-ignore_words = None
-ignore_words_regex = None
-
-[sqlfluff:rules:L042]
-# By default, allow subqueries in from clauses, but not join clauses
-forbid_subquery_in = join
-
-[sqlfluff:rules:L047]
-# Consistent syntax to count all rows
-prefer_count_1 = False
-prefer_count_0 = True
-
-[sqlfluff:rules:L051]
-# Fully qualify JOIN clause
-fully_qualify_join_types = inner
-
-[sqlfluff:rules:L052]
-# Semi-colon formatting approach
-multiline_newline = False
-require_final_semicolon = False
-
-[sqlfluff:rules:L054]
-# GROUP BY/ORDER BY column references
-group_by_and_order_by_style = consistent
-
-[sqlfluff:rules:L057]
-## Special characters in identifiers
-unquoted_identifiers_policy = all
-quoted_identifiers_policy = all
-allow_space_in_identifier = False
-additional_allowed_characters = "-."
-ignore_words = None
-ignore_words_regex = None
-
-[sqlfluff:rules:L059]
-# Policy on quoted and unquoted identifiers
-prefer_quoted_identifiers = False
-ignore_words = None
-ignore_words_regex = None
-force_enable = False
+[sqlfluff:rules:capitalisation.types]
+extended_capitalisation_policy = upper
 
-[sqlfluff:rules:L062]
-# Comma separated list of blocked words that should not be used
-blocked_words = None
+[sqlfluff:rules:convention.blocked_words]
 # Regex of blocked SQL that should not be used.
-# Can be overridden with `-- noqa: L062` for those chapters using secondary pages
+# Can be overridden with `-- noqa: CV09` for those chapters using secondary pages
 # TABLESAMPLE - sometimes used for testing. Shouldn't be used in production as not random.
 # sample_data - sometimes used for testing. Shouldn't be used in production.
 # Block 2022_05_12 (contains secondary pages)
@@ -272,19 +57,11 @@ blocked_words = None
 # Block 2021_06_01 (probably forgot to update month to July for 2021)
 blocked_regex = (TABLESAMPLE|sample_data|2022_?05_?12|2022_?06_?09|2022_?07_?01|2021_?06_?01)
 
-[sqlfluff:rules:L063]
-# Data Types
-extended_capitalisation_policy = upper
-# Comma separated list of words to ignore for this rule
-ignore_words = None
-ignore_words_regex = None
+[sqlfluff:rules:convention.count_rows]
+prefer_count_0 = True
 
-[sqlfluff:rules:L064]
-# Consistent usage of preferred quotes for quoted literals
+[sqlfluff:rules:convention.quoted_literals]
 preferred_quoted_literal_style = single_quotes
-# Disabled for dialects that do not support single and double quotes for quoted literals (e.g. Postgres)
-force_enable = False
 
-[sqlfluff:rules:L066]
-min_alias_length = None
-max_alias_length = None
+[sqlfluff:rules:references.special_chars]
+additional_allowed_characters = "-."

sql/2019/accessibility/09_06.sql

@@ -22,7 +22,8 @@ FROM (
     client,
     total,
     page,
-    id)
+    id
+)
 WHERE
   freq > 1
 GROUP BY

sql/2019/accessibility/09_07.sql

@@ -24,7 +24,8 @@ FROM (
     _TABLE_SUFFIX AS client,
     getMediaElements(payload) AS media_elements
   FROM
-    `httparchive.pages.2019_07_01_*`)
+    `httparchive.pages.2019_07_01_*`
+)
 WHERE
   'audio' IN UNNEST(media_elements) OR
   'video' IN UNNEST(media_elements)

sql/2019/accessibility/09_08.sql

@@ -12,4 +12,5 @@ FROM (
     JSON_EXTRACT_SCALAR(report, "$.audits['html-has-lang'].score") = '1' AS has_lang,
     JSON_EXTRACT_SCALAR(report, "$.audits['html-valid-lang'].score") = '1' AS valid_lang
   FROM
-    `httparchive.lighthouse.2019_07_01_mobile`)
+    `httparchive.lighthouse.2019_07_01_mobile`
+)

sql/2019/accessibility/09_14.sql

@@ -16,7 +16,8 @@ FROM (
     `httparchive.almanac.summary_response_bodies`
   WHERE
     date = '2019-07-01' AND
-    firstHtml)
+    firstHtml
+)
 JOIN
   (SELECT _TABLE_SUFFIX AS client, COUNT(0) AS total FROM `httparchive.pages.2019_07_01_*` GROUP BY _TABLE_SUFFIX)
 USING (client)

sql/2019/accessibility/09_15.sql

@@ -18,6 +18,7 @@ FROM (
     `httparchive.almanac.summary_response_bodies`
   WHERE
     date = '2019-07-01' AND
-    firstHtml)
+    firstHtml
+)
 GROUP BY
   client

sql/2019/accessibility/09_18.sql

@@ -26,7 +26,8 @@ FROM (
     _TABLE_SUFFIX AS client,
     getTableElements(payload) AS table_elements
   FROM
-    `httparchive.pages.2019_07_01_*`)
+    `httparchive.pages.2019_07_01_*`
+)
 WHERE
   'table' IN UNNEST(table_elements)
 GROUP BY