Merge branch 'master' into mmolinaro/fix-543

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+exclude =
+  tests,
+  docs,
+  dist
+max-complexity = 31
+statistics = True
+show-source = True

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/python-app.yml

@@ -15,36 +15,34 @@ on:
 jobs:
   test:
     name: Run tests on ${{ matrix.py }}
-    runs-on: ubuntu-20.04  # keep it on 20.04 to have Python 3.6 available
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         py:
+          #- "3.13-dev"
           - "3.12"
           - "3.11"
           - "3.10"
           - "3.9"
           - "3.8"
-          - "3.7"
-          - "3.6"
+          - "pypy-3.10"
           - "pypy-3.9"
           - "pypy-3.8"
-          - "pypy-3.7"
 
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.py }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.py }}
         allow-prereleases: true
         check-latest: true
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip flit
-        flit install --deps=develop
+        python -m pip install --upgrade pip hatch
     - name: Lint with flake8
-      run: flake8 sqlparse --count --max-complexity=31 --show-source --statistics
-    - name: Test with pytest
-      run: pytest --cov=sqlparse
+      run: hatch run flake8
+    - name: Test with pytest and coverage
+      run: hatch run cov
     - name: Publish to codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4

.gitignore

@@ -4,9 +4,7 @@ dist/
 build/
 MANIFEST
 .coverage
-.tox/
 .cache/
 *.egg-info/
 htmlcov/
-coverage.xml
 .pytest_cache

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# python:
+#   install:
+#     - requirements: docs/requirements.txt

AUTHORS

@@ -8,6 +8,7 @@ project: https://bitbucket.org/gutworth/six.
 
 Alphabetical list of contributors:
 * Adam Greenhall <agreenhall@lyft.com>
+* Adam Johnson <me@adamj.eu>
 * Aki Ariga <chezou+github@gmail.com>
 * Alexander Beedie <ayembee@gmail.com>
 * Alexey Malyshev <nostrict@gmail.com>
@@ -32,6 +33,7 @@ Alphabetical list of contributors:
 * Fredy Wijaya <fredy.wijaya@gmail.com>
 * Gavin Wahl <gwahl@fusionbox.com>
 * Georg Traar <georg@crate.io>
+* griff <70294474+griffatrasgo@users.noreply.github.com>
 * Hugo van Kemenade <hugovk@users.noreply.github.com>
 * hurcy <cinyoung.hur@gmail.com>
 * Ian Robertson <ian.robertson@capitalone.com>

CHANGELOG

@@ -3,20 +3,32 @@ Development Version
 
 Notable Changes
 
-* Drop support for Python 3.5.
+* Drop support for Python 3.5, 3.6, and 3.7.
 * Python 3.12 is now supported (pr725, by hugovk).
 
 Enhancements:
 
 * Splitting statements now allows to remove the semicolon at the end.
   Some database backends love statements without semicolon (issue742).
 * Support TypedLiterals in get_parameters (pr649, by Khrol).
+* Improve splitting of Transact SQL when using GO keyword (issue762).
+* Support for some JSON operators (issue682).
+* Improve formatting of statements containing JSON operators (issue542).
+* Support for BigQuery and Snowflake keywords (pr699, by griffatrasgo).
 
 Bug Fixes
 
 * Ignore dunder attributes when creating Tokens (issue672).
 * Allow operators to precede dollar-quoted strings (issue763).
 * Fix parsing of nested order clauses (issue745, pr746 by john-bodley).
+* Thread-safe initialization of Lexer class (issue730).
+* Classify TRUNCATE as DDL and GRANT/REVOKE as DCL keywords (based on pr719
+  by josuc1, thanks for bringing this up!).
+* Fix parsing of PRIMARY KEY (issue740).
+
+Other
+
+* Optimize performance of matching function (pr799, by admachainz).
 
 
 Release 0.4.4 (Apr 18, 2023)

README.rst

@@ -11,7 +11,7 @@ python-sqlparse - Parse SQL statements
 sqlparse is a non-validating SQL parser for Python.
 It provides support for parsing, splitting and formatting SQL statements.
 
-The module is compatible with Python 3.6+ and released under the terms of the
+The module is compatible with Python 3.8+ and released under the terms of the
 `New BSD license <https://opensource.org/licenses/BSD-3-Clause>`_.
 
 Visit the project page at https://github.com/andialbrecht/sqlparse for

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
-requires = ["flit_core >=3.2,<4"]
-build-backend = "flit_core.buildapi"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
 [project]
 name = "sqlparse"
@@ -16,8 +16,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.6",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
@@ -28,7 +26,7 @@ classifiers = [
     "Topic :: Database",
     "Topic :: Software Development",
 ]
-requires-python = ">=3.6"
+requires-python = ">=3.8"
 
 [project.urls]
 Home = "https://github.com/andialbrecht/sqlparse"
@@ -42,35 +40,63 @@ sqlformat = "sqlparse.__main__:main"
 
 [project.optional-dependencies]
 dev = [
-    "importlib_metadata<5; python_version <= '3.7'",
-    "flake8",
+    "hatch",
     "build",
 ]
-test = [
-    "pytest",
-    "pytest-cov",
-]
 doc = [
     "sphinx",
 ]
-tox = [
-    "virtualenv<20.22.0",   # 20.22.0 dropped Python 3.6 support
-    "tox<4.5.0",  # >=4.5.0 requires virtualenv>=20.22
+
+[tool.hatch.version]
+path = "sqlparse/__init__.py"
+
+[tool.hatch.envs.default]
+dependencies = [
+    "coverage[toml]>=6.5",
+    "pytest",
+    # switch to ruff, but fix problems first
+    # but check defaults!
+    # https://hatch.pypa.io/1.9/config/static-analysis/#default-settings
+    "flake8",
+]
+[tool.hatch.envs.default.scripts]
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+cov-report = [
+    "- coverage combine",
+    "coverage report",
+]
+cov = [
+    "test-cov",
+    "cov-report",
 ]
+check = "flake8 sqlparse/"
 
-[tool.flit.sdist]
-include = [
-    "docs/source/",
-    "docs/sqlformat.1",
-    "docs/Makefile",
-    "tests/*.py", "tests/files/*.sql",
-    "LICENSE",
-    "TODO",
-    "AUTHORS",
-    "CHANGELOG",
-    "Makefile",
-    "tox.ini",
+[[tool.hatch.envs.all.matrix]]
+python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
+[tool.hatch.envs.types]
+dependencies = [
+    "mypy>=1.0.0",
 ]
+[tool.hatch.envs.types.scripts]
+check = "mypy --install-types --non-interactive {args:sqlparse tests}"
 
 [tool.coverage.run]
-omit = ["sqlparse/__main__.py"]
+source_pkgs = ["sqlparse", "tests"]
+branch = true
+parallel = true
+omit = [
+    "sqlparse/__main__.py",
+]
+
+[tool.coverage.paths]
+sqlparse = ["sqlparse"]
+tests = ["tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "no cov",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+]

sqlparse/engine/grouping.py

@@ -139,7 +139,12 @@ def post(tlist, pidx, tidx, nidx):
 
 def group_period(tlist):
     def match(token):
-        return token.match(T.Punctuation, '.')
+        for ttype, value in ((T.Punctuation, '.'),
+                             (T.Operator, '->'),
+                             (T.Operator, '->>')):
+            if token.match(ttype, value):
+                return True
+        return False
 
     def valid_prev(token, idx):
         sqlcls = sql.SquareBrackets, sql.Identifier
@@ -153,7 +158,7 @@ def valid_next(token, idx):
     def post(tlist, pidx, tidx, nidx):
         # next_ validation is being performed here. issue261
         sqlcls = sql.SquareBrackets, sql.Function
-        ttypes = T.Name, T.String.Symbol, T.Wildcard
+        ttypes = T.Name, T.String.Symbol, T.Wildcard, T.String.Single
         next_ = tlist[nidx] if nidx is not None else None
         valid_next = imt(next_, i=sqlcls, t=ttypes)
 

sqlparse/engine/statement_splitter.py

@@ -99,7 +99,12 @@ def process(self, stream):
             self.tokens.append(sql.Token(ttype, value))
 
             # Check if we get the end of a statement
-            if self.level <= 0 and ttype is T.Punctuation and value == ';':
+            # Issue762: Allow GO (or "GO 2") as statement splitter.
+            # When implementing a language toggle, it's not only to add
+            # keywords it's also to change some rules, like this splitting
+            # rule.
+            if (self.level <= 0 and ttype is T.Punctuation and value == ';') \
+                    or (ttype is T.Keyword and value.split()[0] == 'GO'):
                 self.consume_ws = True
 
         # Yield pending statement (if any)