add support for python 3.12

toluaina · Nov 23, 2023 · a19bfc4 · a19bfc4
1 parent 1d55c2d
commit a19bfc4
Show file tree

Hide file tree

Showing 15 changed files with 92 additions and 129 deletions.
diff --git a/.github/workflows/python-build.yml b/.github/workflows/python-build.yml
@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     services:
       postgres:
         image: debezium/postgres:16

diff --git a/.gitignore b/.gitignore
@@ -90,6 +90,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.virtualenv/
 
 # Spyder project settings
 .spyderproject

diff --git a/pgsync/constants.py b/pgsync/constants.py
@@ -2,10 +2,13 @@
 PGSync Constants.
 
 This module contains constants used in PGSync.
-It includes constants for relationship types, relationship variants, node attributes, relationship attributes, 
-relationship foreign keys, tg_op, JSONB operators, Elasticsearch types, Elasticsearch mapping parameters, 
-transform types, default postgres schema, built-in schemas, primary key identifier, logical decoding output plugin, 
-trigger function, materialized views, primary key delimiter, and replication slot patterns.
+It includes constants for relationship types, relationship variants,
+node attributes, relationship attributes, relationship foreign keys,
+tg_op, JSONB operators, Elasticsearch types,
+Elasticsearch mapping parameters, transform types, default postgres schema,
+built-in schemas, primary key identifier, logical decoding output plugin,
+trigger function, materialized views, primary key delimiter,
+and replication slot patterns.
 """
 
 import re

diff --git a/pgsync/search_client.py b/pgsync/search_client.py
@@ -4,6 +4,7 @@
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import boto3
+import elastic_transport
 import elasticsearch
 import elasticsearch_dsl
 import opensearch_dsl
@@ -40,7 +41,7 @@ def __init__(self):
             self.__client: elasticsearch.Elasticsearch = get_search_client(
                 url,
                 client=elasticsearch.Elasticsearch,
-                connection_class=elasticsearch.RequestsHttpConnection,
+                node_class=elastic_transport.RequestsHttpNode,
             )
             try:
                 self.major_version: int = int(
@@ -257,7 +258,7 @@ def _create_setting(
         """Create Elasticsearch/OpenSearch setting and mapping if required."""
         body: dict = defaultdict(lambda: defaultdict(dict))
 
-        if not self.__client.indices.exists(index):
+        if not self.__client.indices.exists(index=index):
             if setting:
                 body.update(**{"settings": {"index": setting}})
 
@@ -343,18 +344,17 @@ def _build_mapping(
 def get_search_client(
     url: str,
     client: Union[opensearchpy.OpenSearch, elasticsearch.Elasticsearch],
-    connection_class: Union[
-        opensearchpy.RequestsHttpConnection,
-        elasticsearch.RequestsHttpConnection,
-    ],
+    connection_class: Optional[opensearchpy.RequestsHttpConnection] = None,
+    node_class: Optional[elastic_transport.RequestsHttpNode] = None,
 ) -> Union[opensearchpy.OpenSearch, elasticsearch.Elasticsearch]:
     """
     Returns a search client based on the specified parameters.
 
     Args:
         url (str): The URL of the search client.
         client (Union[opensearchpy.OpenSearch, elasticsearch.Elasticsearch]): The search client to use.
-        connection_class (Union[opensearchpy.RequestsHttpConnection, elasticsearch.RequestsHttpConnection]): The connection class to use.
+        connection_class (opensearchpy.RequestsHttpConnection): The connection class to use.
+        node_class (elastic_transport.RequestsHttpNode): The node class to use.
 
     Returns:
         Union[opensearchpy.OpenSearch, elasticsearch.Elasticsearch]: The search client.
@@ -365,19 +365,34 @@ def get_search_client(
     if settings.OPENSEARCH_AWS_HOSTED or settings.ELASTICSEARCH_AWS_HOSTED:
         credentials = boto3.Session().get_credentials()
         service: str = "aoss" if settings.OPENSEARCH_AWS_SERVERLESS else "es"
-        return client(
-            hosts=[url],
-            http_auth=AWS4Auth(
-                credentials.access_key,
-                credentials.secret_key,
-                settings.ELASTICSEARCH_AWS_REGION,
-                service,
-                session_token=credentials.token,
-            ),
-            use_ssl=True,
-            verify_certs=True,
-            connection_class=connection_class,
-        )
+        if settings.OPENSEARCH:
+            return client(
+                hosts=[url],
+                http_auth=AWS4Auth(
+                    credentials.access_key,
+                    credentials.secret_key,
+                    settings.ELASTICSEARCH_AWS_REGION,
+                    service,
+                    session_token=credentials.token,
+                ),
+                use_ssl=True,
+                verify_certs=True,
+                connection_class=connection_class,
+            )
+        elif settings.ELASTICSEARCH:
+            return client(
+                hosts=[url],
+                http_auth=AWS4Auth(
+                    credentials.access_key,
+                    credentials.secret_key,
+                    settings.ELASTICSEARCH_AWS_REGION,
+                    service,
+                    session_token=credentials.token,
+                ),
+                use_ssl=True,
+                verify_certs=True,
+                node_class=node_class,
+            )
     else:
         hosts: List[str] = [url]
         # API
@@ -413,7 +428,6 @@ def get_search_client(
         ssl_context: Optional[Any] = settings.ELASTICSEARCH_SSL_CONTEXT
         ssl_show_warn: bool = settings.ELASTICSEARCH_SSL_SHOW_WARN
         # Transport
-        use_ssl: bool = settings.ELASTICSEARCH_USE_SSL
         timeout: float = settings.ELASTICSEARCH_TIMEOUT
         return client(
             hosts=hosts,
@@ -433,6 +447,6 @@ def get_search_client(
             ssl_version=ssl_version,
             ssl_context=ssl_context,
             ssl_show_warn=ssl_show_warn,
-            use_ssl=use_ssl,
+            # use_ssl=use_ssl,
             timeout=timeout,
         )
diff --git a/pgsync/sync.py b/pgsync/sync.py
@@ -1313,6 +1313,13 @@ def receive(self, nthreads_polldb: int) -> None:
             # start a background worker thread to show status
             self.status()
 
+    def join(self):
+        """Join all threads"""
+        self.poll_redis().join()
+        self.poll_db().join()
+        self.truncate_slots().join()
+        self.status().join()
+
 
 @click.command()
 @click.option(
@@ -1449,6 +1456,7 @@ def main(
                 sync.pull()
                 if daemon:
                     sync.receive(nthreads_polldb)
+                sync.join()
 
 
 if __name__ == "__main__":

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,3 +1,3 @@
 [tool.black]
 line-length = 79
-target-version = ['py38', 'py39', 'py310', 'py311']
+target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
diff --git a/requirements/base.in b/requirements/base.in
@@ -2,12 +2,12 @@
 black
 boto3
 click
-elasticsearch==7.13.4 # pin this to version 7.13.4 for compatibility with OpenSearch https://opensearch.org/docs/clients/index/
-elasticsearch-dsl>=6.0.0,<8.0.0
+elasticsearch
+elasticsearch-dsl
 environs
 faker
 isort
-opensearch-dsl>=2.0.1
+opensearch-dsl
 psycopg2-binary
 python-dotenv
 redis

diff --git a/requirements/prod.txt → requirements/base.txt b/requirements/prod.txt → requirements/base.txt
@@ -2,21 +2,21 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --output-file=requirements/prod.txt requirements/prod.in
+#    pip-compile --output-file=requirements/base.txt requirements/base.in
 #
 async-timeout==4.0.3
     # via redis
 black==23.11.0
     # via -r requirements/base.in
-boto3==1.29.3
+boto3==1.29.6
     # via -r requirements/base.in
-botocore==1.32.3
+botocore==1.32.6
     # via
     #   boto3
     #   s3transfer
 certifi==2023.11.17
     # via
-    #   elasticsearch
+    #   elastic-transport
     #   opensearch-py
     #   requests
 charset-normalizer==3.3.2
@@ -25,15 +25,17 @@ click==8.1.7
     # via
     #   -r requirements/base.in
     #   black
-elasticsearch==7.13.4
+elastic-transport==8.10.0
+    # via elasticsearch
+elasticsearch==8.11.0
     # via
     #   -r requirements/base.in
     #   elasticsearch-dsl
-elasticsearch-dsl==7.4.1
+elasticsearch-dsl==8.11.0
     # via -r requirements/base.in
 environs==9.5.0
     # via -r requirements/base.in
-faker==20.0.3
+faker==20.1.0
     # via -r requirements/base.in
 greenlet==3.0.1
     # via sqlalchemy
@@ -51,7 +53,7 @@ mypy-extensions==1.0.0
     # via black
 opensearch-dsl==2.1.0
     # via -r requirements/base.in
-opensearch-py==2.4.1
+opensearch-py==2.4.2
     # via opensearch-dsl
 packaging==23.2
     # via
@@ -86,7 +88,6 @@ s3transfer==0.7.0
     # via boto3
 six==1.16.0
     # via
-    #   elasticsearch-dsl
     #   opensearch-dsl
     #   opensearch-py
     #   python-dateutil
@@ -102,6 +103,6 @@ typing-extensions==4.8.0
 urllib3==1.26.18
     # via
     #   botocore
-    #   elasticsearch
+    #   elastic-transport
     #   opensearch-py
     #   requests
diff --git a/requirements/dev.in b/requirements/dev.in
@@ -2,19 +2,9 @@
 
 coverage
 flake8
-flake8_docstrings
-flake8-debugger
-flake8-print
-flake8-todo
-flake8-isort
 freezegun
 mock
-pip-tools
 pre-commit
 pytest
 pytest-cov
-pytest-runner
 pytest-mock
-pytest-sugar
-pytest-runner
-setuptools