switch from pymssql to pyodbc

pymssql does not work with python 3.8 and the project is
discontinued pymssql/pymssql#668
Let's move to the recommended (and already used) pyodbc

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
@@ -29,8 +29,8 @@ jobs:
     - name: Install databricks dependencies
       run: sudo bash toucan_connectors/install_scripts/databricks.sh
 
-    - name: Install azure_mssql dependencies
-      run: sudo bash toucan_connectors/install_scripts/azure_mssql.sh
+    - name: Install mssql dependencies
+      run: sudo bash toucan_connectors/install_scripts/mssql.sh
 
     - name: install
       run: make install

README.md

@@ -19,11 +19,12 @@ You can then install:
 
 You should be able to run basic tests `pytest tests/test_connector.py`
 
-:warning: On macOS, to test and use
-- `azure_mssql` and `mssql` connectors, you must install `freetds`
-running for instance `brew install freetds`.
-- `postgres` connector, you must install `postgresql` running for instance `brew install postgres`.
-You can install the library with `env LDFLAGS='-L/usr/local/lib -L/usr/local/opt/openssl/lib -L/usr/local/opt/readline/lib' pip install psycopg2`
+:warning: To test and use `mssql` (and `azure_mssql`) you need to install the Microsoft ODBC driver for SQL Server for
+[Linux](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15)
+or [MacOS](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver15)
+
+:warning: On macOS, to test the `postgres` connector, you need to install `postgresql` by running for instance `brew install postgres`.
+You can then install the library with `env LDFLAGS='-L/usr/local/lib -L/usr/local/opt/openssl/lib -L/usr/local/opt/readline/lib' pip install psycopg2`
 
 ## Testing a connector
 If you want to run the tests for another connector, you can install the extra dependencies  

setup.py

@@ -22,7 +22,7 @@
     'http_api': auth_deps,
     'lightspeed': bearer_deps,
     'mongo': ['pymongo>=3.6.1'],
-    'mssql': ['pymssql>=2.1.3,<3.0'],
+    'mssql': ['pyodbc'],
     'mysql': ['PyMySQL>=0.8.0'],
     'odata': auth_deps + ['tctc_odata'],
     'oracle_sql': ['cx_Oracle>=6.2.1'],

tests/docker-compose.yml

@@ -18,7 +18,6 @@ mssql:
   image: microsoft/mssql-server-linux
   environment:
     - ACCEPT_EULA=Y
-    - MSSQL_PID=Developer
     - SA_PASSWORD=Il0veT0uc@n!
   ports:
     - 1433:1433

tests/mssql/test_mssql.py

@@ -2,7 +2,7 @@
 
 import pandas as pd
 import pydantic
-import pymssql
+import pyodbc
 import pytest
 
 from toucan_connectors.mssql.mssql_connector import MSSQLConnector, MSSQLDataSource
@@ -15,7 +15,12 @@ def check_and_feed(host_port):
         This method does not only check that the server is on
         but also feeds the database once it's up !
         """
-        conn = pymssql.connect(host='127.0.0.1', port=host_port, user='SA', password='Il0veT0uc@n!')
+        conn = pyodbc.connect(
+            driver='{ODBC Driver 17 for SQL Server}',
+            server=f'127.0.0.1,{host_port}',
+            user='SA',
+            password='Il0veT0uc@n!',
+        )
         cur = conn.cursor()
         cur.execute('SELECT 1;')
 
@@ -29,7 +34,7 @@ def check_and_feed(host_port):
         cur.close()
         conn.close()
 
-    return service_container('mssql', check_and_feed, pymssql.Error)
+    return service_container('mssql', check_and_feed, pyodbc.Error)
 
 
 @pytest.fixture
@@ -60,6 +65,7 @@ def test_datasource(mssql_datasource):
 def test_connection_params():
     connector = MSSQLConnector(name='my_mssql_con', host='myhost', user='myuser')
     assert connector.get_connection_params(None) == {
+        'driver': '{ODBC Driver 17 for SQL Server}',
         'server': 'myhost',
         'user': 'myuser',
         'as_dict': True,
@@ -73,18 +79,18 @@ def test_connection_params():
         connect_timeout=60,
     )
     assert connector.get_connection_params('mydb') == {
-        'server': 'myhost',
+        'driver': '{ODBC Driver 17 for SQL Server}',
+        'server': 'myhost,123',
         'user': 'myuser',
         'as_dict': True,
         'password': 'mypass',
-        'port': 123,
-        'login_timeout': 60,
+        'timeout': 60,
         'database': 'mydb',
     }
 
 
 def test_mssql_get_df(mocker):
-    snock = mocker.patch('pymssql.connect')
+    snock = mocker.patch('pyodbc.connect')
     reasq = mocker.patch('pandas.read_sql')
 
     mssql_connector = MSSQLConnector(
@@ -99,11 +105,11 @@ def test_mssql_get_df(mocker):
     mssql_connector.get_df(datasource)
 
     snock.assert_called_once_with(
+        driver='{ODBC Driver 17 for SQL Server}',
         as_dict=True,
-        server='localhost',
+        server='127.0.0.1,22',
         user='SA',
         password='Il0veT0uc@n!',
-        port=22,
         database='mydb',
     )
 

toucan_connectors/install_scripts/azure_mssql.sh

@@ -0,0 +1 @@
+mssql.sh

toucan_connectors/install_scripts/mssql.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+if [[ -e ~/mssql-installed ]]; then
+    echo "MSSQL connector dependencies are already installed."
+    exit
+fi
+
+apt-get update
+apt-get install -fyq curl
+curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+source /etc/os-release &&\
+    curl "https://packages.microsoft.com/config/${ID}/${VERSION_ID}/prod.list" \
+    | tee /etc/apt/sources.list.d/mssql-release.list
+apt-get update
+ACCEPT_EULA=Y apt-get -y install msodbcsql17 unixodbc-dev
+
+touch ~/mssql-installed

toucan_connectors/mssql/mssql_connector.py

@@ -1,5 +1,5 @@
 import pandas as pd
-import pymssql
+import pyodbc
 from pydantic import Field, SecretStr, constr
 
 from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource
@@ -42,20 +42,25 @@ class MSSQLConnector(ToucanConnector):
     )
 
     def get_connection_params(self, database):
+        server = self.host
+        if server == 'localhost':
+            server = '127.0.0.1'  # localhost is not understood by pyodbc
+        if self.port is not None:
+            server += f',{self.port}'
         con_params = {
-            'server': self.host,
-            'user': self.user,
+            'driver': '{ODBC Driver 17 for SQL Server}',
+            'server': server,
             'database': database,
+            'user': self.user,
             'password': self.password.get_secret_value() if self.password else None,
-            'port': self.port,
-            'login_timeout': self.connect_timeout,
+            'timeout': self.connect_timeout,
             'as_dict': True,
         }
         # remove None values
         return {k: v for k, v in con_params.items() if v is not None}
 
     def _retrieve_data(self, datasource):
-        connection = pymssql.connect(**self.get_connection_params(datasource.database))
+        connection = pyodbc.connect(**self.get_connection_params(datasource.database))
         df = pd.read_sql(datasource.query, con=connection)
 
         connection.close()