Skip to content

Commit

Permalink
fix: handle postgres connect via cloud run
Browse files Browse the repository at this point in the history
  • Loading branch information
stdavis authored and jacobdadams committed Jan 12, 2023
1 parent 302ac9b commit f46d993
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 3 deletions.
6 changes: 6 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
{
"cSpell.words": [
"AGRC",
"cloudsql",
"Codecov",
"Dataframe",
"drivername",
"Dtype",
"featurelayer",
"featureset",
Expand All @@ -13,14 +15,18 @@
"instafail",
"isort",
"knownhosts",
"lowres",
"naturalearth",
"Overwriter",
"palletjack",
"PARENTOBJECTID",
"PGSQL",
"psycopg",
"pylint",
"pytest",
"reclassifier",
"sftploader",
"sqlalchemy",
"unclassed",
"yapf"
],
Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
'pygsheets==2.0.*',
'geopandas==0.12.*',
'SQLAlchemy==1.4.*',
'pg8000==1.29.*',
'psycopg2-binary==2.9.*',
'numpy==1.23.*', #: Pinned to fix "module 'numpy' has no attribute 'str'" error
],
Expand Down
28 changes: 25 additions & 3 deletions src/palletjack/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import logging
import mimetypes
import os
import re
from io import BytesIO
from pathlib import Path
Expand All @@ -12,8 +13,8 @@
import pandas as pd
import pysftp
import requests
import sqlalchemy
from googleapiclient.http import MediaIoBaseDownload
from sqlalchemy import create_engine

from . import utils

Expand Down Expand Up @@ -439,8 +440,29 @@ class PostgresLoader:
def __init__(self, host, database, username, password, port=5432):
self._class_logger = logging.getLogger(__name__).getChild(self.__class__.__name__)

connection_string = f'postgresql://{username}:{password}@{host}:{port}/{database}'
self.engine = create_engine(connection_string)
if os.environ.get('FUNCTION_TARGET') is not None: #: this is an env var specific to cloud functions
self._class_logger.info('running in GCF, using unix socket')
self.engine = sqlalchemy.create_engine(
sqlalchemy.engine.url.URL.create(
drivername='postgresql+pg8000',
username=username,
password=password,
database=database,
query={'unix_sock': f'/cloudsql/{host}/.s.PGSQL.{port}'}, #: requires the pg8000 package
)
)
else:
self._class_logger.info('running locally, using traditional host connection')
self.engine = sqlalchemy.create_engine(
sqlalchemy.engine.url.URL.create(
drivername='postgresql',
username=username,
password=password,
database=database,
host=host,
port=port,
)
)

def read_table_into_dataframe(self, table_name, index_column, crs, spatial_column):
"""Read a table into a dataframe
Expand Down

0 comments on commit f46d993

Please sign in to comment.