Add autocommit and retry to read only sql sessions for CAI temp table. (

#2672)

google/cloud/forseti/services/db.py

@@ -138,7 +138,9 @@ def create_scoped_readonly_session(engine):
     Returns:
         object: Scoped session maker.
     """
-    session = sessionmaker(bind=engine, autocommit=False, autoflush=False)()
+    # Set session autocommit to true to ensure each query gets a new connection
+    # from the pool.
+    session = sessionmaker(bind=engine, autocommit=True, autoflush=False)()
 
     return ScopedSession(session, auto_commit=False, readonly=True)
 
@@ -152,6 +154,8 @@ def create_readonly_session(engine):
     Returns:
         object: Scoped session maker.
     """
-    session = sessionmaker(bind=engine, autocommit=False, autoflush=False)()
+    # Set session autocommit to true to ensure each query gets a new connection
+    # from the pool.
+    session = sessionmaker(bind=engine, autocommit=True, autoflush=False)()
     stub_out_flush_operation(session)
     return session

google/cloud/forseti/services/inventory/storage.py

@@ -17,6 +17,7 @@
 import json
 import enum
 
+from retrying import retry
 from sqlalchemy import and_
 from sqlalchemy import BigInteger
 from sqlalchemy import case
@@ -803,9 +804,13 @@ def populate_cai_data(data, session):
         return num_rows
 
     @staticmethod
+    @retry(wait_exponential_multiplier=1000, wait_exponential_max=10000,
+           stop_max_attempt_number=5)
     def iter_cai_assets(content_type, asset_type, parent_name, session):
         """Iterate the objects in the cai temporary table.
 
+        Retries query on exception up to 5 times.
+
         Args:
             content_type (ContentTypes): The content type to return.
             asset_type (str): The asset type to return.
@@ -831,9 +836,13 @@ def iter_cai_assets(content_type, asset_type, parent_name, session):
             yield row.extract_asset_data(content_type)
 
     @staticmethod
+    @retry(wait_exponential_multiplier=1000, wait_exponential_max=10000,
+           stop_max_attempt_number=5)
     def fetch_cai_asset(content_type, asset_type, name, session):
         """Returns a single resource from the cai temporary store.
 
+        Retries query on exception up to 5 times.
+
         Args:
             content_type (ContentTypes): The content type to return.
             asset_type (str): The asset type to return.