Merge pull request #33 from hotgluexyz/fix/optimize

added option to fetch just body
hotgluexyz · Sep 6, 2022 · 0164ea4 · 0164ea4
2 parents 298eeb4 + 67ed362
commit 0164ea4
Show file tree

Hide file tree

Showing 5 changed files with 10 additions and 6 deletions.
diff --git a/setup.py b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name='tap-netsuite',
-    version='1.5.14',
+    version='1.5.15',
     description='Singer.io tap for extracting data from the NetSuite SOAP',
     author='hotglue',
     url='https://hotglue.xyz/',

diff --git a/tap_netsuite/__init__.py b/tap_netsuite/__init__.py
@@ -263,6 +263,7 @@ def main_impl():
                       ns_consumer_secret=CONFIG.get('ns_consumer_secret'),
                       ns_token_key=CONFIG.get('ns_token_key'),
                       ns_token_secret=CONFIG.get('ns_token_secret'),
+                      fetch_child=CONFIG.get('fetch_child', True),
                       is_sandbox=CONFIG.get('is_sandbox'),
                       default_start_date=CONFIG.get('start_date'),
                       select_fields_by_default=CONFIG.get('select_fields_by_default'), )

diff --git a/tap_netsuite/netsuite/__init__.py b/tap_netsuite/netsuite/__init__.py
@@ -97,6 +97,7 @@ def __init__(self,
                  ns_consumer_secret=None,
                  ns_token_key=None,
                  ns_token_secret=None,
+                 fetch_child=None,
                  is_sandbox=True,
                  select_fields_by_default=None,
                  default_start_date=None):
@@ -106,6 +107,7 @@ def __init__(self,
         self.ns_consumer_secret = ns_consumer_secret
         self.ns_token_key = ns_token_key
         self.ns_token_secret = ns_token_secret
+        self.ns_fetch_child = fetch_child
         self.is_sandbox = is_sandbox
         self.select_fields_by_default = select_fields_by_default is True or (
                 isinstance(select_fields_by_default, str) and select_fields_by_default.lower() == 'true')
@@ -136,6 +138,7 @@ def connect_tba(self, caching=True):
             consumer_secret=self.ns_consumer_secret,
             token_key=self.ns_token_key,
             token_secret=self.ns_token_secret,
+            fetch_child=self.ns_fetch_child,
             caching=caching
         )
         self.ns_client = nc

diff --git a/tap_netsuite/netsuite/netsuite_client.py b/tap_netsuite/netsuite/netsuite_client.py
@@ -2,11 +2,11 @@
 
 
 class ExtendedNetSuiteClient(NetSuiteClient):
-    def __init__(self, account=None, caching=True, caching_timeout=2592000):
+    def __init__(self, account=None, fetch_child=True, caching=True, caching_timeout=2592000):
         NetSuiteClient.__init__(self, account, caching, caching_timeout)
         # self.set_search_preferences(page_size=100, return_search_columns=True)
         self._search_preferences = self.SearchPreferences(
-            bodyFieldsOnly=False,
-            pageSize=100,
+            bodyFieldsOnly=not(fetch_child),
+            pageSize=500,
             returnSearchColumns=True
         )
diff --git a/tap_netsuite/netsuite/netsuite_connection.py b/tap_netsuite/netsuite/netsuite_connection.py
@@ -25,11 +25,11 @@
 
 
 class ExtendedNetSuiteConnection:
-    def __init__(self, account, consumer_key, consumer_secret, token_key, token_secret, caching=True):
+    def __init__(self, account, consumer_key, consumer_secret, token_key, token_secret, fetch_child=True, caching=True):
         # NetSuiteConnection.__init__(self, account, consumer_key, consumer_secret, token_key, token_secret)
         # ns_client: NetSuiteClient = self.client
 
-        ns_client = ExtendedNetSuiteClient(account=account, caching=caching)
+        ns_client = ExtendedNetSuiteClient(account=account, fetch_child=fetch_child, caching=caching)
         ns_client.connect_tba(
             consumer_key=consumer_key,
             consumer_secret=consumer_secret,