From d6ae4eaed86bb8e3df4a68a91ccfdb2cd3ea032d Mon Sep 17 00:00:00 2001
From: Martin Levy <mahtin@mahtin.com>
Date: Wed, 27 Dec 2023 11:49:53 -0800
Subject: [PATCH] added Content-Type processing on add, removed old api web
 checking (hence remove bs4), refactoring of all add methods, network now
 handles data/json cleanly, results returned based on Content-Type via new
 code, setup/requirements removed BeautifulSoup/bs4

---
 CloudFlare/api_decode_from_web.py | 202 +++++++-------
 CloudFlare/api_v4.py              |  47 ++--
 CloudFlare/cloudflare.py          | 447 +++++++++++++-----------------
 CloudFlare/network.py             | 109 +++-----
 CloudFlare/utils.py               |  28 +-
 Makefile                          |   4 +-
 requirements.txt                  |   2 +-
 setup.py                          |   2 +-
 8 files changed, 382 insertions(+), 459 deletions(-)

diff --git a/CloudFlare/api_decode_from_web.py b/CloudFlare/api_decode_from_web.py
index fd7bd84..a6fc6b0 100644
--- a/CloudFlare/api_decode_from_web.py
+++ b/CloudFlare/api_decode_from_web.py
@@ -1,103 +1,105 @@
 """ API extras for Cloudflare API"""
 
-import sys
-import datetime
+# TODO - remove at some point
 
-API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
-
-class mybs4():
-    """ mybs4 """
-
-    _BeautifulSoup = None
-    _Comment = None
-
-    def __init__(self):
-        """ __init__ """
-        pass
-
-    def _available(self):
-        """ _available() """
-        if not mybs4._BeautifulSoup:
-            try:
-                from bs4 import BeautifulSoup, Comment
-                mybs4._BeautifulSoup = BeautifulSoup
-                mybs4._Comment = Comment
-                self.Comment = mybs4._Comment
-            except ImportError:
-                return False
-        return True
-
-    def BeautifulSoup(self, content, parser):
-        """ BeautifulSoup() """
-        self._available()
-        return mybs4._BeautifulSoup(content, parser)
-
-my_bs4 = mybs4()
-
-def do_section(section):
-    """ API extras for Cloudflare API"""
-
-    cmds = []
-    # look for deprecated first in section
-    deprecated = False
-    deprecated_date = ''
-    deprecated_already = False
-    for tag2 in section.find_all('h3'):
-        # <h3 class="text-warning" data-reactid="13490">Deprecation Warning</h3>
-        if 'Deprecation Warning' in str(tag2):
-            deprecated = True
-            break
-    for tag2 in section.find_all('p'):
-        # <p class="deprecation-date" data-reactid="13491">End of life Date: November 2, 2020</p>
-        if 'End of life Date:' in str(tag2):
-            for child in tag2.children:
-                deprecated_date = str(child).replace('End of life Date:','').strip()
-                try:
-                    # clean up date
-                    d = datetime.datetime.strptime(deprecated_date, '%B %d, %Y')
-                    if d <= datetime.datetime.now():
-                        # already done!
-                        deprecated_already = True
-                    deprecated_date = d.strftime('%Y-%m-%d')
-                except ValueError:
-                    # Lets not worry about all the date formats that could show-up. Leave as a string
-                    pass
-                break
-        if deprecated_date != '':
-            break
-    # look for all API calls in section
-    for tag2 in section.find_all('pre'):
-        cmd = []
-        for child in tag2.children:
-            if isinstance(child, my_bs4.Comment):
-                # remove <!-- react-text ... -> parts
-                continue
-            cmd.append(str(child).strip())
-        if len(cmd) == 0:
-            continue
-        action = cmd[0]
-        if action == '' or action not in API_TYPES:
-            continue
-        cmd = ''.join(cmd[1:])
-        if cmd[0] != '/':
-            cmd = '/' + cmd
-        v = {'action': action, 'cmd': cmd, 'deprecated': deprecated, 'deprecated_date': deprecated_date, 'deprecated_already': deprecated_already}
-        cmds.append(v)
-    return cmds
-
-def api_decode_from_web(content):
-    """ API extras for Cloudflare API"""
-
-    soup = my_bs4.BeautifulSoup(content, 'html.parser')
-
-    for child in soup.find_all('p'):
-        t = child.get_text()
-        if 'Last modified' in t:
-            sys.stderr.write("Retrieved API: %s\n" % (t.strip().replace('\n', ' ')))
-            break
-
-    all_cmds = []
-    for section in soup.find_all('section'):
-        all_cmds += do_section(section)
-
-    return sorted(all_cmds, key=lambda v: v['cmd'])
+#import sys
+#import datetime
+#
+#API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
+#
+#class mybs4():
+#    """ mybs4 """
+#
+#    _BeautifulSoup = None
+#    _Comment = None
+#
+#    def __init__(self):
+#        """ __init__ """
+#        pass
+#
+#    def _available(self):
+#        """ _available() """
+#        if not mybs4._BeautifulSoup:
+#            try:
+#                from bs4 import BeautifulSoup, Comment
+#                mybs4._BeautifulSoup = BeautifulSoup
+#                mybs4._Comment = Comment
+#                self.Comment = mybs4._Comment
+#            except ImportError:
+#                return False
+#        return True
+#
+#    def BeautifulSoup(self, content, parser):
+#        """ BeautifulSoup() """
+#        self._available()
+#        return mybs4._BeautifulSoup(content, parser)
+#
+#my_bs4 = mybs4()
+#
+#def do_section(section):
+#    """ API extras for Cloudflare API"""
+#
+#    cmds = []
+#    # look for deprecated first in section
+#    deprecated = False
+#    deprecated_date = ''
+#    deprecated_already = False
+#    for tag2 in section.find_all('h3'):
+#        # <h3 class="text-warning" data-reactid="13490">Deprecation Warning</h3>
+#        if 'Deprecation Warning' in str(tag2):
+#            deprecated = True
+#            break
+#    for tag2 in section.find_all('p'):
+#        # <p class="deprecation-date" data-reactid="13491">End of life Date: November 2, 2020</p>
+#        if 'End of life Date:' in str(tag2):
+#            for child in tag2.children:
+#                deprecated_date = str(child).replace('End of life Date:','').strip()
+#                try:
+#                    # clean up date
+#                    d = datetime.datetime.strptime(deprecated_date, '%B %d, %Y')
+#                    if d <= datetime.datetime.now():
+#                        # already done!
+#                        deprecated_already = True
+#                    deprecated_date = d.strftime('%Y-%m-%d')
+#                except ValueError:
+#                    # Lets not worry about all the date formats that could show-up. Leave as a string
+#                    pass
+#                break
+#        if deprecated_date != '':
+#            break
+#    # look for all API calls in section
+#    for tag2 in section.find_all('pre'):
+#        cmd = []
+#        for child in tag2.children:
+#            if isinstance(child, my_bs4.Comment):
+#                # remove <!-- react-text ... -> parts
+#                continue
+#            cmd.append(str(child).strip())
+#        if len(cmd) == 0:
+#            continue
+#        action = cmd[0]
+#        if action == '' or action not in API_TYPES:
+#            continue
+#        cmd = ''.join(cmd[1:])
+#        if cmd[0] != '/':
+#            cmd = '/' + cmd
+#        v = {'action': action, 'cmd': cmd, 'deprecated': deprecated, 'deprecated_date': deprecated_date, 'deprecated_already': deprecated_already}
+#        cmds.append(v)
+#    return cmds
+#
+#def api_decode_from_web(content):
+#    """ API extras for Cloudflare API"""
+#
+#    soup = my_bs4.BeautifulSoup(content, 'html.parser')
+#
+#    for child in soup.find_all('p'):
+#        t = child.get_text()
+#        if 'Last modified' in t:
+#            sys.stderr.write("Retrieved API: %s\n" % (t.strip().replace('\n', ' ')))
+#            break
+#
+#    all_cmds = []
+#    for section in soup.find_all('section'):
+#        all_cmds += do_section(section)
+#
+#    return sorted(all_cmds, key=lambda v: v['cmd'])
diff --git a/CloudFlare/api_v4.py b/CloudFlare/api_v4.py
index b4cfa2e..758708f 100644
--- a/CloudFlare/api_v4.py
+++ b/CloudFlare/api_v4.py
@@ -110,7 +110,7 @@ def zones(self):
     self.add('AUTH', 'zones', 'custom_pages')
     self.add('AUTH', 'zones', 'dns_records')
     self.add('AUTH', 'zones', 'dns_records/export')
-    self.add('AUTH', 'zones', 'dns_records/import')
+    self.add('AUTH', 'zones', 'dns_records/import', content_type={'POST':'multipart/form-data'})
     self.add('AUTH', 'zones', 'dns_records/scan')
     self.add('VOID', 'zones', 'dns_settings')
     self.add('AUTH', 'zones', 'dns_settings/use_apex_ns')
@@ -185,6 +185,15 @@ def zones_settings(self):
     self.add('AUTH', 'zones', 'settings/waf')
     self.add('AUTH', 'zones', 'settings/webp')
     self.add('AUTH', 'zones', 'settings/websockets')
+    self.add('VOID', 'zones', 'settings/zaraz')
+    self.add('VOID', 'zones', 'settings/zaraz/v2')
+    self.add('AUTH', 'zones', 'settings/zaraz/v2/config')
+    self.add('AUTH', 'zones', 'settings/zaraz/v2/default')
+    self.add('AUTH', 'zones', 'settings/zaraz/v2/export')
+    self.add('AUTH', 'zones', 'settings/zaraz/v2/history')
+    self.add('AUTH', 'zones', 'settings/zaraz/v2/history/configs')
+    self.add('AUTH', 'zones', 'settings/zaraz/v2/publish')
+    self.add('AUTH', 'zones', 'settings/zaraz/v2/workflow')
 
 def zones_analytics(self):
     """ zones analytics """
@@ -399,7 +408,7 @@ def accounts(self):
 
     self.add('VOID', 'accounts', 'dlp')
     self.add('AUTH', 'accounts', 'dlp/datasets')
-    self.add('AUTH', 'accounts', 'dlp/datasets', 'upload')
+    self.add('AUTH', 'accounts', 'dlp/datasets', 'upload', content_type={'POST':'application/octet-stream'})
     self.add('VOID', 'accounts', 'dlp/patterns')
     self.add('AUTH', 'accounts', 'dlp/patterns/validate')
     self.add('AUTH', 'accounts', 'dlp/payload_log')
@@ -448,7 +457,7 @@ def accounts(self):
     self.add('AUTH', 'accounts', 'storage/kv/namespaces')
     self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'bulk')
     self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'keys')
-    self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'values')
+    self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'values', content_type={'PUT':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'storage/kv/namespaces', 'metadata')
 
     self.add('AUTH', 'accounts', 'subscriptions')
@@ -458,9 +467,9 @@ def accounts(self):
     self.add('VOID', 'accounts', 'vectorize')
     self.add('AUTH', 'accounts', 'vectorize/index')
     self.add('AUTH', 'accounts', 'vectorize/indexes')
-    self.add('AUTH', 'accounts', 'vectorize/indexes', 'insert')
+    self.add('AUTH', 'accounts', 'vectorize/indexes', 'insert', content_type={'POST':'application/x-ndjson'})
     self.add('AUTH', 'accounts', 'vectorize/indexes', 'query')
-    self.add('AUTH', 'accounts', 'vectorize/indexes', 'upsert')
+    self.add('AUTH', 'accounts', 'vectorize/indexes', 'upsert', content_type={'POST':'application/x-ndjson'})
 
     self.add('AUTH', 'accounts', 'virtual_dns')
     self.add('VOID', 'accounts', 'virtual_dns', 'dns_analytics')
@@ -475,7 +484,7 @@ def accounts(self):
     self.add('VOID', 'accounts', 'workers/dispatch')
     self.add('AUTH', 'accounts', 'workers/dispatch/namespaces')
     self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts')
-    self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'content')
+    self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'content', content_type={'PUT':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'settings')
     self.add('AUTH', 'accounts', 'workers/dispatch/namespaces', 'scripts', 'tags')
     self.add('AUTH', 'accounts', 'workers/domains')
@@ -485,15 +494,15 @@ def accounts(self):
     self.add('AUTH', 'accounts', 'workers/queues')
     self.add('AUTH', 'accounts', 'workers/queues', 'consumers')
     self.add('AUTH', 'accounts', 'workers/scripts')
-    self.add('AUTH', 'accounts', 'workers/scripts', 'content')
+    self.add('AUTH', 'accounts', 'workers/scripts', 'content', content_type={'PUT':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'workers/scripts', 'content/v2')
     self.add('AUTH', 'accounts', 'workers/scripts', 'schedules')
-    self.add('AUTH', 'accounts', 'workers/scripts', 'settings')
+    self.add('AUTH', 'accounts', 'workers/scripts', 'settings', content_type={'PATCH':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'workers/scripts', 'tails')
     self.add('AUTH', 'accounts', 'workers/scripts', 'usage-model')
     self.add('VOID', 'accounts', 'workers/services')
     self.add('VOID', 'accounts', 'workers/services', 'environments')
-    self.add('AUTH', 'accounts', 'workers/services', 'environments', 'content')
+    self.add('AUTH', 'accounts', 'workers/services', 'environments', 'content', content_type={'PUT':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'workers/services', 'environments', 'settings')
     self.add('AUTH', 'accounts', 'workers/subdomain')
 
@@ -505,7 +514,7 @@ def accounts_addressing(self):
     self.add('AUTH', 'accounts', 'addressing/address_maps', 'accounts')
     self.add('AUTH', 'accounts', 'addressing/address_maps', 'ips')
     self.add('AUTH', 'accounts', 'addressing/address_maps', 'zones')
-    self.add('AUTH', 'accounts', 'addressing/loa_documents')
+    self.add('AUTH', 'accounts', 'addressing/loa_documents', content_type={'POST':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'addressing/loa_documents', 'download')
     self.add('AUTH', 'accounts', 'addressing/prefixes')
     self.add('VOID', 'accounts', 'addressing/prefixes', 'bgp')
@@ -559,7 +568,7 @@ def accounts_stream(self):
     self.add('AUTH', 'accounts', 'stream')
     self.add('AUTH', 'accounts', 'stream', 'audio')
     self.add('AUTH', 'accounts', 'stream', 'audio/copy')
-    self.add('AUTH', 'accounts', 'stream', 'captions')
+    self.add('AUTH', 'accounts', 'stream', 'captions', content_type={'PUT':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'stream', 'embed')
     self.add('AUTH', 'accounts', 'stream', 'downloads')
     self.add('AUTH', 'accounts', 'stream', 'token')
@@ -568,7 +577,7 @@ def accounts_stream(self):
     self.add('AUTH', 'accounts', 'stream/direct_upload')
     self.add('AUTH', 'accounts', 'stream/keys')
 #   self.add('AUTH', 'accounts', 'stream/preview')
-    self.add('AUTH', 'accounts', 'stream/watermarks')
+    self.add('AUTH', 'accounts', 'stream/watermarks', content_type={'POST':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'stream/webhook')
     self.add('AUTH', 'accounts', 'stream/live_inputs')
     self.add('AUTH', 'accounts', 'stream/live_inputs', 'outputs')
@@ -666,7 +675,7 @@ def accounts_extras(self):
     """ extras """
 
     self.add('VOID', 'accounts', 'ai')
-    self.add('AUTH', 'accounts', 'ai/run')
+    self.add('AUTH', 'accounts', 'ai/run', content_type={'POST':['application/json','application/octet-stream']})
     self.add('AUTH', 'accounts', 'ai/run/proxy')
 
     self.add('VOID', 'accounts', 'alerting')
@@ -737,14 +746,14 @@ def accounts_extras(self):
     self.add('AUTH', 'accounts', 'gateway/rules')
 
     self.add('VOID', 'accounts', 'images')
-    self.add('AUTH', 'accounts', 'images/v1')
+    self.add('AUTH', 'accounts', 'images/v1', content_type={'POST':'multipart/form-data'})
     self.add('AUTH', 'accounts', 'images/v1', 'blob')
     self.add('AUTH', 'accounts', 'images/v1/config')
     self.add('AUTH', 'accounts', 'images/v1/keys')
     self.add('AUTH', 'accounts', 'images/v1/stats')
     self.add('AUTH', 'accounts', 'images/v1/variants')
     self.add('AUTH', 'accounts', 'images/v2')
-    self.add('AUTH', 'accounts', 'images/v2/direct_upload')
+    self.add('AUTH', 'accounts', 'images/v2/direct_upload', content_type={'POST':'multipart/form-data'})
 
     self.add('VOID', 'accounts', 'intel')
     self.add('VOID', 'accounts', 'intel-phishing')
@@ -757,7 +766,7 @@ def accounts_extras(self):
     self.add('AUTH', 'accounts', 'intel/domain/bulk')
     self.add('AUTH', 'accounts', 'intel/indicator-feeds')
     self.add('AUTH', 'accounts', 'intel/indicator-feeds', 'data')
-    self.add('AUTH', 'accounts', 'intel/indicator-feeds', 'snapshot')
+    self.add('AUTH', 'accounts', 'intel/indicator-feeds', 'snapshot', content_type={'PUT':'multipart/form-data'})
     self.add('VOID', 'accounts', 'intel/indicator-feeds/permissions')
     self.add('AUTH', 'accounts', 'intel/indicator-feeds/permissions/add')
     self.add('AUTH', 'accounts', 'intel/indicator-feeds/permissions/remove')
@@ -776,7 +785,7 @@ def accounts_extras(self):
 
     self.add('VOID', 'accounts', 'pages')
     self.add('AUTH', 'accounts', 'pages/projects')
-    self.add('AUTH', 'accounts', 'pages/projects', 'deployments')
+    self.add('AUTH', 'accounts', 'pages/projects', 'deployments', content_type={'POST':'multipart/form-data'})
     self.add('VOID', 'accounts', 'pages/projects', 'deployments', 'history')
     self.add('AUTH', 'accounts', 'pages/projects', 'deployments', 'history/logs')
     self.add('AUTH', 'accounts', 'pages/projects', 'deployments', 'retry')
@@ -845,7 +854,7 @@ def zones_extras(self):
 
     self.add('VOID', 'zones', 'hostnames')
     self.add('AUTH', 'zones', 'hostnames/settings')
-    self.add('AUTH', 'zones', 'snippets')
+    self.add('AUTH', 'zones', 'snippets', content_type={'PUT':'multipart/form-data'})
     self.add('AUTH', 'zones', 'snippets', 'content')
     self.add('AUTH', 'zones', 'snippets/snippet_rules')
 
@@ -907,7 +916,7 @@ def zones_api_gateway(self):
     self.add('AUTH', 'zones', 'api_gateway/schemas')
     self.add('VOID', 'zones', 'api_gateway/settings')
     self.add('AUTH', 'zones', 'api_gateway/settings/schema_validation')
-    self.add('AUTH', 'zones', 'api_gateway/user_schemas')
+    self.add('AUTH', 'zones', 'api_gateway/user_schemas', content_type={'POST':'multipart/form-data'})
     self.add('AUTH', 'zones', 'api_gateway/user_schemas', 'operations')
 
 
diff --git a/CloudFlare/cloudflare.py b/CloudFlare/cloudflare.py
index 7787cca..2511010 100644
--- a/CloudFlare/cloudflare.py
+++ b/CloudFlare/cloudflare.py
@@ -9,7 +9,7 @@
 from .read_configs import read_configs
 from .api_v4 import api_v4
 from .api_extras import api_extras
-from .api_decode_from_web import api_decode_from_web
+#from .api_decode_from_web import api_decode_from_web # TODO - remove at some point
 from .api_decode_from_openapi import api_decode_from_openapi
 from .exceptions import CloudFlareError, CloudFlareAPIError, CloudFlareInternalError
 
@@ -65,12 +65,50 @@ def __del__(self):
                 del self.network
                 self.network = None
 
-        def _add_headers(self, headers):
+        def _add_headers(self, method, content_type, data, files):
             """ Add default headers """
-            headers['User-Agent'] = self.user_agent
-            headers['Content-Type'] = 'application/json'
-
-        def _add_auth_headers(self, headers, method):
+            self.headers = {}
+            self.headers['User-Agent'] = self.user_agent
+            if method == 'GET':
+                # no content type needed - except we throw in a default just for grin's
+                self.headers['Content-Type'] = 'application/json'
+            elif content_type is not None and method in content_type:
+                    # this api endpoint and this method requires a specific content type.
+                    ct = content_type[method]
+                    if isinstance(ct, list):
+                        # How do we choose from more than one content type?
+                        for t in ct:
+                            # we have to match against the data type - arggg! how? XXX/TODO - use first for now
+                            self.headers['Content-Type'] = t
+                            break
+                    else:
+                        self.headers['Content-Type'] = ct
+            else:
+                # default choice 
+                self.headers['Content-Type'] = 'application/json'
+
+            # now adjust Content-Type based on data and files
+            if method != 'GET':
+                if self.headers['Content-Type'] == 'application/json' and isinstance(data, str):
+                    # passing javascript vs JSON
+                    self.headers['Content-Type'] = 'application/javascript'
+                if self.headers['Content-Type'] == 'application/json' and isinstance(data, (bytes,bytearray)):
+                    # passing binary file vs JSON
+                    self.headers['Content-Type'] = 'application/octet-stream'
+                if data and self.headers['Content-Type'] == 'multipart/form-data':
+                    # convert from params to files (i.e multipart/form-data)
+                    files = {}
+                    for k,v in data.items():
+                        # files[k] = (None, v, 'application/json')
+                        files[k] = (None, json.dumps(v), 'application/json')
+                    data = None
+                if files:
+                    # overwrite Content-Type as we are uploading data
+                    self.headers['Content-Type'] = 'multipart/form-data'
+                    # however something isn't right and this works ... look at again later!
+                    del self.headers['Content-Type']
+
+        def _add_auth_headers(self, method):
             """ Add authentication headers """
 
             v = 'email' + '.' + method.lower()
@@ -93,22 +131,22 @@ def _add_auth_headers(self, headers, method):
 
             if api_email is None and api_token is not None:
                 # post issue-114 - token is used
-                headers['Authorization'] = 'Bearer %s' % (api_token)
+                self.headers['Authorization'] = 'Bearer %s' % (api_token)
             elif api_email is None and api_key is not None:
                 # pre issue-114 - key is used vs token - backward compat
-                headers['Authorization'] = 'Bearer %s' % (api_key)
+                self.headers['Authorization'] = 'Bearer %s' % (api_key)
             elif api_email is not None and api_key is not None:
                 # boring old school email/key methodology (token ignored)
-                headers['X-Auth-Email'] = api_email
-                headers['X-Auth-Key'] = api_key
+                self.headers['X-Auth-Email'] = api_email
+                self.headers['X-Auth-Key'] = api_key
             elif api_email is not None and api_token is not None:
                 # boring old school email/key methodology (token ignored)
-                headers['X-Auth-Email'] = api_email
-                headers['X-Auth-Key'] = api_token
+                self.headers['X-Auth-Email'] = api_email
+                self.headers['X-Auth-Key'] = api_token
             else:
                 raise CloudFlareInternalError(0, 'coding issue!')
 
-        def _add_certtoken_headers(self, headers, method):
+        def _add_certtoken_headers(self, method):
             """ Add authentication headers """
 
             v = 'certtoken' + '.' + method.lower()
@@ -119,56 +157,58 @@ def _add_certtoken_headers(self, headers, method):
 
             if api_certtoken is None:
                 raise CloudFlareAPIError(0, 'no cert token defined')
-            headers['X-Auth-User-Service-Key'] = api_certtoken
+            self.headers['X-Auth-User-Service-Key'] = api_certtoken
 
-        def do_no_auth(self, method, parts, identifiers, params=None, data=None, files=None):
+        def do_not_available(self, method, parts, identifiers, params=None, data=None, content_type=None, files=None):
             """ Cloudflare v4 API"""
 
-            headers = {}
-            self._add_headers(headers)
-            return self._call(method, headers, parts, identifiers, params, data, files)
+            # base class simply returns not available - no processing of any arguments
+            raise CloudFlareAPIError(0, 'call not available')
 
-        def do_auth(self, method, parts, identifiers, params=None, data=None, files=None):
+        def do_no_auth(self, method, parts, identifiers, params=None, data=None, content_type=None, files=None):
             """ Cloudflare v4 API"""
 
-            headers = {}
-            self._add_headers(headers)
-            self._add_auth_headers(headers, method)
-            if isinstance(data, str):
-                # passing javascript vs JSON
-                headers['Content-Type'] = 'application/javascript'
-            if files:
-                # overwrite Content-Type as we are uploading data
-                headers['Content-Type'] = 'multipart/form-data'
-                # however something isn't right and this works ... look at again later!
-                del headers['Content-Type']
-            return self._call(method, headers, parts, identifiers, params, data, files)
-
-        def do_auth_unwrapped(self, method, parts, identifiers, params=None, data=None, files=None):
+            self._add_headers(method, content_type, data, files)
+            # We decide at this point if we are sending json or string data
+            if isinstance(data, (str,bytes,bytearray)):
+                return self._call(method, parts, identifiers, params, data, None, files)
+            else:
+                return self._call(method, parts, identifiers, params, None, data, files)
+
+        def do_auth(self, method, parts, identifiers, params=None, data=None, content_type=None, files=None):
             """ Cloudflare v4 API"""
 
-            headers = {}
-            self._add_headers(headers)
-            self._add_auth_headers(headers, method)
-            if isinstance(data, str):
-                # passing javascript vs JSON
-                headers['Content-Type'] = 'application/javascript'
-            if files:
-                # overwrite Content-Type as we are uploading data
-                headers['Content-Type'] = 'multipart/form-data'
-                # however something isn't right and this works ... look at again later!
-                del headers['Content-Type']
-            return self._call_unwrapped(method, headers, parts, identifiers, params, data, files)
-
-        def do_certauth(self, method, parts, identifiers, params=None, data=None, files=None):
+            self._add_headers(method, content_type, data, files)
+            self._add_auth_headers(method)
+            # We decide at this point if we are sending json or string data
+            if isinstance(data, (str,bytes,bytearray)):
+                return self._call(method, parts, identifiers, params, data, None, files)
+            else:
+                return self._call(method, parts, identifiers, params, None, data, files)
+
+        def do_auth_unwrapped(self, method, parts, identifiers, params=None, data=None, content_type=None, files=None):
             """ Cloudflare v4 API"""
 
-            headers = {}
-            self._add_headers(headers)
-            self._add_certtoken_headers(headers, method)
-            return self._call(method, headers, parts, identifiers, params, data, files)
+            self._add_headers(method, content_type, data, files)
+            self._add_auth_headers(method)
+            # We decide at this point if we are sending json or string data
+            if isinstance(data, (str,bytes,bytearray)):
+                return self._call_unwrapped(method, parts, identifiers, params, data, None, files)
+            else:
+                return self._call_unwrapped(method, parts, identifiers, params, None, data, files)
 
-        def _call_network(self, method, headers, parts, identifiers, params, data, files):
+        def do_certauth(self, method, parts, identifiers, params=None, data=None, content_type=None, files=None):
+            """ Cloudflare v4 API"""
+
+            self._add_headers(method, content_type, data, files)
+            self._add_certtoken_headers(method)
+            # We decide at this point if we are sending json or string data
+            if isinstance(data, (str,bytes,bytearray)):
+                return self._call(method, parts, identifiers, params, data, None, files)
+            else:
+                return self._call(method, parts, identifiers, params, None, data, files)
+
+        def _call_network(self, method, headers, parts, identifiers, params, data_str, data_json, files):
             """ Cloudflare v4 API"""
 
             if (method is None) or (parts[0] is None):
@@ -177,7 +217,7 @@ def _call_network(self, method, headers, parts, identifiers, params, data, files
 
             # By this point we know that parts[] has 5 elements and identifiers[] has 4 elements (even if some are None)
 
-            if parts[1] is not None or (data is not None and method == 'GET'):
+            if parts[1] is not None or (data_str is not None and method == 'GET'):
                 if identifiers[0] is None:
                     raise CloudFlareAPIError(0, 'You must specify first identifier')
                 if identifiers[1] is None:
@@ -210,20 +250,20 @@ def _call_network(self, method, headers, parts, identifiers, params, data, files
                     if parts[4]:
                         url += '/' + parts[4]
 
-            if files and data:
-                # Can't send data and form data - so move data into files and send as multipart/form-data
+            if files and data_json:
+                # Can't send data_json and form data - so move data_json into files and send as multipart/form-data
                 new_files = []
                 new_files += [(f, (files[f].name, files[f])) for f in files]
-                new_files += [(d, (None, data[d])) for d in data]
+                new_files += [(d, (None, data_json[d])) for d in data_json]
                 files = tuple(new_files)
-                data = None
+                data_json = None
 
             if self.logger:
-                msg = build_curl(method, url, headers, params, data, files)
+                msg = build_curl(method, url, headers, params, data_str, data_json, files)
                 self.logger.debug('Call: emulated curl command ...\n%s', msg)
 
             try:
-                response = self.network(method, url, headers, params, data, files)
+                response = self.network(method, url, headers, params, data_str, data_json, files)
             except requests_RequestException as e:
                 if self.logger:
                     self.logger.debug('Call: requests exception! "%s"', e)
@@ -254,11 +294,14 @@ def _call_network(self, method, headers, parts, identifiers, params, data, files
             response_code = response.status_code
             response_data = response.content
             if not isinstance(response_data, (str, bytes, bytearray)):
-                response_data = response_data.decode("utf-8")
+                response_data = response_data.decode('utf-8')
 
             if self.logger:
                 if 'text/' == response_type[0:5] or response_type in ['application/javascript', 'application/json']:
-                    self.logger.debug('Response: %d, %s, %s', response_code, response_type, response_data[0:100])
+                    if len(response_data) > 100:
+                        self.logger.debug('Response: %d, %s, %s...', response_code, response_type, response_data[0:100])
+                    else:
+                        self.logger.debug('Response: %d, %s, %s', response_code, response_type, response_data)
                 else:
                     self.logger.debug('Response: %d, %s, %s', response_code, response_type, '...')
 
@@ -304,13 +347,13 @@ def _call_network(self, method, headers, parts, identifiers, params, data, files
 
             return [response_type, response_code, response_data]
 
-        def _raw(self, method, headers, parts, identifiers, params, data, files):
+        def _raw(self, method, headers, parts, identifiers, params, data_str, data_json, files):
             """ Cloudflare v4 API"""
 
             [response_type, response_code, response_data] = self._call_network(method,
                                                                                headers, parts,
                                                                                identifiers,
-                                                                               params, data, files)
+                                                                               params, data_str, data_json, files)
 
             if response_type == 'application/json':
                 # API says it's JSON; so it better be parsable as JSON
@@ -393,7 +436,7 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                         response_data = {'success': False,
                                          'code': response_code,
                                          'result': response_data}
-            elif response_type in ['text/plain', 'application/octet-stream']:
+            elif response_type in ['text/plain', 'text/csv', 'application/octet-stream']:
                 # API says it's text; but maybe it's actually JSON? - should be fixed in API
                 if hasattr(response_data, 'decode'):
                     response_data = response_data.decode('utf-8')
@@ -413,8 +456,10 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                         response_data = {'success': False,
                                          'code': response_code,
                                          'result': response_data}
-            elif response_type in ['text/javascript', 'application/javascript']:
+            elif response_type in ['text/javascript', 'application/javascript', 'text/html']:
                 # used by Cloudflare workers
+                if hasattr(response_data, 'decode'):
+                    response_data = response_data.decode('utf-8')
                 if response_code == requests_codes.ok:
                     # 200 ok
                     response_data = {'success': True,
@@ -424,18 +469,6 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
                     response_data = {'success': False,
                                      'code': response_code,
                                      'result': str(response_data)}
-            elif response_type == 'text/html':
-                # used by media for preview
-                if response_code == requests_codes.ok:
-                    # 200 ok
-                    response_data = {'success': True,
-                                     'result': str(response_data)}
-                else:
-                    # 3xx & 4xx errors
-                    response_data = {'success': False,
-                                     'code': response_code,
-                                     'result': str(response_data)}
-
             elif response_type[0:6] in ['audio/', 'image/', 'video/']:
                 # raw - just pass thru
                 if response_code == requests_codes.ok:
@@ -460,10 +493,10 @@ def _raw(self, method, headers, parts, identifiers, params, data, files):
             # it would be nice to return the error code and content type values; but not quite yet
             return response_data
 
-        def _call(self, method, headers, parts, identifiers, params, data, files):
+        def _call(self, method, parts, identifiers, params, data_str, data_json, files):
             """ Cloudflare v4 API"""
 
-            response_data = self._raw(method, headers, parts, identifiers, params, data, files)
+            response_data = self._raw(method, self.headers, parts, identifiers, params, data_str, data_json, files)
 
             # Sanatize the returned results - just in case API is messed up
             if 'success' not in response_data:
@@ -555,15 +588,20 @@ def _call(self, method, headers, parts, identifiers, params, data, files):
                     result = response_data
             if self.logger:
                 if isinstance(result, str):
+                    if len(result) > 100:
+                        self.logger.debug('Response: %s...', result[0:100].replace('\n', ' '))
+                    else:
+                        self.logger.debug('Response: %s', result.replace('\n', ' '))
+                elif isinstance(result, (bytes,bytearray)):
                     self.logger.debug('Response: %s', result[0:100])
                 else:
                     self.logger.debug('Response: %s', '...')
             return result
 
-        def _call_unwrapped(self, method, headers, parts, identifiers, params, data, files):
+        def _call_unwrapped(self, method, parts, identifiers, params, data_str, data_json, files):
             """ Cloudflare v4 API"""
 
-            response_data = self._raw(method, headers, parts, identifiers, params, data, files)
+            response_data = self._raw(method, self.headers, parts, identifiers, params, data_str, data_json, files)
             if self.logger:
                 self.logger.debug('Response: %s', response_data)
             result = response_data
@@ -589,59 +627,16 @@ def _read_from_web(self, url):
 
             return response.text
 
-    class _add_unused():
+    class add_base():
         """ Cloudflare v4 API"""
 
-        def __init__(self, base, parts):
-            """ Cloudflare v4 API"""
-
-            self._base = base
-            self._parts_unused = parts
-
-        def __call__(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            # This is the same as a get()
-            return self.get(identifier1, identifier2, identifier3, identifier4, params=params, data=data)
-
-        def __str__(self):
-            """ Cloudflare v4 API"""
-
-            return '[%s]' % ('/' + '/:id/'.join(filter(None, self._parts_unused)))
-
-        def get(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            raise CloudFlareAPIError(0, 'not found')
-
-        def patch(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            raise CloudFlareAPIError(0, 'not found')
-
-        def post(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None, files=None):
-            """ Cloudflare v4 API"""
-
-            raise CloudFlareAPIError(0, 'not found')
-
-        def put(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            raise CloudFlareAPIError(0, 'not found')
-
-        def delete(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            raise CloudFlareAPIError(0, 'not found')
-
-    class _add_no_auth():
-        """ Cloudflare v4 API"""
-
-        def __init__(self, base, parts):
+        def __init__(self, base, parts, content_type=None):
             """ Cloudflare v4 API"""
 
             self._base = base
             self._parts = parts
+            self._content_type = content_type
+            self._do = None
 
         def __call__(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
             """ Cloudflare v4 API"""
@@ -657,165 +652,99 @@ def __str__(self):
         def get(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
             """ Cloudflare v4 API"""
 
-            return self._base.do_no_auth('GET', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+            return self._do('GET', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type)
 
         def patch(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
             """ Cloudflare v4 API"""
 
-            raise CloudFlareAPIError(0, 'patch() call not available for this endpoint')
+            return self._do('PATCH', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type)
 
         def post(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None, files=None):
             """ Cloudflare v4 API"""
 
-            raise CloudFlareAPIError(0, 'post() call not available for this endpoint')
+            return self._do('POST', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type, files)
 
-        def put(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
+        def put(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None, files=None):
             """ Cloudflare v4 API"""
 
-            raise CloudFlareAPIError(0, 'put() call not available for this endpoint')
+            return self._do('PUT', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type, files)
 
         def delete(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
             """ Cloudflare v4 API"""
 
-            raise CloudFlareAPIError(0, 'delete() call not available for this endpoint')
+            return self._do('DELETE', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type)
 
-    class _add_with_auth():
+    class _add_unused(add_base):
         """ Cloudflare v4 API"""
 
-        def __init__(self, base, parts):
-            """ Cloudflare v4 API"""
-
-            self._base = base
-            self._parts = parts
-
-        def __call__(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
+        def __init__(self, base, parts, content_type):
             """ Cloudflare v4 API"""
 
-            # This is the same as a get()
-            return self.get(identifier1, identifier2, identifier3, identifier4, params=params, data=data)
+            super().__init__(base, parts, content_type)
+            self._do = self._base.do_not_available
 
-        def __str__(self):
-            """ Cloudflare v4 API"""
-
-            return '[%s]' % ('/' + '/:id/'.join(filter(None, self._parts)))
-
-        def get(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            return self._base.do_auth('GET', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
-
-        def patch(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            return self._base.do_auth('PATCH', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
-
-        def post(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None, files=None):
-            """ Cloudflare v4 API"""
-
-            return self._base.do_auth('POST', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, files)
-
-        def put(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            return self._base.do_auth('PUT', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
-
-        def delete(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            return self._base.do_auth('DELETE', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
-
-    class _add_with_auth_unwrapped():
+    class _add_no_auth(add_base):
         """ Cloudflare v4 API"""
 
-        def __init__(self, base, parts):
-            """ Cloudflare v4 API"""
-
-            self._base = base
-            self._parts = parts
-
-        def __call__(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            # This is the same as a get()
-            return self.get(identifier1, identifier2, identifier3, identifier4, params=params, data=data)
-
-        def __str__(self):
-            """ Cloudflare v4 API"""
-
-            return '[%s]' % ('/' + '/:id/'.join(filter(None, self._parts)))
-
-        def get(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
+        def __init__(self, base, parts, content_type):
             """ Cloudflare v4 API"""
 
-            return self._base.do_auth_unwrapped('GET', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+            super().__init__(base, parts, content_type)
+            self._do = self._base.do_no_auth
+            self._valid = True
 
         def patch(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
             """ Cloudflare v4 API"""
 
-            return self._base.do_auth_unwrapped('PATCH', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+            return self._base.do_not_available('PATCH', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type)
 
         def post(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None, files=None):
             """ Cloudflare v4 API"""
 
-            return self._base.do_auth_unwrapped('POST', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, files)
+            return self._base.do_not_available('POST', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type, files)
 
         def put(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
             """ Cloudflare v4 API"""
 
-            return self._base.do_auth_unwrapped('PUT', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+            return self._base.do_not_available('PUT', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type)
 
         def delete(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
             """ Cloudflare v4 API"""
 
-            return self._base.do_auth_unwrapped('DELETE', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+            return self._base.do_not_available('DELETE', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, self._content_type)
 
-    class _add_with_cert_auth():
+    class _add_with_auth(add_base):
         """ Cloudflare v4 API"""
 
-        def __init__(self, base, parts):
-            """ Cloudflare v4 API"""
-
-            self._base = base
-            self._parts = parts
-
-        def __call__(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            # This is the same as a get()
-            return self.get(identifier1, identifier2, identifier3, identifier4, params=params, data=data)
-
-        def __str__(self):
-            """ Cloudflare v4 API"""
-
-            return '[%s]' % ('/' + '/:id/'.join(filter(None, self._parts)))
-
-        def get(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
+        def __init__(self, base, parts, content_type):
             """ Cloudflare v4 API"""
 
-            return self._base.do_certauth('GET', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+            super().__init__(base, parts, content_type)
+            self._do = self._base.do_auth
+            self._valid = True
 
-        def patch(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
-
-            return self._base.do_certauth('PATCH', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+    class _add_with_auth_unwrapped(add_base):
+        """ Cloudflare v4 API"""
 
-        def post(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None, files=None):
+        def __init__(self, base, parts, content_type):
             """ Cloudflare v4 API"""
 
-            return self._base.do_certauth('POST', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data, files)
-
-        def put(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
-            """ Cloudflare v4 API"""
+            super().__init__(base, parts, content_type)
+            self._do = self._base.do_auth_unwrapped
+            self._valid = True
 
-            return self._base.do_certauth('PUT', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+    class _add_with_cert_auth(add_base):
+        """ Cloudflare v4 API"""
 
-        def delete(self, identifier1=None, identifier2=None, identifier3=None, identifier4=None, params=None, data=None):
+        def __init__(self, base, parts, content_type):
             """ Cloudflare v4 API"""
 
-            return self._base.do_certauth('DELETE', self._parts, [identifier1, identifier2, identifier3, identifier4], params, data)
+            super().__init__(base, parts, content_type)
+            self._do = self._base.do_certauth
+            self._valid = True
 
-    def add(self, t, p1, p2=None, p3=None, p4=None, p5=None):
-        """add api call to class"""
+    def add(self, t, p1, p2=None, p3=None, p4=None, p5=None, content_type=None):
+        """ add api call to class"""
 
         a = []
         if p1:
@@ -864,15 +793,15 @@ def add(self, t, p1, p2=None, p3=None, p4=None, p5=None):
             pass
 
         if t == 'VOID':
-            f = self._add_unused(self._base, parts)
+            f = self._add_unused(self._base, parts, content_type=None)
         elif t == 'OPEN':
-            f = self._add_no_auth(self._base, parts)
+            f = self._add_no_auth(self._base, parts, content_type=content_type)
         elif t == 'AUTH':
-            f = self._add_with_auth(self._base, parts)
+            f = self._add_with_auth(self._base, parts, content_type=content_type)
         elif t == 'AUTH_UNWRAPPED':
-            f = self._add_with_auth_unwrapped(self._base, parts)
+            f = self._add_with_auth_unwrapped(self._base, parts, content_type=content_type)
         elif t == 'CERT':
-            f = self._add_with_cert_auth(self._base, parts)
+            f = self._add_with_cert_auth(self._base, parts, content_type=content_type)
         else:
             # should never happen
             raise CloudFlareAPIError(0, 'api load type mismatch')
@@ -888,11 +817,11 @@ def add(self, t, p1, p2=None, p3=None, p4=None, p5=None):
                 setattr(branch, name, f)
 
     def api_list(self):
-        """recursive walk of the api tree returning a list of api calls"""
+        """ recursive walk of the api tree returning a list of api calls"""
         return self._api_list(m=self)
 
     def _api_list(self, m=None, s=''):
-        """recursive walk of the api tree returning a list of api calls"""
+        """ recursive walk of the api tree returning a list of api calls"""
         w = []
         for n in sorted(dir(m)):
             if n[0] == '_':
@@ -907,32 +836,30 @@ def _api_list(self, m=None, s=''):
                 # really should not happen!
                 raise CloudFlareAPIError(0, '%s: not found - should not happen' % (n))
             d = dir(a)
-            if '_base' in d:
-                # it's a known api call - lets show the result and continue down the tree
+            if '_base' not in d:
+                continue
+            # it's a known api call - lets show the result and continue down the tree
+            if '_parts' in d and '_valid' in d:
                 if 'delete' in d or 'get' in d or 'patch' in d or 'post' in d or 'put' in d:
                     # only show the result if a call exists for this part
-                    if '_parts_unused' in d:
-                            # This is an uncallable endpoint - presently no way to return this info
-                            # w.append(str(a)[1:-1] + ' ; UNUSED')
+                    if n[-1] == '_':
+                        if  keyword.iskeyword(n[:-1]):
+                            # should always be a keyword - but now nothing needs to be done
                             pass
-                    if '_parts' in d:
-                        if n[-1] == '_':
-                            if  keyword.iskeyword(n[:-1]):
-                                # should always be a keyword - but now nothing needs to be done
-                                pass
-                            # remove the extra keyword postfix'ed with underscore
-                            w.append(str(a)[1:-1])
-                        else:
-                            # handle underscores by returning the actual API call vs the method name
-                            w.append(str(a)[1:-1])
-                # now recurse downwards into the tree
-                w = w + self._api_list(a, s + '/' + n)
+                        # remove the extra keyword postfix'ed with underscore
+                        w.append(str(a)[1:-1])
+                    else:
+                        # handle underscores by returning the actual API call vs the method name
+                        w.append(str(a)[1:-1])
+            # now recurse downwards into the tree
+            w = w + self._api_list(a, s + '/' + n)
         return w
 
-    def api_from_web(self):
-        """ Cloudflare v4 API"""
-
-        return api_decode_from_web(self._base.api_from_web())
+    # TODO - remove at some point
+    #def api_from_web(self):
+    #    """ Cloudflare v4 API"""
+    #
+    #    return api_decode_from_web(self._base.api_from_web())
 
     def api_from_openapi(self, url):
         """ Cloudflare v4 API"""
diff --git a/CloudFlare/network.py b/CloudFlare/network.py
index 5def88e..e4619f1 100644
--- a/CloudFlare/network.py
+++ b/CloudFlare/network.py
@@ -2,12 +2,12 @@
 
 from urllib.parse import urlparse
 
+import sys
 import requests
 from requests.adapters import HTTPAdapter
 
 from .exceptions import CloudFlareAPIError
 
-
 class CFnetwork:
     """Network for Cloudflare API"""
 
@@ -21,7 +21,7 @@ def __init__(
         self.max_request_retries = max_request_retries
         self.session = None
 
-    def __call__(self, method, url, headers=None, params=None, data=None, files=None):
+    def __call__(self, method, url, headers=None, params=None, data_str=None, data_json=None, files=None):
         """Network for Cloudflare API"""
 
         if self.use_sessions:
@@ -40,85 +40,52 @@ def __call__(self, method, url, headers=None, params=None, data=None, files=None
         method = method.upper()
 
         if method == 'GET':
+            # no data or files
             r = self.session.get(
                 url,
                 headers=headers,
                 params=params,
-                data=data,
                 timeout=self.global_request_timeout,
             )
         elif method == 'POST':
-            if isinstance(data, str):
-                r = self.session.post(
-                    url,
-                    headers=headers,
-                    params=params,
-                    data=data,
-                    files=files,
-                    timeout=self.global_request_timeout,
-                )
-            else:
-                r = self.session.post(
-                    url,
-                    headers=headers,
-                    params=params,
-                    json=data,
-                    files=files,
-                    timeout=self.global_request_timeout,
-                )
+            r = self.session.post(
+                url,
+                headers=headers,
+                params=params,
+                data=data_str,
+                json=data_json,
+                files=files,
+                timeout=self.global_request_timeout,
+            )
         elif method == 'PUT':
-            if isinstance(data, str):
-                r = self.session.put(
-                    url,
-                    headers=headers,
-                    params=params,
-                    data=data,
-                    timeout=self.global_request_timeout,
-                )
-            else:
-                r = self.session.put(
-                    url,
-                    headers=headers,
-                    params=params,
-                    json=data,
-                    timeout=self.global_request_timeout,
-                )
+            r = self.session.put(
+                url,
+                headers=headers,
+                params=params,
+                data=data_str,
+                json=data_json,
+                files=files,
+                timeout=self.global_request_timeout,
+            )
         elif method == 'DELETE':
-            if isinstance(data, str):
-                r = self.session.delete(
-                    url,
-                    headers=headers,
-                    params=params,
-                    data=data,
-                    timeout=self.global_request_timeout,
-                )
-            else:
-                r = self.session.delete(
-                    url,
-                    headers=headers,
-                    params=params,
-                    json=data,
-                    timeout=self.global_request_timeout,
-                )
+            r = self.session.delete(
+                url,
+                headers=headers,
+                params=params,
+                data=data_str,
+                json=data_json,
+                timeout=self.global_request_timeout,
+            )
         elif method == 'PATCH':
-            if isinstance(data, str):
-                r = self.session.request(
-                    'PATCH',
-                    url,
-                    headers=headers,
-                    params=params,
-                    data=data,
-                    timeout=self.global_request_timeout,
-                )
-            else:
-                r = self.session.request(
-                    'PATCH',
-                    url,
-                    headers=headers,
-                    params=params,
-                    json=data,
-                    timeout=self.global_request_timeout,
-                )
+            r = self.session.request(
+                'PATCH',
+                url,
+                headers=headers,
+                params=params,
+                data=data_str,
+                json=data_json,
+                timeout=self.global_request_timeout,
+            )
         else:
             # should never happen
             raise CloudFlareAPIError(0, 'method not supported')
diff --git a/CloudFlare/utils.py b/CloudFlare/utils.py
index aac10b0..4ee9014 100644
--- a/CloudFlare/utils.py
+++ b/CloudFlare/utils.py
@@ -33,7 +33,7 @@ def sanitize_secrets(secrets):
 
     return secrets_copy
 
-def build_curl(method, url, headers, params, data, files):
+def build_curl(method, url, headers, params, data_str, data_json, files):
     """ misc utilities  for Cloudflare API"""
 
     msg = []
@@ -52,13 +52,29 @@ def build_curl(method, url, headers, params, data, files):
         if k is None:
             continue
         msg.append('            -H "%s: %s" \\' % (k, h[k]))
-    # data
-    if data is not None:
+    # data_str
+    if data_str is not None:
+        if isinstance(data_str, (bytes,bytearray)):
+            if len(data_str) > 100:
+                msg.append('            --data-binary \'%s ...\' \\' % (str(data_str[0:100]).replace('\n', '\n')))
+            else:
+                msg.append('            --data-binary \'%s\' \\' % (str(data_str).replace('\n', '\n')))
+        else:
+            if len(data_str) > 100:
+                msg.append('            --data \'%s ...\' \\' % (str(data_str[0:100]).replace('\n', ' ')))
+            else:
+                msg.append('            --data \'%s\' \\' % (str(data_str).replace('\n', ' ')))
+    # data_json
+    if data_json is not None:
+        print('===', type(data_json), len(data_json))
         try:
-            str_data = json.dumps(data)
+            s = json.dumps(data_json)
         except:
-            str_data = str(data)
-        msg.append('            --data \'%s\' \\' % (str_data.replace('\n', ' ')))
+            s = str(data_json)
+        if len(s) > 100:
+            msg.append('            --data \'%s ...\' \\' % (s[0:100].replace('\n', ' ')))
+        else:
+            msg.append('            --data \'%s\' \\' % (s.replace('\n', ' ')))
     # files
     if files is not None:
         if isinstance(files, (list, tuple)):
diff --git a/Makefile b/Makefile
index 8839ebd..82c177c 100644
--- a/Makefile
+++ b/Makefile
@@ -96,7 +96,7 @@ lint:
 openapi:
 	@tmp=/tmp/_$$$$_ ; \
 	$(PYTHON) -m cli4 --dump | sort > $$tmp.1 ; \
-	$(PYTHON) -m cli4 --openapi $(OPENAPI_URL) | sed -e 's/^[A-Z][A-Z]*  *//' -e 's/?.*//' -e 's/\/:[a-z][A-Za-z_]*/\/:id/g' -e 's/\/:[a-z][A-Za-z_]*}/\/:id/g' -e 's/:id\/:id/:id/' -e 's/\/:id$$//' -e 's/\/:id$$//' -e 's/\/:id ;/ ;/' -e 's/ ; Content-Type: .*//' -e 's/\/$$//' | sort -u > $$tmp.2 ; \
+	$(PYTHON) -m cli4 --openapi $(OPENAPI_URL) | tee $$tmp.5 | sed -e 's/^[A-Z][A-Z]*  *//' -e 's/?.*//' -e 's/\/:[a-z][A-Za-z_]*/\/:id/g' -e 's/\/:[a-z][A-Za-z_]*}/\/:id/g' -e 's/:id\/:id/:id/' -e 's/\/:id$$//' -e 's/\/:id$$//' -e 's/\/:id ;/ ;/' -e 's/ ; Content-Type: .*//' -e 's/\/$$//' | sort -u > $$tmp.2 ; \
 	egrep -v '; deprecated' < $$tmp.2 | sed -e 's/ ; .*//' | diff $$tmp.1 - > $$tmp.3 ; \
 	echo "In code:" ; \
 	egrep '< ' < $$tmp.3 | sed -e 's/< /    /' | sort | tee $$tmp.4 ; \
@@ -104,6 +104,8 @@ openapi:
 	egrep '> ' < $$tmp.3 | sed -e 's/> /    /' | sort | sed -e "s/\//self.add('AUTH', '/" -e "s/$$/'\)/" -e "s/\/:id\//', '/g" ; \
 	echo "Deprecated:" ; \
 	egrep '; deprecated' < $$tmp.2 | while read cmd x deprecated deprecated_date ; do egrep "$$cmd" $$tmp.4 | sed -e "s/$$/ ; deprecated $$deprecated_date/" ; done | sort | uniq ; \
+	echo "Content-Type's:" ; \
+	egrep ';' < $$tmp.5 | egrep -v '; deprecated' | egrep -v ' ; Content-Type: application/json' | sed -e 's/^/    /' ; \
 	rm $$tmp.?
 
 TUNA_CLI4_TEST_COMMAND = "/ips"
diff --git a/requirements.txt b/requirements.txt
index c10f440..6675549 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 requests>=2.4.2
 pyyaml
 jsonlines
-beautifulsoup4
 pytest
+pytest-cov
diff --git a/setup.py b/setup.py
index 30bb483..3cd716c 100755
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,7 @@ def main():
         packages=['cli4', 'examples']+find_packages(),
         include_package_data=True,
         data_files = [('share/man/man1', ['cli4/cli4.1'])],
-        install_requires=['requests', 'pyyaml', 'jsonlines', 'beautifulsoup4'],
+        install_requires=['requests', 'pyyaml', 'jsonlines'],
         keywords='cloudflare',
         entry_points={
             'console_scripts': [