From 387f7aa3d8706fc56f3369c0fabe125542f97700 Mon Sep 17 00:00:00 2001 From: kaigong-iex Date: Fri, 18 Mar 2022 22:03:15 +0000 Subject: [PATCH] code autogen --- pyEX/client.py | 81 ++- pyEX/common/urls.py | 28 +- pyEX/platform/__init__.py | 40 +- pyEX/platform/platform.py | 1280 +++++++++++++++++++++++++++++-------- 4 files changed, 1114 insertions(+), 315 deletions(-) diff --git a/pyEX/client.py b/pyEX/client.py index d94bd8d..e0e51ad 100644 --- a/pyEX/client.py +++ b/pyEX/client.py @@ -140,28 +140,36 @@ from .options import optionExpirations, options, optionsDF, stockOptions, stockOptionsDF from .points import points, pointsDF from .platform import ( - createDataset, - createDatasetAsync, - deleteData, - deleteDataAsync, - deleteDataset, - deleteDatasetAsync, + createDataJob, + createDataJobAsync, + listDataJobs, + listDataJobsAsync, + queryDataJob, + queryDataJobAsync, + listDataJobsById, + listDataJobsByIdAsync, + getDataJobLogFile, + getDataJobLogFileAsync, + awsOnboarding, + awsOnboardingAsync, + getplatformswaggerjson, + getplatformswaggerjsonAsync, listDatasets, listDatasetsAsync, - listDatasetsDF, - listJobs, + getDataset, + getDatasetAsync, + registerDataset, + registerDatasetAsync, loadData, loadDataAsync, modifyDataset, modifyDatasetAsync, - modifyData, - modifyDataAsync, - query, - queryAsync, - queryDF, - queryMeta, - queryMetaAsync, - queryMetaDF, + deleteDataset, + deleteDatasetAsync, + deleteData, + deleteDataAsync, + getDataSourceContent, + getDataSourceContentAsync, ) from .premium import ( accountingQualityAndRiskMatrixAuditAnalytics, @@ -819,29 +827,36 @@ DEFAULT_API_LIMIT = 5 _INCLUDE_FUNCTIONS_PLATFORM = [ - # Platform - ("createDataset", createDataset), - ("createDatasetAsync", createDatasetAsync), - ("deleteData", deleteData), - ("deleteDataAsync", deleteDataAsync), - ("deleteDataset", deleteDataset), - ("deleteDatasetAsync", deleteDatasetAsync), + ("createDataJob", createDataJob), + ("createDataJobAsync", createDataJobAsync), + ("listDataJobs", listDataJobs), + ("listDataJobsAsync", listDataJobsAsync), + ("queryDataJob", queryDataJob), + ("queryDataJobAsync", queryDataJobAsync), + ("listDataJobsById", listDataJobsById), + ("listDataJobsByIdAsync", listDataJobsByIdAsync), + ("getDataJobLogFile", getDataJobLogFile), + ("getDataJobLogFileAsync", getDataJobLogFileAsync), + ("awsOnboarding", awsOnboarding), + ("awsOnboardingAsync", awsOnboardingAsync), + ("getplatformswaggerjson", getplatformswaggerjson), + ("getplatformswaggerjsonAsync", getplatformswaggerjsonAsync), ("listDatasets", listDatasets), ("listDatasetsAsync", listDatasetsAsync), - ("listDatasetsDF", listDatasetsDF), - ("listJobs", listJobs), + ("getDataset", getDataset), + ("getDatasetAsync", getDatasetAsync), + ("registerDataset", registerDataset), + ("registerDatasetAsync", registerDatasetAsync), ("loadData", loadData), ("loadDataAsync", loadDataAsync), ("modifyDataset", modifyDataset), ("modifyDatasetAsync", modifyDatasetAsync), - ("modifyData", modifyData), - ("modifyDataAsync", modifyDataAsync), - ("query", query), - ("queryAsync", queryAsync), - ("queryDF", queryDF), - ("queryMeta", queryMeta), - ("queryMetaAsync", queryMetaAsync), - ("queryMetaDF", queryMetaDF), + ("deleteDataset", deleteDataset), + ("deleteDatasetAsync", deleteDatasetAsync), + ("deleteData", deleteData), + ("deleteDataAsync", deleteDataAsync), + ("getDataSourceContent", getDataSourceContent), + ("getDataSourceContentAsync", getDataSourceContentAsync), ] _INCLUDE_FUNCTIONS_RULES = [ diff --git a/pyEX/common/urls.py b/pyEX/common/urls.py index 0ec05a1..51f3e48 100644 --- a/pyEX/common/urls.py +++ b/pyEX/common/urls.py @@ -92,6 +92,7 @@ def _post( token_in_params=True, filter="", format="json", + maximumValidationErrors=None, ): token = token or os.environ.get("IEX_TOKEN") if version == "sandbox": @@ -105,6 +106,7 @@ def _post( token_in_params=token_in_params, format=format, filter=filter, + maximumValidationErrors=maximumValidationErrors, ) return _postIEXCloud( url=url, @@ -116,6 +118,7 @@ def _post( token_in_params=token_in_params, format=format, filter=filter, + maximumValidationErrors=maximumValidationErrors, ) @@ -129,6 +132,7 @@ def _put( token_in_params=True, filter="", format="json", + maximumValidationErrors=None, ): token = token or os.environ.get("IEX_TOKEN") if version == "sandbox": @@ -142,6 +146,7 @@ def _put( token_in_params=token_in_params, format=format, filter=filter, + maximumValidationErrors=maximumValidationErrors, ) return _putIEXCloud( url=url, @@ -153,6 +158,7 @@ def _put( token_in_params=token_in_params, format=format, filter=filter, + maximumValidationErrors=maximumValidationErrors, ) @@ -409,11 +415,7 @@ async def _getIEXCloudBaseAsync( urlparse(url).geturl(), proxy=_PYEX_PROXIES, params=params ) as resp: - resp = requests.get( - urlparse(url).geturl(), proxies=_PYEX_PROXIES, params=params - ) - - if resp.status_code == 429: + if resp.status == 429: tries += 1 await asyncio.sleep(random() * 0.5 * tries) @@ -422,7 +424,7 @@ async def _getIEXCloudBaseAsync( return await resp.json() elif format == "binary": return await resp.read() - return resp.text() + return await resp.text() else: # break and hit the exception case break @@ -479,6 +481,7 @@ def _pppIEXCloudBase( format="json", token_in_params=True, verb="post", + maximumValidationErrors=None, ): """for iex cloud""" url = base_url.format(version=version) + url @@ -494,6 +497,9 @@ def _pppIEXCloudBase( if filter: params["filter"] = filter + if maximumValidationErrors: + params["maximumValidationErrors"] = maximumValidationErrors + if _PYEX_DEBUG: print(urlparse(url).geturl()) @@ -522,6 +528,7 @@ def _postIEXCloud( filter="", format="json", token_in_params=True, + maximumValidationErrors=None, ): return _pppIEXCloudBase( base_url=_URL_PREFIX_CLOUD, @@ -534,6 +541,7 @@ def _postIEXCloud( filter=filter, format=format, token_in_params=token_in_params, + maximumValidationErrors=maximumValidationErrors, verb="post", ) @@ -548,6 +556,7 @@ def _putIEXCloud( filter="", format="json", token_in_params=True, + maximumValidationErrors=None, ): return _pppIEXCloudBase( base_url=_URL_PREFIX_CLOUD, @@ -560,6 +569,7 @@ def _putIEXCloud( filter=filter, format=format, token_in_params=token_in_params, + maximumValidationErrors=maximumValidationErrors, verb="put", ) @@ -633,7 +643,7 @@ async def _pppIEXCloudBaseAsync( if resp.status == 200: if format == "json": return await resp.json() - return resp.text() + return await resp.text() raise PyEXception("Response %d - " % resp.status, await resp.text()) @@ -725,6 +735,7 @@ def _postIEXCloudSandbox( filter="", format="json", token_in_params=True, + maximumValidationErrors=None, ): return _pppIEXCloudBase( base_url=_URL_PREFIX_CLOUD_SANDBOX, @@ -737,6 +748,7 @@ def _postIEXCloudSandbox( filter=filter, format=format, token_in_params=token_in_params, + maximumValidationErrors=maximumValidationErrors, verb="post", ) @@ -751,6 +763,7 @@ def _putIEXCloudSandbox( filter="", format="json", token_in_params=True, + maximumValidationErrors=None, ): return _pppIEXCloudBase( base_url=_URL_PREFIX_CLOUD_SANDBOX, @@ -763,6 +776,7 @@ def _putIEXCloudSandbox( filter=filter, format=format, token_in_params=token_in_params, + maximumValidationErrors=maximumValidationErrors, verb="put", ) diff --git a/pyEX/platform/__init__.py b/pyEX/platform/__init__.py index 8aeda1a..93a715d 100644 --- a/pyEX/platform/__init__.py +++ b/pyEX/platform/__init__.py @@ -7,26 +7,34 @@ # from .platform import ( - createDataset, - createDatasetAsync, - deleteData, - deleteDataset, - deleteDataAsync, - deleteDatasetAsync, + createDataJob, + createDataJobAsync, + listDataJobs, + listDataJobsAsync, + queryDataJob, + queryDataJobAsync, + listDataJobsById, + listDataJobsByIdAsync, + getDataJobLogFile, + getDataJobLogFileAsync, + awsOnboarding, + awsOnboardingAsync, + getplatformswaggerjson, + getplatformswaggerjsonAsync, listDatasets, listDatasetsAsync, - listDatasetsDF, - listJobs, + getDataset, + getDatasetAsync, + registerDataset, + registerDatasetAsync, loadData, loadDataAsync, modifyDataset, modifyDatasetAsync, - modifyData, - modifyDataAsync, - query, - queryAsync, - queryDF, - queryMeta, - queryMetaAsync, - queryMetaDF, + deleteDataset, + deleteDatasetAsync, + deleteData, + deleteDataAsync, + getDataSourceContent, + getDataSourceContentAsync, ) diff --git a/pyEX/platform/platform.py b/pyEX/platform/platform.py index d1d345b..b222720 100644 --- a/pyEX/platform/platform.py +++ b/pyEX/platform/platform.py @@ -33,6 +33,7 @@ def _queryURL( id="", key="", subkey="", + date="", range=None, calendar=False, limit=1, @@ -62,6 +63,8 @@ def _queryURL( base_url += "/{}".format(_quoteSymbols(key)) if provider and id and key and subkey: base_url += "/{}".format(_quoteSymbols(subkey)) + if provider and id and key and subkey and date: + base_url += "/{}".format(_quoteSymbols(date)) base_url += "?" @@ -145,6 +148,18 @@ def _queryMetaUrl(provider="", id="", key="", subkey=""): return url +def _queryURLLoose( + basePath, + **extra_params, +): + base_url = basePath + base_url += "?" + if extra_params: + base_url += "&".join("{}={}".format(k, v) for k, v in extra_params.items()) + + return base_url + + def queryMeta( provider="", id="", @@ -289,391 +304,1138 @@ async def queryAsync( ) -def listJobs( +# CRUD code +# def listJobs( +# provider, +# type="ingest", +# jobId="", +# token="", +# version="stable", +# filter="", +# format="json", +# ): +# url = "jobs/{}/{}".format(provider, type) + +# if jobId: +# url += "/{}".format(jobId) + +# return _get(url=url, token=token, version=version, filter=filter, format=format) + + +# def listDatasets( +# provider="CORE", id="", token="", version="stable", filter="", format="json" +# ): +# base_url = _queryURL(provider=provider, id=id) +# return _get( +# url=base_url, token=token, version=version, filter=filter, format=format +# ) + + +# @wraps(listDatasets) +# def listDatasetsDF(*args, **kwargs): +# return pd.DataFrame(listDatasets(*args, **kwargs)) + + +# async def listDatasetsAsync( +# provider="CORE", id="", token="", version="stable", filter="", format="json" +# ): +# base_url = _queryURL(provider=provider, id=id) +# return await _getAsync( +# url=base_url, token=token, version=version, filter=filter, format=format +# ) + + +# def createDataset( +# provider, id="", schema=None, token="", version="stable", filter="", format="json" +# ): +# base_url = _queryURL(provider=provider, limit=None, basePath="datasets") + +# # TODO schema validation +# return _post( +# url=base_url, +# json=schema, +# token=token, +# version=version, +# token_in_params=True, +# format=format, +# ) + + +# async def createDatasetAsync( +# provider, id="", schema=None, token="", version="stable", filter="", format="json" +# ): +# base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + +# # TODO schema validation +# return await _postAsync( +# url=base_url, +# json=schema, +# token=token, +# version=version, +# token_in_params=True, +# format=format, +# ) + + +# def loadData( +# provider, +# id, +# data, +# dataType="", +# token="", +# version="stable", +# filter="", +# format="json", +# overwrite=False, +# maximumValidationErrors=None, +# ): +# base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + +# # data interpolation +# if not dataType: +# data, headers = _interpolateDatatype(data) +# else: +# headers = {"content-type": dataType} + +# # TODO schema validation +# if overwrite: +# return _put( +# url=base_url, +# data=data, +# token=token, +# version=version, +# filter=filter, +# format=format, +# token_in_params=True, +# headers=headers, +# maximumValidationErrors=maximumValidationErrors, +# ) +# else: +# return _post( +# url=base_url, +# data=data, +# token=token, +# version=version, +# filter=filter, +# format=format, +# token_in_params=True, +# headers=headers, +# maximumValidationErrors=maximumValidationErrors, +# ) + + +# async def loadDataAsync( +# provider, id, data, token="", version="stable", filter="", format="json" +# ): +# base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") +# # TODO schema validation +# return await _putAsync( +# url=base_url, +# data=data, +# token=token, +# version=version, +# token_in_params=True, +# format=format, +# ) + + +# def modifyDataset( +# provider, id, schema, token="", version="stable", filter="", format="json" +# ): +# base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") +# return _patch( +# url=base_url, +# json=schema, +# token=token, +# version=version, +# token_in_params=True, +# format=format, +# ) + + +# async def modifyDatasetAsync( +# provider, id, schema, token="", version="stable", filter="", format="json" +# ): +# base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") +# return await _patchAsync( +# url=base_url, +# json=schema, +# token=token, +# version=version, +# token_in_params=True, +# format=format, +# ) + + +# def modifyData( +# transforms=None, +# provider="CORE", +# id="", +# key="", +# subkey="", +# range=None, +# calendar=False, +# limit=1, +# subattribute="", +# dateField=None, +# from_=None, +# to_=None, +# on=None, +# last=0, +# first=0, +# sort="", +# interval=None, +# token="", +# version="stable", +# filter="", +# format="json", +# **extra_params, +# ): +# base_url = _queryURL( +# provider=provider, +# id=id, +# key=key, +# subkey=subkey, +# range=range, +# calendar=calendar, +# limit=limit, +# subattribute=subattribute, +# dateField=dateField, +# from_=from_, +# to_=to_, +# on=on, +# last=last, +# first=first, +# sort=sort, +# interval=interval, +# basePath="datasets", +# **extra_params, +# ) +# return _patch( +# url=base_url, +# json=transforms, +# token=token, +# version=version, +# token_in_params=True, +# format=format, +# ) + + +# async def modifyDataAsync( +# transforms=None, +# provider="CORE", +# id="", +# key="", +# subkey="", +# range=None, +# calendar=False, +# limit=1, +# subattribute="", +# dateField=None, +# from_=None, +# to_=None, +# on=None, +# last=0, +# first=0, +# sort="", +# interval=None, +# token="", +# version="stable", +# filter="", +# format="json", +# **extra_params, +# ): +# base_url = _queryURL( +# provider=provider, +# id=id, +# key=key, +# subkey=subkey, +# range=range, +# calendar=calendar, +# limit=limit, +# subattribute=subattribute, +# dateField=dateField, +# from_=from_, +# to_=to_, +# on=on, +# last=last, +# first=first, +# sort=sort, +# interval=interval, +# basePath="datasets", +# **extra_params, +# ) +# return await _patchAsync( +# url=base_url, +# json=transforms, +# token=token, +# version=version, +# token_in_params=True, +# format=format, +# ) + + +# def deleteData( +# provider="CORE", +# id="", +# key="", +# subkey="", +# date="", +# range=None, +# calendar=False, +# subattribute="", +# dateField=None, +# from_=None, +# to_=None, +# on=None, +# last=0, +# first=0, +# sort="", +# interval=None, +# token="", +# version="stable", +# filter="", +# format="json", +# **extra_params, +# ): +# base_url = _queryURL( +# provider=provider, +# id=id, +# key=key, +# subkey=subkey, +# date=date, +# range=range, +# calendar=calendar, +# subattribute=subattribute, +# dateField=dateField, +# from_=from_, +# to_=to_, +# on=on, +# last=last, +# first=first, +# sort=sort, +# interval=interval, +# basePath="datasets", +# **extra_params, +# ) +# return _delete( +# url=base_url, token=token, version=version, filter=filter, format=format +# ) + + +# async def deleteDataAsync( +# provider="CORE", +# id="", +# key="", +# subkey="", +# date="", +# range=None, +# calendar=False, +# limit=1, +# subattribute="", +# dateField=None, +# from_=None, +# to_=None, +# on=None, +# last=0, +# first=0, +# sort="", +# interval=None, +# token="", +# version="stable", +# filter="", +# format="json", +# **extra_params, +# ): +# base_url = _queryURL( +# provider=provider, +# id=id, +# key=key, +# subkey=subkey, +# date=date, +# range=range, +# calendar=calendar, +# limit=limit, +# subattribute=subattribute, +# dateField=dateField, +# from_=from_, +# to_=to_, +# on=on, +# last=last, +# first=first, +# sort=sort, +# interval=interval, +# basePath="datasets", +# **extra_params, +# ) +# return await _deleteAsync( +# url=base_url, token=token, version=version, filter=filter, format=format +# ) + + +# def deleteDataset( +# provider="CORE", +# id="", +# token="", +# version="stable", +# filter="", +# format="json", +# **extra_params, +# ): +# base_url = _queryURL( +# provider=provider, +# id=id, +# limit=None, +# basePath="datasets", +# ) +# return _delete( +# url=base_url, token=token, version=version, filter=filter, format=format +# ) + + +# async def deleteDatasetAsync( +# provider="CORE", +# id="", +# token="", +# version="stable", +# filter="", +# format="json", +# **extra_params, +# ): +# base_url = _queryURL( +# provider=provider, +# id=id, +# limit=None, +# basePath="datasets", +# ) +# return await _deleteAsync( +# url=base_url, token=token, version=version, filter=filter, format=format +# ) + + +def createDataJob( provider, - type="ingest", - jobId="", + type, + json, + dataType="", token="", version="stable", filter="", format="json", + **extra_params, ): - url = "jobs/{}/{}".format(provider, type) - - if jobId: - url += "/{}".format(jobId) - - return _get(url=url, token=token, version=version, filter=filter, format=format) - - -def listDatasets( - provider="CORE", id="", token="", version="stable", filter="", format="json" -): - base_url = _queryURL(provider=provider, id=id) - return _get( - url=base_url, token=token, version=version, filter=filter, format=format - ) - -@wraps(listDatasets) -def listDatasetsDF(*args, **kwargs): - return pd.DataFrame(listDatasets(*args, **kwargs)) - - -async def listDatasetsAsync( - provider="CORE", id="", token="", version="stable", filter="", format="json" -): - base_url = _queryURL(provider=provider, id=id) - return await _getAsync( - url=base_url, token=token, version=version, filter=filter, format=format + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}", ) + headers = {"content-type": "application/json"} -def createDataset( - provider, id="", schema=None, token="", version="stable", filter="", format="json" -): - base_url = _queryURL(provider=provider, limit=None, basePath="datasets") - - # TODO schema validation return _post( url=base_url, - json=schema, - token=token, - version=version, - token_in_params=True, - format=format, - ) - - -async def createDatasetAsync( - provider, id="", schema=None, token="", version="stable", filter="", format="json" -): - base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") - - # TODO schema validation - return await _postAsync( - url=base_url, - json=schema, + json=json, token=token, version=version, - token_in_params=True, + filter=filter, format=format, + headers=headers, ) -def loadData( +async def createDataJobAsync( provider, - id, - data, + type, + json, dataType="", token="", version="stable", filter="", format="json", + **extra_params, ): - base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") - # data interpolation - if not dataType: - data, headers = _interpolateDatatype(data) - else: - headers = {"content-type": dataType} + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}", + ) + + headers = {"content-type": "application/json"} - # TODO schema validation - return _put( + return await _postAsync( url=base_url, - data=data, + json=json, token=token, version=version, filter=filter, format=format, - token_in_params=True, headers=headers, ) -async def loadDataAsync( - provider, id, data, token="", version="stable", filter="", format="json" +def listDataJobs( + provider, type, token="", version="stable", filter="", format="json", **extra_params ): - base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") - # TODO schema validation - return await _putAsync( - url=base_url, - data=data, - token=token, - version=version, - token_in_params=True, - format=format, - ) + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}", + ) -def modifyDataset( - provider, id, schema, token="", version="stable", filter="", format="json" -): - base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") - return _patch( - url=base_url, - json=schema, - token=token, - version=version, - token_in_params=True, - format=format, + return _get( + url=base_url, token=token, version=version, filter=filter, format=format ) -async def modifyDatasetAsync( - provider, id, schema, token="", version="stable", filter="", format="json" +async def listDataJobsAsync( + provider, type, token="", version="stable", filter="", format="json", **extra_params ): - base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") - return await _patchAsync( - url=base_url, - json=schema, - token=token, - version=version, - token_in_params=True, - format=format, + + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}", ) + return await _getAsync( + url=base_url, token=token, version=version, filter=filter, format=format + ) -def modifyData( - transforms=None, - provider="CORE", - id="", - key="", - subkey="", - range=None, - calendar=False, - limit=1, - subattribute="", - dateField=None, - from_=None, - to_=None, - on=None, - last=0, - first=0, + +def queryDataJob( + provider, + type, + limit=0, + from_="", + to_="", + last="", + first="", sort="", - interval=None, token="", version="stable", filter="", format="json", **extra_params, ): - base_url = _queryURL( - provider=provider, - id=id, - key=key, - subkey=subkey, - range=range, - calendar=calendar, + + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}/query", limit=limit, - subattribute=subattribute, - dateField=dateField, from_=from_, to_=to_, - on=on, last=last, first=first, sort=sort, - interval=interval, - basePath="datasets", - **extra_params, ) - return _patch( - url=base_url, - json=transforms, - token=token, - version=version, - token_in_params=True, - format=format, + + return _get( + url=base_url, token=token, version=version, filter=filter, format=format ) -async def modifyDataAsync( - transforms=None, - provider="CORE", - id="", - key="", - subkey="", - range=None, - calendar=False, - limit=1, - subattribute="", - dateField=None, - from_=None, - to_=None, - on=None, - last=0, - first=0, +async def queryDataJobAsync( + provider, + type, + limit=0, + from_="", + to_="", + last="", + first="", sort="", - interval=None, token="", version="stable", filter="", format="json", **extra_params, ): - base_url = _queryURL( - provider=provider, - id=id, - key=key, - subkey=subkey, - range=range, - calendar=calendar, + + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}/query", limit=limit, - subattribute=subattribute, - dateField=dateField, from_=from_, to_=to_, - on=on, last=last, first=first, sort=sort, - interval=interval, - basePath="datasets", - **extra_params, ) - return await _patchAsync( - url=base_url, - json=transforms, - token=token, - version=version, - token_in_params=True, - format=format, + + return await _getAsync( + url=base_url, token=token, version=version, filter=filter, format=format ) -def deleteData( - provider="CORE", - id="", - key="", - subkey="", - range=None, - calendar=False, - limit=1, - subattribute="", - dateField=None, - from_=None, - to_=None, - on=None, - last=0, - first=0, - sort="", - interval=None, +def listDataJobsById( + provider, + type, + id, token="", version="stable", filter="", format="json", **extra_params, ): - base_url = _queryURL( - provider=provider, - id=id, - key=key, - subkey=subkey, - range=range, - calendar=calendar, - limit=limit, - subattribute=subattribute, - dateField=dateField, - from_=from_, - to_=to_, - on=on, - last=last, - first=first, - sort=sort, - interval=interval, - basePath="datasets", - **extra_params, + + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}/{id}", ) - return _delete( + + return _get( url=base_url, token=token, version=version, filter=filter, format=format ) -async def deleteDataAsync( - provider="CORE", - id="", - key="", - subkey="", - range=None, - calendar=False, - limit=1, - subattribute="", - dateField=None, - from_=None, - to_=None, - on=None, - last=0, - first=0, - sort="", - interval=None, +async def listDataJobsByIdAsync( + provider, + type, + id, token="", version="stable", filter="", format="json", **extra_params, ): - base_url = _queryURL( - provider=provider, - id=id, - key=key, - subkey=subkey, - range=range, - calendar=calendar, - limit=limit, - subattribute=subattribute, - dateField=dateField, - from_=from_, - to_=to_, - on=on, - last=last, - first=first, - sort=sort, - interval=interval, - basePath="datasets", - **extra_params, + + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}/{id}", ) - return await _deleteAsync( + + return await _getAsync( url=base_url, token=token, version=version, filter=filter, format=format ) -def deleteDataset( - provider="CORE", - id="", +def getDataJobLogFile( + provider, + type, + id, token="", version="stable", filter="", format="json", **extra_params, ): - base_url = _queryURL( - provider=provider, - id=id, - limit=None, - basePath="datasets", + + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}/{id}/invalid-records-log", ) - return _delete( + + return _get( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +async def getDataJobLogFileAsync( + provider, + type, + id, + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURLLoose( + basePath=f"jobs/{provider}/{type}/{id}/invalid-records-log", + ) + + return await _getAsync( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +def awsOnboarding( + provider, + arnrole="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURLLoose( + basePath=f"aws-onboarding/{provider}/configure-aws-s3", + arnrole=arnrole, + ) + + return _get( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +async def awsOnboardingAsync( + provider, + arnrole="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURLLoose( + basePath=f"aws-onboarding/{provider}/configure-aws-s3", + arnrole=arnrole, + ) + + return await _getAsync( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +def getplatformswaggerjson( + token="", version="stable", filter="", format="json", **extra_params +): + + base_url = _queryURLLoose( + basePath="platform/swagger-json", + ) + + return _get( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +async def getplatformswaggerjsonAsync( + token="", version="stable", filter="", format="json", **extra_params +): + + base_url = _queryURLLoose( + basePath="platform/swagger-json", + ) + + return await _getAsync( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +def listDatasets( + provider, + limit=0, + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, limit=None, basePath="datasets") + + return _get( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +async def listDatasetsAsync( + provider, + limit=0, + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, limit=None, basePath="datasets") + + return await _getAsync( url=base_url, token=token, version=version, filter=filter, format=format ) +def getDataset( + provider, id, token="", version="stable", filter="", format="json", **extra_params +): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + return _get( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +async def getDatasetAsync( + provider, id, token="", version="stable", filter="", format="json", **extra_params +): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + return await _getAsync( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +def registerDataset( + provider, + json, + dataType="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, limit=None, basePath="datasets") + + headers = {"content-type": "application/json"} + + return _post( + url=base_url, + json=json, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + +async def registerDatasetAsync( + provider, + json, + dataType="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, limit=None, basePath="datasets") + + headers = {"content-type": "application/json"} + + return await _postAsync( + url=base_url, + json=json, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + +def loadData( + provider, + id, + data=None, + dataType="", + overwrite=False, + maximumValidationErrors=None, + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + if not dataType: + data, headers = _interpolateDatatype(data) + else: + headers = {"content-type": dataType} + + if overwrite: + return _put( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + token_in_params=True, + headers=headers, + maximumValidationErrors=maximumValidationErrors, + ) + else: + return _post( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + token_in_params=True, + headers=headers, + maximumValidationErrors=maximumValidationErrors, + ) + + +async def loadDataAsync( + provider, + id, + data=None, + dataType="", + overwrite=False, + maximumValidationErrors=None, + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + if not dataType: + data, headers = _interpolateDatatype(data) + else: + headers = {"content-type": dataType} + + if overwrite: + return await _putAsync( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + token_in_params=True, + headers=headers, + maximumValidationErrors=maximumValidationErrors, + ) + else: + return await _postAsync( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + token_in_params=True, + headers=headers, + maximumValidationErrors=maximumValidationErrors, + ) + + +def modifyDataset( + provider, + id, + json, + dataType="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + headers = {"content-type": "application/json"} + + return _patch( + url=base_url, + json=json, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + +async def modifyDatasetAsync( + provider, + id, + json, + dataType="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + headers = {"content-type": "application/json"} + + return await _patchAsync( + url=base_url, + json=json, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + +def deleteDataset( + provider, + id, + data=None, + dataType="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + if not dataType: + data, headers = _interpolateDatatype(data) + else: + headers = {"content-type": dataType} + + return _delete( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + async def deleteDatasetAsync( - provider="CORE", - id="", + provider, + id, + data=None, + dataType="", token="", version="stable", filter="", format="json", **extra_params, ): + + base_url = _queryURL(provider=provider, id=id, limit=None, basePath="datasets") + + if not dataType: + data, headers = _interpolateDatatype(data) + else: + headers = {"content-type": dataType} + + return await _deleteAsync( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + +def deleteData( + provider, + id, + key, + subkey="", + date="", + data=None, + dataType="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + base_url = _queryURL( provider=provider, id=id, + key=key, + subkey=subkey, + date=date, limit=None, basePath="datasets", ) + + if not dataType: + data, headers = _interpolateDatatype(data) + else: + headers = {"content-type": dataType} + + return _delete( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + +async def deleteDataAsync( + provider, + id, + key, + subkey="", + date="", + data=None, + dataType="", + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURL( + provider=provider, + id=id, + key=key, + subkey=subkey, + date=date, + limit=None, + basePath="datasets", + ) + + if not dataType: + data, headers = _interpolateDatatype(data) + else: + headers = {"content-type": dataType} + return await _deleteAsync( + url=base_url, + data=data, + token=token, + version=version, + filter=filter, + format=format, + headers=headers, + ) + + +def getDataSourceContent( + provider, + objectId, + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURLLoose( + basePath=f"sample-data-source/{provider}/{objectId}", + ) + + return _get( + url=base_url, token=token, version=version, filter=filter, format=format + ) + + +async def getDataSourceContentAsync( + provider, + objectId, + token="", + version="stable", + filter="", + format="json", + **extra_params, +): + + base_url = _queryURLLoose( + basePath=f"sample-data-source/{provider}/{objectId}", + ) + + return await _getAsync( url=base_url, token=token, version=version, filter=filter, format=format )