Issue 51 data availability

doc/source/Code_Examples/Discover_Data_Availability.md

@@ -0,0 +1,37 @@
+# Discover Data Availability for Data Products
+
+```python
+# Get the token from your Oceans 3.0 profile page
+from onc import ONC
+
+onc = ONC("YOUR_TOKEN")
+```
+
+## [/dataAvailability/dataproducts](https://data.oceannetworks.ca/OpenAPI#get-/dataAvailability/dataproducts)
+
+### Get data availability from a specific location and a device category
+
+Return which data products are available with _deviceCategoryCode_ "**BPR**" at location Barkley Upper Slope (
+_locationCode_:"**NCBC**").
+
+```python
+
+params = {
+    "deviceCategoryCode": "BPR",
+    "locationCode": "NCBC",
+}
+onc.getDataAvailability(params)
+```
+
+### Get data availability from a specific device with a specific extension
+
+Return which data products are available with _deviceCode_ "**BPR_BC**" and extension "**raw**"
+
+```python
+
+params = {
+    "deviceCode": "BPR_BC",
+    "extension": "raw",
+}
+onc.getDataAvailability(params)
+```

doc/source/Code_Examples/index.md

@@ -12,6 +12,7 @@ Discover_Deployments.ipynb
 Discover_Device_Categories.ipynb
 Discover_Properties.ipynb
 Discover_Data_Products.ipynb
+Discover_Data_Availability.ipynb
 Download_Data_Products.ipynb
 Request_Real_Time_Data.ipynb
 Download_Archived_Files.ipynb

src/onc/modules/_DataProductFile.py

@@ -23,14 +23,13 @@ def __init__(self, dpRunId: int, index: str, baseUrl: str, token: str):
         self._retries = 0
         self._status = 202
         self._downloaded = False
-        self._baseUrl = f"{baseUrl}api/dataProductDelivery"
+        self._baseUrl = f"{baseUrl}api/dataProductDelivery/download"
         self._filePath = ""
         self._fileSize = 0
         self._runningTime = 0
         self._downloadingTime = 0
 
         self._filters = {
-            "method": "download",
             "token": token,
             "dpRunId": dpRunId,
             "index": index,

src/onc/modules/_MultiPage.py

@@ -23,7 +23,7 @@ def getAllPages(self, service: str, url: str, filters: dict):
         """
         # pop archivefiles extension
         extension = None
-        if service == "archivefiles" and "extension" in filters:
+        if service.startswith("archivefile") and "extension" in filters:
             extension = filters["extension"]
             del filters["extension"]
 
@@ -84,7 +84,7 @@ def _doPageRequest(
         @param extension: Only provide for archivefiles filtering
         Returns a tuple (jsonResponse, duration)
         """
-        if service == "archivefiles":
+        if service.startswith("archivefile"):
             response, duration = self.parent()._doRequest(url, filters, getTime=True)
             response = self.parent()._filterByExtension(response, extension)
         else:
@@ -97,7 +97,7 @@ def _catenateData(self, response: object, nextResponse: object, service: str):
         Concatenates the data results from nextResponse into response
         Compatible with the row structure of different services
         """
-        if service == "scalardata":
+        if service.startswith("scalardata"):
             keys = response["sensorData"][0]["data"].keys()
 
             for sensorData in response["sensorData"]:
@@ -111,11 +111,11 @@ def _catenateData(self, response: object, nextResponse: object, service: str):
                 for key in keys:
                     sensorData["data"][key] += nextSensor["data"][key]
 
-        elif service == "rawdata":
+        elif service.startswith("rawdata"):
             for key in response["data"]:
                 response["data"][key] += nextResponse["data"][key]
 
-        elif service == "archivefiles":
+        elif service.startswith("archivefile"):
             response["files"] += nextResponse["files"]
 
     def _estimatePages(self, response: object, service: str):
@@ -154,13 +154,13 @@ def _rowCount(self, response, service: str):
         """
         Returns the number of records in the response
         """
-        if service == "scalardata":
+        if service.startswith("scalardata"):
             return len(response["sensorData"][0]["data"]["sampleTimes"])
 
-        elif service == "rawdata":
+        elif service.startswith("rawdata"):
             return len(response["data"]["times"])
 
-        elif service == "archivefiles":
+        elif service.startswith("archivefile"):
             return len(response["files"])
 
         return 0
@@ -171,15 +171,16 @@ def _responseTimespan(self, response, service: str):
         Returns a timedelta object
         """
         # grab the first and last sample times
-        if service in ["scalardata", "rawdata"]:
-            if service == "scalardata":
-                first = response["sensorData"][0]["data"]["sampleTimes"][0]
-                last = response["sensorData"][0]["data"]["sampleTimes"][-1]
-            elif service == "rawdata":
-                first = response["data"]["times"][0]
-                last = response["data"]["times"][-1]
-
-        elif service == "archivefiles":
+
+        if service.startswith("scalardata"):
+            first = response["sensorData"][0]["data"]["sampleTimes"][0]
+            last = response["sensorData"][0]["data"]["sampleTimes"][-1]
+
+        elif service.startswith("rawdata"):
+            first = response["data"]["times"][0]
+            last = response["data"]["times"][-1]
+
+        elif service.startswith("archivefile"):
             row0 = response["files"][0]
             if isinstance(row0, str):
                 regExp = r"\d{8}T\d{6}\.\d{3}Z"
@@ -198,7 +199,6 @@ def _responseTimespan(self, response, service: str):
                 last = response["files"][-1]["dateFrom"]
 
         # compute the timedelta
-        # print(first, last)
         dateFirst = dateutil.parser.parse(first)
         dateLast = dateutil.parser.parse(last)
         return dateLast - dateFirst

src/onc/modules/_OncArchive.py

@@ -23,15 +23,15 @@ def getArchivefileByLocation(self, filters: dict, allPages: bool):
 
         The filenames obtained can be used to download files using the ``downloadArchivefile`` method.
         """  # noqa: E501
-        return self._getList(filters, by="location", allPages=allPages)
+        return self._getList(filters, service="archivefile/location", allPages=allPages)
 
     def getArchivefileByDevice(self, filters: dict, allPages: bool):
         """
         Return a list of archived files from a specific device.
 
         The filenames obtained can be used to download files using the ``downloadArchivefile`` method.
         """  # noqa: E501
-        return self._getList(filters, by="device", allPages=allPages)
+        return self._getList(filters, service="archivefile/device", allPages=allPages)
 
     def getArchivefile(self, filters: dict, allPages: bool):
         return self._delegateByFilters(
@@ -49,16 +49,15 @@ def getArchivefileUrl(self, filename: str) -> str:
         """
         Return an archivefile absolute download URL for a filename
         """
-        url = self._serviceUrl("archivefile")
+        url = self._serviceUrl("archivefile/download")
         token = self._config("token")
-        return f"{url}/download?filename={filename}&token={token}"
+        return f"{url}?filename={filename}&token={token}"
 
     def downloadArchivefile(self, filename: str = "", overwrite: bool = False):
-        url = self._serviceUrl("archivefiles")
+        url = self._serviceUrl("archivefile/download")
 
         filters = {
             "token": self._config("token"),
-            "method": "getFile",
             "filename": filename,
         }
 
@@ -151,15 +150,14 @@ def downloadDirectArchivefile(
             "stats": {"totalSize": size, "downloadTime": time, "fileCount": successes},
         }
 
-    def _getList(self, filters: dict, by: str = "location", allPages: bool = False):
+    def _getList(
+        self, filters: dict, service: str = "location", allPages: bool = False
+    ):
         """
         Wraps archivefiles getArchivefileByLocation and getArchivefileByDevice methods
         """
-        url = self._serviceUrl("archivefiles")
+        url = self._serviceUrl(service)
         filters["token"] = self._config("token")
-        filters["method"] = (
-            "getListByLocation" if by == "location" else "getListByDevice"
-        )
 
         # parse and remove the artificial parameter extension
         extension = None
@@ -169,7 +167,7 @@ def _getList(self, filters: dict, by: str = "location", allPages: bool = False):
 
         if allPages:
             mp = _MultiPage(self)
-            result = mp.getAllPages("archivefiles", url, filters2)
+            result = mp.getAllPages(service, url, filters2)
         else:
             if "extension" in filters2:
                 del filters2["extension"]

src/onc/modules/_OncDelivery.py

@@ -65,9 +65,8 @@ def requestDataProduct(self, filters: dict):
         """
         Data product request
         """
-        filters["method"] = "request"
         filters["token"] = self._config("token")
-        url = "{:s}api/dataProductDelivery".format(self._config("baseUrl"))
+        url = f"{self._config('baseUrl')}api/dataProductDelivery/request"
         response = self._doRequest(url, filters)
 
         self._estimatePollPeriod(response)
@@ -93,15 +92,14 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool):
         print(
             f"To cancel the running data product, run 'onc.cancelDataProduct({dpRequestId})'"  # noqa: E501
         )
-        url = f"{self._config('baseUrl')}api/dataProductDelivery"
+        url = f"{self._config('baseUrl')}api/dataProductDelivery/run"
         runResult = {"runIds": [], "fileCount": 0, "runTime": 0, "requestCount": 0}
 
         start = time()
         while status != "complete":
             response = requests.get(
                 url,
                 {
-                    "method": "run",
                     "token": self._config("token"),
                     "dpRequestId": dpRequestId,
                 },
@@ -283,9 +281,8 @@ def _countFilesInProduct(self, runId: int):
         Given a runId, polls the "download" method to count files.
         Uses HTTP HEAD to avoid downloading the files.
         """
-        url = f"{self._config('baseUrl')}api/dataProductDelivery"
+        url = f"{self._config('baseUrl')}api/dataProductDelivery/download"
         filters = {
-            "method": "download",
             "token": self._config("token"),
             "dpRunId": runId,
             "index": 1,

src/onc/modules/_OncDiscovery.py

@@ -10,9 +10,8 @@ class _OncDiscovery(_OncService):
     def __init__(self, parent: object):
         super().__init__(parent)
 
-    def _discoveryRequest(self, filters: dict, service: str, method: str = "get"):
+    def _discoveryRequest(self, filters: dict, service: str):
         url = self._serviceUrl(service)
-        filters["method"] = method
         filters["token"] = self._config("token")
 
         result = self._doRequest(url, filters)
@@ -25,7 +24,7 @@ def getLocations(self, filters: dict):
 
     def getLocationHierarchy(self, filters: dict):
         filters = filters or {}
-        return self._discoveryRequest(filters, service="locations", method="getTree")
+        return self._discoveryRequest(filters, service="locations/tree")
 
     def getDeployments(self, filters: dict):
         filters = filters or {}
@@ -47,6 +46,10 @@ def getDataProducts(self, filters: dict):
         filters = filters or {}
         return self._discoveryRequest(filters, service="dataProducts")
 
+    def getDataAvailability(self, filters: dict):
+        filters = filters or {}
+        return self._discoveryRequest(filters, service="dataAvailability/dataproducts")
+
     def _sanitizeBooleans(self, data: list):
         """
         Enforce json values of "true" and "false" are correctly parsed as bool.

src/onc/modules/_OncRealTime.py

@@ -19,7 +19,7 @@ def getScalardataByLocation(self, filters: dict, allPages: bool):
         See https://wiki.oceannetworks.ca/display/O2A/scalardata+service
         for usage and available filters
         """
-        return self._getDirectAllPages(filters, "scalardata", "getByLocation", allPages)
+        return self._getDirectAllPages(filters, "scalardata/location", allPages)
 
     def getScalardataByDevice(self, filters: dict, allPages: bool):
         """
@@ -28,7 +28,7 @@ def getScalardataByDevice(self, filters: dict, allPages: bool):
         See https://wiki.oceannetworks.ca/display/O2A/scalardata+service
         for usage and available filters.
         """
-        return self._getDirectAllPages(filters, "scalardata", "getByDevice", allPages)
+        return self._getDirectAllPages(filters, "scalardata/device", allPages)
 
     def getScalardata(self, filters: dict, allPages: bool):
         return self._delegateByFilters(
@@ -45,7 +45,7 @@ def getRawdataByLocation(self, filters: dict, allPages: bool):
         See https://wiki.oceannetworks.ca/display/O2A/rawdata+service
         for usage and available filters.
         """
-        return self._getDirectAllPages(filters, "rawdata", "getByLocation", allPages)
+        return self._getDirectAllPages(filters, "rawdata/location", allPages)
 
     def getRawdataByDevice(self, filters: dict, allPages: bool):
         """
@@ -54,7 +54,7 @@ def getRawdataByDevice(self, filters: dict, allPages: bool):
         See https://wiki.oceannetworks.ca/display/O2A/rawdata+service
         for usage and available filters.
         """
-        return self._getDirectAllPages(filters, "rawdata", "getByDevice", allPages)
+        return self._getDirectAllPages(filters, "rawdata/device", allPages)
 
     def getRawdata(self, filters: dict, allPages: bool):
         return self._delegateByFilters(
@@ -68,9 +68,7 @@ def getSensorCategoryCodes(self, filters: dict):
         updated_filters = filters | {"returnOptions": "excludeScalarData"}
         return self.getScalardata(updated_filters, False)["sensorData"]
 
-    def _getDirectAllPages(
-        self, filters: dict, service: str, method: str, allPages: bool
-    ) -> Any:
+    def _getDirectAllPages(self, filters: dict, service: str, allPages: bool) -> Any:
         """
         Keeps downloading all scalar or raw data pages until finished.
 
@@ -83,7 +81,6 @@ def _getDirectAllPages(
         # prepare filters for first page request
         filters = filters or {}
         url = self._serviceUrl(service)
-        filters["method"] = method
         filters["token"] = self._config("token")
 
         # if sensorCategoryCodes is an array, join it into a comma-separated string

src/onc/modules/_OncService.py

@@ -95,15 +95,23 @@ def _serviceUrl(self, service: str):
         """
         if service in [
             "locations",
+            "locations/tree",
             "deployments",
             "devices",
             "deviceCategories",
             "properties",
             "dataProducts",
-            "archivefiles",
-            "archivefile",
-            "scalardata",
-            "rawdata",
+            "dataAvailability/dataproducts",
+            "archivefile/device",
+            "archivefile/location",
+            "archivefile/download",
+            "scalardata/location",
+            "scalardata/device",
+            "rawdata/location",
+            "rawdata/device",
+            "dataProductDelivery/request",
+            "dataProductDelivery/run",
+            "dataProductDelivery/download",
         ]:
             return f"{self._config('baseUrl')}api/{service}"