Feature/issue 11

.gitignore

@@ -67,4 +67,6 @@ target/
 
 hydrocronapi/controllers/__pycache__/
 
-.idea
+.idea
+.env
+docker/dynamodb

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
     - Issue 8 - Hydrocron API implementation with mysql local database
 ### Changed 
+    - Issue 8 - Hydrocron API implementation with dynamodb local database
+    - Issue 8 - Rearrange database code
+    - Issue 8 - Rearrange tests
+    - Issue 8 - Add script to load data
 ### Deprecated 
 ### Removed
 ### Fixed

README.md

@@ -9,10 +9,21 @@ as CSV and geoJSON.
 Python 3.10+
 
 ## Usage
-Before starting the server you must first start a local database instance. The easiest method is to use docker
+Before starting the server you must first start a local database instance. The easiest method is to use docker. 
+First, make sure you have installed Docker and AWS CLI. To configure AWS local variables:
 
 ```
-docker run --name hydrocrondb -e MYSQL_DATABASE=test -e MYSQL_ROOT_HOST='%' -e MYSQL_ROOT_PASSWORD=my-secret-pw -p 3306:3306 -v $(pwd)/mysql/20230601_test.sql:/docker-entrypoint-initdb.d/20230601_test.sql -d --rm mysql:latest
+aws configure
+    AWS Access Key ID: a
+    AWS Secret Acces Key: a
+    Default region name: us-west-2
+    Default output format: None
+```
+
+Next step is to run docker compose up:
+
+```
+docker compose up
 ```
 
 To run the server, please execute the following from the root directory:

docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3.8'
+services:
+ dynamodb-local:
+   command: "-jar DynamoDBLocal.jar -sharedDb -dbPath ./data"
+   image: "amazon/dynamodb-local:latest"
+   container_name: dynamodb-local
+   ports:
+     - "8000:8000"
+   volumes:
+     - "./docker/dynamodb:/home/dynamodblocal/data"
+   working_dir: /home/dynamodblocal

example.env

@@ -0,0 +1,3 @@
+HYDROCRON_dynamodb_endpoint_url=http://localhost:8000/
+AWS_ACCESS_KEY_ID=DUMMYIDEXAMPLE
+AWS_SECRET_ACCESS_KEY=DUMMYEXAMPLEKEY

hydrocronapi/__main__.py

@@ -1,14 +1,8 @@
 #!/usr/bin/env python3
 
-import connexion
-
-
 def main():
-    app = connexion.App(__name__, specification_dir='./swagger/')
-    app.add_api('swagger.yaml',
-                arguments={'title': 'Get time series data from SWOT observations for reaches, nodes, and/or lakes'},
-                pythonic_params=True)
-    app.run(port=8080)
+    from hydrocronapi import hydrocron
+    hydrocron.flask_app.run(port=8080)
 
 
 if __name__ == '__main__':

hydrocronapi/controllers/subset.py

@@ -3,11 +3,10 @@
 import time
 from datetime import datetime
 from typing import Generator
+from hydrocronapi import hydrocron
 
 from shapely import Polygon, Point
 
-import hydrocronapi.data_access.db
-
 logger = logging.getLogger()
 
 
@@ -37,9 +36,9 @@ def getsubset_get(feature, subsetpolygon, start_time, end_time, output, fields):
 
     start = time.time()
     if feature.lower() == 'reach':
-        results = hydrocronapi.data_access.db.get_reach_series(start_time, end_time)
+        results = hydrocron.data_repository.get_reach_series_by_feature_id(feature, start_time, end_time)
     elif feature.lower() == 'node':
-        results = hydrocronapi.data_access.db.get_node_series(start_time, end_time)
+        results = hydrocron.data_repository.get_node_series_by_feature_id(feature, start_time, end_time)
     else:
         return {}
     end = time.time()
@@ -67,8 +66,8 @@ def format_subset_json(results: Generator, polygon, exact, time):
     -------
 
     """
-    # Fetch all results from mySQL query
-    results = list(results)
+    # Fetch all results from query
+    results = results['Items']
 
     data = {}
 
@@ -81,11 +80,10 @@ def format_subset_json(results: Generator, polygon, exact, time):
 
         data['status'] = "200 OK"
         data['time'] = str(time) + " ms."
-        # data['search on'] = {"featureID": feature_id}
+        # data['search on'] = {"feature_id": feature_id}
         data['type'] = "FeatureCollection"
         data['features'] = []
         i = 0
-        print(len(results))
         total = len(results)
         for t in results:
             flag_polygon = False
@@ -118,14 +116,14 @@ def format_subset_json(results: Generator, polygon, exact, time):
                         geometry = geometry.replace("'", "")
                         feature_type = 'Point'
                     if 'LINESTRING' in t['geometry']:
-                        geometry = t['geometry'].replace('"LINESTRING (', '').replace(')"', '')
+                        geometry = t['geometry'].replace('LINESTRING (', '').replace(')', '')
                         geometry = geometry.replace('"', '')
                         geometry = geometry.replace("'", "")
                         feature_type = 'LineString'
 
                     feature['geometry']['type'] = feature_type
                     if feature_type == 'LineString':
-                        for p in geometry.split("; "):
+                        for p in geometry.split(", "):
                             (x, y) = p.split(" ")
                             feature['geometry']['coordinates'].append([float(x), float(y)])
                             feature['properties']['time'] = datetime.fromtimestamp(
@@ -141,10 +139,8 @@ def format_subset_json(results: Generator, polygon, exact, time):
                         feature['properties']['wse'] = float(t['wse'])
 
                     data['features'].append(feature)
-                    print(str(i) + "/" + str(total))
                     i += 1
 
-        print('fin')
         data['hits'] = i
 
     return data
@@ -164,8 +160,8 @@ def format_subset_csv(results: Generator, polygon, exact, time, fields):
     -------
 
     """
-    # Fetch all results from mySQL query
-    results = list(results)
+    # Fetch all results from query
+    results = results['Items']
 
     data = {}
 
@@ -177,31 +173,23 @@ def format_subset_csv(results: Generator, polygon, exact, time, fields):
     else:
         csv = fields + '\n'
         fields_set = fields.split(", ")
-        print(fields_set)
         for t in results:
             flag_polygon = False
             if t['time'] != '-999999999999':  # and (t['width'] != '-999999999999')):
                 point = Point(float(t['p_lon']), float(t['p_lat']))
                 if polygon.contains(point):
                     if 'reach_id' in fields_set:
-                        print('yes feature_id')
                         csv += t['reach_id']
                         csv += ','
-                        print(csv)
                     if 'time_str' in fields_set:
-                        print('yes time_str')
                         csv += t['time_str']
                         csv += ','
-                        print(csv)
                     if 'wse' in fields_set:
-                        csv += t['wse']
+                        csv += str(t['wse'])
                         csv += ','
-                        print(csv)
                     if 'geometry' in fields_set:
                         csv += t['geometry'].replace('; ', ', ')
                         csv += ','
-                        print(csv)
                     csv += '\n'
-        print(csv)
 
     return csv

hydrocronapi/controllers/timeseries.py

@@ -2,8 +2,7 @@
 import time
 from datetime import datetime
 from typing import Generator
-
-import hydrocronapi.data_access.db
+from hydrocronapi import hydrocron
 
 logger = logging.getLogger()
 
@@ -29,42 +28,30 @@ def gettimeseries_get(feature, feature_id, start_time, end_time, output, fields)
     :rtype: None
     """
 
-    # If I'm too lazy to type in the UI
-    '''
-    if (feature_id == "CBBTTTSNNNNNN"):
-        feature_id = "73254700251"
-    if (feature == "Reach"): 
-        feature = "reach"
-    if (start_time == "2022-08-04T00:00:00+00:00"):
-        start_time = "2022-08-04 10:15:33"
-    if (end_time == "2022-08-22T12:59:59+00:00"):
-        end_time = "2022-08-22 10:16:38"
-    '''
-
     start_time = start_time.replace("T", " ")[0:19]
     end_time = end_time.replace("T", " ")[0:19]
     start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S")
     end_time = datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S")
 
     start = time.time()
     if feature.lower() == 'reach':
-        results = hydrocronapi.data_access.db.get_reach_series_by_feature_id(feature_id, start_time, end_time)
+        results = hydrocron.data_repository.get_reach_series_by_feature_id(feature_id, start_time, end_time)
     elif feature.lower() == 'node':
-        results = hydrocronapi.data_access.db.get_node_series_by_feature_id(feature_id, start_time, end_time)
+        results = hydrocron.data_repository.get_node_series_by_feature_id(feature_id, start_time, end_time)
     else:
         return {}
     end = time.time()
 
     data = ""
     if output == 'geojson':
-        data = format_json(results, feature_id, True, round((end - start) * 1000, 3))
+        data = format_json(results, feature_id, start_time, end_time, True, round((end - start) * 1000, 3))
     if output == 'csv':
         data = format_csv(results, feature_id, True, round((end - start) * 1000, 3), fields)
 
     return data
 
 
-def format_json(results: Generator, feature_id, exact, time):
+def format_json(results: Generator, feature_id, start_time, end_time, exact, time):
     """
 
     Parameters
@@ -79,7 +66,7 @@ def format_json(results: Generator, feature_id, exact, time):
 
     """
     # Fetch all results
-    results = list(results)
+    results = results['Items']
 
     data = {}
 
@@ -91,13 +78,18 @@ def format_json(results: Generator, feature_id, exact, time):
     else:
         data['status'] = "200 OK"
         data['time'] = str(time) + " ms."
-        # data['search on'] = {"featureID": feature_id}
+        # data['search on'] = {"feature_id": feature_id}
         data['type'] = "FeatureCollection"
         data['features'] = []
         i = 0
-        print(len(results))
+        #st = float(time.mktime(start_time.timetuple()) - 946710000)
+        #et = float(time.mktime(end_time.timetuple()) - 946710000)
+        #TODO: process type of feature_id (i.e. reach_id or node_id)
+
+
         for t in results:
-            if t['time'] != '-999999999999':  # and (t['width'] != '-999999999999')):
+            #TODO: Coordinate to filter in the database instance: if t['reach_id'] == feature_id and t['time'] > start_time and t['time'] < end_time and t['time'] != '-999999999999':  # and (t['width'] != '-999999999999')):
+            if t['reach_id'] == feature_id and t['time'] != '-999999999999':  # and (t['width'] != '-999999999999')):
                 feature = {'properties': {}, 'geometry': {}, 'type': "Feature"}
                 feature['geometry']['coordinates'] = []
                 feature_type = ''
@@ -107,13 +99,12 @@ def format_json(results: Generator, feature_id, exact, time):
                     geometry = geometry.replace("'", "")
                     feature_type = 'Point'
                 if 'LINESTRING' in t['geometry']:
-                    geometry = t['geometry'].replace('"LINESTRING (', '').replace(')"', '')
+                    geometry = t['geometry'].replace('LINESTRING (', '').replace(')', '')
                     geometry = geometry.replace('"', '')
                     geometry = geometry.replace("'", "")
                     feature_type = 'LineString'
                 feature['geometry']['type'] = feature_type
-                print(geometry)
-                for p in geometry.split("; "):
+                for p in geometry.split(", "):
                     (x, y) = p.split(" ")
                     if feature_type == 'LineString':
                         feature['geometry']['coordinates'].append([float(x), float(y)])
@@ -147,11 +138,10 @@ def format_csv(results: Generator, feature_id, exact, time, fields):
 
     """
     # Fetch all results
-    results = list(results)
+    results = results['Items']
 
     data = {}
 
-    print(fields)
     if results is None:
         data['error'] = f"404: Results with the specified Feature ID {feature_id} were not found."
     elif len(results) > 5750000:
@@ -161,30 +151,20 @@ def format_csv(results: Generator, feature_id, exact, time, fields):
         # csv = "feature_id, time_str, wse, geometry\n"
         csv = fields + '\n'
         fieldsSet = fields.split(", ")
-        print(fieldsSet)
-        if 'time_str' in fieldsSet:
-            print('yes1 time_str')
         for t in results:
             if t['time'] != '-999999999999':  # and (t['width'] != '-999999999999')):
                 if 'reach_id' in fieldsSet:
-                    print('yes feature_id')
                     csv += t['reach_id']
                     csv += ','
-                    print(csv)
                 if 'time_str' in fieldsSet:
-                    print('yes time_str')
                     csv += t['time_str']
                     csv += ','
-                    print(csv)
                 if 'wse' in fieldsSet:
-                    csv += t['wse']
+                    csv += str(t['wse'])
                     csv += ','
-                    print(csv)
                 if 'geometry' in fieldsSet:
                     csv += t['geometry'].replace('; ', ', ')
                     csv += ','
-                    print(csv)
                 csv += '\n'
-                print(csv)
 
     return csv