Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
Moved to Standard, added health check
- Loading branch information
Showing
9 changed files
with
177 additions
and
48 deletions.
There are no files selected for viewing
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,13 +1,3 @@ | ||
runtime: python | ||
env: flex | ||
entrypoint: gunicorn -b :$PORT main:app | ||
|
||
runtime_config: | ||
python_version: 3 | ||
|
||
manual_scaling: | ||
instances: 1 | ||
resources: | ||
cpu: 1 | ||
memory_gb: 0.5 | ||
disk_size_gb: 10 | ||
runtime: python37 | ||
env: standard | ||
entrypoint: gunicorn -b :$PORT main:app |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
import json | ||
|
||
import pandas as pd | ||
import requests | ||
import datalab.storage as gcs | ||
from datetime import datetime, time | ||
|
||
|
||
def update_bike_data(request): | ||
bike_data = pd.read_json( | ||
"https://api.jcdecaux.com/vls/v1/stations?contract=dublin&apiKey=6e5c2a98e60a3336ecaede8f8c8688da25144692") | ||
|
||
weather = get_weather() | ||
|
||
print("Updating bike data...") | ||
for _index, row in bike_data.iterrows(): | ||
update_record(row, weather) | ||
print("Bike data refresh complete!") | ||
|
||
return "Success!" | ||
|
||
|
||
def get_weather(): | ||
print("Updating weather data...") | ||
js = requests.get( | ||
"https://api.openweathermap.org/data/2.5/weather?lat=53.277717&lon=-6.218428&APPID" | ||
"=d8d0b9ed5f181cfbdf3330b0037aff7d&units=metric").text | ||
|
||
request_weather = json.loads(js) | ||
|
||
parsed_weather = {'rain': 0.0} | ||
|
||
if "rain" in request_weather and "1h" in request_weather["rain"]: | ||
parsed_weather['rain'] = request_weather["rain"]["1h"] | ||
|
||
parsed_weather['temperature'] = request_weather['main']['temp'] | ||
parsed_weather['humidity'] = request_weather['main']['humidity'] | ||
parsed_weather['wind_speed'] = request_weather['wind']['speed'] | ||
|
||
if "visibility" in request_weather: | ||
parsed_weather['visibility'] = request_weather['visibility'] | ||
else: | ||
parsed_weather['visibility'] = 0 | ||
|
||
print("Weather data update complete!") | ||
|
||
return parsed_weather | ||
|
||
|
||
def datetime_to_seconds(t): | ||
return (t.hour * 60 + t.minute) * 60 + t.second | ||
|
||
|
||
def seconds_in_previous_days(t): | ||
return t.timetuple().tm_yday * 24 * 60 * 60 | ||
|
||
|
||
def nearest_time(items, pivot): | ||
return min(items, key=lambda x: abs(x - pivot)) | ||
|
||
|
||
def is_time_between(begin_time, end_time, check_time=None): | ||
# If check time is not given, default to current UTC time | ||
check_time = check_time or datetime.utcnow().time() | ||
if begin_time < end_time: | ||
return begin_time <= check_time <= end_time | ||
else: # crosses midnight | ||
return check_time >= begin_time or check_time <= end_time | ||
|
||
|
||
def category(quantity): | ||
if quantity == 0: | ||
return 'empty' | ||
elif quantity < 2: | ||
return 'very low' | ||
elif quantity < 5: | ||
return 'low' | ||
elif quantity < 10: | ||
return 'moderate' | ||
else: | ||
return "high" | ||
|
||
|
||
def update_record(station, weather): | ||
print(f"Updating {station['address']}") | ||
if "/" in station['address']: | ||
data = pd.read_csv('gs://dbikes-planner.appspot.com/station_records/Princes Street.csv') | ||
else: | ||
data = pd.read_csv(f"gs://dbikes-planner.appspot.com/station_records/{station['address']}.csv") | ||
|
||
epoch_time = station['last_update'] | ||
|
||
entry_datetime = datetime.fromtimestamp(epoch_time / 1000) | ||
|
||
if is_time_between(time(3, 30), time(5, 0), entry_datetime.time()): | ||
return | ||
|
||
last_line = data.tail(3).to_csv() | ||
|
||
if entry_datetime.isoformat() in last_line: | ||
return | ||
|
||
day_index = entry_datetime.weekday() | ||
|
||
if day_index <= 4: | ||
day_type = 0 | ||
else: | ||
day_type = 10 | ||
|
||
new_row = {'available_bikes': station['available_bikes'], | ||
'available_bike_stands': station['available_bike_stands'], | ||
'time_of_day': datetime_to_seconds(entry_datetime), 'type_of_day': day_type, | ||
'day_of_year': entry_datetime.timetuple().tm_yday, 'iso_date': entry_datetime.isoformat(), | ||
'temperature': weather['temperature'], 'relative_humidity': weather['humidity'], | ||
'wind_speed': weather['wind_speed'], 'rain': weather['rain'], 'visibility': weather['visibility'], | ||
'bike_availability': category(station['available_bikes']), | ||
'bike_stand_availability': category(station['available_bike_stands']), | ||
'unix_timestamp': entry_datetime.timestamp() // 3600} | ||
|
||
new_row_dataframe = pd.DataFrame(new_row, index=[0]) | ||
|
||
combined_df = pd.concat([data, new_row_dataframe], ignore_index=True) | ||
|
||
gcs.Bucket('dbikes-planner.appspot.com').item(f'station_records/{station["address"]}.csv') \ | ||
.write_to(combined_df.to_csv(index=False), 'text/csv') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
runtime: python | ||
env: flex | ||
entrypoint: gunicorn -b :$PORT main:app | ||
|
||
runtime_config: | ||
python_version: 3 | ||
|
||
manual_scaling: | ||
instances: 1 | ||
resources: | ||
cpu: 1 | ||
memory_gb: 0.5 | ||
disk_size_gb: 10 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
from . import routes | ||
|
||
from flask import request | ||
from google.cloud import datastore | ||
|
||
|
||
@routes.route('/health-check') | ||
def health_check(): | ||
return 'OK!' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.