-
Notifications
You must be signed in to change notification settings - Fork 1
/
looper_goes16aws.py
188 lines (149 loc) · 7.06 KB
/
looper_goes16aws.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
# -*- coding: utf-8 -*-
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Created on 26 Nov 2018
#
# @author: rhamilton
"""Main loop for GOES-16 reprojection and animation service.
"""
from __future__ import division, print_function, absolute_import
import gc
import memory_profiler as mprof
import os
import time
from datetime import datetime as dt
from ligmos.utils import confparsers, files, logs
from nightshift.goes import plot, aws
from nightshift.common import maps, utils
def main(outdir, creds, sleep=150., keephours=24.,
forceDown=False, forceRegen=False):
"""
'outdir' is the *base* directory for outputs, stuff will be put into
subdirectories inside of it.
'keephours' is the number of hours of data to keep on hand. Old stuff
is deleted to keep things managable
(REMOVED FROM CALLING SEQUENCE)
'vidhours' is the number of hours of data to make into a GIF (or MP4).
6 hours equates to about 72 images in the video
Tailored for a single channel/band of output ONLY. To have multiple bands
outputting to a single directory this NEEDS some restructuring!!!
"""
vidhours = 4.
aws_keyid = creds['s3_RO']['aws_access_key_id']
aws_secretkey = creds['s3_RO']['aws_secret_access_key']
dout = outdir + "/raws/"
pout = outdir + "/pngs/"
lout = outdir + "/nows/"
cfiles = "./nightshift/resources/cb_2018_us_county_5m/"
# in degrees; for spatially filtering map shapefiles
mapcenter = [-111.4223, 34.7443]
filterRadius = 7.
# Filename to copy the last/latest image into for easier web integration
# Ok to just hardcopy these since they'll be staticly named
vid1 = "%s/g16aws_latest.gif" % (lout)
# What the base/first part of the output filename will be
staticname = 'goes16'
nstaticfiles = 48
# Need this for parsing the filename into a dt obj
dtfmt = "%Y%j%H%M%S%f"
# Prepare some things for plotting so we don't have to do it
# forever in the main loop body
rclasses = ["Interstate", "Federal"]
# On the assumption that we'll plot something, downselect the full
# road database into the subset we want
print("Parsing road data...")
print("\tClasses: %s" % (rclasses))
# roads will be a dict with keys of rclasses and values of geometries
roads = maps.parseRoads(rclasses,
center=mapcenter, centerRad=filterRadius)
for rkey in rclasses:
print("%s: %d found within %d degrees of center" % (rkey,
len(roads[rkey]),
filterRadius))
print("Parsing county data...")
counties = maps.parseCounties(cfiles + "cb_2018_us_county_5m.shp",
center=mapcenter, centerRad=filterRadius)
print("%d counties found within %d degrees of center" % (len(counties),
filterRadius))
# Construct/grab the color map.
# Purposefully leaving this hardcoded here for now, because it's
# so easy to make a god damn mess of the colormap if you don't know
# what you're doing.
vmin, vmax = 160, 330
gcmap = plot.getCMap(vmin=vmin, vmax=vmax)
print("Starting infinite loop...")
while True:
# 'keephours' is time (in hours!) to search for new files relative
# to the present.
# If they exist, they'll be skipped unless forceDown is True
when = dt.utcnow()
print("Looking for files!")
ffiles = aws.GOESAWSgrab(aws_keyid, aws_secretkey, when, dout,
timedelta=keephours, forceDown=forceDown)
print("Found the following files:")
for f in ffiles:
print(os.path.basename(f.key))
print("Making the plots...")
# TODO: Return the projection coordinates (and a timestamp of them)
# so they can be reused between loop cycles.
nplots = plot.makePlots(dout, pout, mapcenter, cmap=gcmap,
roads=roads, counties=counties,
forceRegen=forceRegen)
print("%03d plots done!" % (nplots))
# ... Do what the function says! Return a list of current files
# which will then be used as the input for the GIF/video
#
# NOTE: I'm literally adding a 'fudge' factor here because the initial
# AWS/data query has a resolution of 1 hour, so there can sometimes
# be fighting of downloading/deleting/redownloading/deleting ...
fudge = 1.
# Now we look for old files. Looking is ok! We won't actually act
# unless there's a valid reason to do so.
ofiles = dtfmt + "_C13.png"
curpngs, oldpngs = files.findOldFiles(pout, "*.png", when,
maxage=keephours+fudge,
dtfmt=ofiles)
ofiles = dtfmt + "_C13.nc"
curraws, oldraws = files.findOldFiles(dout, "*.nc", when,
maxage=keephours+fudge,
dtfmt=ofiles)
if nplots > 0:
# Remove the dead/old ones
# BUT notice that this is only if we made new files!
files.deleteOldFiles(oldpngs)
files.deleteOldFiles(oldraws)
print("%d, %d raw and png files remain within %.1f + %.1f hours" %
(len(curraws), len(curpngs), keephours, fudge))
print("Copying the latest/last files to an accessible spot...")
# Move our files to the set of static filenames. This will
# check (cpng) to see if there are actually any files that
# are new, and if so it'll shuffle the files into the correct
# order of static filenames.
# This will stamp files that are > 4 hours old with a warning
utils.copyStaticFilenames(curpngs, lout,
staticname, nstaticfiles,
errorAge=2.25, errorStamp=True)
print("GARBAGE UPDATE")
print(gc.get_stats())
print("MEMORY UPDATE")
print(mprof.memory_usage(timestamps=True, include_children=True,
multiprocess=True))
print("Sleeping for %03d seconds..." % (sleep))
time.sleep(sleep)
if __name__ == "__main__":
outdir = "./outputs/goes/"
awsconf = "./config/awsCreds.conf"
forceDownloads = False
forceRegenPlot = False
logname = './outputs/logs/goesmcgoesface.log'
# Set up logging (using ligmos' quick 'n easy wrapper)
logs.setup_logging(logName=logname, nLogs=30)
# NOTE: We just take a shortcut and use the rawParser here, since we
# don't have much in the way of configuration
creds = confparsers.rawParser(awsconf)
main(outdir, creds, sleep=90.,
forceDown=forceDownloads, forceRegen=forceRegenPlot)
print("Exiting!")