Merge branch 'tc-dat' into clean

DataProcess/DataProcess.py

@@ -496,17 +496,17 @@ def _speed(self, dist, dt, indicator, initIndex):
             speed_no_init = pjoin(self.processPath, 'speed_no_init')
             # Extract all speeds
             self.logger.debug('Outputting data into %s' % all_speed)
-            header = 'all cyclone speed in km/hour'
+            header = 'all cyclone speed in m/s'
             flSaveFile(all_speed, speed, header, fmt='%6.2f')
 
             # Extract initial speeds
             self.logger.debug('Outputting data into %s' % init_speed)
-            header = 'initial cyclone speed in km/hour'
+            header = 'initial cyclone speed in m/s'
             flSaveFile(init_speed, initSpeed, header, fmt='%f')
 
             # Extract speeds, excluding initial speeds
             self.logger.debug('Outputting data into %s' % speed_no_init)
-            header = 'cyclone speed without initial ones in km/hour'
+            header = 'cyclone speed without initial ones in m/s'
             flSaveFile(speed_no_init, speedNoInit, header, fmt='%6.2f')
 
     def _pressure(self, pressure, indicator):

Evaluate/interpolateTracks.py

@@ -8,7 +8,7 @@
 
 from Utilities.maputils import latLon2Azi
 from Utilities.loadData import loadTrackFile
-from Utilities.track import Track
+from Utilities.track import Track, ncSaveTracks
 
 LOG = logging.getLogger(__name__)
 LOG.addHandler(logging.NullHandler())
@@ -292,6 +292,7 @@ def parseTracks(configFile, trackFile, source, delta, outputFile=None,
 
     if outputFile:
         # Save data to file:
-        saveTracks(results, outputFile)
+        ncSaveTracks(outputFile, results)
+
 
     return results

PlotInterface/maps.py

@@ -485,7 +485,8 @@ def subplot(self, axes, subfigure):
         CB = mapobj.colorbar(CS, location='right', pad='5%',
                              fig=self, ax=axes, ticks=lvls[::2])
         CB.set_label(cbarlab)
-        mapobj.barbs(xgrid, ygrid, xdata, ydata, length=5, linewidth=0.5)
+        mapobj.barbs(xgrid, ygrid, xdata, ydata, length=5, linewidth=0.5, 
+                     latlon=True)
         axes.set_title(title)
         self.addGraticule(axes, mapobj)
         self.addCoastline(mapobj)

TrackGenerator/TrackGenerator.py

@@ -143,25 +143,21 @@ class TrackGenerator(object):
     :param maxTimeSteps: the maximum number of tropical cyclone time
                          steps that will be simulated.
 
-    :type  sizeMean: float (default: 57.0)
+    :type  sizeMean: float (default: 46.5.0)
     :param sizeMean: the fallback average tropical cyclone size to use
                      when the empirical distribution data cannot be
-                     loaded from file. The default value is taken from
-                     McConochie et al. (2004)
+                     loaded from file.
 
-
-    :type  sizeStdDev: float (default: 0.6)
+    :type  sizeStdDev: float (default: 0.5)
     :param sizeStdDev: the fallback standard deviation of the tropical
                        cyclone size to use when the empirical
                        distribution data cannot be loaded from file.
-                       The default value is taken from McConochie et al.
-                       (2004).
 
     """
 
     def __init__(self, processPath, gridLimit, gridSpace, gridInc, mslp,
                  landfall, innerGridLimit=None, dt=1.0, maxTimeSteps=360,
-                 sizeMean=57.0, sizeStdDev=0.6):
+                 sizeMean=46.5, sizeStdDev=0.5):
         self.processPath = processPath
         self.gridLimit = gridLimit
         self.gridSpace = gridSpace
@@ -773,7 +769,7 @@ def _singleTrack(self, cycloneNumber, initLon, initLat, initSpeed,
         penv[0] = initEnvPressure
         rmax[0] = initRmax
         land[0] = 0
-        dist[0] = self.dt * initSpeed
+        dist[0] = self.dt * speed[0]
 
         timestep = timedelta(self.dt/24.)
 

Utilities/track.py

@@ -261,8 +261,9 @@ def ncSaveTracks(trackfile, tracks,
 
     # Fidget with the dtype to convert :class:`datetime` objects to floats:
     track_dtype = np.dtype(tracks[0].data.dtype)
+    dtidx = track_dtype.names.index('Datetime')
     track_dtype = track_dtype.descr
-    track_dtype[1] = ('Datetime', 'f4')
+    track_dtype[dtidx] = ('Datetime', 'f8')
     track_dtype = np.dtype(track_dtype)
 
     for n, t in enumerate(tracks):
@@ -273,7 +274,7 @@ def ncSaveTracks(trackfile, tracks,
         tdtype = tdata.createCompoundType(track_dtype, 'track_dtype')
 
         dims = tdata.createDimension('time', None)
-        times = tdata.createVariable('time', 'f4', ('time',),
+        times = tdata.createVariable('time', 'f8', ('time',),
                                      zlib=True, complevel=8, shuffle=True)
         tvar = tdata.createVariable('track', tdtype, ('time',),
                                     zlib=True, complevel=8, shuffle=True)

Utilities/tracks2shp.py

@@ -24,9 +24,9 @@
 
 # For all observation points/line segments:
 OBSFIELD_NAMES = ('Indicator', 'TCID', 'Year', 'Month',
-                  'Day', 'Hour', 'Minute', 'TimeElapsed', 'Longitude',
-                  'Latitude', 'Speed', 'Bearing', 'CentralPressure',
-                  'WindSpeed', 'rMax', 'EnvPressure')
+                  'Day', 'Hour', 'Minute', 'TElapsed', 'Longitude',
+                  'Latitude', 'Speed', 'Bearing', 'Pcentre',
+                  'MaxWind', 'rMax', 'Penv')
 OBSFIELD_TYPES = ('N',)*16
 OBSFIELD_WIDTH = (1, 6, 4, 2, 2, 2, 2, 6, 7, 7, 6, 6, 7, 6, 6, 7)
 OBSFIELD_PREC =  (0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 1, 1, 1, 1, 1, 1)
@@ -38,7 +38,7 @@
 
 # For storing events as a single polyline:
 EVENTFIELD_NAMES = ('TCID', 'Year', 'Month', 'Day', 'Hour', 'Minute', 'Age',
-                    'MinPressure', 'MaxWindSpeed' )
+                    'MinCP', 'MaxWind' )
 EVENTFIELD_TYPES = ('N',)*9
 EVENTFIELD_WIDTH = (6, 4, 2, 2, 2, 2, 6, 7, 7)
 EVENTFIELD_PREC =  (0, 0, 0, 0, 0, 0, 2, 2, 1)

database/__init__.py

@@ -27,6 +27,7 @@
 - Check for existing database and create new/replace existing db if
   configuration settings have changed. Requires config settings to
   be stored in the db (?).
+- Separate the query functions to separate files.
 
 
 
@@ -267,14 +268,17 @@ def getLocations(self):
         :raises: `sqlite3.Error` if unable to retrieve the locations.
         """
         try:
-            cur = self.execute("SELECT locId, locLon, locLat FROM tblLocations")
+            cur = self.execute(("SELECT locId, locName, locLon, locLat "
+                                "FROM tblLocations"))
         except sqlite3.Error as err:
             log.exception("Cannot retrieve locations from tblLocations: {0}".\
                           format(err.args[0]))
             raise
         else:
             locations = cur.fetchall()
 
+        locations = np.rec.fromrecords(locations,
+                                       names=("locId,locName,locLon,locLat"))
         return locations
 
     def generateEventTable(self):
@@ -290,15 +294,18 @@ def generateEventTable(self):
                  _tblEvents_.
 
         """
+        log.info("Inserting records into tblEvents")
+
         fileList = os.listdir(self.windfieldPath)
-        fileList = [f for f in fileList if os.path.isfile(f)]
+        fileList = [f for f in fileList if 
+                    os.path.isfile(pjoin(self.windfieldPath, f))]
 
         pattern = re.compile(r'\d+')
         params = []
         for n, f in enumerate(sorted(fileList)):
             log.debug("Processing {0}".format(f))
             sim, num = pattern.findall(f)
-            eventId = "%s-%s" % (sim, num)
+            eventId = "%03d-%05d" % (int(sim), int(num))
             fname = pjoin(self.windfieldPath, f)
             si = os.stat(fname)
             dtWindfieldFile = datetime.fromtimestamp(int(si.st_mtime))
@@ -309,6 +316,7 @@ def generateEventTable(self):
                            dtTrackFile, dtWindfieldFile, tcrm_version,
                            "", datetime.now()))
 
+
         try:
             self.executemany(INSEVENTS, params)
         except sqlite3.Error as err:
@@ -355,8 +363,7 @@ def processEvent(self, filename, locations):
         log.debug("Processing {0}".format(pjoin(self.windfieldPath, filename)))
         pattern = re.compile(r'\d+')
         sim, num = pattern.findall(filename)
-        eventId = "%s-%s" % (sim, num)
-
+        eventId = "%03d-%05d" % (int(sim), int(num))
         ncobj = Dataset(pjoin(self.windfieldPath, filename))
         lon = ncobj.variables['lon'][:]
         lat = ncobj.variables['lat'][:]
@@ -368,7 +375,7 @@ def processEvent(self, filename, locations):
         params = list()
 
         for loc in locations:
-            locId, locLon, locLat = loc
+            locId, locName, locLon, locLat = loc
             i = find_index(lon, locLon)
             j = find_index(lat, locLat)
             locVm = vmax[j, i]
@@ -427,8 +434,8 @@ def processHazard(self):
         params = []
         for k, year in enumerate(years):
             for loc in locations:
-                locId, locLon, locLat = loc
-                log.debug("Extracting data for location: {0}".format(locId))
+                locId, locName, locLon, locLat = loc
+                log.debug("Extracting data for location: {0}".format(locName))
                 i = find_index(lon, locLon)
                 j = find_index(lat, locLat)
                 locWspd = wspd[k, j, i]
@@ -463,7 +470,7 @@ def processTracks(self):
         """
         log.info("Inserting records into tblTracks")
         locations = self.getLocations()
-        points = [Point(loc[1], loc[2]) for loc in locations]
+        points = [Point(loc[2], loc[3]) for loc in locations]
 
         files = os.listdir(self.trackPath)
         trackfiles = [pjoin(self.trackPath, f) for f in files \
@@ -475,7 +482,7 @@ def processTracks(self):
                 continue
             distances = track.minimumDistance(points)
             for (loc, dist) in zip(locations, distances):
-                locParams = (loc[0], "%s-%s"%(track.trackId),
+                locParams = (loc[0], "%03d-%05d"%(track.trackId),
                              dist, None, None, "", datetime.now())
 
                 params.append(locParams)
@@ -621,15 +628,16 @@ def locationRecords(hazard_db, locId):
 
     """
 
-    query = ("SELECT w.locId, l.locName, w.wspd, w.eventId "
+    query = ("SELECT w.locId, l.locName, w.wspd, w.umax, w.vmax, w.eventId "
              "FROM tblWindSpeed w "
              "INNER JOIN tblLocations l "
              "ON w.locId = l.locId "
              "WHERE l.locId = ? ORDER BY w.wspd ASC")
     cur = hazard_db.execute(query, (locId,))
     results = cur.fetchall()
     results = np.rec.fromrecords(results,
-                                 names=('locId,locName,wspd,eventId'))
+                                 names=('locId,locName,wspd,'
+                                        'umax,vmax,eventId'))
 
     return results
 
@@ -662,13 +670,43 @@ def locationPassage(hazard_db, locId, distance=50):
              "JOIN tblWindSpeed w on w.eventId = t.eventId "
              "JOIN tblEvents e on e.eventId = t.eventId "
              "WHERE t.distClosest < ? and l.locId = ?")
-    cur = hazard_db.execute(query, (locId, distance))
+    cur = hazard_db.execute(query, (distance, locId))
     results = cur.fetchall()
     results = np.rec.fromrecords(results,
                                  names=('locId,locName,eventId,'
                                         'distClosest,wspd,eventFile'))
     return results
 
+def locationPassageWindSpeed(hazard_db, locId, speed, distance):
+    """
+    Select records from _tblWindSpeed_, _tblTracks_ and _tblEvents_ that
+    generate a defined wind speed and pass within a given distance
+    of the location.
+
+    :param hazard_db: :class:`HazardDatabase` instance.
+    :param int locId: Location identifier.
+    :param float speed: Minimum wind speed (m/s).
+    :param float distance: Distance threshold (kilometres).
+
+    """
+
+    query = ("SELECT l.locName, w.wspd, w.umax, w.vmax, w.eventId, "
+             "t.distClosest, e.eventMaxWind, e.eventMinPressure "
+             "FROM tblLocations l "
+             "JOIN tblWindSpeed w on l.locId = w.locId "
+             "JOIN tblEvents e ON e.eventId = w.eventId "
+             "JOIN tblTracks t ON w.locId = t.locId AND w.eventId = t.eventId "
+             "WHERE l.locId = ? and w.wspd > ? AND t.distClosest <= ? "
+             "ORDER BY w.wspd ASC")
+
+    cur = hazard_db.execute(query, (locId, speed, distance))
+    results = cur.fetchall()
+    results = np.rec.fromrecords(results,
+                                 names=('locName,wspd,umax,vmax,eventId,'
+                                        'distClosest,maxwind,pmin'))
+
+    return results
+
 def locationReturnPeriodEvents(hazard_db, locId, return_period):
     """
     Select all records from tblEvents where the wind speed is
@@ -731,3 +769,23 @@ def locationAllReturnLevels(hazard_db, locId):
                                         'wspd,wspdLower,wspdUpper'))
 
     return results
+
+def selectEvents(hazard_db):
+    """
+    Select all events from _tblEvents_.
+
+    :param hazard_db: :class:`HazardDatabase` instance.
+
+    :returns: :class:`numpy.recarray` containing the full listing of each
+              event in the table.
+
+    """
+
+    query = "SELECT * FROM tblEvents ORDER BY eventMaxWind ASC"
+    cur = hazard_db.execute(query)
+    results = cur.fetchall()
+    names = ("eventNum,eventId,eventFile,eventTrackFile,eventMaxWind,"
+             "eventMinPressure,dtTrackFile,dtWindfieldFile,tcrmVer,"
+             "Comments,dtCreated")
+    results = np.rec.fromrecords(results, names=names)
+    return results

hazard/__init__.py

@@ -648,6 +648,7 @@ def aggregateWindFields(inputPath, numSimulations, tilelimits):
         filespec = pjoin(inputPath, "gust.*-%05d.nc"%year)
         fileList = glob(filespec)
         if len(fileList) == 0:
+            log.debug("No files for year: {0}".format(year))
             Vm[year, :, :] = np.zeros((ysize, xsize), dtype='f')
             continue
 

tcevent.py

@@ -177,7 +177,7 @@ def main(configFile):
     source = config.get('DataProcess', 'Source')
     delta = 1/12.
     outputPath = pjoin(config.get('Output','Path'), 'tracks')
-    outputTrackFile = pjoin(outputPath, "tracks.interp.csv")
+    outputTrackFile = pjoin(outputPath, "tracks.interp.nc")
 
     # This will save interpolated track data in TCRM format:
     interpTrack = interpolateTracks.parseTracks(configFile, trackFile,

tests/test_windmodels.py

@@ -166,7 +166,7 @@ def setUp(self):
         pkl_file.close()
 
     def test_Kepert(self):
-        profile = WindProfileModel(0.0, 0.0, 1000., 1000., self.rMax, WindSpeedModel)
+        profile = WindProfileModel(-15, 0.0, 1000., 1000., self.rMax, WindSpeedModel)
         profile.f = self.f
         windField = KepertWindField(profile)
         windField.V = self.V

wind/__init__.py

@@ -176,7 +176,7 @@ def localWindField(self, i):
         eP = convert(self.track.EnvPressure[i], 'hPa', 'Pa')
         cP = convert(self.track.CentralPressure[i], 'hPa', 'Pa')
         rMax = self.track.rMax[i]
-        vFm = self.track.Speed[i]
+        vFm = convert(self.track.Speed[i], 'kmh', 'mps')
         thetaFm = bearing2theta(self.track.Bearing[i] * np.pi/180.),
         thetaMax = self.thetaMax
 
@@ -218,7 +218,7 @@ def regionalExtremes(self, gridLimit, timeStepCallback=None):
         :param timeStepCallback: the function to be called on each time step.
         """
         if len(self.track.data) > 0:
-            envPressure = self.track.EnvPressure[0]
+            envPressure = convert(self.track.EnvPressure[0], 'hPa', 'Pa')
         else:
             envPressure = np.NaN
 
@@ -308,6 +308,9 @@ def regionalExtremes(self, gridLimit, timeStepCallback=None):
                 P < pressure[jmin:jmax, imin:imax],
                 P, pressure[jmin:jmax, imin:imax])
 
+        #log.info("Vmax: {0}; Vt: {1}; Pmin: {2}".format(gust.max(), 
+        #                                                 self.track.Speed.max()/3.6, 
+        #                                                 self.track.CentralPressure.min()))
         return gust, bearing, UU, VV, pressure, lonGrid / 100., latGrid / 100.