version 0.8.0

cran · Dec 18, 2017 · 63788e0 · 63788e0
1 parent 672c3d0
commit 63788e0
Show file tree

Hide file tree

Showing 15 changed files with 179 additions and 168 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -2,13 +2,14 @@ Package: worldmet
 Type: Package
 Title: Import Surface Meteorological Data from NOAA Integrated Surface
         Database (ISD)
-Version: 0.7.5
-Date: 2017-01-25
+Version: 0.8.0
+Date: 2017-12-15
 Authors@R: c(person("David", "Carslaw", role = c("aut", "cre"), email =
     "david.carslaw@york.ac.uk"))
 ByteCompile: true
 Depends: R (>= 3.2.0)
-Imports: openair, plyr, dplyr, leaflet, RCurl, readr, zoo
+Imports: openair, doParallel, parallel, foreach, plyr, dplyr, leaflet,
+        readr, zoo
 Maintainer: David Carslaw <david.carslaw@york.ac.uk>
 Description: Functions to import data from more than 30,000 surface
     meteorological sites around the world managed by the National Oceanic and Atmospheric Administration (NOAA) Integrated Surface
@@ -18,9 +19,9 @@ URL: http://github.com/davidcarslaw/worldmet
 BugReports: http://github.com/davidcarslaw/worldmet/issues
 LazyLoad: true
 LazyData: true
-RoxygenNote: 5.0.1
+RoxygenNote: 6.0.1
 NeedsCompilation: no
-Packaged: 2017-01-25 10:36:19 UTC; DC23
+Packaged: 2017-12-18 15:29:15 UTC; David
 Author: David Carslaw [aut, cre]
 Repository: CRAN
-Date/Publication: 2017-01-26 12:08:20
+Date/Publication: 2017-12-18 15:40:33 UTC
diff --git a/MD5 b/MD5
@@ -1,18 +1,19 @@
-ee56e89a45cdd75fe6663dcd66a83a20 *DESCRIPTION
-41b51e8724b02957fc9006e05727029c *NAMESPACE
+09f9bdf63a073ecde8cf0f31a8145e78 *DESCRIPTION
+2dbbc4f80734d2ebbafc11750ff7eb73 *NAMESPACE
 c907682c4ac72c8f0ae11146430a592a *R/exportADMS.R
-91c5e2e7955c451fbacfdfa2a7439320 *R/getMeta.R
-24d05868485997988bf40b7dee359ae2 *R/metNOAA.R
+20fc7ad7978a1160a351139d461fd0f1 *R/getMeta.R
+3e3d2d2964be0fd5e9b18bce78c1c490 *R/metNOAA.R
 14fcd05358778bdc3207bed2db1ddc83 *R/meta.R
-821f31580226b09a684f3c8349b19b9d *R/sysdata.rda
+966fbfcf2da1001f070f1107e4ec0686 *R/sysdata.rda
 cf4532f8d10db6d94136193ccf0f1cdb *R/weatherCodes.R
 1ff037620039bef68eeeab3e4a5e68d1 *R/worldmet-package.R
-5444a97c8b920565ebe0544f3eb43170 *README.md
-821f31580226b09a684f3c8349b19b9d *data/meta.rda
+cc167598e5b17a10afb4d83e73e8a827 *README.md
+966fbfcf2da1001f070f1107e4ec0686 *data/meta.rda
 8699a0e8e38705634dd0ae8f861ea0c4 *data/weatherCodes.rda
-265b7bf2f6e2263c74b7cec6d621e43a *man/exportADMS.Rd
-21c14d6ea101c06126bf0a9c2a96fcc1 *man/getMeta.Rd
-c99887a99c9c765940df377b4c136a18 *man/importNOAA.Rd
-79517c932c8b1d331c648f7db528a02a *man/meta.Rd
-a741b9fa5a114c5f5e6845a507639014 *man/weatherCodes.Rd
-303a9d0e1a2cf5672f113a09b37b9d81 *man/worldmet.Rd
+84f21841c2bbb1c09c3ffbcf11ec2145 *inst/images/map.PNG
+ffaf8662444fd21bd33c21c40e7cb9f6 *man/exportADMS.Rd
+1018f48bb736b43d53e7ab416cbc8aff *man/getMeta.Rd
+f6d3bcadfbf98522f4b5370b11912bc0 *man/importNOAA.Rd
+f2cefd98affecb5b3afaf90b79db7036 *man/meta.Rd
+ec57f553d8801d0013614cbc37a6f6c7 *man/weatherCodes.Rd
+6ab42224111ef0481fb97bb9eb324d79 *man/worldmet.Rd
diff --git a/NAMESPACE b/NAMESPACE
@@ -3,15 +3,18 @@
 export(exportADMS)
 export(getMeta)
 export(importNOAA)
-import(RCurl)
+import(doParallel)
+import(foreach)
 import(openair)
+import(parallel)
 import(plyr)
 import(readr)
 importFrom(dplyr,"%>%")
 importFrom(leaflet,addCircles)
 importFrom(leaflet,addMarkers)
 importFrom(leaflet,addTiles)
 importFrom(leaflet,leaflet)
+importFrom(utils,download.file)
 importFrom(utils,head)
 importFrom(utils,read.csv)
 importFrom(utils,write.table)

diff --git a/R/getMeta.R b/R/getMeta.R
@@ -124,11 +124,10 @@ getMeta <- function(site = "heathrow", lat = NA, lon = NA,
 
     }
 
-
+    dat <- dplyr::rename(meta, latitude = LAT, longitude = LON)   
 
   if (plot) {
-
-    dat <- dplyr::rename(meta, latitude = LAT, longitude = LON) 
+
 
     if (!"dist" %in% names(dat)) dat$dist <- NA
 
@@ -162,12 +161,8 @@ getMetaLive <- function(...) {
 
     ## downloads the whole thing fresh
 
-    ## use RCurl 
-    bin <- getBinaryURL("ftp://ftp.ncdc.noaa.gov/pub/data/noaa/isd-history.csv",
-                        ssl.verifypeer = FALSE)
-    tmp <- tempfile()
-    writeBin(bin, tmp)
-    meta <-  read.csv(tmp, header = FALSE, skip = 21)
+    url <- "https://www1.ncdc.noaa.gov/pub/data/noaa/isd-history.csv"
+    meta <- suppressMessages(read.csv(url, skip = 21))
 
     ## names in the meta file
     names(meta) <- c("USAF", "WBAN","STATION", "CTRY", "ST", "CALL", "LAT",

diff --git a/R/metNOAA.R b/R/metNOAA.R
@@ -1,124 +1,143 @@
 ##' Main function for importing meteorological data
-##' 
-##' This is the main function to import data from the NOAA Integrated Surface 
-##' Database (ISD). The ISD contains detailed surface meteorological data from 
-##' around the world for over 30,000 locations. For general information of the 
-##' ISD see \url{https://www.ncdc.noaa.gov/isd} and the map here 
+##'
+##' This is the main function to import data from the NOAA Integrated Surface
+##' Database (ISD). The ISD contains detailed surface meteorological data from
+##' around the world for over 30,000 locations. For general information of the
+##' ISD see \url{https://www.ncdc.noaa.gov/isd} and the map here
 ##' \url{https://gis.ncdc.noaa.gov/map/viewer/#app=cdo&cfg=cdo&theme=hourly&layers=1}.
-##' 
+##'
 ##' Note the following units for the main variables:
-##' 
+##'
 ##' \describe{
-##' 
-##' \item{date}{Date/time in POSIXct format. \strong{Note the time zone is GMT 
-##' (UTC) and may need to be adjusted to merge with other local data. See 
+##'
+##' \item{date}{Date/time in POSIXct format. \strong{Note the time zone is GMT
+##' (UTC) and may need to be adjusted to merge with other local data. See
 ##' details below.}}
-##' 
+##'
 ##' \item{lat}{Latitude in decimal degrees (-90 to 90).}
-##' 
-##' \item{lon}{Longitude in decimal degrees (-180 to 180). Negative numbers are 
+##'
+##' \item{lon}{Longitude in decimal degrees (-180 to 180). Negative numbers are
 ##' west of the Greenwich Meridian.}
-##' 
+##'
 ##' \item{elev}{Elevention of site in metres.}
-##' 
+##'
 ##' \item{wd}{Wind direction in degrees. 90 is from the east.}
-##' 
+##'
 ##' \item{ws}{Wind speed in m/s.}
-##' 
-##' \item{sky_ceiling}{The height above ground level (AGL) of the lowest cloud 
-##' or obscuring phenomena layer aloft with 5/8 or more summation total sky 
-##' cover, which may be predominantly opaque, or the vertical visibility into a 
+##'
+##' \item{sky_ceiling}{The height above ground level (AGL) of the lowest cloud
+##' or obscuring phenomena layer aloft with 5/8 or more summation total sky
+##' cover, which may be predominantly opaque, or the vertical visibility into a
 ##' surface-based obstruction.}
-##' 
+##'
 ##' \item{visibility}{The visibility in metres.}
-##' 
+##'
 ##' \item{air_temp}{Air temperature in degrees Celcius.}
-##' 
+##'
 ##' \item{dew_point}{The dew point temperature in degrees Celcius.}
-##' 
+##'
 ##' \item{sea_level_press}{The sea level pressure in millibars.}
-##' 
+##'
 ##' \item{RH}{The relative humidity (\%).}
-##' 
+##'
 ##' \item{cl_1,  ...,  cl_3}{Cloud cover for different layers in Oktas (1-8).}
-##' 
+##'
 ##' \item{cl}{Maximum of cl_1 to cl_3 cloud cover in Oktas (1-8).}
-##' 
+##'
 ##' \item{cl_1_height, ..., cl_3_height}{Height of the cloud base for each later
 ##' in metres.}
 ##'
 ##' \item{precip_12}{12-hour precipitation in mm.}
 ##'
 ##' \item{precip_6}{6-hour precipitation in mm.}
 ##'
-##' \item{precip}{Based on the 12 hourly and 6 hourly totals,
-##' \code{precip}} spreads the 6-hourly totals across the previous
-##' 6-hours to provide an indication of hourly precipitation.
-##' 
-##' \item{pwc}{The description of the present weather description (if 
+##' \item{precip}{Based on the 12 hourly and 6 hourly totals, \code{precip}}
+##' spreads the 6-hourly totals across the previous 6-hours to provide an
+##' indication of hourly precipitation.
+##'
+##' \item{pwc}{The description of the present weather description (if
 ##' available).}
-##' 
+##'
 ##' }
-##' 
-##' The data are returned in GMT (UTC). It may be necessary to adjust the time 
-##' zone when comining with other data. For example, if air quality data were 
-##' available for Beijing with time zone set to "Etc/GMT-8" (note the negative 
-##' offset even though Beijing is ahead of GMT. See the \code{openair} package 
-##' and manual for more details), then the time zone of the met data can be 
-##' changed to be the same. One way of doing this would be \code{attr(met$date, 
-##' "tzone") <- "Etc/GMT-8"} for a meteorological data frame called \code{met}. 
+##'
+##' The data are returned in GMT (UTC). It may be necessary to adjust the time
+##' zone when comining with other data. For example, if air quality data were
+##' available for Beijing with time zone set to "Etc/GMT-8" (note the negative
+##' offset even though Beijing is ahead of GMT. See the \code{openair} package
+##' and manual for more details), then the time zone of the met data can be
+##' changed to be the same. One way of doing this would be \code{attr(met$date,
+##' "tzone") <- "Etc/GMT-8"} for a meteorological data frame called \code{met}.
 ##' The two data sets could then be merged based on \code{date}.
-##' 
+##'
 ##' @title Import meteorological data
-##'   
-##' @param code The identifing code as a character string. The code is
-##'     a combination of the USAF and the WBAN unique identifiers. The
-##'     codes are sperated by a \dQuote{-} e.g. \code{code =
-##'     "037720-99999"}.
+##'
+##' @param code The identifing code as a character string. The code is a
+##'   combination of the USAF and the WBAN unique identifiers. The codes are
+##'   sperated by a \dQuote{-} e.g. \code{code = "037720-99999"}.
 ##' @param year The year to import. This can be a vector of years e.g.
-##'     \code{year = 2000:2005}.
-##' @param hourly Should hourly means be calculated? The default is
-##'     \code{TRUE}. If \code{FALSE} then the raw data are returned.
-##' @param precip Should precipitation measurements be returned? If
-##'     \code{TRUE} the 12-hourly and 6-hourly totals are returned (if
-##'     available). In addition, an hourly sequence is also returned,
-##'     as described below.
-##' @param PWC Description of the present weather conditions (if
-##'     available).
+##'   \code{year = 2000:2005}.
+##' @param hourly Should hourly means be calculated? The default is \code{TRUE}.
+##'   If \code{FALSE} then the raw data are returned.
+##' @param precip Should precipitation measurements be returned? If \code{TRUE}
+##'   the 12-hourly and 6-hourly totals are returned (if available). In
+##'   addition, an hourly sequence is also returned, as described below.
+##' @param PWC Description of the present weather conditions (if available).
+##' @param parallel Should the importing use mutiple processors? By default the
+##'   number of cores - 1 are used.
 ##' @export
 ##' @import openair
 ##' @import plyr
-##' @import RCurl
 ##' @import readr
+##' @import doParallel parallel foreach
 ##' @importFrom dplyr %>%
-##' @importFrom utils head read.csv write.table
+##' @importFrom utils head read.csv write.table download.file
 ##' @importFrom leaflet addCircles addMarkers addTiles leaflet
-##' @return Returns a data frame of surface observations. The data
-##'     frame is consistent for use with the \code{openair}
-##'     package. NOTE! the data are returned in GMT (UTC) time zone
-##'     format. Users may wish to express the data in other time zones
-##'     e.g. to merge with air pollution data.
-##' @seealso \code{\link{getMeta}} to obtain the codes based on
-##'     various site search approaches.
+##' @return Returns a data frame of surface observations. The data frame is
+##'   consistent for use with the \code{openair} package. NOTE! the data are
+##'   returned in GMT (UTC) time zone format. Users may wish to express the data
+##'   in other time zones e.g. to merge with air pollution data.
+##' @seealso \code{\link{getMeta}} to obtain the codes based on various site
+##'   search approaches.
 ##' @author David Carslaw
-##' @examples 
-##' 
+##' @examples
+##'
 ##' \dontrun{
 ##' ## use Beijing airport code (see getMeta example)
 ##' dat <- importNOAA(code = "545110-99999", year = 2010:2011)
 ##' }
 importNOAA <- function(code = "037720-99999", year = 2014,
-                       hourly = TRUE, precip = FALSE, PWC = FALSE) {
+                       hourly = TRUE, precip = FALSE, PWC = FALSE,
+                       parallel = TRUE) {
 
     ## main web site https://www.ncdc.noaa.gov/isd
 
     ## formats document ftp://ftp.ncdc.noaa.gov/pub/data/noaa/ish-format-document.pdf
 
     ## gis map https://gis.ncdc.noaa.gov/map/viewer/#app=cdo&cfg=cdo&theme=hourly&layers=1
 
-    ## go through each of the years selected
-    dat <- plyr::ldply(year, getDat, code = code, hourly = hourly,
-                       precip = precip, PWC = PWC)
+    ## go through each of the years selected, use parallel processing
+
+    i <- NULL
+
+    if (parallel) {
+
+    cl <- makeCluster(detectCores() - 1)
+    registerDoParallel(cl)
+
+    dat <- foreach (i = year, .combine = "rbind", 
+                           .packages = "plyr", .export = "getDat") %dopar%
+      getDat(year = i, code = code, hourly = hourly,
+             precip = precip, PWC = PWC)
+
+    stopCluster(cl)
+
+    } else {
+
+      dat <- getDat(year = year, code = code, hourly = hourly,
+                    precip = precip, PWC = PWC)
+
+    }
+
 
     return(dat)
 
@@ -127,25 +146,26 @@ importNOAA <- function(code = "037720-99999", year = 2014,
 getDat <- function(code, year, hourly, precip, PWC) {
 
   month = day = hour = minute = NULL
-
+
+
     ## location of data
-    file.name <- paste0("ftp://ftp.ncdc.noaa.gov/pub/data/noaa/",
+    file.name <- paste0("https://www1.ncdc.noaa.gov/pub/data/noaa/",
                         year, "/", code, "-", year, ".gz")
 
-    ## Use RCurl - works behind vpn
+    # Download file to temp directory
+    tmp <- paste0(tempdir(), basename(file.name))
+
     ## deal with any missing data, issue warning
 
-    bin <- try(getBinaryURL(file.name, ssl.verifypeer = FALSE), TRUE)
+    bin <- try(download.file(file.name, tmp, quiet = TRUE, mode = "wb"))
 
     if (inherits(bin, "try-error")) {
 
         warning(call. = FALSE, paste0("Data for ", year, " does not exist on server"))
         return()
     }
 
-    tmp <- paste0(tempdir(), basename(file.name))
-    writeBin(bin, tmp)
-
+
     column_widths <- c(4, 6, 5, 4, 2, 2, 2, 2, 1, 6, 7, 5, 5, 5, 4, 3,
                        1, 1, 4, 1, 5, 1, 1, 1, 6, 1, 1, 1, 5, 1, 5, 1,
                        5, 1)

diff --git a/R/sysdata.rda b/R/sysdata.rda
diff --git a/README.md b/README.md
@@ -32,10 +32,10 @@ library(worldmet)
 ## note code to be used in importNOAA
 
 getMeta(site = "heathrow")
-##        USAF  WBAN  STATION CTRY ST CALL    LAT    LON ELEV(M)      BEGIN
-## 1687 037720 99999 HEATHROW   UK    EGLL 51.478 -0.461    25.3 1948-12-01
-##             END         code
-## 1687 2016-07-04 037720-99999
+##        USAF  WBAN  STATION CTRY ST CALL latitude longitude ELEV(M)
+## 1669 037720 99999 HEATHROW   UK    EGLL   51.478    -0.461    25.3
+##           BEGIN        END         code dist
+## 1669 1948-12-01 2017-12-15 037720-99999   NA
 ```
 
 Often we have a latitude / longitude of interest. A search can be made based on supplied decimal coordinates and the top `n` nearest sites are returned. The map shows the location searched by the user (red dot) and markers showing the nearest meteorological stations. Click on a station marker to obtain the code and other basic information.
@@ -46,16 +46,14 @@ Often we have a latitude / longitude of interest. A search can be made based on
 info <- getMeta(lat = 40, lon = 116.9)
 ```
 
-<!--html_preserve-->
+<img src="inst/images/map.PNG" alt="map of Beijing area" width="75%" />
 
-<script type="application/json" data-for="htmlwidget-4aae08a8dcbdfe001f3a">{"x":{"calls":[{"method":"addTiles","args":["http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",null,null,{"minZoom":0,"maxZoom":18,"maxNativeZoom":null,"tileSize":256,"subdomains":"abc","errorTileUrl":"","tms":false,"continuousWorld":false,"noWrap":false,"zoomOffset":0,"zoomReverse":false,"opacity":1,"zIndex":null,"unloadInvisibleTiles":null,"updateWhenIdle":null,"detectRetina":false,"reuseTiles":false,"attribution":"&copy; <a href=\"http://openstreetmap.org\">OpenStreetMap\u003c/a> contributors, <a href=\"http://creativecommons.org/licenses/by-sa/2.0/\">CC-BY-SA\u003c/a>"}]},{"method":"addMarkers","args":[[40.08,39.1,39.124,39.65,40.417,41.2,40.967,40.4,39.433,38.733],[116.585,117.167,117.346,118.1,115.5,116.633,117.917,118.95,118.9,115.483],null,null,null,{"clickable":true,"draggable":false,"keyboard":true,"title":"","alt":"","zIndexOffset":0,"opacity":1,"riseOnHover":false,"riseOffset":250},["BEIJING - CAPITAL INTERNATIONAL AIRPORT<br/>Code: 545110-99999<br/>Start: 1945-10-31<br/>End: 2016-07-04<br/>Distance (km) 28.3","TIANJIN<br/>Code: 545270-99999<br/>Start: 1956-08-20<br/>End: 2016-07-04<br/>Distance (km) 102.7","BINHAI<br/>Code: 545273-99999<br/>Start: 1981-11-25<br/>End: 2016-07-04<br/>Distance (km) 104.6","TANGSHAN<br/>Code: 545340-99999<br/>Start: 1956-08-20<br/>End: 2016-07-04<br/>Distance (km) 109.6","HUAILAI<br/>Code: 544050-99999<br/>Start: 1956-08-20<br/>End: 2016-07-04<br/>Distance (km) 127.6","FENGNING<br/>Code: 543080-99999<br/>Start: 1957-06-01<br/>End: 2016-07-04<br/>Distance (km) 135.3","CHENGDE<br/>Code: 544230-99999<br/>Start: 1956-08-20<br/>End: 2016-07-04<br/>Distance (km) 137.7","QINGLONG<br/>Code: 544360-99999<br/>Start: 1957-06-02<br/>End: 2016-07-04<br/>Distance (km) 179.7","LETING<br/>Code: 545390-99999<br/>Start: 1957-06-01<br/>End: 2016-07-04<br/>Distance (km) 182.3","BAODING<br/>Code: 546020-99999<br/>Start: 1956-08-20<br/>End: 2016-07-04<br/>Distance (km) 186.2"],null,null,null,null]},{"method":"addCircles","args":[40,116.9,200,null,null,{"lineCap":null,"lineJoin":null,"clickable":true,"pointerEvents":null,"className":"","stroke":true,"color":"red","weight":20,"opacity":0.5,"fill":true,"fillColor":"red","fillOpacity":0.2,"dashArray":null},"Search location<br/>Lat = 40<br/>Lon = 116.9",null,null]}],"limits":{"lat":[38.733,41.2],"lng":[115.483,118.95]}},"evals":[],"jsHooks":[]}</script>
-<!--/html_preserve-->
 To obtain the data the user must supply a `code` (see above) and year or years of interest. For example, to download data for Heathrow Airport in 2010 (code 037720-99999):
 
 ``` r
 dat <- importNOAA(code = "037720-99999", year = 2010)
 head(dat)
-## # A tibble: 6 × 23
+## # A tibble: 6 x 23
 ##                  date   usaf  wban         code  station    lat   lon
 ##                <dttm>  <chr> <int>        <chr>    <chr>  <dbl> <dbl>
 ## 1 2010-01-01 00:00:00 037720 99999 037720-99999 HEATHROW 51.483 -0.45

diff --git a/data/meta.rda b/data/meta.rda
diff --git a/inst/images/map.PNG b/inst/images/map.PNG