version 2.1.0

cran · Jan 1, 2015 · ffaec6a · ffaec6a
1 parent 87d3629
commit ffaec6a
Show file tree

Hide file tree

Showing 10 changed files with 126 additions and 166 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,14 +1,14 @@
 Package: rNOMADS
 Type: Package
-Title: An interface to the NOAA Operational Model Archive and
+Title: An Interface to the NOAA Operational Model Archive and
         Distribution System
-Version: 2.0.4
-Date: 2014-10-30
+Version: 2.1.0
+Date: 2015-01-01
 Authors@R: c(person(given="Daniel C.",family="Bowman", role=c("aut", "cre"),
         email="daniel.bowman@unc.edu"))
-Depends: R (>= 3.1.1)
-Imports: scrapeR (>= 0.1.6), stringr (>= 0.6.2), fields (>= 6.7.6), XML
-        (>= 3.98.1.1), GEOmap (>= 2.1), MBA, RCurl (>= 1.95-4.1)
+Depends: R (>= 3.1.1), XML (>= 3.98.1.1), rvest (>= 0.1.0)
+Imports: scrapeR (>= 0.1.6), stringr (>= 0.6.2), fields (>= 6.7.6),
+        GEOmap (>= 2.1), MBA, RCurl (>= 1.95-4.1)
 Description: An interface to the National Oceanic and Atmospheric Administration's Operational Model Archive and Distribution System (NOMADS) that allows R users to quickly and efficiently download global and regional weather model data for processing.  rNOMADS currently supports a variety of models ranging from global weather data to an altitude of 40 km, to high resolution regional weather models, to wave and sea ice models. It can also retrieve archived NOMADS models.  rNOMADS can retrieve binary data in grib format as well as import ascii data directly into R by interfacing with the GrADS-DODS system.
 License: GPL (>= 3)
 URL: https://r-forge.r-project.org/projects/rnomads/ (subversion
@@ -17,10 +17,10 @@ URL: https://r-forge.r-project.org/projects/rnomads/ (subversion
         (GrADS-DODS examples)
         http://www.unc.edu/~haksaeng/rNOMADS/rNOMADS_grib_examples.pdf
         (grib examples)
-Packaged: 2014-10-30 16:49:43 UTC; bowman
+Packaged: 2015-01-01 18:30:03 UTC; dantayaga
 Maintainer: Daniel C. Bowman <daniel.bowman@unc.edu>
 MailingList: rnomads-user@lists.r-forge.r-project.org
 Author: Daniel C. Bowman [aut, cre]
 NeedsCompilation: no
 Repository: CRAN
-Date/Publication: 2014-10-30 19:34:56
+Date/Publication: 2015-01-01 19:50:38
diff --git a/MD5 b/MD5
@@ -1,18 +1,18 @@
-5c62a403e3b02eb7d9a0f87392f8f7fb *DESCRIPTION
-8b54e5a89fbda3af5e077053d40bec76 *NAMESPACE
-8abbabb4c5e982216309d093bd0d635e *NEWS
-7c5fca27d886b469d97698afa0a9ffe5 *R/GetArchiveGrib.R
+b46a5029e81eee324861a02938814746 *DESCRIPTION
+506c8824b59e5ae2f94769222626d5bb *NAMESPACE
+e9a187a5ba2b8eea6eac9a6830fedb62 *NEWS
+3ddc31c9de1a822a5de8a24352acbae1 *R/GetArchiveGrib.R
 8020a4b69d6501b25feac2416f654dc4 *R/GetDODS.R
 cfb79120a31c49156f63e668f7c80428 *R/GetRealTimeGrib.R
-19eb65157610abefb202413aed4d0ce8 *R/Models.R
+c248763b06b3e13c46565811b4e7b2dd *R/Models.R
 b6964c6927a28e96302311336685d4d0 *R/RNomadsTools.R
 987f1ef4e08f89fee97ad6946fa9b218 *R/ReadGrib.R
 655d7615473825378142c4b6aac9cedd *man/ArchiveGribGrab.Rd
 ca122cdc954c5e0fa0977d46cc3c2cbe *man/BuildProfile.Rd
 0bdff541e2fbd21d7b2e205fca3e8656 *man/CheckNOMADSArchive.Rd
 78a1c39a63cd2c4c757dcecd9c0a5b72 *man/CrawlModels.Rd
-2324490e6f5e475a30e11b1b5f2755cc *man/DODSGrab.Rd
-a4fca2864f41036f248a9050c465224b *man/GetClosestGFSForecasts.Rd
+b3633bfc7006186d5725263bf96dc9e3 *man/DODSGrab.Rd
+2c7d527a08ceb8af8e2924ea2184ce4f *man/GetClosestGFSForecasts.Rd
 21d357865da97c9ba14b0d569b2312f7 *man/GetDODSDates.Rd
 e5e218d8ec46db674c43db45a54aabb4 *man/GetDODSModelRunInfo.Rd
 a33d3af929c58a20e1e2833cf845dc0b *man/GetDODSModelRuns.Rd
@@ -21,8 +21,8 @@ b7627ff676fdd6877314c50cf64a0e24 *man/GribGrab.Rd
 43004dfb2d645efdba52442e8e94bea1 *man/MagnitudeAzimuth.Rd
 967f910540742f7a909af4354e14b6c1 *man/ModelGrid.Rd
 df43cf3706870565770577a17e4ec5a1 *man/NOMADSArchiveList.Rd
-47cdec1459286f1d5dad62f7580d828e *man/NOMADSRealTimeList.Rd
-f0374004ef53afcf22888903501e9cf8 *man/ParseModelPage.Rd
+261b6cc4586df17ae4723977b94a6267 *man/NOMADSRealTimeList.Rd
+1a778ec84395e20e9c5c2e9eae620c65 *man/ParseModelPage.Rd
 403a88fbcdad4097d917304bbd90ddac *man/PlotWindProfile.Rd
 fac12ae053b5a476a7e12b4a8fe10043 *man/RTModelProfile.Rd
 b4dd595a80d66cc9be93e53a3a4da0c1 *man/ReadGrib.Rd

diff --git a/NAMESPACE b/NAMESPACE
@@ -1 +1,2 @@
 exportPattern("^[[:alpha:]]+")
+import(XML, rvest)
diff --git a/NEWS b/NEWS
@@ -49,3 +49,8 @@ Made it possible for GetDODS to accept multiple variables
 Fixed date handing in DODSGrab.
 It now returns actual forecast dates instead of numbers.
 Fixed an insidious bug in BuildProfile that constructed profiles located over places not specified by the user.
+
+2.1.0
+Changed NOMADSRealTimeList() so that it scans the NOMADS Real Time model repository web site instead of depending on a hard-coded model list.
+This means the rNOMADS real time model list will always be up to date.
+I have not yet made the change for NOMADSArchiveList(); that will come in a future version.
diff --git a/R/GetArchiveGrib.R b/R/GetArchiveGrib.R
@@ -43,7 +43,9 @@ ArchiveGribGrab <- function(abbrev, model.date, model.run, pred, local.dir = "."
     #Get model info and set up URL to archive
     model.url <- NOMADSArchiveList("grib", abbrev=abbrev)$url
     download.url <- paste0(model.url, paste(model.date[1:6], collapse = ""), "/", paste(model.date, collapse = ""), "/") 
-    file.part <- paste0(paste(model.date, collapse = ""), "_", sprintf("%02.f", model.run), "00_", sprintf("%03.f", pred), suffix)
+    file.part <- paste0(paste(model.date, collapse = ""), "_", 
+    sprintf("%02.f", as.numeric(model.run)), 
+    "00_", sprintf("%03.f", as.numeric(pred)), suffix)
 
     #Find out which grib files are in the archive
     doc <- XML::htmlParse(download.url)

diff --git a/R/Models.R b/R/Models.R
@@ -3,6 +3,12 @@ NOMADSRealTimeList <- function(url.type, abbrev = NULL) {
     #Returns a list of model abbreviations for real time models, a short description, and URL for each model offered by the NOMADS server
     #If a specific model abbreviation is requested, the abbreviation is checked against the model list.
     #If a match is found, information is returned about that model; otherwise an error occurs
+    #
+    #
+    #A big shout out to user hrbrmstr on Stack Overflow for providing the table parsing code in this function
+    #http://stackoverflow.com/questions/27592575/dropped-rows-using-readhtmltable-in-r
+    #http://stackoverflow.com/users/1457051/hrbrmstr
+    #
     #INPUTS
     #    URL.TYPE determines which URL to return: one for downloading GRIB files (grib) or one for downloading dods data via DODS (dods)
     #    ABBREV is the model abbreviation that rNOMADS uses to figure out which model you want.
@@ -17,153 +23,87 @@ NOMADSRealTimeList <- function(url.type, abbrev = NULL) {
         stop("URL type must be either \"grib\" or \"dods\"!")
     }
 
-abbrevs <- c(
-    "fnl",
-    "gfs",
-    "gfs_hd",
-    "gfs_2p5",
-    "gens",
-    "gensbc_precip",
-    "gensbc",
-    "gensbc_ndgd",
-    "naefsbc",
-    "naefsbc_ndgd",
-    "ngac_a2d",
-    "ngac_a3d",
-    "ngac_aod",
-    "cfs_flx",
-    "cfs_pgb",
-    "aqm_daily",
-    "aqm_ozone_1hr",
-    "hiresak",
-    "hiresconus",
-    "hiresguam",
-    "hireshi",
-    "hirespr",
-    "nam_ak",
-    "nam",
-    "nam_na",
-    "nam_crb",
-    "nam_pac",
-    "nam_alaskanest",
-    "nam_conusnest",
-    "nam_hawaiinest",
-    "nam_priconest",
-    "akrtma",
-    "rtma2p5",
-    "gurtma",
-    "hirtma",
-    "prrtma",
-    "hrrr_2d",
-    "rap",
-    "rap32",
-    "narre",
-    "sref",
-    "srefbc",
-    "sref_na",
-    "sref_132",
-    "ofs",
-    "rtofs_hires",
-    "rtofs",
-    "seaice",
-    "wave",
-    "glw",
-    "wave_multi",
-    "hurricane_wave",
-    "nfcens",
-    "estofs",
-    "cmcens",
-    "fens",
-    "ncom")
+    base.url <- "http://nomads.ncep.noaa.gov/"
+    trim <- function(x) gsub("^[[:space:]]+|[[:space:]]+$", "", x)
 
-    names <- c(
-    "Final Operational Global Forecast System ",
-    "Global Forecast System 1x1 Degree ",
-    "Global Forecast System 0.5x0.5 Degree ",
-    "Global Forecast System 2.5x2.5 Degree ",
-    "Global Forecast System Ensemble",
-    "Global Forecast System Ensemble Precipitation Bias Corrected ",
-    "Global Forecast System Ensemble Bias Corrected ",
-    "Global Forecast System Ensemble National Digital Guidance Database Bias Corrected ",
-    "North American Ensemble Forecast System Bias Corrected ",
-    "North American Ensemble Forecast System National Digital Guidance Database Bias Corrected ",
-    "NOAA Environmental Modeling System Global Forecast System Aerosol Component 2D",
-    "NOAA Environmental Modeling System Global Forecast System Aerosol Component 3D",
-    "NOAA Environmental Modeling System Global Forecast System Aerosol Optical Depth",
-    "Climate Forecast System Flux Products",
-    "Climate Forecast System 3D Pressure Products",
-    "Air Quality Daily Maximum",
-    "Air Quality Hourly Surface Ozone",
-    "High Res Window Alaska",
-    "High Res Window - Continental United States",
-    "High Res Window - Guam",
-    "High Res Window - Hawaii",
-    "High Res Window - Puerto Rico",
-    "North American Mesoscale 12 km - Alaska",
-    "North American Mesoscale 12 km - Continental United States",
-    "North American Mesoscale 12 km - North America",
-    "North American Mesoscale 12 km - Caribbean and Central America", 
-    "North American Mesoscale 12 km - Pacific",
-    "North American Mesoscale Nest - Alaska",
-    "North American Mesoscale Nest - Continental United States",
-    "North American Mesoscale Nest - Hawaii",
-    "North American Mesoscale Nest - Puerto Rico",
-    "Real-Time Mesoscale Analysis - Alaska",
-    "Real Time Mesoscale Analysis - Continental United States 2.5 km Resolution",
-    "Real Time Mesoscale Analysis - Guam",
-    "Real Time Mesoscale Analysis - Hawaii",
-    "Real Time Mesoscale Analysis - Puerto Rico",
-    "High Resolution Rapid Refresh",
-    "Rapid Refresh Weather Prediction System",
-    "Rapid Refresh Weather Prediction System - 32 km Resolution",
-    "North American Rapid Refresh Ensemble",
-    "Short Range Ensemble Forecast - Continental United States 40 km",
-    "Short Range Ensemble Forecast - Continental United States 40 km Bias Corrected",
-    "Short Range Ensemble Forecast - Continental United States 32 km",
-    "Short Range Ensemble Forecast - Continental United States 16 km",
-    "Real Time Ocean Forecast System - Atlantic",
-    "Real Time Ocean Forecast System - Atlantic High Resolution",
-    "Real Time Ocean Forecast System - Global",
-    "Sea Ice",
-    "Operational Ocean Wave Predictions",
-    "Operational Ocean Wave Predictions - Great Lakes",
-    "Multi-grid Wave",
-    "Hurricane Wave",
-    "Combined Wave Ensemble",
-    "Extratropical Surge and Tide Operational Forecast System",
-    "Canadian Meterological Center Global Ensemble",
-    "Fleet Numerical Meteorology and Oceanography Ensemble Forecast System",
-    "U.S. Navy Operational Global Ocean Model") 
+    # neither rvest::html nor rvest::html_session liked it, hence using XML::htmlParse
+    doc <- XML::htmlParse(base.url)
 
+    ds <- doc %>% html_nodes(xpath="//table/descendant::th[@class='nomads'][1]/../../
+                                            descendant::td[contains(., 'http')]/
+                                            preceding-sibling::td[3]")
+    data.set <- ds %>% html_text() %>% trim()
 
-    if(!is.null(abbrev)) {
-        i <- which(abbrevs == abbrev)
-        if(length(i) == 0) {
-            stop(paste("The model you searched for:\"", abbrev, "\"is not included in rNOMADS.  Sorry!"))
-        } else {
-            if(url.type == "grib") {
-                 url <- paste0("http://nomads.ncep.noaa.gov/cgi-bin/filter_", abbrev, ".pl")
-            } else {
-                url <- paste0("http://nomads.ncep.noaa.gov:9090/dods/", abbrev, "/")
-            }
-        }
-    } else {
-        if(url.type == "grib") {
-           url <- paste0("http://nomads.ncep.noaa.gov/cgi-bin/filter_", abbrevs, ".pl") 
-        } else {
-           url <- paste0("http://nomads.ncep.noaa.gov:9090/dods/", abbrevs, "/")
-        }
-
-   }
+#    html_text() %>% trim()
 
-    #Handle known exceptions
-    if(url.type == "dods") {
-        url <- stringr::str_replace(url, "hrrr_2d", "hrrr")
-    }
+    grib.filter <- doc %>% html_nodes(xpath="//table/descendant::th[@class='nomads'][1]/../../
+                                  descendant::td[contains(., 'http')]/preceding-sibling::td[1]") %>%
+   sapply(function(x) {
+     ifelse(x %>% xpathApply("boolean(./a)"),
+           x %>% html_node("a") %>% html_attr("href"),
+           NA)
+    })
 
-    return(list(abbrevs = abbrevs, names = names, url = url))
-}
+   http.link <- doc %>% html_nodes("a[href^='/pub/data/']") %>% html_attr("href")
 
+   gds.alt <- doc %>% html_nodes(xpath="//table/descendant::th[@class='nomads'][1]/../../
+                              descendant::td[contains(., 'http')]/following-sibling::td[1]") %>%
+   sapply(function(x) {
+    ifelse(x %>% xpathApply("boolean(./a)"),
+           x %>% html_node("a") %>% html_attr("href"),
+           NA)
+   })
+
+   grib.abbrevs <- stringr::str_replace(stringr::str_replace(basename(grib.filter), "filter_", ""), ".pl", "")
+   dods.abbrevs <- basename(gds.alt)
+   dods.base.url <- "http://nomads.ncep.noaa.gov:9090/dods/"
+   if(is.null(abbrev)) {
+       if(url.type == "grib") {
+          good.abbrevs <- which(!is.na(grib.abbrevs))
+          model.list <- list(abbrev = grib.abbrevs[good.abbrevs], 
+               name = data.set[good.abbrevs], 
+               url = paste0(base.url, 
+               grib.filter[good.abbrevs]))
+       } else {
+          good.abbrevs <- which(!is.na(dods.abbrevs))
+          model.list <- list(abbrev = dods.abbrevs[good.abbrevs], 
+              name = data.set[good.abbrevs], 
+              url = paste0(dods.base.url, 
+              basename(gds.alt[good.abbrevs]), "/"))
+       }
+  } else {
+      if(url.type == "grib") {
+           abbrev.ind <- which(abbrev == grib.abbrevs)
+           if(length(abbrev.ind) > 0) {
+               model.list <- list(abbrev = grib.abbrevs[abbrev.ind], 
+                   name = data.set[abbrev.ind], 
+                   url = paste0(base.url, grib.filter[abbrev.ind]))
+           } else {
+                stop(paste0("The model you searched for: \"", 
+                    abbrev, 
+                    "\" is not included in NOMADS real time ", 
+                    url.type,
+                     " model products.  Sorry!"))    
+           }
+      } else {
+           abbrev.ind <- which(abbrev == dods.abbrevs)
+           if(length(abbrev.ind) > 0) {
+               model.list <- list(abbrev = dods.abbrevs[abbrev.ind], 
+                   name = data.set[abbrev.ind], 
+                   url = paste0(dods.base.url, basename(gds.alt[abbrev.ind]), "/"))
+           } else {
+                stop(paste0("The model you searched for: \"", 
+                    abbrev, 
+                    "\" is not included in NOMADS real time ", 
+                    url.type, 
+                     " model products.  Sorry!"))
+           }
+       }
+   }
+
+   return(model.list)
+}
+
 NOMADSArchiveList <- function(url.type, abbrev = NULL) {
     #Returns a list of model abbreviations for archived models, a short description, and URL for each model offered by the NOMADS server
     #If a specific model abbreviation is requested, the abbreviation is checked against the model list.

diff --git a/man/DODSGrab.Rd b/man/DODSGrab.Rd
@@ -1,7 +1,7 @@
 \name{DODSGrab}
 \alias{DODSGrab}
 \title{
-Download model data from the NOMADS server using the DODS - GrADS system.
+Download model data from the NOMADS server using the GrADS-DODS system.
 }
 \description{
 This function interfaces with the NOMADS server to download weather, ocean, and sea ice data.

diff --git a/man/GetClosestGFSForecasts.Rd b/man/GetClosestGFSForecasts.Rd
@@ -55,7 +55,10 @@ Daniel C. Bowman \email{daniel.bowman@unc.edu}
 
 #Figure out which forecasts to use
 forecast.date <- as.POSIXlt(Sys.time(), tz = "GMT")
-\dontrun{forecasts <- GetClosestGFSForecasts(forecast.date)}
+
+\dontrun{
+forecasts <- GetClosestGFSForecasts(forecast.date)
+}
 
 #Get levels
 pressure <- c(1, 2, 3, 5, 7,

diff --git a/man/NOMADSRealTimeList.Rd b/man/NOMADSRealTimeList.Rd
@@ -4,7 +4,7 @@
 Models available for download through rNOMADS
 }
 \description{
-A list of abbreviations, names and URLs for the NOMADS models.
+Scans the NOMADS Real Time web site to generate a list of available model products.
 Users can refer to this list to find out more information about the available models, and rNOMADS uses the abbreviations to determine which URLs to scan and download.
 }
 \usage{
@@ -28,14 +28,20 @@ NOMADSRealTimeList(url.type, abbrev = NULL)
 Daniel C. Bowman \email{daniel.bowman@unc.edu}
 }
 
+\note{
+A big thanks to user hrbrmstr at Stack Overflow for helping with the NCEP web site parsing code.
+}
 \seealso{
 \code{\link{WebCrawler}}, \code{\link{ParseModelPage}}, \code{\link{NOMADSArchiveList}}, \code{\link{GribGrab}}, \code{\link{DODSGrab}}
 }
 \examples{
 
-#The full model list in rNOMADS
+\dontrun{
+#Grib filter
+model.list <- NOMADSRealTimeList("grib")
 
+#DODS interface
 model.list <- NOMADSRealTimeList("dods")
-
+}
 }
 \keyword{ documentation }