Skip to content

Commit

Permalink
cloud: Fix COARDs emissions for remote simulations
Browse files Browse the repository at this point in the history
  • Loading branch information
@ctessum
ctessum committed Jul 7, 2020
1 parent d5e071f commit 435408c
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 36 deletions.
6 changes: 0 additions & 6 deletions cloud/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ func init() {
func TestClient_fake(t *testing.T) {
checkConfig := func(cmd []string) {
wantCmd := []string{"inmap", "run", "steady",
"--EmissionMaskGeoJSON=",
"--EmissionUnits=tons/year",
"--EmissionsShapefiles=file://test/test/test_user/test_job/258bbcefe8c0073d6f323351463be9e9685e74bb92e367ca769b9536ed247213.shp",
"--InMAPData=file://test/test/test_user/test_job/434bf26e3fda1ef9cef7e1fa6cc6b5174d11a22b19cbe10d256adc83b2a97d44.ncf",
Expand All @@ -60,18 +59,13 @@ func TestClient_fake(t *testing.T) {
"--VarGrid.VariableGridDy=4000", "--VarGrid.VariableGridXo=-4000", "--VarGrid.VariableGridYo=-4000",
"--VarGrid.Xnests=2,2,2", "--VarGrid.Ynests=2,2,2",
"--VariableGridData=file://test/test/test_user/test_job/26b310adcf36530acdb518bd74b61355b2a2e7825c20a07f3631db412c655881.gob",
"--aep.GridRef=no_default",
"--aep.InventoryConfig.COARDSFiles={}\n",
"--aep.InventoryConfig.COARDSYear=0",
"--aep.InventoryConfig.InputUnits=no_default",
"--aep.InventoryConfig.NEIFiles={}\n",
"--aep.SCCExactMatch=true",
"--aep.SpatialConfig.GridName=inmap",
"--aep.SpatialConfig.InputSR=+proj=longlat",
"--aep.SpatialConfig.MaxCacheEntries=10",
"--aep.SpatialConfig.SpatialCache=",
"--aep.SrgShapefileDirectory=no_default",
"--aep.SrgSpec=no_default",
"--aep.SrgSpecType=no_default",
}
if len(cmd) != len(wantCmd) {
Expand Down
12 changes: 5 additions & 7 deletions cloud/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ import (

func TestRunInputFromViper(t *testing.T) {
cfg := inmaputil.InitializeConfig()
cfg.Viper.Set("aep.InventoryConfig.COARDSFiles", map[string][]string{
"xxx": {"../emissions/aep/testdata/emis_coards_hawaii.nc", "../emissions/aep/testdata/emis_coards_hawaii.nc"},
"yyy": {"../emissions/aep/testdata/emis_coards_hawaii.nc"}})
js, err := cloud.JobSpec(cfg.Root, cfg.Viper, "test_job", []string{"run", "steady"}, cfg.InputFiles(), 1)
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -65,20 +68,15 @@ func TestRunInputFromViper(t *testing.T) {
"--VarGrid.PopDensityThreshold": "0.0055",
"--VarGrid.VariableGridDy": "4000",
"--EmissionUnits": "tons/year",
"--EmissionMaskGeoJSON": "",
"--LogFile": "",
"--aep.GridRef": "no_default",
"--aep.InventoryConfig.COARDSFiles": "{}\n",
"--aep.InventoryConfig.COARDSFiles": "ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc,ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc,ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc",
"--aep.InventoryConfig.COARDSYear": "0",
"--aep.InventoryConfig.InputUnits": "no_default",
"--aep.InventoryConfig.NEIFiles": "{}\n",
"--aep.SCCExactMatch": "true",
"--aep.SpatialConfig.GridName": "inmap",
"--aep.SpatialConfig.InputSR": "+proj=longlat",
"--aep.SpatialConfig.MaxCacheEntries": "10",
"--aep.SpatialConfig.SpatialCache": "",
"--aep.SrgShapefileDirectory": "no_default",
"--aep.SrgSpec": "no_default",
"--aep.SrgSpecType": "no_default",
}
if len(js.Args) != len(wantArgs)*2 {
Expand Down Expand Up @@ -110,6 +108,7 @@ func TestRunInputFromViper(t *testing.T) {
"764874ad5081665459c67d40607f68df6fc689aa695b4822e012aef84cba5394.prj": 432,
"26b310adcf36530acdb518bd74b61355b2a2e7825c20a07f3631db412c655881.gob": 21276,
"434bf26e3fda1ef9cef7e1fa6cc6b5174d11a22b19cbe10d256adc83b2a97d44.ncf": 14284,
"ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc": 3484,
}
if len(js.FileData) != len(wantFiles) {
t.Errorf("incorrect number of files: %d != %d", len(js.FileData), len(wantFiles))
Expand Down Expand Up @@ -156,7 +155,6 @@ func TestSRPredictInputFromViper(t *testing.T) {
"--EmissionsShapefiles": "258bbcefe8c0073d6f323351463be9e9685e74bb92e367ca769b9536ed247213.shp",
"--OutputFile": "inmap_output.shp",
"--OutputVariables": "{\"PrimPM25\":\"PrimaryPM25\"}",
"--EmissionMaskGeoJSON": "",
"--SR.OutputFile": "${INMAP_ROOT_DIR}/cmd/inmap/testdata/output_${InMAPRunType}.shp",
"--VarGrid.GridProj": "+proj=lcc +lat_1=33.000000 +lat_2=45.000000 +lat_0=40.000000 +lon_0=-97.000000 +x_0=0 +y_0=0 +a=6370997.000000 +b=6370997.000000 +to_meter=1",
}
Expand Down
43 changes: 32 additions & 11 deletions cloud/jobspec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ func JobSpec(root *cobra.Command, config *viper.Viper, name string, cmdArgs, inp
argVal = ""
vals := stringsFromInterface(config.Get(f.Name))
for i, val := range vals {
if val == "no_default" {
continue
}
val, visitErr = localFileToRunInput(val, js)
if visitErr != nil {
return
Expand All @@ -103,12 +106,12 @@ func JobSpec(root *cobra.Command, config *viper.Viper, name string, cmdArgs, inp
}
}
}
if argVal != "false" {
if argVal == "true" {
js.Args = append(js.Args, fmt.Sprintf("--%s", f.Name), "true")
} else {
js.Args = append(js.Args, fmt.Sprintf("--%s", f.Name), argVal)
}
switch {
case argVal == "false", argVal == "" && f.Name != "LogFile":
case argVal == "true":
js.Args = append(js.Args, fmt.Sprintf("--%s", f.Name), "true")
default:
js.Args = append(js.Args, fmt.Sprintf("--%s", f.Name), argVal)
}
})
if visitErr != nil {
Expand All @@ -120,16 +123,34 @@ func JobSpec(root *cobra.Command, config *viper.Viper, name string, cmdArgs, inp
func stringsFromInterface(val interface{}) []string {
switch t := val.(type) {
case string:
return []string{val.(string)}
if t == "{}" || t == "{}\n" {
return []string{}
}
return []string{t}
case []string:
return val.([]string)
return t
case []interface{}:
valSlice := val.([]interface{})
s := make([]string, len(valSlice))
for i, v := range valSlice {
s := make([]string, len(t))
for i, v := range t {
s[i] = fmt.Sprint(v)
}
return s
case map[string][]string:
var s []string
for _, vs := range t {
for _, v := range vs {
s = append(s, v)
}
}
return s
case map[string]interface{}:
var s []string
for _, vs := range t {
for _, v := range vs.([]interface{}) {
s = append(s, v.(string))
}
}
return s
default:
panic(fmt.Errorf("cloud.JobSpec: invalid file field type %T", t))
}
Expand Down
24 changes: 12 additions & 12 deletions inmaputil/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -795,17 +795,17 @@ variable specifies the information to be output.`,
name: "aep.InventoryConfig.NEIFiles",
usage: `NEIFiles lists National Emissions Inventory emissions files. The file names can include environment variables. The format is map[sector name][list of files].
`,
defaultVal: map[string][]string{},
//isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
defaultVal: map[string][]string{},
isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
},
{
name: "aep.InventoryConfig.COARDSFiles",
usage: `COARDSFiles lists COARDS-compliant NetCDF emission files (NetCDF 4 and greater not supported). Information regarding the COARDS NetCDF conventions are available here: https://ferret.pmel.noaa.gov/Ferret/documentation/coards-netcdf-conventions. The file names can include environment variables. The format is map[sector name][list of files]. For COARDS files, the sector name will also be used as the SCC code.
`,
defaultVal: map[string][]string{},
//isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
defaultVal: map[string][]string{},
isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
},
{
name: "aep.InventoryConfig.COARDSYear",
Expand All @@ -824,9 +824,9 @@ variable specifies the information to be output.`,
name: "aep.SrgSpec",
usage: `SrgSpec gives the location of the surrogate specification file. It is used for assigning spatial locations to emissions records.
`,
defaultVal: "no_default",
//isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
defaultVal: "no_default",
isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
},
{
name: "aep.SrgSpecType",
Expand All @@ -846,9 +846,9 @@ variable specifies the information to be output.`,
name: "aep.GridRef",
usage: `GridRef specifies the locations of the spatial surrogate gridding reference files used for processing emissions. It is used for assigning spatial locations to emissions records.
`,
defaultVal: []string{"no_default"},
//isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
defaultVal: []string{"no_default"},
isInputFile: true,
flagsets: []*pflag.FlagSet{cfg.steadyCmd.Flags(), cfg.cloudStartCmd.Flags()},
},
{
name: "aep.SCCExactMatch",
Expand Down

0 comments on commit 435408c

Please sign in to comment.