Skip to content

Commit

Permalink
Merge 3d71ba1 into 435408c
Browse files Browse the repository at this point in the history
  • Loading branch information
@ctessum
ctessum committed Jul 13, 2020
2 parents 435408c + 3d71ba1 commit 008e233
Show file tree
Hide file tree
Showing 7 changed files with 248 additions and 31 deletions.
163 changes: 163 additions & 0 deletions cloud/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,3 +191,166 @@ func TestClient_fake(t *testing.T) {
}
})
}

func TestClient_fakeCOARDS(t *testing.T) {
checkConfig := func(cmd []string) {
wantCmd := []string{"inmap", "run", "steady",
"--EmissionUnits=tons/year",
"--EmissionsShapefiles=file://test/test/test_user/test_job/258bbcefe8c0073d6f323351463be9e9685e74bb92e367ca769b9536ed247213.shp",
"--InMAPData=file://test/test/test_user/test_job/434bf26e3fda1ef9cef7e1fa6cc6b5174d11a22b19cbe10d256adc83b2a97d44.ncf",
"--LogFile=file://test/test/test_user/test_job/LogFile",
"--NumIterations=0",
"--OutputFile=file://test/test/test_user/test_job/OutputFile.shp",
"--OutputVariables={\"TotalPM25\":\"PrimaryPM25 + pNH4 + pSO4 + pNO3 + SOA\",\"TotalPopD\":\"(exp(log(1.078)/10 * TotalPM25) - 1) * TotalPop * AllCause / 100000\"}\n",
"--VarGrid.CensusFile=file://test/test/test_user/test_job/72f6717ef5f6f9600378fe5b192776ba142b3e93311c3dfd0b67bfecbe399990.shp",
"--VarGrid.CensusPopColumns=TotalPop,WhiteNoLat,Black,Native,Asian,Latino",
"--VarGrid.GridProj=+proj=lcc +lat_1=33.000000 +lat_2=45.000000 +lat_0=40.000000 +lon_0=-97.000000 +x_0=0 +y_0=0 +a=6370997.000000 +b=6370997.000000 +to_meter=1",
"--VarGrid.HiResLayers=1",
"--VarGrid.MortalityRateColumns={\"AllCause\":\"TotalPop\",\"AsianMort\":\"Asian\",\"BlackMort\":\"Black\",\"LatinoMort\":\"Latino\",\"NativeMort\":\"Native\",\"WhNoLMort\":\"WhiteNoLat\"}\n",
"--VarGrid.MortalityRateFile=file://test/test/test_user/test_job/764874ad5081665459c67d40607f68df6fc689aa695b4822e012aef84cba5394.shp",
"--VarGrid.PopConcThreshold=1e-09", "--VarGrid.PopDensityThreshold=0.0055",
"--VarGrid.PopGridColumn=TotalPop", "--VarGrid.PopThreshold=40000", "--VarGrid.VariableGridDx=4000",
"--VarGrid.VariableGridDy=4000", "--VarGrid.VariableGridXo=-4000", "--VarGrid.VariableGridYo=-4000",
"--VarGrid.Xnests=2,2,2", "--VarGrid.Ynests=2,2,2",
"--VariableGridData=file://test/test/test_user/test_job/26b310adcf36530acdb518bd74b61355b2a2e7825c20a07f3631db412c655881.gob",
"--aep.GridRef=file://test/test/test_user/test_job/d471298031ee531438f90ae92878df0aae1f76fb81424e1f223bf7a602a1864c.txt",
"--aep.InventoryConfig.COARDSFiles={\"all\":[\"file://test/test/test_user/test_job/ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc\"]}",
"--aep.InventoryConfig.COARDSYear=2016",
"--aep.InventoryConfig.InputUnits=tons",
"--aep.SCCExactMatch=true",
"--aep.SpatialConfig.GridName=inmap",
"--aep.SpatialConfig.InputSR=+proj=longlat",
"--aep.SpatialConfig.MaxCacheEntries=10",
"--aep.SrgShapefileDirectory=no_default",
"--aep.SrgSpec=file://test/test/test_user/test_job/f299df4d61e915c2d415b18ceaa1339a2cd7f8481d7d3b6d13675bc0516a5c00.json",
"--aep.SrgSpecType=OSM",
}
if len(cmd) != len(wantCmd) {
t.Errorf("wrong command length: %d != %d", len(cmd), len(wantCmd))
}
for i, a := range cmd {
if i >= len(wantCmd) {
t.Errorf("command element %d: '%s' != ''", i, a)
} else if a != wantCmd[i] {
t.Errorf("command element %d: '%s' != '%s'", i, a, wantCmd[i])
}
}
}

checkRun := func(o []byte, err error) {
if err != nil {
t.Error(err)
}
for _, l := range strings.Split(string(o), "\n") {
if strings.Contains(strings.ToLower(l), "error") || strings.Contains(l, "no such") {
t.Log(l)
}
}
}
cfg := inmaputil.InitializeConfig()
cfg.Set("aep.InventoryConfig.COARDSFiles", "{\"all\":[\"${INMAP_ROOT_DIR}/emissions/aep/testdata/emis_coards_hawaii.nc\"]}")
cfg.Set("aep.SrgSpec", "${INMAP_ROOT_DIR}/emissions/aep/testdata/srgspec_osm.json")
cfg.Set("aep.SrgSpecType", "OSM")
cfg.Set("aep.GridRef", []string{"${INMAP_ROOT_DIR}/emissions/aep/aeputil/testdata/gridref_osm.txt"})
cfg.Set("aep.InventoryConfig.InputUnits", "tons")
cfg.Set("aep.InventoryConfig.COARDSYear", 2016)

c, err := cloud.NewFakeClient(checkConfig, checkRun, "file://test/test", cfg.Root, cfg.Viper, cfg.InputFiles(), cfg.OutputFiles())
if err != nil {
t.Fatal(err)
}
os.Mkdir("test", os.ModePerm)
defer os.RemoveAll("test")

jobSpec, err := cloud.JobSpec(cfg.Root, cfg.Viper, "test_job", []string{"run", "steady"}, cfg.InputFiles(), 1)
if err != nil {
t.Fatal(err)
}
ctx := context.WithValue(context.Background(), "user", "test_user")

t.Run("RunJob", func(t *testing.T) {
status, err := c.RunJob(ctx, jobSpec)
if err != nil {
t.Fatal(err)
}
wantStatus := &cloudrpc.JobStatus{
Status: cloudrpc.Status_Complete,
StartTime: 1359849600,
CompletionTime: 1359936000,
}
if !reflect.DeepEqual(wantStatus, status) {
t.Errorf("status:\n%+v\n!=\n%+v", status, wantStatus)
}
})

t.Run("Status", func(t *testing.T) {
status, err := c.Status(ctx, &cloudrpc.JobName{
Version: inmap.Version,
Name: "test_job",
})
if err != nil {
t.Fatal(err)
}
wantStatus := &cloudrpc.JobStatus{
Status: cloudrpc.Status_Complete,
StartTime: 1359849600,
CompletionTime: 1359936000,
}
if !reflect.DeepEqual(wantStatus, status) {
t.Errorf("status:\n%+v\n!=\n%+v", status, wantStatus)
}
})

t.Run("Output", func(t *testing.T) {
output, err := c.Output(ctx, &cloudrpc.JobName{
Version: inmap.Version,
Name: "test_job",
})
if err != nil {
t.Fatal(err)
}
wantFiles := map[string]int{
"LogFile": 94100,
"OutputFile.shp": 2276,
"OutputFile.dbf": 465,
"OutputFile.shx": 228,
"OutputFile.prj": 431,
}
if len(output.Files) != len(wantFiles) {
t.Errorf("wrong number of files: %d != %d", len(output.Files), len(wantFiles))
}
for name, data := range output.Files {
if wantLen, ok := wantFiles[name]; ok {
if len(data) != wantLen && name != "LogFile" {
t.Errorf("wrong file length for %s: %d != %d", name, len(data), wantLen)
}
} else {
t.Errorf("missing files '%s'", name)
}
}
})
t.Run("Delete", func(t *testing.T) {
_, err := c.Delete(ctx, &cloudrpc.JobName{
Version: inmap.Version,
Name: "test_job",
})
if err != nil {
t.Fatal(err)
}

// Ensure the directory is empty
err = filepath.Walk("test", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
fmt.Println(path)
return fmt.Errorf("found file %s in directory that should have been deleted", path)
})
if err != nil {
t.Error(err)
}
})
}
4 changes: 2 additions & 2 deletions cloud/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ func (c *Client) stageInputs(ctx context.Context, job *cloudrpc.JobSpec) error {
return err
}
for i, arg := range job.Args {
if fname == arg {
job.Args[i] = url.Scheme + "://" + url.Hostname() + "/" + filePath
if strings.Contains(arg, fname) {
job.Args[i] = strings.Replace(arg, fname, url.Scheme+"://"+url.Hostname()+"/"+filePath, -1)
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion cloud/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func TestRunInputFromViper(t *testing.T) {
"--VarGrid.VariableGridDy": "4000",
"--EmissionUnits": "tons/year",
"--LogFile": "",
"--aep.InventoryConfig.COARDSFiles": "ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc,ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc,ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc",
"--aep.InventoryConfig.COARDSFiles": "{\"xxx\":[\"ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc\",\"ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc\"],\"yyy\":[\"ffe280d818c1549074d0e15cfb74377b891287d7f81a4ad9038d0f65b12f6642.nc\"]}",
"--aep.InventoryConfig.COARDSYear": "0",
"--aep.InventoryConfig.InputUnits": "no_default",
"--aep.SCCExactMatch": "true",
Expand Down
67 changes: 44 additions & 23 deletions cloud/jobspec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,20 +90,40 @@ func JobSpec(root *cobra.Command, config *viper.Viper, name string, cmdArgs, inp
argVal := val
if _, ok := inputFields[f.Name]; ok {
argVal = ""
vals := stringsFromInterface(config.Get(f.Name))
for i, val := range vals {
if val == "no_default" {
continue
arg := config.Get(f.Name)
vals := stringsFromInterface(arg)
switch v := vals.(type) {
case []string:
for i, val := range v {
if val == "no_default" {
continue
}
val, visitErr = localFileToRunInput(val, js)
if visitErr != nil {
return
}
if i == 0 {
argVal += val
} else {
argVal += "," + val
}
}
val, visitErr = localFileToRunInput(val, js)
if visitErr != nil {
return
case map[string][]string:
for k, vals := range v {
for i, val := range vals {
v[k][i], visitErr = localFileToRunInput(val, js)
if visitErr != nil {
return
}
}
}
if i == 0 {
argVal += val
} else {
argVal += "," + val
b := bytes.NewBuffer(nil)
if err := json.NewEncoder(b).Encode(v); err != nil {
panic(err)
}
argVal = strings.TrimSpace(b.String())
default:
panic(fmt.Errorf("invalid type %T", vals))
}
}
switch {
Expand All @@ -120,13 +140,20 @@ func JobSpec(root *cobra.Command, config *viper.Viper, name string, cmdArgs, inp
return js, nil
}

func stringsFromInterface(val interface{}) []string {
// stringsFromInterface takes an interface{} and returns either a []string or a map[string][]string
func stringsFromInterface(val interface{}) interface{} {
switch t := val.(type) {
case string:
if t == "{}" || t == "{}\n" {
return []string{}
}
return []string{t}
// Try to decode as JSON.
d := json.NewDecoder(bytes.NewBuffer([]byte(t)))
var v interface{}
if err := d.Decode(&v); err != nil {
return []string{t} // If decoding JSON doesn't work, return string.
}
return stringsFromInterface(v)
case []string:
return t
case []interface{}:
Expand All @@ -136,18 +163,12 @@ func stringsFromInterface(val interface{}) []string {
}
return s
case map[string][]string:
var s []string
for _, vs := range t {
for _, v := range vs {
s = append(s, v)
}
}
return s
return t
case map[string]interface{}:
var s []string
for _, vs := range t {
s := make(map[string][]string)
for k, vs := range t {
for _, v := range vs.([]interface{}) {
s = append(s, v.(string))
s[k] = append(s[k], v.(string))
}
}
return s
Expand Down
26 changes: 22 additions & 4 deletions inmaputil/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,25 +193,43 @@ func VarGridConfig(cfg *viper.Viper) (*inmap.VarGridConfig, error) {

// aeputilConfig unmarshals an aeputil inventory and spatial configuration.
func aeputilConfig(cfg *viper.Viper) (*aeputil.InventoryConfig, *aeputil.SpatialConfig, error) {
outChan := outChan()

neiFiles, err := getStringMapStringSlice("aep.InventoryConfig.NEIFiles", cfg)
if err != nil {
return nil, nil, fmt.Errorf("inmaputil: parsing config variable aep.InventoryConfig.NEIFiles: %v", err)
}
for k, vs := range neiFiles {
for i, v := range vs {
neiFiles[k][i] = maybeDownload(context.TODO(), os.ExpandEnv(v), outChan)
}
}

coardsFiles, err := getStringMapStringSlice("aep.InventoryConfig.COARDSFiles", cfg)
if err != nil {
return nil, nil, fmt.Errorf("inmaputil: parsing config variable aep.InventoryConfig.COARDSFiles: %v", err)
}
for k, vs := range coardsFiles {
for i, v := range vs {
coardsFiles[k][i] = maybeDownload(context.TODO(), os.ExpandEnv(v), outChan)
}
}

srgSpec := maybeDownload(context.TODO(), os.ExpandEnv(cfg.GetString("aep.SrgSpec")), outChan)
var gridRef []string
for _, g := range cfg.GetStringSlice("aep.GridRef") {
gridRef = append(gridRef, maybeDownload(context.TODO(), g, outChan))
}

i := &aeputil.InventoryConfig{
NEIFiles: neiFiles,
COARDSFiles: coardsFiles,
COARDSYear: cfg.GetInt("aep.InventoryConfig.COARDSYear"),
InputUnits: cfg.GetString("aep.InventoryConfig.InputUnits"),
SrgSpec: os.ExpandEnv(cfg.GetString("aep.SrgSpec")),
SrgSpec: srgSpec,
SrgSpecType: cfg.GetString("aep.SrgSpecType"),
SrgShapefileDirectory: cfg.GetString("aep.SrgShapefileDirectory"),
GridRef: cfg.GetStringSlice("aep.GridRef"),
GridRef: gridRef,
SCCExactMatch: cfg.GetBool("aep.SCCExactMatch"),
}
i.PolsToKeep = aep.Speciation{
Expand All @@ -223,11 +241,11 @@ func aeputilConfig(cfg *viper.Viper) (*aeputil.InventoryConfig, *aeputil.Spatial
}

s := &aeputil.SpatialConfig{
SrgSpec: os.ExpandEnv(cfg.GetString("aep.SrgSpec")),
SrgSpec: srgSpec,
SrgSpecType: cfg.GetString("aep.SrgSpecType"),
SrgShapefileDirectory: cfg.GetString("aep.SrgShapefileDirectory"),
SCCExactMatch: cfg.GetBool("aep.SCCExactMatch"),
GridRef: cfg.GetStringSlice("aep.GridRef"),
GridRef: gridRef,
OutputSR: os.ExpandEnv(cfg.GetString("VarGrid.GridProj")),
InputSR: cfg.GetString("aep.SpatialConfig.InputSR"),
SpatialCache: cfg.GetString("aep.SpatialConfig.SpatialCache"),
Expand Down
3 changes: 2 additions & 1 deletion inmaputil/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,15 @@ func downloadBlob(ctx context.Context, path string, c chan string) string {
if err != nil {
panic(fmt.Errorf("inmaputil: failed creating temporary download directory: %v", err))
}
ext := filepath.Ext(url.Path)
fnames := expandShp(url.Path)
for _, fname := range fnames {
w, err := os.Create(filepath.Join(dir, filepath.Base(fname)))
if err != nil {
panic(fmt.Errorf("inmaputil: failed creating file for download: %v", err))
}
bucketPath := strings.TrimPrefix(url.Path, "/")
bucketPath = bucketPath[0:len(bucketPath)-4] + filepath.Ext(fname)
bucketPath = bucketPath[0:len(bucketPath)-len(ext)] + filepath.Ext(fname)
r, err := bucket.NewReader(ctx, bucketPath, nil)
if err != nil {
c <- err.Error()
Expand Down
14 changes: 14 additions & 0 deletions inmaputil/inmap_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,20 @@ func TestInMAPDynamic_coards(t *testing.T) {
}
}

func TestInMAPDynamic_coardsflag(t *testing.T) {
cfg := InitializeConfig()
cfg.Set("static", false)
cfg.Set("createGrid", false) // this isn't used for the dynamic grid
os.Setenv("InMAPRunType", "dynamic_coards")
cfg.Set("config", "../cmd/inmap/configExample_coards.toml")
cfg.Set("aep.InventoryConfig.COARDSFiles", "{\"all\":[\"${INMAP_ROOT_DIR}/emissions/aep/testdata/emis_coards_hawaii.nc\"]}")
cfg.Root.SetArgs([]string{"run", "steady"})
defer os.Remove(os.ExpandEnv("$INMAP_ROOT_DIR/cmd/inmap/testdata/output_dynamic_coards.log"))
defer inmap.DeleteShapefile(os.ExpandEnv("$INMAP_ROOT_DIR/cmd/inmap/testdata/output_dynamic_coards.shp"))
if err := cfg.Root.Execute(); err != nil {
t.Fatal(err)
}
}
func TestInMAPDynamicRemote_http(t *testing.T) {
cfg := InitializeConfig()
if err := os.Mkdir("test_bucket", os.ModePerm); err != nil {
Expand Down

0 comments on commit 008e233

Please sign in to comment.