Skip to content

Commit

Permalink
Updated code for script to JSON conversion
Browse files Browse the repository at this point in the history
  • Loading branch information
@goelakash
goelakash committed May 24, 2016
1 parent 53335d4 commit 3356342
Show file tree
Hide file tree
Showing 33 changed files with 104 additions and 619 deletions.
17 changes: 8 additions & 9 deletions lib/parse_script_to_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@


def parse_script_to_json(script_file):
definition = open(SCRIPT_DIR+script_file+'.script', 'rb')
definition = open(SCRIPT_DIR + script_file + ".script", 'rb')

values = {}
urls = {}
Expand All @@ -30,7 +30,7 @@ def parse_script_to_json(script_file):
try:
tables[last_table]
except:
tables[table_name] = {'replace_columns': replace}
tables[table_name] = {'replace_columns': str(replace)}
elif key == "*nulls":
if last_table:
nulls = [eval(v) for v in [v.strip()
Expand All @@ -40,10 +40,10 @@ def parse_script_to_json(script_file):
except KeyError:
if replace:
tables[last_table] = {
'replace_columns': replace}
'replace_columns': str(replace)}
else:
tables[last_table] = {}
tables[last_table]['cleanup'] = {"nulls" :nulls}
tables[last_table]['cleanup'] = "Cleanup(correct_invalid_value, nulls=" + str(nulls) + ")"
elif key == "replace":
replace = [(v.split(',')[0].strip(), v.split(',')[1].strip())
for v in [v.strip() for v in value.split(';')]]
Expand Down Expand Up @@ -82,10 +82,7 @@ def parse_script_to_json(script_file):
except:
e = str(value)

tables[last_table][key] = str(e) if e.__class__ != str else "'" + e + "'"
elif key == ["ref","tags","urls"]:
# general script attributes
values[key] = value
tables[last_table][key] = "'" + str(e) + "'"
else:
values[key] = str(value)

Expand All @@ -112,11 +109,13 @@ def parse_script_to_json(script_file):
with open(JSON_DIR+script_file + '.json', 'w') as json_file:
json.dump(values,json_file,sort_keys=True, indent=4,
separators=(',', ': '))
json_file.write('\n')
json_file.close()

definition.close()

if __name__=="__main__":
for file in os.listdir(SCRIPT_DIR):
if file[-6:]=="script":
# print file
parse_script_to_json(file[:-7])
parse_script_to_json(file[:-7])
8 changes: 2 additions & 6 deletions scripts/AmnioteLH.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,7 @@
"shortname": "AmnioteDB",
"tables": {
"main": {
"cleanup": {
"nulls": [
-999
]
},
"cleanup": "Cleanup(correct_invalid_value, nulls=[-999])",
"columns": [
[
"record_id",
Expand Down Expand Up @@ -116,4 +112,4 @@
"urls": {
"main": "http://esapubs.org/archive/ecol/E096/269/Data_Files/Amniote_Database_Aug_2015.csv"
}
}
}
2 changes: 1 addition & 1 deletion scripts/Baldridge2012.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@
"main": "http://files.figshare.com/2023547/Species_abundances.csv",
"sites": "http://files.figshare.com/2023504/Sites_table_abundances.csv"
}
}
}
8 changes: 2 additions & 6 deletions scripts/Dryad_tamburello_etal_2013.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,7 @@
"shortname": "HomeRanges",
"tables": {
"ranges": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
}
},
"tags": [
Expand All @@ -19,4 +15,4 @@
"urls": {
"ranges": "http://datadryad.org/bitstream/handle/10255/dryad.84768/Tamburelloetal_HomeRangeDatabase.csv"
}
}
}
50 changes: 9 additions & 41 deletions scripts/EA_Chuetal2013.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,60 +6,28 @@
"shortname": "Steppe_plants_2013",
"tables": {
"allrecords_cover": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"allrecords_density": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"daily_climate": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"quad_info": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"quad_inventory": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"quad_stocking_rate": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"species_list": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"species_name_changes": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
}
},
"tags": [
Expand All @@ -78,4 +46,4 @@
"species_list": "http://esapubs.org/archive/ecol/E094/128/species_list.csv",
"species_name_changes": "http://esapubs.org/archive/ecol/E094/128/species_name_changes.csv"
}
}
}
38 changes: 7 additions & 31 deletions scripts/EA_Colin2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,46 +6,22 @@
"shortname": "PhytoplankonBiovolume",
"tables": {
"bvd_genus_ag": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"bvd_genus_raw": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"bvd_raw": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"bvd_species_ag": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"bvd_species_raw": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"taxa_table": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
}
},
"tags": [
Expand All @@ -60,4 +36,4 @@
"bvd_species_raw": "http://esapubs.org/archive/ecol/E095/257/bvd_species_raw_030614.csv",
"taxa_table": "http://esapubs.org/archive/ecol/E095/257/taxa_table_030614.csv"
}
}
}
9 changes: 2 additions & 7 deletions scripts/EA_Fray_Jorge2013.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,12 +167,7 @@
]
},
"precip": {
"cleanup": {
"nulls": [
".",
"n/a"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['.', 'n/a'])"
},
"seed_89_02": {
"columns": [
Expand Down Expand Up @@ -257,4 +252,4 @@
"precip": "http://esapubs.org/archive/ecol/E094/084/data/Fray_Jorge_Precipitation_1989-2005.csv",
"seed_89_02": "http://esapubs.org/archive/ecol/E094/084/data/Fray_Jorge_Seeds_1989-2002.txt"
}
}
}
8 changes: 2 additions & 6 deletions scripts/EA_Luo.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,7 @@
"shortname": "CForBioData",
"tables": {
"biomass": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
}
},
"tags": [
Expand All @@ -19,4 +15,4 @@
"urls": {
"biomass": "http://esapubs.org/archive/ecol/E095/177/CForBioData_v1.0.txt"
}
}
}
26 changes: 5 additions & 21 deletions scripts/EA_Matter_2014.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,32 +5,16 @@
"shortname": "Matter2014",
"tables": {
"Dispersal": {
"cleanup": {
"nulls": [
"n/a"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['n/a'])"
},
"Landscape": {
"cleanup": {
"nulls": [
"n/a"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['n/a'])"
},
"PopulationAbunMark_Recap": {
"cleanup": {
"nulls": [
"n/a"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['n/a'])"
},
"PopulationAbunTransect": {
"cleanup": {
"nulls": [
"n/a"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['n/a'])"
}
},
"tags": [
Expand All @@ -44,4 +28,4 @@
"PopulationAbunMark_Recap": "http://esapubs.org/archive/ecol/E095/258/PopulationAbunMark-Recap.csv",
"PopulationAbunTransect": "http://esapubs.org/archive/ecol/E095/258/PopulationAbunTransect.csv"
}
}
}
14 changes: 3 additions & 11 deletions scripts/EA_Pelissier2011.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,10 @@
"shortname": "TreeWesternGhats",
"tables": {
"ind_loc_girth": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
},
"sp_list": {
"cleanup": {
"nulls": [
"NA"
]
}
"cleanup": "Cleanup(correct_invalid_value, nulls=['NA'])"
}
},
"tags": [
Expand All @@ -28,4 +20,4 @@
"ind_loc_girth": "http://esapubs.org/archive/ecol/E092/115/UPSP_Demo_data.txt",
"sp_list": "http://esapubs.org/archive/ecol/E092/115/UPSP_Species_list2.txt"
}
}
}
Loading

0 comments on commit 3356342

Please sign in to comment.