Merge pull request #1014 from henrykironde/pre-release

Update version before release

CHANGES.md

@@ -1,3 +1,54 @@
+# v2.1.0
+
+## Major changes
+- Add Python interface
+- Add Retriever to conda
+- Auto complete of Retriever commands on Unix systems
+
+## Minor changes
+
+- Add license to datasets
+- Change the structure of raw data from string to list
+- Add testing on any modified dataset
+- Improve memory usage in cross-tab processing
+- Add capabilitiy for datasets to use custom Encoding
+- Use new Python interface for regression testing
+- Use Frictionless Data specification terminology for internals
+
+## New datasets
+- Add ant dataset and weather data to the portal dataset
+- NYC TreesCount
+- PREDICTS
+- aquatic_animal_excretion
+- biodiversity_response
+- bird_migration_data
+- chytr_disease_distr
+- croche_vegetation_data
+- dicerandra_frutescens
+- flensburg_food_web
+- great_basin_mammal_abundance
+- macroalgal_communities
+- macrocystis_variation
+- marine_recruitment_data
+- mediter_basin_plant_traits
+- nematode_traits
+- ngreatplains-flowering-dates
+- portal-dev
+- portal
+- predator_prey_body_ratio
+- predicts
+- socean_diet_data
+- species_exctinction_rates
+- streamflow_conditions
+- tree_canopy_geometries
+- turtle_offspring_nesting
+- Add vertnet individual datasets
+  vertnet_amphibians
+  vertnet_birds
+  vertnet_fishes
+  vertnet_mammals
+  vertnet_reptiles
+
 # v2.0.0
 
 ## Major changes

retriever/__main__.py

@@ -17,7 +17,7 @@
 from retriever.engines import engine_list, choose_engine
 from retriever.lib.datapackage import create_json, edit_json, delete_json, get_script_filename
 from retriever.lib.datasets import datasets, dataset_names, license
-from retriever.lib.defaults import sample_script, CITATION, ENCODING
+from retriever.lib.defaults import sample_script, CITATION, ENCODING, SCRIPT_SEARCH_PATHS
 from retriever.lib.get_opts import parser
 from retriever.lib.repository import check_for_updates
 from retriever.lib.scripts import SCRIPT_LIST
@@ -40,6 +40,10 @@ def main():
     else:
         # otherwise, parse them
 
+        if not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) and not \
+                [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1])
+                 if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]:
+            check_for_updates()
         script_list = SCRIPT_LIST()
 
         args = parser.parse_args()

retriever/_version.py

@@ -1 +1 @@
-__version__ = 'v2.1.dev'
+__version__ = 'v2.1.0'

retriever/engines/download_only.py

@@ -119,7 +119,7 @@ def dummy_method(self, *args, **kwargs):
                 'auto_create_table',
                 'insert_data_from_url',
                 }
-remove_methods = ['insert_data_from_file', 'create_db']
+remove_methods = ['insert_data_from_file', 'create_db', "create_table"]
 for name, method in methods:
     if (name not in keep_methods and
                 'download' not in name and

retriever/lib/download.py

@@ -1,11 +1,13 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import os
+
 from retriever.engines import choose_engine
+from retriever.lib.defaults import DATA_DIR, SCRIPT_WRITE_PATH
 from retriever.lib.scripts import SCRIPT_LIST
 from retriever.lib.tools import name_matches
-
-script_list = SCRIPT_LIST()
+from retriever.lib.repository import check_for_updates
 
 
 def download(dataset, path='./', quiet=False, subdir=False, debug=False):
@@ -18,7 +20,10 @@ def download(dataset, path='./', quiet=False, subdir=False, debug=False):
         'quiet': quiet
     }
     engine = choose_engine(args)
-
+    script_list = SCRIPT_LIST()
+    if not script_list or not os.listdir(SCRIPT_WRITE_PATH):
+        check_for_updates()
+        script_list = SCRIPT_LIST(force_compile=False)
     scripts = name_matches(script_list, args['dataset'])
     if scripts:
         for script in scripts:

retriever/lib/install.py

@@ -4,9 +4,10 @@
 import os
 
 from retriever.engines import choose_engine
-from retriever.lib.defaults import DATA_DIR
+from retriever.lib.defaults import DATA_DIR, SCRIPT_WRITE_PATH
 from retriever.lib.scripts import SCRIPT_LIST
 from retriever.lib.tools import name_matches
+from retriever.lib.repository import check_for_updates
 
 
 def _install(args, use_cache, debug):
@@ -15,6 +16,9 @@ def _install(args, use_cache, debug):
     engine.use_cache = use_cache
 
     script_list = SCRIPT_LIST()
+    if not script_list or not os.listdir(SCRIPT_WRITE_PATH):
+        check_for_updates()
+        script_list = SCRIPT_LIST(force_compile=False)
     data_sets_scripts = name_matches(script_list, args['dataset'])
     if data_sets_scripts:
         for data_sets_script in data_sets_scripts:

retriever_installer.iss

@@ -7,7 +7,7 @@
 ; (To generate a new GUID, click Tools | Generate GUID inside the IDE.)
 AppId={{0467A404-8C83-42A2-8C25-0903FBD14D9C}
 AppName=Data Retriever
-AppVersion=2.0.0
+AppVersion=2.1.0
 AppPublisher=Weecology
 AppPublisherURL=http://data-retriever.org
 AppSupportURL=http://data-retriever.org

setup.py

@@ -16,7 +16,11 @@
     os.system("cp hooks/pre-commit .git/hooks/pre-commit")
     os.system("chmod +x .git/hooks/pre-commit")
 
-__version__ = 'v2.1.dev'
+app_data = "~/.retriever/scripts"
+if os.path.exists(app_data):
+    os.system("rm -r {}".format(app_data))
+
+__version__ = 'v2.1.0'
 with open(os.path.join("retriever", "_version.py"), "w") as version_file:
     version_file.write("__version__ = " + "'" + __version__ + "'\n")
     version_file.close()
@@ -105,7 +109,7 @@ def clean_version(v):
     from retriever.compile import compile
     from retriever.lib.repository import check_for_updates
 
-    compile()
     check_for_updates(False)
+    compile()
 except:
     pass

version.txt

@@ -1,4 +1,4 @@
-v2.1.dev
+v2.1.0
 abalone_age.json,1.2.1
 amniote_life_hist.py,2.0.2
 antarctic_breed_bird.json,1.2.1
@@ -8,7 +8,7 @@ biodiversity_response.json,1.0.1
 biomass_allometry_db.py,1.4.2
 bird_migration_data.json,1.0.1
 bird_size.json,1.2.2
-breast_cancer_wi.json,1.1.1
+breast_cancer_wi.json,1.2.1
 breed_bird_survey.py,1.4.1
 breed_bird_survey_50stop.py,1.4.1
 butterfly_population_network.json,1.2.1