Merge pull request #1408 from CartoDB/do-e2e-tests

Do e2e tests
CartoDB · Jan 7, 2020 · 76dc16d · 76dc16d
2 parents c51530b + d083fa3
commit 76dc16d
Show file tree

Hide file tree

Showing 28 changed files with 433 additions and 30 deletions.
diff --git a/.gitignore b/.gitignore
@@ -57,6 +57,7 @@ node_modules
 htmlcov
 test_*.json
 .pytest_cache
+tmp_file.csv
 
 # Sphinx documentation
 docs/build/

diff --git a/cartoframes/data/clients/bigquery_client.py b/cartoframes/data/clients/bigquery_client.py
@@ -10,7 +10,7 @@
 
 from ...auth import get_default_credentials
 from ...core.logger import log
-from ...utils.utils import timelogger
+from ...utils.utils import timelogger, is_ipython_notebook
 
 
 _GCS_CHUNK_SIZE = 25 * 1024 * 1024  # 25MB. This must be a multiple of 256 KB per the API specification.
@@ -186,7 +186,9 @@ def _get_table(self, project, dataset, table):
 
 
 def _rows_to_file(rows, file_path, column_names=None, progress_bar=True):
-    if progress_bar:
+    show_progress_bar = progress_bar and is_ipython_notebook()
+
+    if show_progress_bar:
         pb = tqdm.tqdm_notebook(total=rows.total_rows)
 
     with open(file_path, 'w') as csvfile:
@@ -197,5 +199,5 @@ def _rows_to_file(rows, file_path, column_names=None, progress_bar=True):
 
         for row in rows:
             csvwriter.writerow(row.values())
-            if progress_bar:
+            if show_progress_bar:
                 pb.update(1)
diff --git a/cartoframes/data/observatory/catalog/dataset.py b/cartoframes/data/observatory/catalog/dataset.py
@@ -396,7 +396,7 @@ def _join_geographies_geodataframes(geographies_gdf1, geographies_gdf2):
         return join_gdf['id'].unique()
 
     @check_do_enabled
-    def to_csv(self, file_path, credentials=None):
+    def to_csv(self, file_path, credentials=None, limit=None):
         """Download dataset data as a local csv file. You need Data Observatory enabled in your CARTO
         account, please contact us at support@carto.com for more information.
 
@@ -409,6 +409,7 @@ def to_csv(self, file_path, credentials=None):
                 credentials of CARTO user account. If not provided,
                 a default credentials (if set with :py:meth:`set_default_credentials
                 <cartoframes.auth.set_default_credentials>`) will be used.
+            limit (int, optional): number of rows to be downloaded.
 
         :raises CartoException: If you have not a valid license for the dataset being downloaded.
         :raises ValueError: If the credentials argument is not valid.
@@ -419,10 +420,10 @@ def to_csv(self, file_path, credentials=None):
             raise Exception('You are not subscribed to this Dataset yet. '
                             'Please, use the subscribe method first.')
 
-        self._download(_credentials, file_path)
+        self._download(_credentials, file_path, limit)
 
     @check_do_enabled
-    def to_dataframe(self, credentials=None):
+    def to_dataframe(self, credentials=None, limit=None):
         """Download dataset data as a pandas.DataFrame. You need Data Observatory enabled in your CARTO
         account, please contact us at support@carto.com for more information.
 
@@ -434,6 +435,7 @@ def to_dataframe(self, credentials=None):
                 credentials of CARTO user account. If not provided,
                 a default credentials (if set with :py:meth:`set_default_credentials
                 <cartoframes.auth.set_default_credentials>`) will be used.
+            limit (int, optional): number of rows to be downloaded.
 
         Returns:
             pandas.DataFrame
@@ -447,7 +449,7 @@ def to_dataframe(self, credentials=None):
             raise Exception('You are not subscribed to this Dataset yet. '
                             'Please, use the subscribe method first.')
 
-        return self._download(_credentials)
+        return self._download(_credentials, limit=limit)
 
     @check_do_enabled
     def subscribe(self, credentials=None):

diff --git a/cartoframes/data/observatory/catalog/entity.py b/cartoframes/data/observatory/catalog/entity.py
@@ -107,7 +107,7 @@ def _get_print_id(self):
 
         return self.id
 
-    def _download(self, credentials, file_path=None):
+    def _download(self, credentials, file_path=None, limit=None):
         if not self._is_available_in('bq'):
             raise CartoException('{} is not ready for Download. Please, contact us for more information.'.format(self))
 
@@ -122,7 +122,11 @@ def _download(self, credentials, file_path=None):
         project, dataset, table = full_remote_table_name.split('.')
 
         column_names = bq_client.get_table_column_names(project, dataset, table)
+
         query = 'SELECT * FROM `{}`'.format(full_remote_table_name)
+        if limit:
+            query = '{} LIMIT {}'.format(query, limit)
+
         job = bq_client.query(query)
 
         if file_path:

diff --git a/cartoframes/data/observatory/catalog/geography.py b/cartoframes/data/observatory/catalog/geography.py
@@ -199,7 +199,7 @@ def get_all(cls, filters=None, credentials=None):
         return cls._entity_repo.get_all(filters, credentials)
 
     @check_do_enabled
-    def to_csv(self, file_path, credentials=None):
+    def to_csv(self, file_path, credentials=None, limit=None):
         """Download geography data as a local csv file. You need Data Observatory enabled in your CARTO
         account, please contact us at support@carto.com for more information.
 
@@ -212,6 +212,7 @@ def to_csv(self, file_path, credentials=None):
                 credentials of CARTO user account. If not provided,
                 a default credentials (if set with :py:meth:`set_default_credentials
                 <cartoframes.auth.set_default_credentials>`) will be used.
+            limit (int, optional): number of rows to be downloaded.
 
         :raises CartoException: If you have not a valid license for the dataset being downloaded.
         :raises ValueError: If the credentials argument is not valud.
@@ -222,10 +223,10 @@ def to_csv(self, file_path, credentials=None):
             raise Exception('You are not subscribed to this Geography yet. '
                             'Please, use the subscribe method first.')
 
-        self._download(_credentials, file_path)
+        self._download(_credentials, file_path, limit)
 
     @check_do_enabled
-    def to_dataframe(self, credentials=None):
+    def to_dataframe(self, credentials=None, limit=None):
         """Download geography data as a pandas.DataFrame. You need Data Observatory enabled in your CARTO
         account, please contact us at support@carto.com for more information.
 
@@ -237,6 +238,7 @@ def to_dataframe(self, credentials=None):
                 credentials of CARTO user account. If not provided,
                 a default credentials (if set with :py:meth:`set_default_credentials
                 <cartoframes.auth.set_default_credentials>`) will be used.
+            limit (int, optional): number of rows to be downloaded.
 
         Returns:
             pandas.DataFrame
@@ -250,7 +252,7 @@ def to_dataframe(self, credentials=None):
             raise Exception('You are not subscribed to this Geography yet. '
                             'Please, use the subscribe method first.')
 
-        return self._download(_credentials)
+        return self._download(_credentials, limit=limit)
 
     @check_do_enabled
     def subscribe(self, credentials=None):

diff --git a/cartoframes/data/observatory/catalog/utils.py b/cartoframes/data/observatory/catalog/utils.py
@@ -1,20 +1,6 @@
 from .subscriptions import trigger_subscription
 from .subscription_info import fetch_subscription_info
-
-
-def is_ipython_notebook():
-    """
-    Detect whether we are in a Jupyter notebook.
-    """
-    try:
-        cfg = get_ipython().config
-        if 'IPKernelApp' in cfg:
-            return True
-        else:
-            return False
-    except NameError:
-        return False
-
+from ....utils.utils import is_ipython_notebook
 
 if is_ipython_notebook():
     from IPython.display import display

diff --git a/cartoframes/utils/utils.py b/cartoframes/utils/utils.py
@@ -425,3 +425,17 @@ def fn(*args, **kw):
             else:
                 raise e
     return fn
+
+
+def is_ipython_notebook():
+    """
+    Detect whether we are in a Jupyter notebook.
+    """
+    try:
+        cfg = get_ipython().config
+        if 'IPKernelApp' in cfg:
+            return True
+        else:
+            return False
+    except NameError:
+        return False
diff --git a/tests/e2e/data/client/test_bigquery_client.py b/tests/e2e/data/client/test_bigquery_client.py
@@ -41,6 +41,7 @@ def result(self):
         return ResponseMock(self.response)
 
 
+@pytest.mark.skip()
 class TestBigQueryClient(unittest.TestCase):
     def setUp(self):
         if (os.environ.get('APIKEY') is None or os.environ.get('USERNAME') is None):

diff --git a/tests/e2e/data/client/test_data_obs_client.py b/tests/e2e/data/client/test_data_obs_client.py
@@ -22,6 +22,7 @@
 warnings.filterwarnings('ignore')
 
 
+@pytest.mark.skip()
 class TestDataObsClient(unittest.TestCase, _UserUrlLoader):
     """Tests for cartoframes.client.DataObsClient"""
 

diff --git a/tests/e2e/data/observatory/__init__.py b/tests/e2e/data/observatory/__init__.py
diff --git a/tests/e2e/data/observatory/catalog/files/private-dataset.csv b/tests/e2e/data/observatory/catalog/files/private-dataset.csv
@@ -0,0 +1,2 @@
+BLOCKGROUP,HHDCY,MLTCY44111,MLTCY44112,MLTCY44121,MLTCY44122,MLTCY44131,MLTCY44132,MLTCY44211,MLTCY44221,MLTCY44229,MLTCY44311,MLTCY44411,MLTCY44412,MLTCY44413,MLTCY44419,MLTCY44421,MLTCY44422,MLTCY44511,MLTCY44512,MLTCY44521,MLTCY44522,MLTCY44523,MLTCY44529,MLTCY44531,MLTCY44611,MLTCY44612,MLTCY44613,MLTCY44619,MLTCY44719,MLTCY44811,MLTCY44812,MLTCY44813,MLTCY44814,MLTCY44815,MLTCY44819,MLTCY44821,MLTCY44831,MLTCY44832,MLTCY45111,MLTCY45112,MLTCY45113,MLTCY45114,MLTCY45121,MLTCY45211,MLTCY45291,MLTCY45299,MLTCY45311,MLTCY45321,MLTCY45322,MLTCY45331,MLTCY45391,MLTCY45392,MLTCY45393,MLTCY45399,MLTCY45411,MLTCY45421,MLTCY45431,MLTCY45439,MLTCY7211,MLTCY7212,MLTCY7213,MLTCY7221,MLTCY7222,MLTCY7223,MLTCY7224,RSGCY44111,RSGCY44112,RSGCY44121,RSGCY44122,RSGCY44131,RSGCY44132,RSGCY44211,RSGCY44221,RSGCY44229,RSGCY44311,RSGCY44411,RSGCY44412,RSGCY44413,RSGCY44419,RSGCY44421,RSGCY44422,RSGCY44511,RSGCY44512,RSGCY44521,RSGCY44522,RSGCY44523,RSGCY44529,RSGCY44531,RSGCY44611,RSGCY44612,RSGCY44613,RSGCY44619,RSGCY44719,RSGCY44811,RSGCY44812,RSGCY44813,RSGCY44814,RSGCY44815,RSGCY44819,RSGCY44821,RSGCY44831,RSGCY44832,RSGCY45111,RSGCY45112,RSGCY45113,RSGCY45114,RSGCY45121,RSGCY45211,RSGCY45291,RSGCY45299,RSGCY45311,RSGCY45321,RSGCY45322,RSGCY45331,RSGCY45391,RSGCY45392,RSGCY45393,RSGCY45399,RSGCY45411,RSGCY45421,RSGCY45431,RSGCY45439,RSGCY7211,RSGCY7212,RSGCY7213,RSGCY7221,RSGCY7222,RSGCY7223,RSGCY7224,geoid,do_date
+10159819031,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10159819031,2018
diff --git a/tests/e2e/data/observatory/catalog/files/private-geography.csv b/tests/e2e/data/observatory/catalog/files/private-geography.csv
@@ -0,0 +1,2 @@
+geoid,do_label,do_area,do_perimeter,do_num_vertices,geom
+240317046003,Block Group 3,449179.761,3074.888,27,"POLYGON((-77.123199 39.001205, -77.122886 38.996192, -77.122749 38.995662, -77.122566 38.9953, -77.121902 38.994705, -77.121162 38.9943, -77.120552 38.99422, -77.120574 38.99319, -77.119644 38.993232, -77.119194 38.993194, -77.118645 38.992882, -77.118217 38.992908, -77.118034 38.993309, -77.118019 38.993625, -77.118301 38.994338, -77.117309 38.994487, -77.116706 38.997055, -77.115943 39.000225, -77.114944 38.99966, -77.114746 39.00061, -77.116394 39.001152, -77.117065 39.00127, -77.118049 39.001274, -77.119689 39.001029, -77.120559 39.000968, -77.121215 39.001003, -77.123199 39.001205))"
diff --git a/tests/e2e/data/observatory/catalog/files/public-dataset.csv b/tests/e2e/data/observatory/catalog/files/public-dataset.csv
@@ -0,0 +1,3 @@
+geoid,nonfamily_households,family_households,median_year_structure_built,rent_burden_not_computed,rent_over_50_percent,rent_40_to_50_percent,rent_35_to_40_percent,rent_30_to_35_percent,rent_25_to_30_percent,rent_20_to_25_percent,rent_15_to_20_percent,rent_10_to_15_percent,rent_under_10_percent,total_pop,male_pop,female_pop,median_age,white_pop,black_pop,asian_pop,hispanic_pop,amerindian_pop,other_race_pop,two_or_more_races_pop,not_hispanic_pop,commuters_by_public_transportation,households,median_income,income_per_capita,housing_units,vacant_housing_units,vacant_housing_units_for_rent,vacant_housing_units_for_sale,median_rent,percent_income_spent_on_rent,owner_occupied_housing_units,million_dollar_housing_units,mortgaged_housing_units,families_with_young_children,two_parent_families_with_young_children,two_parents_in_labor_force_families_with_young_children,two_parents_father_in_labor_force_families_with_young_children,two_parents_mother_in_labor_force_families_with_young_children,two_parents_not_in_labor_force_families_with_young_children,one_parent_families_with_young_children,father_one_parent_families_with_young_children,father_in_labor_force_one_parent_families_with_young_children,commute_10_14_mins,commute_15_19_mins,commute_20_24_mins,commute_25_29_mins,commute_30_34_mins,commute_45_59_mins,aggregate_travel_time_to_work,income_less_10000,income_10000_14999,income_15000_19999,income_20000_24999,income_25000_29999,income_30000_34999,income_35000_39999,income_40000_44999,income_45000_49999,income_50000_59999,income_60000_74999,income_75000_99999,income_100000_124999,income_125000_149999,income_150000_199999,income_200000_or_more,renter_occupied_housing_units_paying_cash_median_gross_rent,owner_occupied_housing_units_lower_value_quartile,owner_occupied_housing_units_median_value,owner_occupied_housing_units_upper_value_quartile,married_households,occupied_housing_units,housing_units_renter_occupied,dwellings_1_units_detached,dwellings_1_units_attached,dwellings_2_units,dwellings_3_to_4_units,dwellings_5_to_9_units,dwellings_10_to_19_units,dwellings_20_to_49_units,dwellings_50_or_more_units,mobile_homes,housing_built_2005_or_later,housing_built_2000_to_2004,housing_built_1939_or_earlier,male_under_5,male_5_to_9,male_10_to_14,male_15_to_17,male_18_to_19,male_20,male_21,male_22_to_24,male_25_to_29,male_30_to_34,male_35_to_39,male_40_to_44,male_45_to_49,male_50_to_54,male_55_to_59,male_60_61,male_62_64,male_65_to_66,male_67_to_69,male_70_to_74,male_75_to_79,male_80_to_84,male_85_and_over,female_under_5,female_5_to_9,female_10_to_14,female_15_to_17,female_18_to_19,female_20,female_21,female_22_to_24,female_25_to_29,female_30_to_34,female_35_to_39,female_40_to_44,female_45_to_49,female_50_to_54,female_55_to_59,female_60_to_61,female_62_to_64,female_65_to_66,female_67_to_69,female_70_to_74,female_75_to_79,female_80_to_84,female_85_and_over,white_including_hispanic,black_including_hispanic,amerindian_including_hispanic,asian_including_hispanic,commute_5_9_mins,commute_35_39_mins,commute_40_44_mins,commute_60_89_mins,commute_90_more_mins,households_retirement_income,armed_forces,civilian_labor_force,employed_pop,unemployed_pop,not_in_labor_force,pop_16_over,pop_in_labor_force,asian_male_45_54,asian_male_55_64,black_male_45_54,black_male_55_64,hispanic_male_45_54,hispanic_male_55_64,white_male_45_54,white_male_55_64,bachelors_degree_2,bachelors_degree_or_higher_25_64,children,children_in_single_female_hh,commuters_by_bus,commuters_by_car_truck_van,commuters_by_carpool,commuters_by_subway_or_elevated,commuters_drove_alone,different_house_year_ago_different_city,different_house_year_ago_same_city,employed_agriculture_forestry_fishing_hunting_mining,employed_arts_entertainment_recreation_accommodation_food,employed_construction,employed_education_health_social,employed_finance_insurance_real_estate,employed_information,employed_manufacturing,employed_other_services_not_public_admin,employed_public_administration,employed_retail_trade,employed_science_management_admin_waste,employed_transportation_warehousing_utilities,employed_wholesale_trade,female_female_households,four_more_cars,gini_index,graduate_professional_degree,group_quarters,high_school_including_ged,households_public_asst_or_food_stamps,in_grades_1_to_4,in_grades_5_to_8,in_grades_9_to_12,in_school,in_undergrad_college,less_than_high_school_graduate,male_45_64_associates_degree,male_45_64_bachelors_degree,male_45_64_graduate_degree,male_45_64_less_than_9_grade,male_45_64_grade_9_12,male_45_64_high_school,male_45_64_some_college,male_45_to_64,male_male_households,management_business_sci_arts_employed,no_car,no_cars,not_us_citizen_pop,occupation_management_arts,occupation_natural_resources_construction_maintenance,occupation_production_transportation_material,occupation_sales_office,occupation_services,one_car,two_cars,three_cars,pop_25_64,pop_determined_poverty_status,population_1_year_and_over,population_3_years_over,poverty,sales_office_employed,some_college_and_associates_degree,walked_to_work,worked_at_home,workers_16_and_over,associates_degree,bachelors_degree,high_school_diploma,less_one_year_college,masters_degree,one_year_more_college,pop_25_years_over,commute_35_44_mins,commute_60_more_mins,commute_less_10_mins,commuters_16_over,hispanic_any_race,pop_5_years_over,speak_only_english_at_home,speak_spanish_at_home,speak_spanish_at_home_low_english,pop_15_and_over,pop_never_married,pop_now_married,pop_separated,pop_widowed,pop_divorced,do_date
+47157005000,160.0,183.0,1981.0,18.0,38.0,41.0,9.0,47.0,33.0,39.0,42.0,11.0,0.0,1042.0,415.0,627.0,28.9,16.0,1004.0,0.0,0.0,0.0,0.0,22.0,1042.0,52.0,343.0,14410.0,8747.0,465.0,122.0,14.0,0.0,391.0,30.5,65.0,0.0,20.0,75.0,0.0,0.0,0.0,0.0,0.0,75.0,0.0,0.0,4.0,59.0,108.0,10.0,20.0,13.0,,79.0,105.0,44.0,38.0,13.0,18.0,11.0,0.0,0.0,7.0,10.0,18.0,0.0,0.0,0.0,0.0,507.0,26600.0,,231300.0,26.0,343.0,278.0,195.0,0.0,13.0,63.0,69.0,40.0,25.0,55.0,5.0,0.0,10.0,81.0,44.0,38.0,27.0,26.0,30.0,0.0,0.0,22.0,11.0,13.0,31.0,11.0,15.0,21.0,41.0,0.0,2.0,0.0,5.0,26.0,5.0,31.0,16.0,24.0,76.0,47.0,34.0,17.0,19.0,42.0,32.0,34.0,13.0,38.0,19.0,25.0,26.0,22.0,0.0,6.0,22.0,38.0,36.0,13.0,6.0,38.0,16.0,1004.0,0.0,0.0,7.0,0.0,7.0,0.0,5.0,47.0,0.0,304.0,233.0,71.0,445.0,749.0,304.0,0.0,0.0,23.0,35.0,0.0,0.0,13.0,3.0,0.0,1.0,316.0,278.0,52.0,162.0,21.0,0.0,141.0,15.0,38.0,0.0,21.0,24.0,57.0,0.0,0.0,16.0,14.0,4.0,19.0,33.0,43.0,2.0,0.0,6.0,0.4372,13.0,45.0,235.0,147.0,90.0,41.0,111.0,331.0,63.0,181.0,7.0,0.0,1.0,2.0,18.0,41.0,10.0,79.0,0.0,54.0,76.0,151.0,0.0,54.0,11.0,80.0,30.0,58.0,122.0,64.0,0.0,328.0,1042.0,1019.0,999.0,552.0,30.0,135.0,7.0,0.0,233.0,28.0,0.0,188.0,22.0,13.0,85.0,564.0,7.0,5.0,7.0,233.0,0.0,,,,,,,,,,,20132017
+72051990021,0.0,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,0.0,0.0,0.0,0.0,,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,0.0,0.0,0.0,,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,,,,,,,,,20132017
diff --git a/tests/e2e/data/observatory/catalog/files/public-geography.csv b/tests/e2e/data/observatory/catalog/files/public-geography.csv
@@ -0,0 +1,3 @@
+geoid,do_label,do_area,do_perimeter,do_num_vertices,geom
+36047020200,202,134862.223,1609.684,5,"POLYGON((-74.010042 40.624686, -74.014203 40.620685, -74.012002 40.619353, -74.007839 40.623355, -74.010042 40.624686))"
+17031161000,1610,426284.325,2672.28,5,"POLYGON((-87.73397 41.953578, -87.733748 41.946293, -87.727401 41.946373, -87.727618 41.953662, -87.73397 41.953578))"