From 7f7d70d7686f3b25882bed08fdf1a8d893925b12 Mon Sep 17 00:00:00 2001 From: ivanzvonkov Date: Sun, 21 Aug 2022 04:44:50 -0400 Subject: [PATCH 1/2] Stable 0.1.3 --- buildings-example/openmapflow.yaml | 2 +- crop-mask-example/openmapflow.yaml | 2 +- maize-example/openmapflow.yaml | 2 +- openmapflow/constants.py | 2 +- openmapflow/labeled_dataset.py | 4 ++-- openmapflow/labeled_dataset_custom.py | 6 +++--- openmapflow/notebooks/new_data.ipynb | 18 +++++++++++++----- openmapflow/templates/train.py | 2 ++ 8 files changed, 24 insertions(+), 14 deletions(-) diff --git a/buildings-example/openmapflow.yaml b/buildings-example/openmapflow.yaml index 536959e3..43510949 100644 --- a/buildings-example/openmapflow.yaml +++ b/buildings-example/openmapflow.yaml @@ -1,4 +1,4 @@ -version: 0.1.3rc1 +version: 0.1.3 project: buildings-example description: OpenMapFlow buildings example gcloud: diff --git a/crop-mask-example/openmapflow.yaml b/crop-mask-example/openmapflow.yaml index 258ead01..346ff08d 100644 --- a/crop-mask-example/openmapflow.yaml +++ b/crop-mask-example/openmapflow.yaml @@ -1,4 +1,4 @@ -version: 0.1.3rc1 +version: 0.1.3 project: crop-mask-example description: OpenMapFlow crop mask example gcloud: diff --git a/maize-example/openmapflow.yaml b/maize-example/openmapflow.yaml index 2517d7be..d2ba7436 100644 --- a/maize-example/openmapflow.yaml +++ b/maize-example/openmapflow.yaml @@ -1,4 +1,4 @@ -version: 0.1.3rc1 +version: 0.1.3 project: maize-example description: OpenMapFlow maize example gcloud: diff --git a/openmapflow/constants.py b/openmapflow/constants.py index 13fb6971..3eb68a5d 100644 --- a/openmapflow/constants.py +++ b/openmapflow/constants.py @@ -13,7 +13,7 @@ TEMPLATE_README = TEMPLATES_DIR / "README.md" TEMPLATE_DEPLOY_YML = TEMPLATES_DIR / "github-deploy.yaml" TEMPLATE_TEST_YML = TEMPLATES_DIR / "github-test.yaml" -VERSION = "0.1.3rc1" +VERSION = "0.1.3" # -------------- Dataframe column names -------------------------------------- SOURCE = "source" diff --git a/openmapflow/labeled_dataset.py b/openmapflow/labeled_dataset.py index 70d246cb..2f4763bd 100644 --- a/openmapflow/labeled_dataset.py +++ b/openmapflow/labeled_dataset.py @@ -107,13 +107,13 @@ def summary(self, df: pd.DataFrame) -> str: + "\n" ) - def load_df(self, skip_to_np: bool = False) -> pd.DataFrame: + def load_df(self, skip_to_np: bool = False, check_eo_data: bool = True) -> pd.DataFrame: """Load dataset (labels + earth observation data) as a DataFrame""" if not self.df_path.exists(): print(self.create_dataset()) df = pd.read_csv(self.df_path) df = df[clean_df_condition(df)].copy() - if df[EO_DATA].isnull().any(): + if check_eo_data and df[EO_DATA].isnull().any(): raise ValueError( f"{self.dataset} has missing earth observation data, " + "run openmapflow create-datasets" diff --git a/openmapflow/labeled_dataset_custom.py b/openmapflow/labeled_dataset_custom.py index 2bc8d895..f1014055 100644 --- a/openmapflow/labeled_dataset_custom.py +++ b/openmapflow/labeled_dataset_custom.py @@ -41,7 +41,7 @@ from openmapflow.engineer import calculate_ndvi, fillna, load_tif, remove_bands from openmapflow.labeled_dataset import LabeledDataset, clean_df_condition from openmapflow.raw_labels import RawLabels -from openmapflow.utils import memoized, tqdm +from openmapflow.utils import tqdm temp_dir = tempfile.gettempdir() @@ -72,7 +72,6 @@ def _distance_point_from_center(lat_idx: int, lon_idx: int, tif) -> int: return x_dist + y_dist -@memoized def _generate_bbox_from_paths() -> Dict[Path, BBox]: cloud_eo_uris = get_cloud_tif_list(BucketNames.LABELED_EO) return { @@ -369,12 +368,13 @@ def set_df(i, start, eo_paths, lon, lat, pbar): df.at[i, EO_LON] = eo_lon df.at[i, EO_FILE] = eo_file df.at[i, EO_STATUS] = EO_STATUS_COMPLETE + return True with tqdm( total=len(df_with_eo_files), desc="Extracting matched earth observation data", ) as pbar: - np.vectorize(set_df)( + np.vectorize(set_df, otypes="b")( i=df_with_eo_files.index, start=df_with_eo_files[START], eo_paths=df_with_eo_files[MATCHING_EO_FILES], diff --git a/openmapflow/notebooks/new_data.ipynb b/openmapflow/notebooks/new_data.ipynb index 21e5c2a9..80694ccb 100644 --- a/openmapflow/notebooks/new_data.ipynb +++ b/openmapflow/notebooks/new_data.ipynb @@ -24,6 +24,15 @@ "If you don't already have one, obtain a Github Personal Access Token using the steps [here](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). Save this token somewhere private." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install \"ipywidgets>=7,<8\" -q # https://github.com/googlecolab/colabtools/issues/3020" + ] + }, { "cell_type": "code", "execution_count": null, @@ -81,7 +90,7 @@ }, "outputs": [], "source": [ - "!pip install -r requirements.txt -q\n", + "!pip install openmapflow[data] dvc[gdrive] -q\n", "!pip install pyyaml==5.4.1 -q" ] }, @@ -280,8 +289,7 @@ "outputs": [], "source": [ "# Pushing to remote storage\n", - "!dvc commit {DataPaths.RAW_LABELS} -f -q\n", - "!dvc commit {DataPaths.DATASETS} -f -q\n", + "!dvc commit\n", "!dvc push" ] }, @@ -297,7 +305,7 @@ "commit_message = input(\"Commit message: \")\n", "!git add .\n", "!git commit -m '{commit_message}'\n", - "!git push " + "!git push --set-upstream origin {branch_name}" ] }, { @@ -340,4 +348,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/openmapflow/templates/train.py b/openmapflow/templates/train.py index a8b29f7f..04a71da9 100644 --- a/openmapflow/templates/train.py +++ b/openmapflow/templates/train.py @@ -169,6 +169,8 @@ def forward(self, x): # ------------------------ Model saving -------------------------- if lowest_validation_loss == val_loss: + # Some models in tsai need to be modified to be TorchScriptable + # https://github.com/timeseriesAI/tsai/issues/561 sm = torch.jit.script(model) model_path = model_path_from_name(model_name=model_name) if model_path.exists(): From 0d5eac773eadcfa0e701b51d5ce3da1416850350 Mon Sep 17 00:00:00 2001 From: ivanzvonkov Date: Sun, 21 Aug 2022 04:50:01 -0400 Subject: [PATCH 2/2] black reformat --- openmapflow/labeled_dataset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openmapflow/labeled_dataset.py b/openmapflow/labeled_dataset.py index 2f4763bd..3f9e8467 100644 --- a/openmapflow/labeled_dataset.py +++ b/openmapflow/labeled_dataset.py @@ -107,7 +107,9 @@ def summary(self, df: pd.DataFrame) -> str: + "\n" ) - def load_df(self, skip_to_np: bool = False, check_eo_data: bool = True) -> pd.DataFrame: + def load_df( + self, skip_to_np: bool = False, check_eo_data: bool = True + ) -> pd.DataFrame: """Load dataset (labels + earth observation data) as a DataFrame""" if not self.df_path.exists(): print(self.create_dataset())