From bbce34e9ba1cf08e522dca03daa5a361383fab44 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 2 Feb 2018 12:43:16 -0500 Subject: [PATCH 01/11] Turn JSON string into UTF-8 Previously this returned a byte string which could not be parsed into JSON on Python 3. --- rasterfoundry/aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasterfoundry/aws/s3.py b/rasterfoundry/aws/s3.py index 7f66569..48b499b 100644 --- a/rasterfoundry/aws/s3.py +++ b/rasterfoundry/aws/s3.py @@ -109,7 +109,7 @@ def download_to_string(uri): file_buffer = io.BytesIO() s3.download_fileobj( parsed_uri.netloc, parsed_uri.path[1:], file_buffer) - return str(file_buffer.getvalue()) + return file_buffer.getvalue().decode('utf-8') finally: file_buffer.close() else: From 80495f4687c5eee41d51b2ea9ceb72dd580c02ac Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 2 Feb 2018 12:44:28 -0500 Subject: [PATCH 02/11] Update spec.yml * Put quotes around yes and no so they are not interpreted as true/false. * Change match to unsure to match RF backend * Add definition for scene order response --- rasterfoundry/spec.yml | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/rasterfoundry/spec.yml b/rasterfoundry/spec.yml index e9a1b09..666b95f 100644 --- a/rasterfoundry/spec.yml +++ b/rasterfoundry/spec.yml @@ -1127,10 +1127,11 @@ paths: - Imagery parameters: - $ref: '#/parameters/uuid' - # TODO: fill in all possible responses responses: 200: description: Order of scenes in project for mosaic purposes + schema: + $ref: '#/definitions/SceneOrder' put: summary: Set a z-index order for scenes within the specified project tags: @@ -3844,10 +3845,10 @@ definitions: quality: type: string enum: - - YES - - NO - - MISS - - MATCH + - "YES" + - "NO" + - "MISS" + - "UNSURE" projectId: type: string format: uuid @@ -3874,10 +3875,10 @@ definitions: quality: type: string enum: - - YES - - NO - - MISS - - MATCH + - "YES" + - "NO" + - "MISS" + - "UNSURE" geometry: $ref: "#/definitions/Geometry" @@ -3903,10 +3904,10 @@ definitions: quality: type: string enum: - - YES - - NO - - MISS - - MATCH + - "YES" + - "NO" + - "MISS" + - "UNSURE" geometry: $ref: "#/definitions/Geometry" @@ -4025,3 +4026,16 @@ definitions: type: array items: $ref: '#/definitions/ShapeFeatureCreate' + + SceneOrder: + type: object + allOf: + - $ref: '#/definitions/PaginatedResponse' + - type: object + properties: + results: + type: array + items: + type: string + format: uuid + description: id of scene From 39e6b4692ef390d4d086ce76513029ea5b2b0ea8 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 2 Feb 2018 13:54:24 -0500 Subject: [PATCH 03/11] Sort image IDs by z-index order This will help us generate training chips correctly. Also, remove the quality field so that we let the RF backend set the appropriate default value. --- rasterfoundry/models/project.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rasterfoundry/models/project.py b/rasterfoundry/models/project.py index e6c0a55..0ed5564 100644 --- a/rasterfoundry/models/project.py +++ b/rasterfoundry/models/project.py @@ -180,8 +180,7 @@ def post_annotations(self, annotations_uri): 'label': properties['class_name'], 'description': '', 'machineGenerated': True, - 'confidence': properties['score'], - 'quality': 'YES' + 'confidence': properties['score'] } self.api.client.Imagery.post_projects_uuid_annotations( @@ -192,7 +191,15 @@ def get_image_source_uris(self): source_uris = [] scenes = self.api.client.Imagery.get_projects_uuid_scenes(uuid=self.id) \ .result().results + scene_order = self.api.client.Imagery.get_projects_uuid_order(uuid=self.id) \ + .result().results + + id_to_scene = {} for scene in scenes: + id_to_scene[scene.id] = scene + sorted_scenes = [id_to_scene[scene_id] for scene_id in scene_order] + + for scene in sorted_scenes: for image in scene.images: source_uris.append(image.sourceUri) From 3e74a7fcf4f5d25974d9ca429407ca58ec034add Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 2 Feb 2018 14:38:10 -0500 Subject: [PATCH 04/11] Reverse ordering of images This is so it matches the VRT convention that images later in the list are on top. --- rasterfoundry/models/project.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rasterfoundry/models/project.py b/rasterfoundry/models/project.py index 0ed5564..afde487 100644 --- a/rasterfoundry/models/project.py +++ b/rasterfoundry/models/project.py @@ -187,7 +187,7 @@ def post_annotations(self, annotations_uri): uuid=self.id, annotations=rf_annotations).future.result() def get_image_source_uris(self): - """Return the sourceUris of images associated with this project""" + """Return sourceUris of images for with this project sorted by z-index.""" source_uris = [] scenes = self.api.client.Imagery.get_projects_uuid_scenes(uuid=self.id) \ .result().results @@ -197,7 +197,8 @@ def get_image_source_uris(self): id_to_scene = {} for scene in scenes: id_to_scene[scene.id] = scene - sorted_scenes = [id_to_scene[scene_id] for scene_id in scene_order] + # Need to reverse so that order is from bottom-most to top-most layer. + sorted_scenes = [id_to_scene[scene_id] for scene_id in reversed(scene_order)] for scene in sorted_scenes: for image in scene.images: From 0c8d765bee4a677c0fcc865a24430a01d2dfb6aa Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Fri, 2 Feb 2018 17:21:55 -0500 Subject: [PATCH 05/11] Handle pagination --- rasterfoundry/models/project.py | 25 +++++++++++++++++++------ rasterfoundry/spec.yml | 2 ++ rasterfoundry/utils.py | 23 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/rasterfoundry/models/project.py b/rasterfoundry/models/project.py index afde487..eab619b 100644 --- a/rasterfoundry/models/project.py +++ b/rasterfoundry/models/project.py @@ -186,19 +186,32 @@ def post_annotations(self, annotations_uri): self.api.client.Imagery.post_projects_uuid_annotations( uuid=self.id, annotations=rf_annotations).future.result() + def get_scenes(self): + def get_page(page): + return self.api.client.Imagery.get_projects_uuid_scenes( + uuid=self.id, page=page).result() + + return get_all_paginated(get_page) + + def get_ordered_scene_ids(self): + def get_page(page): + return self.api.client.Imagery.get_projects_uuid_order( + uuid=self.id, page=page).result() + + # Need to reverse so that order is from bottom-most to top-most layer. + return list(reversed(get_all_paginated(get_page))) + def get_image_source_uris(self): """Return sourceUris of images for with this project sorted by z-index.""" source_uris = [] - scenes = self.api.client.Imagery.get_projects_uuid_scenes(uuid=self.id) \ - .result().results - scene_order = self.api.client.Imagery.get_projects_uuid_order(uuid=self.id) \ - .result().results + + scenes = self.get_scenes() + ordered_scene_ids = self.get_ordered_scene_ids() id_to_scene = {} for scene in scenes: id_to_scene[scene.id] = scene - # Need to reverse so that order is from bottom-most to top-most layer. - sorted_scenes = [id_to_scene[scene_id] for scene_id in reversed(scene_order)] + sorted_scenes = [id_to_scene[scene_id] for scene_id in ordered_scene_ids] for scene in sorted_scenes: for image in scene.images: diff --git a/rasterfoundry/spec.yml b/rasterfoundry/spec.yml index 666b95f..368cab6 100644 --- a/rasterfoundry/spec.yml +++ b/rasterfoundry/spec.yml @@ -1127,6 +1127,8 @@ paths: - Imagery parameters: - $ref: '#/parameters/uuid' + - $ref: '#/parameters/pageSize' + - $ref: '#/parameters/page' responses: 200: description: Order of scenes in project for mosaic purposes diff --git a/rasterfoundry/utils.py b/rasterfoundry/utils.py index 2c9609b..4fcdd9b 100644 --- a/rasterfoundry/utils.py +++ b/rasterfoundry/utils.py @@ -90,3 +90,26 @@ def mkdir_p(path): pass else: raise + + +def get_all_paginated(get_page_fn, list_field='results'): + """Get all objects from a paginated endpoint. + + Args: + get_page_fn: function that takes a page number and returns results + list_field: field in the results that contains the list of objects + + Returns: + List of all objects from a paginated endpoint + """ + has_next = True + all_results = [] + page = 0 + while has_next: + paginated_results = get_page_fn(page) + has_next = paginated_results.hasNext + page = paginated_results.page + 1 + for result in getattr(paginated_results, list_field): + all_results.append(result) + + return all_results From c6de7657be206aeff22639fc028c2ed3992d9376 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Mon, 5 Feb 2018 14:12:36 -0500 Subject: [PATCH 06/11] Save annotations for project to GeoJSON on S3 Also refactor s3.py and utils.py to make the utilities for uploading and downloading to S3 or local files more general. --- rasterfoundry/aws/s3.py | 28 +++++++++++++++++++++------- rasterfoundry/models/project.py | 28 +++++++++++++++++++++++++--- rasterfoundry/utils.py | 30 ------------------------------ 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/rasterfoundry/aws/s3.py b/rasterfoundry/aws/s3.py index 48b499b..a22ede9 100644 --- a/rasterfoundry/aws/s3.py +++ b/rasterfoundry/aws/s3.py @@ -3,10 +3,13 @@ from urllib.parse import urlparse import json import io +import os import boto3 from botocore.exceptions import ClientError +from ..utils import mkdir_p + s3 = boto3.client('s3') @@ -102,15 +105,26 @@ def unauthorize_bucket(bucket_name): return resp['ResponseMetadata']['HTTPStatusCode'] -def download_to_string(uri): - parsed_uri = urlparse(uri) +def file_to_str(file_uri): + parsed_uri = urlparse(file_uri) if parsed_uri.scheme == 's3': - try: - file_buffer = io.BytesIO() + with io.BytesIO() as file_buffer: s3.download_fileobj( parsed_uri.netloc, parsed_uri.path[1:], file_buffer) return file_buffer.getvalue().decode('utf-8') - finally: - file_buffer.close() else: - raise ValueError('uri needs to be for s3') + with open(file_uri, 'r') as file_buffer: + return file_buffer.read() + + +def str_to_file(content_str, file_uri): + parsed_uri = urlparse(file_uri) + if parsed_uri.scheme == 's3': + bucket = parsed_uri.netloc + key = parsed_uri.path[1:] + with io.BytesIO(bytes(content_str, encoding='utf-8')) as str_buffer: + s3.upload_fileobj(str_buffer, bucket, key) + else: + mkdir_p(os.path.dirname(file_uri)) + with open(file_uri, 'w') as content_file: + content_file.write(content_str) diff --git a/rasterfoundry/models/project.py b/rasterfoundry/models/project.py index eab619b..b1f77fe 100644 --- a/rasterfoundry/models/project.py +++ b/rasterfoundry/models/project.py @@ -3,12 +3,14 @@ import uuid import json import copy +from datetime import date, datetime from .. import NOTEBOOK_SUPPORT from ..decorators import check_notebook from ..exceptions import GatewayTimeoutException from .map_token import MapToken -from ..aws.s3 import download_to_string +from ..aws.s3 import str_to_file, file_to_str +from ..utils import get_all_paginated if NOTEBOOK_SUPPORT: from ipyleaflet import ( @@ -171,8 +173,8 @@ def tms(self): ) def post_annotations(self, annotations_uri): - annotations = json.loads(download_to_string(annotations_uri)) - # Convert annotations to RF format. + annotations = json.loads(file_to_str(annotations_uri)) + # Convert RV annotations to RF format. rf_annotations = copy.deepcopy(annotations) for feature in rf_annotations['features']: properties = feature['properties'] @@ -186,6 +188,26 @@ def post_annotations(self, annotations_uri): self.api.client.Imagery.post_projects_uuid_annotations( uuid=self.id, annotations=rf_annotations).future.result() + def get_annotations(self): + def get_page(page): + return self.api.client.Imagery.get_projects_uuid_annotations( + uuid=self.id, page=page).result() + + return get_all_paginated(get_page, list_field='features') + + def save_annotations_json(self, output_uri): + features = self.get_annotations() + geojson = {'features': [feature._as_dict() for feature in features]} + + def json_serial(obj): + """JSON serializer for objects not serializable by default json code.""" + if isinstance(obj, (datetime, date)): + return obj.isoformat() + raise TypeError('Type {} not serializable'.format(str(type(obj)))) + + geojson_str = json.dumps(geojson, default=json_serial) + str_to_file(geojson_str, output_uri) + def get_scenes(self): def get_page(page): return self.api.client.Imagery.get_projects_uuid_scenes( diff --git a/rasterfoundry/utils.py b/rasterfoundry/utils.py index 4fcdd9b..43f11df 100644 --- a/rasterfoundry/utils.py +++ b/rasterfoundry/utils.py @@ -1,10 +1,5 @@ from future.standard_library import install_aliases # noqa install_aliases() # noqa -from urllib.parse import urlparse -from os.path import join -import tempfile -import uuid -import json import os import errno @@ -57,31 +52,6 @@ def start_raster_vision_job(self, job_name, command): return job_id -def upload_raster_vision_config(config_dict, config_uri_root): - """Upload a config file to S3 - - Args: - config_dict: a dictionary to turn into a JSON file to upload - config_uri_root: the root of the URI to upload the config to - - Returns: - remote URI of the config file generate using a UUID - """ - with tempfile.NamedTemporaryFile('w') as config_file: - json.dump(config_dict, config_file) - config_uri = join( - config_uri_root, '{}.json'.format(uuid.uuid1())) - s3 = boto3.resource('s3') - parsed_uri = urlparse(config_uri) - # Rewind file to beginning so that full content will be loaded. - # Without this line 0 bytes are uploaded. - config_file.seek(0) - s3.meta.client.upload_file( - config_file.name, parsed_uri.netloc, parsed_uri.path[1:]) - - return config_uri - - def mkdir_p(path): try: os.makedirs(path) From 5396f8e32934ae73e6b004bec4ee474675cbdb85 Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Mon, 5 Feb 2018 14:14:07 -0500 Subject: [PATCH 07/11] Update save_project_config to save annotation files For each project, save the annotations to a file and then generate a project config file based on them. --- rasterfoundry/api.py | 42 ++++++++++++++++++++++++++------------- rasterfoundry/settings.py | 2 +- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/rasterfoundry/api.py b/rasterfoundry/api.py index 0b504e1..fa699f4 100644 --- a/rasterfoundry/api.py +++ b/rasterfoundry/api.py @@ -1,5 +1,6 @@ import os import json +import uuid from bravado.requests_client import RequestsClient from bravado.client import SwaggerClient @@ -8,7 +9,8 @@ from .models import Project, MapToken, Analysis from .exceptions import RefreshTokenException -from .utils import mkdir_p +from .aws.s3 import str_to_file +from .settings import RV_TEMP_URI SPEC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'spec.yml') @@ -145,47 +147,59 @@ def get_scenes(self, **kwargs): kwargs['bbox'] = ','.join(str(x) for x in bbox) return self.client.Imagery.get_scenes(**kwargs).result() - def get_project_configs(self, project_ids, annotation_uris): + def get_project_config(self, project_ids, annotations_uris=None): """Get data needed to create project config file for prep_train_data The prep_train_data script requires a project config files which lists the images and annotation URIs associated with each project - that will be used to generate training data. + that will be used to generate training data. If the annotation_uris + are not specified, an annotation file for each project will be + generated and saved to S3. Args: project_ids: list of project ids to make training data from - annotation_uris: list of corresponding annotation URIs + annotations_uris: optional list of corresponding annotation URIs Returns: Object of form [{'images': [...], 'annotations':...}, ...] """ project_configs = [] - for project_id, annotation_uri in zip(project_ids, annotation_uris): + for project_ind, project_id in enumerate(project_ids): proj = Project( self.client.Imagery.get_projects_uuid(uuid=project_id).result(), self) + + if annotations_uris is None: + annotations_uri = os.path.join( + RV_TEMP_URI, 'annotations', '{}.json'.format(uuid.uuid4())) + proj.save_annotations_json(annotations_uri) + else: + annotations_uri = annotations_uris[project_ind] + image_uris = proj.get_image_source_uris() project_configs.append({ 'id': project_id, 'images': image_uris, - 'annotations': annotation_uri + 'annotations': annotations_uri }) return project_configs - def write_project_configs(self, project_ids, annotation_uris, output_path): - """Write project config file to disk. + def save_project_config(self, project_ids, output_uri, + annotations_uris=None): + """Save project config file. This file is needed by Raster Vision to prepare training data, make predictions, and evaluate predictions. Args: project_ids: list of project ids to make training data from - annotation_uris: list of corresponding annotation URIs output_path: where to write the project config file + annotations_uris: optional list of corresponding annotation URIs """ - project_configs = self.get_project_configs( - project_ids, annotation_uris) - mkdir_p(os.path.dirname(output_path)) - with open(output_path, 'w') as output_file: - json.dump(project_configs, output_file, sort_keys=True, indent=4) + project_config = self.get_project_config( + project_ids, annotations_uris) + project_config_str = json.dumps( + project_config, sort_keys=True, indent=4) + + str_to_file(project_config_str, output_uri) diff --git a/rasterfoundry/settings.py b/rasterfoundry/settings.py index df904a0..358e401 100644 --- a/rasterfoundry/settings.py +++ b/rasterfoundry/settings.py @@ -1,4 +1,4 @@ RV_CPU_QUEUE = 'raster-vision-cpu' RV_CPU_JOB_DEF = 'raster-vision-cpu' -RV_PROJ_CONFIG_DIR_URI = 's3://raster-vision-lf-dev/detection/configs/projects/rf-generated' +RV_TEMP_URI = 's3://raster-vision-lf-dev/detection/rf-generated' DEVELOP_BRANCH = 'develop' From 218abca4c53733a7cb9a1c62ef1e1962162f77bf Mon Sep 17 00:00:00 2001 From: Lewis Fishgold Date: Mon, 5 Feb 2018 14:15:01 -0500 Subject: [PATCH 08/11] Make annotation quality check optional When we post these, we should leave the quality blank so that the we let the backend be responsible for setting the default value. --- rasterfoundry/spec.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/rasterfoundry/spec.yml b/rasterfoundry/spec.yml index 368cab6..6d2226c 100644 --- a/rasterfoundry/spec.yml +++ b/rasterfoundry/spec.yml @@ -2022,7 +2022,6 @@ parameters: description: Quality check on annotations in: query type: string - required: true enum: - PASS - FAIL From efc943cc466de953f72e4eb6f57e8d2056c34bd5 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Mon, 19 Mar 2018 15:35:10 -0400 Subject: [PATCH 09/11] Point to user-configable swagger spec location (#49) --- README.rst | 11 +++++++++++ rasterfoundry/api.py | 27 +++++++++++++++++++-------- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cbed75c..912bb85 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,17 @@ Usage # Get TMS URl without token one_project.tms() +Versions +~~~~~~~~ + +The latest version of `rasterfoundry` always points to the most recently released swagger spec in +the raster-foundry/raster-foundy-api-spec repository. If you need to point to a different spec +version, either install a version of the python client that refers to the appropriate spec, or +set the `RF_API_SPEC_PATH` environment variable to a url or local file path pointing to the +version of the spec that you want to use. + +Generally this shouldn't matter, because the Raster Foundry API shouldn't have breaking changes. + Installation ------------ diff --git a/rasterfoundry/api.py b/rasterfoundry/api.py index fa699f4..5c39eca 100644 --- a/rasterfoundry/api.py +++ b/rasterfoundry/api.py @@ -1,19 +1,27 @@ -import os import json +import os import uuid -from bravado.requests_client import RequestsClient from bravado.client import SwaggerClient -from bravado.swagger_model import load_file +from bravado.requests_client import RequestsClient +from bravado.swagger_model import load_file, load_url from simplejson import JSONDecodeError -from .models import Project, MapToken, Analysis -from .exceptions import RefreshTokenException from .aws.s3 import str_to_file +from .exceptions import RefreshTokenException +from .models import Analysis, MapToken, Project from .settings import RV_TEMP_URI -SPEC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'spec.yml') +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +SPEC_PATH = os.getenv( + 'RF_API_SPEC_PATH', + 'https://raw.githubusercontent.com/raster-foundry/raster-foundry-api-spec/master/spec.yml' +) class API(object): @@ -34,7 +42,10 @@ def __init__(self, refresh_token=None, api_token=None, self.http = RequestsClient() self.scheme = scheme - spec = load_file(SPEC_PATH) + if urlparse(SPEC_PATH).netloc: + spec = load_url(SPEC_PATH) + else: + spec = load_file(SPEC_PATH) spec['host'] = host spec['schemes'] = [scheme] From 9550e0a5c586d0273b573f0e9b2e7b59673e2c94 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Tue, 27 Mar 2018 14:57:43 -0400 Subject: [PATCH 10/11] Support async exports (#50) * Include export and visualization examples with analyses * Add shapely dependency to setup.py * Fixup spec Note that the export spec works for posting exports, but doesn't work for listing exports. There's something wrong with telling bravado that `mask` is an `object`, I think, or at least -- when you remove `mask` from the section on exports `api.client.get_exports` works. * Add export sugar to projects and analyses * Remove unused imports * Add polling * Flake8 * -Add a new export example notebook. -Update models. -Update spec. * -Finish the export notebook. -Update spec. -Update gitignore. * Clean up async export handling notebooks and models * -Fix export notebook bugs. -Update export model to expose configuration of rastersize on an export supported by backend. * -Update export file name logic. -Update and clean up export notebook. * Flake 8 * Expose raster_size param in create_export method of project model. --- .gitignore | 3 + examples/Analyses.ipynb | 178 +++++++++++++++++++++++++- examples/Export.ipynb | 209 +++++++++++++++++++++++++++++++ examples/Projects.ipynb | 74 ++++++----- examples/Uploads.ipynb | 2 +- rasterfoundry/api.py | 22 +++- rasterfoundry/models/__init__.py | 1 + rasterfoundry/models/analysis.py | 26 ++-- rasterfoundry/models/export.py | 172 +++++++++++++++++++++++++ rasterfoundry/models/project.py | 44 ++++--- rasterfoundry/spec.yml | 44 ++----- setup.py | 3 +- 12 files changed, 673 insertions(+), 105 deletions(-) create mode 100644 examples/Export.ipynb create mode 100644 rasterfoundry/models/export.py diff --git a/.gitignore b/.gitignore index d8ed680..0fb8bc5 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,6 @@ ENV/ /site /.idea/ /raster-foundry-python-client.iml + +# MacOS +.DS_Store diff --git a/examples/Analyses.ipynb b/examples/Analyses.ipynb index 1c616de..3c277d3 100644 --- a/examples/Analyses.ipynb +++ b/examples/Analyses.ipynb @@ -11,6 +11,13 @@ "api = API(refresh_token=refresh_token)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Getting analyses from the client" + ] + }, { "cell_type": "code", "execution_count": null, @@ -21,7 +28,21 @@ "source": [ "analyses = api.analyses\n", "analysis = analyses[0]\n", - "analyses" + "analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualizing analyses" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Comparing to projects" ] }, { @@ -37,25 +58,170 @@ "project.compare(analysis, m)\n", "m" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exporting analyses using tile layers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from ipyleaflet import DrawControl\n", + "dc = DrawControl()\n", + "\n", + "m = analysis.get_map()\n", + "m.add_control(dc)\n", + "analysis.add_to(m)\n", + "m\n", + "\n", + "# Draw a polygon on the map below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "last_draw = dc.last_draw\n", + "last_draw" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Calculate a bounding box from the last draw" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def snap_to_360(x_coord):\n", + " \"\"\"Snap an x coordinate to [-180, 180]\n", + " \n", + " Coordinates coming back from the API for some projects can be\n", + " outside this range, and coordinates in the bbox outside this\n", + " range make the export API upset. When it's upset, it returns\n", + " an array with just a single 0 in it, which is not an accurate\n", + " representation of the project normally.\n", + " \"\"\"\n", + " return x_coord - round((x_coord + 180) / 360, 0) * 360\n", + "\n", + "def geom_to_bbox(geom):\n", + " coords = geom['geometry']['coordinates'][0]\n", + " min_x = snap_to_360(min([point[0] for point in coords]))\n", + " min_y = min([point[1] for point in coords])\n", + " max_x = snap_to_360(max([point[0] for point in coords]))\n", + " max_y = max([point[1] for point in coords])\n", + " return ','.join(map(str, [min_x, min_y, max_x, max_y]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "bbox = geom_to_bbox(last_draw)\n", + "bbox" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Export as GeoTIFF\n", + "\n", + "Note: this example requires\n", + "[`numpy`](http://www.numpy.org/),\n", + "[`matplotlib`](http://matplotlib.org/), and a fairly recent version of\n", + "[`rasterio`](https://mapbox.github.io/rasterio/).\n", + "\n", + "If you don't have them, you can run the cell at the bottom of this notebook,\n", + "provided your pip installation directory is writable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from rasterio.io import MemoryFile\n", + "import matplotlib.pyplot as plt\n", + "\n", + "thumbnail = analysis.get_thumbnail(bbox=bbox, zoom=8)\n", + "\n", + "with MemoryFile(thumbnail) as memfile:\n", + " with memfile.open() as dataset:\n", + " plt.imshow(dataset.read(1), cmap='RdBu')\n", + " \n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Request an asynchronous export\n", + "\n", + "Asynchronous export is good when you have a large analysis or bounding box, or when you need a high\n", + "zoom level in the resulting geotiff. Creating an asynchronous export requires only a\n", + "bbox and zoom level for this analysis. Created exports run remotely. For examples of what\n", + "you can do with exports, check out the [Exports](./Export.ipynb) notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "export = analysis.create_export(bbox=bbox, zoom=13)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "pip install numpy matplotlib rasterio==1.0a12" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 2", "language": "python", - "name": "python3" + "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.4" + "pygments_lexer": "ipython2", + "version": "2.7.12" } }, "nbformat": 4, diff --git a/examples/Export.ipynb b/examples/Export.ipynb new file mode 100644 index 0000000..36bc205 --- /dev/null +++ b/examples/Export.ipynb @@ -0,0 +1,209 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exports\n", + "\n", + "This notebooks shows the steps on how we create an export using a project instance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from rasterfoundry.api import API\n", + "refresh_token = ''\n", + "api = API(refresh_token=refresh_token)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "api.projects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create an export directly\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a polygon as the mask for the export" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ipyleaflet import DrawControl\n", + "\n", + "project = api.projects[-1]\n", + "m = project.get_map()\n", + "\n", + "dc = DrawControl()\n", + "m.add_control(dc)\n", + "project.add_to(m)\n", + "m\n", + "\n", + "# Draw a polygon in the map below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def snap_to_360(x_coord):\n", + " \"\"\"Snap an x coordinate to [-180, 180]\n", + " \n", + " Coordinates coming back from the API for some projects can be\n", + " outside this range, and coordinates in the bbox outside this\n", + " range make the export API upset. When it's upset, it returns\n", + " an array with just a single 0 in it, which is not an accurate\n", + " representation of the project normally.\n", + " \"\"\"\n", + " return x_coord - round((x_coord + 180) / 360, 0) * 360\n", + "\n", + "def geom_to_bbox(geom):\n", + " coords = geom['geometry']['coordinates'][0]\n", + " min_x = snap_to_360(min([point[0] for point in coords]))\n", + " min_y = min([point[1] for point in coords])\n", + " max_x = snap_to_360(max([point[0] for point in coords]))\n", + " max_y = max([point[1] for point in coords])\n", + " return ','.join(map(str, [min_x, min_y, max_x, max_y]))\n", + "\n", + "bbox = geom_to_bbox(dc.last_draw)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create an export through a project instance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_export = project.create_export(bbox=bbox, zoom=8, raster_size=1000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Wait until the export is finished" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "completed_export = project_export.wait_for_completion()\n", + "# it will say 'EXPORTED' as the output of this block if the export is finished\n", + "completed_export.export_status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# When the export is done, we will get a list of download URLs\n", + "project_export.files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Download and plot the export\n", + "\n", + "Note: this example requires\n", + "[`numpy`](http://www.numpy.org/),\n", + "[`matplotlib`](http://matplotlib.org/), and a fairly recent version of\n", + "[`rasterio`](https://mapbox.github.io/rasterio/).\n", + "\n", + "If you don't have them, you can run the cell at the bottom of this notebook,\n", + "provided your pip installation directory is writable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from rasterio.io import MemoryFile\n", + "import matplotlib.pyplot as plt\n", + "\n", + "data = project_export.download_file_bytes()\n", + "\n", + "with MemoryFile(data) as memfile:\n", + " with memfile.open() as dataset:\n", + " plt.imshow(dataset.read(1), cmap='viridis')\n", + " \n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Installs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "pip install rasterio==1.0a12 matplotlib" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/Projects.ipynb b/examples/Projects.ipynb index 0c8ac96..552807b 100644 --- a/examples/Projects.ipynb +++ b/examples/Projects.ipynb @@ -50,9 +50,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# Change the index in projects[3] to a value within your list of projects\n", @@ -74,9 +72,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# Change the index in projects[4] to a value within your list of projects\n", @@ -90,21 +86,21 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Project export" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Export as PNG" + "## Project export\n", + "\n", + "### Synchronous export with the tile server\n", + "\n", + "Synchronous export is good when your project doesn't cover a large\n", + "area, or when you don't need a high zoom level in the resulting\n", + "geotiff.\n", + "\n", + "#### Export as PNG" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true, "scrolled": true }, "outputs": [], @@ -118,9 +114,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "Image(project.png(bbox, zoom=15))" @@ -130,14 +124,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Export as GeoTIFF" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Display in the notebook\n", + "#### Export as GeoTIFF\n", + "\n", + "#### Display in the notebook\n", "\n", "Note: this example requires\n", "[`numpy`](http://www.numpy.org/),\n", @@ -158,9 +147,9 @@ "import matplotlib.pyplot as plt\n", "project = api.projects[3]\n", "bbox = '-121.726057,37.278423,-121.231672,37.377250'\n", - "data = project.geotiff(bbox, zoom=17)\n", + "thumbnail = project.geotiff(bbox, zoom=10)\n", "\n", - "with MemoryFile(data) as memfile:\n", + "with MemoryFile(thumbnail) as memfile:\n", " with memfile.open() as dataset:\n", " plt.imshow(dataset.read(1), cmap='RdBu')\n", " \n", @@ -171,7 +160,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Save as a file" + "#### Save as a file" ] }, { @@ -187,6 +176,27 @@ " outf.write(data)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Asynchronous export\n", + "\n", + "Asynchronous export is good when you have a large project, or when you need a high\n", + "zoom level in the resulting geotiff. Creating an asynchronous export requires only a\n", + "bbox and zoom level for this project. Created exports run remotely. For examples of what\n", + "you can do with exports, check out the [Exports](./Export.ipynb) notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "export = project.create_export(bbox, zoom=10)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -197,13 +207,11 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "%%bash\n", - "pip install numpy matplotlib rasterio==1.0a8" + "pip install numpy matplotlib rasterio==1.0a12" ] } ], diff --git a/examples/Uploads.ipynb b/examples/Uploads.ipynb index b0737a6..906334d 100644 --- a/examples/Uploads.ipynb +++ b/examples/Uploads.ipynb @@ -151,7 +151,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.12" + "version": "2.7.14" } }, "nbformat": 4, diff --git a/rasterfoundry/api.py b/rasterfoundry/api.py index 5c39eca..2dd448d 100644 --- a/rasterfoundry/api.py +++ b/rasterfoundry/api.py @@ -7,9 +7,10 @@ from bravado.swagger_model import load_file, load_url from simplejson import JSONDecodeError + from .aws.s3 import str_to_file from .exceptions import RefreshTokenException -from .models import Analysis, MapToken, Project +from .models import Analysis, MapToken, Project, Export from .settings import RV_TEMP_URI try: @@ -47,6 +48,7 @@ def __init__(self, refresh_token=None, api_token=None, else: spec = load_file(SPEC_PATH) + self.app_host = host spec['host'] = host spec['schemes'] = [scheme] @@ -147,6 +149,24 @@ def analyses(self): analyses.append(Analysis(analysis, self)) return analyses + @property + def exports(self): + """List exports a user has access to + + Returns: + List[Export] + """ + has_next = True + page = 0 + exports = [] + while has_next: + paginated_exports = self.client.Imagery.get_exports(page=page).result() + has_next = paginated_exports.hasNext + page = paginated_exports.page + 1 + for export in paginated_exports.results: + exports.append(Export(export, self)) + return exports + def get_datasources(self, **kwargs): return self.client.Datasources.get_datasources(**kwargs).result() diff --git a/rasterfoundry/models/__init__.py b/rasterfoundry/models/__init__.py index 41b6aeb..6d1a9a1 100644 --- a/rasterfoundry/models/__init__.py +++ b/rasterfoundry/models/__init__.py @@ -2,3 +2,4 @@ from .map_token import MapToken # NOQA from .upload import Upload # NOQA from .analysis import Analysis # NOQA +from .export import Export # NOQA diff --git a/rasterfoundry/models/analysis.py b/rasterfoundry/models/analysis.py index 85a8780..3f4abcf 100644 --- a/rasterfoundry/models/analysis.py +++ b/rasterfoundry/models/analysis.py @@ -2,6 +2,7 @@ import requests from .. import NOTEBOOK_SUPPORT +from .export import Export from .project import Project from ..decorators import check_notebook from ..exceptions import GatewayTimeoutException @@ -36,16 +37,7 @@ def __init__(self, analysis, api): self.name = analysis.name self.id = analysis.id - def get_export(self, bbox, zoom=10): - """Download this Analysis as a single band tiff - - Args: - bbox (str): Bounding box(formatted as 'x1,y1,x2,y2') for the download - zoom (number): Zoom level for the download - - Returns: - str - """ + def get_thumbnail(self, bbox, zoom, raw=False): export_path = self.EXPORT_TEMPLATE.format(analysis=self.id) request_path = '{scheme}://{host}{export_path}'.format( scheme=self.api.scheme, host=self.api.tile_host, export_path=export_path @@ -57,6 +49,7 @@ def get_export(self, bbox, zoom=10): 'bbox': bbox, 'zoom': zoom, 'token': self.api.api_token, + 'colorCorrect': 'false' if raw else 'true' } ) if response.status_code == requests.codes.gateway_timeout: @@ -67,6 +60,19 @@ def get_export(self, bbox, zoom=10): response.raise_for_status() return response + def create_export(self, bbox, zoom=10, **exportOpts): + """Download this Analysis as a single band tiff + + Args: + bbox (str): Bounding box(formatted as 'x1,y1,x2,y2') for the download + zoom (int): Zoom level for the download + exportOpts (dict): Additional parameters to pass to an async export job + + Returns: + Export + """ + return Export.create_export(self.api, bbox=bbox, zoom=zoom, analysis=self, **exportOpts) + def tms(self, node=None): """Returns a TMS URL for this project diff --git a/rasterfoundry/models/export.py b/rasterfoundry/models/export.py new file mode 100644 index 0000000..adbd188 --- /dev/null +++ b/rasterfoundry/models/export.py @@ -0,0 +1,172 @@ +"""An Export is a job to get underlying geospatial data out of Raster Foundry""" + +import logging +import time + +import requests +from shapely.geometry import mapping, box, MultiPolygon +from bravado import exception + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) +logger.setLevel('INFO') + + +class Export(object): + def __repr__(self): + return ''.format(self.name) + + def __init__(self, export, api): + """Instantiate a new Export + + Args: + export (Export): generated Export object from specification + api (API): api used to make requests on behalf of the export + """ + + self._export = export + # Someday, exports will have names, but not yet + self.name = export.id + self.id = export.id + self.api = api + self.export_status = export.exportStatus + self.path = export.exportOptions.source + + @property + def files(self): + try: + fnames_res = self.api.client.Imagery.get_exports_uuid_files(uuid=self.id).result() + fnames = filter( + lambda name: name.upper() != 'RFUploadAccessTestFile'.upper(), + fnames_res) + return [ + 'https://{app_host}/api/exports/{export_id}/files/{file_name}'.format( + app_host=self.api.app_host, + export_id=self.id, + file_name=fname) for fname in fnames] + except exception.HTTPNotFound: + logger.info("The files can't be found until an export is completed") + + @classmethod + def poll_export_status(cls, + api, + export_id, + until=['EXPORTED', 'FAILED'], + delay=15): + """Poll the status of an export until it is done + + Note: if you don't include FAILED in the until parameter, polling will continue even + if the export has failed. + + Args: + api (API): API to use for requests + export_id (str): UUID of the export to poll for + until ([str]): list of statuses to indicate completion + delay (int): how long to wait between attempts + + Returns: + Export + """ + + if 'FAILED' not in until: + logger.warn( + 'Not including FAILED in until can result in states in which this ' + 'function will never return. You may have left off FAILED by accident. ' + 'If that is the case, you should include FAILED in until and try again.' + ) + export = api.client.Imagery.get_exports_uuid(uuid=export_id).result() + while export.exportStatus not in until: + time.sleep(delay) + export = api.client.Imagery.get_exports_uuid( + uuid=export_id).result() + return Export(export, api) + + @classmethod + def create_export(cls, + api, + bbox, + zoom, + project=None, + analysis=None, + visibility='PRIVATE', + source=None, + export_type='S3', + raster_size=4000): + """Create an asynchronous export job for a project or analysis + + Only one of project_id or analysis_id should be specified + + Args: + api (API): API to use for requests + bbox (str): comma-separated bounding box of region to export + zoom (int): the zoom level for performing the export + project (Project): the project to export + analysis (Analysis): the analysis to export + visibility (Visibility): what the export's visibility should be set to + source (str): the destination for the exported files + export_type (str): one of 'S3', 'LOCAL', or 'DROPBOX' + raster_size (int): desired tiff size after export, 4000 by default - same as backend + + Returns: + An export object + """ + + if project is not None and analysis is not None: + raise ValueError( + 'Ambiguous export target -- only one of project or analysis should ' + 'be specified') + elif project is None and analysis is None: + raise ValueError( + 'Nothing to export -- one of project or analysis must be specified' + ) + elif project is not None: + update_dict = { + 'projectId': project.id, + 'organizationId': project._project.organizationId + } + else: + update_dict = { + 'toolRunId': analysis.id, + 'organizationId': analysis._analysis.organizationId + } + + box_poly = MultiPolygon([box(*map(float, bbox.split(',')))]) + + export_create = { + 'exportOptions': { + 'mask': mapping(box_poly), + 'resolution': zoom, + 'rasterSize': raster_size + }, + 'projectId': None, + 'exportStatus': 'TOBEEXPORTED', + 'exportType': export_type, + 'visibility': visibility, + 'toolRunId': None, + 'organizationId': None + } + export_create.update(update_dict) + return Export( + api.client.Imagery.post_exports(Export=export_create).result(), + api) + + def wait_for_completion(self): + """Wait until this export succeeds or fails, returning the completed export + + Returns: + Export + """ + return self.__class__.poll_export_status(self.api, self.id) + + def download_file_bytes(self, index=0): + """Download the exported file from this export to memory + + Args: + index (int): which of this export's files to download + + Returns: + a binary file + """ + resp = requests.get(self.files[index], params={'token': self.api.api_token}) + resp.raise_for_status() + return resp.content diff --git a/rasterfoundry/models/project.py b/rasterfoundry/models/project.py index b1f77fe..fd78ff8 100644 --- a/rasterfoundry/models/project.py +++ b/rasterfoundry/models/project.py @@ -1,15 +1,17 @@ """A Project is a collection of zero or more scenes""" -import requests -import uuid -import json import copy +import json +import uuid from datetime import date, datetime +import requests + +from .export import Export +from .map_token import MapToken from .. import NOTEBOOK_SUPPORT +from ..aws.s3 import file_to_str, str_to_file from ..decorators import check_notebook from ..exceptions import GatewayTimeoutException -from .map_token import MapToken -from ..aws.s3 import str_to_file, file_to_str from ..utils import get_all_paginated if NOTEBOOK_SUPPORT: @@ -90,19 +92,7 @@ def get_map_token(self): if resp.results: return MapToken(resp.results[0], self.api) - def get_export(self, bbox, zoom=10, export_format='png', raw=False): - """Download this project as a file - - PNGs will be returned if the export_format is anything other than tiff - - Args: - bbox (str): Bounding box (formatted as 'x1,y1,x2,y2') for the download - export_format (str): Requested download format - - Returns: - str - """ - + def get_thumbnail(self, bbox, zoom, export_format, raw): headers = self.api.http.session.headers.copy() headers['Accept'] = 'image/{}'.format( export_format @@ -133,6 +123,20 @@ def get_export(self, bbox, zoom=10, export_format='png', raw=False): response.raise_for_status() return response + def create_export(self, bbox, zoom=10, raster_size=4000): + """Create an export job for this project + + Args: + bbox (str): Bounding box (formatted as 'x1,y1,x2,y2') for the download + zoom (int): Zoom level for the download + raster_size (int): desired tiff size after export, 4000 by default - same as backend + + Returns: + Export + """ + return Export.create_export( + self.api, bbox=bbox, zoom=zoom, project=self, raster_size=raster_size) + def geotiff(self, bbox, zoom=10, raw=False): """Download this project as a geotiff @@ -146,7 +150,7 @@ def geotiff(self, bbox, zoom=10, raw=False): str """ - return self.get_export(bbox, zoom, 'tiff', raw).content + return self.get_thumbnail(bbox, zoom, 'tiff', raw).content def png(self, bbox, zoom=10, raw=False): """Download this project as a png @@ -161,7 +165,7 @@ def png(self, bbox, zoom=10, raw=False): str """ - return self.get_export(bbox, zoom, 'png', raw).content + return self.get_thumbnail(bbox, zoom, 'png', raw).content def tms(self): """Return a TMS URL for a project""" diff --git a/rasterfoundry/spec.yml b/rasterfoundry/spec.yml index 6d2226c..b668950 100644 --- a/rasterfoundry/spec.yml +++ b/rasterfoundry/spec.yml @@ -1887,11 +1887,17 @@ paths: in: path type: string description: The filename of the file the user wishes to download. Filenames of an export need to first be fetched. + - name: token + required: true + in: query + type: string + description: API token responses: 200: description: The content of the reqested file schema: - type: file + type: string + format: binary 404: description: | UUID does not reference an export that exists, the user has access to, and has finished successfully. The filename may not exist @@ -2371,14 +2377,11 @@ parameters: type: string required: false enum: - - CREATED + - NOTEXPORTED + - TOBEEXPORTED - EXPORTING - EXPORTED - - QUEUED - - PROCESSING - - COMPLETE - FAILED - - ABORTED source: name: source description: Feed source URI @@ -3642,23 +3645,15 @@ definitions: projectId: type: string format: uuid - sceneIds: - type: array - items: - type: string - format: uuid exportStatus: type: string description: Status of export enum: - - CREATED + - NOTEXPORTED + - TOBEEXPORTED - EXPORTING - EXPORTED - - QUEUED - - PROCESSING - - COMPLETE - FAILED - - ABORTED exportType: type: string description: Source of exports @@ -3684,20 +3679,6 @@ definitions: - $ref: '#/definitions/BaseModel' - $ref: '#/definitions/UserTrackingMixin' - $ref: '#/definitions/ExportCreate' - - type: object - properties: - exportStatus: - type: string - description: Status of export - enum: - - CREATED - - EXPORTING - - EXPORTED - - QUEUED - - PROCESSING - - COMPLETE - - FAILED - - ABORTED ExportPaginated: allOf: - $ref: '#/definitions/PaginatedResponse' @@ -3725,9 +3706,6 @@ definitions: crs: type: integer description: epsg projection code - mask: - type: object - description: GeoJSON multipolygon stitch: type: boolean description: stitch tiles into a single geotiff if possible diff --git a/setup.py b/setup.py index 20d8c0b..59f5a65 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,8 @@ 'requests >= 2.9.1', 'bravado >= 8.4.0', 'boto3 >= 1.4.4', - 'future >= 0.16.0' + 'future >= 0.16.0', + 'shapely >= 1.6.4post1' ], extras_require={ 'notebook': [ From c8a43c9c96e47a36742b6b9308a11978031ac67e Mon Sep 17 00:00:00 2001 From: James Santucci Date: Mon, 14 May 2018 10:14:40 -0400 Subject: [PATCH 11/11] 0.6.0 Update changelog, setup.py, and release instructions --- CHANGELOG.rst | 153 +++++++++++++++++++++++++++++++++++++++----------- README.rst | 9 ++- setup.py | 2 +- 3 files changed, 129 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c8d8da..ad0287f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,45 +1,132 @@ -0.5.0 ------- +Change Log +========== -- Add RV integration for creating project files and running predictions on prjects -- Support analysis preview and raw export -- Add option to allow for raw (non-color-corrected) exports -- Updates to posting projects +`0.6.0 `__ (2018-05-14) +-------------------------------------------------------------------------------------------------- -0.4.1 ------ +`Full +Changelog `__ -- Minor usability fixes (#26) -- Fix ipyleaflet install (#25) +**Merged pull requests:** -0.4.0 ------ +- Support async exports + `#50 `__ +- Point to user-configable swagger spec location + `#49 `__ +- Updates for Raster Vision integration + `#44 `__ -- Add *args and **kwargs to no_op function in notebook support check (#24) -- Include Uploads example notebook (#23) +`0.5.0 `__ (2018-01-26) +-------------------------------------------------------------------------------------------------- -0.3.0 ------ -- Install cryptography 2.0+ (#22) +`Full +Changelog `__ -0.2.1 ------ +**Merged pull requests:** -- Correct repo references (#21) +- Add method to post annotations for project + `#42 `__ +- Support analysis preview and raw export + `#40 `__ +- Add option to allow for raw (non-color-corrected) exports + `#38 `__ +- Add method to evaluate a raster-vision model + `#36 `__ +- Add options to start\_prep\_train\_data + `#34 `__ +- Add Planet Upload Support + `#33 `__ +- Add method to start prep\_train\_data job + `#32 `__ +- Start raster-vision predict job + `#30 `__ +- Include missing install command in README + `#28 `__ -0.2.0 ------ +`0.4.1 `__ (2017-09-27) +-------------------------------------------------------------------------------------------------- -- Make imports in api.py relative (#20) -- Improve scene search and spec cleanup (#18) -- Support local file uploads (#15) -- Add scenes post body parameters to spec (#13) -- Include token with api request (#12) -- Fix install for Mac OS (#9) -- Add convenience methods for exports and visualization to projects (#4) -- Add leaflet support to project model (#2) +`Full +Changelog `__ -0.1.0 ------ +**Merged pull requests:** -- Initial release. +- Minor usability fixes + `#26 `__ +- Fix ipyleaflet install + `#25 `__ + +`0.4.0 `__ (2017-09-07) +-------------------------------------------------------------------------------------------------- + +`Full +Changelog `__ + +**Merged pull requests:** + +- Add \*args and \*\*kwargs to no\_op function in notebook support + check + `#24 `__ +- Include Uploads example notebook + `#23 `__ + +`0.3.0 `__ (2017-08-15) +-------------------------------------------------------------------------------------------------- + +`Full +Changelog `__ + +**Merged pull requests:** + +- Bump cryptography version to 2.x + `#22 `__ + +`0.2.1 `__ (2017-08-10) +-------------------------------------------------------------------------------------------------- + +`Full +Changelog `__ + +**Merged pull requests:** + +- Correct repo references + `#21 `__ + +`0.2.0 `__ (2017-08-10) +-------------------------------------------------------------------------------------------------- + +`Full +Changelog `__ + +**Merged pull requests:** + +- Make imports in api.py relative + `#20 `__ +- Add scene search, make various fixes + `#18 `__ +- Support local file uploads from client + `#15 `__ +- Add scenes post body parameters + `#13 `__ +- Include token with api request + `#12 `__ +- Fix install for Mac OS + `#9 `__ +- Add convenience methods to projects + `#4 `__ +- Add leaflet support to project model + `#2 `__ + +`0.1.0 `__ (2017-05-04) +-------------------------------------------------------------------------------------------------- + +`Full +Changelog `__ + +**Merged pull requests:** + +- Port Python client from main repository + `#1 `__ + +- *This Change Log was automatically generated by + `github\_changelog\_generator `__* diff --git a/README.rst b/README.rst index 912bb85..7f342b5 100644 --- a/README.rst +++ b/README.rst @@ -81,7 +81,14 @@ Releases are automatically published to PyPI through Travis CI when commits are .. code:: bash $ git flow release start X.Y.Z - $ vim CHANGELOG.rst + $ docker run -ti --rm -v "${PWD}:/changelog" -w "/changelog" "quay.io/azavea/github-changelog-generator:v1.14.3" \ + --token "${GITHUB_TOKEN}" \ + --future-release "X.Y.Z" \ + --no-issues \ + --no-author \ + --release-branch develop + $ pandoc CHANGELOG.md -o CHANGELOG.rst + $ rm CHANGELOG.md $ vim setup.py $ git commit -m "X.Y.Z" $ git flow release publish X.Y.Z diff --git a/setup.py b/setup.py index 59f5a65..2c8e2cd 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setuptools.setup( name="rasterfoundry", - version="0.5.0", + version="0.6.0", description='A Python client for Raster Foundry, a web platform for ' 'combining, analyzing, and publishing raster data.', long_description=open('README.rst').read(),