Workflows #111

sdc50 · 2018-09-14T21:42:08Z

Testing notebooks

stickler-ci · 2018-09-14T21:42:34Z

quest/__init__.py

-
-
-logging.getLogger('ulmo').propagate = False
+from . import tools # NOQA


E261 at least two spaces before inline comment

stickler-ci · 2018-09-14T21:42:34Z

quest/api/__init__.py

@@ -157,3 +159,8 @@
    cancel_tasks,
    remove_tasks,
 )
+
+from .workflows import (


E402 module level import not at top of file

stickler-ci · 2018-09-14T21:42:34Z

quest/api/workflows.py

+        datasets = list(matched.values())
+
+
+    # stage the data for download, optional parameters can be provided here


E303 too many blank lines (2)

stickler-ci · 2018-09-14T21:42:34Z

quest/api/workflows.py

+    db = get_db()
+    with db_session:
+        datasets = db.Dataset.select(lambda d: d.feature.name in collection_features
+                                     and d.status == DatasetStatus.DOWNLOADED,


W503 line break before binary operator

stickler-ci · 2018-09-14T21:42:35Z

quest/api/workflows.py

+    with db_session:
+        datasets = db.Dataset.select(lambda d: d.feature.name in collection_features
+                                     and d.status == DatasetStatus.DOWNLOADED,
+                                     )# and d.options == download_options)


E261 at least two spaces before inline comment

stickler-ci · 2018-09-14T21:42:35Z

quest/util/misc.py

@@ -308,6 +327,11 @@ def parse_service_uri(uri):

    return provider, service, feature

+def setattr_on_dataframe(df, attr, value, warnings_filter='ignore'):


E302 expected 2 blank lines, found 1

stickler-ci · 2018-09-14T21:42:35Z

quest/util/param_util.py

+        service_param.default = default or services[0]
+        super().__init__(name='')
+
+class PublisherSelector(param.Parameterized):


E302 expected 2 blank lines, found 1

stickler-ci · 2018-09-14T21:42:35Z

quest/util/__init__.py

@@ -4,5 +4,5 @@
 from .config import get_settings, save_settings, update_settings, update_settings_from_file
 from .log import logger, log_to_console, log_to_file
 from . import param_util as param
-from .param_util import format_json_options, NamedString, Service
+from .param_util import format_json_options, NamedString, ServiceSelector, PublisherSelector


F401 '.param_util.ServiceSelector' imported but unused
F401 '.param_util.format_json_options' imported but unused
F401 '.param_util.NamedString' imported but unused
F401 '.param_util.PublisherSelector' imported but unused

stickler-ci · 2018-09-14T21:42:36Z

quest_io_plugins/xyHdf5.py

@@ -1,10 +1,13 @@
 import json
 import os
+import warnings


F401 'warnings' imported but unused

stickler-ci · 2018-09-14T21:42:36Z

quest_tool_plugins/timeseries/timeseries.py

@@ -7,7 +7,7 @@
 import param

 from .ts_base import TsBase
-from quest.util import unit_registry, unit_list
+from quest.util import unit_registry, unit_list, setattr_on_dataframe


F401 'quest.util.unit_registry' imported but unused
F401 'quest.util.unit_list' imported but unused

stickler-ci · 2018-09-17T15:51:12Z

quest/util/misc.py

+        setattr(df, attr, value)
+
+
+def setattr_on_dataframe(df, attr, value, warnings_filter='ignore'):


F811 redefinition of unused 'setattr_on_dataframe' from line 308

coveralls · 2018-09-17T22:12:00Z

Pull Request Test Coverage Report for Build 418

0 of 0 changed or added relevant lines in 0 files are covered.
118 unchanged lines in 4 files lost coverage.
Overall coverage decreased (-0.2%) to 69.585%

Files with Coverage Reduction	New Missed Lines	%
/Users/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/quest/util/config.py	3	89.55%
/Users/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/quest/util/param_util.py	21	69.05%
/Users/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/quest/util/misc.py	34	76.51%
/Users/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/quest/api/datasets.py	60	54.85%

Totals
Change from base Build 387:	-0.2%
Covered Lines:	1526
Relevant Lines:	2193

💛 - Coveralls

AaronV77 · 2018-09-18T19:37:06Z

@sdc50 why is the WMTS notebook being added to this PR?

stickler-ci · 2018-09-18T21:16:36Z

test/test_util_misc.py

+
+    expected = [180]
+    actual = quest.util.listify(expected[0])
+    assert expected == actual


W292 no newline at end of file

sdc50 · 2018-09-19T13:25:04Z

@AaronV77 I've made changes to the notebooks to leverage the new workflows interface. I've also made running the notebooks part of the tests. I'm just trying to get everything working on CI so it's ready to merge after PR #110 get merged in.

AaronV77 · 2018-09-19T13:47:39Z

@sdc50 alright that makes sense. Can you make sure that the notebook(s) output are cleared out. Thanks

stickler-ci · 2018-09-21T18:31:50Z

quest/plugins/base/service_base.py

+            del catalog_entries['bbox']
+
+        if {'latitude', 'longitude'}.issubset(catalog_entries.columns):
+            fn = lambda row: Point((


E731 do not assign a lambda expression, use a def

stickler-ci · 2018-09-21T18:31:50Z

quest/plugins/base/service_base.py

+        extra_fields = list(set(catalog_entries.columns.tolist()) - set(reserved_catalog_entry_fields))
+        # change NaN to None so it can be JSON serialized properly
+        catalog_entries['metadata'] = [{k: None if v != v else v for k, v in record.items()}
+                                for record in catalog_entries[extra_fields].to_dict(orient='records')]


E128 continuation line under-indented for visual indent

stickler-ci · 2018-09-21T18:31:50Z

quest/plugins/base/service_base.py

+        params = self.get_parameters(catalog_ids=catalog_entries)
+        if isinstance(params, pd.DataFrame):
+            groups = params.groupby('service_id').groups
+            catalog_entries['parameters'] = catalog_entries.index.map(lambda x: ','.join(filter(None, params.loc[groups[x]]['parameter'].tolist())) if x in groups.keys() else '')


E501 line too long (178 > 120 characters)

stickler-ci · 2018-09-21T18:31:50Z

quest_tool_plugins/raster/rst_reprojection.py


-        with rasterio.open(dst) as f:
+        with rasterio.open(file_path) as f:


E303 too many blank lines (2)

stickler-ci · 2018-09-21T18:31:51Z

quest/plugins/base/service_base.py

 import param
+import ulmo
+from shapely.geometry import box, Point, shape


F401 'shapely.geometry.shape' imported but unused

stickler-ci · 2018-09-21T18:31:51Z

quest/plugins/base/service_base.py

+
+        if {'latitude', 'longitude'}.issubset(catalog_entries.columns):
+            fn = lambda row: Point((
+                                    float(row['longitude']),


E126 continuation line over-indented for hanging indent

stickler-ci · 2018-09-21T18:31:51Z

quest/plugins/base/service_base.py

+            fn = lambda row: Point((
+                                    float(row['longitude']),
+                                    float(row['latitude'])
+                                    ))


E123 closing bracket does not match indentation of opening bracket's line

sdc50 · 2018-09-21T19:49:13Z

@AaronV77 This is ready for review. I just pulled the usgs_ned:19-arc-second tests out since the service isn't being consistent right now.

* Testing notebooks

stickler-ci reviewed Sep 14, 2018

View reviewed changes

sdc50 force-pushed the workflows branch from d025742 to 6647b30 Compare September 17, 2018 13:48

stickler-ci reviewed Sep 17, 2018

View reviewed changes

sdc50 force-pushed the workflows branch from a7aed91 to fce04bb Compare September 18, 2018 13:33

stickler-ci reviewed Sep 18, 2018

View reviewed changes

sdc50 force-pushed the workflows branch 2 times, most recently from bd0e75e to b0acfd6 Compare September 21, 2018 18:31

stickler-ci reviewed Sep 21, 2018

View reviewed changes

sdc50 assigned AaronV77 Sep 21, 2018

sdc50 force-pushed the workflows branch from 7fd9259 to f08c5af Compare September 24, 2018 16:26

AaronV77 approved these changes Sep 24, 2018

View reviewed changes

* Add Workflows to API

9057721

* Testing notebooks

sdc50 force-pushed the workflows branch from da62212 to 9057721 Compare September 24, 2018 18:43

sdc50 merged commit 589bca8 into erdc:master Sep 24, 2018

This was referenced Sep 24, 2018

Pony 0.7.3 Versioning #91

Closed

Rename Filters and Features #89

Closed

No tool plugin tests #102

Open

Clarify the use of "services" in the codebase #59

Closed

sdc50 deleted the workflows branch September 28, 2018 15:20

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Workflows #111

Workflows #111

sdc50 commented Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 14, 2018

stickler-ci Sep 17, 2018

coveralls commented Sep 17, 2018 •

edited

Loading

AaronV77 commented Sep 18, 2018

stickler-ci Sep 18, 2018

sdc50 commented Sep 19, 2018

AaronV77 commented Sep 19, 2018

stickler-ci Sep 21, 2018

stickler-ci Sep 21, 2018

stickler-ci Sep 21, 2018

stickler-ci Sep 21, 2018

stickler-ci Sep 21, 2018

stickler-ci Sep 21, 2018

stickler-ci Sep 21, 2018

sdc50 commented Sep 21, 2018



		logging.getLogger('ulmo').propagate = False
		from . import tools # NOQA

		datasets = list(matched.values())


		# stage the data for download, optional parameters can be provided here

		@@ -308,6 +327,11 @@ def parse_service_uri(uri):

		return provider, service, feature

		def setattr_on_dataframe(df, attr, value, warnings_filter='ignore'):

		setattr(df, attr, value)


		def setattr_on_dataframe(df, attr, value, warnings_filter='ignore'):


		with rasterio.open(dst) as f:
		with rasterio.open(file_path) as f:

Workflows #111

Workflows #111

Conversation

sdc50 commented Sep 14, 2018

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

coveralls commented Sep 17, 2018 • edited Loading

Pull Request Test Coverage Report for Build 418

💛 - Coveralls

AaronV77 commented Sep 18, 2018

Choose a reason for hiding this comment

sdc50 commented Sep 19, 2018

AaronV77 commented Sep 19, 2018

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

sdc50 commented Sep 21, 2018

coveralls commented Sep 17, 2018 •

edited

Loading