diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000000..f3bdb16b20 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,62 @@ +name: lint + +on: + pull_request: + paths: + - '**' + + push: + paths: + - '**' + - '!.github/**' + - '.github/workflows/lint.yaml' + - '!docker/**' + - '!docs/**' + - '!contrib/**' + +jobs: + flake8: + name: flake8 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - run: python -m pip install flake8 pep8-naming + # ignore and ignore-names list as per setup.cfg + - name: flake8 + uses: liskin/gh-problem-matcher-wrap@v1.0.1 + with: + linters: flake8 + run: | + flake8 . --exclude Dockerfile --ignore=E711,E226,W503,W504,E124,F841,W605 --ignore-names=W,H,A,S,R,T,WS,X,Y,Z,XX,YY,XY,B,M,N,L,NX,NY + + pylint: + runs-on: ubuntu-latest + name: Pylint + steps: + - name: checkout git + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install dependencies and run pylint + run: | + pip install --upgrade -e '.[test]' + pip install pylint + pylint -j 2 --reports no datacube + + + pycodestyle: + name: pycodestyle + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Run pycodestyle + run: | + pip install --upgrade -e '.[test]' + pip install pycodestyle + pycodestyle tests integration_tests examples --max-line-length 120 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ded4dc39b8..872091cdac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,22 +81,6 @@ jobs: fi EOF - - name: Check Code Style - run: | - docker run --rm \ - -v $(pwd):/code \ - -e SKIP_DB=yes \ - ${{ matrix.docker_image }} \ - pycodestyle tests integration_tests examples --max-line-length 120 - - - name: Lint Code - run: | - docker run --rm \ - -v $(pwd):/code \ - -e SKIP_DB=yes \ - ${{ matrix.docker_image }} \ - pylint -j 2 --reports no datacube - - name: Run Tests run: | cat <`_ for usage of the :class:`xarray.Dataset` and :class:`xarray.DataArray` objects. @@ -279,7 +280,8 @@ def load(self, product=None, measurements=None, output_crs=None, resolution=None :param list(str) measurements: Measurements name or list of names to be included, as listed in :meth:`list_measurements`. - These will be loaded as individual ``xr.DataArray`` variables in the output ``xarray.Dataset`` object. + These will be loaded as individual ``xr.DataArray`` variables in + the output ``xarray.Dataset`` object. If a list is specified, the measurements will be returned in the order requested. By default all available measurements are included. @@ -289,8 +291,8 @@ def load(self, product=None, measurements=None, output_crs=None, resolution=None For example: ``'x', 'y', 'time', 'crs'``. :param str output_crs: - The CRS of the returned data, for example ``EPSG:3577``. If no CRS is supplied, the CRS of the stored data is used - if available. + The CRS of the returned data, for example ``EPSG:3577``. + If no CRS is supplied, the CRS of the stored data is used if available. This differs from the ``crs`` parameter desribed above, which is used to define the CRS of the coordinates in the query itself. @@ -358,7 +360,8 @@ def load(self, product=None, measurements=None, output_crs=None, resolution=None :param function dataset_predicate: Optional. A function that can be passed to restrict loaded datasets. A predicate function should - take a :class:`datacube.model.Dataset` object (e.g. as returned from :meth:`find_datasets`) and return a boolean. + take a :class:`datacube.model.Dataset` object (e.g. as returned from :meth:`find_datasets`) and + return a boolean. For example, loaded data could be filtered to January observations only by passing the following predicate function that returns True for datasets acquired in January:: diff --git a/datacube/api/grid_workflow.py b/datacube/api/grid_workflow.py index 0195ff51d5..5e16ce921d 100644 --- a/datacube/api/grid_workflow.py +++ b/datacube/api/grid_workflow.py @@ -3,9 +3,7 @@ # Copyright (c) 2015-2020 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging -import numpy import xarray -from itertools import groupby from collections import OrderedDict import pandas as pd diff --git a/datacube/api/query.py b/datacube/api/query.py index b164519958..ea843dc7ab 100644 --- a/datacube/api/query.py +++ b/datacube/api/query.py @@ -46,7 +46,7 @@ def __init__(self, group_by_func, dimension, units, sort_key=None, group_key=Non self.sort_key = sort_key if group_key is None: - group_key = lambda datasets: group_by_func(datasets[0]) + group_key = lambda datasets: group_by_func(datasets[0]) # noqa: E731 self.group_key = group_key diff --git a/datacube/config.py b/datacube/config.py index f7960d6c44..34933d2d3d 100755 --- a/datacube/config.py +++ b/datacube/config.py @@ -10,7 +10,7 @@ from pathlib import Path import configparser from urllib.parse import unquote_plus, urlparse, parse_qsl -from typing import Any, Dict, Iterable, MutableMapping, Optional, Tuple, Union, cast +from typing import Any, Dict, Iterable, MutableMapping, Optional, Tuple, Union PathLike = Union[str, 'os.PathLike[Any]'] diff --git a/datacube/drivers/netcdf/_safestrings.py b/datacube/drivers/netcdf/_safestrings.py index 2f9dd47d31..7ac7630a3c 100644 --- a/datacube/drivers/netcdf/_safestrings.py +++ b/datacube/drivers/netcdf/_safestrings.py @@ -75,7 +75,7 @@ def __getitem__(self, name): return _VariableProxy(var) #: pylint: disable=invalid-name - def createVariable(self, *args, **kwargs): + def createVariable(self, *args, **kwargs): # noqa: N802 new_var = super(_NC4DatasetProxy, self).createVariable(*args, **kwargs) return _VariableProxy(new_var) diff --git a/datacube/drivers/netcdf/writer.py b/datacube/drivers/netcdf/writer.py index 2c5d5c3c2d..11bbbc6bcd 100644 --- a/datacube/drivers/netcdf/writer.py +++ b/datacube/drivers/netcdf/writer.py @@ -192,7 +192,7 @@ def _write_lcc2_params(crs_var, crs): # e.g. http://spatialreference.org/ref/sr-org/mexico-inegi-lambert-conformal-conic/ crs_var.grid_mapping_name = cf['grid_mapping_name'] - crs_var.standard_parallel = cf['standard_parallel'] + crs_var.standard_parallel = cf['standard_parallel'] crs_var.latitude_of_projection_origin = cf['latitude_of_projection_origin'] crs_var.longitude_of_central_meridian = cf['longitude_of_central_meridian'] crs_var.false_easting = cf['false_easting'] diff --git a/datacube/drivers/postgis/_connections.py b/datacube/drivers/postgis/_connections.py index 56ea4c974d..3112e3b151 100755 --- a/datacube/drivers/postgis/_connections.py +++ b/datacube/drivers/postgis/_connections.py @@ -21,7 +21,7 @@ from sqlalchemy import event, create_engine, text from sqlalchemy.engine import Engine -from sqlalchemy.engine.url import URL as EngineUrl +from sqlalchemy.engine.url import URL as EngineUrl # noqa: N811 import datacube from datacube.index.exceptions import IndexSetupError diff --git a/datacube/drivers/postgis/_core.py b/datacube/drivers/postgis/_core.py index 9071b92e36..bfda5a7987 100644 --- a/datacube/drivers/postgis/_core.py +++ b/datacube/drivers/postgis/_core.py @@ -9,12 +9,12 @@ import logging from datacube.drivers.postgis.sql import (INSTALL_TRIGGER_SQL_TEMPLATE, - SCHEMA_NAME, TYPES_INIT_SQL, - UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE, - ADDED_COLUMN_MIGRATE_SQL_TEMPLATE, - UPDATE_TIMESTAMP_SQL, - escape_pg_identifier, - pg_column_exists, pg_exists) + SCHEMA_NAME, TYPES_INIT_SQL, + UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE, + ADDED_COLUMN_MIGRATE_SQL_TEMPLATE, + UPDATE_TIMESTAMP_SQL, + escape_pg_identifier, + pg_column_exists) from sqlalchemy import MetaData from sqlalchemy.engine import Engine from sqlalchemy.schema import CreateSchema @@ -40,7 +40,7 @@ def install_timestamp_trigger(connection): from . import _schema - TABLE_NAMES = [ + TABLE_NAMES = [ # noqa: N806 _schema.METADATA_TYPE.name, _schema.PRODUCT.name, _schema.DATASET.name, @@ -53,9 +53,10 @@ def install_timestamp_trigger(connection): connection.execute(UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name)) connection.execute(INSTALL_TRIGGER_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name)) + def install_added_column(connection): from . import _schema - TABLE_NAME = _schema.DATASET_LOCATION.name + TABLE_NAME = _schema.DATASET_LOCATION.name # noqa: N806 connection.execute(ADDED_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=TABLE_NAME)) @@ -113,7 +114,7 @@ def ensure_db(engine, with_permissions=True): _LOG.info("Creating added column.") install_added_column(c) c.execute('commit') - except: + except: # noqa: E722 c.execute('rollback') raise finally: diff --git a/datacube/drivers/postgis/_schema.py b/datacube/drivers/postgis/_schema.py index 187ce99873..173ddae7f6 100644 --- a/datacube/drivers/postgis/_schema.py +++ b/datacube/drivers/postgis/_schema.py @@ -72,7 +72,7 @@ # Typing note: sqlalchemy-stubs doesn't handle this legitimate calling pattern. Column('metadata_type_ref', None, ForeignKey(METADATA_TYPE.c.id), nullable=False), # type: ignore[call-overload] # Typing note: sqlalchemy-stubs doesn't handle this legitimate calling pattern. - Column('dataset_type_ref', None, ForeignKey(PRODUCT.c.id), index=True, nullable=False), # type: ignore[call-overload] + Column('dataset_type_ref', None, ForeignKey(PRODUCT.c.id), index=True, nullable=False), # type: ignore[call-overload] # noqa: E501 Column('metadata', postgres.JSONB, index=False, nullable=False), diff --git a/datacube/drivers/postgis/sql.py b/datacube/drivers/postgis/sql.py index 0fd72f1d9b..c616304826 100644 --- a/datacube/drivers/postgis/sql.py +++ b/datacube/drivers/postgis/sql.py @@ -18,6 +18,7 @@ class CreateView(Executable, ClauseElement): inherit_cache = True + def __init__(self, name, select): self.name = name self.select = select diff --git a/datacube/drivers/postgres/_connections.py b/datacube/drivers/postgres/_connections.py index 3e0373b6c1..56db60bfe7 100755 --- a/datacube/drivers/postgres/_connections.py +++ b/datacube/drivers/postgres/_connections.py @@ -21,7 +21,7 @@ from sqlalchemy import event, create_engine, text from sqlalchemy.engine import Engine -from sqlalchemy.engine.url import URL as EngineUrl +from sqlalchemy.engine.url import URL as EngineUrl # noqa: N811 import datacube from datacube.index.exceptions import IndexSetupError diff --git a/datacube/drivers/postgres/_core.py b/datacube/drivers/postgres/_core.py index 5d26dcf91c..1638f7faa2 100644 --- a/datacube/drivers/postgres/_core.py +++ b/datacube/drivers/postgres/_core.py @@ -14,7 +14,7 @@ ADDED_COLUMN_MIGRATE_SQL_TEMPLATE, UPDATE_TIMESTAMP_SQL, escape_pg_identifier, - pg_column_exists, pg_exists) + pg_column_exists) from sqlalchemy import MetaData from sqlalchemy.engine import Engine from sqlalchemy.schema import CreateSchema @@ -40,7 +40,7 @@ def install_timestamp_trigger(connection): from . import _schema - TABLE_NAMES = [ + TABLE_NAMES = [ # noqa: N806 _schema.METADATA_TYPE.name, _schema.PRODUCT.name, _schema.DATASET.name, @@ -53,9 +53,10 @@ def install_timestamp_trigger(connection): connection.execute(UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name)) connection.execute(INSTALL_TRIGGER_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name)) + def install_added_column(connection): from . import _schema - TABLE_NAME = _schema.DATASET_LOCATION.name + TABLE_NAME = _schema.DATASET_LOCATION.name # noqa: N806 connection.execute(ADDED_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=TABLE_NAME)) @@ -113,7 +114,7 @@ def ensure_db(engine, with_permissions=True): _LOG.info("Creating added column.") install_added_column(c) c.execute('commit') - except: + except: # noqa: E722 c.execute('rollback') raise finally: diff --git a/datacube/drivers/postgres/_schema.py b/datacube/drivers/postgres/_schema.py index 187ce99873..708ff9dc1c 100644 --- a/datacube/drivers/postgres/_schema.py +++ b/datacube/drivers/postgres/_schema.py @@ -72,7 +72,7 @@ # Typing note: sqlalchemy-stubs doesn't handle this legitimate calling pattern. Column('metadata_type_ref', None, ForeignKey(METADATA_TYPE.c.id), nullable=False), # type: ignore[call-overload] # Typing note: sqlalchemy-stubs doesn't handle this legitimate calling pattern. - Column('dataset_type_ref', None, ForeignKey(PRODUCT.c.id), index=True, nullable=False), # type: ignore[call-overload] + Column('dataset_type_ref', None, ForeignKey(PRODUCT.c.id), index=True, nullable=False), # type: ignore[call-overload] # noqa: E501 Column('metadata', postgres.JSONB, index=False, nullable=False), diff --git a/datacube/drivers/postgres/sql.py b/datacube/drivers/postgres/sql.py index e715929c4d..cac004a820 100644 --- a/datacube/drivers/postgres/sql.py +++ b/datacube/drivers/postgres/sql.py @@ -18,6 +18,7 @@ class CreateView(Executable, ClauseElement): inherit_cache = True + def __init__(self, name, select): self.name = name self.select = select diff --git a/datacube/helpers.py b/datacube/helpers.py index 12e0572eb1..cf9c3da5b1 100644 --- a/datacube/helpers.py +++ b/datacube/helpers.py @@ -9,8 +9,6 @@ """ import numpy as np -import rasterio # type: ignore[import] -import warnings DEFAULT_PROFILE = { 'blockxsize': 256, diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 5bf6e2b283..e996cc560c 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -238,9 +238,11 @@ def get_all(self) -> Iterable[MetadataType]: :returns: All available MetadataType models """ + QueryField = Union[str, float, int, Range, datetime.datetime] QueryDict = Mapping[str, QueryField] + class AbstractProductResource(ABC): """ Abstract base class for the Product portion of an index api. @@ -909,23 +911,28 @@ class AbstractIndex(ABC): @property @abstractmethod - def url(self) -> str: pass + def url(self) -> str: + pass @property @abstractmethod - def users(self) -> AbstractUserResource: pass + def users(self) -> AbstractUserResource: + pass @property @abstractmethod - def metadata_types(self) -> AbstractMetadataTypeResource: pass + def metadata_types(self) -> AbstractMetadataTypeResource: + pass @property @abstractmethod - def products(self) -> AbstractProductResource: pass + def products(self) -> AbstractProductResource: + pass @property @abstractmethod - def datasets(self) -> AbstractDatasetResource: pass + def datasets(self) -> AbstractDatasetResource: + pass @classmethod @abstractmethod @@ -946,10 +953,12 @@ def get_dataset_fields(cls, @abstractmethod def init_db(self, with_default_types: bool = True, - with_permissions: bool = True) -> bool: pass + with_permissions: bool = True) -> bool: + pass @abstractmethod - def close(self) -> None: pass + def close(self) -> None: + pass def __enter__(self): return self @@ -973,8 +982,7 @@ def connect_to_index(config: LocalConfig, @staticmethod @abstractmethod - def metadata_type_from_doc( - definition: dict + def metadata_type_from_doc(definition: dict ) -> MetadataType: pass @@ -1002,4 +1010,4 @@ def transform(self): @property def bounds(self): - return Dataset.bounds.__get__(self) \ No newline at end of file + return Dataset.bounds.__get__(self) diff --git a/datacube/index/memory/_datasets.py b/datacube/index/memory/_datasets.py index 09f73a2a36..bcc7ec2f25 100755 --- a/datacube/index/memory/_datasets.py +++ b/datacube/index/memory/_datasets.py @@ -329,9 +329,9 @@ def get_datasets_for_location(self, uri: str, mode: Optional[str] = None) -> Ite raise ValueError(f"Unsupported query mode: {mode}") ids: Set[DSID] = set() if mode == "exact": - test: Callable[[str], bool] = lambda l: l == uri + test: Callable[[str], bool] = lambda l: l == uri # noqa: E741 else: - test = lambda l: l.startswith(uri) + test = lambda l: l.startswith(uri) # noqa: E731 for id_, locs in self.locations.items(): for loc in locs: if test(loc): @@ -400,9 +400,10 @@ def _search( if not product_queries: raise ValueError(f"No products match source filter: {source_filter}") if len(product_queries) > 1: - raise RuntimeError(f"Multiproduct source_filters are not supported. Try adding 'product' field.") + raise RuntimeError("Multiproduct source_filters are not supported. Try adding 'product' field.") source_queries, source_product = product_queries[0] - source_exprs = tuple(fields.to_expressions(source_product.metadata_type.dataset_fields.get, **source_queries)) + source_exprs = tuple(fields.to_expressions(source_product.metadata_type.dataset_fields.get, + **source_queries)) else: source_product = None source_exprs = () @@ -461,10 +462,10 @@ def _search_flat( **query: QueryField ) -> Iterable[Dataset]: return cast(Iterable[Dataset], self._search( - return_format=self.RET_FORMAT_DATASETS, - limit=limit, - source_filter=source_filter, - **query) + return_format=self.RET_FORMAT_DATASETS, + limit=limit, + source_filter=source_filter, + **query) ) def _search_grouped( @@ -474,10 +475,10 @@ def _search_grouped( **query: QueryField ) -> Iterable[Tuple[Iterable[Dataset], Product]]: return cast(Iterable[Tuple[Iterable[Dataset], Product]], self._search( - return_format=self.RET_FORMAT_PRODUCT_GROUPED, - limit=limit, - source_filter=source_filter, - **query) + return_format=self.RET_FORMAT_PRODUCT_GROUPED, + limit=limit, + source_filter=source_filter, + **query) ) def _get_prod_queries(self, **query: QueryField) -> Iterable[Tuple[Mapping[str, QueryField], Product]]: @@ -502,8 +503,8 @@ def search_returning(self, for ds in self.search(limit=limit, **query): # type: ignore[arg-type] ds_fields = get_dataset_fields(ds.type.metadata_type.definition) result_vals = { - fn: ds_fields[fn].extract(ds.metadata_doc) # type: ignore[attr-defined] - for fn in field_names + fn: ds_fields[fn].extract(ds.metadata_doc) # type: ignore[attr-defined] + for fn in field_names } yield result_type(**result_vals) @@ -515,8 +516,8 @@ def count_by_product(self, **query: QueryField) -> Iterable[Tuple[Product, int]] yield (prod, len(list(datasets))) def count_by_product_through_time(self, - period: str, - **query: QueryField + period: str, + **query: QueryField ) -> Iterable[ Tuple[ Product, @@ -542,6 +543,7 @@ def _expand_period( if precision <= 0: raise ValueError('Invalid period string. Must specify a natural number of days, weeks, months or years') unit = match.group("unit") + def next_period(prev: datetime.datetime) -> datetime.datetime: if unit == 'day': return prev + datetime.timedelta(days=precision) @@ -597,7 +599,7 @@ def _product_period_count( ], ] ]: - YieldType = Tuple[Product, Iterable[Tuple[Range, int]]] + YieldType = Tuple[Product, Iterable[Tuple[Range, int]]] # noqa: N806 query = dict(query) try: start, end = cast(Range, query.pop('time')) @@ -664,7 +666,7 @@ def get_product_time_bounds(self, product: str) -> Tuple[datetime.datetime, date min_time = dsmin if max_time is None or dsmax > max_time: max_time = dsmax - return (cast(datetime.datetime, min_time), cast(datetime.datetime,max_time)) + return (cast(datetime.datetime, min_time), cast(datetime.datetime, max_time)) # pylint: disable=redefined-outer-name def search_returning_datasets_light( @@ -678,6 +680,7 @@ def search_returning_datasets_light( custom_fields = build_custom_fields(custom_offsets) else: custom_fields = {} + def make_ds_light(ds: Dataset) -> Tuple: fields = { fname: ds.metadata_type.dataset_fields[fname] diff --git a/datacube/index/memory/_fields.py b/datacube/index/memory/_fields.py index a80b07ed93..1ad6a7cd63 100644 --- a/datacube/index/memory/_fields.py +++ b/datacube/index/memory/_fields.py @@ -2,6 +2,7 @@ from datacube.model.fields import SimpleField, Field, get_dataset_fields as generic_get_dataset_fields from datacube.index.abstract import Offset + # TODO: SimpleFields cannot handle non-metadata fields because e.g. the extract API expects a doc, not a Dataset model def get_native_fields() -> MutableMapping[str, Field]: return { @@ -37,11 +38,13 @@ def get_native_fields() -> MutableMapping[str, Field]: ), } + def get_dataset_fields(metadata_definition: Mapping[str, Any]) -> Mapping[str, Field]: fields = get_native_fields() fields.update(generic_get_dataset_fields(metadata_definition)) return fields + def build_custom_fields(custom_offsets: Mapping[str, Offset]): return { name: SimpleField( diff --git a/datacube/index/memory/_products.py b/datacube/index/memory/_products.py index d565c3a274..9688491482 100644 --- a/datacube/index/memory/_products.py +++ b/datacube/index/memory/_products.py @@ -10,7 +10,7 @@ from datacube.model import DatasetType as Product from datacube.utils import changes, jsonify_document, _readable_offset from datacube.utils.changes import AllowPolicy, Change, Offset, check_doc_unchanged, get_doc_changes, classify_changes -from typing import Iterable, Iterator, Mapping, Tuple, Union, cast +from typing import Iterable, Iterator, Mapping, Tuple, cast _LOG = logging.getLogger(__name__) @@ -36,7 +36,7 @@ def add(self, product: Product, allow_table_lock: bool = False) -> Product: if mdt is None: _LOG.warning(f'Adding metadata_type "{product.metadata_type.name}" as it doesn\'t exist') product.metadata_type = self.metadata_type_resource.add(product.metadata_type, - allow_table_lock=allow_table_lock) + allow_table_lock=allow_table_lock) clone = self.clone(product) clone.id = self.next_id self.next_id += 1 diff --git a/datacube/index/memory/_users.py b/datacube/index/memory/_users.py index 1fb2b0e7a6..6e597853f1 100644 --- a/datacube/index/memory/_users.py +++ b/datacube/index/memory/_users.py @@ -5,6 +5,7 @@ from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource + class User: def __init__(self, username: str, password: str, role: str, description: Optional[str] = None): @@ -29,7 +30,7 @@ def __init__(self) -> None: "agdc_ingest", "agdc_manage", "agdc_admin", - + # For forwards compatibility with future driver(s) "odc_user", "odc_ingest", diff --git a/datacube/index/null/_datasets.py b/datacube/index/null/_datasets.py index eb315dd8c6..93f27bd84d 100755 --- a/datacube/index/null/_datasets.py +++ b/datacube/index/null/_datasets.py @@ -2,10 +2,10 @@ # # Copyright (c) 2015-2020 ODC Contributors # SPDX-License-Identifier: Apache-2.0 -from typing import Iterable, Union, Optional from datacube.index.abstract import AbstractDatasetResource, DSID from datacube.model import Dataset, DatasetType +from typing import Iterable class DatasetResource(AbstractDatasetResource): diff --git a/datacube/index/null/_users.py b/datacube/index/null/_users.py index 139df489ca..cc91c5dae0 100644 --- a/datacube/index/null/_users.py +++ b/datacube/index/null/_users.py @@ -5,6 +5,7 @@ from typing import Iterable, Optional, Tuple from datacube.index.abstract import AbstractUserResource + class UserResource(AbstractUserResource): def __init__(self) -> None: pass diff --git a/datacube/index/postgis/_datasets.py b/datacube/index/postgis/_datasets.py index baeb41a087..f6418f2b09 100755 --- a/datacube/index/postgis/_datasets.py +++ b/datacube/index/postgis/_datasets.py @@ -9,7 +9,7 @@ import logging import warnings from collections import namedtuple -from typing import Iterable, Tuple, Union, List, Optional +from typing import Iterable, Union, List from uuid import UUID from sqlalchemy import select, func diff --git a/datacube/index/postgis/_metadata_types.py b/datacube/index/postgis/_metadata_types.py index 178948f4ed..3f33f98144 100644 --- a/datacube/index/postgis/_metadata_types.py +++ b/datacube/index/postgis/_metadata_types.py @@ -3,7 +3,6 @@ # Copyright (c) 2015-2020 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging -import warnings from cachetools.func import lru_cache @@ -14,6 +13,7 @@ _LOG = logging.getLogger(__name__) + class MetadataTypeResource(AbstractMetadataTypeResource): def __init__(self, db): """ diff --git a/datacube/index/postgis/_users.py b/datacube/index/postgis/_users.py index 59fc414b60..85a2ac9e34 100644 --- a/datacube/index/postgis/_users.py +++ b/datacube/index/postgis/_users.py @@ -6,6 +6,7 @@ from datacube.index.abstract import AbstractUserResource from datacube.drivers.postgis import PostGisDb + class UserResource(AbstractUserResource): def __init__(self, db: PostGisDb) -> None: """ diff --git a/datacube/index/postgis/index.py b/datacube/index/postgis/index.py index 3b55a1a4a3..c35b6834ab 100644 --- a/datacube/index/postgis/index.py +++ b/datacube/index/postgis/index.py @@ -42,7 +42,7 @@ class Index(AbstractIndex): def __init__(self, db: PostGisDb) -> None: # POSTGIS driver is not stable with respect to database schema or internal APIs. _LOG.warning("""WARNING: The POSTGIS index driver implementation is considered EXPERIMENTAL. -WARNING: +WARNING: WARNING: Database schema and internal APIs may change significantly between releases. Use at your own risk.""") self._db = db @@ -74,7 +74,7 @@ def url(self) -> str: @classmethod def from_config(cls, config, application_name=None, validate_connection=True): db = PostGisDb.from_config(config, application_name=application_name, - validate_connection=validate_connection) + validate_connection=validate_connection) return cls(db) @classmethod diff --git a/datacube/index/postgres/_datasets.py b/datacube/index/postgres/_datasets.py index 8e8c412fa4..e01d92085e 100755 --- a/datacube/index/postgres/_datasets.py +++ b/datacube/index/postgres/_datasets.py @@ -9,7 +9,7 @@ import logging import warnings from collections import namedtuple -from typing import Iterable, Tuple, Union, List, Optional +from typing import Iterable, Union, List from uuid import UUID from sqlalchemy import select, func @@ -29,8 +29,6 @@ # It's a public api, so we can't reorganise old methods. # pylint: disable=too-many-public-methods, too-many-lines - - class DatasetResource(AbstractDatasetResource): """ :type _db: datacube.drivers.postgres._connections.PostgresDb @@ -164,7 +162,6 @@ def process_bunch(dss, main_ds, transaction): if main_ds.uris is not None: self._ensure_new_locations(main_ds, transaction=transaction) - _LOG.info('Indexing %s', dataset.id) if with_lineage: diff --git a/datacube/index/postgres/_metadata_types.py b/datacube/index/postgres/_metadata_types.py index b315158a62..941559bd98 100644 --- a/datacube/index/postgres/_metadata_types.py +++ b/datacube/index/postgres/_metadata_types.py @@ -3,7 +3,6 @@ # Copyright (c) 2015-2020 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import logging -import warnings from cachetools.func import lru_cache @@ -14,6 +13,7 @@ _LOG = logging.getLogger(__name__) + class MetadataTypeResource(AbstractMetadataTypeResource): def __init__(self, db): """ diff --git a/datacube/index/postgres/_users.py b/datacube/index/postgres/_users.py index dffce7638f..f9cfd14152 100644 --- a/datacube/index/postgres/_users.py +++ b/datacube/index/postgres/_users.py @@ -6,6 +6,7 @@ from datacube.index.abstract import AbstractUserResource from datacube.drivers.postgres import PostgresDb + class UserResource(AbstractUserResource): def __init__(self, db: PostgresDb) -> None: """ diff --git a/datacube/model/__init__.py b/datacube/model/__init__.py index 0e91017177..c2ee3c4be8 100644 --- a/datacube/model/__init__.py +++ b/datacube/model/__init__.py @@ -8,7 +8,6 @@ import logging import math -import warnings from collections import OrderedDict from datetime import datetime from pathlib import Path @@ -21,7 +20,7 @@ from datacube.utils import geometry, without_lineage_sources, parse_time, cached_property, uri_to_local_path, \ schema_validated, DocReader from .fields import Field, get_dataset_fields -from ._base import Range, ranges_overlap +from ._base import Range, ranges_overlap # noqa: F401 _LOG = logging.getLogger(__name__) diff --git a/datacube/model/utils.py b/datacube/model/utils.py index 6a0b82c34b..7540942755 100644 --- a/datacube/model/utils.py +++ b/datacube/model/utils.py @@ -25,7 +25,7 @@ from yaml import SafeDumper # type: ignore -class BadMatch(Exception): +class BadMatch(Exception): # noqa: N818 pass diff --git a/datacube/scripts/cli_app.py b/datacube/scripts/cli_app.py index 3729e7e902..820d8179eb 100644 --- a/datacube/scripts/cli_app.py +++ b/datacube/scripts/cli_app.py @@ -10,12 +10,12 @@ from datacube.ui.click import cli -import datacube.scripts.dataset -import datacube.scripts.ingest -import datacube.scripts.product -import datacube.scripts.metadata -import datacube.scripts.system -import datacube.scripts.user +import datacube.scripts.dataset # noqa: F401 +import datacube.scripts.ingest # noqa: F401 +import datacube.scripts.product # noqa: F401 +import datacube.scripts.metadata # noqa: F401 +import datacube.scripts.system # noqa: F401 +import datacube.scripts.user # noqa: F401 if __name__ == '__main__': diff --git a/datacube/scripts/dataset.py b/datacube/scripts/dataset.py index 62cc120dde..1b5d964d09 100644 --- a/datacube/scripts/dataset.py +++ b/datacube/scripts/dataset.py @@ -235,7 +235,7 @@ def parse_update_rules(keys_that_can_change): default='keep', help=dedent(''' What to do with previously recorded dataset location(s) - + \b - 'keep': keep as alternative location [default] - 'archive': mark as archived @@ -487,7 +487,8 @@ def uri_search_cmd(index: Index, paths: List[str], search_mode): @click.option('--archive-derived', '-d', help='Also recursively archive derived datasets', is_flag=True, default=False) @click.option('--dry-run', help="Don't archive. Display datasets that would get archived", is_flag=True, default=False) -@click.option('--all', "all_ds", help="Ignore id list - archive ALL non-archived datasets (warning: may be slow on large databases)", +@click.option('--all', "all_ds", + help="Ignore id list - archive ALL non-archived datasets (warning: may be slow on large databases)", is_flag=True, default=False) @click.argument('ids', nargs=-1) @ui.pass_index() @@ -496,7 +497,8 @@ def archive_cmd(index: Index, archive_derived: bool, dry_run: bool, all_ds: bool if all_ds: datasets_for_archive = {dsid: True for dsid in index.datasets.get_all_dataset_ids(archived=False)} else: - datasets_for_archive = {UUID(dataset_id): exists for dataset_id, exists in zip(ids, index.datasets.bulk_has(ids))} + datasets_for_archive = {UUID(dataset_id): exists + for dataset_id, exists in zip(ids, index.datasets.bulk_has(ids))} if False in datasets_for_archive.values(): for dataset_id, exists in datasets_for_archive.items(): @@ -528,11 +530,13 @@ def archive_cmd(index: Index, archive_derived: bool, dry_run: bool, all_ds: bool help="Only restore derived datasets that were archived " "this recently to the original dataset", default=10 * 60) -@click.option('--all', "all_ds", help="Ignore id list - restore ALL archived datasets (warning: may be slow on large databases)", +@click.option('--all', "all_ds", + help="Ignore id list - restore ALL archived datasets (warning: may be slow on large databases)", is_flag=True, default=False) @click.argument('ids', nargs=-1) @ui.pass_index() -def restore_cmd(index: Index, restore_derived: bool, derived_tolerance_seconds: int, dry_run: bool, all_ds: bool, ids: List[str]): +def restore_cmd(index: Index, restore_derived: bool, derived_tolerance_seconds: int, + dry_run: bool, all_ds: bool, ids: List[str]): tolerance = datetime.timedelta(seconds=derived_tolerance_seconds) if all_ds: ids = index.datasets.get_all_dataset_ids(archived=True) # type: ignore[assignment] @@ -570,7 +574,8 @@ def within_tolerance(dataset): @dataset_cmd.command('purge', help="Purge archived datasets") @click.option('--dry-run', help="Don't archive. Display datasets that would get archived", is_flag=True, default=False) -@click.option('--all', "all_ds", help="Ignore id list - purge ALL archived datasets (warning: may be slow on large databases)", +@click.option('--all', "all_ds", + help="Ignore id list - purge ALL archived datasets (warning: may be slow on large databases)", is_flag=True, default=False) @click.argument('ids', nargs=-1) @ui.pass_index() @@ -578,7 +583,8 @@ def purge_cmd(index: Index, dry_run: bool, all_ds: bool, ids: List[str]): if all_ds: datasets_for_archive = {dsid: True for dsid in index.datasets.get_all_dataset_ids(archived=True)} else: - datasets_for_archive = {UUID(dataset_id): exists for dataset_id, exists in zip(ids, index.datasets.bulk_has(ids))} + datasets_for_archive = {UUID(dataset_id): exists + for dataset_id, exists in zip(ids, index.datasets.bulk_has(ids))} # Check for non-existent datasets if False in datasets_for_archive.values(): diff --git a/datacube/scripts/product.py b/datacube/scripts/product.py index 993a584386..9cd5d0f439 100644 --- a/datacube/scripts/product.py +++ b/datacube/scripts/product.py @@ -42,7 +42,7 @@ def add_products(index, allow_exclusive_lock, files): Add or update products in the generic index. """ def on_ctrlc(sig, frame): - echo(f'''Can not abort `product add` without leaving database in bad state. + echo('''Can not abort `product add` without leaving database in bad state. This operation requires constructing a bunch of indexes and this takes time, the bigger your database the longer it will take. Just wait a bit.''') @@ -151,15 +151,15 @@ def _write_tab(products): echo('No products discovered :(') return - output_columns=('id', 'name', 'description', 'ancillary_quality', - 'product_type', 'gqa_abs_iterative_mean_xy', - 'gqa_ref_source', 'sat_path', - 'gqa_iterative_stddev_xy', 'time', 'sat_row', - 'orbit', 'gqa', 'instrument', 'gqa_abs_xy', 'crs', - 'resolution', 'tile_size', 'spatial_dimensions') + output_columns = ('id', 'name', 'description', 'ancillary_quality', + 'product_type', 'gqa_abs_iterative_mean_xy', + 'gqa_ref_source', 'sat_path', + 'gqa_iterative_stddev_xy', 'time', 'sat_row', + 'orbit', 'gqa', 'instrument', 'gqa_abs_xy', 'crs', + 'resolution', 'tile_size', 'spatial_dimensions') # If the intersection of desired columns with available columns is empty, just use whatever IS in df - output_columns=tuple(col for col in output_columns if col in df.columns) or df.columns - echo(df.to_string(columns=output_columns,justify='left',index=False)) + output_columns = tuple(col for col in output_columns if col in df.columns) or df.columns + echo(df.to_string(columns=output_columns, justify='left', index=False)) def _default_lister(products): diff --git a/datacube/storage/_rio.py b/datacube/storage/_rio.py index 4c585a8f37..8f3e0a3bd8 100644 --- a/datacube/storage/_rio.py +++ b/datacube/storage/_rio.py @@ -6,7 +6,6 @@ Driver implementation for Rasterio based reader. """ import logging -import warnings import contextlib from contextlib import contextmanager from threading import RLock diff --git a/datacube/storage/masking.py b/datacube/storage/masking.py index 56afc07aad..4d17547f2c 100644 --- a/datacube/storage/masking.py +++ b/datacube/storage/masking.py @@ -2,9 +2,9 @@ # # Copyright (c) 2015-2020 ODC Contributors # SPDX-License-Identifier: Apache-2.0 +from datacube.utils.masking import * # noqa: F401, F403 + import warnings warnings.warn("datacube.storage.masking has moved to datacube.utils.masking", category=DeprecationWarning) - -from datacube.utils.masking import * diff --git a/datacube/testutils/__init__.py b/datacube/testutils/__init__.py index fac472152b..a0ea3f8726 100644 --- a/datacube/testutils/__init__.py +++ b/datacube/testutils/__init__.py @@ -27,8 +27,6 @@ from datacube.utils import read_documents, SimpleDocNav from datacube.utils.geometry import GeoBox, CRS -from datacube.model.fields import parse_search_field - _DEFAULT = object() diff --git a/datacube/testutils/geom.py b/datacube/testutils/geom.py index cdc8778f22..1bf892f217 100644 --- a/datacube/testutils/geom.py +++ b/datacube/testutils/geom.py @@ -41,7 +41,7 @@ ''' -def mkA(rot=0, scale=(1, 1), shear=0, translation=(0, 0)): +def mkA(rot=0, scale=(1, 1), shear=0, translation=(0, 0)): # noqa: N802 return Affine.translation(*translation)*Affine.rotation(rot)*Affine.shear(shear)*Affine.scale(*scale) @@ -85,7 +85,7 @@ def norm_v(v): return (s, -vmin*s) - A_rot = Affine.rotation(deg) + A_rot = Affine.rotation(deg) # noqa: N806 x, y = apply_affine(A_rot, x, y) sx, tx = norm_v(x) diff --git a/datacube/ui/click.py b/datacube/ui/click.py index bff2943fe9..49ecbf39df 100644 --- a/datacube/ui/click.py +++ b/datacube/ui/click.py @@ -77,7 +77,7 @@ def emit(self, record): try: msg = self.format(record) click.echo(msg, err=True) - except: # noqa: E772 pylint: disable=bare-except + except: # pylint: disable=bare-except # noqa: E722 self.handleError(record) diff --git a/datacube/utils/_misc.py b/datacube/utils/_misc.py index c9d9e1b439..b797b3ace6 100644 --- a/datacube/utils/_misc.py +++ b/datacube/utils/_misc.py @@ -8,7 +8,7 @@ import os -class DatacubeException(Exception): +class DatacubeException(Exception): # noqa: N818 """Your Data Cube has malfunctioned""" pass diff --git a/datacube/utils/dask.py b/datacube/utils/dask.py index a934003134..9bb51a076b 100644 --- a/datacube/utils/dask.py +++ b/datacube/utils/dask.py @@ -257,12 +257,11 @@ def _save_blob_to_file(data: Union[bytes, str], def _save_blob_to_s3(data: Union[bytes, str], url: str, profile: Optional[str] = None, - creds = None, + creds=None, region_name: Optional[str] = None, with_deps=None, **kw) -> Tuple[str, bool]: from botocore.errorfactory import ClientError - from botocore.credentials import ReadOnlyCredentials from botocore.exceptions import BotoCoreError from .aws import s3_dump, s3_client diff --git a/datacube/utils/dates.py b/datacube/utils/dates.py index 10fbcfdc3c..9045d8a494 100644 --- a/datacube/utils/dates.py +++ b/datacube/utils/dates.py @@ -135,7 +135,7 @@ def mk_time_coord(dts, name='time', units=None): def _mk_parse_time() -> Callable[[Union[str, datetime]], datetime]: try: - import ciso8601 # pylint: disable=wrong-import-position + import ciso8601 # pylint: disable=wrong-import-position # noqa: F401 return _parse_time_ciso8601 except ImportError: # pragma: no cover return _parse_time_generic # pragma: no cover diff --git a/datacube/utils/documents.py b/datacube/utils/documents.py index 70018d75d6..635658e6ec 100644 --- a/datacube/utils/documents.py +++ b/datacube/utils/documents.py @@ -262,7 +262,7 @@ def remove_implicit_resolver(cls, tag_to_remove): NoDatesSafeLoader.remove_implicit_resolver('tag:yaml.org,2002:timestamp') -class InvalidDocException(Exception): +class InvalidDocException(Exception): # noqa: N818 pass diff --git a/datacube/utils/geometry/tools.py b/datacube/utils/geometry/tools.py index 7594299219..e6dc8516ee 100644 --- a/datacube/utils/geometry/tools.py +++ b/datacube/utils/geometry/tools.py @@ -380,7 +380,7 @@ def pt_tr(pts): return pt_tr -def compute_axis_overlap(Ns: int, Nd: int, s: float, t: float) -> Tuple[slice, slice]: +def compute_axis_overlap(Ns: int, Nd: int, s: float, t: float) -> Tuple[slice, slice]: # noqa: N803 """ s, t define linear transform from destination coordinate space to source >> x_s = s * x_d + t @@ -435,7 +435,7 @@ def compute_axis_overlap(Ns: int, Nd: int, s: float, t: float) -> Tuple[slice, s return (src, dst) -def box_overlap(src_shape, dst_shape, ST, tol): +def box_overlap(src_shape, dst_shape, ST, tol): # noqa: N803 """ Given two image planes whose coordinate systems are related via scale and translation only, find overlapping regions within both. diff --git a/datacube/utils/py.py b/datacube/utils/py.py index c469d92eaf..de00a15d1b 100644 --- a/datacube/utils/py.py +++ b/datacube/utils/py.py @@ -43,7 +43,7 @@ def ignore_exceptions_if(ignore_errors, errors=None): yield -class cached_property(object): # pylint: disable=invalid-name +class cached_property(object): # pylint: disable=invalid-name # noqa: N801 """ A property that is only computed once per instance and then replaces itself with an ordinary attribute. Deleting the attribute resets the diff --git a/datacube/virtual/expr.py b/datacube/virtual/expr.py index 5fbfc10d82..23fb9f08a6 100644 --- a/datacube/virtual/expr.py +++ b/datacube/virtual/expr.py @@ -67,11 +67,11 @@ def formula_parser(): @lark.v_args(inline=True) class FormulaEvaluator(lark.Transformer): - from operator import not_, or_, and_, xor # type: ignore[misc] - from operator import eq, ne, le, ge, lt, gt # type: ignore[misc] - from operator import add, sub, mul, truediv, floordiv # type: ignore[misc] - from operator import neg, pos, inv # type: ignore[misc] - from operator import mod, pow, lshift, rshift # type: ignore[misc] + from operator import not_, or_, and_, xor # type: ignore[misc] + from operator import eq, ne, le, ge, lt, gt # type: ignore[misc] + from operator import add, sub, mul, truediv, floordiv # type: ignore[misc] + from operator import neg, pos, inv # type: ignore[misc] + from operator import mod, pow, lshift, rshift # type: ignore[misc] float_literal = float int_literal = int @@ -114,6 +114,7 @@ def var_name(self, key): return TypeEvaluator().transform(parser.parse(formula)) + def evaluate_data(formula, env, parser, evaluator): """ Evaluates a formula given a parser, a corresponding evaluator class, and an environment. diff --git a/datacube/virtual/impl.py b/datacube/virtual/impl.py index 9340a29c80..0d11c2aa05 100644 --- a/datacube/virtual/impl.py +++ b/datacube/virtual/impl.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence -from functools import reduce + from typing import Any, Dict, List, Optional, cast from typing import Mapping as TypeMapping @@ -22,7 +22,7 @@ import yaml from datacube import Datacube -from datacube.api.core import select_datasets_inside_polygon, output_geobox +from datacube.api.core import output_geobox from datacube.api.grid_workflow import _fast_slice from datacube.api.query import Query, query_group_by from datacube.model import Measurement, DatasetType @@ -38,7 +38,7 @@ from .utils import select_unique, select_keys, reject_keys, merge_search_terms -class VirtualProductException(Exception): +class VirtualProductException(Exception): # noqa: N818 """ Raised if the construction of the virtual product cannot be validated. """ diff --git a/datacube/virtual/transformations.py b/datacube/virtual/transformations.py index 2347f26937..9c929eacc7 100644 --- a/datacube/virtual/transformations.py +++ b/datacube/virtual/transformations.py @@ -439,6 +439,7 @@ def result(output_var, output_desc): def year(time): return time.astype('datetime64[Y]') + def fiscal_year(time): """" This function supports group-by financial years @@ -448,10 +449,10 @@ def convert_to_quarters(x): return df.apply(lambda x: numpy.datetime64(str(x.to_period('Q-JUN').qyear))).values ds = xarray.apply_ufunc(convert_to_quarters, - time, - input_core_dims=[["time"]], - output_core_dims=[["time"]], - vectorize=True) + time, + input_core_dims=[["time"]], + output_core_dims=[["time"]], + vectorize=True) df = time['time'].to_series() years = df.apply(lambda x: numpy.datetime64(str(x.to_period('Q-JUN').qyear))).values diff --git a/datacube/virtual/utils.py b/datacube/virtual/utils.py index ccd5267671..27cb0acada 100644 --- a/datacube/virtual/utils.py +++ b/datacube/virtual/utils.py @@ -5,7 +5,6 @@ """ Utilities to facilitate virtual product implementation. """ import warnings -import math def select_unique(things): diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index 8ff326af2f..0635e73ddd 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -12,6 +12,7 @@ v1.8.next - Fix broken paths in api docs. (:pull:`1277`) - Fix readthedocs build. (:pull:`1269`) - Added doc change comparison for tuple and list types with identical values (:pull:`1281`) +- Added flake8 to Github action workflow and correct code base per flake8 rules (:pull:`1285`) - Add `dataset id` check to dataset doc resolve to prevent `uuid` returning error when `id` used in `None` (:pull:`1287`) v1.8.7 (7 June 2022) diff --git a/docs/click_utils.py b/docs/click_utils.py index 2df871e6b0..0b658d8eef 100644 --- a/docs/click_utils.py +++ b/docs/click_utils.py @@ -81,4 +81,3 @@ def setup(app): 'parallel_read_safe': False, 'parallel_write_safe': False, } - diff --git a/docs/conf.py b/docs/conf.py index 2d7160d380..14ce310c28 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,7 +5,7 @@ import os import sys -from bs4 import BeautifulSoup as bs +from bs4 import BeautifulSoup as bs # noqa: N813 # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -171,6 +171,7 @@ numfig = True + def custom_page_funcs(app, pagename, templatename, context, doctree): def get_autosummary_toc(): @@ -230,11 +231,11 @@ def get_class_toc(): 'title': '', 'menu_items': [] } - out_section['title'] = section.find_previous_sibling('p').text.replace(':','') + out_section['title'] = section.find_previous_sibling('p').text.replace(':', '') matches = section.find_all('tr') for match in matches: link = match.find(class_="internal") - + if link != None: title = link['title'] if title != None: @@ -253,7 +254,6 @@ def get_class_toc(): context['get_autosummary_toc'] = get_autosummary_toc - def setup(app): # Fix bug where code isn't being highlighted app.add_css_file('pygments.css') diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index dde91c98d0..718d1eacbd 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -433,7 +433,7 @@ def telemetry_metadata_type_doc(): @pytest.fixture def ga_metadata_type_doc(): - _FULL_EO_METADATA = Path(__file__).parent.joinpath('extensive-eo-metadata.yaml') + _FULL_EO_METADATA = Path(__file__).parent.joinpath('extensive-eo-metadata.yaml') # noqa: N806 [(path, eo_md_type)] = datacube.utils.read_documents(_FULL_EO_METADATA) return eo_md_type diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index dfda7e0cb4..776bff812d 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -474,8 +474,8 @@ def test_update_metadata_type(index, default_metadata_type): index.metadata_types.update_document(different_mt_doc, allow_unsafe_updates=True) updated_type = index.metadata_types.get_by_name(mt_doc['name']) assert ( - isinstance(updated_type.dataset_fields['time'], PgrNumericRangeDocField) - or isinstance(updated_type.dataset_fields['time'], PgsNumericRangeDocField) + isinstance(updated_type.dataset_fields['time'], PgrNumericRangeDocField) + or isinstance(updated_type.dataset_fields['time'], PgsNumericRangeDocField) ) diff --git a/integration_tests/index/test_null_index.py b/integration_tests/index/test_null_index.py index 5642277f58..6eb3809698 100644 --- a/integration_tests/index/test_null_index.py +++ b/integration_tests/index/test_null_index.py @@ -32,7 +32,7 @@ def test_null_user_resource(null_config): dc.index.users.grant_role("role1", "user1", "user2") -def test_null_user_resource(null_config): +def test_null_metadata_types_resource(null_config): with Datacube(config=null_config, validate_connection=True) as dc: assert dc.index.metadata_types.get_all() == [] with pytest.raises(NotImplementedError) as e: diff --git a/integration_tests/index/test_search.py b/integration_tests/index/test_search.py index 7855da8592..63ef173419 100644 --- a/integration_tests/index/test_search.py +++ b/integration_tests/index/test_search.py @@ -986,7 +986,7 @@ def test_cli_info(index: Index, ' sat_path: {begin: 116, end: 116}', ' sat_row: {begin: 74, end: 84}', " time: {begin: '2014-07-26T23:48:00.343853', end: '2014-07-26T23:52:00.343853'}", - ] + ] assert expected_lines == output_lines # Check indexed time separately, as we don't care what timezone it's displayed in. diff --git a/integration_tests/test_end_to_end.py b/integration_tests/test_end_to_end.py index 90b4260c99..74b80db2d8 100644 --- a/integration_tests/test_end_to_end.py +++ b/integration_tests/test_end_to_end.py @@ -11,7 +11,7 @@ from datacube.api.query import query_group_by from datacube.api.core import Datacube -from integration_tests.utils import assert_click_command, prepare_test_ingestion_configuration +from integration_tests.utils import prepare_test_ingestion_configuration PROJECT_ROOT = Path(__file__).parents[1] CONFIG_SAMPLES = PROJECT_ROOT / 'docs/config_samples/' diff --git a/integration_tests/test_model.py b/integration_tests/test_model.py index 1c983130e5..4d2c68113e 100644 --- a/integration_tests/test_model.py +++ b/integration_tests/test_model.py @@ -2,7 +2,6 @@ # # Copyright (c) 2015-2020 ODC Contributors # SPDX-License-Identifier: Apache-2.0 -import pytest from datacube.model import Dataset, DatasetType from typing import List diff --git a/setup.cfg b/setup.cfg index 81af225efd..7dce40ae36 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,5 +15,11 @@ ignore = # W50{3,4}: line break before/after binary operator W503 W504 + # E124 closing bracket does not match visual indentation + E124 + # F841 local variable 'e' is assigned to but never used + F841 + # W605 invalid escape sequence '\*' + W605 ignore-names = W,H,A,S,R,T,WS,X,Y,Z,XX,YY,XY,B,M,N,L,NX,NY diff --git a/tests/drivers/test_rio_reader.py b/tests/drivers/test_rio_reader.py index 626983e9b5..1402b5823d 100644 --- a/tests/drivers/test_rio_reader.py +++ b/tests/drivers/test_rio_reader.py @@ -53,7 +53,7 @@ def test_rio_rd_entry(): def test_rd_internals_crs(): - from rasterio.crs import CRS as RioCRS + from rasterio.crs import CRS as RioCRS # noqa: N811 assert _dc_crs(None) is None assert _dc_crs(RioCRS()) is None diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index c32b346e59..87dde2b67b 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -77,4 +77,4 @@ def test_band_info(): ds = mk_sample_dataset(bands, uri='/not/a/uri') band = BandInfo(ds, 'a') - assert(band.uri_scheme is '') + assert(band.uri_scheme is '') # noqa: F632 diff --git a/tests/storage/test_netcdfwriter.py b/tests/storage/test_netcdfwriter.py index 41390ca018..cb88cc0b15 100644 --- a/tests/storage/test_netcdfwriter.py +++ b/tests/storage/test_netcdfwriter.py @@ -238,35 +238,39 @@ def test_chunksizes(tmpnetcdf_filename): EXAMPLE_FLAGS_DEF = { - 'band_1_saturated': { - 'bits': 0, - 'values': { - 0: True, - 1: False - }, - 'description': 'Band 1 is saturated'}, - 'band_2_saturated': { - 'bits': 1, - 'values': { - 0: True, - 1: False - }, - 'description': 'Band 2 is saturated'}, - 'band_3_saturated': { - 'bits': 2, - 'values': { - 0: True, - 1: False - }, - 'description': 'Band 3 is saturated'}, - 'land_sea': { - 'bits': 9, - 'values': { - 0: 'sea', - 1: 'land' - }, - 'description': 'Land/Sea observation'}, - } + 'band_1_saturated': { + 'bits': 0, + 'values': { + 0: True, + 1: False + }, + 'description': 'Band 1 is saturated' + }, + 'band_2_saturated': { + 'bits': 1, + 'values': { + 0: True, + 1: False + }, + 'description': 'Band 2 is saturated' + }, + 'band_3_saturated': { + 'bits': 2, + 'values': { + 0: True, + 1: False + }, + 'description': 'Band 3 is saturated' + }, + 'land_sea': { + 'bits': 9, + 'values': { + 0: 'sea', + 1: 'land' + }, + 'description': 'Land/Sea observation' + }, +} def test_measurements_model_netcdfflags(): diff --git a/tests/test_dynamic_db_passwd.py b/tests/test_dynamic_db_passwd.py index 72936aa30c..62579b99b0 100644 --- a/tests/test_dynamic_db_passwd.py +++ b/tests/test_dynamic_db_passwd.py @@ -20,11 +20,9 @@ def next_token(base): def test_dynamic_password(): - url = URL.create( - 'postgresql', - host="fake_host", database="fake_database", port=6543, - username="fake_username", password="fake_password" - ) + url = URL.create('postgresql', + host="fake_host", database="fake_database", port=6543, + username="fake_username", password="fake_password") engine = PostgresDb._create_engine(url) counter[0] = 0 last_base[0] = None diff --git a/tests/test_geometry.py b/tests/test_geometry.py index 213c7e78e1..07928b3b97 100644 --- a/tests/test_geometry.py +++ b/tests/test_geometry.py @@ -922,7 +922,6 @@ def test_3d_point_converted_to_2d_point(): def test_crs(): - CRS = geometry.CRS custom_crs = geometry.CRS("""PROJCS["unnamed", GEOGCS["Unknown datum based upon the custom spheroid", DATUM["Not specified (based on custom spheroid)", SPHEROID["Custom spheroid",6371007.181,0]], @@ -947,7 +946,7 @@ def test_crs(): assert crs.dimensions == ('latitude', 'longitude') assert crs.epsg == 4326 - crs2 = CRS(crs) + crs2 = geometry.CRS(crs) assert crs2 == crs assert crs.proj == crs2.proj @@ -1217,8 +1216,8 @@ def test_fit(): def run_test(A, n, tol=1e-5): X = [(uniform(0, 1), uniform(0, 1)) for _ in range(n)] - Y = [A*x for x in X] - A_ = affine_from_pts(X, Y) + Y = [A*x for x in X] # noqa: N806 + A_ = affine_from_pts(X, Y) # noqa: N806 assert get_diff(A, A_) < tol diff --git a/tests/test_testutils.py b/tests/test_testutils.py index 875d656a9f..91ee584ce6 100644 --- a/tests/test_testutils.py +++ b/tests/test_testutils.py @@ -3,9 +3,8 @@ # Copyright (c) 2015-2020 ODC Contributors # SPDX-License-Identifier: Apache-2.0 import pytest -from datacube.model import Dataset from datacube.testutils.threads import FakeThreadPoolExecutor -from datacube.testutils import mk_sample_xr_dataset, mk_sample_product, mk_sample_dataset +from datacube.testutils import mk_sample_xr_dataset, mk_sample_dataset from datacube.testutils.io import native_geobox diff --git a/tests/test_utils_docs.py b/tests/test_utils_docs.py index b7ffcc4fae..74a523b969 100644 --- a/tests/test_utils_docs.py +++ b/tests/test_utils_docs.py @@ -324,7 +324,7 @@ def test_simple_doc_nav(): def node(name, **kwargs): return dict(id=name, lineage=dict(source_datasets=kwargs)) - A, _, C, _, _ = make_graph_abcde(node) + A, _, C, _, _ = make_graph_abcde(node) # noqa: N806 rdr = SimpleDocNav(A) assert rdr.doc == A diff --git a/tests/test_utils_other.py b/tests/test_utils_other.py index 8d98dce668..76646dcb54 100644 --- a/tests/test_utils_other.py +++ b/tests/test_utils_other.py @@ -14,7 +14,6 @@ import numpy as np import pytest -import rasterio import xarray as xr from dateutil.parser import parse from hypothesis import given @@ -44,7 +43,7 @@ pick_uri, uri_resolve, is_vsipath, normalise_path, default_base_dir) from datacube.utils.io import check_write_path -from datacube.testutils import mk_sample_product, remove_crs +from datacube.testutils import mk_sample_product def test_stats_dates():