diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index dd07802..4bfa3af 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ { "name": "VS Code DEV Container for AWS CDK development", "image": "jsii/superchain:1-buster-slim-node16" -} \ No newline at end of file +} diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 54eddf3..01aff37 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ ## :warning: Checklist if your PR is changing anything else than documentation - [ ] Posted the link to a successful manually triggered deployment workflow (successful including the resources destruction) -## Merge request description \ No newline at end of file +## Merge request description diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4a206d1..3990633 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,7 +12,7 @@ on: required: false DS_RELEASE_BOT_PRIVATE_KEY: required: false - + jobs: build_and_package: name: Build and package diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 0455e7b..f662266 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -2,7 +2,7 @@ name: Deployment on: workflow_dispatch: - + jobs: build_package_and_deploy: name: Build, package and deploy @@ -30,7 +30,7 @@ jobs: - name: Generate distribution packages run: npm run package - + - name: Install deployment environment id: install_deploy_env run: | @@ -40,7 +40,7 @@ jobs: pip install dist/python/*.gz cd integration_tests/cdk pip install -r requirements.txt - npm install + npm install deactivate cd - @@ -56,7 +56,7 @@ jobs: PROJECT_ID: ${{ steps.short-sha.outputs.sha }} run: | source .deployment_venv/bin/activate - + # synthesize the stack cd integration_tests/cdk npx cdk synth --debug --all --require-approval never @@ -65,7 +65,7 @@ jobs: npx cdk deploy --ci --all --require-approval never deactivate cd - - + - name: Tear down any infrastructure if: always() env: diff --git a/.github/workflows/distribute.yaml b/.github/workflows/distribute.yaml index 960d4a4..2e604e6 100644 --- a/.github/workflows/distribute.yaml +++ b/.github/workflows/distribute.yaml @@ -18,7 +18,7 @@ jobs: with: name: python path: dist - + - run: pip install twine - run: twine upload dist/* diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..1ba24ca --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,26 @@ +name: Lint + +on: + push: + branches: + - main + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index 72301b1..3719585 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -10,15 +10,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [3.9] + python: [3.11] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} + - name: Install Tox and any other packages run: pip install tox + - name: Run Tox # Run tox using the version of Python in `PATH` run: tox -e py diff --git a/.gitignore b/.gitignore index 0ce8378..b54838e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,4 @@ __pycache__ .tox tests/*.egg* tests/*venv* -tests/__pycache__ \ No newline at end of file +tests/__pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..5378869 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +# Optionally both commit and push +default_stages: [commit] + +# Regex for files to exclude +# Don't lint the generated JSON metadata files +exclude: "diagrams/" + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.4 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format diff --git a/README.md b/README.md index 698f7a2..83f4ffc 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,4 @@ _Warning_: If you rebase `main`, you must ensure that the commits referenced by ## Tests -Each pull request to `main` is added to a [merge queue](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions) so that a "deployment test" workflow can run before the merge actually happens. If the deployment fails, the merge is cancelled. Here is [the definition of this workflow](https://github.com/developmentseed/eoapi-cdk/blob/main/.github/workflows/deploy.yaml) and the [tests definition](https://github.com/developmentseed/eoapi-cdk/blob/main/tests). \ No newline at end of file +Each pull request to `main` is added to a [merge queue](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions) so that a "deployment test" workflow can run before the merge actually happens. If the deployment fails, the merge is cancelled. Here is [the definition of this workflow](https://github.com/developmentseed/eoapi-cdk/blob/main/.github/workflows/deploy.yaml) and the [tests definition](https://github.com/developmentseed/eoapi-cdk/blob/main/tests). diff --git a/integration_tests/cdk/README.md b/integration_tests/cdk/README.md index abc32f6..bd70e8d 100644 --- a/integration_tests/cdk/README.md +++ b/integration_tests/cdk/README.md @@ -8,11 +8,11 @@ This is a wrapper CDK code that is used to test a deployment of the `eoapi-cdk` - python - docker - node -- AWS credentials environment variables configured to point to an account. +- AWS credentials environment variables configured to point to an account. ## Installation -Install python dependencies with +Install python dependencies with ``` python -m venv .venv @@ -28,7 +28,7 @@ pip install eoapi-cdk Or alternatively, compile and package from the root of this repository to get the python version of the constructs locally. -Also install node dependencies with +Also install node dependencies with ``` npm install @@ -42,7 +42,7 @@ npx cdk --version ## Deployment -First, synthesize the app +First, synthesize the app ``` npx cdk synth --all @@ -52,4 +52,4 @@ Then, deploy ``` npx cdk deploy --all --require-approval never -``` \ No newline at end of file +``` diff --git a/integration_tests/cdk/app.py b/integration_tests/cdk/app.py index da66306..1a04621 100644 --- a/integration_tests/cdk/app.py +++ b/integration_tests/cdk/app.py @@ -1,28 +1,19 @@ -from config import build_app_config, AppConfig -from aws_cdk import ( - Stack, - aws_ec2, - aws_rds, - App, - RemovalPolicy -) +from aws_cdk import App, RemovalPolicy, Stack, aws_ec2, aws_rds +from config import AppConfig, build_app_config from constructs import Construct from eoapi_cdk import ( PgStacApiLambda, PgStacDatabase, - TitilerPgstacApiLambda, TiPgApiLambda, + TitilerPgstacApiLambda, ) class VpcStack(Stack): - def __init__(self, scope: Construct, app_config: AppConfig, id: str, **kwargs) -> None: - super().__init__( - scope, - id=id, - tags=app_config.tags, - **kwargs - ) + def __init__( + self, scope: Construct, app_config: AppConfig, id: str, **kwargs + ) -> None: + super().__init__(scope, id=id, tags=app_config.tags, **kwargs) self.vpc = aws_ec2.Vpc( self, @@ -31,7 +22,7 @@ def __init__(self, scope: Construct, app_config: AppConfig, id: str, **kwargs) - aws_ec2.SubnetConfiguration( name="ingress", subnet_type=aws_ec2.SubnetType.PUBLIC, cidr_mask=24 ), - ] + ], ) self.vpc.add_interface_endpoint( @@ -88,7 +79,7 @@ def __init__( ), allocated_storage=app_config.db_allocated_storage, instance_type=aws_ec2.InstanceType(app_config.db_instance_type), - removal_policy=RemovalPolicy.DESTROY + removal_policy=RemovalPolicy.DESTROY, ) pgstac_db.db.connections.allow_default_port_from_any_ipv4() @@ -101,7 +92,7 @@ def __init__( "description": f"{app_config.stage} STAC API", }, db=pgstac_db.db, - db_secret=pgstac_db.pgstac_secret + db_secret=pgstac_db.pgstac_secret, ) TitilerPgstacApiLambda( @@ -145,10 +136,7 @@ def __init__( pgstac_infra_stack_id = f"pgstac{app_config.project_id}" pgstac_infra_stack = pgStacInfraStack( - scope=app, - vpc=vpc_stack.vpc, - app_config=app_config, - id=pgstac_infra_stack_id + scope=app, vpc=vpc_stack.vpc, app_config=app_config, id=pgstac_infra_stack_id ) app.synth() diff --git a/integration_tests/cdk/cdk.json b/integration_tests/cdk/cdk.json index d9313bf..294adbb 100644 --- a/integration_tests/cdk/cdk.json +++ b/integration_tests/cdk/cdk.json @@ -29,4 +29,4 @@ "aws-cn" ] } -} \ No newline at end of file +} diff --git a/integration_tests/cdk/config.py b/integration_tests/cdk/config.py index cf4dedd..28bef30 100644 --- a/integration_tests/cdk/config.py +++ b/integration_tests/cdk/config.py @@ -7,15 +7,9 @@ class AppConfig(BaseSettings): - model_config = SettingsConfigDict( - env_file=".env" - ) - aws_default_account: str = pydantic.Field( - description="AWS account ID" - ) - project_id: str = pydantic.Field( - description="Project ID", default="eoapicdk" - ) + model_config = SettingsConfigDict(env_file=".env") + aws_default_account: str = pydantic.Field(description="AWS account ID") + project_id: str = pydantic.Field(description="Project ID", default="eoapicdk") stage: str = pydantic.Field(description="Stage of deployment", default="test") # because of its validator, `tags` should always come after `project_id` and `stage` tags: Dict[str, str] | None = pydantic.Field( diff --git a/integration_tests/cdk/package.json b/integration_tests/cdk/package.json index f16c04f..e5a5a6b 100644 --- a/integration_tests/cdk/package.json +++ b/integration_tests/cdk/package.json @@ -5,4 +5,3 @@ "aws-cdk": "2.130.0" } } - \ No newline at end of file diff --git a/lib/database/bootstrapper_runtime/handler.py b/lib/database/bootstrapper_runtime/handler.py index 7c937d0..0db9ae0 100644 --- a/lib/database/bootstrapper_runtime/handler.py +++ b/lib/database/bootstrapper_runtime/handler.py @@ -217,9 +217,7 @@ def handler(event, context): ) with psycopg.connect(eoapi_db_admin_conninfo, autocommit=True) as conn: with conn.cursor() as cur: - print( - f"Registering Extension in '{eoapi_params['dbname']}' database..." - ) + print(f"Registering Extension in '{eoapi_params['dbname']}' database...") register_extensions(cursor=cur) print("Starting PgSTAC Migration ") @@ -247,18 +245,18 @@ def handler(event, context): customization(cursor=cur, params=params) # Make sure the user can access the database - eoapi_user_dsn = ( - "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( - dbname=eoapi_params["dbname"], - user=eoapi_params["username"], - password=eoapi_params["password"], - host=admin_params["host"], - port=admin_params["port"], - ) + eoapi_user_dsn = "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( + dbname=eoapi_params["dbname"], + user=eoapi_params["username"], + password=eoapi_params["password"], + host=admin_params["host"], + port=admin_params["port"], ) print("Checking eoAPI user access to the PgSTAC database...") with PgstacDB(dsn=eoapi_user_dsn, debug=True) as pgdb: - print(f" OK - User has access to pgstac db, pgstac schema version: {pgdb.version}") + print( + f" OK - User has access to pgstac db, pgstac schema version: {pgdb.version}" + ) except Exception as e: print(f"Unable to bootstrap database with exception={e}") diff --git a/lib/index.ts b/lib/index.ts index 6f543fd..a1c5a45 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -5,4 +5,4 @@ export * from "./stac-api"; export * from "./titiler-pgstac-api"; export * from "./stac-browser"; export * from "./tipg-api"; -export * from "./utils"; \ No newline at end of file +export * from "./utils"; diff --git a/lib/ingestor-api/index.ts b/lib/ingestor-api/index.ts index e39f7ce..25ad480 100644 --- a/lib/ingestor-api/index.ts +++ b/lib/ingestor-api/index.ts @@ -46,7 +46,7 @@ export class StacIngestor extends Construct { ), ], }); - + const handler = this.buildApiLambda({ table: this.table, env, @@ -113,7 +113,7 @@ export class StacIngestor extends Construct { subnetSelection: undefined | ec2.SubnetSelection lambdaFunctionOptions?: CustomLambdaFunctionProps; }): lambda.Function { - + const handler = new lambda.Function(this, "api-handler", { // defaults runtime: lambda.Runtime.PYTHON_3_11, @@ -162,7 +162,7 @@ export class StacIngestor extends Construct { lambdaFunctionOptions?: CustomLambdaFunctionProps; }): lambda.Function { - + const handler = new lambda.Function(this, "stac-ingestor",{ // defaults runtime: lambda.Runtime.PYTHON_3_11, @@ -319,17 +319,17 @@ export interface StacIngestorProps { readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions; /** - * Can be used to override the default lambda function properties. + * Can be used to override the default lambda function properties. * * @default - default settings are defined in the construct. */ readonly apiLambdaFunctionOptions?: CustomLambdaFunctionProps; /** - * Can be used to override the default lambda function properties. + * Can be used to override the default lambda function properties. * * @default - default settings are defined in the construct. */ readonly ingestorLambdaFunctionOptions?: CustomLambdaFunctionProps; - -} \ No newline at end of file + +} diff --git a/lib/ingestor-api/runtime/Dockerfile b/lib/ingestor-api/runtime/Dockerfile index 02eb7da..8bfafd2 100644 --- a/lib/ingestor-api/runtime/Dockerfile +++ b/lib/ingestor-api/runtime/Dockerfile @@ -13,4 +13,4 @@ RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --n RUN mkdir -p /asset/src COPY runtime/src/*.py /asset/src/ -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"] diff --git a/lib/ingestor-api/runtime/dev_requirements.txt b/lib/ingestor-api/runtime/dev_requirements.txt index 81c05ba..9f888ae 100644 --- a/lib/ingestor-api/runtime/dev_requirements.txt +++ b/lib/ingestor-api/runtime/dev_requirements.txt @@ -1,2 +1,3 @@ httpx moto[dynamodb, ssm]>=4.0.9,<5.0 +pytest diff --git a/lib/ingestor-api/runtime/src/dependencies.py b/lib/ingestor-api/runtime/src/dependencies.py index 837dd3f..2788942 100644 --- a/lib/ingestor-api/runtime/src/dependencies.py +++ b/lib/ingestor-api/runtime/src/dependencies.py @@ -52,9 +52,9 @@ def decode_token( claims.validate() return claims - except errors.JoseError: # + except errors.JoseError as e: logger.exception("Unable to decode token") - raise HTTPException(status_code=403, detail="Bad auth token") + raise HTTPException(status_code=403, detail="Bad auth token") from e def get_username_from_token( @@ -88,7 +88,7 @@ def fetch_ingestion( ): try: return db.fetch_one(username=username, ingestion_id=ingestion_id) - except services.NotInDb: + except services.NotInDb as e: raise HTTPException( status_code=404, detail="No ingestion found with provided ID" - ) + ) from e diff --git a/lib/ingestor-api/runtime/src/main.py b/lib/ingestor-api/runtime/src/main.py index 6d217f6..46c635d 100644 --- a/lib/ingestor-api/runtime/src/main.py +++ b/lib/ingestor-api/runtime/src/main.py @@ -9,9 +9,7 @@ ) -@app.get( - "/ingestions", response_model=schemas.ListIngestionResponse, tags=["Ingestion"] -) +@app.get("/ingestions", response_model=schemas.ListIngestionResponse, tags=["Ingestion"]) async def list_ingestions( list_request: schemas.ListIngestionRequest = Depends(), db: services.Database = Depends(dependencies.get_db), @@ -78,8 +76,7 @@ def cancel_ingestion( raise HTTPException( status_code=400, detail=( - "Unable to delete ingestion if status is not " - f"{schemas.Status.queued}" + "Unable to delete ingestion if status is not " f"{schemas.Status.queued}" ), ) return ingestion.cancel(db) @@ -100,7 +97,7 @@ def publish_collection(collection: schemas.StacCollection): raise HTTPException( status_code=400, detail=(f"Unable to publish collection: {e}"), - ) + ) from e @app.delete( @@ -114,7 +111,7 @@ def delete_collection(collection_id: str): return {f"Successfully deleted: {collection_id}"} except Exception as e: print(e) - raise HTTPException(status_code=400, detail=(f"{e}")) + raise HTTPException(status_code=400, detail=(f"{e}")) from e @app.get("/auth/me") diff --git a/lib/ingestor-api/runtime/src/schemas.py b/lib/ingestor-api/runtime/src/schemas.py index 79db560..90c764b 100644 --- a/lib/ingestor-api/runtime/src/schemas.py +++ b/lib/ingestor-api/runtime/src/schemas.py @@ -116,7 +116,7 @@ def __post_init_post_parse__(self) -> None: try: self.next = json.loads(base64.b64decode(self.next)) - except (UnicodeDecodeError, binascii.Error): + except (UnicodeDecodeError, binascii.Error) as e: raise RequestValidationError( [ error_wrappers.ErrorWrapper( @@ -126,7 +126,7 @@ def __post_init_post_parse__(self) -> None: "query.next", ) ] - ) + ) from e class ListIngestionResponse(BaseModel): diff --git a/lib/ingestor-api/runtime/src/services.py b/lib/ingestor-api/runtime/src/services.py index ef263fe..2f59e69 100644 --- a/lib/ingestor-api/runtime/src/services.py +++ b/lib/ingestor-api/runtime/src/services.py @@ -22,8 +22,8 @@ def fetch_one(self, username: str, ingestion_id: str): ) try: return schemas.Ingestion.parse_obj(response["Item"]) - except KeyError: - raise NotInDb("Record not found") + except KeyError as e: + raise NotInDb("Record not found") from e def fetch_many( self, status: str, next: dict = None, limit: int = None diff --git a/lib/ingestor-api/runtime/src/validators.py b/lib/ingestor-api/runtime/src/validators.py index a5e63a5..e424ae2 100644 --- a/lib/ingestor-api/runtime/src/validators.py +++ b/lib/ingestor-api/runtime/src/validators.py @@ -37,7 +37,7 @@ def s3_object_is_accessible(bucket: str, key: str): except client.exceptions.ClientError as e: raise ValueError( f"Asset not accessible: {e.__dict__['response']['Error']['Message']}" - ) + ) from e def url_is_accessible(href: str): @@ -49,7 +49,7 @@ def url_is_accessible(href: str): except requests.exceptions.HTTPError as e: raise ValueError( f"Asset not accessible: {e.response.status_code} {e.response.reason}" - ) + ) from e @functools.cache diff --git a/lib/ingestor-api/runtime/tests/test_ingestor.py b/lib/ingestor-api/runtime/tests/test_ingestor.py index 72e393b..7282178 100644 --- a/lib/ingestor-api/runtime/tests/test_ingestor.py +++ b/lib/ingestor-api/runtime/tests/test_ingestor.py @@ -52,7 +52,7 @@ def test_handler( ingestor.handler(dynamodb_stream_event, {}) load_items.assert_called_once_with( creds="", - ingestions=list([example_ingestion]), + ingestions=[example_ingestion], ) response = mock_table.get_item( Key={"created_by": example_ingestion.created_by, "id": example_ingestion.id} diff --git a/lib/ingestor-api/runtime/tests/test_registration.py b/lib/ingestor-api/runtime/tests/test_registration.py index f83cbe0..3f5b267 100644 --- a/lib/ingestor-api/runtime/tests/test_registration.py +++ b/lib/ingestor-api/runtime/tests/test_registration.py @@ -74,9 +74,7 @@ def test_create(self, client_authenticated, collection_exists, asset_exists): ) assert response.status_code == 201 - assert collection_exists.called_once_with( - self.example_ingestion.item.collection - ) + assert collection_exists.called_once_with(self.example_ingestion.item.collection) stored_data = self.db.fetch_many(status="queued")["items"] assert len(stored_data) == 1 @@ -119,10 +117,8 @@ def test_validates_missing_assets( assert response.status_code == 422, "should get validation error" for asset_type in self.example_ingestion.item.assets.keys(): assert any( - [ - err["loc"] == ["body", "assets", asset_type, "href"] - for err in response.json()["detail"] - ] + err["loc"] == ["body", "assets", asset_type, "href"] + for err in response.json()["detail"] ), "should reference asset type in validation error response" assert ( len(self.db.fetch_many(status="queued")["items"]) == 0 diff --git a/lib/ingestor-api/runtime/tests/test_utils.py b/lib/ingestor-api/runtime/tests/test_utils.py index cdba105..ffd47cb 100644 --- a/lib/ingestor-api/runtime/tests/test_utils.py +++ b/lib/ingestor-api/runtime/tests/test_utils.py @@ -28,7 +28,7 @@ def dbcreds(): def test_load_items(loader, pgstacdb, example_ingestion, dbcreds): import src.utils as utils - utils.load_items(dbcreds, list([example_ingestion])) + utils.load_items(dbcreds, [example_ingestion]) loader.return_value.load_items.assert_called_once_with( file=jsonable_encoder([example_ingestion.item]), insert_mode=Methods.upsert, diff --git a/lib/stac-api/index.ts b/lib/stac-api/index.ts index 77c4bc2..9fe86d8 100644 --- a/lib/stac-api/index.ts +++ b/lib/stac-api/index.ts @@ -19,7 +19,7 @@ export class PgStacApiLambda extends Construct { constructor(scope: Construct, id: string, props: PgStacApiLambdaProps) { super(scope, id); - + console.log(props) console.log(props.lambdaFunctionOptions); this.stacApiLambdaFunction = new lambda.Function(this, "lambda", { @@ -51,7 +51,7 @@ export class PgStacApiLambda extends Construct { this.stacApiLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432)); } const stacApi = new HttpApi(this, `${Stack.of(this).stackName}-stac-api`, { - defaultDomainMapping: props.stacApiDomainName ? { + defaultDomainMapping: props.stacApiDomainName ? { domainName: props.stacApiDomainName } : undefined, defaultIntegration: new HttpLambdaIntegration("integration", this.stacApiLambdaFunction), @@ -104,4 +104,3 @@ export interface PgStacApiLambdaProps { */ readonly lambdaFunctionOptions?: CustomLambdaFunctionProps; } - diff --git a/lib/stac-api/runtime/Dockerfile b/lib/stac-api/runtime/Dockerfile index 3f8e0ac..380eb4e 100644 --- a/lib/stac-api/runtime/Dockerfile +++ b/lib/stac-api/runtime/Dockerfile @@ -10,4 +10,4 @@ RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --n RUN mkdir -p /asset/src COPY runtime/src/*.py /asset/src/ -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"] diff --git a/lib/stac-api/runtime/requirements.txt b/lib/stac-api/runtime/requirements.txt index 744fce5..cbc95bb 100644 --- a/lib/stac-api/runtime/requirements.txt +++ b/lib/stac-api/runtime/requirements.txt @@ -4,4 +4,4 @@ stac-fastapi.pgstac==2.4.8 stac-fastapi.types==2.4.8 # https://github.com/stac-utils/stac-fastapi/pull/466 pygeoif==0.7 -starlette_cramjam \ No newline at end of file +starlette_cramjam diff --git a/lib/stac-browser/index.ts b/lib/stac-browser/index.ts index df8beb0..d7263fc 100644 --- a/lib/stac-browser/index.ts +++ b/lib/stac-browser/index.ts @@ -28,12 +28,12 @@ export class StacBrowser extends Construct { websiteIndexDocument: props.websiteIndexDocument }) } - + // if props.cloudFrontDistributionArn is defined and props.bucketArn is not defined, add a bucket policy to allow read access from the cloudfront distribution if (props.cloudFrontDistributionArn && !props.bucketArn) { this.bucket.addToResourcePolicy(new PolicyStatement({ sid: 'AllowCloudFrontServicePrincipal', - effect: Effect.ALLOW, + effect: Effect.ALLOW, actions: ['s3:GetObject'], principals: [new ServicePrincipal('cloudfront.amazonaws.com')], resources: [this.bucket.arnForObjects('*')], @@ -44,7 +44,7 @@ export class StacBrowser extends Construct { } })); } - + // add the compiled code to the bucket as a bucket deployment this.bucketDeployment = new s3_deployment.BucketDeployment(this, 'BucketDeployment', { destinationBucket: this.bucket, @@ -59,12 +59,12 @@ export class StacBrowser extends Construct { } private buildApp(props: StacBrowserProps, cloneDirectory: string): string { - + // Define where to clone and build const githubRepoUrl = 'https://github.com/radiantearth/stac-browser.git'; - // Maybe the repo already exists in cloneDirectory. Try checking out the desired version and if it fails, delete and reclone. + // Maybe the repo already exists in cloneDirectory. Try checking out the desired version and if it fails, delete and reclone. try { console.log(`Checking if a valid cloned repo exists with version ${props.githubRepoTag}...`) execSync(`git checkout tags/${props.githubRepoTag}`, { cwd: cloneDirectory }); @@ -116,9 +116,9 @@ export class StacBrowser extends Construct { export interface StacBrowserProps { /** - * Bucket ARN. If specified, the identity used to deploy the stack must have the appropriate permissions to create a deployment for this bucket. + * Bucket ARN. If specified, the identity used to deploy the stack must have the appropriate permissions to create a deployment for this bucket. * In addition, if specified, `cloudFrontDistributionArn` is ignored since the policy of an imported resource can't be modified. - * + * * @default - No bucket ARN. A new bucket will be created. */ @@ -126,7 +126,7 @@ export interface StacBrowserProps { /** * STAC catalog URL. Overrides the catalog URL in the stac-browser configuration. - */ + */ readonly stacCatalogUrl: string; /** @@ -144,11 +144,11 @@ export interface StacBrowserProps { /** * The ARN of the cloudfront distribution that will be added to the bucket policy with read access. * If `bucketArn` is specified, this parameter is ignored since the policy of an imported bucket can't be modified. - * + * * @default - No cloudfront distribution ARN. The bucket policy will not be modified. - */ + */ readonly cloudFrontDistributionArn?: string; - + /** * The name of the index document (e.g. "index.html") for the website. Enables static website * hosting for this bucket. @@ -158,8 +158,8 @@ export interface StacBrowserProps { readonly websiteIndexDocument?: string; /** - * Location in the filesystem where to compile the browser code. - * + * Location in the filesystem where to compile the browser code. + * * @default - DEFAULT_CLONE_DIRECTORY */ readonly cloneDirectory?: string; diff --git a/lib/tipg-api/index.ts b/lib/tipg-api/index.ts index d40a3df..37f8b5c 100644 --- a/lib/tipg-api/index.ts +++ b/lib/tipg-api/index.ts @@ -49,7 +49,7 @@ import { this.tiPgLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432), "allow connections from tipg"); } const tipgApi = new HttpApi(this, `${Stack.of(this).stackName}-tipg-api`, { - defaultDomainMapping: props.tipgApiDomainName ? { + defaultDomainMapping: props.tipgApiDomainName ? { domainName: props.tipgApiDomainName } : undefined, defaultIntegration: new HttpLambdaIntegration("integration", this.tiPgLambdaFunction), @@ -91,11 +91,11 @@ import { * Customized environment variables to send to titiler-pgstac runtime. */ readonly apiEnv?: Record; - + /** - * Custom Domain Name for tipg API. If defined, will create the - * domain name and integrate it with the tipg API. - * + * Custom Domain Name for tipg API. If defined, will create the + * domain name and integrate it with the tipg API. + * * @default - undefined */ readonly tipgApiDomainName?: IDomainName; diff --git a/lib/tipg-api/runtime/src/handler.py b/lib/tipg-api/runtime/src/handler.py index 51f2a18..f077b59 100644 --- a/lib/tipg-api/runtime/src/handler.py +++ b/lib/tipg-api/runtime/src/handler.py @@ -15,9 +15,11 @@ # skipping linting rule that wants all imports at the top from tipg.main import app # noqa: E402 -from tipg.settings import CustomSQLSettings # noqa: E402 -from tipg.settings import DatabaseSettings # noqa: E402 -from tipg.settings import PostgresSettings # noqa: E402; noqa: E402 +from tipg.settings import ( + CustomSQLSettings, # noqa: E402 + DatabaseSettings, # noqa: E402 + PostgresSettings, # noqa: E402; noqa: E402 +) postgres_settings = PostgresSettings() db_settings = DatabaseSettings() diff --git a/lib/titiler-pgstac-api/index.ts b/lib/titiler-pgstac-api/index.ts index 726744d..61708e5 100644 --- a/lib/titiler-pgstac-api/index.ts +++ b/lib/titiler-pgstac-api/index.ts @@ -13,12 +13,12 @@ import { import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; import { Construct } from "constructs"; import { CustomLambdaFunctionProps } from "../utils"; - - // default settings that can be overridden by the user-provided environment. + + // default settings that can be overridden by the user-provided environment. let defaultTitilerPgstacEnv :{ [key: string]: any } = { "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff", - "GDAL_CACHEMAX": "200", + "GDAL_CACHEMAX": "200", "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", "GDAL_INGESTED_BYTES_AT_OPEN": "32768", "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES", @@ -26,7 +26,7 @@ import { CustomLambdaFunctionProps } from "../utils"; "GDAL_HTTP_VERSION": "2", "PYTHONWARNINGS": "ignore", "VSI_CACHE": "TRUE", - "VSI_CACHE_SIZE": "5000000", + "VSI_CACHE_SIZE": "5000000", "DB_MIN_CONN_SIZE": "1", "DB_MAX_CONN_SIZE": "1" } @@ -34,10 +34,10 @@ import { CustomLambdaFunctionProps } from "../utils"; export class TitilerPgstacApiLambda extends Construct { readonly url: string; public titilerPgstacLambdaFunction: lambda.Function; - + constructor(scope: Construct, id: string, props: TitilerPgStacApiLambdaProps) { super(scope, id); - + this.titilerPgstacLambdaFunction = new lambda.Function(this, "lambda", { // defaults runtime: lambda.Runtime.PYTHON_3_11, @@ -55,9 +55,9 @@ import { CustomLambdaFunctionProps } from "../utils"; // if user provided environment variables, merge them with the defaults. environment: props.apiEnv ? { ...defaultTitilerPgstacEnv, ...props.apiEnv, "PGSTAC_SECRET_ARN": props.dbSecret.secretArn } : defaultTitilerPgstacEnv, // overwrites defaults with user-provided configurable properties - ...props.lambdaFunctionOptions, + ...props.lambdaFunctionOptions, }); - + // grant access to buckets using addToRolePolicy if (props.buckets) { props.buckets.forEach(bucket => { @@ -67,46 +67,46 @@ import { CustomLambdaFunctionProps } from "../utils"; })); }); } - + props.dbSecret.grantRead(this.titilerPgstacLambdaFunction); - + if (props.vpc) { this.titilerPgstacLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432), "allow connections from titiler"); } - + const stacApi = new HttpApi(this, `${Stack.of(this).stackName}-titiler-pgstac-api`, { - defaultDomainMapping: props.titilerPgstacApiDomainName ? { - domainName: props.titilerPgstacApiDomainName + defaultDomainMapping: props.titilerPgstacApiDomainName ? { + domainName: props.titilerPgstacApiDomainName } : undefined, defaultIntegration: new HttpLambdaIntegration("integration", this.titilerPgstacLambdaFunction), }); - + this.url = stacApi.url!; - + new CfnOutput(this, "titiler-pgstac-api-output", { exportName: `${Stack.of(this).stackName}-titiler-pgstac-url`, value: this.url, }); } } - + export interface TitilerPgStacApiLambdaProps { /** * VPC into which the lambda should be deployed. */ readonly vpc?: ec2.IVpc; - + /** * RDS Instance with installed pgSTAC. */ readonly db: rds.IDatabaseInstance; - + /** * Subnet into which the lambda should be deployed. */ readonly subnetSelection?: ec2.SubnetSelection; - + /** * Secret containing connection information for pgSTAC database. */ @@ -114,19 +114,19 @@ import { CustomLambdaFunctionProps } from "../utils"; /** * Customized environment variables to send to titiler-pgstac runtime. These will be merged with `defaultTitilerPgstacEnv`. - * The database secret arn is automatically added to the environment variables at deployment. + * The database secret arn is automatically added to the environment variables at deployment. /*/ readonly apiEnv?: Record; /** - * list of buckets the lambda will be granted access to. + * list of buckets the lambda will be granted access to. */ readonly buckets?: string[]; /** * Custom Domain Name Options for Titiler Pgstac API, - * - * @default - undefined. + * + * @default - undefined. */ readonly titilerPgstacApiDomainName?: IDomainName; @@ -138,4 +138,3 @@ import { CustomLambdaFunctionProps } from "../utils"; readonly lambdaFunctionOptions?: CustomLambdaFunctionProps; } - diff --git a/lib/titiler-pgstac-api/runtime/Dockerfile b/lib/titiler-pgstac-api/runtime/Dockerfile index e5cf74e..307425d 100644 --- a/lib/titiler-pgstac-api/runtime/Dockerfile +++ b/lib/titiler-pgstac-api/runtime/Dockerfile @@ -16,4 +16,4 @@ RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/g COPY runtime/src/*.py /asset/ -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"] diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..4ca2480 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,23 @@ +line-length = 90 + +[lint] +select = [ + "F", # flake8 + "C", # flake8-comprehensions + "B", # flake8-bugbear + "I001", # isort +] + +ignore = [ + "E203", + "E266", + "F403", + "E231", + "E501", # line too long, handled by black + "B008", # do not perform function calls in argument defaults + "B905", # ignore zip() without an explicit strict= parameter, only support with python >3.10 +] + + +[lint.mccabe] +max-complexity = 18 diff --git a/tox.ini b/tox.ini index 79db331..2e49f79 100644 --- a/tox.ini +++ b/tox.ini @@ -7,46 +7,10 @@ extras = test envdir = toxenv passenv = AWS_DEFAULT_REGION commands = - pip install flake8 isort black pytest pip install -r ./lib/ingestor-api/runtime/requirements.txt pip install -r ./lib/ingestor-api/runtime/dev_requirements.txt - flake8 - black lib --diff - isort lib python -m pytest -s - -[flake8] -ignore = E203, E266, E501, W503, F403, E231 -exclude = - node_modules - __pycache__ - .git - .tox - *venv* - toxenv* - devenv* - cdk.out - *.egg-info -max-line-length = 90 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 - -[black] -line-length = 90 -exclude = - __pycache__ - .git - .tox - *venv* - toxenv* - devenv* - cdk.out - *.egg-info - -[isort] -profile = black - [pytest] addopts = -ra -q testpaths = lib/ingestor-api