diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/deploy_latest.yml similarity index 77% rename from .github/workflows/ci_cd.yml rename to .github/workflows/deploy_latest.yml index 136711b..a215448 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/deploy_latest.yml @@ -1,4 +1,4 @@ -name: Deploy on EC2 +name: Deploy on EC2 - Latest on: push: @@ -11,9 +11,10 @@ jobs: - uses: actions/checkout@v2 - name: Deploy in EC2 env: - PRIVATE_KEY: ${{secrets.AWS_PRIVATE_KEY}} - HOSTNAME : ${{secrets.HOSTNAME}} + PRIVATE_KEY: ${{secrets.AWS_PRIVATE_KEY_LATEST}} + HOSTNAME : ${{secrets.HOSTNAME_LATEST}} USERNAME : ${{secrets.USERNAME}} + HOST_FQDN: ${{secrets.LATEST_HOST_FQDN}} run: | echo "$PRIVATE_KEY" > private_key && chmod 600 private_key diff --git a/.github/workflows/deploy_stable.yml b/.github/workflows/deploy_stable.yml new file mode 100644 index 0000000..d17f0c8 --- /dev/null +++ b/.github/workflows/deploy_stable.yml @@ -0,0 +1,29 @@ +name: Deploy on EC2 - Stable + +on: + release: + types: + - published + +jobs: + Deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Deploy in EC2 + env: + PRIVATE_KEY: ${{secrets.AWS_PRIVATE_KEY_STABLE}} + HOSTNAME : ${{secrets.HOSTNAME_STABLE}} + USERNAME : ${{secrets.USERNAME}} + HOST_FQDN: ${{secrets.STABLE_HOST_FQDN}} + + run: | + echo "$PRIVATE_KEY" > private_key && chmod 600 private_key + ssh -o StrictHostKeyChecking=no -i private_key ${USERNAME}@${HOSTNAME} ' + cd /home/ubuntu/opencdms-api + git pull origin main + docker-compose -f docker-compose.prod.yml build + docker-compose -f docker-compose.prod.yml stop opencdms-api + sleep 30 + docker-compose -f docker-compose.prod.yml up -d --build + ' \ No newline at end of file diff --git a/README.md b/README.md index 3d41dbf..f90d5d0 100644 --- a/README.md +++ b/README.md @@ -91,4 +91,49 @@ To run the tests, you just need to run `docker-compose -f docker-compose.test.ym Check the logs for error. -### How to access surface, climsoft or mch API +### How to access surface, climsoft, pygeoapi or mch API + +OpenCDMS API server is a FastAPI application. surface, climsoft, pygeoapi, mch servers are +mounted to this FastAPI application. When mounting these child applications, we also have +enforced an Auth Middleware. So, if you want to access the endpoints on these child applications, +you have to make authenticated request. + +To get an access token using default username and password: + +```bash +$ curl -X 'POST' \ + 'http://localhost:5070/auth' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "username": "admin", + "password": "password123" +}' +``` + +Say, it returns + +```bash +{ + "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhZG1pbiIsImV4cCI6MTY0MzgzMDUzMiwidG9rZW5fdHlwZSI6ImFjY2VzcyIsImp0aSI6ImEzM2Q4OWMyLTNlNmEtNDJlYS04MGZjLWViZjEzNTcyZjU5MSIsInVzZXJfaWQiOjF9.dp_wPSDZwL4HAN8JWCWyGRlL0s8gRvWKASUeDPQQygY" +} +``` + +Now you can make request to protected endpoints using this access token as `Bearer` token. + +Here is an example: + +```bash +$ curl -X 'GET' \ + 'http://localhost:5070/climsoft/v1/acquisition-types/?limit=25&offset=0' \ + -H 'accept: application/json' \ + -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhZG1pbiIsImV4cCI6MTY0MzgzMDUzMiwidG9rZW5fdHlwZSI6ImFjY2VzcyIsImp0aSI6ImEzM2Q4OWMyLTNlNmEtNDJlYS04MGZjLWViZjEzNTcyZjU5MSIsInVzZXJfaWQiOjF9.dp_wPSDZwL4HAN8JWCWyGRlL0s8gRvWKASUeDPQQygY' +``` + +which will return something like this: + +```bash +{"message":"Successfully fetched acquisition types.","status":"success","result":[]} +``` + +You can use Postman to make this requests easily. \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c76a37e..e3b6d1b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -52,6 +52,8 @@ services: - surfacedb - mchdb environment: + - PYGEOAPI_CONFIG=/code/pygeoapi-config.yml + - PYGEOAPI_OPENAPI=/code/pygeoapi-openapi.yml - PYTHONPATH=/code/surface/api - CLIMSOFT_DATABASE_URI=mysql+mysqldb://root:password@climsoftdb:3306/climsoft - CLIMSOFT_SECRET_KEY=6v(n93zuas7&k^-zcvkp5kq*3hw49t%ra0nt6!_3(0&4! @@ -123,9 +125,10 @@ services: - MCH_API_ENABLED=true - DEFAULT_USERNAME=admin - DEFAULT_PASSWORD=password123 + labels: - "traefik.enable=true" - - "traefik.http.routers.fastapi.rule=Host(`api.opencdms.org`)" + - "traefik.http.routers.fastapi.rule=Host(`$HOST_FQDN`)" - "traefik.http.routers.fastapi.tls=true" - "traefik.http.routers.fastapi.tls.certresolver=letsencrypt" command: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 11b7b70..75231f0 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -9,6 +9,8 @@ services: expose: - "5000" environment: + - PYGEOAPI_CONFIG=/code/pygeoapi-config.yml + - PYGEOAPI_OPENAPI=/code/pygeoapi-openapi.yml - PYTHONPATH=/code/surface/api - TIMESCALEDB_TELEMETRY=off - PGDATA=/var/lib/postgresql/data/pgdata @@ -80,6 +82,7 @@ services: - MCH_API_ENABLED=true - DEFAULT_USERNAME=admin - DEFAULT_PASSWORD=password123 + depends_on: - test_opencdms_surface_db - test_opencdms_mch_db diff --git a/docker-compose.yml b/docker-compose.yml index 94a059d..7aad8bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: env_file: - .env environment: + - PYGEOAPI_CONFIG=/code/pygeoapi-config.yml + - PYGEOAPI_OPENAPI=/code/pygeoapi-openapi.yml - PYTHONPATH=/code/surface/api - CLIMSOFT_DATABASE_URI=mysql+mysqldb://root:password@opencdms_climsoftdb:3306/climsoft - CLIMSOFT_SECRET_KEY=climsoft-secret-key diff --git a/dockerfile b/dockerfile index 5ade39d..af4d1c2 100644 --- a/dockerfile +++ b/dockerfile @@ -9,8 +9,9 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_NO_CACHE_DIR=1 RUN apt-get update --fix-missing -RUN apt-get install -y g++ libgdal-dev libpq-dev libgeos-dev libproj-dev openjdk-17-jre vim wait-for-it +RUN apt-get install -y g++ libgdal-dev libpq-dev libgeos-dev libproj-dev openjdk-17-jre vim wait-for-it r-base-core libmagick++-dev RUN apt-get install -y curl git && pip install --upgrade pip +RUN R -e "install.packages('magick')" WORKDIR /code @@ -33,6 +34,8 @@ RUN useradd -m opencdms_api_user && chown -R opencdms_api_user /code RUN ["chmod", "+x", "/code/scripts/load_initial_surface_data.sh"] +COPY ["pygeoapi-config.yml", "/code"] + USER opencdms_api_user ENTRYPOINT [ "/bin/sh", "entrypoint.sh" ] diff --git a/entrypoint.sh b/entrypoint.sh index 6e8d033..6c6d181 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,4 +3,5 @@ python surface/api/manage.py migrate python init_climsoft_db.py ./scripts/load_initial_surface_data.sh +pygeoapi openapi generate /code/pygeoapi-config.yml >| /code/pygeoapi-openapi.yml exec $@ diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml new file mode 100644 index 0000000..db26879 --- /dev/null +++ b/pygeoapi-config.yml @@ -0,0 +1,73 @@ +server: + bind: + host: 0.0.0.0 + port: 443 + url: https://${HOST_FQDN}/pygeoapi/ + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + gzip: false + languages: + - en-US + - fr-CA + cors: true + pretty_print: true + limit: 10 + + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + manager: + name: TinyDB + connection: /tmp/pygeoapi-test-process-manager.db + output_dir: /tmp + +logging: + level: ERROR + +metadata: + identification: + title: + en: pygeoapi default instance + fr: instance par défaut de pygeoapi + description: + en: pygeoapi provides an API to geospatial data + fr: pygeoapi fournit une API aux données géospatiales + keywords: + en: + - geospatial + - data + - api + fr: + - géospatiale + - données + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: http://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: Organization Name + url: https://pygeoapi.io + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + windrose-generator: + type: process + processor: + name: opencdms_process.process.climatol.windrose_generator.WindroseProcessor \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 33b33ad..bf4b776 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ mch-api @ git+https://github.com/opencdms/mch-api.git@main mysqlclient numpy opencdms @ git+https://github.com/opencdms/pyopencdms.git@main -climsoft_api @ git+https://github.com/faysal-ishtiaq/climsoft-api.git@package-installer +climsoft_api @ git+https://github.com/opencdms/climsoft-api.git@main packaging pandas passlib @@ -63,3 +63,5 @@ typing-extensions urllib3 uvicorn Werkzeug +Flask-Cors +opencdms_process @ git+https://github.com/opencdms/opencdms-process.git@main diff --git a/src/opencdms_api/main.py b/src/opencdms_api/main.py index 49ec6e3..c5222bf 100644 --- a/src/opencdms_api/main.py +++ b/src/opencdms_api/main.py @@ -17,6 +17,7 @@ from fastapi.templating import Jinja2Templates from fastapi.responses import HTMLResponse from pathlib import Path +from pygeoapi.flask_app import APP as pygeoapi_app # load controllers @@ -31,6 +32,8 @@ def get_app(): app.mount("/mch", AuthMiddleWare(WSGIMiddleware(mch_api_application))) if settings.CLIMSOFT_API_ENABLED is True: app.mount("/climsoft", AuthMiddleWare(climsoft_app)) + app.mount("/pygeoapi", AuthMiddleWare(WSGIMiddleware(pygeoapi_app))) + app.include_router(router) @app.on_event("startup") @@ -78,7 +81,7 @@ def fetch_stations(session: Session = Depends(get_session)): @app.get("/", response_class=HTMLResponse) def root(request: Request): - supported_apis = [] + supported_apis = [{"title": "Pygeoapi", "url": "/pygeoapi"}] if settings.SURFACE_API_ENABLED: supported_apis.append({"title": "Surface API", "url": "/surface"}) if settings.CLIMSOFT_API_ENABLED: