Opencdms api#32

.github/workflows/ci_cd.yml → .github/workflows/deploy_latest.yml

@@ -1,4 +1,4 @@
-name: Deploy on EC2
+name: Deploy on EC2 - Latest
 
 on:
   push:
@@ -11,9 +11,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: Deploy in EC2
         env:
-            PRIVATE_KEY: ${{secrets.AWS_PRIVATE_KEY}}
-            HOSTNAME : ${{secrets.HOSTNAME}}
+            PRIVATE_KEY: ${{secrets.AWS_PRIVATE_KEY_LATEST}}
+            HOSTNAME : ${{secrets.HOSTNAME_LATEST}}
             USERNAME : ${{secrets.USERNAME}}
+            HOST_FQDN: ${{secrets.LATEST_HOST_FQDN}}
 
         run: |
           echo "$PRIVATE_KEY" > private_key && chmod 600 private_key

.github/workflows/deploy_stable.yml

@@ -0,0 +1,29 @@
+name: Deploy on EC2 - Stable
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  Deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Deploy in EC2
+        env:
+            PRIVATE_KEY: ${{secrets.AWS_PRIVATE_KEY_STABLE}}
+            HOSTNAME : ${{secrets.HOSTNAME_STABLE}}
+            USERNAME : ${{secrets.USERNAME}}
+            HOST_FQDN: ${{secrets.STABLE_HOST_FQDN}}
+
+        run: |
+          echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
+          ssh -o StrictHostKeyChecking=no -i private_key ${USERNAME}@${HOSTNAME} '
+            cd /home/ubuntu/opencdms-api
+            git pull origin main
+            docker-compose -f docker-compose.prod.yml build
+            docker-compose -f docker-compose.prod.yml stop opencdms-api
+            sleep 30
+            docker-compose -f docker-compose.prod.yml up -d --build
+          '

README.md

@@ -91,4 +91,49 @@ To run the tests, you just need to run `docker-compose -f docker-compose.test.ym
 
 Check the logs for error.
 
-### How to access surface, climsoft or mch API
+### How to access surface, climsoft, pygeoapi or mch API
+
+OpenCDMS API server is a FastAPI application. surface, climsoft, pygeoapi, mch servers are
+mounted to this FastAPI application. When mounting these child applications, we also have
+enforced an Auth Middleware. So, if you want to access the endpoints on these child applications,
+you have to make authenticated request.
+
+To get an access token using default username and password:
+
+```bash
+$ curl -X 'POST' \
+  'http://localhost:5070/auth' \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+  "username": "admin",
+  "password": "password123"
+}'
+```
+
+Say, it returns 
+
+```bash
+{
+  "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhZG1pbiIsImV4cCI6MTY0MzgzMDUzMiwidG9rZW5fdHlwZSI6ImFjY2VzcyIsImp0aSI6ImEzM2Q4OWMyLTNlNmEtNDJlYS04MGZjLWViZjEzNTcyZjU5MSIsInVzZXJfaWQiOjF9.dp_wPSDZwL4HAN8JWCWyGRlL0s8gRvWKASUeDPQQygY"
+}
+```
+
+Now you can make request to protected endpoints using this access token as `Bearer` token.
+
+Here is an example:
+
+```bash
+$ curl -X 'GET' \
+  'http://localhost:5070/climsoft/v1/acquisition-types/?limit=25&offset=0' \
+  -H 'accept: application/json' \
+  -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhZG1pbiIsImV4cCI6MTY0MzgzMDUzMiwidG9rZW5fdHlwZSI6ImFjY2VzcyIsImp0aSI6ImEzM2Q4OWMyLTNlNmEtNDJlYS04MGZjLWViZjEzNTcyZjU5MSIsInVzZXJfaWQiOjF9.dp_wPSDZwL4HAN8JWCWyGRlL0s8gRvWKASUeDPQQygY'
+```
+
+which will return something like this:
+
+```bash
+{"message":"Successfully fetched acquisition types.","status":"success","result":[]}
+```
+
+You can use Postman to make this requests easily.

docker-compose.prod.yml

@@ -52,6 +52,8 @@ services:
       - surfacedb
       - mchdb
     environment:
+      - PYGEOAPI_CONFIG=/code/pygeoapi-config.yml
+      - PYGEOAPI_OPENAPI=/code/pygeoapi-openapi.yml
       - PYTHONPATH=/code/surface/api
       - CLIMSOFT_DATABASE_URI=mysql+mysqldb://root:password@climsoftdb:3306/climsoft
       - CLIMSOFT_SECRET_KEY=6v(n93zuas7&k^-zcvkp5kq*3hw49t%ra0nt6!_3(0&4!
@@ -123,9 +125,10 @@ services:
       - MCH_API_ENABLED=true
       - DEFAULT_USERNAME=admin
       - DEFAULT_PASSWORD=password123
+
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.fastapi.rule=Host(`api.opencdms.org`)"
+      - "traefik.http.routers.fastapi.rule=Host(`$HOST_FQDN`)"
       - "traefik.http.routers.fastapi.tls=true"
       - "traefik.http.routers.fastapi.tls.certresolver=letsencrypt"
     command:

docker-compose.test.yml

@@ -9,6 +9,8 @@ services:
     expose:
       - "5000"
     environment:
+      - PYGEOAPI_CONFIG=/code/pygeoapi-config.yml
+      - PYGEOAPI_OPENAPI=/code/pygeoapi-openapi.yml
       - PYTHONPATH=/code/surface/api
       - TIMESCALEDB_TELEMETRY=off
       - PGDATA=/var/lib/postgresql/data/pgdata
@@ -80,6 +82,7 @@ services:
       - MCH_API_ENABLED=true
       - DEFAULT_USERNAME=admin
       - DEFAULT_PASSWORD=password123
+
     depends_on:
       - test_opencdms_surface_db
       - test_opencdms_mch_db

docker-compose.yml

@@ -11,6 +11,8 @@ services:
     env_file:
       - .env
     environment:
+      - PYGEOAPI_CONFIG=/code/pygeoapi-config.yml
+      - PYGEOAPI_OPENAPI=/code/pygeoapi-openapi.yml
       - PYTHONPATH=/code/surface/api
       - CLIMSOFT_DATABASE_URI=mysql+mysqldb://root:password@opencdms_climsoftdb:3306/climsoft
       - CLIMSOFT_SECRET_KEY=climsoft-secret-key

dockerfile

@@ -9,8 +9,9 @@ ENV PYTHONFAULTHANDLER=1 \
     PIP_NO_CACHE_DIR=1
 
 RUN apt-get update --fix-missing
-RUN apt-get install -y g++ libgdal-dev libpq-dev libgeos-dev libproj-dev openjdk-17-jre vim wait-for-it
+RUN apt-get install -y g++ libgdal-dev libpq-dev libgeos-dev libproj-dev openjdk-17-jre vim wait-for-it r-base-core libmagick++-dev
 RUN apt-get install -y curl git && pip install --upgrade pip
+RUN R -e "install.packages('magick')"
 
 WORKDIR /code
 
@@ -33,6 +34,8 @@ RUN useradd -m opencdms_api_user && chown -R opencdms_api_user /code
 
 RUN ["chmod", "+x", "/code/scripts/load_initial_surface_data.sh"]
 
+COPY ["pygeoapi-config.yml", "/code"]
+
 USER opencdms_api_user
 
 ENTRYPOINT [ "/bin/sh", "entrypoint.sh" ]

entrypoint.sh

@@ -3,4 +3,5 @@
 python surface/api/manage.py migrate
 python init_climsoft_db.py
 ./scripts/load_initial_surface_data.sh
+pygeoapi openapi generate /code/pygeoapi-config.yml >| /code/pygeoapi-openapi.yml
 exec $@

pygeoapi-config.yml

@@ -0,0 +1,73 @@
+server:
+    bind:
+        host: 0.0.0.0
+        port: 443
+    url: https://${HOST_FQDN}/pygeoapi/
+    mimetype: application/json; charset=UTF-8
+    encoding: utf-8
+    gzip: false
+    languages:
+        - en-US
+        - fr-CA
+    cors: true
+    pretty_print: true
+    limit: 10
+
+    map:
+        url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png
+        attribution: '<a href="https://wikimediafoundation.org/wiki/Maps_Terms_of_Use">Wikimedia maps</a> | Map data &copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
+    manager:
+        name: TinyDB
+        connection: /tmp/pygeoapi-test-process-manager.db
+        output_dir: /tmp
+
+logging:
+    level: ERROR
+
+metadata:
+    identification:
+        title:
+            en: pygeoapi default instance
+            fr: instance par défaut de pygeoapi
+        description:
+            en: pygeoapi provides an API to geospatial data
+            fr: pygeoapi fournit une API aux données géospatiales
+        keywords:
+            en:
+                - geospatial
+                - data
+                - api
+            fr:
+                - géospatiale
+                - données
+                - api
+        keywords_type: theme
+        terms_of_service: https://creativecommons.org/licenses/by/4.0/
+        url: http://example.org
+    license:
+        name: CC-BY 4.0 license
+        url: https://creativecommons.org/licenses/by/4.0/
+    provider:
+        name: Organization Name
+        url: https://pygeoapi.io
+    contact:
+        name: Lastname, Firstname
+        position: Position Title
+        address: Mailing Address
+        city: City
+        stateorprovince: Administrative Area
+        postalcode: Zip or Postal Code
+        country: Country
+        phone: +xx-xxx-xxx-xxxx
+        fax: +xx-xxx-xxx-xxxx
+        email: you@example.org
+        url: Contact URL
+        hours: Hours of Service
+        instructions: During hours of service.  Off on weekends.
+        role: pointOfContact
+
+resources:
+    windrose-generator:
+        type: process
+        processor:
+            name: opencdms_process.process.climatol.windrose_generator.WindroseProcessor

requirements.txt

@@ -33,7 +33,7 @@ mch-api @ git+https://github.com/opencdms/mch-api.git@main
 mysqlclient
 numpy
 opencdms @ git+https://github.com/opencdms/pyopencdms.git@main
-climsoft_api @ git+https://github.com/faysal-ishtiaq/climsoft-api.git@package-installer
+climsoft_api @ git+https://github.com/opencdms/climsoft-api.git@main
 packaging
 pandas
 passlib
@@ -63,3 +63,5 @@ typing-extensions
 urllib3
 uvicorn
 Werkzeug
+Flask-Cors
+opencdms_process @ git+https://github.com/opencdms/opencdms-process.git@main

src/opencdms_api/main.py

@@ -17,6 +17,7 @@
 from fastapi.templating import Jinja2Templates
 from fastapi.responses import HTMLResponse
 from pathlib import Path
+from pygeoapi.flask_app import APP as pygeoapi_app
 
 
 # load controllers
@@ -31,6 +32,8 @@ def get_app():
         app.mount("/mch", AuthMiddleWare(WSGIMiddleware(mch_api_application)))
     if settings.CLIMSOFT_API_ENABLED is True:
         app.mount("/climsoft", AuthMiddleWare(climsoft_app))
+    app.mount("/pygeoapi", AuthMiddleWare(WSGIMiddleware(pygeoapi_app)))
+
     app.include_router(router)
 
     @app.on_event("startup")
@@ -78,7 +81,7 @@ def fetch_stations(session: Session = Depends(get_session)):
 
 @app.get("/", response_class=HTMLResponse)
 def root(request: Request):
-    supported_apis = []
+    supported_apis = [{"title": "Pygeoapi", "url": "/pygeoapi"}]
     if settings.SURFACE_API_ENABLED:
         supported_apis.append({"title": "Surface API", "url": "/surface"})
     if settings.CLIMSOFT_API_ENABLED: