Merge pull request #278 from openreferral/database-postgres

Database postgres

.github/workflows/build_database.yml

@@ -48,5 +48,53 @@ jobs:
          git config --global user.name 'Database Schema Generator'
          git config --global user.email 'hello@openreferral.org'
          git add database/database_mysql.sql
-         git commit -m "Database Schema changed by automatic generation" database/database_mysql.sql
+         git commit -m "Mysql Database Schema changed by automatic generation" database/database_mysql.sql
+         git push
+  postgresql:
+    needs: mysql
+    runs-on: ubuntu-20.04
+    services:
+      postgresql:
+        # 12 is chosen because that matches the version of the client utils we will install
+        image: postgres:12
+        env:
+          POSTGRES_PASSWORD: 1234
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432/tcp
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+        architecture: x64
+    # If the MySQL stage above made changes, we need to pull first
+    - run: git pull
+    - name: Install Ubuntu libs
+      run: sudo apt-get install -y graphviz graphviz-dev postgresql-client-12 libpq-dev
+    - name: Install Python libs
+      run: pip install -r requirements.txt
+    - name: Install Python Libs for DB work
+      run: pip install -r requirements_build_database.in
+    - run: POSTGRESQL_PASSWORD=1234 ./build_database_postgresql.sh
+    - run: cat database/database_postgresql.sql
+    - name: Check for changes
+      run: |
+        if git diff --exit-code; then
+          echo "CHANGED=false" >>${GITHUB_ENV}
+        else
+          echo "CHANGED=true" >>${GITHUB_ENV}
+        fi
+    - name: Commit database back (if changes)
+      if: env.CHANGED == 'true'
+      run: |
+         git config --global user.name 'Database Schema Generator'
+         git config --global user.email 'hello@openreferral.org'
+         git add database/database_postgresql.sql
+         git commit -m "Postgresql Database Schema changed by automatic generation" database/database_postgresql.sql
          git push

README.md

@@ -92,4 +92,5 @@ These are built automatically from the schema by a GitHub action and checked in
 If you want to run the build process locally for any reason, look in the following shell files for help:
 
     build_database_mysql.sh
+    build_database_postgresql.sh
 

build_database_postgresql.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -e
+
+# This script assumes you have:
+# * Postgres client tools installed
+#      Ubuntu: sudo apt-get install -y postgresql-client libpq-dev
+# * The Python libraries in requirements_build_database.in installed
+# * A Postgres database server running locally
+# * The postgres user password for it is in the environmental variable POSTGRESQL_PASSWORD
+# * A database called 'test' can be created and used
+#
+# If you don't have a server running locally, Docker is a good way to do this. Run:
+#    docker run -d --name openreferral-database-dump-postgresql -e POSTGRES_PASSWORD=$POSTGRESQL_PASSWORD --publish 5432:5432/tcp postgres:12
+# Make sure the version of the server matches the version of the client libraries you have installed, or pg_dump will fail.
+
+# get ready temp dir
+echo "READY TEMP DIRECTORY"
+mkdir -p tmp_datapackage_for_database_schemas
+
+# make datapackage
+echo "MAKE DATAPACKAGE"
+code="import os
+import sys
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(BASE_DIR,'python'))
+import openreferral.utils
+openreferral.utils.build_blank_datapackage_for_database_schemas(os.path.join(BASE_DIR, 'datapackage.json'),os.path.join(BASE_DIR, 'tmp_datapackage_for_database_schemas'))"
+echo -e "$code" | python
+
+# Create a database
+echo "CREATE A DATABASE"
+PGPASSWORD=$POSTGRESQL_PASSWORD psql -h 127.0.0.1 -p 5432 -U postgres -w -c "CREATE DATABASE test"
+
+# Create the schema
+echo "CREATE THE SCHEMA"
+code="from frictionless import Package
+package = Package('tmp_datapackage_for_database_schemas/datapackage.json')
+package.to_sql('postgresql://postgres:$POSTGRESQL_PASSWORD@127.0.0.1:5432/test')"
+echo -e "$code" | python
+
+# Dump
+echo "DUMP THE SCHEMA"
+PGPASSWORD=$POSTGRESQL_PASSWORD pg_dump -h 127.0.0.1 -p 5432 -U postgres --no-password -f database/database_postgresql.sql --schema-only test
+
+# Remove dumped lines
+echo "EDIT THE SCHEMA"
+sed -i '/^\-\- Dumped by /d' database/database_postgresql.sql
+sed -i '/^\-\- Dumped from /d' database/database_postgresql.sql