Merge pull request #273 from openreferral/database-schema-mysql-2

Build MySQL Database automatically from schema

.github/workflows/build_database.yml

@@ -0,0 +1,52 @@
+name: Build Databases
+on:
+  push:
+    branches-ignore:
+      - 'master'
+
+jobs:
+  mysql:
+    runs-on: ubuntu-20.04
+    services:
+      mysql:
+        # 8 is chosen because that matches the version of the client utils we will install
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: 1234
+        ports:
+          - 3306:3306/tcp
+        options: >-
+          --health-cmd "/usr/bin/mysql -h 127.0.0.1 --user=root --password=1234 --execute \"SHOW DATABASES;\""
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+        architecture: x64
+    - name: Install Ubuntu libs
+      run: sudo apt-get install -y graphviz graphviz-dev mysql-client-8.0
+    - name: Install Python libs
+      run: pip install -r requirements.txt
+    - name: Install Python Libs for DB work
+      run: pip install -r requirements_build_database.in
+    - run: MYSQL_ROOT_PASSWORD=1234 ./build_database_mysql.sh
+    - run: cat database/database_mysql.sql
+    - name: Check for changes
+      run: |
+        if git diff --exit-code; then
+          echo "CHANGED=false" >>${GITHUB_ENV}
+        else
+          echo "CHANGED=true" >>${GITHUB_ENV}
+        fi
+    - name: Commit database back (if changes)
+      if: env.CHANGED == 'true'
+      run: |
+         git config --global user.name 'Database Schema Generator'
+         git config --global user.email 'hello@openreferral.org'
+         git add database/database_mysql.sql
+         git commit -m "Database Schema changed by automatic generation" database/database_mysql.sql
+         git push

.gitignore

@@ -2,3 +2,5 @@
 /docs/assets/entity_relationship_diagram.svg
 /docs/assets/entity_relationship_diagram_core_tables.svg
 /examples/*/datapackage.json
+/tmp_datapackage_for_database_schemas/*.json
+/tmp_datapackage_for_database_schemas/*.csv

README.md

@@ -84,3 +84,12 @@ Everytime a new example csv is added or datapackage.json is changed, rerun:
     python examples/make_datapackages.py
 
 Everytime a new example directory is added, please add a line above and in `.github/workflows/test_examples.yml`.
+
+## Database Schemas
+
+These are built automatically from the schema by a GitHub action and checked in for you.
+
+If you want to run the build process locally for any reason, look in the following shell files for help:
+
+    build_database_mysql.sh
+

build_database_mysql.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+# This script assumes you have:
+# * Mysql client tools installed
+# * The Python libraries in requirements_build_database.in installed
+# * A Mysql database server running locally
+# * The root password for it is in the environmental variable MYSQL_ROOT_PASSWORD
+# * A database called 'test' can be created and used
+#
+# If you don't have a server running locally, Docker is a good way to do this. Run:
+#    docker run -d --name openreferral-database-dump-mysql -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD  --publish 3306:3306/tcp  mysql:8.0
+
+
+# get ready temp dir
+echo "READY TEMP DIRECTORY"
+mkdir -p tmp_datapackage_for_database_schemas
+
+# make datapackage
+echo "MAKE DATAPACKAGE"
+code="import os
+import sys
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(BASE_DIR,'python'))
+import openreferral.utils
+openreferral.utils.build_blank_datapackage_for_database_schemas(os.path.join(BASE_DIR, 'datapackage.json'),os.path.join(BASE_DIR, 'tmp_datapackage_for_database_schemas'))"
+echo -e "$code" | python
+
+# Create a database
+echo "CREATE A DATABASE"
+mysql -h 127.0.0.1 -P 3306 -u root -p$MYSQL_ROOT_PASSWORD -e "CREATE DATABASE test"
+
+# Create the schema
+echo "CREATE THE SCHEMA"
+code="from frictionless import Package
+package = Package('tmp_datapackage_for_database_schemas/datapackage.json')
+package.to_sql('mysql://root:$MYSQL_ROOT_PASSWORD@127.0.0.1:3306/test')"
+echo -e "$code" | python
+
+# Dump
+echo "DUMP THE SCHEMA"
+mysqldump -h 127.0.0.1 -P 3306 -u root -p$MYSQL_ROOT_PASSWORD --result-file=database/database_mysql.sql --no-data test
+
+# Remove dumped lines
+echo "EDIT THE SCHEMA"
+sed -i '/^\-\- Dump completed on/d' database/database_mysql.sql
+sed -i '/^\-\- Server version/d' database/database_mysql.sql
+sed -i '/^\-\- Host/d' database/database_mysql.sql
+sed -i '/^\-\- MySQL dump/d' database/database_mysql.sql