diff --git a/.env.example b/.env.example index c212198..0c1dccc 100644 --- a/.env.example +++ b/.env.example @@ -25,7 +25,7 @@ TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_te USE_HTTPS_FOR_DEV=false # CKAN core -CKAN_VERSION=2.10.0 +CKAN_VERSION=2.11.4 CKAN_SITE_ID=default CKAN_SITE_URL=http://localhost:5000 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME @@ -57,7 +57,7 @@ TEST_CKAN_REDIS_URL=redis://redis:6379/1 # Datapusher DATAPUSHER_VERSION=0.0.21 CKAN_DATAPUSHER_URL=http://datapusher:8800 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://localhost:5000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000 # Extensions CKAN__PLUGINS="image_view text_view datatables_view datastore datapusher envvars" diff --git a/ckan/Dockerfile b/ckan/Dockerfile deleted file mode 100644 index 17269f0..0000000 --- a/ckan/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM ckan/ckan-base:2.11 - -# Install any extensions needed by your CKAN instance -# See Dockerfile.dev for more details and examples - -# Copy custom initialization scripts -COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/ - -# Apply any patches needed to CKAN core or any of the built extensions (not the -# runtime mounted ones) -COPY --chown=ckan-sys:ckan-sys patches ${APP_DIR}/patches - -USER ckan - -RUN for d in $APP_DIR/patches/*; do \ - if [ -d $d ]; then \ - for f in `ls $d/*.patch | sort -g`; do \ - cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \ - done ; \ - fi ; \ - done diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index b4a12b9..111611c 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,5 +1,12 @@ FROM ckan/ckan-dev:2.11 +USER root + +# Install sudo for development purposes. Also install patch. +RUN apt-get update && apt-get install -y sudo patch && \ + echo "ckan ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \ + rm -rf /var/lib/apt/lists/* + # Install any extensions needed by your CKAN instance # - Make sure to add the plugins to CKAN__PLUGINS in the .env file # - Also make sure all provide all extra configuration options, either by: diff --git a/ckan/setup/prerun.py.override b/ckan/setup/prerun.py.override deleted file mode 100644 index 3d68696..0000000 --- a/ckan/setup/prerun.py.override +++ /dev/null @@ -1,212 +0,0 @@ -import os -import sys -import subprocess -import psycopg2 -try: - from urllib.request import urlopen - from urllib.error import URLError -except ImportError: - from urllib2 import urlopen - from urllib2 import URLError - -import time -import re -import json - -ckan_ini = os.environ.get("CKAN_INI", "/srv/app/ckan.ini") - -RETRY = 5 - - -def update_plugins(): - - plugins = os.environ.get("CKAN__PLUGINS", "") - print(("[prerun] Setting the following plugins in {}:".format(ckan_ini))) - print(plugins) - cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - print("[prerun] Plugins set.") - - -def check_main_db_connection(retry=None): - - conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") - if not conn_str: - print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") - return check_db_connection(conn_str, retry) - - -def check_datastore_db_connection(retry=None): - - conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") - if not conn_str: - print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") - return check_db_connection(conn_str, retry) - - -def check_db_connection(conn_str, retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print("[prerun] Giving up after 5 tries...") - sys.exit(1) - - try: - connection = psycopg2.connect(conn_str) - - except psycopg2.Error as e: - print(str(e)) - print("[prerun] Unable to connect to the database, waiting...") - time.sleep(10) - check_db_connection(conn_str, retry=retry - 1) - else: - connection.close() - - -def check_solr_connection(retry=None): - - if retry is None: - retry = RETRY - elif retry == 0: - print("[prerun] Giving up after 5 tries...") - sys.exit(1) - - url = os.environ.get("CKAN_SOLR_URL", "") - search_url = '{url}/schema/name?wt=json'.format(url=url) - - try: - connection = urlopen(search_url) - except URLError as e: - print(str(e)) - print("[prerun] Unable to connect to solr, waiting...") - time.sleep(10) - check_solr_connection(retry=retry - 1) - else: - import re - conn_info = connection.read() - schema_name = json.loads(conn_info) - if 'ckan' in schema_name['name']: - print('[prerun] Succesfully connected to solr and CKAN schema loaded') - else: - print('[prerun] Succesfully connected to solr, but CKAN schema not found') - - -def init_db(): - - db_command = ["ckan", "-c", ckan_ini, "db", "init"] - print("[prerun] Initializing or upgrading db - start") - try: - subprocess.check_output(db_command, stderr=subprocess.STDOUT) - print("[prerun] Initializing or upgrading db - end") - except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) - print("[prerun] Database not ready, waiting a bit before exit...") - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - - -def init_datastore_db(): - - conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") - if not conn_str: - print("[prerun] Skipping datastore initialization") - return - - datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"] - - connection = psycopg2.connect(conn_str) - cursor = connection.cursor() - - print("[prerun] Initializing datastore db - start") - try: - datastore_perms = subprocess.Popen( - datastore_perms_command, stdout=subprocess.PIPE - ) - - perms_sql = datastore_perms.stdout.read() - # Remove internal pg command as psycopg2 does not like it - perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql) - cursor.execute(perms_sql) - for notice in connection.notices: - print(notice) - - connection.commit() - - print("[prerun] Initializing datastore db - end") - print(datastore_perms.stdout.read()) - except psycopg2.Error as e: - print("[prerun] Could not initialize datastore") - print(str(e)) - - except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) - print("[prerun] Database not ready, waiting a bit before exit...") - time.sleep(5) - sys.exit(1) - else: - print(e.output) - raise e - finally: - cursor.close() - connection.close() - - -def create_sysadmin(): - - name = os.environ.get("CKAN_SYSADMIN_NAME") - password = os.environ.get("CKAN_SYSADMIN_PASSWORD") - email = os.environ.get("CKAN_SYSADMIN_EMAIL") - - if name and password and email: - - # Check if user exists - command = ["ckan", "-c", ckan_ini, "user", "show", name] - - out = subprocess.check_output(command) - if b"User:None" not in re.sub(b"\s", b"", out): - print("[prerun] Sysadmin user exists, skipping creation") - return - - # Create user - command = [ - "ckan", - "-c", - ckan_ini, - "user", - "add", - name, - "password=" + password, - "email=" + email, - ] - - subprocess.call(command) - print("[prerun] Created user {0}".format(name)) - - # Make it sysadmin - command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name] - - subprocess.call(command) - print("[prerun] Made user {0} a sysadmin".format(name)) - - -if __name__ == "__main__": - - maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true" - - if maintenance: - print("[prerun] Maintenance mode, skipping setup...") - else: - check_main_db_connection() - init_db() - update_plugins() - check_datastore_db_connection() - init_datastore_db() - check_solr_connection() - create_sysadmin() - \ No newline at end of file diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override deleted file mode 100755 index 0c8409c..0000000 --- a/ckan/setup/start_ckan.sh.override +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh - -# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) -ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx - -# Set up the Secret key used by Beaker and Flask -# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini -then - echo "Setting beaker.session.secret in ini file" - ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') - ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" - ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" -fi - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py - -echo "Set up ckan.datapusher.api_token in the CKAN config file" -ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Set the common uwsgi options -UWSGI_OPTS="--plugins http,python \ - --socket /tmp/uwsgi.sock \ - --wsgi-file /srv/app/wsgi.py \ - --module wsgi:application \ - --uid 92 --gid 92 \ - --http 0.0.0.0:5000 \ - --master --enable-threads \ - --lazy-apps \ - -p 2 -L -b 32768 --vacuum \ - --harakiri $UWSGI_HARAKIRI" - -if [ $? -eq 0 ] -then - # Start supervisord - supervisord --configuration /etc/supervisord.conf & - # Start uwsgi - sudo -u ckan -EH uwsgi $UWSGI_OPTS -else - echo "[prerun] failed...not starting CKAN." -fi diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override deleted file mode 100755 index 93ce814..0000000 --- a/ckan/setup/start_ckan_development.sh.override +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/sh - -# Install any local extensions in the src_extensions volume -echo "Looking for local extensions to install..." -echo "Extension dir contents:" -ls -la $SRC_EXTENSIONS_DIR -for i in $SRC_EXTENSIONS_DIR/* -do - if [ -d $i ]; - then - - if [ -f $i/pip-requirements.txt ]; - then - pip install -r $i/pip-requirements.txt - echo "Found requirements file in $i" - fi - if [ -f $i/requirements.txt ]; - then - pip install -r $i/requirements.txt - echo "Found requirements file in $i" - fi - if [ -f $i/dev-requirements.txt ]; - then - pip install -r $i/dev-requirements.txt - echo "Found dev-requirements file in $i" - fi - if [ -f $i/setup.py ]; - then - cd $i - python3 $i/setup.py develop - echo "Found setup.py file in $i" - cd $APP_DIR - fi - - # Point `use` in test.ini to location of `test-core.ini` - if [ -f $i/test.ini ]; - then - echo "Updating \`test.ini\` reference to \`test-core.ini\` for plugin $i" - ckan config-tool $i/test.ini "use = config:../../src/ckan/test-core.ini" - fi - fi -done - -# Set debug to true -echo "Enabling debug mode" -ckan config-tool $CKAN_INI -s DEFAULT "debug = true" - -# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) -ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx - -# Set up the Secret key used by Beaker and Flask -# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini -then - echo "Setting beaker.session.secret in ini file" - ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') - ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" - ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" -fi - -# Update the plugins setting in the ini file with the values defined in the env var -echo "Loading the following plugins: $CKAN__PLUGINS" -ckan config-tool $CKAN_INI "ckan.plugins = $CKAN__PLUGINS" - -# Update test-core.ini DB, SOLR & Redis settings -echo "Loading test settings into test-core.ini" -ckan config-tool $SRC_DIR/ckan/test-core.ini \ - "sqlalchemy.url = $TEST_CKAN_SQLALCHEMY_URL" \ - "ckan.datastore.write_url = $TEST_CKAN_DATASTORE_WRITE_URL" \ - "ckan.datastore.read_url = $TEST_CKAN_DATASTORE_READ_URL" \ - "solr_url = $TEST_CKAN_SOLR_URL" \ - "ckan.redis.url = $TEST_CKAN_REDIS_URL" - -# Run the prerun script to init CKAN and create the default admin user -sudo -u ckan -EH python3 prerun.py - -echo "Set up ckan.datapusher.api_token in the CKAN config file" -ckan config-tool $CKAN_INI "ckan.datapusher.api_token=$(ckan -c $CKAN_INI user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" - -# Run any startup scripts provided by images extending this one -if [[ -d "/docker-entrypoint.d" ]] -then - for f in /docker-entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: Running init file $f"; . "$f" ;; - *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; - *) echo "$0: Ignoring $f (not an sh or py file)" ;; - esac - echo - done -fi - -# Start supervisord -supervisord --configuration /etc/supervisord.conf & - -# Start the development server with automatic reload -sudo -u ckan -EH ckan -c $CKAN_INI run -H 0.0.0.0 \ No newline at end of file