Merge pull request #94 from qld-gov-au/develop

Develop to master - improve spam protection

.ahoy.yml

@@ -12,14 +12,14 @@ commands:
       ahoy clean
       ahoy build-network
       ahoy up -- --build --force-recreate
-      ahoy install-site
       ahoy title "Build complete"
       ahoy doctor
       ahoy info 1
 
   build-network:
     usage: Ensure that the amazeeio network exists.
     cmd: |
+      ahoy title "Creating amazeeio Docker network"
       docker network prune -f > /dev/null
       docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
 
@@ -28,63 +28,74 @@ commands:
     cmd: |
       ahoy line "Project                  : " ${PROJECT}
       ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
-      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
-      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "DB port on host          : " $(docker port $(sh bin/docker-compose.sh ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(sh bin/docker-compose.sh ps -q solr) 8983 | cut -d : -f 2)
       ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
 
   up:
     usage: Build and start Docker containers.
     cmd: |
-      docker-compose up -d "$@"
-      sleep 10
-      docker-compose logs
+      ahoy title "Building and starting Docker containers"
+      sh bin/docker-compose.sh up -d "$@"
+      echo "Initialising database schema"
+      ahoy cli '$APP_DIR/bin/init.sh'
+      echo "Waiting for containers to start listening..."
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
-      if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
-      if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
       export DOCTOR_CHECK_CLI=0
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+    cmd: |
+      ahoy title 'Stopping and removing old containers, images, volumes, networks'
+      if [ -f "docker-compose.yml" ]; then sh bin/docker-compose.sh down --volumes; fi
 
   start:
     usage: Start existing Docker containers.
-    cmd: docker-compose start "$@"
+    cmd: sh bin/docker-compose.sh start "$@"
 
   stop:
     usage: Stop running Docker containers.
-    cmd: docker-compose stop "$@"
+    cmd: sh bin/docker-compose.sh stop "$@"
 
   restart:
     usage: Restart all stopped and running Docker containers.
-    cmd: docker-compose restart "$@"
+    cmd: sh bin/docker-compose.sh restart "$@"
 
   logs:
     usage: Show Docker logs.
-    cmd: docker-compose logs "$@"
+    cmd: sh bin/docker-compose.sh logs "$@"
 
   pull:
     usage: Pull latest docker images.
-    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate && cd $APP_DIR && sh'; fi
+    cmd: |
+      CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
+      if [ "${#}" -ne 0 \]; then
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+      else
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+      fi
 
   doctor:
     usage: Find problems with current project setup.
-    cmd: .docker/scripts/doctor.sh "$@"
+    cmd: bin/doctor.sh "$@"
 
   install-site:
-    usage: Install a site.
+    usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli '$APP_DIR/scripts/init.sh'
+      ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
 
   clean:
     usage: Remove containers and all build files.
     cmd: |
+      ahoy title "Cleaning up old builds"
       ahoy down
       # Remove other directories.
       # @todo: Add destinations below.
@@ -101,37 +112,46 @@ commands:
 
   flush-redis:
     usage: Flush Redis cache.
-    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+    cmd: docker exec -i $(sh bin/docker-compose.sh ps -q redis) redis-cli flushall > /dev/null
 
   lint:
     usage: Lint code.
     cmd: |
+      ahoy title 'Check for lint'
       ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
   copy-local-files:
     usage: Update files from local repo.
     cmd: |
-      docker cp . $(docker-compose ps -q ckan):/srv/app/
-      docker cp .docker/scripts $(docker-compose ps -q ckan):/srv/app/
-      docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI'
+      docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
+      docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
+      ahoy title 'Run unit tests'
       ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
     usage: Run BDD tests.
     cmd: |
-      ahoy start-ckan-job-worker &
+      ahoy title 'Run scenario tests'
+      ahoy cli "rm -f test/screenshots/*"
+      ahoy start-ckan-job-worker
       ahoy start-mailmock &
       sleep 5 &&
-      ahoy cli "behave -k ${*:-test/features}" --tags @smoke && \
-      ahoy cli "behave -k ${*:-test/features}" || \
-      [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      if [ "$BEHAVE_TAG" = "" ]; then
+        (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \
+         && ahoy cli "behave -k ${*:-test/features} --tags=-smoke" \
+        ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      else
+        # run tests with the specified tag
+        ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \
+          || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      fi
       ahoy stop-mailmock
       ahoy stop-ckan-job-worker
 
@@ -152,7 +172,7 @@ commands:
     cmd: |
       ahoy title 'Starting default CKAN background job worker'
       ahoy cli "ckan_cli jobs clear && \
-                ckan_cli jobs worker"
+                ckan_cli jobs worker &"
 
   stop-ckan-job-worker:
     usage: Stops CKAN background job worker

.docker/Dockerfile-template.ckan

@@ -2,6 +2,7 @@ FROM openknowledge/ckan-dev:{CKAN_VERSION}
 
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_VERSION={CKAN_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
 ENV PYTHON={PYTHON}
 
@@ -14,17 +15,13 @@ RUN apk add --no-cache build-base \
 
 COPY .docker/test.ini $CKAN_INI
 
-RUN sed -i "s@SITE_URL@${CKAN_SITE_URL}@g" $CKAN_INI
-
 COPY . ${APP_DIR}/
 
-COPY .docker/scripts ${APP_DIR}/scripts
-
-COPY .docker/scripts/ckan_cli /usr/bin/
+COPY bin/ckan_cli /usr/bin/
 
-RUN chmod +x ${APP_DIR}/scripts/*.sh /usr/bin/ckan_cli
+RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
 
 # Init current extension.
-RUN ${APP_DIR}/scripts/init-ext.sh
+RUN ${APP_DIR}/bin/init-ext.sh
 
-CMD ["/srv/app/scripts/serve.sh"]
+CMD ["/srv/app/bin/serve.sh"]

.docker/test.ini

@@ -179,6 +179,9 @@ ckan.datarequests.default_organisation = open-data-administration-data-requests
 # Enable or disable circumstances for closing data requests. Default value is False
 ckan.datarequests.enable_closing_circumstances = True
 
+ckan.comments.check_for_profanity = True
+ckan.comments.bad_words_file = /srv/app/ckanext/datarequests/bad_words.txt
+
 # Logging configuration
 [loggers]
 keys = root, ckan, ckanext

.env

@@ -13,7 +13,7 @@
 PROJECT="ckanext-datarequests"
 
 # Docker Compose project name. All containers will have this name.
-COMPOSE_PROJECT_NAME="$PROJECT"
+COMPOSE_PROJECT_NAME="ckanext-datarequests"
 
 # Flag to allow code linting failures. 0=enforce, 1=ignore
 ALLOW_LINT_FAIL=0

.flake8

@@ -3,7 +3,6 @@
 
 exclude =
     ckan
-    scripts
 
 # Extended output format.
 format = pylint

.github/workflows/test.yml

@@ -15,30 +15,30 @@ jobs:
 
     name: Test on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: drevops/ci-builder
+    container: drevops/ci-builder:23.7.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         timeout-minutes: 2
 
       - name: Build
-        run: .circleci/build.sh
+        run: bin/build.sh
         timeout-minutes: 15
 
       - name: Test
-        run: .circleci/test.sh
+        run: bin/test.sh
         timeout-minutes: 20
 
       - name: Retrieve screenshots
         if: failure()
-        run: .circleci/process-artifacts.sh
+        run: bin/process-artifacts.sh
         timeout-minutes: 1
 
       - name: Upload screenshots
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots

.circleci/build.sh → bin/build.sh

@@ -2,8 +2,7 @@
 ##
 # Build site in CI.
 #
-set -e
-set -x
+set -ex
 
 # Process Docker Compose configuration. This is used to avoid multiple
 # docker-compose.yml files.
@@ -24,7 +23,8 @@ else
 fi
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
+    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \
     | sed "s|{PYTHON}|$PYTHON|g" \
-    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" > .docker/Dockerfile.ckan
+    > .docker/Dockerfile.ckan
 
-ahoy build || (ahoy logs; exit 1)
+ahoy build

.docker/scripts/ckan_cli → bin/ckan_cli

@@ -59,14 +59,14 @@ fi
 if [ "$COMMAND" = "ckan" ]; then
     # adjust args to match ckan expectations
     COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
-    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
     # adjust args to match paster expectations
     COMMAND=$1
-    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     if [ "$1" = "show" ]; then shift; fi
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}
 else

.docker/scripts/create-test-data.sh → bin/create-test-data.sh

@@ -2,14 +2,14 @@
 ##
 # Create some example content for extension BDD tests.
 #
-set -e
+set -ex
 
 CKAN_ACTION_URL=${CKAN_SITE_URL}api/action
 CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
 CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
 CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
 
-. ${APP_DIR}/scripts/activate
+. ${APP_DIR}/bin/activate
 
 add_user_if_needed () {
     echo "Adding user '$2' ($1) with email address [$3]"
@@ -25,7 +25,7 @@ ckan_cli sysadmin add "${CKAN_USER_NAME}"
 API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
 if [ "$API_KEY" = "None" ]; then
     echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..."
-    API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]')
+    API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |tail -1 | tr -d '[:space:]')
 fi
 
 ##
@@ -36,7 +36,7 @@ echo "Adding ckan.datarequests.closing_circumstances:"
 
 curl -LsH "Authorization: ${API_KEY}" \
     --header "Content-Type: application/json" \
-    --data '{"ckan.datarequests.closing_circumstances":"Released as open data|nominate_dataset\nOpen dataset already exists|nominate_dataset\nPartially released|nominate_dataset\nTo be released as open data at a later date|nominate_approximate_date\nData openly available elsewhere\nNot suitable for release as open data\nRequested data not available/cannot be compiled\nRequestor initiated closure"}' \
+    --data '{"ckan.datarequests.closing_circumstances": "Released as open data|nominate_dataset\nOpen dataset already exists|nominate_dataset\nPartially released|nominate_dataset\nTo be released as open data at a later date|nominate_approximate_date\nData openly available elsewhere\nNot suitable for release as open data\nRequested data not available/cannot be compiled\nRequestor initiated closure"}' \
     ${CKAN_ACTION_URL}/config_option_update
 
 ##
@@ -60,11 +60,12 @@ echo "Creating ${TEST_ORG_TITLE} organisation:"
 
 TEST_ORG=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \
+    --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
+        "description": "Organisation for testing issues"}' \
     ${CKAN_ACTION_URL}/organization_create
 )
 
-TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON $APP_DIR/scripts/extract-id.py)
+TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
 
 echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
 
@@ -104,7 +105,7 @@ DR_ORG=$( \
     ${CKAN_ACTION_URL}/organization_create
 )
 
-DR_ORG_ID=$(echo $DR_ORG | $PYTHON $APP_DIR/scripts/extract-id.py)
+DR_ORG_ID=$(echo $DR_ORG | $PYTHON $APP_DIR/bin/extract-id.py)
 
 echo "Assigning test users to ${DR_ORG_TITLE} Organisation:"
 
@@ -138,7 +139,7 @@ Closed_DR=$( \
 echo $Closed_DR
 
 # Get the ID of that newly created Data Request
-CLOSE_DR_ID=$(echo $Closed_DR | $PYTHON $APP_DIR/scripts/extract-id.py)
+CLOSE_DR_ID=$(echo $Closed_DR | $PYTHON $APP_DIR/bin/extract-id.py)
 echo $CLOSE_DR_ID
 
 echo "Closing Data Request:"
@@ -159,4 +160,4 @@ curl -LsH "Authorization: ${API_KEY}" \
 
 ckan_cli search-index rebuild
 
-. ${APP_DIR}/scripts/deactivate
+. ${APP_DIR}/bin/deactivate