[DNM] test publishing after testing

.github/workflows/built_and_test.yml

@@ -12,6 +12,12 @@ on:
 env:
   env_file: ${{ github.event.inputs.env_file || '.env' }}
 
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.TEST_DOCKER_HUB_ACCESS_TOKEN }}
+  DOCKER_HUB_REPOSITORY_NAME: toanwmde
+  DOCKER_HUB_ID: toanwmde
+  WORKFLOW_RUN_NUMBER: ${{ github.run_id }}
+
 jobs:
   build_mediawiki:
     runs-on: ubuntu-latest
@@ -139,6 +145,13 @@ jobs:
           if-no-files-found: error
           path: artifacts/wikibase-bundle.docker.tar.gz
 
+      - name: Archive metadata artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: BuildMetadata
+          if-no-files-found: error
+          path: artifacts/build_metadata_*.env
+
   build_quickstatements:
     runs-on: ubuntu-latest
     steps:
@@ -247,7 +260,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [ repo, fedprops, repo-client, quickstatements ]
+        suite: [ pingback ]
     needs:
       - build_wikibase_bundle
       - build_queryservice
@@ -280,3 +293,33 @@ jobs:
           name: TestArtifacts
           path: |
             test/log
+
+  should_publish:
+    needs:
+      - test_wikibase
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: c-py/action-dotenv-to-setenv@v2
+        with:
+          env-file: variables.env
+      - uses: c-py/action-dotenv-to-setenv@v2
+        with:
+          env-file: ${{ env.env_file }}
+
+      - name: Cache git_cache repos
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-build-metadata
+        with:
+          path: artifacts/LastBuildMetadata
+          key: cache-build-metadata
+
+      - name: Get BuildMetadata artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: BuildMetadata
+          path: artifacts/
+
+      - name: Check if something changed
+        run: bash publish/should_publish.sh

.github/workflows/publish.yml

@@ -15,7 +15,7 @@ on:
 env:
   env_file: ${{ github.event.inputs.env_file }}
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+  DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.TEST_DOCKER_HUB_ACCESS_TOKEN }}
   DOCKER_HUB_REPOSITORY_NAME: toanwmde
   DOCKER_HUB_ID: toanwmde
   WORKFLOW_RUN_NUMBER: ${{ github.event.inputs.workflow_run_number }}

.github/workflows/test.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [ repo, fedprops, repo-client ]
+        suite: [ repo, fedprops, repo-client, quickstatements, pingback ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

Docker/build/Wikibase/Dockerfile

@@ -44,7 +44,9 @@ COPY htaccess /var/www/html/.htaccess
 RUN ln -s /var/www/html/ /var/www/html/w
 
 ENV MW_SITE_NAME=wikibase-docker\
-    MW_SITE_LANG=en
+    MW_SITE_LANG=en\
+    MW_WG_JOB_RUN_RATE=2\
+    WIKIBASE_PINGBACK=false
 
 ENTRYPOINT ["/bin/bash"]
 CMD ["/entrypoint.sh"]

Docker/build/Wikibase/LocalSettings.php.template

@@ -48,13 +48,19 @@ ${DOLLAR}wgRCMaxAge = 365 * 24 * 3600;
 
 wfLoadSkin( 'Vector' );
 
+// T276905 - default set in Dockerfile
+${DOLLAR}wgJobRunRate = ${MW_WG_JOB_RUN_RATE};
+
 ## Wikibase
 # Load Wikibase repo & client with the example / default settings.
 require_once "${DOLLAR}IP/extensions/Wikibase/repo/Wikibase.php";
 require_once "${DOLLAR}IP/extensions/Wikibase/repo/ExampleSettings.php";
 require_once "${DOLLAR}IP/extensions/Wikibase/client/WikibaseClient.php";
 require_once "${DOLLAR}IP/extensions/Wikibase/client/ExampleSettings.php";
 
+#Pingback
+${DOLLAR}wgWBRepoSettings['wikibasePingback'] = ${WIKIBASE_PINGBACK};
+
 foreach (glob("LocalSettings.d/*.php") as ${DOLLAR}filename)
 {
     include ${DOLLAR}filename;

Docker/build/WikibaseBundle/Dockerfile

@@ -17,6 +17,15 @@ FROM ${WIKIBASE_IMAGE_NAME}:latest
 RUN rm -rf /var/www/html/vendor
 COPY --from=composer /var/www/html /var/www/html
 COPY extra-install.sh /extra-install.sh
+
+# copy extension settings
+COPY LocalSettings.WikibaseManifest.php /var/www/html/LocalSettings.d/WikibaseManifest.php
 COPY LocalSettings.OAuth.php /var/www/html/LocalSettings.d/OAuth.php
+COPY LocalSettings.UniversalLanguageSelector.php /var/www/html/LocalSettings.d/UniversalLanguageSelector.php
+COPY LocalSettings.EntitySchema.php /var/www/html/LocalSettings.d/EntitySchema.php
 COPY LocalSettings.Scribunto.php /var/www/html/LocalSettings.d/Scribunto.php
+COPY LocalSettings.VisualEditor.php /var/www/html/LocalSettings.d/VisualEditor.php
+COPY LocalSettings.cldr.php /var/www/html/LocalSettings.d/cldr.php
+
+# copy template for envsubst
 COPY LocalSettings.Elasticsearch.php.template /var/www/html/LocalSettings.template.d/

Docker/build/WikibaseBundle/LocalSettings.EntitySchema.php

@@ -0,0 +1,5 @@
+<?php
+
+// https://www.mediawiki.org/wiki/Extension:EntitySchema
+## EntitySchema Configuration
+wfLoadExtension( 'EntitySchema' );

Docker/build/WikibaseBundle/LocalSettings.UniversalLanguageSelector.php

@@ -0,0 +1,5 @@
+<?php
+
+// https://www.mediawiki.org/wiki/Extension:UniversalLanguageSelector
+## UniversalLanguageSelector Extension
+wfLoadExtension( 'UniversalLanguageSelector' );

Docker/build/WikibaseBundle/LocalSettings.VisualEditor.php

@@ -0,0 +1,6 @@
+<?php
+
+// https://www.mediawiki.org/wiki/Extension:VisualEditor
+## VisualEditor Extension
+wfLoadExtension( 'VisualEditor' );
+wfLoadExtension( 'Parsoid', 'vendor/wikimedia/parsoid/extension.json' );

Docker/build/WikibaseBundle/LocalSettings.WikibaseManifest.php

@@ -0,0 +1,7 @@
+<?php
+
+// https://www.mediawiki.org/wiki/Extension:WikibaseManifest
+
+## WikibaseManifest Configuration
+$wgEnableRestAPI = true;
+wfLoadExtension( 'WikibaseManifest' );

Docker/build/WikibaseBundle/LocalSettings.cldr.php

@@ -0,0 +1,5 @@
+<?php
+
+// https://www.mediawiki.org/wiki/Extension:CLDR
+## CLDR Extension
+wfLoadExtension( 'cldr' );

Docker/publish/download_artifacts/download_artifacts.py

@@ -19,6 +19,14 @@
         print('Downloading %d bytes from %s' % (artifact['size_in_bytes'], artifact['name']))
 
         r = requests.get( artifact['archive_download_url'], allow_redirects=True, headers={"Authorization": 'Bearer ' + os.getenv('GITHUB_TOKEN') } )
+        if r.status_code != 200:
+            errorMessage = "Unknown error (http " + str(r.status_code) + ")."
+            try:
+                errorMessage = r.json()['message']
+            except:
+                pass
+            raise Exception("Github API request failed: " + errorMessage)
+
         zipfilePath = os.path.join('/zips' ,artifact['name'] + '.zip')
         open(zipfilePath, 'wb').write(r.content)
 

Docker/publish/tag_git/Dockerfile

@@ -1,9 +1,4 @@
 FROM docker-registry.wikimedia.org/wikimedia-buster:latest
 
-RUN apt-get update && \
-    apt-get install --yes --no-install-recommends git ca-certificates=201* && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
-
 COPY ./tag_git.sh /tag_git.sh
-RUN mkdir /repo
 ENTRYPOINT [ "/bin/bash", "/tag_git.sh" ]

Docker/publish/tag_git/tag_git.sh

@@ -15,44 +15,26 @@ if [ -z "$WIKIBASE_BRANCH_NAME" ] || \
 [ -z "$WMDE_RELEASE_VERSION" ] || \
 [ -z "$METADATA_WDQS_UI_COMMIT_HASH" ] || \
 [ -z "$METADATA_WIKIBASE_COMMIT_HASH" ] || \
+[ -z "$METADATA_WIKIBASEMANIFEST_COMMIT_HASH" ] || \
 [ -z "$WORKFLOW_RUN_NUMBER" ] ; then
     echo "A variable is required but isn't set. You should pass it to docker. See: https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file";
     exit 1;
 fi
 
-function tag_and_push {
-    REPOSITORY_CACHE_NAME=$1
-    CHECKOUT_DIR=$2
-    BRANCH_NAME=$3
-    COMMIT_HASH=$4
-    REMOTE_URL=$5
-
-    git clone --single-branch --branch "${BRANCH_NAME}" "$REPOSITORY_CACHE_NAME" "$CHECKOUT_DIR"
-    cd "$CHECKOUT_DIR"
-
-    echo "Tagging $WMDE_RELEASE_VERSION at $COMMIT_HASH"
-    git tag --force -a "$WMDE_RELEASE_VERSION" "$COMMIT_HASH" -m "Tagging: $WMDE_RELEASE_VERSION Build: $WORKFLOW_RUN_NUMBER"
-
-    if [ -z "$DRY_RUN" ]; then
-        git remote set-url origin "$REMOTE_URL"
-        git push --tags
-    else
-        echo "DRY RUN! Not pushing anything"
-    fi
-
-    cd -
+function echo_tag {
+    COMMIT_HASH=$1
+    REPO_NAME=$2
+    echo
+    echo "Use the following tag on $REPO_NAME"
+    echo "git tag --force -a \"$WMDE_RELEASE_VERSION\" \"$COMMIT_HASH\" -m \"Tagging: $WMDE_RELEASE_VERSION Build: $WORKFLOW_RUN_NUMBER\""
+    echo
 }
 
 # tag and push Wikibase
-tag_and_push "/git_cache/Wikibase.git" \
-    "/repo/Wikibase" \
-    "$WIKIBASE_BRANCH_NAME" \
-    "$METADATA_WIKIBASE_COMMIT_HASH" \
-    ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Wikibase
+echo_tag "$METADATA_WIKIBASE_COMMIT_HASH" "Wikibase"
 
 # tag and push queryservice ui
-tag_and_push "/git_cache/services/wikidata-query-gui.git" \
-    "/repo/wdqs-frontend" \
-    master \
-    "$METADATA_WDQS_UI_COMMIT_HASH" \
-    ssh://gerrit.wikimedia.org:29418/wikidata/query/gui
+echo_tag "$METADATA_WDQS_UI_COMMIT_HASH" "Queryservice UI"
+
+# tag and push WikibaseManifest
+echo_tag "$METADATA_WIKIBASEMANIFEST_COMMIT_HASH" "Wikibase Manifest"

Docker/test/selenium/data/entityschema.sx

@@ -0,0 +1,3 @@
+<something> {
+    wdt:P1 [wd:Q1]
+  }

Docker/test/selenium/helpers/default-functions.js

@@ -4,6 +4,7 @@ const axios = require( 'axios' );
 const assert = require( 'assert' );
 const exec = require( 'child_process' ).exec;
 const _ = require( 'lodash' );
+const MWBot = require( 'mwbot' );
 
 const defaultFunctions = function () {
 
@@ -14,6 +15,48 @@ const defaultFunctions = function () {
 		return axios.get( url );
 	} );
 
+	/**
+	 * Execute query on database
+	 */
+	browser.addCommand( 'dbQuery', function async( query, config ) {
+		if ( !config ) {
+			config = {
+				user: process.env.DB_USER,
+				pass: process.env.DB_PASS,
+				database: process.env.DB_NAME
+			};
+		}
+
+		return browser.dockerExecute(
+			process.env.DOCKER_MYSQL_NAME,
+			'mysql --user "' + config.user + '"' +
+			' --password="' + config.pass + '" "' + config.database + '"' +
+			" -e '" + query + "'"
+		);
+	} );
+
+	/**
+	 * Delete a claim by guid or pipe-separated list of guids
+	 */
+	browser.addCommand( 'deleteClaim', function async( claimGuid ) {
+		const bot = new MWBot( {
+			apiUrl: `${browser.config.baseUrl}/api.php`
+		} );
+
+		return new Promise( ( resolve ) => {
+			bot.getEditToken()
+				.then( () => {
+					bot.request( {
+						action: 'wbremoveclaims',
+						claim: claimGuid,
+						token: bot.editToken
+					} ).then( ( response ) => {
+						resolve( response );
+					} );
+				} );
+		} );
+	} );
+
 	/**
 	 * Execute docker command on container and get output
 	 */
@@ -43,10 +86,27 @@ const defaultFunctions = function () {
 	 */
 	browser.addCommand( 'editPage', function editPage( host, title, content ) {
 		browser.url( host + '/wiki/' + title + '?action=edit' );
+
+		// wait for javascript to settle
+		browser.pause( 5 * 1000 );
+
+		// this shows up one time for anonymous users (VisualEditor)
+		const startEditbutton = $( '.oo-ui-messageDialog-actions .oo-ui-flaggedElement-progressive' );
+		if ( startEditbutton.elementId ) {
+			startEditbutton.click();
+		}
+
+		// fill out form
 		$( '#wpTextbox1' ).waitForDisplayed();
 		$( '#wpTextbox1' ).setValue( content );
-		$( '#wpSave' ).click();
-		browser.pause( 1 * 1000 );
+
+		// save page
+		browser.execute( function () {
+			$( '#editform.mw-editform' ).submit();
+		}, this );
+
+		browser.pause( 2 * 1000 );
+
 		$( '#bodyContent' ).waitForDisplayed();
 		return $( '#bodyContent' ).getText();
 	} );

Docker/test/selenium/package.json

@@ -32,6 +32,7 @@
 		"test:repo-client": "wdio wdio.conf.js --spec specs/repo-client/*",
 		"test:fedprops": "wdio wdio.conf.js --spec specs/federated-properties/*",
 		"test:quickstatements": "wdio wdio.conf.js --spec specs/quickstatements/*",
+		"test:pingback": "wdio wdio.conf.js --spec specs/pingback/*",
 		"lint": "npx eslint .",
 		"fix": "npx eslint . --fix"
 	},

Docker/test/selenium/specs/federated-properties/item.js

@@ -33,8 +33,10 @@ describe( 'Fed props Item', function () {
 		browser.call( () => WikibaseApi.createItem( Util.getTestString( itemLabel ), data ) );
 
 		browser.url( process.env.MW_SERVER + '/wiki/Item:' + itemId );
-		$( '.wikibase-statementgroupview-property' ).getText().includes( propertyValue ); // value is the label
-		$( '.wikibase-snakview-value-container' ).getText().includes( propertyValue );
+
+		const actualPropertyValue = $( '.wikibase-statementgroupview-property' ).getText();
+		assert( actualPropertyValue.includes( propertyValue ) ); // value is the label
+
 		$( '.wikibase-toolbarbutton.wikibase-toolbar-item.wikibase-toolbar-button.wikibase-toolbar-button-add' ).waitForDisplayed();
 	} );