Merge commit '42951edfd4edd68f112077ba05078934e8a6b8f6'

* commit '42951edfd4edd68f112077ba05078934e8a6b8f6': (62 commits)
  Poke interwiki name and source name
  Use assert.deepStrictEqual
  $ is webdriver, not jqeury /o\
  mariadb:10.9
  Use 1.37.6
  Try mw-content-text for edit page content
  Use assert package consistently
  Fix test: entityschema.js
  Use assert.strictEqual in tests
  debug logging of mw in tests
  mwlogs, continue on error & cleaner ls output
  mwlog, more ls output, and no /
  Poke the logging again...
  Try outputting mw logs again
  fix: Fix commandline argument passing to runBlazegraph.sh (wmde#364)
  Add wgWBClientSettings['itemAndPropertySourceName']
  Revert "Revert "Revert "Output MediaWiki logs in CI too"""
  Revert "More general path for mwlog output"
  More general path for mwlog output
  Revert "Revert "Output MediaWiki logs in CI too""
  ...

.github/workflows/lint.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: c-py/action-dotenv-to-setenv@v3
+      - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: variables.env
       - name: cache
@@ -43,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: c-py/action-dotenv-to-setenv@v3
+      - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: variables.env
       - name: cache

.github/workflows/publish.yml

@@ -22,10 +22,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: c-py/action-dotenv-to-setenv@v3
+      - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: variables.env
-      - uses: c-py/action-dotenv-to-setenv@v3
+      - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: ${{ env.env_file }}
       - name: run the publish script

.github/workflows/test.yml

@@ -17,10 +17,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        databaseImageName: [ 'mariadb:10.3' ] # 'mysql:5.6' disabled https://phabricator.wikimedia.org/T296066
+        databaseImageName: [ 'mariadb:10.9' ] # 'mysql:5.6' disabled https://phabricator.wikimedia.org/T296066
         suite: [
           repo,
-          fedprops,
+          # Disable the fed props tests for now, as the feature has changed
+          #fedprops,
           repo_client,
           quickstatements,
           pingback,
@@ -30,16 +31,17 @@ jobs:
           base__repo,
           base__repo_client,
           base__pingback,
-          base__fedprops,
+          # Disable the fed props tests for now, as the feature has changed
+          #base__fedprops,
         ]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v3
-      - uses: c-py/action-dotenv-to-setenv@v3
+      - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: variables.env
-      - uses: c-py/action-dotenv-to-setenv@v3
+      - uses: c-py/action-dotenv-to-setenv@v4
         with:
           env-file: ${{ env.env_file }}
 
@@ -61,7 +63,7 @@ jobs:
           SUITE=${{ matrix.suite }} node report.js
 
       - name: Archive docker test artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: always()
         with:
           name: TestArtifacts

.github/workflows/test_example.yml

@@ -20,7 +20,7 @@ jobs:
         run: make test-example SUITE=example
 
       - name: Archive test example artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: always()
         with:
           name: TestExampleArtifacts

CHANGES.md

@@ -4,6 +4,28 @@ This file is only intended to serve the developers of this repository and of the
 
 This provides an overview of the releases that have been made using this release pipeline.
 
+## WORK IN PROGRESS October 2022: First 1.37 release
+
+The [standard MediaWiki upgrade process](https://www.mediawiki.org/wiki/Manual:Upgrading) can be followed for this release.
+
+- [MediaWiki release notes](https://www.mediawiki.org/wiki/Release_notes/1.37)
+- [Wikibase release notes](https://github.com/wikimedia/Wikibase/blob/REL1_37/RELEASE-NOTES-1.37)
+
+The Query service was also updated from `0.3.97` to `0.3.115`, but no additional upgrade process is needed.
+
+| Suite Version | Version Info | Date availbile | Run number | Release task  |
+|---------------|----------------------|----------------|------------| --------------|
+| wmde.9 ([spec](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.9/versions/wmde9.env),[metadata](https://github.com/wmde/wikibase-release-pipeline/blob/main/versions/wmde9)) | MediaWiki (1.37.6, PHP 7.4.30), WDQS (0.3.115)    | TODO | [TODO](https://github.com/wmde/wikibase-release-pipeline/actions/runs/TODO) | [T317890](https://phabricator.wikimedia.org/T317890) |
+
+## September 2022: Security releases
+
+| Suite Version | MediaWiki release | Date availbile | Run number | Release task  |
+|---------------|----------------------|----------------|------------| --------------|
+| wmde.8 ([spec](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.8/versions/wmde8.env),[metadata](https://github.com/wmde/wikibase-release-pipeline/blob/ec8b9e0bb25b9524df5671ad7785a66683598920/versions/wmde8)) | 1.36.4    | September 2022 | [2971822356](https://github.com/wmde/wikibase-release-pipeline/actions/runs/2971822356) | [T316707](https://phabricator.wikimedia.org/T316707) |
+| wmde.7 ([spec](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.7/versions/wmde7.env),[metadata](https://github.com/wmde/wikibase-release-pipeline/blob/ec8b9e0bb25b9524df5671ad7785a66683598920/versions/wmde7)) | 1.35.7    | September 2022 | [2971057584](https://github.com/wmde/wikibase-release-pipeline/actions/runs/2971057584) | [T314881](https://phabricator.wikimedia.org/T314881) |
+
+Releases that bring a handfull of small security fixes to MediaWiki & Wikibase, along with small patches to other components.
+
 ## February 2022: Security releases
 
 | Suite Version | Compatable MediaWiki | Date availbile | Run number | Release task |

Docker/build/QuickStatements/entrypoint.sh

@@ -9,7 +9,7 @@ for i in "${REQUIRED_VARIABLES[@]}"; do
     fi
 done
 
-if [[ -v "$OAUTH_CONSUMER_KEY" && "$OAUTH_CONSUMER_SECRET" ]]; then
+if [[ -v OAUTH_CONSUMER_KEY && -v OAUTH_CONSUMER_SECRET ]]; then
     envsubst < /templates/oauth.ini > /quickstatements/data/oauth.ini;
 fi
 

Docker/build/WDQS-frontend/Dockerfile

@@ -17,9 +17,7 @@ RUN apk --no-cache add --virtual build-dependencies ca-certificates git jq make
 
 # TODO do npm build instead of leaving any dev node modules hanging around
 RUN mv package.json package.json.orig \
-    && jq 'delpaths([["devDependencies","karma-qunit"],["devDependencies","qunitjs"],["devDependencies","sinon"]])' \
-        > package.json < package.json.orig \
-    && jq 'setpath(["devDependencies","less"]; "~2.7.1")' \
+    && jq 'delpaths([["devDependencies","karma-qunit"],["devDependencies","sinon"]])' \
         > package.json < package.json.orig \
     && npm install \
     && npm run build

Docker/build/WDQS/Dockerfile

@@ -29,7 +29,8 @@ ENV MEMORY=""\
     HOST="0.0.0.0"\
     WDQS_ENTITY_NAMESPACES="120,122"\
     WIKIBASE_SCHEME="http"\
-    WIKIBASE_MAX_DAYS_BACK="90"
+    WIKIBASE_MAX_DAYS_BACK="90"\
+    BLAZEGRAPH_EXTRA_OPTS=""
 
 WORKDIR /wdqs
 

Docker/build/WDQS/README.md

@@ -6,7 +6,7 @@ Wikibase specific Blazegraph image.
 
 [WDQS](https://gerrit.wikimedia.org/r/admin/repos/wikidata/query/rdf) exposes by default some endpoints and methods that reveal internal details or functionality that might allow for abuse of the system. With the example docker configuration we are using the [WDQS-proxy](../WDQS-proxy/README.md) which filters out all long-running or unwanted requests.
 
-When running WDQS in a setup without WDQS-proxy **please consider disabling these endpoints in some other way**. For more information on how this is tested in this pipeline see the test-cases in [queryservice.js](../../test/selenium/specs/repo/queryservice.js) 
+When running WDQS in a setup without WDQS-proxy **please consider disabling these endpoints in some other way**. For more information on how this is tested in this pipeline see the test-cases in [queryservice.js](../../test/selenium/specs/repo/queryservice.js)
 
 ### Upgrading
 
@@ -34,7 +34,9 @@ Variable                 | Default            | Since   | Description
 `WDQS_PORT`              | "9999"             | 0.2.5   | Port of the WDQS host (this service)
 `WDQS_ENTITY_NAMESPACES` | "120,122"          | 0.2.5   | Wikibase Namespaces to load data from
 `WIKIBASE_MAX_DAYS_BACK` | "90"               | 0.3.0   | Max days updater is allowed back from now
+`BLAZEGRAPH_EXTRA_OPTS`  | ""                 | wmde.9  | Extra options to be passed to Blazegraph
 
+Note on `BLAZEGRAPH_EXTRA_OPTS`: These are options that are directly passed to the Blazegraph jar. That means they must be prefixed with `-D`. Example: `-Dhttps.proxyHost=http://my.proxy.com -Dhttps.proxyPort=3128`. See [the Wikidata Query Service User Manual](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Configurable_properties) for all available options.
 
 ### Filesystem layout
 

Docker/build/WDQS/entrypoint.sh

@@ -13,7 +13,7 @@ done
 
 set -eu
 
-export BLAZEGRAPH_OPTS="-DwikibaseHost=${WIKIBASE_HOST}"
+export BLAZEGRAPH_OPTS="${BLAZEGRAPH_EXTRA_OPTS} -DwikibaseHost=${WIKIBASE_HOST}"
 export UPDATER_OPTS="-DwikibaseHost=${WIKIBASE_HOST} -DwikibaseMaxDaysBack=${WIKIBASE_MAX_DAYS_BACK}"
 
 envsubst < /templates/mwservices.json > /wdqs/mwservices.json

Docker/build/WDQS/runBlazegraph.sh

@@ -3,4 +3,4 @@
 
 cd /wdqs || exit
 
-./runBlazegraph.sh
+./runBlazegraph.sh "$@"

Docker/build/Wikibase/LocalSettings.php.REL1_35.template

@@ -21,6 +21,7 @@ ${DOLLAR}wgDebugLogGroups = array(
 	'resourceloader' => '/var/log/mediawiki/resourceloader.log',
 	'exception' => '/var/log/mediawiki/exception.log',
 	'error' => '/var/log/mediawiki/error.log',
+	'fatal' => '/var/log/mediawiki/fatal.log',
 );
 
 ## Site Settings

Docker/build/Wikibase/LocalSettings.php.template

@@ -21,6 +21,7 @@ ${DOLLAR}wgDebugLogGroups = array(
 	'resourceloader' => '/var/log/mediawiki/resourceloader.log',
 	'exception' => '/var/log/mediawiki/exception.log',
 	'error' => '/var/log/mediawiki/error.log',
+	'fatal' => '/var/log/mediawiki/fatal.log',
 );
 
 ## Site Settings

Docker/test/selenium/README.md

@@ -63,7 +63,7 @@ The behavior of the tests can be modified with several environment variables.
 When working on the browser tests,
 you’ll want to consult the documentation of the following libraries we use:
 
-* [WebdriverIO](http://webdriver.io/api.html) for controlling the browser
+* [WebdriverIO](https://webdriver.io/docs/api) for controlling the browser
   (`browser`, `$`, `waitForVisible`, …)
 * [Mocha](https://mochajs.org/) as the general testing framework
   (`describe`, `it`, `before`, …)

Docker/test/selenium/helpers/default-functions.js

@@ -133,8 +133,8 @@ const defaultFunctions = function () {
 
 		browser.pause( 2 * 1000 );
 
-		$( '#bodyContent' ).waitForDisplayed();
-		return $( '#bodyContent' ).getText();
+		$( '#mw-content-text' ).waitForDisplayed();
+		return $( '#mw-content-text' ).getText();
 	} );
 
 	/**
@@ -150,7 +150,7 @@ const defaultFunctions = function () {
 		const changes = result.data.query.recentchanges;
 		const foundResult = _.find( changes, expectedChange );
 
-		assert( result.status === 200 );
+		assert.strictEqual( result.status, 200 );
 
 		if ( !foundResult ) {
 			console.error( 'Could not find:' );

Docker/test/selenium/specs/confirm_edit/confirm-edit.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { assert } = require( 'console' );
+const assert = require( 'assert' );
 
 describe( 'ConfirmEdit', function () {
 
@@ -13,7 +13,7 @@ describe( 'ConfirmEdit', function () {
 			'paris'
 		);
 
-		assert( executionResult === 'something great' );
+		assert.strictEqual( executionResult, 'something great' );
 
 	} );
 

Docker/test/selenium/specs/elasticsearch/elasticsearch.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { assert } = require( 'console' );
+const assert = require( 'assert' );
 const Util = require( 'wdio-mediawiki/Util' );
 const WikibaseApi = require( 'wdio-wikibase/wikibase.api' );
 
@@ -37,7 +37,7 @@ describe( 'ElasticSearch', function () {
 		// alias should be visible on item page
 		$( '.wikibase-aliasesview-list-item' ).waitForDisplayed();
 		const alias = $( '.wikibase-aliasesview-list-item' ).getText();
-		assert( alias === itemAlias );
+		assert.strictEqual( alias, itemAlias );
 	} );
 
 	it( 'should be able to search case-insensitive', async function () {

Docker/test/selenium/specs/fedprops/item.js

@@ -20,7 +20,7 @@ describe( 'Fed props Item', function () {
 		const success = result.data.success;
 		const searchResults = result.data.search;
 
-		assert( success === 1 );
+		assert.strictEqual( success, 1 );
 		assert( searchResults.length > 0 );
 	} );
 

Docker/test/selenium/specs/fedprops/prefetching.js

@@ -53,11 +53,11 @@ describe( 'Property Prefetching', function () {
 			propertyGuids.push( statement.getAttribute( 'id' ) );
 		} );
 
-		assert( propertyGuids.length === NUM_PROPERTIES );
+		assert.strictEqual( propertyGuids.length, NUM_PROPERTIES );
 
 		propertyGuids.forEach( ( guid ) => {
 			const response = browser.deleteClaim( guid );
-			assert( response.success === 1 );
+			assert.strictEqual( response.success, 1 );
 		} );
 	} );
 

Docker/test/selenium/specs/pingback/pingback.js

@@ -11,17 +11,17 @@ describe( 'Pingback', function () {
 		browser.pause( 5 * 1000 );
 
 		const sqlResult = browser.dbQuery( 'SELECT * from updatelog where ul_key LIKE "WikibasePingback%"' );
-		assert( sqlResult.includes( 'WikibasePingback\t' ) === true );
-		assert( sqlResult.includes( 'WikibasePingback-1.' ) === true );
+		assert.strictEqual( sqlResult.includes( 'WikibasePingback\t' ), true );
+		assert.strictEqual( sqlResult.includes( 'WikibasePingback-1.' ), true );
 
 		const result = browser.makeRequest( process.env.PINGBACK_BEACON_SERVER );
-		assert( result.data.length === 2 );
+		assert.strictEqual( result.data.length, 2 );
 
 		const requestData = JSON.parse(
 			Object.keys( result.data[ 0 ] )[ 0 ].replace( ';', '' )
 		);
 
-		assert( requestData.schema === 'WikibasePingback' );
+		assert.strictEqual( requestData.schema, 'WikibasePingback' );
 	} );
 
 } );

Docker/test/selenium/specs/quickstatements/quickstatements.js

@@ -60,8 +60,8 @@ describe( 'QuickStatements Service', function () {
 		const responseQ1 = browser.makeRequest( process.env.MW_SERVER + '/wiki/Special:EntityData/Q1.json' );
 		const responseQ2 = browser.makeRequest( process.env.MW_SERVER + '/wiki/Special:EntityData/Q2.json' );
 
-		assert( responseQ1.data.entities.Q1.id === 'Q1' );
-		assert( responseQ2.data.entities.Q2.id === 'Q2' );
+		assert.strictEqual( responseQ1.data.entities.Q1.id, 'Q1' );
+		assert.strictEqual( responseQ2.data.entities.Q2.id, 'Q2' );
 	} );
 
 	it( 'Should be able to add an alias to an item', function () {
@@ -111,7 +111,7 @@ describe( 'QuickStatements Service', function () {
 		browser.executeQuickStatement( 'Q1|' + propertyId + '|"Will it blend?"' );
 
 		const responseQ1 = browser.makeRequest( process.env.MW_SERVER + '/wiki/Special:EntityData/Q1.json' );
-		assert( responseQ1.data.entities.Q1.claims[ propertyId ][ 0 ].type === 'statement' );
+		assert.strictEqual( responseQ1.data.entities.Q1.claims[ propertyId ][ 0 ].type, 'statement' );
 
 	} );
 
@@ -168,7 +168,7 @@ describe( 'QuickStatements Service', function () {
 		browser.executeQuickStatement( 'Q1|' + propertyIdItem + '|Q1|' + propertyIdItem + '|Q1' );
 
 		const responseQ1 = browser.makeRequest( process.env.MW_SERVER + '/wiki/Special:EntityData/Q1.json' );
-		assert( responseQ1.data.entities.Q1.claims[ propertyId ][ 0 ].type === 'statement' );
+		assert.strictEqual( responseQ1.data.entities.Q1.claims[ propertyId ][ 0 ].type, 'statement' );
 
 	} );
 
@@ -183,7 +183,7 @@ describe( 'QuickStatements Service', function () {
 		const response = browser.makeRequest( process.env.MW_SERVER + '/w/api.php?action=wbgetclaims&format=json&entity=' + itemId );
 		const refValue = getReferenceValue( response, propertyIdItem, propertyIdItem );
 
-		assert( refValue.id === 'Q2' );
+		assert.strictEqual( refValue.id, 'Q2' );
 	} );
 
 	it( 'Should be able to add a property with "url" reference', function () {
@@ -198,7 +198,7 @@ describe( 'QuickStatements Service', function () {
 		const response = browser.makeRequest( process.env.MW_SERVER + '/w/api.php?action=wbgetclaims&format=json&entity=' + itemId );
 		const refValue = getReferenceValue( response, propertyIdItem, propertyURL );
 
-		assert( refValue === 'https://www.wikidata.org' );
+		assert.strictEqual( refValue, 'https://www.wikidata.org' );
 	} );
 
 	it( 'Should be able to add a property with "string" reference', function () {
@@ -211,7 +211,7 @@ describe( 'QuickStatements Service', function () {
 		const response = browser.makeRequest( process.env.MW_SERVER + '/w/api.php?action=wbgetclaims&format=json&entity=' + itemId );
 		const refValue = getReferenceValue( response, propertyIdItem, propertyId );
 
-		assert( refValue === 'some string' );
+		assert.strictEqual( refValue, 'some string' );
 	} );
 
 	it( 'Should be able to add and remove a property on an item', function () {
@@ -221,12 +221,12 @@ describe( 'QuickStatements Service', function () {
 		browser.executeQuickStatement( itemId + '|' + propertyIdItem + '|Q1' );
 
 		let response = browser.makeRequest( process.env.MW_SERVER + '/wiki/Special:EntityData/' + itemId + '.json' );
-		assert( ( propertyIdItem in response.data.entities[ itemId ].claims ) === true );
+		assert.strictEqual( ( propertyIdItem in response.data.entities[ itemId ].claims ), true );
 
 		browser.executeQuickStatement( '-' + itemId + '|' + propertyIdItem + '|Q1' );
 
 		response = browser.makeRequest( process.env.MW_SERVER + '/wiki/Special:EntityData/' + itemId + '.json' );
-		assert( ( propertyIdItem in response.data.entities[ itemId ].claims ) === false );
+		assert.strictEqual( ( propertyIdItem in response.data.entities[ itemId ].claims ), false );
 
 	} );
 
@@ -237,7 +237,7 @@ describe( 'QuickStatements Service', function () {
 		browser.executeQuickStatement( 'MERGE|Q1|Q2' );
 
 		const responseQ2 = browser.makeRequest( process.env.MW_SERVER + '/wiki/Special:EntityData/Q2.json' );
-		assert( responseQ2.data.entities.Q1.id === 'Q1' );
+		assert.strictEqual( responseQ2.data.entities.Q1.id, 'Q1' );
 	} );
 
 } );

Docker/test/selenium/specs/repo/api.js

@@ -30,8 +30,8 @@ describe( 'Wikibase API', function () {
 				() => WikibaseApi.createItem( Util.getTestString( itemLabel ), data )
 			);
 
-			assert( itemId.startsWith( 'Q' ) === true );
-			assert( propertyId.startsWith( 'P' ) === true );
+			assert.strictEqual( itemId.startsWith( 'Q' ), true );
+			assert.strictEqual( propertyId.startsWith( 'P' ), true );
 		}
 
 	} );