Support the azure-entities signature serialization format

libraries/postgres/README.md

@@ -315,8 +315,6 @@ The `ser` property, short for "serialization", is an integener defining the colu
 When a table migration occurs that adds a column to the signature, a new `ser` must be introduced, with the old version continuing to verify the old data.
 Note that removing a signed column is impossible without dropping all rows from the table, as otherwise existing signatures would become invalid.
 
-Care must be taken when constructing a serialization for data previously signed by taskcluster-lib-entities, as the serialization is done at the byte level and involves carefully formatted inputs for each column.
-
 ### Signing and Verification API
 
 Both signing and verification expect a row object an an array of functions for serialization of the row, indexed by `ser`:
@@ -345,6 +343,22 @@ const valid = db.verifySignature({
 });
 ```
 
+The `azureEntitiesSerialization` utility function performs part of the serialization operation used by azure-entities.
+It requires as input an object mapping property names to the *hashed* version of their values, where the hashes are defined by each entity value type.
+The function sorts the properties and encodes them using the 32-bit run-length-encoding introduced by azure-entities.
+
+```javascript
+const signingSerializations = [
+  row => azureEntitiesSerialization({
+    // strings hash as themselves
+    id: row.id,
+    name: row.name,
+    // hash value of an integer column is the string form of the integer
+    count: row.count.toString(),
+  }),
+];
+```
+
 ### Updating Tables With New Keys or New Serializations
 
 Every service with signed data should have a periodic task that updates all rows of the table to use the current key, version, and serialization version.

libraries/postgres/src/index.js

@@ -1,7 +1,9 @@
+const {pick} = require('lodash');
+
 module.exports = {
   Schema: require('./Schema'),
   Database: require('./Database'),
   ...require('./constants'),
   Keyring: require('./Keyring'),
-  ignorePgErrors: require('./util').ignorePgErrors,
+  ...pick(require('./util'), ['ignorePgErrors', 'azureEntitiesSerialization']),
 };

libraries/postgres/src/util.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const path = require('path');
+const assert = require('assert');
 const {SYNTAX_ERROR} = require('./constants');
 
 exports.ignorePgErrors = async (promise, ...codes) => {
@@ -76,3 +77,40 @@ exports.annotateError = (query, err) => {
     }
   }
 };
+
+/**
+ * Utility function to serialize the way azure-entities always did
+ */
+exports.azureEntitiesSerialization = hashedValues => {
+  const res = [];
+
+  // Order keys for consistency
+  const keys = Object.keys(hashedValues).sort();
+
+  const n = keys.length;
+  for (let i = 0; i < n; i++) {
+    const property = keys[i];
+    const value = hashedValues[property];
+    assert.equal(typeof value, 'string', `${property} value is not a string`);
+
+    // Hash [uint32 - len(property)] [bytes - property]
+    const propLenBuf = Buffer.alloc(4);
+    propLenBuf.writeUInt32BE(Buffer.byteLength(property, 'utf8'), 0);
+    res.push(propLenBuf);
+    res.push(Buffer.from(property, 'utf8'));
+
+    // Hash [uint32 - len(value)] [bytes - value]
+    let len;
+    if (typeof value === 'string') {
+      len = Buffer.byteLength(value, 'utf8');
+    } else {
+      len = value.length;
+    }
+    const valLenBuf = Buffer.alloc(4);
+    valLenBuf.writeUInt32BE(len, 0);
+    res.push(valLenBuf);
+    res.push(Buffer.from(value, 'utf8'));
+  }
+
+  return Buffer.concat(res);
+};

libraries/postgres/test/database_test.js

@@ -2,6 +2,7 @@ const helper = require('./helper');
 const {
   Schema,
   Database,
+  azureEntitiesSerialization,
   READ,
   WRITE,
   DUPLICATE_OBJECT,
@@ -788,7 +789,7 @@ helper.dbSuite(path.basename(__filename), function() {
   });
 
   suite('signing', function() {
-    const azureSigningKey = 'top-secret';
+    const azureSigningKey = 'no-way-you-can-guess-this';
     const pgSigningKey = 'classified';
 
     const signingSerializations = [
@@ -807,7 +808,7 @@ helper.dbSuite(path.basename(__filename), function() {
       assert.equal(signature.v, 0);
       assert.equal(signature.ser, 1);
       assert.equal(signature.kid, 'azure');
-      assert.equal(signature.sig, 'A2kh+yXvCqNYF7fL0Nhyo+TcnE6oXsN0k3aylLUzUOtS4PsNj82V+bd7BsLKxNePJ347VbMRgqwUKc3A5xh6vQ==');
+      assert.equal(signature.sig, 'zTZt8KxusLG+9mMULV9YjWsMsOU337I1D43kxkl9yur/aGrRAMdp9Yqn4nPpmXpElPbuiEBXH9lUMJzYexOnEg==');
 
       assert(db.verifySignature({row, signature, signingSerializations}));
     });
@@ -825,7 +826,7 @@ helper.dbSuite(path.basename(__filename), function() {
       assert.equal(signature.v, 0);
       assert.equal(signature.ser, 0);
       assert.equal(signature.kid, 'azure');
-      assert.equal(signature.sig, 'yfsG8ASoKihprcz49EGA6sGQdKoslKzp8LPoaZ/lmYyO8QS3TwQnWOYUErBSMhlcUVz5S8JVcJhTNkroy0i4+g==');
+      assert.equal(signature.sig, 'RmFt8jIVO2mR0YgQBSCJTzNV2uCTyJnm0kiNpOuekV9STHHC1sAvwXxNONKolpHmMgELxzun7PW9ytQUoXs5qg==');
 
       assert(db.verifySignature({row, signature, signingSerializations}));
     });
@@ -836,7 +837,7 @@ helper.dbSuite(path.basename(__filename), function() {
       assert.equal(signature.v, 0);
       assert.equal(signature.ser, 1);
       assert.equal(signature.kid, 'azure');
-      assert.equal(signature.sig, 'A2kh+yXvCqNYF7fL0Nhyo+TcnE6oXsN0k3aylLUzUOtS4PsNj82V+bd7BsLKxNePJ347VbMRgqwUKc3A5xh6vQ==');
+      assert.equal(signature.sig, 'zTZt8KxusLG+9mMULV9YjWsMsOU337I1D43kxkl9yur/aGrRAMdp9Yqn4nPpmXpElPbuiEBXH9lUMJzYexOnEg==');
 
       // This new_db does not have azureSigningKey as it's current key but can still read old encryptions
       const new_db = await Database.setup({schema, readDbUrl: helper.dbUrl, writeDbUrl: helper.dbUrl,
@@ -845,5 +846,24 @@ helper.dbSuite(path.basename(__filename), function() {
         ]});
       assert(new_db.verifySignature({row, signature, signingSerializations}));
     });
+
+    test('azure-compatible signing produces the same signature', async function() {
+      // this is taken from the azure-entities tests
+      const row = {
+        id: 'ZqZrh4PeQp6eS6alJNANLg',
+        name: 'stable entity',
+        count: 42,
+      };
+      const signingSerializations = [
+        row => azureEntitiesSerialization({
+          id: row.id,
+          name: row.name,
+          count: row.count.toString(),
+        }),
+      ];
+      const signature = db.sign({row, signingSerializations});
+      assert.equal(signature.sig,
+        'Ngc8HXokZRUuUJadEPtlYXbDPrV/C52eCR6aviiyLtaxvaV1LrWy0tFOjx0LzsiCd2Tq2dciEtL65cIfK8ohTQ==');
+    });
   });
 });