FIR-9: Identity enhancements

Makefile

@@ -31,43 +31,43 @@ mocks-$(strip $(1))-$(strip $(2)): ${MOCKERY}
 	${MOCKERY} --case underscore --dir $(1) --name $(2) --outpkg $(3) --output mocks/$(strip $(3))
 endef
 
-$(eval $(call makemock, pkg/blockchain,            Plugin,             blockchainmocks))
-$(eval $(call makemock, pkg/blockchain,            Callbacks,          blockchainmocks))
-$(eval $(call makemock, pkg/database,              Plugin,             databasemocks))
-$(eval $(call makemock, pkg/database,              Callbacks,          databasemocks))
-$(eval $(call makemock, pkg/publicstorage,         Plugin,             publicstoragemocks))
-$(eval $(call makemock, pkg/publicstorage,         Callbacks,          publicstoragemocks))
-$(eval $(call makemock, pkg/events,                Plugin,             eventsmocks))
-$(eval $(call makemock, pkg/events,                PluginAll,          eventsmocks))
-$(eval $(call makemock, pkg/events,                Callbacks,          eventsmocks))
-$(eval $(call makemock, pkg/identity,              Plugin,             identitymocks))
-$(eval $(call makemock, pkg/identity,              Callbacks,          identitymocks))
-$(eval $(call makemock, pkg/dataexchange,          Plugin,             dataexchangemocks))
-$(eval $(call makemock, pkg/dataexchange,          Callbacks,          dataexchangemocks))
-$(eval $(call makemock, pkg/tokens,                Plugin,             tokenmocks))
-$(eval $(call makemock, pkg/tokens,                Callbacks,          tokenmocks))
-$(eval $(call makemock, pkg/wsclient,              WSClient,           wsmocks))
-$(eval $(call makemock, internal/txcommon,         Helper,             txcommonmocks))
-$(eval $(call makemock, internal/identity,         Manager,            identitymanagermocks))
-$(eval $(call makemock, internal/batchpin,         Submitter,          batchpinmocks))
-$(eval $(call makemock, internal/sysmessaging,     SystemEvents,       sysmessagingmocks))
-$(eval $(call makemock, internal/sysmessaging,     MessageSender,      sysmessagingmocks))
-$(eval $(call makemock, internal/sysmessaging,     LocalNodeInfo,      sysmessagingmocks))
-$(eval $(call makemock, internal/syncasync,        Bridge,             syncasyncmocks))
-$(eval $(call makemock, internal/data,             Manager,            datamocks))
-$(eval $(call makemock, internal/batch,            Manager,            batchmocks))
-$(eval $(call makemock, internal/broadcast,        Manager,            broadcastmocks))
-$(eval $(call makemock, internal/privatemessaging, Manager,            privatemessagingmocks))
-$(eval $(call makemock, internal/definitions,      DefinitionHandlers, definitionsmocks))
-$(eval $(call makemock, internal/events,           EventManager,       eventmocks))
-$(eval $(call makemock, internal/networkmap,       Manager,            networkmapmocks))
-$(eval $(call makemock, internal/assets,           Manager,            assetmocks))
-$(eval $(call makemock, internal/contracts,        Manager,            contractmocks))
-$(eval $(call makemock, internal/oapiffi,          FFISwaggerGen,      oapiffimocks))
-$(eval $(call makemock, internal/orchestrator,     Orchestrator,       orchestratormocks))
-$(eval $(call makemock, internal/apiserver,        Server,             apiservermocks))
-$(eval $(call makemock, internal/apiserver,        IServer,            apiservermocks))
-$(eval $(call makemock, internal/metrics,          Manager,            metricsmocks))
+$(eval $(call makemock, pkg/blockchain,            Plugin,               blockchainmocks))
+$(eval $(call makemock, pkg/blockchain,            Callbacks,            blockchainmocks))
+$(eval $(call makemock, pkg/database,              Plugin,               databasemocks))
+$(eval $(call makemock, pkg/database,              Callbacks,            databasemocks))
+$(eval $(call makemock, pkg/publicstorage,         Plugin,               publicstoragemocks))
+$(eval $(call makemock, pkg/publicstorage,         Callbacks,            publicstoragemocks))
+$(eval $(call makemock, pkg/events,                Plugin,               eventsmocks))
+$(eval $(call makemock, pkg/events,                PluginAll,            eventsmocks))
+$(eval $(call makemock, pkg/events,                Callbacks,            eventsmocks))
+$(eval $(call makemock, pkg/identity,              Plugin,               identitymocks))
+$(eval $(call makemock, pkg/identity,              Callbacks,            identitymocks))
+$(eval $(call makemock, pkg/dataexchange,          Plugin,               dataexchangemocks))
+$(eval $(call makemock, pkg/dataexchange,          Callbacks,            dataexchangemocks))
+$(eval $(call makemock, pkg/tokens,                Plugin,               tokenmocks))
+$(eval $(call makemock, pkg/tokens,                Callbacks,            tokenmocks))
+$(eval $(call makemock, pkg/wsclient,              WSClient,             wsmocks))
+$(eval $(call makemock, internal/txcommon,         Helper,               txcommonmocks))
+$(eval $(call makemock, internal/identity,         Manager,              identitymanagermocks))
+$(eval $(call makemock, internal/batchpin,         Submitter,            batchpinmocks))
+$(eval $(call makemock, internal/sysmessaging,     SystemEvents,         sysmessagingmocks))
+$(eval $(call makemock, internal/sysmessaging,     MessageSender,        sysmessagingmocks))
+$(eval $(call makemock, internal/sysmessaging,     LocalNodeInfo,        sysmessagingmocks))
+$(eval $(call makemock, internal/syncasync,        Bridge,               syncasyncmocks))
+$(eval $(call makemock, internal/data,             Manager,              datamocks))
+$(eval $(call makemock, internal/batch,            Manager,              batchmocks))
+$(eval $(call makemock, internal/broadcast,        Manager,              broadcastmocks))
+$(eval $(call makemock, internal/privatemessaging, Manager,              privatemessagingmocks))
+$(eval $(call makemock, internal/contracts,        Manager,              contractmocks))
+$(eval $(call makemock, internal/assets,           Manager,              assetmocks))
+$(eval $(call makemock, internal/definitions,      DefinitionHandlers,   definitionsmocks))
+$(eval $(call makemock, internal/metrics,          Manager,              metricsmocks))
+$(eval $(call makemock, internal/events,           EventManager,         eventmocks))
+$(eval $(call makemock, internal/networkmap,       Manager,              networkmapmocks))
+$(eval $(call makemock, internal/oapiffi,          FFISwaggerGen,        oapiffimocks))
+$(eval $(call makemock, internal/orchestrator,     Orchestrator,         orchestratormocks))
+$(eval $(call makemock, internal/apiserver,        Server,               apiservermocks))
+$(eval $(call makemock, internal/apiserver,        IServer,              apiservermocks))
 
 firefly-nocgo: ${GOFILES}
 		CGO_ENABLED=0 $(VGO) build -o ${BINARY_NAME}-nocgo -ldflags "-X main.buildDate=`date -u +\"%Y-%m-%dT%H:%M:%SZ\"` -X main.buildVersion=$(BUILD_VERSION)" -tags=prod -tags=prod -v

db/migrations/postgres/000063_create_identities_table.down.sql

@@ -0,0 +1,97 @@
+BEGIN;
+
+CREATE TABLE orgs (
+  seq            SERIAL          PRIMARY KEY,
+  id             UUID            NOT NULL,
+  message_id     UUID            NOT NULL,
+  name           VARCHAR(64)     NOT NULL,
+  parent         VARCHAR(1024),
+  identity       VARCHAR(1024)   NOT NULL,
+  description    VARCHAR(4096)   NOT NULL,
+  profile        TEXT,
+  created        BIGINT          NOT NULL
+);
+
+CREATE UNIQUE INDEX orgs_id ON orgs(id);
+CREATE UNIQUE INDEX orgs_identity ON orgs(identity);
+CREATE UNIQUE INDEX orgs_name ON orgs(name);
+
+CREATE TABLE nodes (
+  seq            SERIAL          PRIMARY KEY,
+  id             UUID            NOT NULL,  
+  message_id     UUID            NOT NULL,
+  owner          VARCHAR(1024)   NOT NULL,
+  name           VARCHAR(64)     NOT NULL,
+  description    VARCHAR(4096)   NOT NULL,
+  dx_peer        VARCHAR(256),
+  dx_endpoint    TEXT,
+  created        BIGINT          NOT NULL
+);
+
+CREATE UNIQUE INDEX nodes_id ON nodes(id);
+CREATE UNIQUE INDEX nodes_owner ON nodes(owner,name);
+CREATE UNIQUE INDEX nodes_peer ON nodes(dx_peer);
+
+-- We only reconstitute orgs that were dropped during the original up migration.
+-- These have the UUID of the verifier set to the same UUID as the org.
+INSERT INTO orgs (
+    id,
+    parent,
+    message_id,
+    name,
+    description,
+    profile,
+    created,
+    identity
+  ) SELECT 
+    i.id,
+    COALESCE(pv.value, '') as parent,
+    i.messages_claim,
+    i.name,
+    i.description,
+    i.profile,
+    i.created,
+    v.value as identity
+  FROM identities as i
+  LEFT JOIN verifiers v ON v.hash = REPLACE(i.id::text,'-','') || REPLACE(i.id::text,'-','')
+  LEFT JOIN verifiers pv ON pv.hash = REPLACE(i.parent::text,'-','') ||  REPLACE(i.parent::text,'-','')
+  WHERE i.did LIKE 'did:firefly:org/%' AND v.hash IS NOT NULL;
+
+-- We only reconstitute nodes that were dropped during the original up migration.
+-- These have the Hash of the verifier set to the bytes from the UUID of the node (by taking the string and removing the dashes).
+INSERT INTO nodes (
+    id,
+    owner,
+    message_id,
+    name,
+    description,
+    dx_endpoint,
+    created,
+    dx_peer
+  ) SELECT 
+    i.id,
+    COALESCE(pv.value, '') as owner,
+    i.messages_claim,
+    i.name,
+    i.description,
+    i.profile,
+    i.created,
+    v.value as dx_peer
+  FROM identities as i
+  LEFT JOIN verifiers v ON v.hash = REPLACE(i.id::text,'-','') || REPLACE(i.id::text,'-','')
+  LEFT JOIN verifiers pv ON pv.hash = REPLACE(i.parent::text,'-','') || REPLACE(i.parent::text,'-','')
+  WHERE i.did LIKE 'did:firefly:node/%' AND v.hash IS NOT NULL;
+
+DROP INDEX identities_id;
+DROP INDEX identities_did;
+DROP INDEX identities_name;
+
+DROP TABLE IF EXISTS identities;
+
+DROP INDEX verifiers_hash;
+DROP INDEX verifiers_value;
+DROP INDEX verifiers_identity;
+
+DROP TABLE IF EXISTS verifiers;
+
+COMMIT;

db/migrations/postgres/000063_create_identities_table.up.sql

@@ -0,0 +1,143 @@
+BEGIN;
+
+CREATE TABLE identities (
+  seq                   SERIAL          PRIMARY KEY,
+  id                    UUID            NOT NULL,
+  did                   VARCHAR(256)    NOT NULL,
+  parent                UUID,
+  messages_claim        UUID            NOT NULL,
+  messages_verification UUID,
+  messages_update       UUID,
+  itype                 VARCHAR(64)     NOT NULL,
+  namespace             VARCHAR(64)     NOT NULL,
+  name                  VARCHAR(64)     NOT NULL,
+  description           VARCHAR(4096)   NOT NULL,
+  profile               TEXT,
+  created               BIGINT          NOT NULL,
+  updated               BIGINT          NOT NULL
+);
+
+CREATE UNIQUE INDEX identities_id ON identities(id);
+CREATE UNIQUE INDEX identities_did ON identities(did);
+CREATE UNIQUE INDEX identities_name ON identities(itype, namespace, name);
+
+CREATE TABLE verifiers (
+  seq            SERIAL          PRIMARY KEY,
+  hash           CHAR(64)        NOT NULL,
+  identity       UUID            NOT NULL,
+  vtype          VARCHAR(256)    NOT NULL,
+  namespace      VARCHAR(64)     NOT NULL,
+  value          TEXT            NOT NULL,
+  created        BIGINT          NOT NULL
+);
+
+CREATE UNIQUE INDEX verifiers_hash ON verifiers(hash);
+CREATE UNIQUE INDEX verifiers_value ON verifiers(vtype, namespace, value);
+CREATE INDEX verifiers_identity ON verifiers(identity);
+
+INSERT INTO identities (
+    id,
+    did,
+    parent,
+    messages_claim,
+    itype,
+    namespace,
+    name,
+    description,
+    profile,
+    created,
+    updated
+  ) SELECT 
+    o1.id,
+    'did:firefly:org/' || o1.name,
+    o2.id,
+    o1.message_id,
+    'org',
+    'ff_system',
+    o1.name,
+    o1.description,
+    o1.profile,
+    o1.created,
+    o1.created
+  FROM orgs as o1
+  LEFT JOIN orgs o2 ON o2.identity = o1.parent;
+
+INSERT INTO identities (
+    id,
+    did,
+    parent,
+    messages_claim,
+    itype,
+    namespace,
+    name,
+    description,
+    profile,
+    created,
+    updated
+  ) SELECT 
+    n.id,
+    'did:firefly:node/' || n.name,
+    o.id,
+    n.message_id,
+    'node',
+    'ff_system',
+    n.name,
+    n.description,
+    n.dx_endpoint,
+    n.created,
+    n.created
+  FROM nodes as n
+  LEFT JOIN orgs o ON o.identity = n.owner;
+
+INSERT INTO verifiers (
+    hash,
+    namespace,
+    identity,
+    vtype,
+    value,
+    created
+  ) SELECT 
+    REPLACE(o.id::text, '-', '') || REPLACE(o.id::text, '-', ''), -- to avoid the need for hashing in the migration, use the convenient fact the UUID is known hex - have to write it twice to fill the 32B --
+    'ff_system',
+    o.id,
+    'ethereum_address',
+    o.identity,
+    o.created    
+  FROM orgs as o WHERE o.identity LIKE '0x%';
+
+INSERT INTO verifiers (
+    hash,
+    namespace,
+    identity,
+    vtype,
+    value,
+    created
+  ) SELECT 
+    REPLACE(o.id::text, '-', '') || REPLACE(o.id::text, '-', ''), -- to avoid the need for hashing in the migration, use the convenient fact the UUID is known hex - have to write it twice to fill the 32B --
+    'ff_system',
+    o.id,
+    'fabric_msp_id',
+    o.identity,
+    o.created
+  FROM orgs as o WHERE o.identity NOT LIKE '0x%';
+
+INSERT INTO verifiers (
+    hash,
+    namespace,
+    identity,
+    vtype,
+    value,
+    created
+  ) SELECT 
+    REPLACE(n.id::text, '-', '') || REPLACE(n.id::text, '-', ''), -- to avoid the need for hashing in the migration, use the convenient fact the UUID is known hex - have to write it twice to fill the 32B --
+    'ff_system',
+    n.id,
+    'dx_peer_id',
+    n.dx_peer,
+    n.created
+  FROM nodes as n;
+
+DROP TABLE orgs;
+DROP TABLE nodes;
+
+COMMIT;

db/migrations/postgres/000064_allow_null_data.down.sql

@@ -0,0 +1 @@
+-- No down migration for this one

db/migrations/postgres/000064_allow_null_data.up.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE data ALTER COLUMN value DROP NOT NULL;
+
+COMMIT;

db/migrations/postgres/000065_pin_signer_event_correlator.down.sql

@@ -0,0 +1,4 @@
+BEGIN;
+ALTER TABLE pins DROP COLUMN signer;
+ALTER TABLE events DROP COLUMN cid;
+COMMIT;

db/migrations/postgres/000065_pin_signer_event_correlator.up.sql

@@ -0,0 +1,6 @@
+BEGIN;
+ALTER TABLE pins ADD COLUMN signer TEXT;
+UPDATE pins SET signer = '';
+
+ALTER TABLE events ADD COLUMN cid UUID;
+COMMIT;