feat(plugin-persistence-fabric): add new fabric persistence plugin

1. Add a new plugin for storing hyperledger fabric ledger data into a database
2. Add functional tests for plugin and data access layer operations.
operating with fabric-all-in-one docker composition

Tests assume any postgres database, but for final deployment postgres
or supabase is assumed.
Data fed by this plugin can later be visualized by a GUI application or
analyzed directly. ( right now the GUI will have switch to operate
either with Ethereum ledger or Fabric ledger )

Also upgrade ts-node in the rood package.json because it was causing
crashes for the scripts in the ./tools/ directory (such as the webpack
bundle name validator script)

Depends on: #2259
Depends on: #2265

Co-authored-by: Peter Somogyvari <peter.somogyvari@accenture.com>

Signed-off-by: Barnaba Pawełczak <barnaba.pawelczak@fujitsu.com>
Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>

package.json

@@ -151,7 +151,7 @@
     "tape-promise": "4.0.0",
     "ts-jest": "28.0.2",
     "ts-loader": "9.2.5",
-    "ts-node": "10.7.0",
+    "ts-node": "10.9.1",
     "typescript": "4.3.5",
     "webpack": "5.76.0",
     "webpack-bundle-analyzer": "4.4.2",

packages/cactus-plugin-ledger-connector-fabric/src/main/json/openapi.json

@@ -487,6 +487,9 @@
                             "type": "string",
                             "nullable": true
                         }
+                    },
+                    "responseType":{
+                        "type": "string"
                     }
                 }
             },

packages/cactus-plugin-ledger-connector-fabric/src/main/kotlin/generated/openapi/kotlin-client/src/main/kotlin/org/openapitools/client/models/RunTransactionRequest.kt

@@ -39,6 +39,7 @@ import com.squareup.moshi.Json
  * @param transientData 
  * @param gatewayOptions 
  * @param endorsingParties 
+ * @param responseType 
  */
 
 data class RunTransactionRequest (
@@ -72,7 +73,10 @@ data class RunTransactionRequest (
     val gatewayOptions: GatewayOptions? = null,
 
     @Json(name = "endorsingParties")
-    val endorsingParties: kotlin.collections.List<kotlin.String>? = null
+    val endorsingParties: kotlin.collections.List<kotlin.String>? = null,
+
+    @Json(name = "responseType")
+    val responseType: kotlin.String? = null
 
 )
 

packages/cactus-plugin-ledger-connector-fabric/src/main/typescript/generated/openapi/typescript-axios/api.ts

@@ -974,6 +974,12 @@ export interface RunTransactionRequest {
      * @memberof RunTransactionRequest
      */
     endorsingParties?: Array<string>;
+    /**
+     * 
+     * @type {string}
+     * @memberof RunTransactionRequest
+     */
+    responseType?: string;
 }
 /**
  * 

packages/cactus-plugin-ledger-connector-fabric/src/main/typescript/plugin-ledger-connector-fabric.ts

@@ -130,6 +130,7 @@ import { querySystemChainCode } from "./common/query-system-chain-code";
  * found in the https://github.com/hyperledger/fabric-samples repository.
  */
 export const K_DEFAULT_CLI_CONTAINER_GO_PATH = "/opt/gopath/";
+export const JSONstringResponseType = "JSONstring";
 
 /**
  * The command that will be used to issue docker commands while controlling
@@ -951,8 +952,10 @@ export class PluginLedgerConnectorFabric
   protected async createGateway(req: RunTransactionRequest): Promise<Gateway> {
     if (req.gatewayOptions) {
       return this.createGatewayWithOptions(req.gatewayOptions);
-    } else {
+    } else if (req.signingCredential) {
       return this.createGatewayLegacy(req.signingCredential);
+    } else {
+      throw new Error("Missing either gatewayOptions or signingCredential");
     }
   }
 
@@ -1087,6 +1090,7 @@ export class PluginLedgerConnectorFabric
       params,
       transientData,
       endorsingParties,
+      responseType: responseType,
     } = req;
 
     try {
@@ -1182,9 +1186,18 @@ export class PluginLedgerConnectorFabric
           throw new Error(`${fnTag} unknown ${message}`);
         }
       }
-      const outUtf8 = out.toString("utf-8");
+      let outResp = "";
+
+      switch (responseType) {
+        case JSONstringResponseType:
+          outResp = JSON.stringify(out);
+          break;
+        default:
+          outResp = out.toString("utf-8");
+      }
+
       const res: RunTransactionResponse = {
-        functionOutput: outUtf8,
+        functionOutput: outResp,
         success,
         transactionId: transactionId,
       };

packages/cactus-plugin-ledger-connector-ubiquity/package.json

@@ -98,4 +98,4 @@
             "runOnChangeOnly": true
         }
     }
-}
+}

packages/cactus-plugin-persistence-fabric/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+

packages/cactus-plugin-persistence-fabric/README.md

@@ -0,0 +1,226 @@
+# `@hyperledger/cactus-plugin-persistence-fabric`
+
+This plugin allows `Cacti` to persist Fabric Block general data and basic information about transactions into some storage (currently to a `PostgreSQL` database, but this concept can be extended further).
+Data in the database can later be analyzed and viewed in a GUI tool.
+GUI tool is in project root directory of Cacti project in GUI folder. cacti/packages/cacti-cmd-gui-app
+
+
+## Summary
+
+- [Remarks](#remarks)
+- [Getting Started](#getting-started)
+- [Endpoints](#endpoints)
+- [Running the tests](#running-the-tests)
+- [Contributing](#contributing)
+- [License](#license)
+- [Acknowledgments](#acknowledgments)
+
+## Remarks
+
+- This plugin was only tested with small test ledgers. Running it to synchronize with old ledgers will take a lot of time.
+- For now, the database schema is not considered public and can change over time (i.e., writing own application that reads data directly from the database is discouraged).
+- All the methods must be called directly on the plugin instance for now.
+
+## Getting Started
+
+Clone the git repository on your local machine. Follow these instructions that will get you a copy of the project up and running on your local machine for development and testing purposes.
+
+### Prerequisites
+
+In the root of the project, execute the command to install and build the dependencies. It will also build this persistence plugin:
+
+```sh
+yarn run configure
+```
+
+### Usage
+
+Instantiate a new `PluginPersistenceFabrickBlock` instance:
+
+There is few ways to use this plugin:
+
+1.  Using Watch block might cause infinite loop
+    and function migrateBlockNrWithTransactions
+2.  Individually using migrateBlockNrWithTransactions
+    if you know which exactly which blocks you want to analyze
+3.  Using plugin function continueBlocksSynchronization in some periodical calls
+4.  Using plugin function continuousBlocksSynchronization in some reasonable time with changeSynchronization - which should break flow
+    this function might continue work for long period
+
+5) Best to start synchronization with initialBlocksSynchronization , which will transfer and parse into database number of blocks equal to edgeOfLedger
+
+6) If there were some issues in network connection or you think that your database might be corrupted and lost some data you might use whichBlocksAreMissingInDdSimple
+   and then showHowManyBlocksMissing if more than 0
+   then use synchronizeOnlyMissedBlocks
+
+```typescript
+import { PluginPersistenceFabric } from "../../../main/typescript/plugin-fabric-persistence-block";
+import { v4 as uuidv4 } from "uuid";
+
+PluginInstance = new PluginPersistenceFabric({
+  gatewayOptions,
+  apiClient,
+  logLevel: testLogLevel,
+  instanceId: uuidv4(),
+  connectionString:
+    "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",
+});
+
+// Initialize the connection to the DB
+PluginInstance.onPluginInit();
+```
+
+Alternatively, import `PluginPersistenceFabric` from the plugin package and use it to create a plugin.
+
+```typescript
+import { PluginPersistenceFabric } from "@hyperledger/plugin-fabric-persistence-block";
+import { PluginImportType } from "@hyperledger/cactus-core-api";
+import { v4 as uuidv4 } from "uuid";
+
+const factory = new PluginFactoryPersistenceFabricBlock({
+  pluginImportType: PluginImportType.Local,
+});
+
+const persistencePlugin = await factory.create({
+  instanceId: uuidv4(),
+  apiClient: new SocketIOApiClient(apiConfigOptions),
+  logLevel: "info",
+  connectionString:
+    "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",
+});
+
+// Initialize the connection to the DB
+persistencePlugin.onPluginInit();
+```
+
+onPluginInit it also creates database structure if this is first time run
+
+// Start synchronization with ledger.
+// To synchronize ledger
+
+you individually
+
+// Show current status of the plugin
+PluginInstance.getStatus();
+
+```
+
+
+```
+
+## Endpoints
+
+### Plugin Methods
+
+- Most of the plugin functionalities are currently not available through OpenAPI interface, please use direct method calls instead.
+
+#### `onPluginInit`
+
+- Should be called before using the plugin.
+
+#### `shutdown`
+
+- Close the connection to the DB, cleanup any allocated resources.
+
+#### `getStatus`
+
+- Get status report of this instance of persistence plugin.
+
+#### `constinousBlocksSynchronization`
+
+- Start the block synchronization process. New blocks from the ledger will be parsed and pushed to the database.
+
+#### `continueBlocksSynchronization`
+
+- Start the block synchronization process. New blocks from the ledger will be parsed and pushed to the database. Should be used periodically.
+
+#### `changeSynchronization`
+
+- Stop the block synchronization process.
+
+#### `whichBlocksAreMissingInDdSimple`
+
+- Walk through all the blocks that could not be synchronized with the DB for some reasons and list them
+
+#### `synchronizeOnlyMissedBlocks`
+
+- Walk through all the blocks that are listed as not be synchronized with the DB for some reasons and try push them into DB from ledger.
+- can try many times to use this
+
+### `setLastBlockConsidered`
+
+- set the last block in ledger which we consider valid by our party and synchronize only to this point in ledger
+  If some blocks above this number are already in database they will not be removed.
+
+#### `initialBlocksSynchronization`
+
+- Synchronize entire first N number of blocks of ledger state with the database. It is a good start and easy to check if everything is correctly set.
+
+## Running the tests
+
+To run all the tests for this persistence fabric plugin to ensure it's working correctly execute the following from the root of the `cactus` project:
+
+```sh
+npx jest cactus-plugin-fabric-persistence-block
+```
+
+## Contributing
+
+We welcome contributions to Hyperledger Cacti in many forms, and there’s always plenty to do!
+
+Please review [CONTIRBUTING.md](../../CONTRIBUTING.md) to get started.
+
+### Quick plugin project walkthrough
+
+#### ./src/main/json/contract_abi
+
+- Contains reference token ABIs used to call and identify token transfers.
+
+#### `./src/main/json/openapi.json`
+
+- Contains OpenAPI definition.
+
+#### `./src/main/sql/schema.sql`
+
+- Database schema for Ethereum data.
+
+#### `./src/main/typescript/web-services`
+
+- Folder that contains web service endpoint definitions if present.
+
+#### `./plugin-fabric-persistence-block`
+
+- Main persistent plugin logic file
+
+#### `./src/test/typescript/integration/`
+
+- Integration test of various plugin functionalities.
+
+### Generating types from the database schema
+
+- Current setup assume use of Supabase that has utility for generating types from the database schema.
+- We use this tool to generate type definitions and store them in `./src/main/typescript/db-client/database.types.ts`
+- Upstream instructions: https://supabase.com/docs/guides/api/generating-types
+- Step by step manual on updating the types (must be done after changing the database schema):
+  - Install `supabase` package
+  - Init and start development supabase server:
+    - `npx supabase init`
+    - `npx supabase start`
+  - Fill in current schema:
+    - `psql -h localhost -p 54322 -U postgres -d postgres -a -f src/main/sql/schema.sql` (password: `postgres`)
+  - Generate the file with type definitions:
+    - `npx supabase gen types typescript --schema public --local > src/main/typescript/db-client/database.types.ts`
+  - Cleanup:
+    - `npx supabase stop`
+    - `rm -rf ./supabase`
+
+#### Insert sample data
+
+- Can be used to test GUI applications without running entire ledger / persistence setup.
+- `psql -h localhost -p 54322 -U postgres -d postgres -a -f src/test/sql/insert-test-data.sql` (password: `postgres`)
+
+## License
+
+This distribution is published under the Apache License Version 2.0 found in the [LICENSE](../../LICENSE) file.
+
+## Acknowledgments

packages/cactus-plugin-persistence-fabric/openapitools.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "node_modules/@openapitools/openapi-generator-cli/config.schema.json",
+  "spaces": 2,
+  "generator-cli": {
+    "version": "5.2.1"
+  }
+}