diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 64d6f2e918..0168cb9753 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -74,12 +74,12 @@ jobs: - name: Check the docs for broken links (root) if: github.event_name == 'pull_request' working-directory: docs - run: bundle exec htmlproofer --disable-external --allow-hash-href --assume-extension ./_site --url-swap '^/firefly/:/' --url-ignore /127.0.0.1/,/localhost/ + run: bundle exec htmlproofer --disable-external --allow-hash-href --allow_missing_href true --swap-urls '^/firefly/:/' --ignore-urls /127.0.0.1/,/localhost/ ./_site - name: Check the docs for broken links (version) if: github.event_name != 'pull_request' working-directory: docs - run: bundle exec htmlproofer --disable-external --allow-hash-href --assume-extension ./_site --url-swap '^/firefly/${{ env.DOCS_VERSION }}/:/' --url-ignore /127.0.0.1/,/localhost/ + run: bundle exec htmlproofer --disable-external --allow-hash-href --allow_missing_href true --swap-urls '^/firefly/${{ env.DOCS_VERSION }}/:/' --ignore-urls /127.0.0.1/,/localhost/ ./_site - name: Deploy docs (version) if: github.event_name == 'push' diff --git a/.vscode/settings.json b/.vscode/settings.json index 10cd5a2c69..01df452d23 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,6 +4,7 @@ ], "go.lintTool": "golangci-lint", "cSpell.words": [ + "actioned", "APIID", "autometa", "bifactory", diff --git a/Makefile b/Makefile index e1778c761e..eb12319e86 100644 --- a/Makefile +++ b/Makefile @@ -102,4 +102,4 @@ manifest: docker: ./docker_build.sh $(DOCKER_ARGS) docs: .ALWAYS - cd docs && bundle install && bundle exec jekyll build && bundle exec htmlproofer --disable-external --allow-hash-href --assume-extension ./_site --url-swap '^/firefly/:/' --url-ignore /127.0.0.1/,/localhost/ + cd docs && bundle install && bundle exec jekyll build && bundle exec htmlproofer --disable-external --allow-hash-href --allow_missing_href true --swap-urls '^/firefly/:/' --ignore-urls /127.0.0.1/,/localhost/ ./_site diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index db1ea6a7de..2d9b8444cc 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -1,20 +1,20 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.0.5) + activesupport (6.0.6) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) zeitwerk (~> 2.2, >= 2.2.2) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) coffee-script (2.4.1) coffee-script-source execjs coffee-script-source (1.11.1) colorator (1.1.0) - commonmarker (0.23.4) + commonmarker (0.23.5) concurrent-ruby (1.1.10) dnsruby (1.61.9) simpleidn (~> 0.1) @@ -25,33 +25,14 @@ GEM ffi (>= 1.15.0) eventmachine (1.2.7) execjs (2.8.1) - faraday (1.10.0) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + faraday (2.5.2) + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.3) - multipart-post (>= 1.2, < 3) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) + faraday-net_http (3.0.0) ffi (1.15.5) forwardable-extended (2.6.0) gemoji (3.0.1) - github-pages (226) + github-pages (227) github-pages-health-check (= 1.17.9) jekyll (= 3.9.2) jekyll-avatar (= 0.7.0) @@ -93,7 +74,7 @@ GEM liquid (= 4.0.3) mercenary (~> 0.3) minima (= 2.5.1) - nokogiri (>= 1.13.4, < 2.0) + nokogiri (>= 1.13.6, < 2.0) rouge (= 3.26.0) terminal-table (~> 1.4) github-pages-health-check (1.17.9) @@ -102,10 +83,10 @@ GEM octokit (~> 4.0) public_suffix (>= 3.0, < 5.0) typhoeus (~> 1.3) - html-pipeline (2.14.1) + html-pipeline (2.14.2) activesupport (>= 2) nokogiri (>= 1.4) - html-proofer (3.19.4) + html-proofer (4.4.0) addressable (~> 2.3) mercenary (~> 0.3) nokogiri (~> 1.13) @@ -113,6 +94,7 @@ GEM rainbow (~> 3.0) typhoeus (~> 1.3) yell (~> 2.0) + zeitwerk (~> 2.5) http_parser.rb (0.8.0) i18n (0.9.5) concurrent-ruby (~> 1.0) @@ -239,20 +221,19 @@ GEM jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - minitest (5.15.0) - multipart-post (2.1.1) - nokogiri (1.13.6-arm64-darwin) + minitest (5.16.3) + nokogiri (1.13.8-arm64-darwin) racc (~> 1.4) - octokit (4.22.0) - faraday (>= 0.9) - sawyer (~> 0.8.0, >= 0.5.3) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) parallel (1.22.1) pathutil (0.16.2) forwardable-extended (~> 2.6) public_suffix (4.0.7) racc (1.6.0) rainbow (3.1.1) - rb-fsevent (0.11.1) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) rexml (3.2.5) @@ -265,9 +246,9 @@ GEM sass-listen (4.0.0) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.8.2) + sawyer (0.9.2) addressable (>= 2.3.5) - faraday (> 0.8, < 2.0) + faraday (>= 0.17.3, < 3) simpleidn (0.2.1) unf (~> 0.1.4) terminal-table (1.8.0) @@ -275,15 +256,15 @@ GEM thread_safe (0.3.6) typhoeus (1.4.0) ethon (>= 0.9.0) - tzinfo (1.2.9) + tzinfo (1.2.10) thread_safe (~> 0.1) unf (0.1.4) unf_ext - unf_ext (0.0.8.1) + unf_ext (0.0.8.2) unicode-display_width (1.8.0) webrick (1.7.0) yell (2.2.2) - zeitwerk (2.5.4) + zeitwerk (2.6.0) PLATFORMS arm64-darwin-21 diff --git a/docs/_config.yml b/docs/_config.yml index 02cdd13af8..56419c2f6a 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -4,7 +4,7 @@ title: Hyperledger FireFly Docs baseurl: /firefly aux_links: "GitHub": - - "//github.com/hyperledger/firefly" + - "https://github.com/hyperledger/firefly" "Wiki": - "https://wiki.hyperledger.org/display/FIR" "Discord": diff --git a/docs/_i18n/en.yml b/docs/_i18n/en.yml index 0661968a0e..ca22a6e661 100644 --- a/docs/_i18n/en.yml +++ b/docs/_i18n/en.yml @@ -4,33 +4,50 @@ langs: site: title: Hyperledger FireFly Docs pages: - home: Home advanced_cli_usage: Advanced CLI Usage - api_spec: API Spec - architecture: Architecture api_post_syntax: API Post Syntax api_query_syntax: API Query Syntax - blockchain_protocols: Blockchain protocols - broadcast_shared_data: Broadcast / shared data + api_spec: API Spec + apps: Apps + arbitrum: Arbitrum Testnet + architecture: Architecture + avalanche: Avalanche Testnet + binance_smart_chain: Binance Smartchain Testnet + blockchain_protocols: Blockchain Protocols broadcast_data: Broadcast data + broadcast_data: Broadcast Data + broadcast_shared_data: Broadcast / Shared Data + chains: Connecting to remote blockchains code_hierarchy: FireFly Code Hierarchy code_overview: FireFly Code Overview code_repositories: Code Repositories configuration_reference: Configuration Reference + connector_framework: Connector Framework contributors: Contributors custom_smart_contracts: Work with custom smart contracts + data_security: Data Security + deterministic: Deterministic Compute + digital_assets: Digital Assets faqs: FAQs + firefly_core: Firefly Core + flows: Flows getting_started: Getting Started - private_data_exchange: Private data exchange + home: Home + introduction: Introduction + key_features: Key Features + moonbeam_testnet: Moonbeam Testnet + multiparty_features: Multiparty Features + multiparty_flow: Multiparty Process Flow + near: NEAR Testnet + optimism: Optimism Testnet + orchestration_engine: Orchestration Engine + polygon_testnet: Polygon Testnet + private_data_exchange: Private Data Exchange reference: Reference + remote_fabric_network: Remote Fabric Network + security: Security + tools: Tools tutorials: Tutorials understanding_firefly: Understanding FireFly - chains: Connecting to remote blockchains - remote_fabric_network: Remote Fabric Network - polygon_testnet: Polygon Testnet - arbitrum: Arbitrum Testnet - avalanche: Avalanche Testnet - binance_smart_chain: Binance Smartchain Testnet - near: NEAR Testnet - optimism: Optimism Testnet - moonbeam_testnet: Moonbeam Testnet + usage_patterns: Usage Patterns + web3_gateway_features: Web3 Gateway Features \ No newline at end of file diff --git a/docs/_i18n/en/index.md b/docs/_i18n/en/index.md index 908c9ce9fa..1c5e2602cb 100644 --- a/docs/_i18n/en/index.md +++ b/docs/_i18n/en/index.md @@ -2,12 +2,19 @@ ![Hyperledger FireFly](./images/hyperledger_firefly_social.png) -Hyperledger FireFly is the first open source Supernode: a complete stack for enterprises to build and scale secure Web3 applications. +Hyperledger FireFly is an [open source Supernode](./overview/supernode_concept.html), a complete stack for enterprises to build and scale secure Web3 applications. -The FireFly API for digital assets, data flows, and blockchain transactions makes it radically faster to build production-ready apps on popular chains and protocols. +The easiest way to understand a FireFly Supernode is to think of it like a toolbox. Connect your existing apps and/or back office systems to the toolbox and within it there are two different sets of tools. One set of tools helps you connect to the Web3 world that already exists, and the other set allows you to build new decentralized applications quickly with security and scalability. + +Head to the [Understanding FireFly](./overview/supernode_concept.html) section for more details. + +## Table of contents - [Understanding FireFly](./overview/) +- [Getting Started](./gettingstarted/) +- [Tutorials](./tutorials/) - [Reference](./reference/) - [Architecture](./architecture/) - [Contributors](./contributors/) - [API Spec](./swagger/swagger.html) +- [FAQs](./faqs/) \ No newline at end of file diff --git a/docs/architecture/blockchain_connector_framework.md b/docs/architecture/blockchain_connector_framework.md new file mode 100644 index 0000000000..c0a451440b --- /dev/null +++ b/docs/architecture/blockchain_connector_framework.md @@ -0,0 +1,190 @@ +--- +layout: default +title: Blockchain Connector Toolkit +parent: pages.architecture +nav_order: 2 +--- + +# Multiparty Event Sequencing +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + + + +## Blockchain Connector Framework + +Hyperledger FireFly has a [multi-tier pluggable architecture](../overview/key_components/connectors.md) for supporting blockchains of +all shapes and sizes. This includes a remote API that allows a microservice connector to +be built from scratch in any programming language. + +It also includes the **Connector Toolkit**, which is a pluggable SDK in Golang that provides +a set of re-usable modules that can be used across blockchain implementations. + +> This is the preferred way to build a new blockchain connector, if you are comfortable +> with coding in Golang and there are language bindings available for the raw RPC interface +> of your blockchain. + +## Connector Toolkit Architecture + +[![FireFly Blockchain Connector Toolkit Architecture](../images/firefly_transaction_manager.jpg)](../images/firefly_transaction_manager.jpg) + +The core component of the FireFly Connector Framework for Blockchains is a Go module +called [FireFly Transaction Manager](https://github.com/hyperledger/firefly-transaction-manager) (FFTM). + +FFTM is responsible for: + +- Submission of transactions to blockchains of all types + - Nonce management - idempotent submission of transactions, and assignment of nonces + - Protocol connectivity decoupled with additional lightweight API connector + - Easy to add additional protocols that conform to normal patterns of TX submission / events + +- Monitoring and updating blockchain operations + - Receipts + - Confirmations + +- Gas calculation and rescue of stalled transactions + - Extensible policy engine + - Gas station API integration + +- Event streaming + - Protocol agnostic event polling/streaming support + - Reliable checkpoint restart + - At least once delivery API + +## Assumptions / Requirements + +The framework is currently constrained to blockchains that adhere to certain basic principals: + +1. Has transactions + - That are signed + - That can optionally have gas semantics (limits and prices, expressed in a blockchain specific way) +2. Has events (or "logs") + - That are emitted as a deterministic outcome of transactions +3. Has blocks + - Containing zero or more transactions, with their associated events + - With a sequential numeric order + - With a hash + - With a parent hash +4. Has finality for transactions & events that can be expressed as a level of confidence over time + - Confirmations: A number of sequential blocks in the canonical chain that contain the transaction + +## Nonce management + +The nonces for transactions is assigned as early as possible in the flow: +- Before the REST API for submission of the transaction occurs +- After the FFCAPI blockchain connector verifies the transaction can be encoded successfully to the chain +- With protection against multiple parallel API requests for the same signing address +- With stateful persistence meaning the connector knows about all nonces it previously allocated, to avoids duplicates + +This "at source" allocation of nonces provides the strictest assurance of order of transactions possible, +because the order is locked in with the coordination of the business logic of the application submitting the transaction. + +As well as protecting against loss of transactions, this protects against duplication of transactions - even in crash +recovery scenarios with a sufficiently reliable persistence layer. + +### Avoid multiple nonce management systems against the same signing key + +FFTM is optimized for cases where all transactions for a given signing address flow through the +same FireFly connector. If you have signing and nonce allocation happening elsewhere, not going through the +FireFly blockchain connector, then it is possible that the same nonce will be allocated in two places. + +> Be careful that the signing keys for transactions you stream through the Nonce Management of the FireFly +> blockchain connector are not used elsewhere. + +If you must have multiple systems performing nonce management against the same keys you use with FireFly nonce management, +you can set the `transactions.nonceStateTimeout` to `0` (or a low threshold like `100ms`) to cause the nonce management +to query the pending transaction pool of the node every time a nonce is allocated. + +This reduces the window for concurrent nonce allocation to be small (basically the same as if you had +multiple simple web/mobile wallets used against the same key), but it does not eliminate it completely it. + +### Why "at source" nonce management was chosen vs. "at target" + +The "at source" approach to ordering used in FFTM could be compared with the "at target" allocation of nonces used in +[EthConnect](https://github.com/hyperledger/firefly-ethconnect)). + +The "at target" approach optimizes for throughput and ability to send new transactions to the chain, +with an at-least-once delivery assurance to the applications. + +An "at target" algorithm as used in EthConnect could resume transaction delivery automatically without operator intervention +from almost all scenarios, including where nonces have been double allocated. + +However, "at target" comes with two compromises that mean FFTM chose the "at source" approach was chosen for FFTM: + +- Individual transactions might fail in certain scenarios, and subsequent transactions will still be streamed to the chain. + While desirable for automation and throughput, this reduces the ordering guarantee for high value transactions. + +- In crash recovery scenarios the assurance is at-least-once delivery for "at target" ordering (rather than "exactly once"), + although the window can be made very small through various optimizations included in the EthConnect codebase. + +## Policy Manager + +The policy manager is a pluggable component that allows rich policy to be applied to the +gas pricing, signing, submission and re-submission of transactions to the blockchain. + +It is executed at regular intervals against transactions in-flight, and is responsible for +evaluating what actions are required to cause those transactions to be executed successfully. + +The policy manager can store custom state in the state store of the FFTM code, which is also +reported in status within the FireFly API/Explorer on the operation. + +A reference implementation is provided that: +- Submits transactions via the underlying FFCAPI +- Estimates gas price in one of three modes: + - Fixed: It is specified via configuration + - Connector: The FFCAPI is used to estimate the gas price (e.g. `eth_gasPrice` for EVM JSON/RPC) + - Gas Oracle: A REST API is called the the result mapped via a Go template +- Re-submits transactions after a configurable stale time + +The reference implementation is available [here](https://github.com/hyperledger/firefly-transaction-manager/blob/main/pkg/policyengines/simple/simple_policy_engine.go) + +## Event Streams + +One of the largest pieces of heavy lifting code in the FFTM codebase, is the event stream +support. This provides a WebSocket (and Webhook) interface that FireFly Core and the Tokens +Connectors connect to in order to receive ordered streams of events from the blockchain. + +The interface down to the blockchain layer is via go channels, and there are lifecycle +interactions over the FFCAPI to the blockchain specific code to add and remove listeners +for different types of blockchain events. + +Some high architectural principals that informed the code: + +- Event Stream + - A delivery stream of events that have been confirmed + - Only events that have reached finality are delivered to an event stream + - FireFly creates a single event stream _per namespace_ from core + - Each token connector creates a single event stream + - If one event stream is blocked, it must not block other streams in the FFTM based runtime +- Listener (/Subscription) + - A blockchain specific specification for a set of events to listen to + - Specifies an initial block to listen from, and will replay all events from that block + - Can have multiple blockchain specific filters, to match multiple events + - The order of delivery _within_ a listener matches the blockchain across all filters + > - Note that the EVMConnect implementation of FFCAPI attempts to make this true across all listeners + > on an event stream. However, this is impossible when a new listener has been added, + > and that listener is catching up from an old block. +- Compatibility + - Has breaking changes from the API of EthConnect + - A component that has been upgraded to support EVMConnect, + can maintain backwards compatibility with EthConnect +- Batching & Checkpoints + - Delivery on event streams is via batches, with a single confirmation for each batch + - At-least-once delivery of batches + - Checkpoints are written after each batch +- Chain head stability + - A configurable section of the head of the chain is considered unstable + - If no events have been delivered for a listener, checkpoints are still moved forwards + - These empty checkpoints will never be written in the unstable head of the chain +- Confirmation manager + - Plugged in between detection of the events, and assembling into batches + - Determines the final order based on order of confirmation on the blockchain + +[![FireFly Connector Toolkit Event Streams](../images/firefly_connector_toolkit_event_streams.jpg)](../images/firefly_connector_toolkit_event_streams.jpg) \ No newline at end of file diff --git a/docs/architecture/multiparty_event_sequencing.md b/docs/architecture/multiparty_event_sequencing.md index fb12ce0cd0..d49219fec6 100644 --- a/docs/architecture/multiparty_event_sequencing.md +++ b/docs/architecture/multiparty_event_sequencing.md @@ -2,7 +2,7 @@ layout: default title: Multiparty Event Sequencing parent: pages.architecture -nav_order: 2 +nav_order: 3 --- # Multiparty Event Sequencing diff --git a/docs/architecture/ping_pong_txflow.md b/docs/architecture/ping_pong_txflow.md index 1ed306e9a4..2470150cf1 100644 --- a/docs/architecture/ping_pong_txflow.md +++ b/docs/architecture/ping_pong_txflow.md @@ -2,7 +2,7 @@ layout: default title: Example Transaction Flow parent: pages.architecture -nav_order: 3 +nav_order: 5 --- # Example Transaction Flow (Ping Pong) diff --git a/docs/faqs/faqs.md b/docs/faqs/index.md similarity index 100% rename from docs/faqs/faqs.md rename to docs/faqs/index.md diff --git a/docs/gettingstarted/firefly_cli.md b/docs/gettingstarted/firefly_cli.md index b73adb869e..521a231ea4 100644 --- a/docs/gettingstarted/firefly_cli.md +++ b/docs/gettingstarted/firefly_cli.md @@ -6,9 +6,11 @@ nav_order: 1 --- # ① Install the FireFly CLI + {: .no_toc } ## Table of contents + {: .no_toc .text-delta } 1. TOC @@ -16,12 +18,6 @@ nav_order: 1 --- -![FireFly CLI](../images/firefly_cli.png) - -## FireFly CLI - -The FireFly CLI can be used to create local FireFly stacks for offline development of blockchain apps. This allows developers to rapidly iterate on their idea without needing to set up a bunch of infrastructure before they can write the first line of code. - ## Prerequisites In order to run the FireFly CLI, you will need a few things installed on your dev machine: @@ -31,9 +27,11 @@ In order to run the FireFly CLI, you will need a few things installed on your de - openssl ### Linux Users + > **NOTE**: For Linux users, it is recommended that you add your user to the `docker` group so that you do not have to run `ff` or `docker` as `root` or with `sudo`. For more information about Docker permissions on Linux, please see [Docker's documentation on the topic](https://docs.docker.com/engine/install/linux-postinstall/). ### Windows Users + > **NOTE**: For Windows users, we recommend that you use [Windows Subsystem for Linux 2 (WSL2)](https://docs.microsoft.com/en-us/windows/wsl/). Binaries provided for Linux will work in this environment. ## Install the CLI @@ -41,6 +39,7 @@ In order to run the FireFly CLI, you will need a few things installed on your de There are several ways to install the FireFly CLI. The easiest way to get up and running with the FireFly CLI is to download a pre-compiled binary of the latest release. ### Download the package for your OS + Go to the [latest release page](https://github.com/hyperledger/firefly-cli/releases/latest) and download the package for your OS and CPU architecture. ### Extract the binary and move it to `/usr/bin/local` @@ -51,11 +50,11 @@ Assuming you downloaded the package from GitHub into your `Downloads` directory, sudo tar -zxf ~/Downloads/firefly-cli_*.tar.gz -C /usr/local/bin ff && rm ~/Downloads/firefly-cli_*.tar.gz ``` -If you downloaded the package from GitHub into a different directory, you will need to change the `tar` command above to wherever the `firefly-cli_*.tar.gz ` file is located. +If you downloaded the package from GitHub into a different directory, you will need to change the `tar` command above to wherever the `firefly-cli_*.tar.gz` file is located. ### macOSUsers - > **NOTE**: On recent versions of macOS, default security settings will prevent the FireFly CLI binary from running, because it was downloaded from the internet. You will need to [allow the FireFly CLI in System Preferences](https://github.com/hyperledger/firefly-cli/blob/main/docs/mac_help.md), before it will run. + > **NOTE**: On recent versions of macOS, default security settings will prevent the FireFly CLI binary from running, because it was downloaded from the internet. You will need to [allow the FireFly CLI in System Preferences](https://github.com/hyperledger/firefly-cli/blob/main/docs/mac_help.md), before it will run. ### Alternative installation method: Install via Go @@ -68,6 +67,7 @@ go install github.com/hyperledger/firefly-cli/ff@latest ## Verify the installation After using either installation method above, you can verify that the CLI is successfully installed by running `ff version`. This should print the current version like this: + ``` { "Version": "v0.0.47", @@ -76,6 +76,7 @@ After using either installation method above, you can verify that the CLI is suc ``` ## Next steps: Start your environment + Now that you've got the FireFly CLI set up on your machine, the next step is to create and start a FireFly stack. -[② Start your environment →](setup_env.md){: .btn .btn-purple .float-right .mb-5} \ No newline at end of file +[② Start your environment →](setup_env.md){: .btn .btn-purple .float-right .mb-5} diff --git a/docs/gettingstarted/sandbox.md b/docs/gettingstarted/sandbox.md index 8b50e2a35c..fe7f83a514 100644 --- a/docs/gettingstarted/sandbox.md +++ b/docs/gettingstarted/sandbox.md @@ -26,9 +26,9 @@ Now that you have a full network of three Supernodes running on your machine, le ## Video walkthrough -This video is a walkthrough of the FireFly Sandbox and FireFly Explorer from the FireFly 1.0 launch webinar. At this point you should be able to follow along and try all these same things on your own machine. +This video is a walkthrough of the FireFly Sandbox and FireFly Explorer from the FireFly 1.0 launch webinar. At this point you should be able to follow along and try all these same things on your own machine. -## What is the FireFly Sandbox? + ## Open the FireFly Sandbox for the first member diff --git a/docs/images/firefly-getting-started-steps.png b/docs/images/firefly-getting-started-steps.png new file mode 100644 index 0000000000..20c6a67fa0 Binary files /dev/null and b/docs/images/firefly-getting-started-steps.png differ diff --git a/docs/images/firefly_architecture_overview.jpg b/docs/images/firefly_architecture_overview.jpg index 6e6cc4b13c..7e211d58e2 100644 Binary files a/docs/images/firefly_architecture_overview.jpg and b/docs/images/firefly_architecture_overview.jpg differ diff --git a/docs/images/firefly_blockchain_connector_framework.png b/docs/images/firefly_blockchain_connector_framework.png new file mode 100644 index 0000000000..c4c504ea20 Binary files /dev/null and b/docs/images/firefly_blockchain_connector_framework.png differ diff --git a/docs/images/firefly_connector_toolkit_event_streams.jpg b/docs/images/firefly_connector_toolkit_event_streams.jpg new file mode 100644 index 0000000000..0bd5ac569a Binary files /dev/null and b/docs/images/firefly_connector_toolkit_event_streams.jpg differ diff --git a/docs/images/firefly_core.png b/docs/images/firefly_core.png new file mode 100644 index 0000000000..e39c114f6c Binary files /dev/null and b/docs/images/firefly_core.png differ diff --git a/docs/images/firefly_data_transport_layers.png b/docs/images/firefly_data_transport_layers.png new file mode 100644 index 0000000000..6a46c7e7e2 Binary files /dev/null and b/docs/images/firefly_data_transport_layers.png differ diff --git a/docs/images/firefly_functionality_overview.png b/docs/images/firefly_functionality_overview.png index c54a4e3aa3..b29c883197 100644 Binary files a/docs/images/firefly_functionality_overview.png and b/docs/images/firefly_functionality_overview.png differ diff --git a/docs/images/firefly_functionality_overview_apps.png b/docs/images/firefly_functionality_overview_apps.png new file mode 100644 index 0000000000..140ba4f915 Binary files /dev/null and b/docs/images/firefly_functionality_overview_apps.png differ diff --git a/docs/images/firefly_functionality_overview_connectivity.png b/docs/images/firefly_functionality_overview_connectivity.png new file mode 100644 index 0000000000..aab49eb7ca Binary files /dev/null and b/docs/images/firefly_functionality_overview_connectivity.png differ diff --git a/docs/images/firefly_functionality_overview_digital_assets.png b/docs/images/firefly_functionality_overview_digital_assets.png new file mode 100644 index 0000000000..5946101ca9 Binary files /dev/null and b/docs/images/firefly_functionality_overview_digital_assets.png differ diff --git a/docs/images/firefly_functionality_overview_flows.png b/docs/images/firefly_functionality_overview_flows.png new file mode 100644 index 0000000000..d01699214c Binary files /dev/null and b/docs/images/firefly_functionality_overview_flows.png differ diff --git a/docs/images/firefly_functionality_overview_orchestration_engine.png b/docs/images/firefly_functionality_overview_orchestration_engine.png new file mode 100644 index 0000000000..30ca31928a Binary files /dev/null and b/docs/images/firefly_functionality_overview_orchestration_engine.png differ diff --git a/docs/images/firefly_functionality_overview_security.png b/docs/images/firefly_functionality_overview_security.png new file mode 100644 index 0000000000..352e196d34 Binary files /dev/null and b/docs/images/firefly_functionality_overview_security.png differ diff --git a/docs/images/firefly_functionality_overview_tools.png b/docs/images/firefly_functionality_overview_tools.png new file mode 100644 index 0000000000..42ee55dc22 Binary files /dev/null and b/docs/images/firefly_functionality_overview_tools.png differ diff --git a/docs/images/firefly_intro_overview.png b/docs/images/firefly_intro_overview.png new file mode 100644 index 0000000000..28987c9aaf Binary files /dev/null and b/docs/images/firefly_intro_overview.png differ diff --git a/docs/images/firefly_orchestration_engine.png b/docs/images/firefly_orchestration_engine.png new file mode 100644 index 0000000000..27635fa529 Binary files /dev/null and b/docs/images/firefly_orchestration_engine.png differ diff --git a/docs/images/firefly_transaction_manager.jpg b/docs/images/firefly_transaction_manager.jpg index d7b9c7f5b4..a658492737 100644 Binary files a/docs/images/firefly_transaction_manager.jpg and b/docs/images/firefly_transaction_manager.jpg differ diff --git a/docs/images/gateway_mode.png b/docs/images/gateway_mode.png new file mode 100644 index 0000000000..20aba8ccd8 Binary files /dev/null and b/docs/images/gateway_mode.png differ diff --git a/docs/images/gateway_multiparty_mode.png b/docs/images/gateway_multiparty_mode.png new file mode 100644 index 0000000000..99ba8e7145 Binary files /dev/null and b/docs/images/gateway_multiparty_mode.png differ diff --git a/docs/images/hyperledger-firefly-namespaces-example-with-org.png b/docs/images/hyperledger-firefly-namespaces-example-with-org.png index cd0a774c8c..2076df4eef 100644 Binary files a/docs/images/hyperledger-firefly-namespaces-example-with-org.png and b/docs/images/hyperledger-firefly-namespaces-example-with-org.png differ diff --git a/docs/images/multiparty_business_process_flow.jpg b/docs/images/multiparty_business_process_flow.jpg new file mode 100644 index 0000000000..53e4c8b8a2 Binary files /dev/null and b/docs/images/multiparty_business_process_flow.jpg differ diff --git a/docs/images/multiparty_business_process_flow.svg b/docs/images/multiparty_business_process_flow.svg deleted file mode 100644 index 2c6ae061e4..0000000000 --- a/docs/images/multiparty_business_process_flow.svg +++ /dev/null @@ -1,1019 +0,0 @@ - -image/svg+xmlStart12InitiateConfirmParty 1AgreedsequenceParty 2Party 3StepTriggerInitiateConfirmStepTrigger3InitiateConfirmEndTriggerMulti- party business process flow diff --git a/docs/images/multiparty_mode.png b/docs/images/multiparty_mode.png new file mode 100644 index 0000000000..73c625e057 Binary files /dev/null and b/docs/images/multiparty_mode.png differ diff --git a/docs/images/multiparty_system.png b/docs/images/multiparty_system.png deleted file mode 100644 index 5e5d061d6e..0000000000 Binary files a/docs/images/multiparty_system.png and /dev/null differ diff --git a/docs/images/multiparty_system1.png b/docs/images/multiparty_system1.png new file mode 100644 index 0000000000..abdcbf7fd7 Binary files /dev/null and b/docs/images/multiparty_system1.png differ diff --git a/docs/images/understanding_firefly1.png b/docs/images/understanding_firefly1.png new file mode 100644 index 0000000000..24ab639084 Binary files /dev/null and b/docs/images/understanding_firefly1.png differ diff --git a/docs/images/without_firefly_with_firefly.png b/docs/images/without_firefly_with_firefly.png deleted file mode 100644 index 4abddc46f5..0000000000 Binary files a/docs/images/without_firefly_with_firefly.png and /dev/null differ diff --git a/docs/index.md b/docs/index.md index c515207352..68527df5a4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,4 +4,5 @@ title: pages.home nav_order: 1 --- -{% tf index.md %} \ No newline at end of file +{% tf index.md %} + diff --git a/docs/overview/blockchain_protocols.md b/docs/overview/blockchain_protocols.md deleted file mode 100644 index 375cdcfd36..0000000000 --- a/docs/overview/blockchain_protocols.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -layout: i18n_page -title: pages.blockchain_protocols -parent: pages.understanding_firefly -nav_order: 4 ---- - -# Blockchain protocols -{: .no_toc } - -## Table of contents -{: .no_toc .text-delta } - -1. TOC -{:toc} - ---- - -## Supporting multiple blockchain protocols - -A blockchain DLT technology is usually the beating heart of a multi-party system, critical to the -new trust and privacy models being established. - -The FireFly API provides an interface for the blockchain tier that is fully pluggable. - -So you can choose the blockchain ecosystem that best meets the functional and -non-functional requirements of your business network, and still benefit from the developer -friendly APIs, event-driven programming model, and on-chain/off-chain coordination provided by FireFly. - -![FireFly Multiple Blockchain Protocols](../images/multi_protocol.png "FireFly Multiple Blockchain Protocols") - -## Core constructs and custom on-chain logic - -Core blockchain programming patterns like pinning proofs to the chain, fungible tokens, -and non-fungible tokens are abstracted into a common API, so that higher level services can -be provided on top of those - such as transaction history for tokens, or on-chain/off-chain -pinning of data. - -> _You can think of these a little like the core CRUD interfaces of a traditional database -> technology. They are the foundation actions that any developer needs to build a multi-party system, -> without necessarily being a blockchain specialist. -> Wherever possible, reference implementations of the on-chain logic should be used, that have been -> peer reviewed through open source collaboration and/or wide production adoption._ - -For fully custom smart contract logic outside of these foundation constructs, FireFly provides -a passthrough mechanism. This means your applications can use the FireFly API and reliable events -for simple access to the blockchain technology, while interacting with rich on-chain -logic. - -So application developers get REST APIs that still abstract away the mechanics of how transaction -submission and events work in the low level RPC interfaces of the blockchain technologies. -However, the on-chain logic can be anything supported by the blockchain itself. - -> _You can think of these a little like the stored procedures of a traditional database technology. -> These allow execution of an exact piece of logic to be performed with a deterministic outcome, -> using only data stored inside of the blockchain itself. One well established way of assuring mutual -> agreement on an outcome in a multi-party system. -> Development of custom smart contracts is usually done by a specialist -> team that understands the technology in detail. They are often subject to -> additional specialist scrutiny such as multi-party review, and external code audit._ - -FireFly is deliberately one step detached from the deployment, upgrade, and maintenance of this -on-chain logic. This approach applies to both the foundational constructs, and the fully custom -on-chain logic. We leave the best practice on this specialist activity to the the individual -blockchain communities. However, the CLI for developers does automate deployment of a reference -set of contracts to get you started. - -A popular approach to innovation in enterprise use cases, is to make small internal customizations -to the operation of standardized peer-reviewed logic (such as token contracts), without updating -the interface. This can allow you to still give the full FireFly API experience to Web/API use -case developers (such as a cached transaction history API for tokens), while allowing the kinds of -innovation only possible by updating the logic on-chain. - -## Blockchain interface plugins - -Different blockchains provide different features, such as multiple separate ledgers (Fabric channels etc.) -or private transaction execution (Tessera etc.). These are mapped to high-level core constructs in the FireFly -model, but with protocol specific configuration that can be passed through from the FireFly API to the -blockchain interface. - -There are sub-communities building the blockchain interfaces for each of the "big 3": - -- Ethereum (Hyperledger Besu, Quorum, Go-ethereum) - - Status: Mature - - Repo: [hyperledger/firefly-ethconnect](https://github.com/hyperledger/firefly-ethconnect) -- Hyperledger Fabric - - Status: Mature - - Repo: [hyperledger/firefly-fabconnect](https://github.com/hyperledger/firefly-fabconnect) -- Corda - - Status: Core model proven. Requires bespoke customization to each CorDapp - - Repo: [hyperledger/firefly-cordaconnect](https://github.com/hyperledger/firefly-cordaconnect) - -## Need help choosing the right blockchain ledger technology? - -The following article might help you compare and contrast: - -- [Enterprise Blockchain Protocols: A Technical Analysis of Ethereum vs Fabric vs Corda](https://www.kaleido.io/blockchain-blog/enterprise-blockchain-protocols-a-technical-analysis-of-ethereum-vs-fabric-vs-corda) diff --git a/docs/overview/firefly_node.md b/docs/overview/firefly_node.md deleted file mode 100644 index 0c623f4985..0000000000 --- a/docs/overview/firefly_node.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -layout: default -title: The FireFly node -parent: pages.understanding_firefly -nav_order: 3 ---- - -# The FireFly node -{: .no_toc } - -## Table of contents -{: .no_toc .text-delta } - -1. TOC -{:toc} - ---- - -## FireFly node runtimes - -A FireFly node is a collection of multiple microservice runtimes with a single unified HTTPS/Websocket API (exposed by the Core). - -![The FireFly node](../images/firefly_node.png "The FireFly node") - -The minimum set of runtimes is as follows: - -- FireFly Core - the API and event server for your multi-party applications -- Blockchain node - the decentralized ledger technology establishing a shared source of truth -- Blockchain interface - transaction submission and event streams for your chosen protocol -- Shared storage node - a network-wide peer-to-peer store of shared data -- Data exchange - for private member to member communications of messages and files - -> _Check out the [FireFly CLI](https://github.com/hyperledger/firefly-cli) to get a -> a multi-party system running on your laptop in minutes._ - -## Pluggable microservices architecture - -The runtimes are pluggable, allowing technology choice, and extensibility. - -- FireFly Core - - Orchestration engine - manages lifecycle of assets and data - - Hosts the API and UI - applications connect here - - Maintains private storage - - Written in Go -- Connectors - - Runtimes that bridge the core to multi-party infrastructure - - Can be written in any language Go, Java, Node.js etc. - - Can be stateful or stateless, depending on requirements - - Can contain significant function, such as managed file transfer, or e2e encryption -- Infrastructure runtimes - - Can be local runtimes, or cloud services - - Blockchain nodes - Fabric, Ethereum, Corda etc. - - Database servers - PostgreSQL, SQLite, CouchDB etc. - - Private messaging - Kafka, RabbitMQ, ActiveMQ, Mosquitto etc. - - Private blob storage - Kubernetes PVCs, AWS S3, Azure File etc. - - Public blob storage - IPFS, etc. - - ... and more - token bridges, trusted compute engines, etc. diff --git a/docs/overview/firefly_supernode.md b/docs/overview/firefly_supernode.md deleted file mode 100644 index 1e9776efdc..0000000000 --- a/docs/overview/firefly_supernode.md +++ /dev/null @@ -1,252 +0,0 @@ ---- -layout: default -title: Introduction to Supernodes -parent: pages.understanding_firefly -nav_order: 1 ---- - -# Introduction to Supernodes -{: .no_toc } - -## Table of contents -{: .no_toc .text-delta } - -1. TOC -{:toc} - ---- - -## What is a Supernode - -Over the last decade of enterprise blockchain projects, architects and developers have realized -that they need much more than a blockchain node for their projects to be successful. - -The development stack needed for an enterprise grade Web3 application, -is just as sophisticated as the stack required for the Web 2.0 applications -that came before. - -A raw blockchain node is simply not enough. - -## Your project with or without a Supernode - -![Without FireFly / with FireFly](../images/without_firefly_with_firefly.png) - -So your choice as a development team for a blockchain project becomes whether you build -and update all of the "plumbing" / "middleware" components needed underneath your business -logic yourself, or whether you look for pre-built solutions. - -The Hyperledger FireFly approach is to allow the community to collaborate on the development and hardening of -these components, across industries and projects. Then fit them into an open source, enterprise grade, -pluggable development and runtime stack... the _Supernode_. - -The application developers then code against these APIs, and can be confident that the business logic that works -on their local laptop against a sandbox, is being written in a way that scales to an enterprise -decentralized application and can be deployed against one or more public/private blockchains in production. - -Thus allowing development teams to focus on differentiation where it matters - at the solution layer. - -## Types of blockchain project that benefit - -There are two main reasons your project might be exploring Supernodes, and considering the open source -approach of Hyperledger FireFly. - -### Solution builders - -Teams building new decentralized Web3 solutions need a full technology stack for -the application, to manage both private and blockchain data. Particularly in the enterprise space -due to data security, regulatory and privacy concerns. - -For theses solutions to be successful, they need decentralized deployment to multiple parties. -Each party needs to customize the deployment to their SecDevOps environment, as well as -onboard it to their key management solution etc. - -So the complexity of requiring a bespoke technology stack for a solution can be a barrier to its adoption. - -Whereas, building on top of a standardized and open technology stack can ease adoption, as well -as radically reducing the amount of engineering needed by the solution developer. - -### Organizations needing a gateway to Web3 - -Organizations are increasingly participating in multiple blockchain projects, and integrating with -digital assets in multiple blockchain ecosystems. - -This means core IT security policy needs to scale to the challenge of adding these connections, -and managing the wallets / signing identities, data flow, and SecDevOps requirements across multiple -projects. - -A gateway tier at the edge between the core systems of the enterprise, and the Web3 transactions, -helps reduce the overhead, and reduce risk. - -## Feature view - -So what makes a Supernode? - -![Hyperledger FireFly features](../images/firefly_functionality_overview.png) - -Let's break down the functionality that is needed for an enterprise blockchain solution. - -## Application features - -Rapidly accelerating development is a key requirement of any Supernode. - -The business logic APIs, web and mobile user experiences for Web3 applications need to be just as rich -and feature-full as the Web 2.0 / centralized applications. - -That means developers skilled in these application layers, must have the tools they need. - -Capabilities fitting their application development toolchain, and optimized to their skillset. - -### API Gateway - -Modern APIs that: - -- Are fast and efficient -- Have rich query support -- Give deterministic outcomes and clear instruction for safe use -- Integrate with your security frameworks like OAuth 2.0 / OpenID Connect single sign-on -- Provide Open API 3 / Swagger definitions -- Come with code SDKs, with rich type information -- Conform as closely as possible to the principles of REST -- Do not pretend to be RESTful in cases when it is impossible to be - -### Event Streams - -The reality is that the only programming paradigm that works for a decentralized solutions, -is an event-driven one. - -All blockchain technologies are for this reason event-driven programming interfaces at their core. - -In an overall solution, those on-chain events must be coordinated with off-chain private -data transfers, and existing core-systems / human workflows. - -This means great event support is a must: - -- Convenient WebSocket APIs that work for your microservices development stack -- Support for Webhooks to integrated serverless functions -- Integration with your core enterprise message queue (MQ) or enterprise service bus (ESB) -- At-least-once delivery assurance, with simple instructions at the application layer - -### API Generation - -The blockchain is going to be at the heart of your Web3 project. While usually small in overall surface -area compared to the lines of code in the traditional application tiers, this kernel of -mission-critical code is what makes your solution transformational compared to a centralized / Web 2.0 solution. - -Whether the smart contract is hand crafted for your project, an existing contract on a public blockchain, -or a built-in pattern of a framework like FireFly - it must be interacted with correctly. - -So there can be no room for misinterpretation in the hand-off between the blockchain -Smart Contract specialist, familiar with EVM contracts in Solidity/Vyper, Fabric chaincode -(or maybe even raw block transition logic in Rust or Go), and the backend/full-stack -application developer / core-system integrator. - -Well documented APIs are the modern norm for this, and it is no different for blockchain. This means: - -- Generating the interface for methods and events on your smart contract -- Providing robust transaction submission, and event streaming -- Publishing the API, version, and location, of your smart contracts to the network - -## Flow features - -Data, value, and process flow are how decentralized systems function. In an enterprise context -not all of this data can be shared with all parties, and some is very sensitive. - -### Private data flow - -Managing the flows of data so that the right information is shared with the right parties, -at the right time, means thinking carefully about what data flows over what channel. - -The number of enterprise solutions where all data can flow directly through the blockchain, -is vanishingly small. - -Coordinating these different data flows is often one of the biggest pieces of heavy lifting solved -on behalf of the application by a robust framework like FireFly: - -- Establishing the identity of participants so data can be shared -- Securing the transfer of data off-chain -- Coordinating off-chain data flow with on-chain data flow -- Managing sequence for deterministic outcomes for all parties -- Integrating off-chain private execution with multi-step stateful business logic - -### Multi-party business process flow - -Web3 has the potential to transform how ecosystems interact. Digitally transforming -legacy process flows, by giving deterministic outcomes that are trusted by all parties, -backed by new forms of digital trust between parties. - -Some of the most interesting use cases require complex multi-step business process across -participants. The Web3 version of business process management, comes with a some new challenges. - -So you need the platform to: - -- Provide a robust event-driven programming model fitting a "state machine" approach -- Integrate with the decentralized application stack of each participant -- Allow integration with the core-systems and human decision making of each participant -- Provide deterministic ordering between all parties -- Provide identity assurance and proofs for data flow / transition logic - -### Data exchange - -Business processes need data, and that data comes in many shapes and sizes. - -The platform needs to handle all of them: - -- Large files and documents, as well as application data -- Uniqueness / Enterprise NFTs - agreement on a single "foreign key" for a record -- Non-repudiation, and acknowledgement of receipt -- Coordination of flows of data, with flows of value - delivery vs. payment scenarios - -## Digital asset features - -The modelling, transfer and management of digital assets is the core programming -foundation of blockchain. - -Yet out of the box, raw blockchains designed to efficiently manage these assets -in large ecosystems, do not come with all the building blocks needed by applications. - -### Token API - -Tokens are such a fundamental construct, that they justify a standard API. -This has been evolving in the industry through standards like ERC-20/ERC-721, -and Web3 signing wallets and that support these. - -Supernodes bring this same standardization to applications. Providing APIs -that work across token standards, and blockchain implementations, providing -consistent and interoperable support. - -This means one application or set of back-end systems, can integrate with multiple -blockchains, and different token implementations. - -Pluggability here is key, so that the rules of governance of each digital -asset ecosystem can be exposed and enforced. Whether tokens are fungible, -non-fungible, or some hybrid in between. - -### Transfer history / audit trail - -For efficiency blockchains seldom provide in their core the ability to -query historical transaction information. Sometimes even the ability -to query balances is unavailable, for blockchains based on a UTXO model. - -So off-chain indexing of transaction history is an absolute must-have -for any digital asset solution, or even a simple wallet application. - -A platform like Hyperledger FireFly provides: - -- Automatic indexing of tokens, whether existing or newly deployed -- Off-chain indexing of fungible and non-fungible asset transfers & balances -- Off-chain indexing of approvals -- Integration with digital identity -- Full extensibility across both token standards and blockchain technologies - -### Wallets - -Wallet and signing-key management is a critical requirement for any -blockchain solution, particularly those involving the transfer -of digital assets between wallets. - -A platform like Hyperledger FireFly provides you the ability to: - -- Integrate multiple different signing/custody solutions in a proven way -- Manage the mapping of off-chain identities to on-chain signing identities -- Provide a plug-point for policy-based decision making on high value transactions -- Manage connections to multiple different blockchain solutions diff --git a/docs/overview/gateway_features.md b/docs/overview/gateway_features.md new file mode 100644 index 0000000000..356688273a --- /dev/null +++ b/docs/overview/gateway_features.md @@ -0,0 +1,132 @@ +--- +layout: i18n_page +title: pages.web3_gateway_features +parent: pages.understanding_firefly +nav_order: 4 +--- + +# Web3 Gateway Features +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +Web3 Gateway features allow your FireFly Supernode to connect to any blockchain ecosystem, public or private. When a chain is connected, the FireFly Supernode may invoke custom smart contracts, interact with tokens, and monitor transactions. A single FireFly Supernode is able to have multiple namespaces, or isolated environments, where each namespace is a connection to a different chain. + +![Gateway Mode](../images/gateway_mode.png "Gateway Mode") + +## Transfer tokenized value + +The [Digital Asset Features](./key_components/digital_assets.md) allow you to connect to +token economies, in multiple blockchains, using the same infrastructure and signing keys. + +The complexities of how each token works, and how each blockchain works, are abstracted +away from you by the Hyperledger FireFly [Connector Framework](./key_components/connectors.md). + +All of the layers of plumbing required to execute a transaction exactly once on a +blockchain, and tracking it through to completion, are part of the stack. Deploy and +configure them once in your Web3 gateway, and use them for multiple use cases in your +enterprise. + +## Invoke any other type of smart contract + +The [API Generation](./key_components/apps.md) features of Hyperledger FireFly, allow +you to generate a convenient and reliable REST API for any smart contract logic. + +Then you just invoke that contract like you would any other API, with all the features +you would expect like an OpenAPI 3.0 specification for the API, and UI explorer. + +The same reliable transaction submission framework is used as for token transfers, +and you can use Hyperledger FireFly as a high volume staging post for those transactions. + +- Handles peaks in workload, drip-feeding transactions onto the chain +- Handles large batch submissions, tracking +- Manages nonce assignment at high volume +- Idempotent APIs assuring that business transactions are submitted exactly once + +> For EVM based chains, these features were significantly enhanced in the new EVMConnect +> connector introduced in v1.1 of FireFly (superseding [EthConnect](https://github.com/hyperledger/firefly-ethconnect)). + +## Index data from the blockchain + +Blockchain nodes are not designed for efficient querying of historical information. Instead +their core function is to provide an ordered ledger of actions+events, along with a consistent +world state at any point in time. + +This means that almost all user experiences and business APIs need a separate +data store, that provides an fast indexed view of the history and current state of the chain. + +As an example, you've probably looked at a _Block Explorer_ for a public blockchain on the web. +Well, you weren't looking directly at the blockchain node. You were querying an off-chain indexed +database, of all the blocks and transactions on that chain. An _indexer_ behind the scenes +was listening to the blockchain and synchronizing the off-chain state. + +Hyperledger FireFly has a built-in indexer for tokens, that maps every token +mint/burn/transfer/approve operation that happens on the the blockchain into the database for +fast query. You just specify which tokens you're interested in, and FireFly takes care of +the rest. + +Additionally, FireFly does the heavy lifting part of indexing for all other types of smart contract +event that might occur. It scrapes the blockchain for the events, formats them into easy to +consume JSON, and reliably delivers them to your application. + +So your application just needs a small bit of code to take those payloads, and insert them +into the database with the right database indexes you need to query your data by. + +## Reliably trigger events in your applications + +One of the most important universal rules about Web3 applications, is that they are event-driven. + +No one party in the system can chose to change the state, instead they must submit transactions +that get ordered against everyone else's transactions, and only once confirmed through the +consensus algorithm are they actioned. + +This means the integration into your application and core systems needs to be event-driven too. + +The same features that support reliable indexing of the blockchain data, allow reliable triggering +of application code, business workflows, and core system integrations. + +> Learn more about the [FireFly Event Bus](../reference/events.html) + +## Manage decentralized data (NFTs etc.) + +Your blockchain transactions are likely to refer to data that is stored off-chain. + +One common example is non-fungible-token (NFT) metadata, images and documents. These are not +a good fit for storing directly in any blockchain ledger, so complimentary decentralized +technologies like the InterPlanetary File System (IPFS) are used to make the data widely +available and resilient outside of the blockchain itself. + +As a publisher or consumer of such metadata from decentralized storage, you need to be confident +you have your own copy safe. So just like with the blockchain data, Hyperledger FireFly can +act as a staging post for this data. + +Structured JSON data can be stored, uploaded and downloaded from the FireFly database. + +Large image/document/video payloads are handled by the pluggable Data Exchange microservice, +which allows you to attach local or cloud storage to manage your copy of the data. + +FireFly then provides a standardized API to allow publishing of this data. So configuring +a reliable gateway to the decentralized storage tier can be done once, and then accessed +from your applications via a single Web3 Gateway. + +## Maintain a private address book + +You need to manage your signing keys, and know the signing keys of others you are +transacting with. A blockchain address like `0x0742e81393ee79C768e84cF57F1bF314F0f31ECe` +is not very helpful for this. + +So Hyperledger FireFly provides a pluggable identity system, built on the foundation of +the Decentralized IDentifier (DID). When in Web3 Gateway Mode these identities are not +shared or published, and simply provide you a local address book. + +You can associate profile information with the identities, for example correlating them +to the identifiers in your own core systems - such as an Identity and Access Management (IAM) +system, or Know Your Customer (KYC) database. + +> Learn more about [Hyperledger FireFly Identities](../reference/identities.html) diff --git a/docs/overview/index.md b/docs/overview/index.md index 2d11133f39..546c5b3ec7 100644 --- a/docs/overview/index.md +++ b/docs/overview/index.md @@ -6,15 +6,3 @@ has_children: true --- # Understanding FireFly -{: .no_toc } - -{: .no_toc .text-delta } - -1. TOC -{:toc} - ---- - -![Hyperledger FireFly features](../images/firefly_functionality_overview.png) - - diff --git a/docs/overview/key_components/apps.md b/docs/overview/key_components/apps.md new file mode 100644 index 0000000000..4be5c79df1 --- /dev/null +++ b/docs/overview/key_components/apps.md @@ -0,0 +1,85 @@ +--- +layout: i18n_page +title: pages.apps +parent: pages.key_features +grand_parent: pages.understanding_firefly +nav_order: 1 +--- + +# Apps +{: .no_toc } + +--- + +![Hyperledger FireFly App Features](../../images/firefly_functionality_overview_apps.png) + +## Apps + +Rapidly accelerating development of applications is a key feature of Hyperledger FireFly. + +The toolkit is designed to support the full-stack of applications in the enterprise Web3 +ecosystem, not just the Smart Contract layer. + +Business logic APIs, back-office system integrations, and web/mobile user experiences are just +as important to the overall Web3 use case. + +These layers require a different developer skill-set to the on-chain Smart Contracts, and those +developers must have the tools they need to work efficiently. + +### API Gateway + +FireFly provides APIs that: + +- Are fast and efficient +- Have rich query support +- Give deterministic outcomes and clear instructions for safe use +- Integrate with your security frameworks like OAuth 2.0 / OpenID Connect single sign-on +- Provide Open API 3 / Swagger definitions +- Come with code SDKs, with rich type information +- Conform as closely as possible to the principles of REST +- Do not pretend to be RESTful in cases when it is impossible to be + +> Learn more about **deploying APIs for custom smart contracts** in [this tutorial](../../tutorials/custom_contracts/) + +### Event Streams + +The reality is that the only programming paradigm that works for a decentralized solutions, +is an event-driven one. + +All blockchain technologies are for this reason event-driven programming interfaces at their core. + +In an overall solution, those on-chain events must be coordinated with off-chain private +data transfers, and existing core-systems / human workflows. + +This means great event support is a must: + +- Convenient WebSocket APIs that work for your microservices development stack +- Support for Webhooks to integrated serverless functions +- Integration with your core enterprise message queue (MQ) or enterprise service bus (ESB) +- At-least-once delivery assurance, with simple instructions at the application layer + +> Learn all about the Hyperledger FireFly **Event Bus**, and **event-driven application architecture**, +> in [this reference section](../../reference/events.html) + +### API Generation + +The blockchain is going to be at the heart of your Web3 project. While usually small in overall surface +area compared to the lines of code in the traditional application tiers, this kernel of +mission-critical code is what makes your solution transformational compared to a centralized / Web 2.0 solution. + +Whether the smart contract is hand crafted for your project, an existing contract on a public blockchain, +or a built-in pattern of a framework like FireFly - it must be interacted with correctly. + +So there can be no room for misinterpretation in the hand-off between the blockchain +Smart Contract specialist, familiar with EVM contracts in Solidity/Vyper, Fabric chaincode +(or maybe even raw block transition logic in Rust or Go), and the backend / full-stack +application developer / core-system integrator. + +Well documented APIs are the modern norm for this, and it is no different for blockchain. + +This means Hyperledger FireFly provides: + +- Generating the interface for methods and events on your smart contract +- Providing robust transaction submission, and event streaming +- Publishing the API, version, and location, of your smart contracts to the network + diff --git a/docs/overview/key_components/connectors.md b/docs/overview/key_components/connectors.md new file mode 100644 index 0000000000..78f0a74056 --- /dev/null +++ b/docs/overview/key_components/connectors.md @@ -0,0 +1,49 @@ +--- +layout: i18n_page +title: pages.connector_framework +parent: pages.key_features +grand_parent: pages.understanding_firefly +nav_order: 3 +--- + +# Connector Framework + +{: .no_toc } + +--- + +![Hyperledger FireFly Connectivity Features](../../images/firefly_functionality_overview_connectivity.png) + +## Pluggable Microservices Architecture + +The ability for every component to be pluggable is at the core of Hyperledger FireFly. + +A microservices approach is used, combining code plug-points in the core runtime, with API extensibility +to remote runtimes implemented in a variety of programming languages. + +[![Hyperledger FireFly Architecture Overview](../../images/firefly_architecture_overview.jpg)](../../images/firefly_architecture_overview.jpg) + +## Extension points + +- Blockchain - a rich framework for extensibility to any blockchain / digital ledger technology (DLT) +- Tokens - mapping token standards and governance models to a common data model +- Shared storage - supporting permissioned and public distributed storage technologies +- Data exchange - private local/storage and encrypted transfer of data +- Identity - flexibility for resolving identities via Decentralized IDentifier (DID) +- Persistence - the local private database + +> Learn more about the [plugin architecture here](../../architecture/plugin_architecture.html) + +## Blockchain Connector Framework + +The most advanced extension point is for the blockchain layer, where multiple layers of extensibility +are provided to support the programming models, and behaviors of different blockchain technologies. + +This framework has been proven with technologies as different as EVM based Layer 2 Ethereum Scaling +solutions like Polygon, all the way to permissioned Hyperledger Fabric networks. + +> Check out instructions to connect to a list of remote blockchain networks [here](../../tutorials/chains/). + +![FireFly Blockchain Connector Framework](../../images/firefly_blockchain_connector_framework.png) + +Find out more about the Blockchain Connector Framework [here](../../architecture/blockchain_connector_framework.html). \ No newline at end of file diff --git a/docs/overview/key_components/digital_assets.md b/docs/overview/key_components/digital_assets.md new file mode 100644 index 0000000000..29d052edfa --- /dev/null +++ b/docs/overview/key_components/digital_assets.md @@ -0,0 +1,74 @@ +--- +layout: i18n_page +title: pages.digital_assets +parent: pages.key_features +grand_parent: pages.understanding_firefly +nav_order: 2 +--- + +# Digital Assets +{: .no_toc } + +--- + +![Hyperledger FireFly Digital Asset Features](../../images/firefly_functionality_overview_digital_assets.png) + +## Digital asset features + +The modelling, transfer and management of digital assets is the core programming +foundation of blockchain. + +Yet out of the box, raw blockchains designed to efficiently manage these assets +in large ecosystems, do not come with all the building blocks needed by applications. + +### Token API + +Token standards have been evolving in the industry through standards +like ERC-20/ERC-721, and the Web3 signing wallets that support these. + +Hyperledger FireFly bring this same standardization to the application tier. +Providing APIs that work across token standards, and blockchain implementations, +providing consistent and interoperable support. + +This means one application or set of back-end systems, can integrate with multiple +blockchains, and different token implementations. + +Pluggability here is key, so that the rules of governance of each digital +asset ecosystem can be exposed and enforced. Whether tokens are fungible, +non-fungible, or some hybrid in between. + +> Learn more about token standards for fungible tokens, and non-fungible +> tokens (NFTs) in [this set of tutorials](../../tutorials/tokens/) + +### Transfer history / audit trail + +For efficiency blockchains do not provide a direct ability to +query historical transaction information. + +Depending on the blockchain technology, even the current balance of your +wallet can be complex to calculate - particularly for blockchain +technologies based on an Unspent Transaction Output (UTXO) model. + +So off-chain indexing of transaction history is an absolute must-have +for any digital asset solution. + +Hyperledger FireFly provides: + +- Automatic indexing of tokens, whether existing or newly deployed +- Off-chain indexing of fungible and non-fungible asset transfers & balances +- Off-chain indexing of approvals +- Integration with digital identity +- Full extensibility across both token standards and blockchain technologies + +### Wallets + +Wallet and signing-key management is a critical requirement for any +blockchain solution, particularly those involving the transfer +of digital assets between wallets. + +Hyperledger FireFly provides you the ability to: + +- Integrate multiple different signing/custody solutions in a proven way +- Manage the mapping of off-chain identities to on-chain signing identities +- Provide a plug-point for policy-based decision making on high value transactions +- Manage connections to multiple different blockchain solutions diff --git a/docs/overview/key_components/flows.md b/docs/overview/key_components/flows.md new file mode 100644 index 0000000000..9bae2d37d4 --- /dev/null +++ b/docs/overview/key_components/flows.md @@ -0,0 +1,98 @@ +--- +layout: i18n_page +title: pages.flows +parent: pages.key_features +grand_parent: pages.understanding_firefly +nav_order: 5 +--- + +# Flows +{: .no_toc } + +--- + +![Hyperledger FireFly Data Flow Features](../../images/firefly_functionality_overview_flows.png) + +## Data flow + +The reality of most Web3 scenarios is that only a small part of the overall use-case +can be represented _inside_ the blockchain or distributed ledger technology. + +Some additional data flow is always required. This does not diminish the value of +executing the kernel of the logic within the blockchain itself. + +Hyperledger FireFly embraces this reality, and allows an organization to keep track +of the relationship between the off-chain data flow, and the on-chain transactions. + +Let's look at a few common examples: + +### Digital Asset Transfers + +Examples of common data flows performed off-chain, include Know Your Customer (KYC) +and Anti Money Laundering (AML) checks that need to be performed and validated +before participating in transactions. + +There might also be document management and business transaction flows required to +verify the conditions are correct to digitally settle a transaction. +Have the goods been delivered? Are the contracts in place? + +In regulated enterprise scenarios it is common to see a 10-to-1 difference in the number +of steps performed off-chain to complete a business transaction, vs. the number +of steps performed on-chain. + +These off-chain data flows might be coordinated with on-chain smart contracts +that lock assets in _digital escrow_ until the off-chain steps are completed by each party, +and protect each party while the steps are being completed. + +> A common form of digital escrow is a Hashed Timelock Contract (HTLC). + +### Non-fungible Tokens (NFTs) and hash-pinning + +The data associated with an NFT might be as simple as a JSON document pointing at an interesting +piece of artwork, or as complex a set of high resolution scans / authenticity documents +representing a _digital twin_ of a real world object. + +Here the concept of a _hash pinning_ is used - allowing anyone who has a copy of the original data +to recreate the hash that is stored in the on-chain record. + +With even the simplest NFT the business data is not stored on-chain, so simple data flow is +always required to publish/download the off-chain data. + +The data might be published publicly for anyone to download, or it might be sensitive and require +a detailed permissioning flow to obtain it from a current holder of that data. + +### Dynamic NFTs and Business Transaction Flow + +In an enterprise context, an NFT might have a dynamic ever-evolving trail of business transaction +data associated with it. Different parties might have different views of that business data, based +on their participation in the business transactions associated with it. + +Here the NFT becomes a like a _foreign key_ integrated across the core systems of a set of enterprises +working together in a set of business transactions. + +The data itself needs to be downloaded, retained, processed and rendered. +Probably integrated to systems, acted upon, and used in multiple exchanges between companies +on different blockchains, or off-chain. + +The business process is accelerated through this **Enterprise NFT** on the blockchain - as all parties +have _matched_ or _bound_ their own private data store to that NFT. This means they are confident +to be executing a business transaction against the same person or thing in the world. + +### Data and Transaction Flow patterns + +Hyperledger FireFly provides the raw tools for building data and transaction flow patterns, such +as storing, hashing and transferring data. It provides the event bus to trigger off-chain +applications and integration to participate in the flows. + +It also provides the higher level flow capabilities that are needed for multiple parties to +build sophisticated transaction flows together, massively simplifying the application logic required: + +- Coordinating the transfer of data off-chain with a blockchain sequencing transaction +- Batching for high throughput transfer via the blockchain and distributed storage technologies +- Managing privacy groups between parties involved in a business transaction +- Masking the relationship between blockchain transactions, for sensitive data + +![Multi-party business process flow](../../images/multiparty_business_process_flow.jpg) + +> Learn more in [Multiparty Process Flows](../multiparty/multiparty_flow.html) + diff --git a/docs/overview/key_components/orchestration_engine.md b/docs/overview/key_components/orchestration_engine.md new file mode 100644 index 0000000000..4474ffd80e --- /dev/null +++ b/docs/overview/key_components/orchestration_engine.md @@ -0,0 +1,56 @@ +--- +layout: i18n_page +title: pages.orchestration_engine +parent: pages.key_features +grand_parent: pages.understanding_firefly +nav_order: 4 +--- + +# Orchestration Engine + +{: .no_toc } + +--- + +![Hyperledger FireFly Orchestration Engine](../../images/firefly_functionality_overview_orchestration_engine.png) + +## FireFly Core + +At the core of Hyperledger FireFly is an event-driven engine that routes, indexed, aggregates, and sequences data +to and from the blockchain, and other connectors. + +![Hyperledger FireFly Orchestration Engine](../../images/firefly_orchestration_engine.png) + +## Data Layer + +Your own private view of the each network you connect: + +- Indexes of all tokens and NFTs that you transact with +- A consistent view across multiple blockchains +- High performance rich query of transaction and data audit trail +- Private data you have received from other parties +- Local copies of data you have download from IPFS or other shared storage tech + +## Event Bus + +Whether a few dozen companies in a private blockchain consortium, or millions of +users connected to a public blockchain network - one thing is always true: + +_Decentralized applications are event-driven._ + +In an enterprise context, you need to think not only about how those events +are being handled and made consistent _within_ the blockchain layer, +but also how those events are being processed and integrated to your core systems. + +FireFly provides you with the reliable streams of events you need, as well +as the interfaces to subscribe to those events and integrate them into your +core systems. + +- Token transfer events, across multiple blockchains, and varied asset types +- Custom smart contract events +- Correlated on-chain and off-chain data events + +![Hyperledger FireFly Event Mode](../../images/firefly_event_model.jpg) + +> Learn more about the event bus and event-driven programming in this +> [reference document](http://localhost:4000/firefly/reference/events.html) \ No newline at end of file diff --git a/docs/overview/key_components/security.md b/docs/overview/key_components/security.md new file mode 100644 index 0000000000..0214b67953 --- /dev/null +++ b/docs/overview/key_components/security.md @@ -0,0 +1,62 @@ +--- +layout: i18n_page +title: pages.security +parent: pages.key_features +grand_parent: pages.understanding_firefly +nav_order: 6 +--- + +# Security +{: .no_toc } + +--- + +![Hyperledger FireFly Security Features](../../images/firefly_functionality_overview_security.png) + +## API Security + +Hyperledger FireFly provides a pluggable infrastructure for authenticating API requests. + +Each [namespace](../../reference/namespaces.html) can be configured with a different authentication +plugin, such that different teams can have different access to resources on the same +FireFly server. + +A reference plugin implementation is provided for HTTP Basic Auth, combined with a `htpasswd` +verification of passwords with a `bcrypt` encoding. + +[See this config section for details](../../reference/config.html#pluginsauth), and the +reference implementation +[in Github](https://github.com/hyperledger/firefly-common/blob/main/pkg/auth/basic/basic_auth.go) + +> Pre-packaged vendor extensions to Hyperledger FireFly are known to be available, addressing more +> comprehensive role-based access control (RBAC) and JWT/OAuth based security models. + +## Data Partitioning and Tenancy + +[Namespaces](../../reference/namespaces.html) also provide a data isolation system for different +applications / teams / tenants sharing a Hyperledger FireFly node. + +![Namespaces](../../images/hyperledger-firefly-namespaces-example-with-org.png) + +Data is partitioned within the FireFly database by namespace. It is also possible to increase the +separation between namespaces, by using separate database configurations. For example to different +databases or table spaces within a single database server, or even to different database servers. + +## Private Data Exchange + +FireFly has a pluggable implementation of a private data transfer bus. This transport supports +both structured data (conforming to agreed data formats), and large unstructured data & documents. + +![Hyperledger FireFly Data Transport Layers](../../images/firefly_data_transport_layers.png) + +A reference microservice implementation is provided for HTTPS point-to-point connectivity with +mutual TLS encryption. + +See the reference implementation +[in Github](https://github.com/hyperledger/firefly-dataexchange-https) + +> Pre-packaged vendor extensions to Hyperledger FireFly are known to be available, addressing +> message queue based reliable delivery of messages, hub-and-spoke connectivity models, chunking +> of very large file payloads, and end-to-end encryption. + +Learn more about these private data flows in [Multiparty Process Flows](../multiparty/multiparty_flow.md). \ No newline at end of file diff --git a/docs/overview/key_components/tools.md b/docs/overview/key_components/tools.md new file mode 100644 index 0000000000..cfa86a1329 --- /dev/null +++ b/docs/overview/key_components/tools.md @@ -0,0 +1,35 @@ +--- +layout: i18n_page +title: pages.tools +parent: pages.key_features +grand_parent: pages.understanding_firefly +nav_order: 7 +--- + +# Tools + +{: .no_toc } + +--- + +![Hyperledger FireFly Tools](../../images/firefly_functionality_overview_tools.png) + +## FireFly CLI + +![FireFly CLI](../../images/firefly_cli.png) + +The FireFly CLI can be used to create local FireFly stacks for offline development of blockchain apps. This allows developers to rapidly iterate on their idea without needing to set up a bunch of infrastructure before they can write the first line of code. + +## FireFly Sandbox + +![FireFly Sandbox](../../images/sandbox/sandbox_broadcast.png) + +The FireFly Sandbox sits logically outside the Supernode, and it acts like an "end-user" application written to use FireFly's API. In your setup, you have one Sandbox per member, each talking to their own FireFly API. The purpose of the Sandbox is to provide a quick and easy way to try out all of the fundamental building blocks that FireFly provides. It also shows developers, through example code snippets, how they would implement the same functionality in their own app's backend. + +> 🗒 Technical details: The FireFly Sandbox is an example "full-stack" web app. It has a backend written in TypeScript / Node.js, and a frontend in TypeScript / React. When you click a button in your browser, the frontend makes a request to the backend, which then uses the [FireFly Node.js SDK](https://www.npmjs.com/package/@hyperledger/firefly-sdk) to make requests to FireFly's API. + +## FireFly Explorer + +The FireFly explorer is a part of FireFly Core itself. It is a view into the system that allows operators to monitor the current state of the system and investigate specific transactions, messages, and events. It is also a great way for developers to see the results of running their code against FireFly's API. + +![FireFly Explorer](../../images/firefly_explorer.png) diff --git a/docs/overview/key_features.md b/docs/overview/key_features.md new file mode 100644 index 0000000000..914188dc0a --- /dev/null +++ b/docs/overview/key_features.md @@ -0,0 +1,18 @@ +--- +layout: i18n_page +title: pages.key_features +parent: pages.understanding_firefly +nav_order: 3 +has_children: true +--- + +# Key Features +{: .no_toc } + +--- +![Hyperledger FireFly features](../images/firefly_functionality_overview.png) + +Hyperledger FireFly provides a rich suite of features for building new applications, and connecting +existing Web3 ecosystems to your business. In this section we introduce each core pillar of functionality. + + diff --git a/docs/overview/broadcast.md b/docs/overview/multiparty/broadcast.md similarity index 74% rename from docs/overview/broadcast.md rename to docs/overview/multiparty/broadcast.md index b3b0ac32c8..510ffe711f 100644 --- a/docs/overview/broadcast.md +++ b/docs/overview/multiparty/broadcast.md @@ -1,19 +1,14 @@ --- layout: i18n_page -title: pages.broadcast_shared_data -parent: pages.understanding_firefly -nav_order: 8 +title: pages.broadcast_data +parent: pages.multiparty_features +grand_parent: pages.understanding_firefly +nav_order: 3 --- # Broadcast / shared data {: .no_toc } -## Table of contents -{: .no_toc .text-delta } - -1. TOC -{:toc} - --- ## Introduction @@ -23,7 +18,7 @@ often that needs to include certain reference data that is available to all parties in the network. The data needs to be "broadcast" to all members, and also need to be available to new members that join the network -![Multi-party Systems](../images/multiparty_system.png "Multi-Party System") +![Multi-party Systems](../../images/multiparty_system1.png "Multi-Party System") ## Blockchain backed broadcast @@ -35,7 +30,8 @@ Using the blockchain also gives a global order of events for these broadcasts, which allows them to be processed by each member in a way that allows them to derive the same result - even though the processing logic on the events themselves is being performed independently by each member. -For more information see _Global sequencing_. + +For more information see [Multiparty Event Sequencing](../../architecture/multiparty_event_sequencing.html). ## Shared data @@ -64,11 +60,11 @@ all parties in the network: - Network map - Organizational identities - Nodes - - See _Identity_ for more information + - See [Identities](../../reference/identities.html) in the reference section for more information - Datatype definitions - - See _Agreed datatypes_ for more information + - See [Datatype](../../reference/types/datatype.html) in the reference section for more information - Namespaces - - See _Namespaces_ for more information + - See [Namespaces](../../reference/namespaces.html) for more information These definitions rely on the same assurances provided by blockchain backed broadcast that FireFly applications do. @@ -77,15 +73,3 @@ broadcast that FireFly applications do. - Deterministic assignment of a namespace+name to an unique item of data - If two parties in the network broadcast the same data at similar times, the same one "wins" for all parties in the network (including the broadcaster) - -## Network Registry - -> _Work in progress_ - -Using a permissioned blockchain and shared data network provides a security mechanism -to protect against broadcast data being published from outside of the network. - -However, some networks might have additional permissioning and security requirements -on joining the network. As such FireFly defines a plug-point for a Network Registry -that defines a way to establish authorization to perform a broadcast (that is decoupled -from the blockchain and shared data tiers themselves). diff --git a/docs/overview/data_exchange.md b/docs/overview/multiparty/data_exchange.md similarity index 93% rename from docs/overview/data_exchange.md rename to docs/overview/multiparty/data_exchange.md index f08572b8ee..5647c7f5e1 100644 --- a/docs/overview/data_exchange.md +++ b/docs/overview/multiparty/data_exchange.md @@ -1,29 +1,24 @@ --- layout: i18n_page title: pages.private_data_exchange -parent: pages.understanding_firefly -nav_order: 7 +parent: pages.multiparty_features +grand_parent: pages.understanding_firefly +nav_order: 2 --- # Private data exchange {: .no_toc } -## Table of contents -{: .no_toc .text-delta } - -1. TOC -{:toc} - --- ## Introduction Private data exchange is the way most enterprise business-to-business communication -happens today. One party private sends data to another, over a pipe that has been +happens today. One party privately sends data to another, over a pipe that has been agreed as sufficiently secure between the two parties. That might be a REST API, SOAP Web Service, FTP / EDI, Message Queue (MQ), or other B2B Gateway technology. -![Multi-party Systems](../images/multiparty_system.png "Multi-Party System") +![Multi-party Systems](../../images/multiparty_system1.png "Multi-Party System") The ability to perform these same private data exchanges within a multi-party system is critical. In fact it's common for the majority of business diff --git a/docs/overview/deterministic_compute.md b/docs/overview/multiparty/deterministic.md similarity index 94% rename from docs/overview/deterministic_compute.md rename to docs/overview/multiparty/deterministic.md index 80634a4cd9..cbc0c90e0a 100644 --- a/docs/overview/deterministic_compute.md +++ b/docs/overview/multiparty/deterministic.md @@ -1,19 +1,14 @@ --- -layout: default -title: Deterministic compute -parent: pages.understanding_firefly -nav_order: 6 +layout: i18n_page +title: pages.deterministic +parent: pages.multiparty_features +grand_parent: pages.understanding_firefly +nav_order: 4 --- -# Deterministic compute +# Deterministic Compute {: .no_toc } -## Table of contents -{: .no_toc .text-delta } - -1. TOC -{:toc} - --- ## Introduction @@ -60,7 +55,7 @@ mature blockchain technology, and all multi-party systems should consider exploi - Gives you an anchor to attach to something in the real world - This is the magic behind non-fungible tokens (NTFs) - The proven technology for this is a shared ledger of its creation, and ownership changes - - _Learn more in the Tokens section_ + - Learn more in the [Tokens](../../tutorials/tokens/index.html) section - An agreed sequence of events - The foundation tool that allows the building of higher level constructs (including tokens) - Not previously available when business ecosystems used HTTP/Messaging transports alone @@ -71,7 +66,7 @@ mature blockchain technology, and all multi-party systems should consider exploi - The glue that binds a piece of private data, to a proof that you have a copy of that data - This is the basis of "pinning" data to the blockchain, without sharing its contents - Care needs to be taken to make sure the data is unique enough to make the hash secure - - _Learn more in the On-chain/off-chain coordination section_ + - Learn more in the [Gateway Features](../gateway_features.html#manage-decentralized-data-nfts-etc) section ## Advanced Cryptography and Privacy Preserving Trusted Compute @@ -106,7 +101,7 @@ in the network. Those businesses then bring on-board an ecosystem of employees a are end-users to the system. So the shared source of truth empowered by the blockchain and other cryptography are not the only -tools that can be used in the toolbox to ensure correct behavior. Regognizing that there are real +tools that can be used in the toolbox to ensure correct behavior. Recognizing that there are real legal entities involved, that are mature and regulated, does not undermine the value of the blockchain components. In fact it enhances it. @@ -122,3 +117,4 @@ business relationships, by being able to agree the order and sequence of a set o tools to digitize processes that previously took physical documents flying round the world, into near-immediate digital agreement where the arbitration of a dispute can be resolved at a tiny fraction of what would have been possible without a shared and immutable audit trail of who said what when. + diff --git a/docs/overview/multiparty/multiparty_flow.md b/docs/overview/multiparty/multiparty_flow.md new file mode 100644 index 0000000000..94fa6da354 --- /dev/null +++ b/docs/overview/multiparty/multiparty_flow.md @@ -0,0 +1,145 @@ +--- +layout: i18n_page +title: pages.multiparty_flow +parent: pages.multiparty_features +grand_parent: pages.understanding_firefly +nav_order: 1 +--- + +# Multiparty Process Flows +{: .no_toc } + +--- + +## Flow features + +Data, value, and process flow are how decentralized systems function. In an enterprise context +not all of this data can be shared with all parties, and some is very sensitive. + +### Private data flow + +Managing the flows of data so that the right information is shared with the right parties, +at the right time, means thinking carefully about what data flows over what channel. + +The number of enterprise solutions where all data can flow directly through the blockchain, +is vanishingly small. + +Coordinating these different data flows is often one of the biggest pieces of heavy lifting solved +on behalf of the application by a robust framework like FireFly: + +- Establishing the identity of participants so data can be shared +- Securing the transfer of data off-chain +- Coordinating off-chain data flow with on-chain data flow +- Managing sequence for deterministic outcomes for all parties +- Integrating off-chain private execution with multi-step stateful business logic + +### Multi-party business process flow + +Web3 has the potential to transform how ecosystems interact. Digitally transforming +legacy process flows, by giving deterministic outcomes that are trusted by all parties, +backed by new forms of digital trust between parties. + +Some of the most interesting use cases require complex multi-step business process across +participants. The Web3 version of business process management, comes with a some new challenges. + +So you need the platform to: + +- Provide a robust event-driven programming model fitting a "state machine" approach +- Integrate with the decentralized application stack of each participant +- Allow integration with the core-systems and human decision making of each participant +- Provide deterministic ordering between all parties +- Provide identity assurance and proofs for data flow / transition logic + +### Data exchange + +Business processes need data, and that data comes in many shapes and sizes. + +The platform needs to handle all of them: + +- Large files and documents, as well as application data +- Uniqueness / Enterprise NFTs - agreement on a single "foreign key" for a record +- Non-repudiation, and acknowledgement of receipt +- Coordination of flows of data, with flows of value - delivery vs. payment scenarios + +## Building multi-party flows + +The ability to globally sequence events _across parties_ is a game changing capability of a multiparty +system. FireFly is designed to allow developers to harnesses that power in the application layer, to build +sophisticated multi-party APIs and user experiences. + +[![Multi-party business process flow](../../images/multiparty_business_process_flow.jpg "Multi-party business process flow")](../../images/multiparty_business_process_flow.jpg) + +- Build multi-party business processes where there is one agreed outcome: + - Agree the trigger, inputs, outputs of each step in the process + - Agree any common "rules of the road" must be adhered to +- Look back at your shared history, when deciding to commit to the next step: + - Fast rich-query cache, backed by a private database + - Initiate the next step through automated or manual decision making + - Only consider a step final once it's multi-party sequence has been confirmed +- Gain big efficiencies in how multi-party business processes work: + - Once locked in, a step is consider final - attested to by the party + - If two parties submit conflicting actions, one wins, and one loses + - Avoids complex compensation logic in the business orchestration layer + - Provides one clear source of truth to quickly resolve multi-party disputes +- Program multi-party apps using the tools you know: + - REST APIs for triggering the next step in a process, and querying history + - WebSockets and Webhooks for events (pluggable to other event transports) + - Remember - each party runs their own copy of the app, with their own private data +- Allow each party to integrate into their existing core systems: + - Realtime or batch + - Human workflows + - Proprietary business logic that is unique to one party +- Avoid sensitive data written to the blockchain: + - Works in bi-lateral and multi-lateral scenarios + - Designed to limit leaking other "metadata" about the transaction as well + - Share partial history with different participants in a +- No requirement to write custom on-chain smart contract logic: + - Can be combined with rich custom on-chain logic as well + +## Innovate fast + +Building a successful multi-party system is often about business experimentation, and business results. +Proving the efficiency gains, and new business models, made possible by working together in a new way +under a new system of trust. + +Things that can get in the way of that innovation, can include concerns over data privacy, technology +maturity, and constraints on autonomy of an individual party in the system. An easy to explain position +on how new technology components are used, where data lives, and how business process independence +is maintained can really help parties make the leap of faith necessary to take the step towards a new +model. + +Keys to success often include building great user experiences that help digitize clunky decades old +manual processes. Also easy to integrate with APIs, what embrace the existing core systems of record +that are establish within each party. + +## Consider the on-chain toolbox too + +There is a huge amount of value that deterministic execution of multi-party logic within the blockchain can add. +However, the more compute is made fully deterministic via a blockchain consensus algorithm validated +by multiple parties beyond those with a business need for access to the data, the more sensitivity +needs to be taken to data privacy. Also bear in mind any data that is used in this processing +becomes immutable - it can never be deleted. + +The core constructs of blockchain are a great place to start. +Almost every process can be enhanced with pre-built fungible and non-fungible tokens, for example. +Maybe it's to build a token economy that enhances the value parties get from the system, +or to encourage healthy participation (and discourage bad behavior). +Or maybe it's to track exactly which party owns a document, asset, or action within a process using NFTs. + +On top of this you can add advanced tools like digital escrow, signature / threshold based voting +on outcomes, and atomic swaps of value/ownership. + +The investment in building this bespoke on-chain logic is higher than building the off-chain pieces +(and there are always some off-chain pieces as we've discussed), so it's about finding the kernel +of value the blockchain can provide to differentiate your solution from a centralized database solution. + +The power provided by deterministic sequencing of events, attested by signatures, and pinned +to private data might be sufficient for some cases. In others the token constructs are the key value +that differentiates the decentralized ecosystem. Whatever it is, it's important it is identified and +crafted carefully. + +> Note that advanced privacy preserving techniques such as zero-knowledge proofs (ZKP) are gaining traction +> and hardening in their production readiness and efficiency. Expect these to play an increasing +> role in the technology stack of multiparty systems (and Hyperledger FireFly) in the future. + +Learn more in the [Deterministic Compute](./deterministic.md) section. diff --git a/docs/overview/multiparty.md b/docs/overview/multiparty_features.md similarity index 50% rename from docs/overview/multiparty.md rename to docs/overview/multiparty_features.md index 22efd359b1..ba5fd46766 100644 --- a/docs/overview/multiparty.md +++ b/docs/overview/multiparty_features.md @@ -1,14 +1,17 @@ --- -layout: default -title: Multi-party Systems +layout: i18n_page +title: pages.multiparty_features parent: pages.understanding_firefly -nav_order: 2 +nav_order: 5 +has_children: true --- # Enterprise multi-party systems + {: .no_toc } ## Table of contents + {: .no_toc .text-delta } 1. TOC @@ -18,7 +21,9 @@ nav_order: 2 ## Introduction -Multi-party systems are a class of application empowered by the technology revolution +Multiparty mode has all the features in Gateway mode with the added benefit of multi-party process flows. + +A multi-party system is a class of application empowered by the technology revolution of blockchain digital ledger technology (DLT), and emerging cryptographic proof technologies like zero-knowledge proofs (ZKPs) and trusted execution environments (TEEs). @@ -26,11 +31,11 @@ By combining these technologies with existing best practice technologies for data security in regulated industries, multi-party systems allow businesses to collaborate in ways previously impossible. -![Multi-party Systems](../images/multiparty_system.png "Multi-Party System") +![Multiparty Mode](../images/multiparty_mode.png "Multiparty Mode") Through agreement on a common source of truth, such as the completion of a step in a business process to proceed, or the existence and ownership of a unique asset, businesses -can cut out huge inefficiencies in existing multi-party processes. +can cut out huge inefficiencies in existing multi-party processes. New business and transaction models can be achieved, unlocking value in assets and data that were previously siloed within a single organization. Governance and incentive @@ -50,8 +55,21 @@ each party, rather than seeking to unify or replace them. Multi-party systems are different from centralized third-party systems, because each party retains sovereignty over: - - Their application instance - - Their private data - - Their business processes - - Their proprietary business logic - - Their internal business processes and IT controls + +- Their application instance +- Their private data +- Their business processes +- Their proprietary business logic +- Their internal business processes and IT controls + +## Use Case Example + +There are many multiparty use cases. An example for healthcare is detailed below. + +Patient care requires multiple entities to work together including healthcare providers, insurance companies, and medical systems. Sharing data between these parties is inefficient and prone to errors and patient information must be kept secure and up to date. Blockchain's shared ledger makes it possible to automate data sharing while ensuring accuracy and privacy. + +In a Multi-party FireFly system, entities are able to share data privately as detailed in the "Data Exchange" section. For example, imagine a scenario where there is one healthcare provide and two insurance companies operating in a multi-party system. Insurance company A may send private data to the healthcare provider that insurance company B is not privy to. While insurance company B may not know the contents of data transferred, it may verify that a transfer of data did occur. This validation is all thats needed to maintain an up to date state of the blockchain. + +In a larger healthcare ecosystem with many members, a similar concept may emerge with multiple variations of members. + + \ No newline at end of file diff --git a/docs/overview/multiparty_process_flow.md b/docs/overview/multiparty_process_flow.md deleted file mode 100644 index ec8e988d2a..0000000000 --- a/docs/overview/multiparty_process_flow.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -layout: default -title: Multi-party process flow -parent: pages.understanding_firefly -nav_order: 9 ---- - -# Multi-party process flow -{: .no_toc } - -## Table of contents -{: .no_toc .text-delta } - -1. TOC -{:toc} - ---- - -## Building multi-party flows - -The ability to globally sequence events _across parties_ is a game changing capability of multi-party -systems. FireFly is designed to allow developers to harnesses that power in the application layer, to build -sophisticated multi-party APIs and user experiences. - -[![Multi-party business process flow](../images/multiparty_business_process_flow.svg "Multi-party business process flow")](../images/multiparty_business_process_flow.svg) - -- Build multi-party business processes where there is one agreed outcome: - - Agree the trigger, inputs, outputs of each step in the process - - Agree any common "rules of the road" must be adhered to -- Look back at your shared history, when deciding to commit to the next step: - - Fast rich-query cache, backed by a private database - - Initiate the next step through automated or manual decision making - - Only consider a step final once it's multi-party sequence has been confirmed -- Gain big efficiencies in how multi-party business processes work: - - Once locked in, a step is consider final - attested to by the party - - If two parties submit conflicting actions, one wins, and one loses - - Avoids complex compensation logic in the business orchestration layer - - Provides one clear source of truth to quickly resolve multi-party disputes -- Program multi-party apps using the tools you know: - - REST APIs for triggering the next step in a process, and querying history - - WebSockets and Webhooks for events (pluggable to other event transports) - - Remember - each party runs their own copy of the app, with their own private data -- Allow each party to integrate into their existing core systems: - - Realtime or batch - - Human workflows - - Proprietary business logic that is unique to one party -- Avoid sensitive data written to the blockchain: - - Works in bi-lateral and multi-lateral scenarios - - Designed to limit leaking other "metadata" about the transaction as well - - Share partial history with different participants in a -- No requirement to write custom on-chain smart contract logic: - - Can be combined with rich custom on-chain logic as well - -## Innovate fast - -Building a successful multi-party system is often about business experimentation, and business results. -Proving the efficiency gains, and new business models, made possible by working together in a new way -under a new system of trust. - -Things that can get in the way of that innovation, can include concerns over data privacy, technology -maturity, and constraints on autonomy of an individual party in the system. An easy to explain position -on how new technology components are used, where data lives, and how business process independence -is maintained can really help parties make the leap of faith necessary to take the step towards a new -model. - -Keys to success often include building great user experiences that help digitize clunky decades old -manual processes. Also easy to integrate with APIs, what embrace the existing core systems of record -that are establish within each party. - -## Consider the on-chain toolbox too - -In the _deterministic compute_ section we talked about the value that deterministic execution -of multi-party logic can have. Either through on-chain execution, or advanced privacy preserving -techniques. - -It's important to state that almost every process can be enhanced with more sophisticated -on-chain constructs like tokens. Maybe it's to build a token economy that enhances the value -parties get from the system, or encourages healthy participation (and discourages leaching value). -Or maybe it's to track exactly which party owns a document, asset, or action within a process using NFTs. - -There are also cases where the foundation constructs are insufficient to implement the level of -automation or efficiency you need in your multi-party process. Here making the investment in building -bespoke on-chain logic, or apply advanced cryptographic techniques, is the linchpin to a successful -multi-party ecosystem. - diff --git a/docs/overview/public_vs_permissioned.md b/docs/overview/public_vs_permissioned.md index 499ef617e3..2e1aee2a63 100644 --- a/docs/overview/public_vs_permissioned.md +++ b/docs/overview/public_vs_permissioned.md @@ -1,4 +1,4 @@ ---- + diff --git a/docs/overview/supernode_concept.md b/docs/overview/supernode_concept.md new file mode 100644 index 0000000000..49fc3f7f1c --- /dev/null +++ b/docs/overview/supernode_concept.md @@ -0,0 +1,59 @@ +--- +layout: i18n_page +title: pages.introduction +parent: pages.understanding_firefly +nav_order: 1 +--- + +# Introduction to Hyperledger FireFly +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +## Your Gateway to Web3 Technologies + +![Hyperledger FireFly features](../images/firefly_intro_overview.png) + +Hyperledger FireFly is an organization's gateway to Web3, including all the blockchain ecosystems that they participate in. + +Multiple blockchains, multiple token economies, and multiple business networks. + +FireFly is not another blockchain implementation, rather it is a pluggable API Orchestration and Data layer, +integrating into all of the different types of decentralized technologies that exist in Web3: + +- Public Blockchains, Layer 2 scaling solutions, Side chains, and App chains +- Permissioned Blockchains and Distributed Ledger Technologies (DLTs) +- Decentralized storage solutions +- Token ecosystems and standards +- Smart Contracts, DeFi solutions and DAOs +- Private off-chain encrypted communication rails +- Advanced cryptography solutions +- Identity frameworks + +## An Open Source Supernode for Web3 Apps + +Hyperledger FireFly is a toolkit for building and connecting new full-stack decentralized applications (dapps), +as well as integrating your existing core systems to the world of Web3. + +It has a runtime engine, and it provides a data layer that synchronizes state from the blockchain and other Web3 technologies. +It exposes an API and Event Bus to your business logic, that is reliable, developer friendly and ready for enterprise use. + +We call this a _Supernode_ - it sits between the application and the underlying infrastructure nodes, +providing layers of additional function. + +![Without FireFly / with FireFly](../images/firefly-getting-started-steps.png) + +The concept of a Supernode has evolved over the last decade of enterprise blockchain projects, as developers realized +that they need much more than a blockchain node for their projects to be successful. + +Without a technology like Hyperledger FireFly, the application layer becomes extremely complex and fragile. +Tens of thousands of lines of complex low-level "plumbing" / "middleware" code is required to integrate the +web3 infrastructure into the application. This code provides zero unique business value to the solution, but can +consume a huge proportion of the engineering budget and maintenance cost if built bespoke within a solution. + diff --git a/docs/overview/usage_patterns.md b/docs/overview/usage_patterns.md new file mode 100644 index 0000000000..19ea49699e --- /dev/null +++ b/docs/overview/usage_patterns.md @@ -0,0 +1,62 @@ +--- +layout: i18n_page +title: pages.usage_patterns +parent: pages.understanding_firefly +nav_order: 2 +--- + +# Usage Patterns +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +There are two modes of usage for Hyperledger Firefly: **Web3 Gateway** and **Multiparty** + +> A single runtime can operate in both of these modes, using different [namespaces](../reference/namespaces.md). + +## Web3 Gateway Mode + +![Gateway Mode](../images/gateway_mode.png) + +Web3 Gateway mode lets you interact with any Web3 application, regardless of whether Hyperledger FireFly +is being used by other members of your business network. + +In this mode you can: +- Transfer tokenized value +- Invoke any other type of smart contract +- Index data from the blockchain +- Reliably trigger events in your applications and back-office core systems +- Manage decentralized data (NFTs etc.) +- Use a _private_ address book to manage signing identities and relationships +- ... and much more + +Learn more about [Web3 Gateway Mode](./gateway_features.html). + +## Multiparty Mode + +Multiparty mode is used to build multi-party systems, with a common application runtime deployed by each enterprise participant. + +![Multiparty Mode](../images/multiparty_mode.png) + +This allows sophisticated applications to be built, that all use the pluggable APIs of Hyperledger FireFly to achieve +end-to-end business value in an enterprise context. + +In this mode you can do everything you could do in Web3 Gateway mode, plus: +- Share and enforce common data formats +- Exchange data privately, via an encrypted data bus + - Structured JSON data payloads + - Large documents +- Coordinate on-chain and off-chain data exchange + - Private data + - Broadcast data +- Mask on-chain activities using hashes +- Use a _shared_ address book to manage signing identities and relationships +- ... and much more + +Learn more about [Multiparty Mode](./multiparty_features.html). diff --git a/docs/reference/events.md b/docs/reference/events.md index 2629df8dd0..d182c530a3 100644 --- a/docs/reference/events.md +++ b/docs/reference/events.md @@ -30,7 +30,7 @@ like NATS, Kafka, and JMS Servers can be connected through plugins. Each application creates one or more [Subscriptions](./types/subscription.md) to identify itself. In this subscription the application can choose to receive all events that -are emitted within a `namespace`, or can use server-side filtering to +are emitted within a `namespace`, or can use server-side filtering to only receive a sub-set of events. The event bus reliably keeps track of which events have been delivered to which @@ -136,7 +136,7 @@ cheat, and must follow the rules. How much of that rule enforcement needs to be executed on-chain vs. off-chain (backed by a deterministic order through the blockchain) is different for each use case. -> Remember that tokens provide a great set of building blocks for on-chain steps in +> Remember that tokens provide a great set of building blocks for on-chain steps in > your decentralized applications. Enterprise NFTs allow generation of a globally > unique ID, and track ownership. Fungible tokens allow value transfer, and can be > extended with smart contracts that to lock/unlock funds in "digital escrow" @@ -246,7 +246,7 @@ API and Events directly. Event aggregation between data arriving off-chain, and the associated ordered proof/transaction events being confirmed on-chain, is a complex orchestration task. -The universal order and additional transaction logic **on-chain must be the +The universal order and additional transaction logic **on-chain must be the source of truth** for when and how an event is processed. However, that event cannot be processed until the off-chain private/broadcast data diff --git a/docs/reference/namespaces.md b/docs/reference/namespaces.md index f4cf68d647..77de679467 100644 --- a/docs/reference/namespaces.md +++ b/docs/reference/namespaces.md @@ -33,7 +33,7 @@ They can be thought of in two basic modes: ### Multi-party Namespaces This namespace is shared with one or more other FireFly nodes. It requires three types of communication plugins - blockchain, data exchange, and shared storage. Organization and node identities must be claimed with an identity broadcast when joining the namespace, which establishes credentials for blockchain and off-chain communication. Shared objects can be defined in the namespace (such as datatypes and token pools), and details of them will be implicitly broadcast to other members. -This type of namespace is used when multiple parties need to share on- and off-chain data and agree upon the ordering and authenticity of that data. For more information, see the [multi-party system](../overview/multiparty.md) overview. +This type of namespace is used when multiple parties need to share on- and off-chain data and agree upon the ordering and authenticity of that data. For more information, see the [multi-party system](../overview/multiparty_features.md) overview. ### Gateway Namespaces diff --git a/docs/reference/types/_includes/tokentransfer_description.md b/docs/reference/types/_includes/tokentransfer_description.md index 1157f0446e..abce382852 100644 --- a/docs/reference/types/_includes/tokentransfer_description.md +++ b/docs/reference/types/_includes/tokentransfer_description.md @@ -16,7 +16,7 @@ Hyperledger FireFly maintains this index automatically for all Token Pools that There is no requirement at all to use FireFly to initiate transfers in Token Pools that Hyperledger FireFly is aware of. FireFly will listen to and update its audit history and balances for all transfers, regardless of whether they were initiated using a FireFly -SuperNode or not. +Supernode or not. So you could for example use Metamask to initiate a transfer directly against an ERC-20/ERC-721 contract directly on your blockchain, and you will see it appear as a transfer. Or initiate diff --git a/docs/tutorials/broadcast_data.md b/docs/tutorials/broadcast_data.md index a82de9ea8a..a7a78c52c2 100644 --- a/docs/tutorials/broadcast_data.md +++ b/docs/tutorials/broadcast_data.md @@ -33,7 +33,7 @@ nav_order: 1 ## Additional info -- Key Concepts: [Broadcast / shared data](../overview/broadcast.md) +- Key Concepts: [Broadcast / shared data](../overview/multiparty/broadcast.html) - Swagger Reference: POST /api/v1/namespaces/{ns}/messages/broadcast ## Example 1: Inline string data @@ -221,7 +221,7 @@ In the sandbox, enter your message into the message field as seen in the screens Notice how the `data` field in the center panel updates in real time. -Click the blue `Run` button. This should return a `202` response immediately in the Server Response section and will populate the right hand panel with transaction information after a few seconds. +Click the blue `Run` button. This should return a `202` response immediately in the Server Response section and will populate the right hand panel with transaction information after a few seconds. ![Broadcast Result](../images/message_broadcast_sample_result.png) diff --git a/docs/tutorials/chains/Arbitrum.md b/docs/tutorials/chains/Arbitrum.md new file mode 100644 index 0000000000..ab23a54e16 --- /dev/null +++ b/docs/tutorials/chains/Arbitrum.md @@ -0,0 +1,111 @@ +--- +layout: i18n_page +title: pages.arbitrum +parent: pages.chains +grand_parent: pages.tutorials +nav_order: 4 +--- + + +# {%t pages.arbitrum %} +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +Starting with FireFly v1.1, it's easy to connect to public Ethereum chains. This guide will walk you through the steps to create a local FireFly development environment and connect it to the Arbitrum Nitro Goerli Rollup Testnet. + +## Previous steps: Install the FireFly CLI +If you haven't set up the FireFly CLI already, please go back to the Getting Started guide and read the section on how to [Install the FireFly CLI](../../gettingstarted/firefly_cli.md). + +[← ① Install the FireFly CLI](../../gettingstarted/firefly_cli.md){: .btn .btn-purple .mb-5} + +## Create an `evmconnect.yml` config file +In order to connect to the Binance Smart Chain testnet, you will need to set a few configuration options for the evmconnect blockchain connector. Create a text file called `evmconnect.yml` with the following contents: + +```yml +confirmations: + required: 4 +policyengine.simple: + fixedGasPrice: null + gasOracle: + mode: connector +``` + +For this tutorial, we will assume this file is saved at `~/Desktop/evmconnect.yml`. If your path is different, you will need to adjust the path in the next command below. + +## Creating a new stack +To create a local FireFly development stack and connect it to the Arbitrum testnet, we will use command line flags to customize the following settings: + + - Create a new stack named `arbitrum` with `1` member + - Disable `multiparty` mode. We are going to be using this FireFly node as a Web3 gateway, and we don't need to communicate with a consortium here + - Connect to an ethereum network + - Use the `evmconnect` blockchain connector + - Use an remote RPC node. This will create a signer locally, so that our signing key never leaves the development machine. + - Set the remote RPC node URL to `https://goerli-rollup.arbitrum.io/rpc` (for a full list of testnet RPC node urls visit https://developer.offchainlabs.com/docs/Public_Chains) + - Set the chain ID to `421613` (the correct ID for the Binance Smart Chain testnet) + - Merge the custom config created above with the generated `evmconnect` config file + +To do this, run the following command: +``` +ff init arbitrum 1 \ + --multiparty=false \ + -b ethereum \ + -c evmconnect \ + -n remote-rpc \ + --remote-node-url https://goerli-rollup.arbitrum.io/rpc \ + --chain-id 421613 \ + --connector-config ~/Desktop/evmconnect.yml +``` + +## Start the stack +Now you should be able to start your stack by running: + +``` +ff start arbitrum +``` + +After some time it should print out the following: + +``` +Web UI for member '0': http://127.0.0.1:5000/ui +Sandbox UI for member '0': http://127.0.0.1:5109 + + +To see logs for your stack run: + +ff logs arbitrum +``` + +## Get some Aribitrum +At this point you should have a working FireFly stack, talking to a public chain. However, you won't be able to run any transactions just yet, because you don't have any way to pay for gas. + +First, you will need to know what signing address your FireFly node is using. To check that, you can run: + +``` +ff accounts list arbitrum +[ + { + "address": "0x225764d1be1f137be23ddfc426b819512b5d0f3e", + "privateKey": "..." + } +] +``` + +Copy the address listed in the output from this command. Next, check out this article [https://medium.com/offchainlabs/new-g%C3%B6rli-testnet-and-getting-rinkeby-ready-for-nitro-3ff590448053](https://medium.com/offchainlabs/new-g%C3%B6rli-testnet-and-getting-rinkeby-ready-for-nitro-3ff590448053) and follow the instructions to send a tweet to the developers. Make sure to change the address to the one in the CLI. + +![Arbitrum Faucet](images/arbitrum_faucet.png) + +### Confirm the transaction on Bscscan +You should be able to go lookup your account on [https://goerli-rollup-explorer.arbitrum.io/](https://goerli-rollup-explorer.arbitrum.io/) and see that you now have a balance of 0.001 ether. Simply paste in your account address to search for it. + + +![Blockscout Scan](images/arbitrum_scan.png) + +## Use the public testnet +Now that you have everything set up, you can follow one of the other FireFly guides such as [Using Tokens](../tokens/index.md) or [Custom Smart Contracts](../custom_contracts/ethereum.md). For detailed instructions on deploying a custom smart contract to Binance Smart Chain, please see the [Arbitrum docs](https://developer.offchainlabs.com/docs/Contract_Deployment) for instructions using various tools. \ No newline at end of file diff --git a/docs/tutorials/chains/NEAR.md b/docs/tutorials/chains/NEAR.md new file mode 100644 index 0000000000..21b3e3b243 --- /dev/null +++ b/docs/tutorials/chains/NEAR.md @@ -0,0 +1,117 @@ +--- +layout: i18n_page +title: pages.near +parent: pages.chains +grand_parent: pages.tutorials +nav_order: 6 +--- + + +# {%t pages.near %} +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +Starting with FireFly v1.1, it's easy to connect to public Ethereum chains. This guide will walk you through the steps to create a local FireFly development environment and connect it to the NEAR testnet. + +## Previous steps: Install the FireFly CLI +If you haven't set up the FireFly CLI already, please go back to the Getting Started guide and read the section on how to [Install the FireFly CLI](../../gettingstarted/firefly_cli.md). + +[← ① Install the FireFly CLI](../../gettingstarted/firefly_cli.md){: .btn .btn-purple .mb-5} + +## Create an `evmconnect.yml` config file +In order to connect to the NEAR testnet, you will need to set a few configuration options for the evmconnect blockchain connector. Create a text file called `evmconnect.yml` with the following contents: + +```yml +confirmations: + required: 4 +policyengine.simple: + fixedGasPrice: null + gasOracle: + mode: connector +``` + +For this tutorial, we will assume this file is saved at `~/Desktop/evmconnect.yml`. If your path is different, you will need to adjust the path in the next command below. + +## Creating a new stack +To create a local FireFly development stack and connect it to the NEAR testnet, we will use command line flags to customize the following settings: + + - Create a new stack named `near` with `1` member + - Disable `multiparty` mode. We are going to be using this FireFly node as a Web3 gateway, and we don't need to communicate with a consortium here + - Connect to an ethereum network + - Use the `evmconnect` blockchain connector + - Use an remote RPC node. This will create a signer locally, so that our signing key never leaves the development machine. + - Set the remote RPC node URL to `https://rpc.testnet.near.org` (RPC nodes for other NEAR networks may be found here https://docs.near.org/api/rpc/setup) + - Set the chain ID to any number (NEAR works with any chain ID) + - Merge the custom config created above with the generated `evmconnect` config file + +To do this, run the following command: +``` +ff init near 1 \ + --multiparty=false \ + -b ethereum \ + -c evmconnect \ + -n remote-rpc \ + --remote-node-url https://rpc.testnet.near.org \ + --chain-id 1 \ + --connector-config ~/Desktop/evmconnect.yml +``` + +## Start the stack +Now you should be able to start your stack by running: + +``` +ff start near +``` + +After some time it should print out the following: + +``` +Web UI for member '0': http://127.0.0.1:5000/ui +Sandbox UI for member '0': http://127.0.0.1:5109 + + +To see logs for your stack run: + +ff logs near +``` + +## Get some NEAR +At this point you should have a working FireFly stack, talking to a public chain. However, you won't be able to run any transactions just yet, because you don't have any way to pay for gas. A testnet faucet can give us some NEAR, the native token for the NEAR protocol. + +First, you will need to know what signing address your FireFly node is using. To check that, you can run: + +``` +ff accounts list near +[ + { + "address": "0xa4ed2a9a99dfdf46f1812c38a1656ff2ad1f61da", + "privateKey": "..." + } +] +``` +Note, for the NEAR protocol, the line labeled privateKey is the address you will use. + +Go to [https://near-faucet.io/](https://near-faucet.io/) and click **Connect with Near Testnet**. Next click **Create Accounte**, make an account ID, and choose a security method. Follow the steps for either the Seedphrase or Ledger Hardware Wallet until your NEAR account is created. Once complete you will be redirected to the original https://near-faucet.io/ page and are now able to request 20 NEAR tokens. + +![NEAR Faucet](images/near_faucet.png) +![NEAR Account](images/near_account.png) +![NEAR Account Name](images/near_account_name.png) +![Fund Account](images/near_fund_account.png) + +### Confirm the transaction on NEAR Testnet Explorer +Once the request for 20 NEAR tokens is completed, go to [https://explorer.testnet.near.org/](https://explorer.testnet.near.org/) and search via your account name. Once it is found click on the link under **Balance Profile** to access your NEAR wallet. From here, click the **Wallet** button in the top left and then **Send**. Choose a denomination of NEAR to send, enter your 64 character string denominated as privateKey from the FireFly CLI, and you will now have funded tokens in your account. + + +![NEAR Scan](images/near_scan.png) +![Account Lookup](images/near_account_lookup.png) +![NEAR Wallet Send Funds](images/near_wallet_send_funds.png) + +## Use the public testnet +Now that you have everything set up, you can follow one of the other FireFly guides such as [Using Tokens](../tokens/index.md) or [Custom Smart Contracts](../custom_contracts/ethereum.md). For detailed instructions on deploying a custom smart contract to NEAR, please see the [NEAR docs](https://docs.near.org/develop/contracts/introduction) for instructions using various tools. \ No newline at end of file diff --git a/docs/tutorials/chains/Optimism.md b/docs/tutorials/chains/Optimism.md new file mode 100644 index 0000000000..f1d742b279 --- /dev/null +++ b/docs/tutorials/chains/Optimism.md @@ -0,0 +1,111 @@ +--- +layout: i18n_page +title: pages.optimism +parent: pages.chains +grand_parent: pages.tutorials +nav_order: 5 +--- + + +# {%t pages.optimism %} +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +Starting with FireFly v1.1, it's easy to connect to public Ethereum chains. This guide will walk you through the steps to create a local FireFly development environment and connect it to the Optimism Goerli testnet. + +## Previous steps: Install the FireFly CLI +If you haven't set up the FireFly CLI already, please go back to the Getting Started guide and read the section on how to [Install the FireFly CLI](../../gettingstarted/firefly_cli.md). + +[← ① Install the FireFly CLI](../../gettingstarted/firefly_cli.md){: .btn .btn-purple .mb-5} + +## Create an `evmconnect.yml` config file +In order to connect to the Optimism testnet, you will need to set a few configuration options for the evmconnect blockchain connector. Create a text file called `evmconnect.yml` with the following contents: + +```yml +confirmations: + required: 4 +policyengine.simple: + fixedGasPrice: null + gasOracle: + mode: connector +``` + +For this tutorial, we will assume this file is saved at `~/Desktop/evmconnect.yml`. If your path is different, you will need to adjust the path in the next command below. + +## Creating a new stack +To create a local FireFly development stack and connect it to the Optimism testnet, we will use command line flags to customize the following settings: + + - Create a new stack named `optimism` with `1` member + - Disable `multiparty` mode. We are going to be using this FireFly node as a Web3 gateway, and we don't need to communicate with a consortium here + - Connect to an ethereum network + - Use the `evmconnect` blockchain connector + - Use an remote RPC node. This will create a signer locally, so that our signing key never leaves the development machine. + - Set the remote RPC node URL to `https://goerli.optimism.io` + - Set the chain ID to `420` (the correct ID for the Optimism testnet) + - Merge the custom config created above with the generated `evmconnect` config file + +To do this, run the following command: +``` +ff init optimism 1 \ + --multiparty=false \ + -b ethereum \ + -c evmconnect \ + -n remote-rpc \ + --remote-node-url https://goerli.optimism.io \ + --chain-id 420 \ + --connector-config ~/Desktop/evmconnect.yml +``` + +## Start the stack +Now you should be able to start your stack by running: + +``` +ff start optimism +``` + +After some time it should print out the following: + +``` +Web UI for member '0': http://127.0.0.1:5000/ui +Sandbox UI for member '0': http://127.0.0.1:5109 + + +To see logs for your stack run: + +ff logs optimism +``` + +## Get some Optimism +At this point you should have a working FireFly stack, talking to a public chain. However, you won't be able to run any transactions just yet, because you don't have any way to pay for gas. A testnet faucet can give us some OP, the native token for Optimism. + +First, you will need to know what signing address your FireFly node is using. To check that, you can run: + +``` +ff accounts list optimism +[ + { + "address": "0x235461d246ab95d367925b4e91bd2755a921fdd8", + "privateKey": "..." + } +] +``` + +Copy the address listed in the output from this command. Go to [https://optimismfaucet.xyz/](https://optimismfaucet.xyz/). You will need to login to your Github account and paste the address in the form. + +![Optimism Faucet](images/optimism_faucet.png) + +### Confirm the transaction on Blockcscout +You should be able to go lookup your account on [Blockscout for Optimism testnet https://blockscout.com/optimism/goerli](https://blockscout.com/optimism/goerli) and see that you now have a balance of 100 OP. Simply paste in your account address to search for it. + + +![Blockscout Scan](images/optimism_scan.png) + +## Use the public testnet +Now that you have everything set up, you can follow one of the other FireFly guides such as [Using Tokens](../tokens/index.md) or [Custom Smart Contracts](../custom_contracts/ethereum.md). For detailed instructions on deploying a custom smart contract to Optimism, please see the [Optimism docs](https://community.optimism.io/docs/developers/build/system-contracts/#getting-contract-artifacts-interfaces-and-abis) for instructions using various tools. \ No newline at end of file diff --git a/docs/tutorials/chains/avalanche.md b/docs/tutorials/chains/avalanche.md index 85fb2df4c7..391ee17450 100644 --- a/docs/tutorials/chains/avalanche.md +++ b/docs/tutorials/chains/avalanche.md @@ -100,7 +100,7 @@ ff accounts list avalanche Copy the address listed in the output from this command. Go to [https://faucet.avax.network/](https://faucet.avax.network/) and paste the address in the form. Make sure that the network you select is Fuji (C-Chain). Click the **Request 2 AVAX** button. -Avalanche Faucet +Avalanche Faucet ### Confirm the transaction on Snowtrace You should be able to go lookup your account on [Snowtrace for the Fuji testnet](https://testnet.snowtrace.io/) and see that you now have a balance of 2 AVAX. Simply paste in your account address or transaction ID to search for it. diff --git a/docs/tutorials/chains/images/near_account.png b/docs/tutorials/chains/images/near_account.png new file mode 100644 index 0000000000..fa3ee55f77 Binary files /dev/null and b/docs/tutorials/chains/images/near_account.png differ diff --git a/docs/tutorials/chains/images/near_account_lookup.png b/docs/tutorials/chains/images/near_account_lookup.png new file mode 100644 index 0000000000..eb2a41cd0d Binary files /dev/null and b/docs/tutorials/chains/images/near_account_lookup.png differ diff --git a/docs/tutorials/chains/images/near_account_name.png b/docs/tutorials/chains/images/near_account_name.png new file mode 100644 index 0000000000..f31dfc7193 Binary files /dev/null and b/docs/tutorials/chains/images/near_account_name.png differ diff --git a/docs/tutorials/chains/images/near_faucet.png b/docs/tutorials/chains/images/near_faucet.png new file mode 100644 index 0000000000..44bb9136bb Binary files /dev/null and b/docs/tutorials/chains/images/near_faucet.png differ diff --git a/docs/tutorials/chains/images/near_fund_account.png b/docs/tutorials/chains/images/near_fund_account.png new file mode 100644 index 0000000000..cfc1b1e424 Binary files /dev/null and b/docs/tutorials/chains/images/near_fund_account.png differ diff --git a/docs/tutorials/chains/images/near_navigate_to_wallet.png b/docs/tutorials/chains/images/near_navigate_to_wallet.png new file mode 100644 index 0000000000..1d5b9ff4d6 Binary files /dev/null and b/docs/tutorials/chains/images/near_navigate_to_wallet.png differ diff --git a/docs/tutorials/chains/images/near_scan.png b/docs/tutorials/chains/images/near_scan.png new file mode 100644 index 0000000000..a93b05598c Binary files /dev/null and b/docs/tutorials/chains/images/near_scan.png differ diff --git a/docs/tutorials/chains/images/near_wallet_send_funds.png b/docs/tutorials/chains/images/near_wallet_send_funds.png new file mode 100644 index 0000000000..b705e1dd94 Binary files /dev/null and b/docs/tutorials/chains/images/near_wallet_send_funds.png differ diff --git a/docs/tutorials/define_datatype.md b/docs/tutorials/define_datatype.md index 4e8985b763..4c3a85da1d 100644 --- a/docs/tutorials/define_datatype.md +++ b/docs/tutorials/define_datatype.md @@ -31,7 +31,7 @@ of datatypes, as is used to broadcast the data itself. ## Additional info -- Key Concepts: [Broadcast / shared data](../overview//broadcast.md) +- Key Concepts: [Broadcast / shared data](../overview/multiparty/broadcast.html) - Swagger: POST /api/v1/namespaces/{ns}/datatypes ### Example 1: Broadcast new datatype @@ -199,7 +199,7 @@ In the sandbox, enter the datatype's name, version, and JSON Schema as seen in t Notice how the `data` field in the center panel updates in real time. -Click the blue `Run` button. This should return a `202` response immediately in the Server Response section and will populate the right hand panel with transaction information after a few seconds. +Click the blue `Run` button. This should return a `202` response immediately in the Server Response section and will populate the right hand panel with transaction information after a few seconds. ![Message Broadcast](../images/message_broadcast_sample_result.png) diff --git a/docs/tutorials/events.md b/docs/tutorials/events.md index e5dc1914b4..3d52203f3b 100644 --- a/docs/tutorials/events.md +++ b/docs/tutorials/events.md @@ -36,7 +36,7 @@ We focus on WebSockets in this getting started guide. ## Additional info -- Key Concepts: [Multi-party process flow](../overview//multiparty_process_flow.md) +- Key Concepts: [Multi-party process flow](../overview/multiparty/multiparty_flow.html) - Reference: _coming soon_ ## WebSockets Example 1: Ephemeral subscription with auto-commit diff --git a/docs/tutorials/private_send.md b/docs/tutorials/private_send.md index 9b7598eeb1..b690f02858 100644 --- a/docs/tutorials/private_send.md +++ b/docs/tutorials/private_send.md @@ -48,7 +48,7 @@ nav_order: 2 ## Additional info -- Key Concepts: [Private data exchange](../overview/data_exchange.md) +- Key Concepts: [Private data exchange](../overview/multiparty/data_exchange.html) - Swagger: POST /api/v1/namespaces/{ns}/messages/private ## Example 1: Pinned private send of in-line string data @@ -283,10 +283,10 @@ Make sure to expand the "Send a Private Message" section. Enter your message int Notice how the `data` field in the center panel updates in real time as you update the message you wish to send. -Click the blue `Run` button. This should return a `202` response immediately in the Server Response section and will populate the right hand panel with transaction information after a few seconds. +Click the blue `Run` button. This should return a `202` response immediately in the Server Response section and will populate the right hand panel with transaction information after a few seconds. ![Private Message result](../images/message_broadcast_sample_result.png) -Go back to the FireFly UI (the URL for this would have been shown in the terminal when you started the stack) and you'll see your successful blockchain transaction. Compare the "Recent Network Changes" widget With private messages, your +Go back to the FireFly UI (the URL for this would have been shown in the terminal when you started the stack) and you'll see your successful blockchain transaction. Compare the "Recent Network Changes" widget With private messages, your ![Successful Transaction](../images/firefly_first_successful_transaction.png) \ No newline at end of file