diff --git a/README.md b/README.md index 8edd7e9..e397358 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,20 @@ This example demonstrates the integration with a TypeScript CLI client applicati This example demonstrates the integration with a TypeScript web client application. See [examples/typescript-web-client/README.md](examples/typescript-web-client/README.md) for more details. +## Architecture + +### Block Diagram + +Block Diagram + +### Sequence Diagram - Consent Request + +Sequence Diagram - Consent Request + +### Sequence Diagram - Track Usage + +Sequence Diagram - Track Usage + ## Support, Feedback, Contributing This project is open to feature requests/suggestions, bug reports etc. via [GitHub issues](https://github.com/SAP/automated-usage-tracking-tool/issues). Contribution and feedback are encouraged and always welcome. For more information about how to contribute, the project structure, as well as additional contribution information, see our [Contribution Guidelines](CONTRIBUTING.md). diff --git a/dataflows/README.md b/dataflows/README.md new file mode 100644 index 0000000..3ae0fb1 --- /dev/null +++ b/dataflows/README.md @@ -0,0 +1,15 @@ +# Dataflows + +## Dataflow - Store Latest Usages + +The [Store Latest Usages Dataflow](store_latest_usages.json) runs every 10 minutes, copying the data from the user's `data.latestUsages` (ClientModify) to `data.usages` (ServerOnly). This process ensures long-term storage of usage data and prevents loss of old usages when adding new ones or in case of user data deletion. + +### Steps + +1. **Read emailAccounts**: Reads data from the `emailAccounts` data source, selecting the `UID`, `profile`, and `data` fields. It uses the `lastUpdatedTimestamp` field to fetch only the updated records. +2. **Process "latestUsages" to "usages"**: Evaluates each record to move new usages from `data.latestUsages` to `data.usages`. It ensures that only new usages are added and avoids duplicates. +3. **Update Lite**: Writes the processed data back to the `accounts.setAccountInfo` API, updating the `data` and `profile` fields and marking the record as `isLite`. + +## Note + +These dataflows are not part of this package but are configured in SAP Customer Data Cloud (CDC). They provide the necessary API keys for this project to function correctly. diff --git a/dataflows/store_latest_usages.json b/dataflows/store_latest_usages.json new file mode 100644 index 0000000..6d08de4 --- /dev/null +++ b/dataflows/store_latest_usages.json @@ -0,0 +1,62 @@ +{ + "apiKey": "4_3OulQC05sfcJ-D5mG6aMNg", + "siteId": 251588122855, + "id": "f0a7b01eda6641e9b036b831e2ea6e8a", + "name": "Move new usages from \"latestUsages\" to \"usages\"", + "status": "published", + "description": "", + "lastRuntime": "2024-10-21T12:57:32.942Z", + "steps": [ + { + "id": "1. Read emailAccounts", + "type": "datasource.read.gigya.account", + "params": { + "select": "UID,profile,data", + "from": "emailAccounts", + "deltaField": "lastUpdatedTimestamp", + "batchSize": 300, + "maxConcurrency": 1, + "keepFieldNamesWithDotAsIs": false + }, + "next": ["2. Process \"latestUsages\" to \"usages\""] + }, + { + "id": "2. Process \"latestUsages\" to \"usages\"", + "type": "record.evaluate", + "params": { + "script": "ZnVuY3Rpb24gcHJvY2VzcyhyZWNvcmQsIGN0eCwgbG9nZ2VyLCBuZXh0KSB7DQogICAgLy8gQmFzZSBWYWxpZGF0aW9uDQogICAgaWYoIXJlY29yZCB8fCAhcmVjb3JkLnByb2ZpbGUgfHwgIXJlY29yZC5kYXRhKSByZXR1cm47DQogICAgDQogICAgLy8gU2V0dXANCiAgICBsZXQgeyBVSUQgfSA9IHJlY29yZDsNCiAgICBsZXQgcmVjID0geyBVSUQsIHByb2ZpbGU6IHt9LCBkYXRhOiB7fX07DQogICAgaWYocmVjb3JkLnByb2ZpbGUuZW1haWwpIHJlYy5wcm9maWxlLmVtYWlsID0gcmVjb3JkLnByb2ZpbGUuZW1haWw7DQoNCiAgICAvLyAjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjDQogICAgLy8gTW92ZSBuZXcgdXNhZ2VzIGZyb20gImxhdGVzdFVzYWdlcyIgdG8gInVzYWdlcyINCiAgICAvLyAjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjDQogICAgDQogICAgLy8gVmFsaWRhdGlvbg0KICAgIGlmKCFyZWNvcmQuZGF0YS5sYXRlc3RVc2FnZXM/Lmxlbmd0aCkgcmV0dXJuOw0KICAgIA0KICAgIC8vIEluaXQgdXNhZ2VzIGlmIG5lY2Vzc2FyeQ0KICAgIGlmKCFyZWNvcmQuZGF0YS51c2FnZXMpIHsNCiAgICAgICAgcmVjb3JkLmRhdGEudXNhZ2VzID0gW107DQogICAgfQ0KICAgIA0KICAgIC8vIEdldCBvbmx5IG5ldyB1c2FnZXMNCiAgICB2YXIgbmV3VXNhZ2VzID0gcmVjb3JkLmRhdGEubGF0ZXN0VXNhZ2VzLmZpbHRlcigobmV3VXNhZ2UpID0+IHsNCiAgICAgICAgdmFyIGlzVXNhZ2VTdG9yZWQgPSByZWNvcmQuZGF0YS51c2FnZXMuZmluZCgoc3RvcmVkVXNhZ2UpID0+IHN0b3JlZFVzYWdlLmlkID09PSBuZXdVc2FnZS5pZCk7DQogICAgICAgIGlmKGlzVXNhZ2VTdG9yZWQpIHsNCiAgICAgICAgICAgIHJldHVybiBmYWxzZTsNCiAgICAgICAgfQ0KICAgICAgICByZXR1cm4gdHJ1ZTsNCiAgICB9KTsNCiAgICANCiAgICAvLyBBZGQgbmV3IHVzYWdlcyBmcm9tICJkYXRhLmxhdGVzdFVzYWdlcyIgdG8gImRhdGEudXNhZ2VzIg0KICAgIHJlYy5kYXRhLnVzYWdlcyA9IFsuLi5yZWNvcmQuZGF0YS51c2FnZXMsIC4uLm5ld1VzYWdlc107DQogICAgDQogICAgLy8gQ2xlYXIgbGF0ZXN0VXNhZ2VzDQogICAgcmVjLmRhdGEubGF0ZXN0VXNhZ2VzID0gW107DQoNCiAgICAvL2xvZ2dlci5pbmZvKCJCZWZvcmU6ICIsIHJlY29yZCk7DQogICAgLy9sb2dnZXIuaW5mbygiQWZ0ZXI6ICIsIHJlYyk7DQogICAgcmV0dXJuIHJlYzsNCn0=", + "ECMAScriptVersion": "12", + "notifyLastRecord": false + }, + "next": ["3. Update Lite"] + }, + { + "id": "3. Update Lite", + "type": "datasource.write.gigya.generic", + "params": { + "apiMethod": "accounts.setAccountInfo", + "maxConnections": 10, + "apiParams": [ + { + "sourceField": "data", + "paramName": "data", + "value": "" + }, + { + "sourceField": "profile", + "paramName": "profile", + "value": "" + }, + { + "sourceField": "", + "paramName": "isLite", + "value": "true" + } + ], + "addResponse": false + } + } + ], + "version": 1, + "updatedByEmail": "l.marques@sap.com" +} diff --git a/docs/architecture/general_block_diagram.png b/docs/architecture/general_block_diagram.png new file mode 100644 index 0000000..7bb36a8 Binary files /dev/null and b/docs/architecture/general_block_diagram.png differ diff --git a/docs/architecture/get_and_save_user_consent.png b/docs/architecture/get_and_save_user_consent.png new file mode 100644 index 0000000..8c72889 Binary files /dev/null and b/docs/architecture/get_and_save_user_consent.png differ diff --git a/docs/architecture/save_new_usage.png b/docs/architecture/save_new_usage.png new file mode 100644 index 0000000..ada623b Binary files /dev/null and b/docs/architecture/save_new_usage.png differ