From 7672ba525e92ab9ae1adf57c9ab03898b4e4201c Mon Sep 17 00:00:00 2001 From: dehann Date: Sun, 14 May 2023 13:43:28 -0700 Subject: [PATCH] fixed blobs entries docs page --- Makefile | 1 + docs/blobs.md | 110 +++++++++++--------- docs/variables.md | 1 + src/navability/entities/blob/blobentry.py | 61 +++++------ src/navability/entities/navabilityclient.py | 6 +- src/navability/services/blobentry.py | 23 ++-- 6 files changed, 105 insertions(+), 97 deletions(-) diff --git a/Makefile b/Makefile index 458b536..722dce9 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ docs_cicd: docs: $(SDKPY_DOCS_BUILDER) build docs/ + $(SDKPY_DOCS_BUILDER) build docs/ docs_firefox: docs firefox docs/_build/html/index.html diff --git a/docs/blobs.md b/docs/blobs.md index e4c2d88..58c1949 100644 --- a/docs/blobs.md +++ b/docs/blobs.md @@ -1,86 +1,100 @@ # Data `BlobEntry=>Blob` -## Getting BlobEntries +Many `BlobEntry`s from various location can all reference the same large and heavy binary data `Blob`. Furthermore, this system allows a distributed usage over a large networked system. -Additional (large) data attached to variables exist in a few different ways. The primary method for storing additional large data blobs with a variable, is to look at the `BlobEntry`s associated with a particular variable. For example: +:::{tip} +`Blob`s are separate from `BlobEntry`s. Various nodes on the graph can have any number of `BlobEntry`s (including duplicates), but the user does not necessary have the need to pull the bandwidth heavy blobs across all parts of the system network. +::: + +## What is a `BlobEntry` + +Additional (large) data attached to variables exist in a few different ways. The primary method for storing additional large data as a `Blob`, however, a blob just "barcoded" binary blob. we use `BlobEntry`s to organize, find, share, distribute, make available and useful across the divergent network how a `Blob` is associated with the graph. + +A blob entry is small about of structed data that holds reference information to find an actual binary blob which is possibly a massive amount of data. + +### Listing BlobEntries on a Node + +The `list` verb is purposefully limited to just returning a list of labels (i.e. `List[string]`), and therefore the following call only returns the labels of `BlobEntry`s attached a particular node (variable, session, robot, etc.): ```python -entries = await listBlobEntries(client, context, "x0") |> fetch +ent_labels = listBlobEntries(fgclient, 'x0') +# ent_labels = await listBlobEntriesAsync(fgclient, 'x0') + # human friendly labels -print([en.label for en in entries]) -# e.g. ['Camera0', 'UserCalib', 'GPS', 'JoystickCmds'] +print('Human friendly BlobEntry labels on x0 are', ent_labels) +# e.g. ['LEFTCAM_1002', 'LEFTCAM_1003', 'UserCalib', 'GPS', 'JoystickCmds'] +``` -# machine friendly system wide unique identifier -print([en.id for en in entries]) +```{eval-rst} +.. autofunction:: navability.services.listBlobEntries +.. autofunction:: navability.services.listBlobEntriesAsync ``` -:::{tip} -`Blob`s are separate from `BlobEntry`s. Various nodes on the graph can have any number of `BlobEntry`s (including duplicates), but the user does not necessary have the need to pull the bandwidth heavy blobs across all parts of the system network. -::: +### Getting a `BlobEntry` + +Using the human friendly label from the `listBlobEntries` above, we can fetch the `BlobEntry` +```python +entry = getBlobEntry(fgclient, 'LEFTCAM_1002') +# entry = await getBlobEntryAsync(fgclient, 'LEFTCAM_1002') +``` + +The entry object is well structured +```{eval-rst} +.. autoclass:: navability.entities.BlobEntry +``` -Data blobs can be fetched via, e.g. using the unique `id` (or `blobId`) of from a `BlobEntry`: +## Getting the associated `Blob` + +A binary `Blob` is basically just a "barcoded" piece of data that can be associated (via individual `BlobEntry`s) multiple times across multiple graph nodes, sessions, or robots. + +Data blobs can be fetched via, e.g. using the unique `.blobId` as primary (or `.originId` as secondary) reference. Also note that the blob itself may also be replicated across any number of blob stores, depending on the application: ```python -blob = await getBlob(client, context, entries[1].id]; checkhash=false) -# b'{"latitude":41.7325,"altitude":2.211,"header":{"stamp":{"secs":1670378554,"nsecs":000624417},"seq":91,"frame_id":"gps","_type":"ROS1/std_msgs/Header"},"status":{"status":0,"service":1},"position_covariance":[0.265225,0.0,0.0,0.0,0.265225,0.0,0.0,0.0,0.556516],"longitude":-49.946944,"_type":"ROS1/sensor_msgs/NavSatFix","position_covariance_type":2}' +blob = getBlob(fgclient, entry.blobId]; checkhash=false) # legacy versions did not use .hash check +# blob = await getBlobAsync(fgclient, entry.blobId]; checkhash=false) ``` -Data blobs are provided in binary format. A blob can be associated via any number of `BlobEntry`s across multiple graph nodes, sessions, or robots. `BlobEntry` also stores a hash value to ensure data consistency which must correspond to the stored hash upon retrieval. The check can be skipped as indicated by the option in the function call above. +The blob contains binary information, for example this `mimeType = application/octet-stream/json; _type=ROS.sensor_msgs.NavSatFix`: +``` +b'{"latitude":41.7325,"altitude":2.211,"header":{"stamp":{"secs":1670378554,"nsecs":000624417},"seq":91,"frame_id":"gps","_type":"ROS1/std_msgs/Header"},"status":{"status":0,"service":1},"position_covariance":[0.265225,0.0,0.0,0.0,0.265225,0.0,0.0,0.0,0.556516],"longitude":-49.946944,"_type":"ROS1/sensor_msgs/NavSatFix","position_covariance_type":2}' +``` +:::{tip} +Depending on the blob store, it may also be possible to retrieve a blob using the `.originId` rather than `.blobId`. +::: :::{tip} A blob is owned by a `user` and only accessible by other users if allowed via approved roles or permissions. ::: :::{tip} -All `blobId`s are unique across the entire distributed system and are immutable. +`BlobEntry.hash` helps ensure data consistency by rehasing the retrieved blob binary data itself. ::: ## Adding BlobEntries -Blobs can be linked to any variable (future node) in the graph. This is easily done by adding a BlobEntry: -```python -res = await addBlobEntry(client, context, 'x12', entries[1].id, entries[1].label, len(blob), entries[1].mimeType) -``` - -:::{tip} -More ubiqitous use of `blob` size was recently introduced to `BlobEntry` and will be unified with less user input as part of SDKs v0.6 upgrades. +:::{warning} +Adding `Blob` or `BlobEntry`s from the Python SDK are under construction and expected to be part of the v0.6.1 release. This functionality has already been released with the JuliaLang SDK. ::: -## BlobEntry Structure - -To simplify many different requirements, a `BlobEntry` has the following field structure: -``` -{ - id: UUID - label: string - description: string - blobstore: string - hash: string - mimeType: string - origin: string - - ## planned future fields - # blobId: UUID - # originId: UUID - # createdTimestamp: datetime - # size: int - # metadata: string -} +Blobs can be linked to any variable (future node) in the graph. This is easily done by adding a BlobEntry: +```python +res = addBlobEntry(fgclient, 'x12', entries[1].id, entries[1].label, len(blob), entries[1].mimeType) +# res = await addBlobEntryAsync(fgclient, 'x12', entries[1].id, entries[1].label, len(blob), entries[1].mimeType) ``` ## Adding New Blobs It is also possible to push data blobs: ```python -blobId = await addBlob(client, "testimage.png", imgblob) +client = NavAbilityHttpsClient() +blobId = await addBlob(fgclient.client, "testimage.png", imgblob) ``` Remember to add at least one BlobEntry somewhere in your session so that you might find it again in the future, see `addBlobEntry` above. +:::{eval-rst} +.. autofunction:: navability.entities.NavAbilityHttpsClient +::: + :::{seealso} See [Tutorial 5 from ICRA 2022 for a more in-depth example of working with data blobs](sdkpynb:python/navability-sdk/icra-5-marineexample). ::: - - \ No newline at end of file diff --git a/docs/variables.md b/docs/variables.md index 6f35240..358649b 100644 --- a/docs/variables.md +++ b/docs/variables.md @@ -55,6 +55,7 @@ The `getVariable` call returns a `Variable` object containing many fields. For ```python print('The tags on this variable are', v0.tags) ``` + which would print the list of tags similar to: ``` The tags on this variable are ["VARIABLE", "POSE", "DOCS_EXAMPLE"] diff --git a/src/navability/entities/blob/blobentry.py b/src/navability/entities/blob/blobentry.py index cdb846a..a6777ef 100644 --- a/src/navability/entities/blob/blobentry.py +++ b/src/navability/entities/blob/blobentry.py @@ -11,46 +11,33 @@ from navability.common.timestamps import TS_FORMAT from navability.common.versions import payload_version -# type BlobEntry { -# # This is created by server-side GraphQL # -# id: ID! @id -# # This is the forced server generated blobId, or the filesystem blobId. # -# blobId: ID! -# # This is the ID at creation at the edge, do whatever you want with this, but make sure you populate it. # -# originId: ID! -# label: String! -# description: String -# hash: String -# mimeType: String -# blobstore: String -# origin: String -# metadata: Metadata -# timestamp: DateTime -# nstime: BigInt -# _type: String! -# _version: String! - -# createdTimestamp: DateTime! @timestamp(operations: [CREATE]) -# lastUpdatedTimestamp: DateTime! @timestamp(operations: [CREATE, UPDATE]) - -# user: [Variable!]! @relationship(type: "DATA_USER", direction: IN) -# robot: [Robot!]! @relationship(type: "DATA_ROBOT", direction: IN) -# session: [Session!]! @relationship(type: "DATA_SESSION", direction: IN) -# variable: [Variable!]! @relationship(type: "DATA_VARIABLE", direction: IN) -# factor: [Factor!]! @relationship(type: "DATA_FACTOR", direction: IN) - -# # NOTE: This is for the unique label constraints -# # In this situation, someone has to own this, so cannot be required -# userLabel: String -# robotLabel: String -# sessionLabel: String -# variableLabel: String -# factorLabel: String -# } - @dataclass() class BlobEntry: + """ A `BlobEntry` is a small about of structured data that holds reference information to find an actual blob. Many `BlobEntry`s + can exist on different graph nodes spanning Robots, and Sessions which can all reference the same `Blob`. A `BlobEntry` + is also a equivalent to a bridging entry between local `.originId` and a remotely assigned `.blobIds`. + + All `.blobId`s are unique across the entire distributed system and are immutable. The `.originId` should be + globally unique except for stochastic `uuid4` collisions that cannot be checked from a main reference owing to practical limitations such as network connectivity. + + Args: + id: Remotely assigned and globally unique identifier for the `BlobEntry` itself (not the `.blobId`). + label: Human friendly label of the `Blob` and also used as unique identifier per node on which a `BlobEntry` is added. E.g. do "LEFTCAM_1", "LEFTCAM_2", ... of you need to repeat a label on the same variable. + blobId: Machine friendly and globally unique identifier of the 'Blob', usually assigned from a common point in the system. This can be used to guarantee unique retrieval of the large data blob. + originId: Machine friendly and locally assigned identifier of the 'Blob'. `.originId`s are mandatory upon first creation at the origin regardless of network access. Separate from `.blobId` since some architectures do not allow edge processes to assign a uuid to data store elements. + timestamp: When the Blob itself was first created. + description: Additional information that can help a different user of the Blob. + blobstore: A hint about where the `Blob` itself might be stored. Remember that a Blob may be duplicated over multiple blobstores. + hash: A hash value to ensure data consistency which must correspond to the stored hash upon retrieval. [Legacy: some usage functions allow the check to be skipped if needed.] + mimeType: MIME description describing the format of binary data in the `Blob`, e.g. 'image/png' or 'application/json; _type=CameraModel'. + origin: Context from which a BlobEntry=>Blob was first created. E.g. user|robot|session|varlabel. + metadata: Additional storage for functional metadata used in some scenarios, e.g. to support advanced features such as `parsejson(base64decode(entry.metadata))['time_sync']`. + createdTimestamp: When the BlobEntry was created. + lastUpdatedTimestamp: Use carefully, but necessary to support advanced usage such as time synchronization over Blob data. + _type: Self type declaration for when duck-typing happens. + _version: Type version of this BlobEntry. + """ id: Optional[UUID] label: str blobId: Optional[UUID] diff --git a/src/navability/entities/navabilityclient.py b/src/navability/entities/navabilityclient.py index 6a49665..6af4c0e 100644 --- a/src/navability/entities/navabilityclient.py +++ b/src/navability/entities/navabilityclient.py @@ -57,10 +57,12 @@ async def mutate(self, options: MutationOptions): class NavAbilityHttpsClient(NavAbilityClient): - """Connection object for queries and mutations to API server. + """Connection object for queries and mutations to API server. Note, this is used but higher level objects such as DFGClient. Args: - NavAbilityClient (NavAbilityClient): the connection object + NavAbilityClient: the connection object to a server (cloud our deployed). + url: Network path to the API (cloud or deployed). + auth_token: Token for auth, likely provided by NavAbility App Connect page. """ def __init__(self, url: str = "https://api.navability.io", auth_token: str = "") -> None: super().__init__() diff --git a/src/navability/services/blobentry.py b/src/navability/services/blobentry.py index dbcf95e..675cb0c 100644 --- a/src/navability/services/blobentry.py +++ b/src/navability/services/blobentry.py @@ -72,20 +72,16 @@ async def listBlobEntriesAsync( fgclient: DFGClient, variableLabel: str, ): - client = fgclient.client - context = fgclient.context - - """ List the blob entries associated with a particular variable. + """ Return a list of `BlobEntry` labels (asynchronous version). Args: - fgclient (NavAbilityClient): client connection to API server - and unique context with (user, robot, session) + fgclient (DFGClient): client connection to API server + with unique context (user, robot, session) variableLabel (string): list data entries connected to which variable - - Returns: - List[String]: A list of `BlobEntry` labels - """ + client = fgclient.client + context = fgclient.context + params = { "userLabel": context.userLabel, "robotLabel": context.robotLabel, @@ -127,6 +123,13 @@ def listBlobEntries( fgclient: DFGClient, variableLabel: str, ): + """ Return a list of `BlobEntry` labels (synchronous version). + + Args: + fgclient (DFGClient): client connection to API server + with unique context (user, robot, session) + variableLabel (string): list data entries connected to which variable + """ tsk = listBlobEntriesAsync(fgclient, variableLabel) return asyncio.run(tsk)