From 0591e3c284995ecfec99f9da340b5095a7b0fad8 Mon Sep 17 00:00:00 2001 From: Joao Meyer <1994meyer@gmail.com> Date: Tue, 18 Aug 2020 23:28:58 -0300 Subject: [PATCH 1/5] Implemented dag get, put, import and export --- ipfshttpclient/client/__init__.py | 3 ++- ipfshttpclient/client/dag.py | 33 ++++++++++++++++++++++++ test/functional/fake_json/data.car | Bin 0 -> 104 bytes test/functional/test_dag.py | 40 +++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 ipfshttpclient/client/dag.py create mode 100644 test/functional/fake_json/data.car create mode 100644 test/functional/test_dag.py diff --git a/ipfshttpclient/client/__init__.py b/ipfshttpclient/client/__init__.py index 8b170103..65a56dc3 100644 --- a/ipfshttpclient/client/__init__.py +++ b/ipfshttpclient/client/__init__.py @@ -22,7 +22,7 @@ from . import block from . import bootstrap from . import config -#TODO: `from . import dag` +from . import dag from . import dht from . import files from . import key @@ -175,6 +175,7 @@ def close(self): # Call this when you're done block = base.SectionProperty(block.Section) bootstrap = base.SectionProperty(bootstrap.Section) config = base.SectionProperty(config.Section) + dag = base.SectionProperty(dag.Section) dht = base.SectionProperty(dht.Section) key = base.SectionProperty(key.Section) name = base.SectionProperty(name.Section) diff --git a/ipfshttpclient/client/dag.py b/ipfshttpclient/client/dag.py new file mode 100644 index 00000000..4660ea2e --- /dev/null +++ b/ipfshttpclient/client/dag.py @@ -0,0 +1,33 @@ +import typing as ty + +from . import base + +from .. import multipart + + +class Section(base.SectionBase): + @base.returns_single_item(base.ResponseBase) + def get(self, cid: str, **kwargs: base.CommonArgs): + args = (str(cid),) + return self._client.request('/dag/get', args, decoder='json', **kwargs) + + @base.returns_single_item(base.ResponseBase) + def put(self, data: ty.IO, **kwargs: base.CommonArgs): + body, headers = multipart.stream_files(data, chunk_size=self.chunk_size) + return self._client.request('/dag/put', decoder='json', data=body, + headers=headers, **kwargs) + + @base.returns_single_item(base.ResponseBase) + def resolve(self, cid: str, **kwargs: base.CommonArgs): + args = (str(cid),) + return self._client.request('/dag/resolve', args, decoder='json', **kwargs) + + @base.returns_single_item(base.ResponseBase) + def imprt(self, data: ty.IO, **kwargs: base.CommonArgs): + body, headers = multipart.stream_files(data, chunk_size=self.chunk_size) + return self._client.request('/dag/import', decoder='json', data=body, + headers=headers, **kwargs) + + def export(self, cid: str, **kwargs: base.CommonArgs): + args = (str(cid),) + return self._client.request('/dag/export', args, **kwargs) diff --git a/test/functional/fake_json/data.car b/test/functional/fake_json/data.car new file mode 100644 index 0000000000000000000000000000000000000000..af2b2e419c59ed6b3f396b0bb1cf1246e9eb6bf6 GIT binary patch literal 104 zcmcColv Date: Wed, 19 Aug 2020 21:02:05 -0300 Subject: [PATCH 2/5] Added docs --- ipfshttpclient/client/dag.py | 116 +++++++++++++++++++++++++++++++++++ test/functional/test_dag.py | 3 + 2 files changed, 119 insertions(+) diff --git a/ipfshttpclient/client/dag.py b/ipfshttpclient/client/dag.py index 4660ea2e..043d7944 100644 --- a/ipfshttpclient/client/dag.py +++ b/ipfshttpclient/client/dag.py @@ -8,26 +8,142 @@ class Section(base.SectionBase): @base.returns_single_item(base.ResponseBase) def get(self, cid: str, **kwargs: base.CommonArgs): + """Get and serialize the DAG node named by CID. + + .. code-block:: python + + >>> client.dag.get('QmTkzDwWqPbnAh5YiV5VwcTLnGdwSNsNTn2aDxdXBFca7D') + {'Data': '\x08\x01', + 'Links': [ + {'Hash': 'Qmd2xkBfEwEs9oMTk77A6jrsgurpF3ugXSg7dtPNFkcNMV', + 'Name': 'Makefile', 'Size': 174}, + {'Hash': 'QmeKozNssnkJ4NcyRidYgDY2jfRZqVEoRGfipkgath71bX', + 'Name': 'example', 'Size': 1474}, + {'Hash': 'QmZAL3oHMQYqsV61tGvoAVtQLs1WzRe1zkkamv9qxqnDuK', + 'Name': 'home', 'Size': 3947}, + {'Hash': 'QmZNPyKVriMsZwJSNXeQtVQSNU4v4KEKGUQaMT61LPahso', + 'Name': 'lib', 'Size': 268261}, + {'Hash': 'QmSY8RfVntt3VdxWppv9w5hWgNrE31uctgTiYwKir8eXJY', + 'Name': 'published-version', 'Size': 55} + ]} + + Parameters + ---------- + cid + Key of the object to retrieve, in CID format + + Returns + ------- + dict + Cid with the address of the dag object + """ args = (str(cid),) return self._client.request('/dag/get', args, decoder='json', **kwargs) @base.returns_single_item(base.ResponseBase) def put(self, data: ty.IO, **kwargs: base.CommonArgs): + """Stores input as a DAG object and returns its key. + + .. code-block:: python + + >>> client.dag.put(io.BytesIO(b''' + ... { + ... "Data": "another", + ... "Links": [ { + ... "Name": "some link", + ... "Hash": "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCV … R39V", + ... "Size": 8 + ... } ] + ... }''')) + {'Cid': { + '/': 'bafyreifgjgbmtykld2e3yncey3naek5xad3h4m2pxmo3of376qxh54qk34' + } + } + + Parameters + ---------- + data + IO stream object with data that should be put + + Returns + ------- + dict + Cid with the address of the dag object + """ body, headers = multipart.stream_files(data, chunk_size=self.chunk_size) return self._client.request('/dag/put', decoder='json', data=body, headers=headers, **kwargs) @base.returns_single_item(base.ResponseBase) def resolve(self, cid: str, **kwargs: base.CommonArgs): + """Resolves a DAG node from its Cid + + .. code-block:: python + + >>> client.dag.resolve('QmTkzDwWqPbnAh5YiV5VwcTLnGdwSNsNTn2aDxdXBFca7D') + {'Cid': { + '/': 'QmTkzDwWqPbnAh5YiV5VwcTLnGdwSNsNTn2aDxdXBFca7D' + } + } + + Parameters + ---------- + cid + Key of the object to resolve, in CID format + + Returns + ------- + dict + Cid with the address of the dag object + """ args = (str(cid),) return self._client.request('/dag/resolve', args, decoder='json', **kwargs) @base.returns_single_item(base.ResponseBase) def imprt(self, data: ty.IO, **kwargs: base.CommonArgs): + """Imports a .car file with a DAG to IPFS + + .. code-block:: python + + >>> with open('data.car', 'rb') as file + ... client.dag.imprt(file) + {'Root': { + 'Cid': { + '/': 'bafyreidepjmjhvhlvp5eyxqpmyyi7rxwvl7wsglwai3cnvq63komq4tdya' + } + } + } + + Parameters + ---------- + data + IO stream object with data that should be imported + + Returns + ------- + dict + Dictionary with the root CID of the DAG imported + """ body, headers = multipart.stream_files(data, chunk_size=self.chunk_size) return self._client.request('/dag/import', decoder='json', data=body, headers=headers, **kwargs) def export(self, cid: str, **kwargs: base.CommonArgs): + """Exports a DAG into a .car file format + + .. code-block:: python + + >>> bytes = client.dag.export('bafyreidepjmjhvhlvp5eyxqpmyyi7rxwvl7wsglwai3cnvq63komq4tdya') + + Parameters + ---------- + cid + Key of the object to export, in CID format + + Returns + ------- + bytes + DAG in a .car format + """ args = (str(cid),) return self._client.request('/dag/export', args, **kwargs) diff --git a/test/functional/test_dag.py b/test/functional/test_dag.py index b643d827..e5eaf9a5 100644 --- a/test/functional/test_dag.py +++ b/test/functional/test_dag.py @@ -23,7 +23,10 @@ def test_put_get_resolve(client): def test_import_export(client): + # This file was created by inserting a simple JSON object into IPFS and + # exporting it using `ipfs dag export > file.car` data_car = conftest.TEST_DIR / 'fake_json' / 'data.car' + with open(data_car, 'rb') as file: response = client.dag.imprt(file) From 260def2bb2f7419ec0d53af025c42019f665ce09 Mon Sep 17 00:00:00 2001 From: Joao Meyer <1994meyer@gmail.com> Date: Wed, 19 Aug 2020 21:14:25 -0300 Subject: [PATCH 3/5] Fixed problem with pathlib in some versions of python --- test/functional/test_dag.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/functional/test_dag.py b/test/functional/test_dag.py index e5eaf9a5..6345020c 100644 --- a/test/functional/test_dag.py +++ b/test/functional/test_dag.py @@ -26,6 +26,7 @@ def test_import_export(client): # This file was created by inserting a simple JSON object into IPFS and # exporting it using `ipfs dag export > file.car` data_car = conftest.TEST_DIR / 'fake_json' / 'data.car' + data_car = str(data_car) with open(data_car, 'rb') as file: response = client.dag.imprt(file) From 996f68d53a93f88f6ff051ff9b437a474e3d2bfb Mon Sep 17 00:00:00 2001 From: Alexander Schlarb Date: Mon, 24 Aug 2020 17:51:52 +0200 Subject: [PATCH 4/5] Improve/streamline the DAG method documentation and typings --- ipfshttpclient/client/dag.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ipfshttpclient/client/dag.py b/ipfshttpclient/client/dag.py index 043d7944..715b8d7f 100644 --- a/ipfshttpclient/client/dag.py +++ b/ipfshttpclient/client/dag.py @@ -1,14 +1,13 @@ -import typing as ty - from . import base from .. import multipart +from .. import utils class Section(base.SectionBase): @base.returns_single_item(base.ResponseBase) - def get(self, cid: str, **kwargs: base.CommonArgs): - """Get and serialize the DAG node named by CID. + def get(self, cid: base.cid_t, **kwargs: base.CommonArgs): + """Retrieves the contents of a DAG node .. code-block:: python @@ -41,8 +40,8 @@ def get(self, cid: str, **kwargs: base.CommonArgs): return self._client.request('/dag/get', args, decoder='json', **kwargs) @base.returns_single_item(base.ResponseBase) - def put(self, data: ty.IO, **kwargs: base.CommonArgs): - """Stores input as a DAG object and returns its key. + def put(self, data: utils.clean_file_t, **kwargs: base.CommonArgs): + """Decodes the given input file as a DAG object and returns their key .. code-block:: python @@ -63,7 +62,7 @@ def put(self, data: ty.IO, **kwargs: base.CommonArgs): Parameters ---------- data - IO stream object with data that should be put + IO stream object of path to a file containing the data to put Returns ------- @@ -75,8 +74,8 @@ def put(self, data: ty.IO, **kwargs: base.CommonArgs): headers=headers, **kwargs) @base.returns_single_item(base.ResponseBase) - def resolve(self, cid: str, **kwargs: base.CommonArgs): - """Resolves a DAG node from its Cid + def resolve(self, cid: base.cid_t, **kwargs: base.CommonArgs): + """Resolves a DAG node from its CID, returning its address and remaining path .. code-block:: python @@ -100,8 +99,8 @@ def resolve(self, cid: str, **kwargs: base.CommonArgs): return self._client.request('/dag/resolve', args, decoder='json', **kwargs) @base.returns_single_item(base.ResponseBase) - def imprt(self, data: ty.IO, **kwargs: base.CommonArgs): - """Imports a .car file with a DAG to IPFS + def imprt(self, data: utils.clean_file_t, **kwargs: base.CommonArgs): + """Imports a .car file with a DAG into IPFS .. code-block:: python @@ -114,6 +113,9 @@ def imprt(self, data: ty.IO, **kwargs: base.CommonArgs): } } + *Note*: This method is named ``.imprt`` (rather than ``.import``) to avoid causing a Python + :exc:`SyntaxError` due to ``import`` being global keyword in Python. + Parameters ---------- data @@ -133,7 +135,10 @@ def export(self, cid: str, **kwargs: base.CommonArgs): .. code-block:: python - >>> bytes = client.dag.export('bafyreidepjmjhvhlvp5eyxqpmyyi7rxwvl7wsglwai3cnvq63komq4tdya') + >>> data = client.dag.export('bafyreidepjmjhvhlvp5eyxqpmyyi7rxwvl7wsglwai3cnvq63komq4tdya') + + *Note*: When exporting larger DAG structures, remember that you can set the *stream* + parameter to ``True`` on any method to have it return results incrementally. Parameters ---------- From 32f3b328cac62b70a79a98e0099d789f66eae66c Mon Sep 17 00:00:00 2001 From: Alexander Schlarb Date: Mon, 24 Aug 2020 17:57:20 +0200 Subject: [PATCH 5/5] Only run IPFS DAG tests on go-IPFS 0.5+ --- test/functional/test_dag.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/functional/test_dag.py b/test/functional/test_dag.py index 6345020c..7a5079e6 100644 --- a/test/functional/test_dag.py +++ b/test/functional/test_dag.py @@ -1,9 +1,15 @@ import io +import pytest + import conftest def test_put_get_resolve(client): + version = tuple(map(int, client.version()["Version"].split('-', 1)[0].split('.'))) + if version < (0, 5): + pytest.skip("IPFS DAG APIs first appeared in go-IPFS 0.5") + data = io.BytesIO(br'{"links": []}') response = client.dag.put(data) @@ -23,10 +29,14 @@ def test_put_get_resolve(client): def test_import_export(client): + version = tuple(map(int, client.version()["Version"].split('-', 1)[0].split('.'))) + if version < (0, 5): + pytest.skip("IPFS DAG APIs first appeared in go-IPFS 0.5") + # This file was created by inserting a simple JSON object into IPFS and # exporting it using `ipfs dag export > file.car` data_car = conftest.TEST_DIR / 'fake_json' / 'data.car' - data_car = str(data_car) + data_car = str(data_car) #PY35 with open(data_car, 'rb') as file: response = client.dag.imprt(file)