Merge 08006c4 into 1d2b52e

.gitignore

@@ -113,5 +113,5 @@ examples/tmp.py
 # mypy
 .mypy_cache
 
-# documentations
-docs/
+# mkdocs outputs
+site/

.readthedocs.yml

@@ -6,12 +6,12 @@
 version: 2
 
 # Build documentation in the docs/ directory with Sphinx
-sphinx:
-  configuration: rst/conf.py
+# sphinx:
+#   configuration: rst/conf.py
 
 # Build documentation with MkDocs
-#mkdocs:
-#  configuration: mkdocs.yml
+mkdocs:
+ configuration: mkdocs.yml
 
 # Optionally build your docs in additional formats such as PDF
 formats:

.vscode/settings.json

@@ -1,7 +1,6 @@
 {
     "python.pythonPath": "${workspaceFolder}/.venv/bin/python3",
     "python.formatting.provider": "black",
-    "autoDocstring.includeName": true,
     "autoDocstring.startOnNewLine": true,
     "autoDocstring.docstringFormat": "google",
     "python.linting.flake8Enabled": true,

CHANGELOG.md

@@ -1,9 +1,14 @@
 # `iotoolz`
 
-## v0.1.0
+## v0.1.0-rc-3
+
+- Converted sphinx docs to mkdocs for better look and feel.
+- Removed the need for a context manager for the iterable.
+- Renamed the 2 abstract methods.
+
+## v0.1.0-rc-2
 
 - Initial Release
 
   - Features:
     - `iotoolz.AbcStream` is an abstract class to represent any IO. It follows the `open` very closely.
-    - `iotoolz.

Makefile

@@ -5,7 +5,7 @@ requirements:
 	poetry export -f requirements.txt --output requirements-dev.txt --dev --extras all
 
 docs: .FORCE requirements
-	poetry run sphinx-build rst docs -b dirhtml -E -P
+	poetry run mkdocs build
 
 check:
 	poetry run isort -c iotoolz
@@ -26,7 +26,7 @@ coveralls: test
 	poetry run coveralls
 
 serve-docs: docs
-	cd docs/  && poetry run python -m http.server 8000
+	cd site/  && poetry run python -m http.server 8000
 
 format:
 	poetry run autoflake -i -r ./iotoolz --remove-all-unused-imports --ignore-init-module-imports --expand-star-imports

README.md

@@ -83,25 +83,33 @@ foo_txt.close()
 
 ```
 
-## Pipe streams
+## Piping streams
 
 `pipe` is method to push data to a sink (similar to NodeJS stream except it has no
 watermark or buffering).
 
 ```py
 from  iotoolz.streams import open_stream
 
-local_file = open_stream("path/to/google.html", content_type="text/html", mode="w")
-temp_file = open_stream("tmp://google.html", content_type="text/html", mode="wb")
+local_file = open_stream(
+    "path/to/google.html", content_type="text/html", mode="w"
+)
+temp_file = open_stream(
+    "tmp://google.html", content_type="text/html", mode="wb"
+)
 
 # when source is closed, all sinks will be closed also
 with open_stream("https://google.com") as source:
     # writes to a temp file then to a local file in sequence
     source.pipe(temp_file).pipe(local_file)
 
 
-local_file2 = open_stream("path/to/google1.html", content_type="text/html", mode="w")
-local_file3 = open_stream("path/to/google2.html", content_type="text/html", mode="w")
+local_file2 = open_stream(
+    "path/to/google1.html", content_type="text/html", mode="w"
+)
+local_file3 = open_stream(
+    "path/to/google2.html", content_type="text/html", mode="w"
+)
 
 # when source is closed, all sinks will be closed also
 with open_stream("tmp://foo_src", mode="w") as source:
@@ -119,20 +127,18 @@ with open_stream("tmp://foo_src", mode="w") as source:
 The abstract class `iotoolz.AbcStream` requires the following methods to be implemented:
 
 ```py
-# This is the material method to get the data from the actual IO resource.
-# It should return an iterable to the data and the corresponding StreamInfo.
-# If resources to the data need to be released, you can also return a ContextManager
-# to the iterable instead.
-def _read_to_iterable(
+# This is the material method to get the data from the actual IO resource and return
+# a Tuple with an Iterable to the data and the corresponding StreamInfo.
+def read_to_iterable_(
     self, uri: str, chunk_size: int, **kwargs
-) -> Tuple[Union[Iterable[bytes], ContextManager[Iterable[bytes]]], StreamInfo]:
+) -> Tuple[Iterable[bytes], StreamInfo]:
     ...
 
 # This is the material method to write the data to the actual IO resource.
 # This method is only triggered when "close" or "save" is called.
 # You should use the "file_" parameter (a file-like obj) to write the current data to
 # the actual IO resource.
-def _write_from_fileobj(
+def write_from_fileobj_(
     self, uri: str, file_: IO[bytes], size: int, **kwargs
 ) -> StreamInfo:
     ...
@@ -146,3 +152,36 @@ Ideally, the implementation of any `AbcStream` class should also provide
 in the future to infer what sort of schemas that will be supported by the class. For
 example, since `https` and `http` are supported by `iotoolz.HttpStream`, all uri that
 starts with `https://` and `http://` can be handled by `iotoolz.HttpStream`.
+
+### Example implementation of a HttpStream using requests
+
+```py
+class HttpStream(AbcStream):
+    supported_schemes = {"http", "https"}
+
+    def read_to_iterable_(
+        self, uri: str, chunk_size: int, **kwargs
+    ) -> Tuple[Iterable[bytes], StreamInfo]:
+        resp = requests.get(uri, stream=True, **cytoolz.dissoc(kwargs, "stream"))
+        resp.raise_for_status()
+        info = StreamInfo(
+            content_type=resp.headers.get("Content-Type"),
+            encoding=resp.encoding,
+            etag=resp.headers.get("etag"),
+        )
+        return resp.iter_content(chunk_size=chunk_size), info
+
+    def write_from_fileobj_(
+        self, uri: str, file_: IO[bytes], size: int, **kwargs
+    ) -> StreamInfo:
+        use_post = kwargs.get("use_post")
+        requests_method = requests.post if use_post else requests.put
+        resp = requests_method(
+            uri,
+            data=requests_toolbelt.StreamingIterator(size, file_),
+            **cytoolz.dissoc(kwargs, "use_post", "data")
+        )
+        resp.raise_for_status()
+        return StreamInfo()
+
+```

docs/changelog.md

@@ -0,0 +1 @@
+{!CHANGELOG.md!}

docs/classes/AbcStream.md

@@ -0,0 +1,3 @@
+# AbcStream
+
+::: iotoolz:AbcStream

docs/classes/FileStream.md

@@ -0,0 +1,5 @@
+# FileStream
+
+`iotoolz.FileStream` is a helper
+
+::: iotoolz:FileStream

docs/classes/HttpStream.md

@@ -0,0 +1,5 @@
+# HttpStream
+
+`iotoolz.HttpStream` is a helper
+
+::: iotoolz:HttpStream

docs/classes/StreamInfo.md

@@ -0,0 +1,5 @@
+# StreamInfo
+
+::: iotoolz:StreamInfo
+    selections:
+        members: true

docs/classes/Streams.md

@@ -0,0 +1,3 @@
+# iotoolz.streams.Streams
+
+::: iotoolz.streams:Streams

docs/classes/TempStream.md

@@ -0,0 +1,5 @@
+# TempStream
+
+`iotoolz.TempStream` is a helper
+
+::: iotoolz:TempStream

docs/index.md

@@ -0,0 +1 @@
+{!README.md!}

docs/streams.md

@@ -0,0 +1,39 @@
+# iotoolz.streams
+
+The module `iotoolz.streams` provides a helper class
+[iotoolz.streams.Streams](./classes/Streams.md) to manage the different concrete
+AbcStream classes.
+
+It also provides a default `iotoolz.streams.Streams` singleton which support most of the
+implemented streams. The singleton object's methods are exposed as module callables:
+
+- `open_stream`: corresponds to `iotoolz.streams.Streams.open`
+- `register_stream` : corresponds to `iotoolz.streams.Streams.register_stream`
+- `set_schema_kwargs`: corresponds to `iotoolz.streams.Streams.set_schema_kwargs`
+
+## Usage
+
+```py
+from iotoolz.streams import open_stream, register_stream, set_schema_kwargs
+
+# do not verify the ssl cert for all https requests
+set_schema_kwargs("https", verify=False)
+
+# print line by line some data in from a https endpoint
+with open_stream("https://foo/bar/data.txt", "r") as stream:
+    for line in stream:
+        print(line)
+
+# Post some binary content to a http endpoint
+with open_stream("https://foo.bar/api/data", "wb") as stream:
+    stream.write(b"hello world")
+
+# Read a file from local path
+with open_stream("path/to/data.csv", "r") as csv_source:
+    # create a tempfile
+    tmpsink = open_stream("tmp://foobar.txt", "w")
+    # pipe content in csv_source to tmpsink
+    csv_source.pipe(tmpsink)
+
+print(tmpsink.read())  # prints contents of csv_source
+```

docs/stylesheets/extra.css

@@ -0,0 +1,17 @@
+.doc-method, .doc-attribute {
+    padding: 5px 5px;
+}
+.doc-method:nth-child(odd),
+.doc-attribute:nth-child(odd) {
+    background-color: #f1f1f1;
+}
+.md-typeset h3 {
+    border-bottom: 2px solid #eeeeee;
+    padding-left: 5px;
+}
+h4.doc-heading {
+    font-size: 0.9rem;
+}
+div.doc-contents {
+    font-size: 0.7rem;
+}

docs/utils.md

@@ -0,0 +1,7 @@
+# iotoolz.utils
+
+::: iotoolz.utils
+    selection:
+        members:
+        - peek_stream
+        - guess_encoding