From 81b78dec1ff80a94c51d6d61c815cfc8978cfc7d Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 11:44:38 +0100 Subject: [PATCH 01/10] WIP: Generate HTML doc from Notebooks --- awswrangler/_config.py | 4 +- awswrangler/s3/_write_excel.py | 2 +- awswrangler/s3/_write_parquet.py | 4 +- awswrangler/s3/_write_text.py | 4 +- building/build-docs.sh | 6 +++ docs/source/conf.py | 10 ++++- docs/source/index.rst | 2 +- docs/source/notebooks.rst | 8 ++++ docs/source/tutorials/.gitignore | 2 + requirements-dev.txt | 3 ++ tutorials/003 - Amazon S3.ipynb | 64 ++++++++++++++++---------------- 11 files changed, 68 insertions(+), 41 deletions(-) create mode 100644 docs/source/notebooks.rst create mode 100644 docs/source/tutorials/.gitignore diff --git a/awswrangler/_config.py b/awswrangler/_config.py index 25e1f2d59..0fedb21fa 100644 --- a/awswrangler/_config.py +++ b/awswrangler/_config.py @@ -364,8 +364,8 @@ def _inject_config_doc(doc: Optional[str], available_configs: Tuple[str, ...]) - if "\n Parameters" not in doc: return doc header: str = ( - "\n Note\n ----" - "\n This functions has arguments that can has default values configured globally through " + "\n\n Note\n ----" + "\n This function has arguments which can be configured globally through " "*wr.config* or environment variables:\n\n" ) args: Tuple[str, ...] = tuple(f" - {x}\n" for x in available_configs) diff --git a/awswrangler/s3/_write_excel.py b/awswrangler/s3/_write_excel.py index dbb191414..1556d1f8d 100644 --- a/awswrangler/s3/_write_excel.py +++ b/awswrangler/s3/_write_excel.py @@ -43,7 +43,7 @@ def to_excel( s3_additional_kwargs : Optional[Dict[str, Any]] Forward to botocore requests. Valid parameters: "ACL", "Metadata", "ServerSideEncryption", "StorageClass", "SSECustomerAlgorithm", "SSECustomerKey", "SSEKMSKeyId", "SSEKMSEncryptionContext", "Tagging", - "RequestPayer", "ExpectedBucketOwner". + "RequestPayer", "ExpectedBucketOwner". e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'} use_threads : bool True to enable concurrent requests, False to disable multiple threads. diff --git a/awswrangler/s3/_write_parquet.py b/awswrangler/s3/_write_parquet.py index 0ebf7f90c..8394bcc97 100644 --- a/awswrangler/s3/_write_parquet.py +++ b/awswrangler/s3/_write_parquet.py @@ -270,7 +270,7 @@ def to_parquet( # pylint: disable=too-many-arguments,too-many-locals s3_additional_kwargs : Optional[Dict[str, Any]] Forward to botocore requests. Valid parameters: "ACL", "Metadata", "ServerSideEncryption", "StorageClass", "SSECustomerAlgorithm", "SSECustomerKey", "SSEKMSKeyId", "SSEKMSEncryptionContext", "Tagging", - "RequestPayer", "ExpectedBucketOwner". + "RequestPayer", "ExpectedBucketOwner". e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'} sanitize_columns : bool True to sanitize columns names (using `wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`) @@ -740,7 +740,7 @@ def store_parquet_metadata( # pylint: disable=too-many-arguments s3_additional_kwargs : Optional[Dict[str, Any]] Forward to botocore requests. Valid parameters: "ACL", "Metadata", "ServerSideEncryption", "StorageClass", "SSECustomerAlgorithm", "SSECustomerKey", "SSEKMSKeyId", "SSEKMSEncryptionContext", "Tagging", - "RequestPayer", "ExpectedBucketOwner". + "RequestPayer", "ExpectedBucketOwner". e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'} boto3_session : boto3.Session(), optional Boto3 Session. The default boto3 session will be used if boto3_session receive None. diff --git a/awswrangler/s3/_write_text.py b/awswrangler/s3/_write_text.py index 48fd72f78..569b693fd 100644 --- a/awswrangler/s3/_write_text.py +++ b/awswrangler/s3/_write_text.py @@ -153,7 +153,7 @@ def to_csv( # pylint: disable=too-many-arguments,too-many-locals,too-many-state s3_additional_kwargs : Optional[Dict[str, Any]] Forward to botocore requests. Valid parameters: "ACL", "Metadata", "ServerSideEncryption", "StorageClass", "SSECustomerAlgorithm", "SSECustomerKey", "SSEKMSKeyId", "SSEKMSEncryptionContext", "Tagging", - "RequestPayer", "ExpectedBucketOwner". + "RequestPayer", "ExpectedBucketOwner". e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'} sanitize_columns : bool True to sanitize columns names or False to keep it as is. @@ -563,7 +563,7 @@ def to_json( s3_additional_kwargs : Optional[Dict[str, Any]] Forward to botocore requests. Valid parameters: "ACL", "Metadata", "ServerSideEncryption", "StorageClass", "SSECustomerAlgorithm", "SSECustomerKey", "SSEKMSKeyId", "SSEKMSEncryptionContext", "Tagging", - "RequestPayer", "ExpectedBucketOwner". + "RequestPayer", "ExpectedBucketOwner". e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'} use_threads : bool True to enable concurrent requests, False to disable multiple threads. diff --git a/building/build-docs.sh b/building/build-docs.sh index 44586a8b3..b379119fc 100755 --- a/building/build-docs.sh +++ b/building/build-docs.sh @@ -3,5 +3,11 @@ set -ex pushd .. rm -rf docs/build docs/source/stubs +for notebook in tutorials/*.ipynb; do + filename=${notebook##*/} + basename=${filename%.ipynb} + echo $pwd + echo "{\"path\": \"../../../tutorials/${filename}\",\"extra-media\": [\"../../../tutorials/_static\"]}" > "docs/source/tutorials/${basename}.nblink" +done make -C docs/ html doc8 --ignore D005,D002 --max-line-length 120 docs/source diff --git a/docs/source/conf.py b/docs/source/conf.py index 57d4e4395..c7621d212 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -35,7 +35,13 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named "sphinx.ext.*") or your custom # ones. -extensions = ["sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", "sphinx.ext.napoleon"] +extensions = [ + "sphinx.ext.autosectionlabel", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "nbsphinx", + "nbsphinx_link", +] language = None @@ -137,3 +143,5 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] + +nbsphinx_allow_errors = True diff --git a/docs/source/index.rst b/docs/source/index.rst index e420fd257..da20fb625 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -61,7 +61,7 @@ Read The Docs what install - Tutorials + notebooks api Community Resources Logging diff --git a/docs/source/notebooks.rst b/docs/source/notebooks.rst new file mode 100644 index 000000000..c4a1369a2 --- /dev/null +++ b/docs/source/notebooks.rst @@ -0,0 +1,8 @@ +Tutorials +========= + +.. toctree:: + :maxdepth: 1 + :glob: + + tutorials/* diff --git a/docs/source/tutorials/.gitignore b/docs/source/tutorials/.gitignore new file mode 100644 index 000000000..c96a04f00 --- /dev/null +++ b/docs/source/tutorials/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 1334f49ee..ffa43e823 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,6 +17,9 @@ cfn-flip==1.2.3 twine==3.3.0 sphinx==3.5.1 sphinx_bootstrap_theme==0.7.1 +nbsphinx==0.8.1 +nbsphinx-link==1.3.0 +IPython==7.19.0 moto==1.3.16 jupyterlab==3.0.8 s3fs==0.4.2 diff --git a/tutorials/003 - Amazon S3.ipynb b/tutorials/003 - Amazon S3.ipynb index 05e729069..140a5d6ee 100644 --- a/tutorials/003 - Amazon S3.ipynb +++ b/tutorials/003 - Amazon S3.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Amazon S3" + "# 3 - Amazon S3" ] }, { @@ -102,14 +102,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 1. CSV files" + "## 1. CSV files" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1.1 Writing CSV files" + "### 1.1 Writing CSV files" ] }, { @@ -129,7 +129,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 1.2 Reading single CSV file" + "### 1.2 Reading single CSV file" ] }, { @@ -155,14 +155,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 1.3 Reading multiple CSV files" + "### 1.3 Reading multiple CSV files" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### 1.3.1 Reading CSV by list" + "#### 1.3.1 Reading CSV by list" ] }, { @@ -188,7 +188,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 1.3.2 Reading CSV by prefix" + "#### 1.3.2 Reading CSV by prefix" ] }, { @@ -214,14 +214,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 2. JSON files" + "## 2. JSON files" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 2.1 Writing JSON files" + "### 2.1 Writing JSON files" ] }, { @@ -250,7 +250,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2.2 Reading single JSON file" + "### 2.2 Reading single JSON file" ] }, { @@ -276,14 +276,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2.3 Reading multiple JSON files" + "### 2.3 Reading multiple JSON files" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### 2.3.1 Reading JSON by list" + "#### 2.3.1 Reading JSON by list" ] }, { @@ -309,7 +309,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 2.3.2 Reading JSON by prefix" + "#### 2.3.2 Reading JSON by prefix" ] }, { @@ -335,7 +335,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 3. Parquet files" + "## 3. Parquet files" ] }, { @@ -349,7 +349,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 3.1 Writing Parquet files" + "### 3.1 Writing Parquet files" ] }, { @@ -369,7 +369,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 3.2 Reading single Parquet file" + "### 3.2 Reading single Parquet file" ] }, { @@ -395,14 +395,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 3.3 Reading multiple Parquet files" + "### 3.3 Reading multiple Parquet files" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### 3.3.1 Reading Parquet by list" + "#### 3.3.1 Reading Parquet by list" ] }, { @@ -428,7 +428,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 3.3.2 Reading Parquet by prefix" + "#### 3.3.2 Reading Parquet by prefix" ] }, { @@ -454,7 +454,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 4. Fixed-width formatted files (only read)" + "## 4. Fixed-width formatted files (only read)" ] }, { @@ -487,7 +487,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 4.1 Reading single FWF file" + "### 4.1 Reading single FWF file" ] }, { @@ -513,14 +513,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 4.2 Reading multiple FWF files" + "### 4.2 Reading multiple FWF files" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### 4.2.1 Reading FWF by list" + "#### 4.2.1 Reading FWF by list" ] }, { @@ -546,7 +546,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 4.2.2 Reading FWF by prefix" + "#### 4.2.2 Reading FWF by prefix" ] }, { @@ -572,14 +572,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 5. Excel files" + "## 5. Excel files" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 5.1 Writing Excel file" + "### 5.1 Writing Excel file" ] }, { @@ -606,7 +606,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 5.2 Reading Excel file" + "### 5.2 Reading Excel file" ] }, { @@ -632,7 +632,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 6. Reading with lastModified filter " + "## 6. Reading with lastModified filter" ] }, { @@ -674,7 +674,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 6.2 Define the Date time and specify the Timezone " + "### 6.2 Define the Date time and specify the Timezone" ] }, { @@ -863,7 +863,7 @@ } }, "source": [ - "## 8.1 Upload object from a file path" + "### 8.1 Upload object from a file path" ] }, { @@ -904,7 +904,7 @@ } }, "source": [ - "## 8.2 Upload object from a file-like object in binary mode" + "### 8.2 Upload object from a file-like object in binary mode" ] }, { @@ -942,7 +942,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 9. Delete objects" + "## 9. Delete objects" ] }, { From 1d960921c905619c68bec8eee4a4d760ce46c383 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 13:15:45 +0100 Subject: [PATCH 02/10] Copy tutorials from Sphinx --- building/build-docs.sh | 6 ------ docs/source/_ext/copy_tutorials.py | 12 ++++++++++++ docs/source/conf.py | 3 +++ requirements-docs.txt | 3 +++ 4 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 docs/source/_ext/copy_tutorials.py diff --git a/building/build-docs.sh b/building/build-docs.sh index b379119fc..44586a8b3 100755 --- a/building/build-docs.sh +++ b/building/build-docs.sh @@ -3,11 +3,5 @@ set -ex pushd .. rm -rf docs/build docs/source/stubs -for notebook in tutorials/*.ipynb; do - filename=${notebook##*/} - basename=${filename%.ipynb} - echo $pwd - echo "{\"path\": \"../../../tutorials/${filename}\",\"extra-media\": [\"../../../tutorials/_static\"]}" > "docs/source/tutorials/${basename}.nblink" -done make -C docs/ html doc8 --ignore D005,D002 --max-line-length 120 docs/source diff --git a/docs/source/_ext/copy_tutorials.py b/docs/source/_ext/copy_tutorials.py new file mode 100644 index 000000000..5228681ad --- /dev/null +++ b/docs/source/_ext/copy_tutorials.py @@ -0,0 +1,12 @@ +from pathlib import Path +import json + + +def setup(app): + for f in Path("../tutorials").glob("*.ipynb"): + with open(f"source/tutorials/{f.stem}.nblink", "w") as output_file: + nb_link = { + "path": f"../../../tutorials/{f.name}", + "extra-media": ["../../../tutorials/_static"] + } + json.dump(nb_link, output_file) diff --git a/docs/source/conf.py b/docs/source/conf.py index c7621d212..e8d67b1db 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,6 +15,8 @@ import sphinx_bootstrap_theme +# Append local Sphinx extensions +sys.path.append(os.path.abspath("./_ext")) # Insert awswrangler"s path into the system. sys.path.insert(0, os.path.abspath("../..")) import awswrangler @@ -41,6 +43,7 @@ "sphinx.ext.napoleon", "nbsphinx", "nbsphinx_link", + "copy_tutorials", ] language = None diff --git a/requirements-docs.txt b/requirements-docs.txt index 60aa0acf0..5aff97643 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,3 +1,6 @@ sphinx==3.5.1 sphinx_bootstrap_theme==0.7.1 +nbsphinx==0.8.1 +nbsphinx-link==1.3.0 +IPython==7.19.0 -e . From 07e81ea2b8bc6ee9f5b1edbd5646de9308b3d7f5 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 13:26:54 +0100 Subject: [PATCH 03/10] Linting --- docs/source/_ext/copy_tutorials.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/source/_ext/copy_tutorials.py b/docs/source/_ext/copy_tutorials.py index 5228681ad..9c1def63d 100644 --- a/docs/source/_ext/copy_tutorials.py +++ b/docs/source/_ext/copy_tutorials.py @@ -1,12 +1,9 @@ -from pathlib import Path import json +from pathlib import Path def setup(app): for f in Path("../tutorials").glob("*.ipynb"): with open(f"source/tutorials/{f.stem}.nblink", "w") as output_file: - nb_link = { - "path": f"../../../tutorials/{f.name}", - "extra-media": ["../../../tutorials/_static"] - } + nb_link = {"path": f"../../../tutorials/{f.name}", "extra-media": ["../../../tutorials/_static"]} json.dump(nb_link, output_file) From 8d5f0eb36c73d2eb15b2073df3a1eb7af70e6c47 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 15:24:32 +0100 Subject: [PATCH 04/10] Test conda rtd --- .readthedocs.yml | 4 ++++ docs/environment.yml | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 .readthedocs.yml create mode 100644 docs/environment.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000..935cd5fe9 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,4 @@ +version: 2 +formats: all +conda: + environment: docs/environment.yml \ No newline at end of file diff --git a/docs/environment.yml b/docs/environment.yml new file mode 100644 index 000000000..b6ec60b04 --- /dev/null +++ b/docs/environment.yml @@ -0,0 +1,13 @@ +channels: + - conda-forge +dependencies: + - python>=3 + - pandoc + - ipykernel + - pip + - pip: + - nbsphinx + - nbsphinx-link + - sphinx + - sphinx_bootstrap_theme + - IPython \ No newline at end of file From f1c4f9c1efe4ae74b961f7c8c1b4f1be45bd40be Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 15:35:20 +0100 Subject: [PATCH 05/10] Add local dir as requirements --- docs/environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/environment.yml b/docs/environment.yml index b6ec60b04..3e8e3cbb1 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -10,4 +10,5 @@ dependencies: - nbsphinx-link - sphinx - sphinx_bootstrap_theme - - IPython \ No newline at end of file + - IPython + - -e . From 7a744b4b4b884afb2f069b1890196ea8729c6846 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 15:41:52 +0100 Subject: [PATCH 06/10] Fix local pip path --- docs/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/environment.yml b/docs/environment.yml index 3e8e3cbb1..8dbc60fdb 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -11,4 +11,4 @@ dependencies: - sphinx - sphinx_bootstrap_theme - IPython - - -e . + - -e .. From 81a8f7676ed6afea556370804c888cb651f8978f Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 15:50:20 +0100 Subject: [PATCH 07/10] Fix sphinx extension setup path --- docs/source/_ext/copy_tutorials.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/_ext/copy_tutorials.py b/docs/source/_ext/copy_tutorials.py index 9c1def63d..6b4b5cb8f 100644 --- a/docs/source/_ext/copy_tutorials.py +++ b/docs/source/_ext/copy_tutorials.py @@ -3,7 +3,8 @@ def setup(app): - for f in Path("../tutorials").glob("*.ipynb"): - with open(f"source/tutorials/{f.stem}.nblink", "w") as output_file: + file_dir = Path(__file__).parent + for f in file_dir.joinpath("../../../tutorials").glob("*.ipynb"): + with open(file_dir.joinpath(f"../tutorials/{f.stem}.nblink"), "w") as output_file: nb_link = {"path": f"../../../tutorials/{f.name}", "extra-media": ["../../../tutorials/_static"]} json.dump(nb_link, output_file) From 7d108fe1b501a8bb69427df9724679bd26577844 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 16:10:02 +0100 Subject: [PATCH 08/10] Delete requirements-docs.txt --- .readthedocs.yml | 2 +- requirements-docs.txt | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 requirements-docs.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 935cd5fe9..7030530af 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,4 @@ version: 2 formats: all conda: - environment: docs/environment.yml \ No newline at end of file + environment: docs/environment.yml diff --git a/requirements-docs.txt b/requirements-docs.txt deleted file mode 100644 index 5aff97643..000000000 --- a/requirements-docs.txt +++ /dev/null @@ -1,6 +0,0 @@ -sphinx==3.5.1 -sphinx_bootstrap_theme==0.7.1 -nbsphinx==0.8.1 -nbsphinx-link==1.3.0 -IPython==7.19.0 --e . From 240f9a21eddf8c83bee2b6423df51871e93dba04 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 18:21:04 +0100 Subject: [PATCH 09/10] Adapt link in docs --- docs/source/_static/css/max_width.css | 3 +++ docs/source/conf.py | 6 +++++- docs/source/index.rst | 2 +- docs/source/notebooks.rst | 8 -------- docs/source/tutorials.rst | 10 ++++++++++ 5 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 docs/source/_static/css/max_width.css delete mode 100644 docs/source/notebooks.rst create mode 100644 docs/source/tutorials.rst diff --git a/docs/source/_static/css/max_width.css b/docs/source/_static/css/max_width.css new file mode 100644 index 000000000..a260de7cc --- /dev/null +++ b/docs/source/_static/css/max_width.css @@ -0,0 +1,3 @@ +div.body { + max-width: 90%; +} \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index e8d67b1db..d1201594d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -92,7 +92,7 @@ "navbar_links": [ ("What is Data Wrangler?", "what"), ("Install", "install"), - ("Tutorials", "https://github.com/awslabs/aws-data-wrangler/tree/main/tutorials", True), + ("Tutorials", "tutorials"), ("API Reference", "api"), ("License", "https://github.com/awslabs/aws-data-wrangler/blob/main/LICENSE.txt", True), ("Contributing", "https://github.com/awslabs/aws-data-wrangler/blob/main/CONTRIBUTING.md", True), @@ -148,3 +148,7 @@ html_static_path = ["_static"] nbsphinx_allow_errors = True + + +def setup(app): + app.add_css_file("css/max_width.css") diff --git a/docs/source/index.rst b/docs/source/index.rst index da20fb625..3f3bc8ad0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -61,7 +61,7 @@ Read The Docs what install - notebooks + tutorials api Community Resources Logging diff --git a/docs/source/notebooks.rst b/docs/source/notebooks.rst deleted file mode 100644 index c4a1369a2..000000000 --- a/docs/source/notebooks.rst +++ /dev/null @@ -1,8 +0,0 @@ -Tutorials -========= - -.. toctree:: - :maxdepth: 1 - :glob: - - tutorials/* diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst new file mode 100644 index 000000000..d934d0719 --- /dev/null +++ b/docs/source/tutorials.rst @@ -0,0 +1,10 @@ +Tutorials +========= + +.. note:: You can also find all Tutorial Notebooks on `GitHub `_. + +.. toctree:: + :maxdepth: 1 + :glob: + + tutorials/* From 894552bf7bfdd0da255ea0c2717e28a65a5c07eb Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Thu, 25 Feb 2021 20:58:03 +0100 Subject: [PATCH 10/10] Update links in docstrings --- awswrangler/athena/_read.py | 32 +++++++++++++-------------- awswrangler/s3/_read_parquet.py | 2 +- awswrangler/s3/_read_text.py | 6 ++--- awswrangler/s3/_write_parquet.py | 4 ++-- awswrangler/s3/_write_text.py | 2 +- docs/source/_static/css/max_width.css | 2 +- docs/source/conf.py | 1 + docs/source/tutorials/.gitignore | 2 +- 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/awswrangler/athena/_read.py b/awswrangler/athena/_read.py index 514de0c2d..aef498274 100644 --- a/awswrangler/athena/_read.py +++ b/awswrangler/athena/_read.py @@ -590,12 +590,12 @@ def read_sql_query( **Related tutorial:** - - `Amazon Athena `_ - - `Athena Cache `_ - - `Global Configurations `_ + - `Amazon Athena `_ + - `Athena Cache `_ + - `Global Configurations `_ **There are two approaches to be defined through ctas_approach parameter:** @@ -642,8 +642,8 @@ def read_sql_query( /athena.html#Athena.Client.get_query_execution>`_ . For a practical example check out the - `related tutorial `_! + `related tutorial `_! Note @@ -853,12 +853,12 @@ def read_sql_table( **Related tutorial:** - - `Amazon Athena `_ - - `Athena Cache `_ - - `Global Configurations `_ + - `Amazon Athena `_ + - `Athena Cache `_ + - `Global Configurations `_ **There are two approaches to be defined through ctas_approach parameter:** @@ -902,8 +902,8 @@ def read_sql_table( /athena.html#Athena.Client.get_query_execution>`_ . For a practical example check out the - `related tutorial `_! + `related tutorial `_! Note diff --git a/awswrangler/s3/_read_parquet.py b/awswrangler/s3/_read_parquet.py index e6ca2889b..d6f726296 100644 --- a/awswrangler/s3/_read_parquet.py +++ b/awswrangler/s3/_read_parquet.py @@ -684,7 +684,7 @@ def read_parquet_table( This function MUST return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/023%20-%20Flexible%20Partitions%20Filter.ipynb + https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/tutorials/023%20-%20Flexible%20Partitions%20Filter.html columns : List[str], optional Names of columns to read from the file(s). validate_schema: diff --git a/awswrangler/s3/_read_text.py b/awswrangler/s3/_read_text.py index 4821a2b0a..ce58aaa34 100644 --- a/awswrangler/s3/_read_text.py +++ b/awswrangler/s3/_read_text.py @@ -217,7 +217,7 @@ def read_csv( This function MUST return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/023%20-%20Flexible%20Partitions%20Filter.ipynb + https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/tutorials/023%20-%20Flexible%20Partitions%20Filter.html pandas_kwargs : KEYWORD arguments forwarded to pandas.read_csv(). You can NOT pass `pandas_kwargs` explicit, just add valid Pandas arguments in the function call and Wrangler will accept it. @@ -359,7 +359,7 @@ def read_fwf( This function MUST return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/023%20-%20Flexible%20Partitions%20Filter.ipynb + https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/tutorials/023%20-%20Flexible%20Partitions%20Filter.html pandas_kwargs: KEYWORD arguments forwarded to pandas.read_fwf(). You can NOT pass `pandas_kwargs` explicit, just add valid Pandas arguments in the function call and Wrangler will accept it. @@ -505,7 +505,7 @@ def read_json( This function MUST return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/023%20-%20Flexible%20Partitions%20Filter.ipynb + https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/tutorials/023%20-%20Flexible%20Partitions%20Filter.html pandas_kwargs: KEYWORD arguments forwarded to pandas.read_json(). You can NOT pass `pandas_kwargs` explicit, just add valid Pandas arguments in the function call and Wrangler will accept it. diff --git a/awswrangler/s3/_write_parquet.py b/awswrangler/s3/_write_parquet.py index 8394bcc97..87293de1f 100644 --- a/awswrangler/s3/_write_parquet.py +++ b/awswrangler/s3/_write_parquet.py @@ -291,7 +291,7 @@ def to_parquet( # pylint: disable=too-many-arguments,too-many-locals concurrent_partitioning: bool If True will increase the parallelism level during the partitions writing. It will decrease the writing time and increase the memory usage. - https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/022%20-%20Writing%20Partitions%20Concurrently.ipynb + https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html mode: str, optional ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True. For details check the related tutorial: @@ -302,7 +302,7 @@ def to_parquet( # pylint: disable=too-many-arguments,too-many-locals If True allows schema evolution (new or missing columns), otherwise a exception will be raised. (Only considered if dataset=True and mode in ("append", "overwrite_partitions")) Related tutorial: - https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/014%20-%20Schema%20Evolution.ipynb + https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/tutorials/014%20-%20Schema%20Evolution.html database : str, optional Glue/Athena catalog: Database name. table : str, optional diff --git a/awswrangler/s3/_write_text.py b/awswrangler/s3/_write_text.py index 569b693fd..dc0c0537e 100644 --- a/awswrangler/s3/_write_text.py +++ b/awswrangler/s3/_write_text.py @@ -173,7 +173,7 @@ def to_csv( # pylint: disable=too-many-arguments,too-many-locals,too-many-state concurrent_partitioning: bool If True will increase the parallelism level during the partitions writing. It will decrease the writing time and increase the memory usage. - https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/022%20-%20Writing%20Partitions%20Concurrently.ipynb + https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html mode : str, optional ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True. For details check the related tutorial: diff --git a/docs/source/_static/css/max_width.css b/docs/source/_static/css/max_width.css index a260de7cc..6aace8bd4 100644 --- a/docs/source/_static/css/max_width.css +++ b/docs/source/_static/css/max_width.css @@ -1,3 +1,3 @@ div.body { max-width: 90%; -} \ No newline at end of file +} diff --git a/docs/source/conf.py b/docs/source/conf.py index d1201594d..53e6b93c7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -148,6 +148,7 @@ html_static_path = ["_static"] nbsphinx_allow_errors = True +nbsphinx_execute = "never" def setup(app): diff --git a/docs/source/tutorials/.gitignore b/docs/source/tutorials/.gitignore index c96a04f00..d6b7ef32c 100644 --- a/docs/source/tutorials/.gitignore +++ b/docs/source/tutorials/.gitignore @@ -1,2 +1,2 @@ * -!.gitignore \ No newline at end of file +!.gitignore