diff --git a/.gitignore b/.gitignore index eccdc755..f7fada41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ -# s3proxy artifacts +# Python testing artifacts +/osfs_local/ /s3proxy* /s3_local/ +smb.conf # Logs logs @@ -166,6 +168,9 @@ ENV/ # mypy .mypy_cache/ +# scratch +/scratch/ + # jetbrains ide stuff *.iml .idea/ diff --git a/Makefile b/Makefile index e4cfa0f8..b4a209eb 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ testjs: ## Clean and Make js tests yarn test -testpy: ## Clean and Make unit tests - python3.7 -m pytest -v jupyterfs/tests --cov=jupyterfs +testpy: ## Clean and Make py tests + python3.7 -m pytest -v jupyterfs/tests --cov=jupyterfs --cov-branch -test: lint ## run the tests - python3.7 -m pytest -v jupyterfs/tests --cov=jupyterfs --junitxml=python_junit.xml --cov-report=xml --cov-branch - yarn test +test: ## run all tests + testpy + testjs lintjs: ## run linter yarn lint @@ -15,8 +15,8 @@ lintpy: ## run linter flake8 jupyterfs setup.py lint: ## run linter - flake8 jupyterfs setup.py - yarn lint + lintpy + lintjs fixjs: ## run autopep8/tslint fix ./node_modules/.bin/tslint --fix src/* @@ -25,8 +25,8 @@ fixpy: ## run autopep8/tslint fix autopep8 --in-place -r -a -a jupyterfs/ fix: ## run autopep8/tslint fix - autopep8 --in-place -r -a -a jupyterfs/ - ./node_modules/.bin/tslint --fix src/* + fixpy + fixjs annotate: ## MyPy type annotation check mypy -s jupyterfs diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..1e42eb3c --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,65 @@ +image: Visual Studio 2019 + +environment: + PYTHON: "C:\\Python37" + +build: off + +services: +- docker + +# mentioned in appveyor docs, doesn't seem to work +# stack: docker, jdk 1.8, node 12, python 3.7 + +# runs before repo checkout +init: +# set the python version permanently (for debugging)... +- ps: "[Environment]::SetEnvironmentVariable(\"Path\", \"$env:PYTHON;$env:PYTHON\\Scripts;$env:Path\", \"Machine\")" +# ...and for this session +- cmd: set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% + +# install latest pip +- python -m pip install --upgrade pip + +# set the node version +- ps: Install-Product node 12 + +# install yarn globally +- npm install -g yarn + +install: +# set up a windows file share named "test" +- ps: ci/enable_os_share_windows.ps1 +- ps: ci/create_os_share_windows.ps1 + +# set up a local s3_proxy server +- bash -c "ci/s3_local.sh" +- ps: Start-Process -FilePath "java" -ArgumentList "-jar","s3proxy","--properties","s3proxy.conf" +- bash -c "sleep 5" +- curl http://127.0.0.1:9000/ + +# install python deps +- pip install --upgrade --upgrade-strategy=eager -e .[dev] + +# install js deps +- yarn + +# print all env vars +- ps: gci env:* | sort-object name + +test_script: +# lint the python code +- python -m flake8 jupyterfs setup.py + +# lint the js code +- yarn lint + +# test the python code +- python -m pytest -v jupyterfs/tests --cov=jupyterfs --cov-branch + +# test the js code +- yarn test + +on_failure: +# enable remote login (via rdp) to windows ci vm on test failure +- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2e4518a0..e1d906de 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,7 +4,7 @@ trigger: - '*' # must quote since "*" is a YAML reserved character; we want a string parameters: -- name: initializationSteps +- name: initSteps type: stepList default: - task: UsePythonVersion@0 @@ -37,27 +37,53 @@ parameters: - bash: env displayName: "List all environment variables" -- name: s3proxySteps + +- name: initDockerSteps type: stepList default: - - bash: nohup ci/s3_local.sh - displayName: "Start up local S3 server" + - task: DockerInstaller@0 + displayName: Docker Installer + inputs: + dockerVersion: 19.03.8 + releaseType: stable - - script: curl http://127.0.0.1:9000/ - displayName: 'Test local S3 server' -- name: s3proxyMacSteps +- name: initDockerStepsMac type: stepList default: - script: | ci/install_docker_desktop_mac.sh ci/start_docker_desktop_mac.sh - docker pull andrewgaul/s3proxy - nohup sudo docker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy & + displayName: 'Setup Docker Desktop Mac' + +- name: initS3proxyStepsLocal + type: stepList + default: + - bash: | + nohup ci/s3_local.sh -d -r sleep 5 - displayName: 'Start s3proxy in a docker' + displayName: "Start up local S3 server" - script: curl http://127.0.0.1:9000/ displayName: 'Test local S3 server' + +- name: initSmbOsWindows + type: stepList + default: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: 'ci/enable_os_share_windows.ps1' + errorActionPreference: 'stop' + failOnStderr: True + displayName: 'Enable os builtin file sharing' + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: 'ci/create_os_share_windows.ps1' + errorActionPreference: 'stop' + failOnStderr: True + displayName: 'Create share on os builtin samba server' + - name: testSteps type: stepList default: @@ -72,6 +98,7 @@ parameters: - script: make testjs displayName: 'Test JS' + - name: publishSteps type: stepList default: @@ -97,8 +124,8 @@ jobs: python.version: '3.7' steps: - - ${{ parameters.initializationSteps }} - - ${{ parameters.s3proxySteps }} + - ${{ parameters.initDockerSteps }} + - ${{ parameters.initSteps }} - ${{ parameters.testSteps }} - ${{ parameters.publishSteps }} @@ -112,22 +139,23 @@ jobs: python.version: '3.7' steps: - - ${{ parameters.s3proxyMacSteps }} - - ${{ parameters.initializationSteps }} + - ${{ parameters.initDockerStepsMac }} + - ${{ parameters.initSteps }} - ${{ parameters.testSteps }} - ${{ parameters.publishSteps }} -- job: 'Windows' - pool: - vmImage: 'vs2017-win2016' - - strategy: - matrix: - Python37: - python.version: '3.7' - - steps: - - ${{ parameters.initializationSteps }} - - ${{ parameters.s3proxySteps }} - - ${{ parameters.testSteps }} - - ${{ parameters.publishSteps }} +# - job: 'Windows' +# pool: +# vmImage: 'windows-2019' + +# strategy: +# matrix: +# Python37: +# python.version: '3.7' + +# steps: +# - ${{ parameters.initSmbOsWindows }} +# - ${{ parameters.initS3proxyStepsLocal }} +# - ${{ parameters.initSteps }} +# - ${{ parameters.testSteps }} +# - ${{ parameters.publishSteps }} diff --git a/ci/create_os_share_windows.ps1 b/ci/create_os_share_windows.ps1 new file mode 100644 index 00000000..f8d35f11 --- /dev/null +++ b/ci/create_os_share_windows.ps1 @@ -0,0 +1,13 @@ +#Requires -RunAsAdministrator + +$ErrorActionPreference = 'Stop' + +$smbuser_passwd = ConvertTo-SecureString "smbuser" -AsPlainText -Force +New-LocalUser "smbuser" -Password $smbuser_passwd + +mkdir C:\shared + +New-SmbShare -Name "test" -Path "C:\shared" -FullAccess smbuser + +# display info on newly-created share +Get-SmbShare diff --git a/ci/enable_os_share_windows.ps1 b/ci/enable_os_share_windows.ps1 new file mode 100644 index 00000000..f8018ba0 --- /dev/null +++ b/ci/enable_os_share_windows.ps1 @@ -0,0 +1,24 @@ +#Requires -RunAsAdministrator + +$ErrorActionPreference = 'Stop' + +# script to overcome all weird windows vm settings on azure + +# disable all password requirements (length, complexity, etc) +# needed to allow for creation of new account smbuser:smbuser +secedit /export /cfg c:\secpol.cfg +(gc C:\secpol.cfg).replace("PasswordComplexity = 1", "PasswordComplexity = 0") | Out-File C:\secpol.cfg +secedit /configure /db c:\windows\security\local.sdb /cfg c:\secpol.cfg /areas SECURITYPOLICY +rm -force c:\secpol.cfg -confirm:$false + +# turn on file sharing +netsh advfirewall firewall set rule group="File and Printer Sharing" new enable=Yes + +# print out some info about local NETBIOS cache +nbtstat -c + +# print out some info about local NETBIOS names +nbtstat -n + +# print out some info about local NETBIOS resolutions +nbtstat -r diff --git a/ci/install_docker_desktop_mac.sh b/ci/install_docker_desktop_mac.sh index f68d6535..bdd742af 100755 --- a/ci/install_docker_desktop_mac.sh +++ b/ci/install_docker_desktop_mac.sh @@ -5,8 +5,7 @@ # https://forums.docker.com/t/docker-for-mac-unattended-installation/27112 brew cask install docker -set -x -echo "Running xattr" +# allow the app to run without confirmation xattr -d -r com.apple.quarantine /Applications/Docker.app # preemptively do docker.app's setup to avoid any gui prompts diff --git a/ci/remove_os_share_windows.ps1 b/ci/remove_os_share_windows.ps1 new file mode 100644 index 00000000..aa8c773d --- /dev/null +++ b/ci/remove_os_share_windows.ps1 @@ -0,0 +1,27 @@ +#Requires -RunAsAdministrator + +# discover true names of errors: +# $Error[0].Exception.GetType().FullName + +$ErrorActionPreference = 'Stop' + +try { + Get-SmbShare -name "test" + Remove-SmbShare -name "test" -force +} +catch [Microsoft.PowerShell.Cmdletization.Cim.CimJobException] { +} + +try { + ls C:\shared + rm -r -fo C:\shared +} +catch [System.Management.Automation.ItemNotFoundException] { +} + +try { + Get-LocalUser -name "smbuser" + Remove-LocalUser -name "smbuser" +} +catch [Microsoft.PowerShell.Commands.UserNotFoundException] { +} diff --git a/ci/s3_local.sh b/ci/s3_local.sh index a848e890..9934eebb 100755 --- a/ci/s3_local.sh +++ b/ci/s3_local.sh @@ -2,6 +2,36 @@ S3PROXY_VERSION=1.7.0 S3PROXY_PORT=9000 +# Usage info +show_help() { +cat << EOF +Usage: ${0##*/} [-d] +Run a local instance of s3_proxy + + -d Detach; run s3_proxy as a background job +EOF +} + +# parse opts +detach= +run= +OPTIND=1 +while getopts hdtr opt; do + case $opt in + h) show_help + exit 0 + ;; + d) detach=-d + ;; + r) run=-r + ;; + *) show_help >&2 + exit 1 + ;; + esac +done +shift "$((OPTIND-1))" # Discard the options and sentinel -- + # make dir for storing s3 files locally mkdir -p s3_local @@ -29,7 +59,13 @@ EOT curl -L https://github.com/gaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy -o s3proxy # run s3proxy as a background job -java -jar s3proxy --properties s3proxy.conf & +if [ "$run" = "-r" ]; then + if [ "$detach" = "-d" ]; then + java -jar s3proxy --properties s3proxy.conf & + else + java -jar s3proxy --properties s3proxy.conf + fi +fi # no explicit wait needed, azure already stalls # for 10 seconds to wait for open process to stop diff --git a/ci/start_docker_desktop_mac.sh b/ci/start_docker_desktop_mac.sh index 6e1ae55e..f5ecc382 100755 --- a/ci/start_docker_desktop_mac.sh +++ b/ci/start_docker_desktop_mac.sh @@ -2,27 +2,34 @@ # refs: # https://stackoverflow.com/a/35979292/425458 - -case $1 in - -h|--help) - echo $'usage: docker-start\n\nStarts Docker (Docker.app) on macOS and waits until the Docker environment is initialized.' - exit 0 - ;; -esac -(( $# )) && { echo "ARGUMENT ERROR: Unexpected argument(s) specified. Use -h for help." >&2; exit 2; } +# https://crossprogramming.com/2019/12/27/use-docker-when-running-integration-tests-with-azure-pipelines.html#self-managed-docker-containers [[ $(uname) == 'Darwin' ]] || { echo "This function only runs on macOS." >&2; exit 2; } -echo "-- Starting Docker.app, if necessary..." - -open -g -a Docker.app || exit +printf "Starting Docker.app, if necessary" # Wait for the server to start up, if applicable. -i=0 +retries=0 while ! docker system info &>/dev/null; do - (( i++ == 0 )) && printf %s '-- Waiting for Docker to finish starting up...' || printf '.' - sleep 1 + if (( retries % 30 == 0 )); then + if pgrep -xq -- "Docker"; then + printf '\nDocker init still running' + else + (( retries != 0 )) && printf '\nDocker not running, restart' + # /Applications/Docker.app/Contents/MacOS/Docker & + open -g -a Docker.app || exit + fi + + if [[ ${retries} -gt 600 ]]; then + printf '\nFailed to run Docker' + exit 1 + fi + fi + + printf '.' + (( retries++)) + sleep 1 done -(( i )) && printf '\n' +(( retries )) && printf '\n' -echo "-- Docker is ready." +echo "Docker is ready" diff --git a/examples/pyfilesystem_s3fs.ipynb b/examples/pyfilesystem_s3fs.ipynb index 2b685843..0aabeb3f 100644 --- a/examples/pyfilesystem_s3fs.ipynb +++ b/examples/pyfilesystem_s3fs.ipynb @@ -7,7 +7,7 @@ "Launch local S3 via s3proxy in a docker container (will not persist data):\n", "\n", "```\n", - "docker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\n", + "docker run --rm -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\n", "```" ] }, @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -31,6 +31,7 @@ "from fs import open_fs\n", "import os\n", "import sys\n", + "import time\n", "\n", "from jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager" ] @@ -44,26 +45,18 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "scrolled": true }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:23:24 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Marker': '', 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url'}\n", - "{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:23:24 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url', 'KeyCount': 0, 'ContinuationToken': '', 'StartAfter': ''}\n" - ] - }, { "data": { "text/plain": [ "{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9',\n", " 'HostId': '',\n", " 'HTTPStatusCode': 200,\n", - " 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:23:24 GMT',\n", + " 'HTTPHeaders': {'date': 'Mon, 30 Mar 2020 18:22:42 GMT',\n", " 'x-amz-request-id': '4442587FB7D0A2F9',\n", " 'location': '/foo',\n", " 'content-length': '0',\n", @@ -72,7 +65,7 @@ " 'Location': '/foo'}" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -82,8 +75,8 @@ "boto_kwargs = dict(\n", " config=botocore.client.Config(signature_version=botocore.UNSIGNED),\n", " endpoint_url='http://127.0.0.1:9000',\n", - "# aws_access_key_id='',\n", - "# aws_secret_access_key='',\n", + " aws_access_key_id='s3_local',\n", + " aws_secret_access_key='s3_local',\n", "# region_name=self.region,\n", ")\n", "\n", @@ -108,8 +101,8 @@ " print(client.list_objects(Bucket=bucket_name))\n", " print(client.list_objects_v2(Bucket=bucket_name))\n", "\n", - " # delete the bucket\n", - " for key in bucket.objects.all():\n", + " # delete the bucket (faster in reverse order)\n", + " for key in reversed(list(bucket.objects.all())):\n", " key.delete()\n", " bucket.delete()\n", "\n", @@ -125,11 +118,18 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "fooman = PyFilesystemContentsManager.open_fs('s3://foo?endpoint_url=http://127.0.0.1:9000')\n", + "s3uri = 's3://{id}:{key}@{bucket}?endpoint_url={endpoint_url}'.format(\n", + " bucket='foo',\n", + " endpoint_url='http://127.0.0.1:9000',\n", + " id='s3_local',\n", + " key='s3_local'\n", + ")\n", + "\n", + "fooman = PyFilesystemContentsManager.open_fs(s3uri)\n", "# fooman.get('')" ] }, @@ -142,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -167,28 +167,31 @@ "text/plain": [ "{'name': 'pyfilesystem_s3fs.ipynb',\n", " 'path': 'root1/leaf1/pyfilesystem_s3fs.ipynb',\n", - " 'last_modified': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\n", + " 'last_modified': datetime.datetime(2020, 3, 30, 18, 22, 52, tzinfo=),\n", " 'created': None,\n", " 'content': None,\n", " 'format': None,\n", " 'mimetype': None,\n", - " 'size': 11229,\n", + " 'size': 48093,\n", " 'writable': True,\n", " 'type': 'notebook'}" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "fooman.save(osman.get('pyfilesystem_s3fs.ipynb'), 'pyfilesystem_s3fs.ipynb')\n", + "thisContent = osman.get('pyfilesystem_s3fs.ipynb')\n", + "\n", "fooman._save_directory('root0', None)\n", "fooman._save_directory('root1', None)\n", "fooman._save_directory('root1/leaf1', None)\n", - "fooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), 'root0/pyfilesystem_s3fs.ipynb')\n", - "fooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), 'root1/leaf1/pyfilesystem_s3fs.ipynb')" + "\n", + "fooman.save(thisContent, 'pyfilesystem_s3fs.ipynb')\n", + "fooman.save(thisContent, 'root0/pyfilesystem_s3fs.ipynb')\n", + "fooman.save(thisContent, 'root1/leaf1/pyfilesystem_s3fs.ipynb')" ] }, { @@ -200,7 +203,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": { "scrolled": true }, @@ -210,48 +213,45 @@ "text/plain": [ "{'name': 'pyfilesystem_s3fs.ipynb',\n", " 'path': 'pyfilesystem_s3fs.ipynb',\n", - " 'last_modified': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\n", + " 'last_modified': datetime.datetime(2020, 3, 30, 18, 22, 52, tzinfo=),\n", " 'created': None,\n", " 'content': {'cells': [{'cell_type': 'markdown',\n", " 'metadata': {},\n", - " 'source': 'Launch local S3 via s3proxy in a docker container (will not persist data):\\n\\n```\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\n```'},\n", + " 'source': 'Launch local S3 via s3proxy in a docker container (will not persist data):\\n\\n```\\ndocker run --rm -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\n```'},\n", " {'cell_type': 'code',\n", " 'execution_count': 1,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", " 'source': '%load_ext autoreload\\n%autoreload 2'},\n", " {'cell_type': 'code',\n", - " 'execution_count': 2,\n", + " 'execution_count': None,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': 'import boto3, botocore\\nfrom fs import open_fs\\nimport os\\nimport sys\\n\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager'},\n", + " 'source': 'import boto3, botocore\\nfrom fs import open_fs\\nimport os\\nimport sys\\nimport time\\n\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager'},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Create a bucket to use as the filesystem'},\n", " {'cell_type': 'code',\n", - " 'execution_count': 18,\n", + " 'execution_count': 2,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [{'name': 'stdout',\n", - " 'output_type': 'stream',\n", - " 'text': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Marker': '', 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url'}\\n{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url', 'KeyCount': 0, 'ContinuationToken': '', 'StartAfter': ''}\\n\"},\n", - " {'data': {'text/plain': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9',\\n 'HostId': '',\\n 'HTTPStatusCode': 200,\\n 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT',\\n 'x-amz-request-id': '4442587FB7D0A2F9',\\n 'location': '/foo',\\n 'content-length': '0',\\n 'server': 'Jetty(9.2.z-SNAPSHOT)'},\\n 'RetryAttempts': 0},\\n 'Location': '/foo'}\"},\n", - " 'execution_count': 18,\n", + " 'outputs': [{'data': {'text/plain': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9',\\n 'HostId': '',\\n 'HTTPStatusCode': 200,\\n 'HTTPHeaders': {'date': 'Mon, 30 Mar 2020 18:16:00 GMT',\\n 'x-amz-request-id': '4442587FB7D0A2F9',\\n 'location': '/foo',\\n 'content-length': '0',\\n 'server': 'Jetty(9.2.z-SNAPSHOT)'},\\n 'RetryAttempts': 0},\\n 'Location': '/foo'}\"},\n", + " 'execution_count': 2,\n", " 'metadata': {},\n", " 'output_type': 'execute_result'}],\n", - " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n# aws_access_key_id='',\\n# aws_secret_access_key='',\\n# region_name=self.region,\\n)\\n\\nclient = boto3.client('s3', **boto_kwargs)\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # dump info on any existing s3 contents\\n for key in bucket.objects.all():\\n print(key)\\n print(client.list_objects(Bucket=bucket_name))\\n print(client.list_objects_v2(Bucket=bucket_name))\\n\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\nclient.create_bucket(Bucket=bucket_name)\"},\n", + " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n aws_access_key_id='s3_local',\\n aws_secret_access_key='s3_local',\\n# region_name=self.region,\\n)\\n\\nclient = boto3.client('s3', **boto_kwargs)\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # dump info on any existing s3 contents\\n for key in bucket.objects.all():\\n print(key)\\n print(client.list_objects(Bucket=bucket_name))\\n print(client.list_objects_v2(Bucket=bucket_name))\\n\\n # delete the bucket (faster in reverse order)\\n for key in reversed(list(bucket.objects.all())):\\n key.delete()\\n bucket.delete()\\n\\nclient.create_bucket(Bucket=bucket_name)\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Set up a S3 PyFilesystemContentsManager'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 3,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': \"fooman = PyFilesystemContentsManager.open_fs('s3://foo?endpoint_url=http://127.0.0.1:9000')\\n# fooman.get('')\"},\n", + " 'source': \"s3uri = 's3://{id}:{key}@{bucket}?endpoint_url={endpoint_url}'.format(\\n bucket='foo',\\n endpoint_url='http://127.0.0.1:9000',\\n id='s3_local',\\n key='s3_local'\\n)\\n\\nfooman = PyFilesystemContentsManager.open_fs(s3uri)\\n# fooman.get('')\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Set up a local PyFilesystemContentsManager'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 4,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", " 'source': \"osman = PyFilesystemContentsManager.open_fs('osfs://%s' % os.getcwd())\\n# osman.get('')\"},\n", @@ -259,27 +259,39 @@ " 'metadata': {},\n", " 'source': '## Create some dirs on our S3 filesystem and save this notebook into them'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 5,\n", " 'metadata': {'trusted': True},\n", - " 'outputs': [],\n", - " 'source': 'fooman.save(osman.get(\\'pyfilesystem_s3fs.ipynb\\'), \\'pyfilesystem_s3fs.ipynb\\')\\nfooman._save_directory(\\'root0\\', None)\\nfooman._save_directory(\\'root1\\', None)\\nfooman._save_directory(\\'root1/leaf1\\', None)\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\'root0/pyfilesystem_s3fs.ipynb\\')\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\')'},\n", + " 'outputs': [{'data': {'text/plain': \"{'name': 'pyfilesystem_s3fs.ipynb',\\n 'path': 'root1/leaf1/pyfilesystem_s3fs.ipynb',\\n 'last_modified': datetime.datetime(2020, 3, 30, 18, 16, 3, tzinfo=),\\n 'created': None,\\n 'content': None,\\n 'format': None,\\n 'mimetype': None,\\n 'size': 47398,\\n 'writable': True,\\n 'type': 'notebook'}\"},\n", + " 'execution_count': 5,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", + " 'source': \"thisContent = osman.get('pyfilesystem_s3fs.ipynb')\\n\\nfooman._save_directory('root0', None)\\nfooman._save_directory('root1', None)\\nfooman._save_directory('root1/leaf1', None)\\n\\nfooman.save(thisContent, 'pyfilesystem_s3fs.ipynb')\\nfooman.save(thisContent, 'root0/pyfilesystem_s3fs.ipynb')\\nfooman.save(thisContent, 'root1/leaf1/pyfilesystem_s3fs.ipynb')\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Retrieve the saved data from our S3 filesystem'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 7,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 7,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 8,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'root0/pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 8,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'root0/pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 9,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 9,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'root1/leaf1/pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'markdown', 'metadata': {}, 'source': '## scratch'},\n", " {'cell_type': 'markdown',\n", @@ -307,10 +319,22 @@ " 'output_type': 'execute_result'}],\n", " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n)\\n\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\nresource.create_bucket(Bucket=bucket_name)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': 26,\n", + " 'execution_count': 8,\n", + " 'metadata': {'trusted': True},\n", + " 'outputs': [],\n", + " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n aws_access_key_id='s3_local',\\n aws_secret_access_key='s3_local',\\n)\\n\\n\\ndef _s3Resource():\\n return boto3.resource('s3', **boto_kwargs)\\n\\n\\ndef _s3CreateBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if not bucket_exists:\\n # create the bucket\\n s3Resource.create_bucket(Bucket=bucket_name)\\n\\n\\ndef _s3DeleteBucket(bucket_name):\\n now = time.time()\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n \\n print('existence check: %s' % (time.time() - now))\\n now = time.time()\\n \\n if bucket_exists:\\n # delete the bucket (faster in reverse order)\\n for key in reversed(list(bucket.objects.all())):\\n key.delete()\\n print('fetch and delete %s: %s' % (key, time.time() - now))\\n now = time.time()\\n bucket.delete()\\n print('delete empty bucket: %s' % (time.time() - now))\"},\n", + " {'cell_type': 'code',\n", + " 'execution_count': 9,\n", + " 'metadata': {'trusted': True},\n", + " 'outputs': [{'name': 'stdout',\n", + " 'output_type': 'stream',\n", + " 'text': \"existence check: 0.015963077545166016\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/pyfilesystem_s3fs.ipynb'): 0.04035472869873047\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/'): 0.004658937454223633\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/'): 0.004182100296020508\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/pyfilesystem_s3fs.ipynb'): 0.004545927047729492\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/'): 0.004651069641113281\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='pyfilesystem_s3fs.ipynb'): 0.004023075103759766\\ndelete empty bucket: 0.0046961307525634766\\n\"}],\n", + " 'source': '_s3DeleteBucket(bucket_name)'},\n", + " {'cell_type': 'code',\n", + " 'execution_count': None,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': \"test_bucket = 'test'\\ntest_endpoint_url = 'http://127.0.0.1:9000'\\n\\n_boto_kw = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url=test_endpoint_url,\\n)\\n\\n\\ndef _s3Resource():\\n return boto3.resource('s3', **_boto_kw)\\n\\n\\ndef _s3CreateBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if not bucket_exists:\\n # create the bucket\\n s3Resource.create_bucket(Bucket=bucket_name)\\n\\n\\ndef _s3DeleteBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if bucket_exists:\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\n# test idempotency\\n_s3CreateBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\\n_s3CreateBucket(test_bucket)\\n_s3CreateBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\"},\n", + " 'source': '# test idempotency\\n_s3CreateBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)\\n_s3CreateBucket(bucket_name)\\n_s3CreateBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)'},\n", " {'cell_type': 'code',\n", " 'execution_count': None,\n", " 'metadata': {'trusted': True},\n", @@ -330,12 +354,12 @@ " 'nbformat_minor': 4},\n", " 'format': 'json',\n", " 'mimetype': None,\n", - " 'size': 11229,\n", + " 'size': 48093,\n", " 'writable': True,\n", " 'type': 'notebook'}" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -347,7 +371,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": { "scrolled": true }, @@ -357,48 +381,45 @@ "text/plain": [ "{'name': 'pyfilesystem_s3fs.ipynb',\n", " 'path': 'root0/pyfilesystem_s3fs.ipynb',\n", - " 'last_modified': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\n", + " 'last_modified': datetime.datetime(2020, 3, 30, 18, 22, 52, tzinfo=),\n", " 'created': None,\n", " 'content': {'cells': [{'cell_type': 'markdown',\n", " 'metadata': {},\n", - " 'source': 'Launch local S3 via s3proxy in a docker container (will not persist data):\\n\\n```\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\n```'},\n", + " 'source': 'Launch local S3 via s3proxy in a docker container (will not persist data):\\n\\n```\\ndocker run --rm -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\n```'},\n", " {'cell_type': 'code',\n", " 'execution_count': 1,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", " 'source': '%load_ext autoreload\\n%autoreload 2'},\n", " {'cell_type': 'code',\n", - " 'execution_count': 2,\n", + " 'execution_count': None,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': 'import boto3, botocore\\nfrom fs import open_fs\\nimport os\\nimport sys\\n\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager'},\n", + " 'source': 'import boto3, botocore\\nfrom fs import open_fs\\nimport os\\nimport sys\\nimport time\\n\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager'},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Create a bucket to use as the filesystem'},\n", " {'cell_type': 'code',\n", - " 'execution_count': 18,\n", + " 'execution_count': 2,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [{'name': 'stdout',\n", - " 'output_type': 'stream',\n", - " 'text': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Marker': '', 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url'}\\n{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url', 'KeyCount': 0, 'ContinuationToken': '', 'StartAfter': ''}\\n\"},\n", - " {'data': {'text/plain': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9',\\n 'HostId': '',\\n 'HTTPStatusCode': 200,\\n 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT',\\n 'x-amz-request-id': '4442587FB7D0A2F9',\\n 'location': '/foo',\\n 'content-length': '0',\\n 'server': 'Jetty(9.2.z-SNAPSHOT)'},\\n 'RetryAttempts': 0},\\n 'Location': '/foo'}\"},\n", - " 'execution_count': 18,\n", + " 'outputs': [{'data': {'text/plain': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9',\\n 'HostId': '',\\n 'HTTPStatusCode': 200,\\n 'HTTPHeaders': {'date': 'Mon, 30 Mar 2020 18:16:00 GMT',\\n 'x-amz-request-id': '4442587FB7D0A2F9',\\n 'location': '/foo',\\n 'content-length': '0',\\n 'server': 'Jetty(9.2.z-SNAPSHOT)'},\\n 'RetryAttempts': 0},\\n 'Location': '/foo'}\"},\n", + " 'execution_count': 2,\n", " 'metadata': {},\n", " 'output_type': 'execute_result'}],\n", - " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n# aws_access_key_id='',\\n# aws_secret_access_key='',\\n# region_name=self.region,\\n)\\n\\nclient = boto3.client('s3', **boto_kwargs)\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # dump info on any existing s3 contents\\n for key in bucket.objects.all():\\n print(key)\\n print(client.list_objects(Bucket=bucket_name))\\n print(client.list_objects_v2(Bucket=bucket_name))\\n\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\nclient.create_bucket(Bucket=bucket_name)\"},\n", + " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n aws_access_key_id='s3_local',\\n aws_secret_access_key='s3_local',\\n# region_name=self.region,\\n)\\n\\nclient = boto3.client('s3', **boto_kwargs)\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # dump info on any existing s3 contents\\n for key in bucket.objects.all():\\n print(key)\\n print(client.list_objects(Bucket=bucket_name))\\n print(client.list_objects_v2(Bucket=bucket_name))\\n\\n # delete the bucket (faster in reverse order)\\n for key in reversed(list(bucket.objects.all())):\\n key.delete()\\n bucket.delete()\\n\\nclient.create_bucket(Bucket=bucket_name)\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Set up a S3 PyFilesystemContentsManager'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 3,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': \"fooman = PyFilesystemContentsManager.open_fs('s3://foo?endpoint_url=http://127.0.0.1:9000')\\n# fooman.get('')\"},\n", + " 'source': \"s3uri = 's3://{id}:{key}@{bucket}?endpoint_url={endpoint_url}'.format(\\n bucket='foo',\\n endpoint_url='http://127.0.0.1:9000',\\n id='s3_local',\\n key='s3_local'\\n)\\n\\nfooman = PyFilesystemContentsManager.open_fs(s3uri)\\n# fooman.get('')\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Set up a local PyFilesystemContentsManager'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 4,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", " 'source': \"osman = PyFilesystemContentsManager.open_fs('osfs://%s' % os.getcwd())\\n# osman.get('')\"},\n", @@ -406,27 +427,39 @@ " 'metadata': {},\n", " 'source': '## Create some dirs on our S3 filesystem and save this notebook into them'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 5,\n", " 'metadata': {'trusted': True},\n", - " 'outputs': [],\n", - " 'source': 'fooman.save(osman.get(\\'pyfilesystem_s3fs.ipynb\\'), \\'pyfilesystem_s3fs.ipynb\\')\\nfooman._save_directory(\\'root0\\', None)\\nfooman._save_directory(\\'root1\\', None)\\nfooman._save_directory(\\'root1/leaf1\\', None)\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\'root0/pyfilesystem_s3fs.ipynb\\')\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\')'},\n", + " 'outputs': [{'data': {'text/plain': \"{'name': 'pyfilesystem_s3fs.ipynb',\\n 'path': 'root1/leaf1/pyfilesystem_s3fs.ipynb',\\n 'last_modified': datetime.datetime(2020, 3, 30, 18, 16, 3, tzinfo=),\\n 'created': None,\\n 'content': None,\\n 'format': None,\\n 'mimetype': None,\\n 'size': 47398,\\n 'writable': True,\\n 'type': 'notebook'}\"},\n", + " 'execution_count': 5,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", + " 'source': \"thisContent = osman.get('pyfilesystem_s3fs.ipynb')\\n\\nfooman._save_directory('root0', None)\\nfooman._save_directory('root1', None)\\nfooman._save_directory('root1/leaf1', None)\\n\\nfooman.save(thisContent, 'pyfilesystem_s3fs.ipynb')\\nfooman.save(thisContent, 'root0/pyfilesystem_s3fs.ipynb')\\nfooman.save(thisContent, 'root1/leaf1/pyfilesystem_s3fs.ipynb')\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Retrieve the saved data from our S3 filesystem'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 7,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 7,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 8,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'root0/pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 8,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'root0/pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 9,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 9,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'root1/leaf1/pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'markdown', 'metadata': {}, 'source': '## scratch'},\n", " {'cell_type': 'markdown',\n", @@ -454,10 +487,22 @@ " 'output_type': 'execute_result'}],\n", " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n)\\n\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\nresource.create_bucket(Bucket=bucket_name)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': 26,\n", + " 'execution_count': 8,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': \"test_bucket = 'test'\\ntest_endpoint_url = 'http://127.0.0.1:9000'\\n\\n_boto_kw = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url=test_endpoint_url,\\n)\\n\\n\\ndef _s3Resource():\\n return boto3.resource('s3', **_boto_kw)\\n\\n\\ndef _s3CreateBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if not bucket_exists:\\n # create the bucket\\n s3Resource.create_bucket(Bucket=bucket_name)\\n\\n\\ndef _s3DeleteBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if bucket_exists:\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\n# test idempotency\\n_s3CreateBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\\n_s3CreateBucket(test_bucket)\\n_s3CreateBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\"},\n", + " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n aws_access_key_id='s3_local',\\n aws_secret_access_key='s3_local',\\n)\\n\\n\\ndef _s3Resource():\\n return boto3.resource('s3', **boto_kwargs)\\n\\n\\ndef _s3CreateBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if not bucket_exists:\\n # create the bucket\\n s3Resource.create_bucket(Bucket=bucket_name)\\n\\n\\ndef _s3DeleteBucket(bucket_name):\\n now = time.time()\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n \\n print('existence check: %s' % (time.time() - now))\\n now = time.time()\\n \\n if bucket_exists:\\n # delete the bucket (faster in reverse order)\\n for key in reversed(list(bucket.objects.all())):\\n key.delete()\\n print('fetch and delete %s: %s' % (key, time.time() - now))\\n now = time.time()\\n bucket.delete()\\n print('delete empty bucket: %s' % (time.time() - now))\"},\n", + " {'cell_type': 'code',\n", + " 'execution_count': 9,\n", + " 'metadata': {'trusted': True},\n", + " 'outputs': [{'name': 'stdout',\n", + " 'output_type': 'stream',\n", + " 'text': \"existence check: 0.015963077545166016\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/pyfilesystem_s3fs.ipynb'): 0.04035472869873047\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/'): 0.004658937454223633\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/'): 0.004182100296020508\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/pyfilesystem_s3fs.ipynb'): 0.004545927047729492\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/'): 0.004651069641113281\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='pyfilesystem_s3fs.ipynb'): 0.004023075103759766\\ndelete empty bucket: 0.0046961307525634766\\n\"}],\n", + " 'source': '_s3DeleteBucket(bucket_name)'},\n", + " {'cell_type': 'code',\n", + " 'execution_count': None,\n", + " 'metadata': {'trusted': True},\n", + " 'outputs': [],\n", + " 'source': '# test idempotency\\n_s3CreateBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)\\n_s3CreateBucket(bucket_name)\\n_s3CreateBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)'},\n", " {'cell_type': 'code',\n", " 'execution_count': None,\n", " 'metadata': {'trusted': True},\n", @@ -477,12 +522,12 @@ " 'nbformat_minor': 4},\n", " 'format': 'json',\n", " 'mimetype': None,\n", - " 'size': 11229,\n", + " 'size': 48093,\n", " 'writable': True,\n", " 'type': 'notebook'}" ] }, - "execution_count": 8, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -494,7 +539,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "scrolled": true }, @@ -504,48 +549,45 @@ "text/plain": [ "{'name': 'pyfilesystem_s3fs.ipynb',\n", " 'path': 'root1/leaf1/pyfilesystem_s3fs.ipynb',\n", - " 'last_modified': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\n", + " 'last_modified': datetime.datetime(2020, 3, 30, 18, 22, 52, tzinfo=),\n", " 'created': None,\n", " 'content': {'cells': [{'cell_type': 'markdown',\n", " 'metadata': {},\n", - " 'source': 'Launch local S3 via s3proxy in a docker container (will not persist data):\\n\\n```\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\n```'},\n", + " 'source': 'Launch local S3 via s3proxy in a docker container (will not persist data):\\n\\n```\\ndocker run --rm -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\n```'},\n", " {'cell_type': 'code',\n", " 'execution_count': 1,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", " 'source': '%load_ext autoreload\\n%autoreload 2'},\n", " {'cell_type': 'code',\n", - " 'execution_count': 2,\n", + " 'execution_count': None,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': 'import boto3, botocore\\nfrom fs import open_fs\\nimport os\\nimport sys\\n\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager'},\n", + " 'source': 'import boto3, botocore\\nfrom fs import open_fs\\nimport os\\nimport sys\\nimport time\\n\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager'},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Create a bucket to use as the filesystem'},\n", " {'cell_type': 'code',\n", - " 'execution_count': 18,\n", + " 'execution_count': 2,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [{'name': 'stdout',\n", - " 'output_type': 'stream',\n", - " 'text': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Marker': '', 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url'}\\n{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT', 'x-amz-request-id': '4442587FB7D0A2F9', 'content-type': 'application/xml; charset=UTF-8', 'transfer-encoding': 'chunked', 'server': 'Jetty(9.2.z-SNAPSHOT)'}, 'RetryAttempts': 0}, 'IsTruncated': False, 'Name': 'foo', 'Prefix': '', 'MaxKeys': 1000, 'EncodingType': 'url', 'KeyCount': 0, 'ContinuationToken': '', 'StartAfter': ''}\\n\"},\n", - " {'data': {'text/plain': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9',\\n 'HostId': '',\\n 'HTTPStatusCode': 200,\\n 'HTTPHeaders': {'date': 'Fri, 20 Mar 2020 07:08:58 GMT',\\n 'x-amz-request-id': '4442587FB7D0A2F9',\\n 'location': '/foo',\\n 'content-length': '0',\\n 'server': 'Jetty(9.2.z-SNAPSHOT)'},\\n 'RetryAttempts': 0},\\n 'Location': '/foo'}\"},\n", - " 'execution_count': 18,\n", + " 'outputs': [{'data': {'text/plain': \"{'ResponseMetadata': {'RequestId': '4442587FB7D0A2F9',\\n 'HostId': '',\\n 'HTTPStatusCode': 200,\\n 'HTTPHeaders': {'date': 'Mon, 30 Mar 2020 18:16:00 GMT',\\n 'x-amz-request-id': '4442587FB7D0A2F9',\\n 'location': '/foo',\\n 'content-length': '0',\\n 'server': 'Jetty(9.2.z-SNAPSHOT)'},\\n 'RetryAttempts': 0},\\n 'Location': '/foo'}\"},\n", + " 'execution_count': 2,\n", " 'metadata': {},\n", " 'output_type': 'execute_result'}],\n", - " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n# aws_access_key_id='',\\n# aws_secret_access_key='',\\n# region_name=self.region,\\n)\\n\\nclient = boto3.client('s3', **boto_kwargs)\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # dump info on any existing s3 contents\\n for key in bucket.objects.all():\\n print(key)\\n print(client.list_objects(Bucket=bucket_name))\\n print(client.list_objects_v2(Bucket=bucket_name))\\n\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\nclient.create_bucket(Bucket=bucket_name)\"},\n", + " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n aws_access_key_id='s3_local',\\n aws_secret_access_key='s3_local',\\n# region_name=self.region,\\n)\\n\\nclient = boto3.client('s3', **boto_kwargs)\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # dump info on any existing s3 contents\\n for key in bucket.objects.all():\\n print(key)\\n print(client.list_objects(Bucket=bucket_name))\\n print(client.list_objects_v2(Bucket=bucket_name))\\n\\n # delete the bucket (faster in reverse order)\\n for key in reversed(list(bucket.objects.all())):\\n key.delete()\\n bucket.delete()\\n\\nclient.create_bucket(Bucket=bucket_name)\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Set up a S3 PyFilesystemContentsManager'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 3,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': \"fooman = PyFilesystemContentsManager.open_fs('s3://foo?endpoint_url=http://127.0.0.1:9000')\\n# fooman.get('')\"},\n", + " 'source': \"s3uri = 's3://{id}:{key}@{bucket}?endpoint_url={endpoint_url}'.format(\\n bucket='foo',\\n endpoint_url='http://127.0.0.1:9000',\\n id='s3_local',\\n key='s3_local'\\n)\\n\\nfooman = PyFilesystemContentsManager.open_fs(s3uri)\\n# fooman.get('')\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Set up a local PyFilesystemContentsManager'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 4,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", " 'source': \"osman = PyFilesystemContentsManager.open_fs('osfs://%s' % os.getcwd())\\n# osman.get('')\"},\n", @@ -553,27 +595,39 @@ " 'metadata': {},\n", " 'source': '## Create some dirs on our S3 filesystem and save this notebook into them'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 5,\n", " 'metadata': {'trusted': True},\n", - " 'outputs': [],\n", - " 'source': 'fooman.save(osman.get(\\'pyfilesystem_s3fs.ipynb\\'), \\'pyfilesystem_s3fs.ipynb\\')\\nfooman._save_directory(\\'root0\\', None)\\nfooman._save_directory(\\'root1\\', None)\\nfooman._save_directory(\\'root1/leaf1\\', None)\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\'root0/pyfilesystem_s3fs.ipynb\\')\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\')'},\n", + " 'outputs': [{'data': {'text/plain': \"{'name': 'pyfilesystem_s3fs.ipynb',\\n 'path': 'root1/leaf1/pyfilesystem_s3fs.ipynb',\\n 'last_modified': datetime.datetime(2020, 3, 30, 18, 16, 3, tzinfo=),\\n 'created': None,\\n 'content': None,\\n 'format': None,\\n 'mimetype': None,\\n 'size': 47398,\\n 'writable': True,\\n 'type': 'notebook'}\"},\n", + " 'execution_count': 5,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", + " 'source': \"thisContent = osman.get('pyfilesystem_s3fs.ipynb')\\n\\nfooman._save_directory('root0', None)\\nfooman._save_directory('root1', None)\\nfooman._save_directory('root1/leaf1', None)\\n\\nfooman.save(thisContent, 'pyfilesystem_s3fs.ipynb')\\nfooman.save(thisContent, 'root0/pyfilesystem_s3fs.ipynb')\\nfooman.save(thisContent, 'root1/leaf1/pyfilesystem_s3fs.ipynb')\"},\n", " {'cell_type': 'markdown',\n", " 'metadata': {},\n", " 'source': '## Retrieve the saved data from our S3 filesystem'},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 7,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 7,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 8,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'root0/pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 8,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'root0/pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': None,\n", + " 'execution_count': 9,\n", " 'metadata': {'scrolled': True, 'trusted': True},\n", - " 'outputs': [],\n", + " 'outputs': [{'data': {'text/plain': '{\\'name\\': \\'pyfilesystem_s3fs.ipynb\\',\\n \\'path\\': \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\',\\n \\'last_modified\\': datetime.datetime(2020, 3, 20, 7, 23, 33, tzinfo=),\\n \\'created\\': None,\\n \\'content\\': {\\'cells\\': [{\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'Launch local S3 via s3proxy in a docker container (will not persist data):\\\\n\\\\n```\\\\ndocker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy\\\\n```\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 1,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'%load_ext autoreload\\\\n%autoreload 2\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 2,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'import boto3, botocore\\\\nfrom fs import open_fs\\\\nimport os\\\\nimport sys\\\\n\\\\nfrom jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create a bucket to use as the filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 18,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'name\\': \\'stdout\\',\\n \\'output_type\\': \\'stream\\',\\n \\'text\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Marker\\': \\'\\', \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\'}\\\\n{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\', \\'HostId\\': \\'\\', \\'HTTPStatusCode\\': 200, \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\', \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\', \\'content-type\\': \\'application/xml; charset=UTF-8\\', \\'transfer-encoding\\': \\'chunked\\', \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'}, \\'RetryAttempts\\': 0}, \\'IsTruncated\\': False, \\'Name\\': \\'foo\\', \\'Prefix\\': \\'\\', \\'MaxKeys\\': 1000, \\'EncodingType\\': \\'url\\', \\'KeyCount\\': 0, \\'ContinuationToken\\': \\'\\', \\'StartAfter\\': \\'\\'}\\\\n\"},\\n {\\'data\\': {\\'text/plain\\': \"{\\'ResponseMetadata\\': {\\'RequestId\\': \\'4442587FB7D0A2F9\\',\\\\n \\'HostId\\': \\'\\',\\\\n \\'HTTPStatusCode\\': 200,\\\\n \\'HTTPHeaders\\': {\\'date\\': \\'Fri, 20 Mar 2020 07:08:58 GMT\\',\\\\n \\'x-amz-request-id\\': \\'4442587FB7D0A2F9\\',\\\\n \\'location\\': \\'/foo\\',\\\\n \\'content-length\\': \\'0\\',\\\\n \\'server\\': \\'Jetty(9.2.z-SNAPSHOT)\\'},\\\\n \\'RetryAttempts\\': 0},\\\\n \\'Location\\': \\'/foo\\'}\"},\\n \\'execution_count\\': 18,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n# aws_access_key_id=\\'\\',\\\\n# aws_secret_access_key=\\'\\',\\\\n# region_name=self.region,\\\\n)\\\\n\\\\nclient = boto3.client(\\'s3\\', **boto_kwargs)\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # dump info on any existing s3 contents\\\\n for key in bucket.objects.all():\\\\n print(key)\\\\n print(client.list_objects(Bucket=bucket_name))\\\\n print(client.list_objects_v2(Bucket=bucket_name))\\\\n\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nclient.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a S3 PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fooman = PyFilesystemContentsManager.open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\n# fooman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Set up a local PyFilesystemContentsManager\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"osman = PyFilesystemContentsManager.open_fs(\\'osfs://%s\\' % os.getcwd())\\\\n# osman.get(\\'\\')\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Create some dirs on our S3 filesystem and save this notebook into them\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'fooman.save(osman.get(\\\\\\'pyfilesystem_s3fs.ipynb\\\\\\'), \\\\\\'pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman._save_directory(\\\\\\'root0\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1\\\\\\', None)\\\\nfooman._save_directory(\\\\\\'root1/leaf1\\\\\\', None)\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root0/pyfilesystem_s3fs.ipynb\\\\\\')\\\\nfooman.save(osman.get(\"pyfilesystem_s3fs.ipynb\"), \\\\\\'root1/leaf1/pyfilesystem_s3fs.ipynb\\\\\\')\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Retrieve the saved data from our S3 filesystem\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root0/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"fpath = \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'\\\\nfooman.get(fpath)\"},\\n {\\'cell_type\\': \\'markdown\\', \\'metadata\\': {}, \\'source\\': \\'## scratch\\'},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'### examine underlying fs-s3fs\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"foofs = open_fs(\\'s3://foo?endpoint_url=http://127.0.0.1:9000\\')\\\\nfoofs.getinfo(\\'\\').raw\\\\nfoofs.getinfo(\\'root1/\\')\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"for fpath in (\\'pyfilesystem_s3fs.ipynb\\', \\'root0/pyfilesystem_s3fs.ipynb\\', \\'root1/leaf1/pyfilesystem_s3fs.ipynb\\'):\\\\n print(foofs.getinfo(fpath).raw)\"},\\n {\\'cell_type\\': \\'markdown\\',\\n \\'metadata\\': {},\\n \\'source\\': \\'## Use only boto3 resource api (not client api)\\'},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 24,\\n \\'metadata\\': {\\'scrolled\\': True, \\'trusted\\': True},\\n \\'outputs\\': [{\\'data\\': {\\'text/plain\\': \"s3.Bucket(name=\\'foo\\')\"},\\n \\'execution_count\\': 24,\\n \\'metadata\\': {},\\n \\'output_type\\': \\'execute_result\\'}],\\n \\'source\\': \"bucket_name = \\'foo\\'\\\\nboto_kwargs = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=\\'http://127.0.0.1:9000\\',\\\\n)\\\\n\\\\nresource = boto3.resource(\\'s3\\', **boto_kwargs)\\\\n\\\\n# check if bucket already exists\\\\nbucket = resource.Bucket(bucket_name)\\\\nbucket_exists = True\\\\ntry:\\\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\\\nexcept botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\nif bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\nresource.create_bucket(Bucket=bucket_name)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': 26,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \"test_bucket = \\'test\\'\\\\ntest_endpoint_url = \\'http://127.0.0.1:9000\\'\\\\n\\\\n_boto_kw = dict(\\\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\\\n endpoint_url=test_endpoint_url,\\\\n)\\\\n\\\\n\\\\ndef _s3Resource():\\\\n return boto3.resource(\\'s3\\', **_boto_kw)\\\\n\\\\n\\\\ndef _s3CreateBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if not bucket_exists:\\\\n # create the bucket\\\\n s3Resource.create_bucket(Bucket=bucket_name)\\\\n\\\\n\\\\ndef _s3DeleteBucket(bucket_name):\\\\n s3Resource = _s3Resource()\\\\n\\\\n # check if bucket already exists\\\\n bucket = s3Resource.Bucket(bucket_name)\\\\n bucket_exists = True\\\\n try:\\\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\\\n except botocore.exceptions.ClientError as e:\\\\n # If it was a 404 error, then the bucket does not exist.\\\\n error_code = e.response[\\'Error\\'][\\'Code\\']\\\\n if error_code == \\'404\\':\\\\n bucket_exists = False\\\\n\\\\n if bucket_exists:\\\\n # delete the bucket\\\\n for key in bucket.objects.all():\\\\n key.delete()\\\\n bucket.delete()\\\\n\\\\n# test idempotency\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3CreateBucket(test_bucket)\\\\n_s3DeleteBucket(test_bucket)\"},\\n {\\'cell_type\\': \\'code\\',\\n \\'execution_count\\': None,\\n \\'metadata\\': {\\'trusted\\': True},\\n \\'outputs\\': [],\\n \\'source\\': \\'\\'}],\\n \\'metadata\\': {\\'kernelspec\\': {\\'display_name\\': \\'Python 3\\',\\n \\'language\\': \\'python\\',\\n \\'name\\': \\'python3\\'},\\n \\'language_info\\': {\\'codemirror_mode\\': {\\'name\\': \\'ipython\\', \\'version\\': 3},\\n \\'file_extension\\': \\'.py\\',\\n \\'mimetype\\': \\'text/x-python\\',\\n \\'name\\': \\'python\\',\\n \\'nbconvert_exporter\\': \\'python\\',\\n \\'pygments_lexer\\': \\'ipython3\\',\\n \\'version\\': \\'3.7.6\\'}},\\n \\'nbformat\\': 4,\\n \\'nbformat_minor\\': 4},\\n \\'format\\': \\'json\\',\\n \\'mimetype\\': None,\\n \\'size\\': 11229,\\n \\'writable\\': True,\\n \\'type\\': \\'notebook\\'}'},\n", + " 'execution_count': 9,\n", + " 'metadata': {},\n", + " 'output_type': 'execute_result'}],\n", " 'source': \"fpath = 'root1/leaf1/pyfilesystem_s3fs.ipynb'\\nfooman.get(fpath)\"},\n", " {'cell_type': 'markdown', 'metadata': {}, 'source': '## scratch'},\n", " {'cell_type': 'markdown',\n", @@ -601,10 +655,22 @@ " 'output_type': 'execute_result'}],\n", " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n)\\n\\nresource = boto3.resource('s3', **boto_kwargs)\\n\\n# check if bucket already exists\\nbucket = resource.Bucket(bucket_name)\\nbucket_exists = True\\ntry:\\n resource.meta.client.head_bucket(Bucket=bucket_name)\\nexcept botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\nif bucket_exists:\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\nresource.create_bucket(Bucket=bucket_name)\"},\n", " {'cell_type': 'code',\n", - " 'execution_count': 26,\n", + " 'execution_count': 8,\n", " 'metadata': {'trusted': True},\n", " 'outputs': [],\n", - " 'source': \"test_bucket = 'test'\\ntest_endpoint_url = 'http://127.0.0.1:9000'\\n\\n_boto_kw = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url=test_endpoint_url,\\n)\\n\\n\\ndef _s3Resource():\\n return boto3.resource('s3', **_boto_kw)\\n\\n\\ndef _s3CreateBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if not bucket_exists:\\n # create the bucket\\n s3Resource.create_bucket(Bucket=bucket_name)\\n\\n\\ndef _s3DeleteBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if bucket_exists:\\n # delete the bucket\\n for key in bucket.objects.all():\\n key.delete()\\n bucket.delete()\\n\\n# test idempotency\\n_s3CreateBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\\n_s3CreateBucket(test_bucket)\\n_s3CreateBucket(test_bucket)\\n_s3DeleteBucket(test_bucket)\"},\n", + " 'source': \"bucket_name = 'foo'\\nboto_kwargs = dict(\\n config=botocore.client.Config(signature_version=botocore.UNSIGNED),\\n endpoint_url='http://127.0.0.1:9000',\\n aws_access_key_id='s3_local',\\n aws_secret_access_key='s3_local',\\n)\\n\\n\\ndef _s3Resource():\\n return boto3.resource('s3', **boto_kwargs)\\n\\n\\ndef _s3CreateBucket(bucket_name):\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n\\n if not bucket_exists:\\n # create the bucket\\n s3Resource.create_bucket(Bucket=bucket_name)\\n\\n\\ndef _s3DeleteBucket(bucket_name):\\n now = time.time()\\n s3Resource = _s3Resource()\\n\\n # check if bucket already exists\\n bucket = s3Resource.Bucket(bucket_name)\\n bucket_exists = True\\n try:\\n s3Resource.meta.client.head_bucket(Bucket=bucket_name)\\n except botocore.exceptions.ClientError as e:\\n # If it was a 404 error, then the bucket does not exist.\\n error_code = e.response['Error']['Code']\\n if error_code == '404':\\n bucket_exists = False\\n \\n print('existence check: %s' % (time.time() - now))\\n now = time.time()\\n \\n if bucket_exists:\\n # delete the bucket (faster in reverse order)\\n for key in reversed(list(bucket.objects.all())):\\n key.delete()\\n print('fetch and delete %s: %s' % (key, time.time() - now))\\n now = time.time()\\n bucket.delete()\\n print('delete empty bucket: %s' % (time.time() - now))\"},\n", + " {'cell_type': 'code',\n", + " 'execution_count': 9,\n", + " 'metadata': {'trusted': True},\n", + " 'outputs': [{'name': 'stdout',\n", + " 'output_type': 'stream',\n", + " 'text': \"existence check: 0.015963077545166016\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/pyfilesystem_s3fs.ipynb'): 0.04035472869873047\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/'): 0.004658937454223633\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/'): 0.004182100296020508\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/pyfilesystem_s3fs.ipynb'): 0.004545927047729492\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/'): 0.004651069641113281\\nfetch and delete s3.ObjectSummary(bucket_name='foo', key='pyfilesystem_s3fs.ipynb'): 0.004023075103759766\\ndelete empty bucket: 0.0046961307525634766\\n\"}],\n", + " 'source': '_s3DeleteBucket(bucket_name)'},\n", + " {'cell_type': 'code',\n", + " 'execution_count': None,\n", + " 'metadata': {'trusted': True},\n", + " 'outputs': [],\n", + " 'source': '# test idempotency\\n_s3CreateBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)\\n_s3CreateBucket(bucket_name)\\n_s3CreateBucket(bucket_name)\\n_s3DeleteBucket(bucket_name)'},\n", " {'cell_type': 'code',\n", " 'execution_count': None,\n", " 'metadata': {'trusted': True},\n", @@ -624,12 +690,12 @@ " 'nbformat_minor': 4},\n", " 'format': 'json',\n", " 'mimetype': None,\n", - " 'size': 11229,\n", + " 'size': 48093,\n", " 'writable': True,\n", " 'type': 'notebook'}" ] }, - "execution_count": 9, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -639,11 +705,25 @@ "fooman.get(fpath)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## scratch" + "# scratch" ] }, { @@ -730,21 +810,21 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "test_bucket = 'test'\n", - "test_endpoint_url = 'http://127.0.0.1:9000'\n", - "\n", - "_boto_kw = dict(\n", + "bucket_name = 'foo'\n", + "boto_kwargs = dict(\n", " config=botocore.client.Config(signature_version=botocore.UNSIGNED),\n", - " endpoint_url=test_endpoint_url,\n", + " endpoint_url='http://127.0.0.1:9000',\n", + " aws_access_key_id='s3_local',\n", + " aws_secret_access_key='s3_local',\n", ")\n", "\n", "\n", "def _s3Resource():\n", - " return boto3.resource('s3', **_boto_kw)\n", + " return boto3.resource('s3', **boto_kwargs)\n", "\n", "\n", "def _s3CreateBucket(bucket_name):\n", @@ -767,6 +847,7 @@ "\n", "\n", "def _s3DeleteBucket(bucket_name):\n", + " now = time.time()\n", " s3Resource = _s3Resource()\n", "\n", " # check if bucket already exists\n", @@ -779,20 +860,57 @@ " error_code = e.response['Error']['Code']\n", " if error_code == '404':\n", " bucket_exists = False\n", - "\n", + " \n", + " print('existence check: %s' % (time.time() - now))\n", + " now = time.time()\n", + " \n", " if bucket_exists:\n", - " # delete the bucket\n", - " for key in bucket.objects.all():\n", + " # delete the bucket (faster in reverse order)\n", + " for key in reversed(list(bucket.objects.all())):\n", " key.delete()\n", + " print('fetch and delete %s: %s' % (key, time.time() - now))\n", + " now = time.time()\n", " bucket.delete()\n", - "\n", + " print('delete empty bucket: %s' % (time.time() - now))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "existence check: 0.015963077545166016\n", + "fetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/pyfilesystem_s3fs.ipynb'): 0.04035472869873047\n", + "fetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/leaf1/'): 0.004658937454223633\n", + "fetch and delete s3.ObjectSummary(bucket_name='foo', key='root1/'): 0.004182100296020508\n", + "fetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/pyfilesystem_s3fs.ipynb'): 0.004545927047729492\n", + "fetch and delete s3.ObjectSummary(bucket_name='foo', key='root0/'): 0.004651069641113281\n", + "fetch and delete s3.ObjectSummary(bucket_name='foo', key='pyfilesystem_s3fs.ipynb'): 0.004023075103759766\n", + "delete empty bucket: 0.0046961307525634766\n" + ] + } + ], + "source": [ + "_s3DeleteBucket(bucket_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "# test idempotency\n", - "_s3CreateBucket(test_bucket)\n", - "_s3DeleteBucket(test_bucket)\n", - "_s3DeleteBucket(test_bucket)\n", - "_s3CreateBucket(test_bucket)\n", - "_s3CreateBucket(test_bucket)\n", - "_s3DeleteBucket(test_bucket)" + "_s3CreateBucket(bucket_name)\n", + "_s3DeleteBucket(bucket_name)\n", + "_s3DeleteBucket(bucket_name)\n", + "_s3CreateBucket(bucket_name)\n", + "_s3CreateBucket(bucket_name)\n", + "_s3DeleteBucket(bucket_name)" ] }, { diff --git a/jupyterfs/tests/conftest.py b/jupyterfs/tests/conftest.py new file mode 100644 index 00000000..58833797 --- /dev/null +++ b/jupyterfs/tests/conftest.py @@ -0,0 +1,20 @@ +import pytest +import sys + +# ref: http://doc.pytest.org/en/latest/example/markers.html#marking-platform-specific-tests-with-pytest + +PLATFORM_INFO = {'darwin': 'mac', 'linux': 'linux', 'win32': 'windows'} +PLATFORMS = set(PLATFORM_INFO.keys()) + +def pytest_configure(config): + # register the platform markers + for info in PLATFORM_INFO.items(): + config.addinivalue_line( + "markers", "{}: mark test to run only on platform == {}".format(*info) + ) + +def pytest_runtest_setup(item): + platforms_for_test = PLATFORMS.intersection(mark.name for mark in item.iter_markers()) + + if platforms_for_test and sys.platform not in platforms_for_test: + pytest.skip('cannot run on platform %s' % sys.platform) diff --git a/jupyterfs/tests/test_pyfilesystem_manager.py b/jupyterfs/tests/test_pyfilesystem_manager.py index c4d85917..2702cb10 100644 --- a/jupyterfs/tests/test_pyfilesystem_manager.py +++ b/jupyterfs/tests/test_pyfilesystem_manager.py @@ -4,21 +4,33 @@ # # This file is part of the jupyter-fs library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. -# -import boto3 -import botocore -from jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager +from pathlib import Path +import pytest +import os +import shutil +import socket +import sys -# these can be anything, but they have to exist -test_aws_access_key_id = 's3_local' -test_aws_secret_access_key = 's3_local' +from jupyterfs.pyfilesystem_manager import PyFilesystemContentsManager +from .utils import s3, samba -test_bucket = 'test' +test_dir = 'test' test_content = 'foo\nbar\nbaz' -test_endpoint_url = 'http://127.0.0.1:9000' test_fname = 'foo.txt' +test_root_osfs = 'osfs_local' + +test_url_s3 = 'http://127.0.0.1/' +test_port_s3 = '9000' + +test_hostname_smb_docker_share = 'TESTNET' +test_name_port_smb_docker_share = 3669 + +test_direct_tcp_smb_os_share = False +test_host_smb_os_share = socket.gethostbyname_ex(socket.gethostname())[2][-1] +test_smb_port_smb_os_share = 139 + _test_file_model = { 'content': test_content, 'format': 'text', @@ -29,88 +41,198 @@ 'writable': True, } -_boto_kw = dict( - config=botocore.client.Config(signature_version=botocore.UNSIGNED), - endpoint_url=test_endpoint_url, -) - - -def _s3BucketExists(bucket_name): - # check if bucket already exists - bucket_exists = True - try: - _s3Resource().meta.client.head_bucket(Bucket=bucket_name) - except botocore.exceptions.ClientError as e: - # If it was a 404 error, then the bucket does not exist. - error_code = e.response['Error']['Code'] - if error_code == '404': - bucket_exists = False - - return bucket_exists - -def _s3ContentsManager(): - s3Uri = 's3://{aws_access_key_id}:{aws_secret_access_key}@{bucket}?endpoint_url={endpoint_url}'.format( - aws_access_key_id=test_aws_access_key_id, - aws_secret_access_key=test_aws_secret_access_key, - bucket=test_bucket, - endpoint_url=test_endpoint_url - ) - return PyFilesystemContentsManager.open_fs(s3Uri) +class _TestBase: + """Contains tests universal to all PyFilesystemContentsManager flavors + """ -def _s3CreateBucket(bucket_name): - if not _s3BucketExists(bucket_name): - # create the bucket - _s3Resource().create_bucket(Bucket=bucket_name) + def _createContentsManager(self): + raise NotImplementedError -def _s3DeleteBucket(bucket_name): - if _s3BucketExists(bucket_name): - bucket = _s3Resource().Bucket(bucket_name) + def testWriteRead(self): + cm = self._createContentsManager() - # delete the bucket - for key in bucket.objects.all(): - key.delete() - bucket.delete() + fpaths = [ + '' + test_fname, + 'root0/' + test_fname, + 'root1/leaf1/' + test_fname, + ] -def _s3Resource(): - return boto3.resource('s3', **_boto_kw) + # set up dir structure + cm._save_directory('root0', None) + cm._save_directory('root1', None) + cm._save_directory('root1/leaf1', None) + + # save to root and tips + cm.save(_test_file_model, fpaths[0]) + cm.save(_test_file_model, fpaths[1]) + cm.save(_test_file_model, fpaths[2]) + # read and check + assert test_content == cm.get(fpaths[0])['content'] + assert test_content == cm.get(fpaths[1])['content'] + assert test_content == cm.get(fpaths[2])['content'] + + +class TestPyFilesystemContentsManager_osfs(_TestBase): + """No extra setup required for this test suite + """ + _test_dir = str(Path(test_root_osfs) / Path(test_dir)) -class TestPyFilesystemContentsManagerS3: @classmethod def setup_class(cls): - _s3DeleteBucket(test_bucket) + shutil.rmtree(test_root_osfs, ignore_errors=True) + os.makedirs(test_root_osfs) def setup_method(self, method): - _s3CreateBucket(test_bucket) + os.makedirs(self._test_dir) def teardown_method(self, method): - _s3DeleteBucket(test_bucket) + shutil.rmtree(self._test_dir, ignore_errors=True) - def test_write_s3_read_s3(self): - s3CM = _s3ContentsManager() + def _createContentsManager(self): + uri = 'osfs://{local_dir}'.format(local_dir=self._test_dir) - fpaths = [ - '' + test_fname, - 'root0/' + test_fname, - 'root1/leaf1/' + test_fname - ] + return PyFilesystemContentsManager.open_fs(uri) - # set up dir structure - s3CM._save_directory('root0', None) - s3CM._save_directory('root1', None) - s3CM._save_directory('root1/leaf1', None) - # save to root and tips - s3CM.save(_test_file_model, fpaths[0]) - s3CM.save(_test_file_model, fpaths[1]) - s3CM.save(_test_file_model, fpaths[2]) +class TestPyFilesystemContentsManager_s3(_TestBase): + """Tests on an instance of s3proxy running in a docker + Manual startup of equivalent docker: - # read and check - assert test_content == s3CM.get(fpaths[0])['content'] - assert test_content == s3CM.get(fpaths[1])['content'] - assert test_content == s3CM.get(fpaths[2])['content'] + docker run --rm -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy + """ + _rootDirUtil = s3.RootDirUtil(dir_name=test_dir, port=test_port_s3, url=test_url_s3) + + @classmethod + def setup_class(cls): + if sys.platform != 'win32': + # start up the server + cls._rootDirUtil.start() + + # delete any existing root + cls._rootDirUtil.delete() + + @classmethod + def teardown_class(cls): + if sys.platform != 'win32': + # stop the server + cls._rootDirUtil.stop() - # @classmethod - # def teardown_class(cls): - # pass + def setup_method(self, method): + self._rootDirUtil.create() + + def teardown_method(self, method): + self._rootDirUtil.delete() + + def _createContentsManager(self): + uri = 's3://{id}:{key}@{bucket}?endpoint_url={url}:{port}'.format( + id=s3.aws_access_key_id, + key=s3.aws_secret_access_key, + bucket=test_dir, + url=test_url_s3.strip('/'), + port=test_port_s3, + ) + + return PyFilesystemContentsManager.open_fs(uri) + + +@pytest.mark.darwin +@pytest.mark.linux +class TestPyFilesystemContentsManager_smb_docker_share(_TestBase): + """(mac/linux only. future: windows) runs its own samba server via + py-docker. Automatically creates and exposes a share from a docker + container. + + Manual startup of equivalent docker: + + docker run --rm -it -p 137:137/udp -p 138:138/udp -p 139:139 -p 445:445 dperson/samba -p -n -u "smbuser;smbuser" -w "TESTNET" + + Docker with a windows guest: + + docker run --rm -it -p 137:137/udp -p 138:138/udp -p 139:139 -p 445:445 mcr.microsoft.com/windows/nanoserver:1809 + """ + _rootDirUtil = samba.RootDirUtil( + dir_name=test_dir, + hostname=test_hostname_smb_docker_share, + name_port=test_name_port_smb_docker_share, + ) + + @classmethod + def setup_class(cls): + # start up the server + cls._rootDirUtil.start() + + # delete any existing root + cls._rootDirUtil.delete() + + @classmethod + def teardown_class(cls): + # stop the server + cls._rootDirUtil.stop() + + def setup_method(self, method): + # create a root + self._rootDirUtil.create() + + def teardown_method(self, method): + # delete any existing root + self._rootDirUtil.delete() + + def _createContentsManager(self): + uri = 'smb://{username}:{passwd}@{host}/{share}?name-port={name_port}'.format( + username=samba.smb_user, + passwd=samba.smb_passwd, + host=socket.gethostbyname(socket.gethostname()), + name_port=test_name_port_smb_docker_share, + share=test_dir, + ) + + cm = PyFilesystemContentsManager.open_fs(uri) + assert cm.dir_exists('.') + return cm + + +@pytest.mark.win32 +class TestPyFilesystemContentsManager_smb_os_share(_TestBase): + """(windows only. future: also mac) Uses the os's buitlin samba server. + Expects a local user "smbuser" with access to a share named "test" + """ + _rootDirUtil = samba.RootDirUtil( + dir_name=test_dir, + host=test_host_smb_os_share, + smb_port=test_smb_port_smb_os_share + ) + + @classmethod + def setup_class(cls): + # delete any existing root + cls._rootDirUtil.delete() + + def setup_method(self, method): + # create a root + self._rootDirUtil.create() + + def teardown_method(self, method): + # delete any existing root + self._rootDirUtil.delete() + + def _createContentsManager(self): + kwargs = dict( + direct_tcp=test_direct_tcp_smb_os_share, + host=test_host_smb_os_share, + hostname=socket.getfqdn(), + passwd=samba.smb_passwd, + share=test_dir, + username=samba.smb_user, + ) + + if test_smb_port_smb_os_share is not None: + uri = 'smb://{username}:{passwd}@{host}:{port}/{share}?hostname={hostname}&direct-tcp={direct_tcp}'.format(port=test_smb_port_smb_os_share, **kwargs) + else: + uri = 'smb://{username}:{passwd}@{host}/{share}?hostname={hostname}&direct-tcp={direct_tcp}'.format(**kwargs) + + cm = PyFilesystemContentsManager.open_fs(uri) + + assert cm.dir_exists('.') + return cm diff --git a/jupyterfs/tests/utils/__init__.py b/jupyterfs/tests/utils/__init__.py new file mode 100644 index 00000000..ac7f1aa8 --- /dev/null +++ b/jupyterfs/tests/utils/__init__.py @@ -0,0 +1,7 @@ +# ***************************************************************************** +# +# Copyright (c) 2019, the jupyter-fs authors. +# +# This file is part of the jupyter-fs library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# diff --git a/jupyterfs/tests/utils/s3.py b/jupyterfs/tests/utils/s3.py new file mode 100644 index 00000000..7d69d3a4 --- /dev/null +++ b/jupyterfs/tests/utils/s3.py @@ -0,0 +1,127 @@ +# ***************************************************************************** +# +# Copyright (c) 2019, the jupyter-fs authors. +# +# This file is part of the jupyter-fs library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. + +import atexit +import boto3 +import botocore +import docker +import time + +__all__ = ['aws_access_key_id', 'aws_secret_access_key', 'RootDirUtil'] + +aws_access_key_id = 's3_local' +aws_secret_access_key = 's3_local' + +def startServer(s3_port=9000): + ports = {'80/tcp': s3_port} + + # init docker + docker_client = docker.from_env(version='auto') + docker_client.info() + + # run a docker container running s3proxy + container = docker_client.containers.run( + 'andrewgaul/s3proxy', + detach=True, + environment={'S3PROXY_AUTHORIZATION': 'none'}, + ports=ports, + remove=True, + tty=True, + # network_mode='host', + ) + + def exitHandler(): + try: + # will raise docker.errors.NotFound if container does not currently exist + docker_client.containers.get(container.id) + + container.kill() + # container.remove() + except docker.errors.NotFound: + pass + + atexit.register(exitHandler) + + # wait for samba to start up + timeout = 0 + while True: + tail = container.logs(tail=1) + if b"main o.g.s.o.e.jetty.server.Server" in tail and b"|::] Started @" in tail: + break + + if timeout >= 100: + raise RuntimeError('docker andrewgaul/s3proxy timed out while starting up') + + timeout += 1 + time.sleep(1) + + return container, exitHandler + +class RootDirUtil: + def __init__( + self, + dir_name, + url, + port, + ): + self.dir_name = dir_name + self.url = url.rstrip('/') + self.port = port + + self._container = None + self._container_exit_handler = None + self._endpoint_url = '{}:{}'.format(self.url, self.port) + + def exists(self): + # check if bucket already exists + bucket_exists = True + try: + self.resource().meta.client.head_bucket(Bucket=self.dir_name) + except botocore.exceptions.ClientError as e: + # If it was a 404 error, then the bucket does not exist. + error_code = e.response['Error']['Code'] + if error_code == '404': + bucket_exists = False + + return bucket_exists + + def create(self): + if not self.exists(): + # create the bucket + self.resource().create_bucket(Bucket=self.dir_name) + + def delete(self): + if self.exists(): + bucket = self.resource().Bucket(self.dir_name) + + # walk the bucket from leaves to roots and delete as you go. + # This avoids deleting non-empty s3 folders (extremely slow) + for key in reversed(list(bucket.objects.all())): + key.delete() + + # delete the bucket + bucket.delete() + + def resource(self): + boto_kw = dict( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + # config=botocore.client.Config(signature_version=botocore.UNSIGNED), + endpoint_url=self._endpoint_url, + ) + + return boto3.resource('s3', **boto_kw) + + def start(self): + self._container, self._container_exit_handler = startServer(s3_port=self.port) + + def stop(self): + if self._container is not None: + self._container_exit_handler() + + self._container = None + self._container_exit_handler = None diff --git a/jupyterfs/tests/utils/samba.py b/jupyterfs/tests/utils/samba.py new file mode 100755 index 00000000..8272124f --- /dev/null +++ b/jupyterfs/tests/utils/samba.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# ***************************************************************************** +# +# Copyright (c) 2019, the jupyter-fs authors. +# +# This file is part of the jupyter-fs library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. + +import atexit +import docker +import os +import shutil +import signal +from smb.SMBConnection import SMBConnection +import socket +import sys +import time + +__all__ = ['smb_user', 'smb_passwd', 'startServer', 'RootDirUtil'] + +smb_user = 'smbuser' +smb_passwd = 'smbuser' + +_dir = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) + + +def startServer(name_port=137): + ports = dict(( + ('137/udp', name_port), + ('138/udp', 138), + ('139/tcp', 139), + ('445/tcp', 445), + )) + + # init docker + docker_client = docker.from_env(version='auto') + docker_client.info() + + # set up smb.conf + shutil.copy(os.path.join(_dir, 'smb.conf.template'), os.path.join(_dir, 'smb.conf')) + + # run the docker container + smb_container = docker_client.containers.run( + 'dperson/samba', 'samba.sh -n -p -u "{user};{passwd}"'.format(user=smb_user, passwd=smb_passwd), + detach=True, + ports=ports, + remove=True, + tmpfs={'/shared': 'size=3G,uid=1000'}, + tty=True, + volumes={ + os.path.join(_dir, "smb.conf"): {"bind": "/etc/samba/smb.conf", "mode": "rw"} + }, + # network_mode='host', + ) + + def exitHandler(): + try: + # will raise docker.errors.NotFound if container does not currently exist + docker_client.containers.get(smb_container.id) + + smb_container.kill() + # smb_container.remove() + except docker.errors.NotFound: + pass + + atexit.register(exitHandler) + + # wait for samba to start up + timeout = 0 + while True: + if b"daemon 'smbd' finished starting up" in smb_container.logs(): + break + + if timeout >= 100: + raise RuntimeError('docker dperson/samba timed out while starting up') + + timeout += 1 + time.sleep(1) + + return smb_container, exitHandler + + +class RootDirUtil: + def __init__( + self, + dir_name, + direct_tcp=False, + host=None, + hostname=None, + my_name='local', + name_port=137, + smb_port=None + ): + self.host = socket.gethostbyname(socket.gethostname()) if host is None else host + self.hostname = socket.getfqdn() if hostname is None else hostname + + self.dir_name = dir_name + self.direct_tcp = direct_tcp + self.my_name = my_name + self.name_port = name_port + self.smb_port = smb_port + + self._container = None + self._container_exit_handler = None + + def exists(self): + conn = self.resource() + + return self.dir_name in conn.listShares() + + def create(self): + """taken care of by smb.conf + """ + pass + + def _delete(self, path, conn): + for p in conn.listPath(self.dir_name, path): + if p.filename != '.' and p.filename != '..': + subpath = os.path.join(path, p.filename) + + if p.isDirectory: + self._delete(subpath, conn) + conn.deleteDirectory(self.dir_name, subpath) + else: + conn.deleteFiles(self.dir_name, subpath) + + def delete(self): + conn = self.resource() + + self._delete('', conn) + + def resource(self): + kwargs = dict( + username=smb_user, + password=smb_passwd, + my_name=self.my_name, + remote_name=self.hostname, + is_direct_tcp=self.direct_tcp, + ) + + conn = SMBConnection(**kwargs) + + # actually connect + if self.smb_port is not None: + assert conn.connect(self.host, port=self.smb_port) + else: + assert conn.connect(self.host) + + return conn + + def start(self): + self._container, self._container_exit_handler = startServer(name_port=self.name_port) + + def stop(self): + if self._container is not None: + self._container_exit_handler() + + self._container = None + self._container_exit_handler = None + + +if __name__ == "__main__": + smb_container, _ = startServer(name_port=3669) + + def sigHandler(signo, frame): + sys.exit(0) + + # make sure the atexit-based docker cleanup runs on ctrl-c + signal.signal(signal.SIGINT, sigHandler) + signal.signal(signal.SIGTERM, sigHandler) + + old_log = '' + while True: + new_log = smb_container.logs() + if old_log != new_log: + print(new_log) + old_log = new_log + + time.sleep(1) diff --git a/jupyterfs/tests/utils/smb.conf.template b/jupyterfs/tests/utils/smb.conf.template new file mode 100644 index 00000000..7f57dd9f --- /dev/null +++ b/jupyterfs/tests/utils/smb.conf.template @@ -0,0 +1,22 @@ +[global] + netbios name = TESTNET + workgroup = TESTGROUP + server string = Samba test server for CI + security = user + guest account = nobody + map to guest = Bad User + + # disable printing services + load printers = no + printing = bsd + printcap name = /dev/null + disable spoolss = yes + +[test] + comment = test + path = /shared + write list = smbuser + guest ok = yes + # getting rid of those annoying .DS_Store files created by Mac users... + veto files = /._*/.DS_Store/ + delete veto files = yes diff --git a/setup.cfg b/setup.cfg index 50c88acd..9632e814 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,8 +5,15 @@ universal=1 description_file = README.md [flake8] -per-file-ignores= - jupyterfs/tests/test_pyfilesystem_manager.py:E302 +# ref: https://flake8.pycqa.org/en/2.5.5/warnings.html#error-codes +# E302: "expected 2 blank lines, found 0" +extend-ignore=E302 +max-line-length=200 + +# config for autopep8 +[pycodestyle] +# autopep8 doesn't seem to know about extend-ignore +ignore=E302 max-line-length=200 [manifix] diff --git a/setup.py b/setup.py index 84a24f64..68ba4efe 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ # # This file is part of the jupyter-fs library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. -# + from codecs import open from os import path from pathlib import Path @@ -32,19 +32,25 @@ 'notebook>=5.7.0', ] -dev_requires = requires + [ - 'autopep8', +test_requires = [ 'boto3', - 'bump2version', - 'codecov', - 'flake8', + 'docker', 'fs-miniofs', + 'fs.smbfs @ git+https://github.com/telamonian/fs.smbfs.git@dont_assume_everyone_ace_exists#egg=fs.smbfs', 'mock', - 'pylint', + 'pysmb', 'pytest', 'pytest-cov', ] +dev_requires = requires + test_requires + [ + 'autopep8', + 'bump2version', + 'codecov', + 'flake8', + 'pylint', +] + data_files_spec = [ # Lab extension installed by default: ('share/jupyter/lab/extensions',