Merge pull request #16 from telamonian/smb_local_ci_tests

[WIP] add tests for local and smb flavors of PyFilesystemContentsManager

.gitignore

@@ -1,6 +1,8 @@
-# s3proxy artifacts
+# Python testing artifacts
+/osfs_local/
 /s3proxy*
 /s3_local/
+smb.conf
 
 # Logs
 logs
@@ -166,6 +168,9 @@ ENV/
 # mypy
 .mypy_cache/
 
+# scratch
+/scratch/
+
 # jetbrains ide stuff
 *.iml
 .idea/

Makefile

@@ -1,12 +1,12 @@
 testjs: ## Clean and Make js tests
 	yarn test
 
-testpy: ## Clean and Make unit tests
-	python3.7 -m pytest -v jupyterfs/tests --cov=jupyterfs
+testpy: ## Clean and Make py tests
+	python3.7 -m pytest -v jupyterfs/tests --cov=jupyterfs --cov-branch
 
-test: lint ## run the tests
-	python3.7 -m pytest -v jupyterfs/tests --cov=jupyterfs --junitxml=python_junit.xml --cov-report=xml --cov-branch
-	yarn test
+test: ## run all tests
+	testpy
+	testjs
 
 lintjs: ## run linter
 	yarn lint
@@ -15,8 +15,8 @@ lintpy: ## run linter
 	flake8 jupyterfs setup.py
 
 lint: ## run linter
-	flake8 jupyterfs setup.py
-	yarn lint
+	lintpy
+	lintjs
 
 fixjs:  ## run autopep8/tslint fix
 	./node_modules/.bin/tslint --fix src/*
@@ -25,8 +25,8 @@ fixpy:  ## run autopep8/tslint fix
 	autopep8 --in-place -r -a -a jupyterfs/
 
 fix:  ## run autopep8/tslint fix
-	autopep8 --in-place -r -a -a jupyterfs/
-	./node_modules/.bin/tslint --fix src/*
+	fixpy
+	fixjs
 
 annotate: ## MyPy type annotation check
 	mypy -s jupyterfs

appveyor.yml

@@ -0,0 +1,65 @@
+image: Visual Studio 2019
+
+environment:
+  PYTHON: "C:\\Python37"
+
+build: off
+
+services:
+- docker
+
+# mentioned in appveyor docs, doesn't seem to work
+# stack: docker, jdk 1.8, node 12, python 3.7
+
+# runs before repo checkout
+init:
+# set the python version permanently (for debugging)...
+- ps: "[Environment]::SetEnvironmentVariable(\"Path\", \"$env:PYTHON;$env:PYTHON\\Scripts;$env:Path\", \"Machine\")"
+# ...and for this session
+- cmd: set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+
+# install latest pip
+- python -m pip install --upgrade pip
+
+# set the node version
+- ps: Install-Product node 12
+
+# install yarn globally
+- npm install -g yarn
+
+install:
+# set up a windows file share named "test"
+- ps: ci/enable_os_share_windows.ps1
+- ps: ci/create_os_share_windows.ps1
+
+# set up a local s3_proxy server
+- bash -c "ci/s3_local.sh"
+- ps: Start-Process -FilePath "java" -ArgumentList "-jar","s3proxy","--properties","s3proxy.conf"
+- bash -c "sleep 5"
+- curl http://127.0.0.1:9000/
+
+# install python deps
+- pip install --upgrade --upgrade-strategy=eager -e .[dev]
+
+# install js deps
+- yarn
+
+# print all env vars
+- ps: gci env:* | sort-object name
+
+test_script:
+# lint the python code
+- python -m flake8 jupyterfs setup.py
+
+# lint the js code
+- yarn lint
+
+# test the python code
+- python -m pytest -v jupyterfs/tests --cov=jupyterfs --cov-branch
+
+# test the js code
+- yarn test
+
+on_failure:
+# enable remote login (via rdp) to windows ci vm on test failure
+- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

azure-pipelines.yml

@@ -4,7 +4,7 @@ trigger:
     - '*'  # must quote since "*" is a YAML reserved character; we want a string
 
 parameters:
-- name: initializationSteps
+- name: initSteps
   type: stepList
   default:
     - task: UsePythonVersion@0
@@ -37,27 +37,53 @@ parameters:
 
     - bash: env
       displayName: "List all environment variables"
-- name: s3proxySteps
+
+- name: initDockerSteps
   type: stepList
   default:
-    - bash: nohup ci/s3_local.sh
-      displayName: "Start up local S3 server"
+    - task: DockerInstaller@0
+      displayName: Docker Installer
+      inputs:
+        dockerVersion: 19.03.8
+        releaseType: stable
 
-    - script: curl http://127.0.0.1:9000/
-      displayName: 'Test local S3 server'
-- name: s3proxyMacSteps
+- name: initDockerStepsMac
   type: stepList
   default:
     - script: |
         ci/install_docker_desktop_mac.sh
         ci/start_docker_desktop_mac.sh
-        docker pull andrewgaul/s3proxy
-        nohup sudo docker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy &
+      displayName: 'Setup Docker Desktop Mac'
+
+- name: initS3proxyStepsLocal
+  type: stepList
+  default:
+    - bash: |
+        nohup ci/s3_local.sh -d -r
         sleep 5
-      displayName: 'Start s3proxy in a docker'
+      displayName: "Start up local S3 server"
 
     - script: curl http://127.0.0.1:9000/
       displayName: 'Test local S3 server'
+
+- name: initSmbOsWindows
+  type: stepList
+  default:
+    - task: PowerShell@2
+      inputs:
+        targetType: 'filePath'
+        filePath: 'ci/enable_os_share_windows.ps1'
+        errorActionPreference: 'stop'
+        failOnStderr: True
+      displayName: 'Enable os builtin file sharing'
+    - task: PowerShell@2
+      inputs:
+        targetType: 'filePath'
+        filePath: 'ci/create_os_share_windows.ps1'
+        errorActionPreference: 'stop'
+        failOnStderr: True
+      displayName: 'Create share on os builtin samba server'
+
 - name: testSteps
   type: stepList
   default:
@@ -72,6 +98,7 @@ parameters:
 
     - script: make testjs
       displayName: 'Test JS'
+
 - name: publishSteps
   type: stepList
   default:
@@ -97,8 +124,8 @@ jobs:
         python.version: '3.7'
 
   steps:
-    - ${{ parameters.initializationSteps }}
-    - ${{ parameters.s3proxySteps }}
+    - ${{ parameters.initDockerSteps }}
+    - ${{ parameters.initSteps }}
     - ${{ parameters.testSteps }}
     - ${{ parameters.publishSteps }}
 
@@ -112,22 +139,23 @@ jobs:
         python.version: '3.7'
 
   steps:
-    - ${{ parameters.s3proxyMacSteps }}
-    - ${{ parameters.initializationSteps }}
+    - ${{ parameters.initDockerStepsMac }}
+    - ${{ parameters.initSteps }}
     - ${{ parameters.testSteps }}
     - ${{ parameters.publishSteps }}
 
-- job: 'Windows'
-  pool:
-    vmImage: 'vs2017-win2016'
-
-  strategy:
-    matrix:
-      Python37:
-        python.version: '3.7'
-
-  steps:
-    - ${{ parameters.initializationSteps }}
-    - ${{ parameters.s3proxySteps }}
-    - ${{ parameters.testSteps }}
-    - ${{ parameters.publishSteps }}
+# - job: 'Windows'
+#   pool:
+#     vmImage: 'windows-2019'
+
+#   strategy:
+#     matrix:
+#       Python37:
+#         python.version: '3.7'
+
+#   steps:
+#     - ${{ parameters.initSmbOsWindows }}
+#     - ${{ parameters.initS3proxyStepsLocal }}
+#     - ${{ parameters.initSteps }}
+#     - ${{ parameters.testSteps }}
+#     - ${{ parameters.publishSteps }}

ci/create_os_share_windows.ps1

@@ -0,0 +1,13 @@
+#Requires -RunAsAdministrator
+
+$ErrorActionPreference = 'Stop'
+
+$smbuser_passwd = ConvertTo-SecureString "smbuser" -AsPlainText -Force
+New-LocalUser "smbuser" -Password $smbuser_passwd
+
+mkdir C:\shared
+
+New-SmbShare -Name "test" -Path "C:\shared" -FullAccess smbuser
+
+# display info on newly-created share
+Get-SmbShare

ci/enable_os_share_windows.ps1

@@ -0,0 +1,24 @@
+#Requires -RunAsAdministrator
+
+$ErrorActionPreference = 'Stop'
+
+# script to overcome all weird windows vm settings on azure
+
+# disable all password requirements (length, complexity, etc)
+# needed to allow for creation of new account smbuser:smbuser
+secedit /export /cfg c:\secpol.cfg
+(gc C:\secpol.cfg).replace("PasswordComplexity = 1", "PasswordComplexity = 0") | Out-File C:\secpol.cfg
+secedit /configure /db c:\windows\security\local.sdb /cfg c:\secpol.cfg /areas SECURITYPOLICY
+rm -force c:\secpol.cfg -confirm:$false
+
+# turn on file sharing
+netsh advfirewall firewall set rule group="File and Printer Sharing" new enable=Yes
+
+# print out some info about local NETBIOS cache
+nbtstat -c
+
+# print out some info about local NETBIOS names
+nbtstat -n
+
+# print out some info about local NETBIOS resolutions
+nbtstat -r

ci/install_docker_desktop_mac.sh

@@ -5,8 +5,7 @@
 # https://forums.docker.com/t/docker-for-mac-unattended-installation/27112
 
 brew cask install docker
-set -x
-echo "Running xattr"
+# allow the app to run without confirmation
 xattr -d -r com.apple.quarantine /Applications/Docker.app
 
 # preemptively do docker.app's setup to avoid any gui prompts

ci/remove_os_share_windows.ps1

@@ -0,0 +1,27 @@
+#Requires -RunAsAdministrator
+
+# discover true names of errors:
+# $Error[0].Exception.GetType().FullName
+
+$ErrorActionPreference = 'Stop'
+
+try {
+    Get-SmbShare -name "test"
+    Remove-SmbShare -name "test" -force
+}
+catch [Microsoft.PowerShell.Cmdletization.Cim.CimJobException] {
+}
+
+try {
+    ls C:\shared
+    rm -r -fo C:\shared
+}
+catch [System.Management.Automation.ItemNotFoundException] {
+}
+
+try {
+    Get-LocalUser -name "smbuser"
+    Remove-LocalUser -name "smbuser"
+}
+catch [Microsoft.PowerShell.Commands.UserNotFoundException] {
+}

ci/s3_local.sh

@@ -2,6 +2,36 @@
 S3PROXY_VERSION=1.7.0
 S3PROXY_PORT=9000
 
+# Usage info
+show_help() {
+cat << EOF
+Usage: ${0##*/} [-d]
+Run a local instance of s3_proxy
+
+    -d    Detach; run s3_proxy as a background job
+EOF
+}
+
+# parse opts
+detach=
+run=
+OPTIND=1
+while getopts hdtr opt; do
+    case $opt in
+        h)  show_help
+            exit 0
+            ;;
+        d)  detach=-d
+            ;;
+        r)  run=-r
+            ;;
+        *)  show_help >&2
+            exit 1
+            ;;
+    esac
+done
+shift "$((OPTIND-1))"   # Discard the options and sentinel --
+
 # make dir for storing s3 files locally
 mkdir -p s3_local
 
@@ -29,7 +59,13 @@ EOT
 curl -L https://github.com/gaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy -o s3proxy
 
 # run s3proxy as a background job
-java -jar s3proxy --properties s3proxy.conf &
+if [ "$run" = "-r" ]; then
+    if [ "$detach" = "-d" ]; then
+        java -jar s3proxy --properties s3proxy.conf &
+    else
+        java -jar s3proxy --properties s3proxy.conf
+    fi
+fi
 
 # no explicit wait needed, azure already stalls
 # for 10 seconds to wait for open process to stop