Merge pull request #53 from andreax79/fs

### Added

- add support for remote filesystem (S3, GCP, etc...) in editor/file browser

### Changed

- mountpoint configuration changed
- download git objects from /repo/<object> instead of /files/~git/<object>
- node packages upgrade
- CodeMirror upgrade

Makefile

@@ -13,7 +13,10 @@ help:
 	@echo "- make npm-watch    Run npm build when files change"
 
 lint:
-	python3 setup.py flake8
+	flake8 airflow_code_editor tests
+
+black:
+	black -S airflow_code_editor tests
 
 tag:
 	@grep -q "## $$(cat airflow_code_editor/VERSION)" changelog.txt || (echo "Missing changelog !!! Update changelog.txt"; exit 1)
@@ -41,7 +44,7 @@ coverage:
 
 codemirror:
 	@rm -rf codemirror_src codemirror.zip
-	@curl -O https://codemirror.net/codemirror.zip
+	@curl -O https://codemirror.net/5/codemirror.zip
 	@unzip codemirror.zip -d codemirror_src
 	@mv codemirror_src/codemirror-*/* codemirror_src
 	@rm codemirror.zip
@@ -55,7 +58,7 @@ codemirror:
 	@python3 update_themes_js.py
 
 npm-build:
-	@NODE_OPTIONS=--openssl-legacy-provider npm run build
+	@npm run build
 
 npm-watch:
 	@NODE_OPTIONS=--openssl-legacy-provider npm run watch

README.md

@@ -35,10 +35,15 @@ If git support is enabled, the DAGs are stored in a Git repository. You may use
     pip install airflow-code-editor
   ```
 
-2. (Optional) Install Black Python code formatter.
+2. Install optional dependencies
+
+* black - Black Python code formatter
+* fs-s3fs - S3FS Amazon S3 Filesystem
+* fs-gcsfs - Google Cloud Storage Filesystem
+* ... other filesystems supported by PyFilesystem - see https://www.pyfilesystem.org/page/index-of-filesystems/
 
   ```bash
-    pip install black
+    pip install black fs-s3fs fs-gcsfs
   ```
 
 3. Restart the Airflow Web Server
@@ -60,12 +65,10 @@ All the settings are optional.
 * **git_author_email** email for the author/committer (default: logged user email)
 * **git_init_repo**  initialize a git repo in DAGs folder (default: True)
 * **root_directory**  root folder (default: Airflow DAGs folder)
-* **mount_name**, **mount1_name**, ...  configure additional file folder name (mount point)
-* **mount_path**, **mount1_path**, ...  configure additional file path
 * **line_length**  Python code formatter - max line length (default: 88)
 * **string_normalization**  Python code formatter - if true normalize string quotes and prefixes (default: False)
+* **mount**, **mount1**, ...  configure additional folder (mount point) - format: name=xxx,path=yyy
 
-Example:
 ```
    [code_editor]
    enabled = True
@@ -76,12 +79,21 @@ Example:
    root_directory = /home/airflow/dags
    line_length = 88
    string_normalization = False
-   mount_name = data
-   mount_path = /home/airflow/data
-   mount1_name = logs
-   mount1_path = /home/airflow/logs
+   mount = name=data,path=/home/airflow/data
+   mount1 = name=logs,path=/home/airflow/logs
+   mount2 = name=data,path=s3://example
 ```
 
+Mount Options:
+
+* **name**: mount name (destination)
+* **path**: local path or PyFilesystem FS URLs - see https://docs.pyfilesystem.org/en/latest/openers.html
+
+Example:
+* name=ftp_server,path=ftp://user:pass@ftp.example.org/private
+* name=data,path=s3://example
+* name=tmp,path=/tmp
+
 You can also set options with the following environment variables:
 
 * AIRFLOW__CODE_EDITOR__ENABLED
@@ -94,20 +106,14 @@ You can also set options with the following environment variables:
 * AIRFLOW__CODE_EDITOR__ROOT_DIRECTORY
 * AIRFLOW__CODE_EDITOR__LINE_LENGTH
 * AIRFLOW__CODE_EDITOR__STRING_NORMALIZATION
-* AIRFLOW__CODE_EDITOR__MOUNT_NAME
-* AIRFLOW__CODE_EDITOR__MOUNT_PATH
-* AIRFLOW__CODE_EDITOR__MOUNT1_NAME, AIRFLOW__CODE_EDITOR__MOUNT2_NAME, ...
-* AIRFLOW__CODE_EDITOR__MOUNT1_PATH, AIRFLOW__CODE_EDITOR__MOUNT2_PATH, ...
+* AIRFLOW__CODE_EDITOR__MOUNT, AIRFLOW__CODE_EDITOR__MOUNT1, AIRFLOW__CODE_EDITOR__MOUNT2, ...
 
 Example:
 ```
    export AIRFLOW__CODE_EDITOR__STRING_NORMALIZATION=True
-   export AIRFLOW__CODE_EDITOR__MOUNT_NAME='data'
-   export AIRFLOW__CODE_EDITOR__MOUNT_PATH='/home/airflow/data'
-   export AIRFLOW__CODE_EDITOR__MOUNT1_NAME='logs'
-   export AIRFLOW__CODE_EDITOR__MOUNT1_PATH='/home/airflow/logs'
-   export AIRFLOW__CODE_EDITOR__MOUNT2_NAME='tmp'
-   export AIRFLOW__CODE_EDITOR__MOUNT2_PATH='/tmp'
+   export AIRFLOW__CODE_EDITOR__MOUNT='name=data,path=/home/airflow/data'
+   export AIRFLOW__CODE_EDITOR__MOUNT1='name=logs,path=/home/airflow/logs'
+   export AIRFLOW__CODE_EDITOR__MOUNT2='name=tmp,path=/tmp'
 ```
 
 ### Development Instructions
@@ -167,3 +173,6 @@ Example:
 * Vue-tree, TreeView control for VueJS - https://github.com/grapoza/vue-tree
 * Splitpanes - https://github.com/antoniandre/splitpanes
 * Axios, Promise based HTTP client for the browser and node.js - https://github.com/axios/axios
+* PyFilesystem2, Python's Filesystem abstraction layer - https://github.com/PyFilesystem/pyfilesystem2
+* Amazon S3 PyFilesystem - https://github.com/PyFilesystem/s3fs
+* Google Cloud Storage PyFilesystem - https://github.com/Othoz/gcsfs

airflow_code_editor/VERSION

@@ -1 +1 @@
-5.2.2
+6.0.0

airflow_code_editor/app_builder_view.py

@@ -98,7 +98,6 @@ def _render(self, template, *args, **kargs):
                 **kargs
             )
 
-
 except (ImportError, ModuleNotFoundError):
     from airflow_code_editor.auth import has_access
     from airflow.www_rbac.decorators import has_dag_access

airflow_code_editor/code_editor_view.py

@@ -15,24 +15,24 @@
 #   limitations under the Licens
 #
 
-import os
-import os.path
 import logging
 import mimetypes
-from flask import abort, request, send_file
+from flask import abort, request
 from flask_wtf.csrf import generate_csrf
 from airflow.version import version
 from airflow_code_editor.commons import HTTP_404_NOT_FOUND
 from airflow_code_editor.tree import get_tree
 from airflow_code_editor.utils import (
     get_plugin_boolean_config,
     get_plugin_int_config,
-    git_absolute_path,
-    execute_git_command,
     error_message,
     normalize_path,
     prepare_api_response,
 )
+from airflow_code_editor.git import (
+    execute_git_command,
+)
+from airflow_code_editor.fs import RootFS
 
 
 __all__ = ["AbstractCodeEditorView"]
@@ -48,22 +48,16 @@ def _index(self):
 
     def _save(self, path=None):
         try:
-            fullpath = git_absolute_path(path)
             mime_type = request.headers.get("Content-Type", "text/plain")
             is_text = mime_type.startswith("text/")
             if is_text:
                 data = request.get_data(as_text=True)
                 # Newline fix (remove cr)
-                data = data.replace("\r", "").rstrip()
-                os.makedirs(os.path.dirname(fullpath), exist_ok=True)
-                with open(fullpath, "w") as f:
-                    f.write(data)
-                    f.write("\n")
+                data = data.replace("\r", "").rstrip() + "\n"
             else:  # Binary file
                 data = request.get_data()
-                os.makedirs(os.path.dirname(fullpath), exist_ok=True)
-                with open(fullpath, "wb") as f:
-                    f.write(data)
+            root_fs = RootFS()
+            root_fs.path(path).write_file(data=data, is_text=is_text)
             return prepare_api_response(path=normalize_path(path))
         except Exception as ex:
             logging.error(ex)
@@ -81,42 +75,49 @@ def _git_repo(self, path):
             return self._git_repo_get(path)
 
     def _git_repo_get(self, path):
-        " Get a file from GIT (invoked by the HTTP GET method) "
-        return execute_git_command(["cat-file", "-p", path])
+        "Get a file from GIT (invoked by the HTTP GET method)"
+        try:
+            # Download git blob - path = '<hash>/<name>'
+            path, attachment_filename = path.split('/', 1)
+        except:
+            # No attachment filename
+            attachment_filename = None
+        response = execute_git_command(["cat-file", "-p", path])
+        if attachment_filename:
+            response.headers["Content-Disposition"] = (
+                'attachment; filename="{0}"'.format(attachment_filename)
+            )
+            try:
+                content_type = mimetypes.guess_type(attachment_filename)[0]
+                if content_type:
+                    response.headers["Content-Type"] = content_type
+            except Exception:
+                pass
+        return response
 
     def _git_repo_post(self, path):
-        " Execute a GIT command (invoked by the HTTP POST method) "
+        "Execute a GIT command (invoked by the HTTP POST method)"
         git_args = request.json.get('args', [])
         return execute_git_command(git_args)
 
     def _load(self, path):
-        " Send the contents of a file to the client "
+        "Send the contents of a file to the client"
         try:
             path = normalize_path(path)
             if path.startswith("~git/"):
                 # Download git blob - path = '~git/<hash>/<name>'
-                _, path, filename = path.split("/", 3)
-                response = execute_git_command(["cat-file", "-p", path])
-                response.headers["Content-Disposition"] = (
-                    'attachment; filename="%s"' % filename
-                )
-                try:
-                    content_type = mimetypes.guess_type(filename)[0]
-                    if content_type:
-                        response.headers["Content-Type"] = content_type
-                except Exception:
-                    pass
-                return response
+                _, path = path.split("/", 1)
+                return self._git_repo_get(path)
             else:
                 # Download file
-                fullpath = git_absolute_path(path)
-                return send_file(fullpath, as_attachment=True)
+                root_fs = RootFS()
+                return root_fs.path(path).send_file(as_attachment=True)
         except Exception as ex:
             logging.error(ex)
             abort(HTTP_404_NOT_FOUND)
 
     def _format(self):
-        " Format code "
+        "Format code"
         try:
             import black
 
@@ -141,7 +142,7 @@ def _format(self):
                 )
             )
 
-    def _tree(self, path, args = {}):
+    def _tree(self, path, args={}):
         return {'value': get_tree(path, args)}
 
     def _ping(self):

airflow_code_editor/commons.py

@@ -15,7 +15,7 @@
 #   limitations under the Licens
 #
 
-import os
+from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Union
 
 __all__ = [
@@ -25,6 +25,7 @@
     'ROUTE',
     'STATIC',
     'CONFIG_SECTION',
+    'DEFAULT_GIT_BRANCH',
     'SUPPORTED_GIT_COMMANDS',
     'HTTP_200_OK',
     'HTTP_404_NOT_FOUND',
@@ -34,7 +35,6 @@
     'VERSION_FILE',
     'VERSION',
     'Args',
-    'Path',
     'GitOutput',
     'TreeOutput',
     'TreeFunc',
@@ -46,6 +46,7 @@
 ROUTE = '/' + PLUGIN_NAME
 STATIC = '/static/' + PLUGIN_NAME
 CONFIG_SECTION = PLUGIN_NAME + '_plugin'
+DEFAULT_GIT_BRANCH = 'main'
 HTTP_200_OK = 200
 HTTP_404_NOT_FOUND = 404
 SUPPORTED_GIT_COMMANDS = [
@@ -99,13 +100,11 @@
     'airflow_code_editor.js',
 ]
 
-VERSION_FILE = os.path.join(os.path.dirname(__file__), "VERSION")
-with open(VERSION_FILE) as f:
-    VERSION = f.read().strip()
+VERSION_FILE = Path(__file__).parent / "VERSION"
+VERSION = VERSION_FILE.read_text().strip()
 
 
 Args = Dict[str, str]
-Path = Optional[str]
 GitOutput = Union[None, bytes, str]
 TreeOutput = List[Dict[str, Any]]
-TreeFunc = Callable[[Path, Args], TreeOutput]
+TreeFunc = Callable[[Optional[str], Args], TreeOutput]