Allow Datapoint/Segmentations upload via Frontend

backend/routes/data.py

@@ -1,8 +1,11 @@
+import os
+import shutil
 import json
 import sqlalchemy as sa
 import uuid
 
 from pathlib import Path
+from zipfile import ZipFile
 
 from flask import jsonify, flash, redirect, url_for, request, send_from_directory
 from flask_jwt_extended import jwt_required, get_jwt_identity
@@ -15,19 +18,22 @@
 
 from . import api
 
+ALLOWED_COMPRESSED_EXTENSIONS = ["zip"]
+ALLOWED_ANNOTATION_EXTENSIONS = ["json"]
 ALLOWED_EXTENSIONS = ["wav", "mp3", "ogg"]
 
-
 @api.route("/audio/<path:file_name>", methods=["GET"])
 @jwt_required
 def send_audio_file(file_name):
     return send_from_directory(app.config["UPLOAD_FOLDER"], file_name)
 
 
-def validate_segmentation(segment):
+def validate_segmentation(segment, without_data=False):
     """Validate the segmentation before accepting the annotation's upload from users
     """
     required_key = {"start_time", "end_time", "transcription"}
+    if without_data:
+        required_key.add("filename") # requried to search the datapoint
 
     if set(required_key).issubset(segment.keys()):
         return True
@@ -190,3 +196,120 @@ def add_data():
         ),
         201,
     )
+
+
+def files_from_zip(zip_file):
+    """Generator for getting files from the a zip file
+
+    Returns:
+        Generator: if valid, generator of files in the zip
+        False: if the file is invalid
+    """
+    with ZipFile(zip_file, "r") as zip_obj:
+        for cfilename in zip_obj.namelist():
+            cfile_extension = Path(cfilename).suffix.lower()
+            if cfile_extension[1:] in ALLOWED_EXTENSIONS:
+                zip_obj.extract(
+                    cfilename,
+                    Path(app.config["UPLOAD_FOLDER"])
+                )
+                yield Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), "data"
+            elif cfile_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS:
+                zip_obj.extract(
+                    cfilename,
+                    Path(app.config["UPLOAD_FOLDER"])
+                )
+                yield Path(app.config["UPLOAD_FOLDER"]).joinpath(cfilename), "annotation"
+
+
+def file_to_database(
+    db,
+    user,
+    project,
+    audio_file,
+    is_marked_for_review,
+    reference_transcription,
+    compressed_file=False
+):
+    """Add data to database and save a copy in the /uploads folder
+
+    TODO:
+        - delete compressed file is there was some error
+    """
+    try:
+
+        if compressed_file:
+            original_filename = os.path.basename(audio_file)
+            extension = Path(original_filename).suffix.lower()
+            filename = f"{str(uuid.uuid4().hex)}{extension}"
+            from_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(
+                original_filename)
+            to_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename)
+            shutil.move(from_path, to_path)
+        else:
+            original_filename = secure_filename(audio_file.filename)
+            extension = Path(original_filename).suffix.lower()
+            filename = f"{str(uuid.uuid4().hex)}{extension}"
+            file_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename)
+            audio_file.save(file_path.as_posix())
+
+        data = Data(
+            project_id=project.id,
+            filename=filename,
+            original_filename=original_filename,
+            reference_transcription=reference_transcription,
+            is_marked_for_review=is_marked_for_review,
+            assigned_user_id=user.id,
+        )
+        db.session.add(data)
+        db.session.flush()
+
+        return True
+    except Exception as e:
+        if compressed_file:
+            shutil.rmtree(path=from_path, ignore_errors=True)
+            shutil.rmtree(path=to_path, ignore_errors=True)
+        app.logger.error("Error in adding the data")
+        app.logger.error(e)
+        return False
+
+
+def annotation_to_database(project, annotation_file):
+    """Add segmentation to database from a json
+    """
+    ret_flag = False
+    try:
+        segmentations = json.load(annotation_file)
+        for _segment in segmentations:
+            validated = validate_segmentation(
+                _segment, without_data=True
+            )
+            if validated:
+                data = Data.query.filter_by(
+                    project_id=project.id,
+                    original_filename=_segment['filename']
+                ).first()
+
+                if data:
+                    new_segment = generate_segmentation(
+                        data_id=data.id,
+                        project_id=project.id,
+                        end_time=_segment["end_time"],
+                        start_time=_segment["start_time"],
+                        transcription=_segment["transcription"],
+                        annotations=_segment.get(
+                            "annotations", {})
+                    )
+                ret_flag = True
+
+        # delete the annotations file from the disk if exists
+        if hasattr(annotation_file, "name") and os.path.exists(annotation_file.name):
+            os.remove(annotation_file.name)
+        elif hasattr(annotation_file, "filename") and os.path.exists(annotation_file.filename):
+            os.remove(annotation_file.filename)
+
+        return ret_flag
+    except Exception as e:
+        app.logger.error("Error in adding the annotations")
+        app.logger.error(e)
+        return False

backend/routes/projects.py

@@ -1,15 +1,26 @@
 import sqlalchemy as sa
 import uuid
 
+from pathlib import Path
 from flask import jsonify, flash, redirect, url_for, request
 from flask_jwt_extended import jwt_required, get_jwt_identity
 from werkzeug.urls import url_parse
+from werkzeug.exceptions import BadRequest
 
 from backend import app, db
 from backend.models import Project, User, Label, Data, Segmentation, LabelValue
 
 from . import api
-from .data import generate_segmentation
+
+from .data import (
+    files_from_zip,
+    file_to_database,
+    ALLOWED_EXTENSIONS,
+    generate_segmentation,
+    annotation_to_database,
+    ALLOWED_ANNOTATION_EXTENSIONS,
+    ALLOWED_COMPRESSED_EXTENSIONS
+)
 
 
 def generate_api_key():
@@ -708,3 +719,102 @@ def get_project_annotations(project_id):
         ),
         200,
     )
+
+
+@api.route("/projects/<int:project_id>/upload", methods=["POST"])
+@jwt_required
+def add_data_to_project(project_id):
+    """Upload data via zip files ro direct supported formats
+
+    """
+    identity = get_jwt_identity()
+    request_user = User.query.filter_by(username=identity["username"]).first()
+    app.logger.info(f"Current user is: {request_user}")
+    is_admin = True if request_user.role.role == "admin" else False
+
+    if is_admin == False:
+        return jsonify(message="Unauthorized access!"), 401
+
+    project = Project.query.get(project_id)
+    files = request.files.items()
+
+    for _, file in files:
+        filename = file.filename
+        file_ext = Path(filename).suffix.lower()
+
+        if file_ext[1:] in ALLOWED_EXTENSIONS:
+            commit_flag = file_to_database(
+                db=db,
+                user=request_user,
+                project=project,
+                audio_file=file,
+                is_marked_for_review=True,
+                reference_transcription="False"
+            )
+
+            if not commit_flag:
+                return (
+                    jsonify(
+                        message="Error during uploading of file",
+                        type="UPLOAD_FAILED",
+                    ),
+                    500
+                )
+
+        elif file_ext[1:] in ALLOWED_ANNOTATION_EXTENSIONS:
+            commit_flag = annotation_to_database(
+                project=project,
+                annotation_file=file
+            )
+
+            if not commit_flag:
+                return (
+                    jsonify(
+                        message="Error during uploading of file",
+                        type="UPLOAD_FAILED",
+                    ),
+                    500
+                )
+
+        elif file_ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS:
+            cmprsd_files = files_from_zip(
+                zip_file=file
+            )
+            for cfilename, filetype in cmprsd_files:
+                if filetype == "data":
+                    commit_flag = file_to_database(
+                        db=db,
+                        user=request_user,
+                        project=project,
+                        audio_file=cfilename,
+                        is_marked_for_review=True,
+                        reference_transcription="False",
+                        compressed_file=True
+                    )
+                else:
+                    cfile = open(cfilename, "r")
+                    commit_flag = annotation_to_database(
+                        project=project,
+                        annotation_file=cfile
+                    )
+
+                if not commit_flag:
+                    return (
+                        jsonify(
+                            message="Error during uploading of file",
+                            type="UPLOAD_FAILED",
+                        ),
+                        500
+                    )
+
+        else:
+            raise BadRequest(description="File format is not supported")
+        db.session.commit()
+
+    return (
+        jsonify(
+            message="Data uploaded succesfully",
+            type="UPLOAD_SUCCESS",
+        ),
+        200
+    )

frontend/package.json

@@ -12,6 +12,7 @@
     "@testing-library/react": "^9.3.2",
     "@testing-library/user-event": "^7.1.2",
     "axios": "^0.19.2",
+    "react-dropzone-uploader": "^2.11.0",
     "bootstrap": "^4.4.1",
     "jquery": "^3.5.0",
     "popper.js": "^1.16.1",

frontend/src/containers/forms/uploadDataForm.js

@@ -0,0 +1,49 @@
+import React from "react";
+import { withRouter } from "react-router";
+import Dropzone from "react-dropzone-uploader";
+import { withStore } from "@spyna/react-store";
+import "react-dropzone-uploader/dist/styles.css";
+
+class UploadDataForm extends React.Component {
+  constructor(props) {
+    super(props);
+
+    const projectId = this.props.projectId;
+
+    this.initialState = {
+      projectId,
+      addDataUrl: `/api/projects/${projectId}/upload`,
+    };
+
+    this.state = Object.assign({}, this.initialState);
+    console.log(this.props);
+  }
+
+  getUploadParams = ({ file, meta }) => {
+    const body = new FormData();
+    body.append("fileField", file);
+    return {
+      url: this.state.addDataUrl,
+      body,
+      headers: {
+        Authorization: localStorage.getItem("access_token"),
+      },
+    };
+  };
+
+  handleSubmit = (files, allFiles) => {
+    allFiles.forEach((f) => f.remove());
+  };
+
+  render() {
+    return (
+      <Dropzone
+        onSubmit={this.handleSubmit}
+        getUploadParams={this.getUploadParams}
+        accept="application/zip,application/x-zip,application/x-zip-compressed,application/octet-stream, audio/*, application/json"
+      />
+    );
+  }
+}
+
+export default withStore(withRouter(UploadDataForm));

frontend/src/containers/modal.js

@@ -2,6 +2,7 @@ import React from "react";
 import Modal from "react-bootstrap/Modal";
 
 import CreateUserForm from "./forms/createUserForm";
+import UploadDataForm from "./forms/uploadDataForm";
 import EditUserForm from "./forms/editUserForm";
 import CreateProjectForm from "./forms/createProjectForm";
 import CreateLabelForm from "./forms/createLabelForm";
@@ -28,6 +29,9 @@ const FormModal = (props) => {
       <Modal.Body>
         {props.formType === "NEW_USER" ? <CreateUserForm /> : null}
         {props.formType === "NEW_PROJECT" ? <CreateProjectForm /> : null}
+        {props.formType === "NEW_DATA" ? (
+          <UploadDataForm projectId={props.projectId} />
+        ) : null}
         {props.formType === "EDIT_USER" ? (
           <EditUserForm userId={props.userId} />
         ) : null}