add files from script

python/dendro/api_helpers/routers/gui/file_routes.py

@@ -62,7 +62,18 @@ async def set_file(project_id, file_name, data: SetFileRequest, github_access_to
     metadata = data.metadata
     is_folder = data.isFolder
 
-    assert size is not None, "size must be specified"
+    if size is None:
+        if content.startswith("url:"):
+            # if the content is a URL, we can get the size from the URL
+            headers = {
+                'Accept-Encoding': 'identity' # don't accept encoding in order to get the actual size
+            }
+            from aiohttp import ClientSession
+            async with ClientSession() as session:
+                async with session.head(content[len("url:"):], headers=headers) as response:
+                    size = int(response.headers['Content-Length'])
+        else:
+            raise Exception("size must be specified")
 
     project = await fetch_project(project_id)
     assert project is not None, f"No project with ID {project_id}"

src/dbInterface/dbInterface.ts

@@ -173,20 +173,13 @@ export const fetchFile = async (projectId: string, fileName: string, auth: Auth)
 }
 
 const headRequest = async (url: string, headers?: any) => {
-    // Cannot use HEAD, because it is not allowed by CORS on DANDI AWS bucket
-    // let headResponse
-    // try {
-    //     headResponse = await fetch(url, {method: 'HEAD'})
-    //     if (headResponse.status !== 200) {
-    //         return undefined
-    //     }
-    // }
-    // catch(err: any) {
-    //     console.warn(`Unable to HEAD ${url}: ${err.message}`)
-    //     return undefined
-    // }
-    // return headResponse
+    // It's annoying that we need to hard-code this...
+    // but if we are fetching from a cloudflare bucket, then we can do the normal HEAD request
+    if ((url.startsWith('https://lindi.neurosift.org')) || (url.startsWith('https://neurosift.org'))) {
+        return normalHeadRequest(url, headers)
+    }
 
+    // Otherwise we cannot use HEAD, because it is not allowed by CORS on DANDI AWS bucket
     // Instead, use aborted GET.
     const controller = new AbortController();
     const signal = controller.signal;
@@ -198,9 +191,32 @@ const headRequest = async (url: string, headers?: any) => {
     return response
 }
 
+const normalHeadRequest = async (url: string, headers?: any) => {
+    try {
+        const response = await fetch(url, {method: 'HEAD', headers})
+        if (response.status !== 200) {
+            return undefined
+        }
+        // display response headers
+        for (const [key, value] of response.headers) {
+            console.log(`${key}: ${value}`);
+        }
+        return response
+    }
+    catch(err: any) {
+        console.warn(`Unable to HEAD ${url}: ${err.message}`)
+        return undefined
+    }
+}
+
 const getSizeForRemoteFile = async (url: string): Promise<number> => {
     const authorizationHeader = getAuthorizationHeaderForUrl(url)
-    const headers = authorizationHeader ? {Authorization: authorizationHeader} : undefined
+    const headers: {[key: string]: string} = authorizationHeader ? {Authorization: authorizationHeader} : {}
+
+    // we are going to need the content-length
+    headers['User-Agent'] = 'Mozilla/5.0'
+
+    // add user agent to avoid 403
     const response = await headRequest(url, headers)
     if (!response) {
         throw Error(`Unable to HEAD ${url}`)
@@ -222,10 +238,12 @@ const getSizeForRemoteFile = async (url: string): Promise<number> => {
 
 export const setUrlFile = async (projectId: string, fileName: string, url: string, metadata: any, auth: Auth): Promise<void> => {
     const reqUrl = `${apiBase}/api/gui/projects/${projectId}/files/${fileName}`
-    const size = await getSizeForRemoteFile(url)
+
+    // the size is now computed on the server side
+    // const size = await getSizeForRemoteFile(url)
     const body = {
         content: `url:${url}`,
-        size,
+        // size,
         metadata
     }
     const response = await putRequest(reqUrl, body, auth)

src/pages/ProjectPage/FileView/FileView.tsx

@@ -48,7 +48,7 @@ const FileViewChild: FunctionComponent<FileViewChildProps> = ({fileName, width,
         if (!files) return undefined
         return files.find(f => (f.fileName === fileName))
     }, [files, fileName])
-    if (fileName.endsWith('.nwb')) {
+    if ((fileName.endsWith('.nwb')) || (fileName.endsWith('.nwb.zarr.json'))) {
         return (
             <NwbFileView
                 fileName={fileName}

src/pages/ProjectPage/FileView/FileViewTable.tsx

@@ -43,9 +43,9 @@ const FileViewTable: FunctionComponent<FileViewTableProps> = ({fileName, additio
                     <td>URL:</td>
                     <td>
                         {theUrl}
-                        {theFile && theFile.size && theFile.size < 50e6 && (
+                        {theFile && theFile.size && theFile.size < 50e6 ? (
                             <>&nbsp;&nbsp;<DownloadLink url={theUrl} baseFileName={getBaseFileName(theFile.fileName)} /></>
-                        )}
+                        ) : ""}
                     </td>
                 </tr>
                 <tr>

src/pages/ProjectPage/FileView/NwbFileView.tsx

@@ -198,9 +198,12 @@ const NwbFileView: FunctionComponent<Props> = ({fileName, width, height}) => {
             const dandiAssetPathEncoded = encodeURIComponent(metadata.dandiAssetPath)
             additionalQueryParams += `&dandiAssetPath=${dandiAssetPathEncoded}`
         }
-        const u = `https://flatironinstitute.github.io/neurosift/?p=/nwb&url=${nwbUrl}${additionalQueryParams}`
+        let u = `https://flatironinstitute.github.io/neurosift/?p=/nwb&url=${nwbUrl}${additionalQueryParams}`
+        if (fileName.endsWith('.json')) {
+            u += '&st=lindi'
+        }
         window.open(u, '_blank')
-    }, [nwbUrl, metadata])
+    }, [nwbUrl, metadata, fileName])
 
     useEffect(() => {
         if (!dandisetId) return

src/pages/ProjectPage/scripts/RunScript/RunScriptWindow.tsx

@@ -4,7 +4,7 @@ import { useProject } from "../../ProjectPageContext"
 import { RunScriptAddJob, RunScriptResult } from "./RunScriptWorkerTypes"
 import { Hyperlink } from "@fi-sci/misc"
 import { useGithubAuth } from "../../../../GithubAuth/useGithubAuth"
-import { DendroProcessingJobDefinition, createJob } from "../../../../dbInterface/dbInterface"
+import { DendroProcessingJobDefinition, createJob, setUrlFile } from "../../../../dbInterface/dbInterface"
 
 type RunScriptWindowProps = {
     width: number
@@ -60,6 +60,18 @@ const RunScriptWindow: FunctionComponent<RunScriptWindowProps> = ({width, height
         return {newJobs, existingJobs}
     }, [jobs, result, projectJobHashes, computeResourceProcessorsByName])
 
+    const {newFiles} = useMemo(() => {
+        if (!files) return {newFiles: undefined, existingFiles: undefined}
+        if (!result) return {newFiles: undefined, existingFiles: undefined}
+        const newFiles: {fileName: string, url: string}[] = []
+        for (const f of result.addedFiles) {
+            if (!files.find(f1 => f1.fileName === f.fileName)) {
+                newFiles.push(f)
+            }
+        }
+        return {newFiles}
+    }, [files, result])
+
     const jobProblems = useMemo(() => {
         if (!result) return undefined
         if (!computeResourceProcessorsByName) return undefined
@@ -81,9 +93,17 @@ const RunScriptWindow: FunctionComponent<RunScriptWindowProps> = ({width, height
 
     const auth = useGithubAuth()
 
-    const handleSubmitJobs = useCallback(async (jobsToSubmit: RunScriptAddJob[]) => {
+    const handleSubmit = useCallback(async () => {
+        const jobsToSubmit = newJobs
+        const filesToAdd = newFiles
+        if (!jobsToSubmit) return
+        if (!filesToAdd) return
         if (!auth.signedIn) return
         if (!files) return
+        for (const f of filesToAdd) {
+            const metaData = {}
+            await setUrlFile(projectId, f.fileName, f.url, metaData, auth)
+        }
         const batchId = createRandomId(8)
         for (const j of jobsToSubmit) {
             console.info(`Submitting job: ${j.processorName}`)
@@ -115,8 +135,16 @@ const RunScriptWindow: FunctionComponent<RunScriptWindowProps> = ({width, height
         }
         refreshFiles()
         refreshJobs()
-        alert('Jobs submitted')
-    }, [auth, projectId, files, computeResourceProcessorsByName, refreshFiles, refreshJobs])
+        if ((newFiles.length > 0) && (newJobs.length > 0)) {
+            alert('New files and jobs have been submitted')
+        }
+        else if (newFiles.length > 0) {
+            alert('New files have been submitted')
+        }
+        else if (newJobs.length > 0) {
+            alert('New jobs have been submitted')
+        }
+    }, [auth, projectId, files, computeResourceProcessorsByName, refreshFiles, refreshJobs, newJobs, newFiles])
 
     return (
         <div style={{position: 'absolute', width, height}}>
@@ -137,18 +165,18 @@ const RunScriptWindow: FunctionComponent<RunScriptWindowProps> = ({width, height
                                 {jobProblems.map((p, i) => <div key={i}>{p}</div>)}
                             </div>
                         ) : (
-                            newJobs && existingJobs && (
+                            newJobs && existingJobs && newFiles && (
                                 <div>
-                                    <div>{newJobs.length} new jobs ({newJobs.length + existingJobs.length} total)</div>
-                                    {newJobs.length > 0 && (
+                                    <div>{newJobs.length} new jobs ({newJobs.length + existingJobs.length} total); {newFiles.length} new files</div>
+                                    {newJobs.length + newFiles.length > 0 && (
                                         saved ? (
                                             <div>
                                                 <Hyperlink
                                                     onClick={() => {
-                                                        handleSubmitJobs(newJobs)
+                                                        handleSubmit()
                                                     }}
                                                 >
-                                                    {newJobs.length > 1 ? `Submit these ${newJobs.length} new jobs` : `Submit this one new job?`}
+                                                    SUBMIT
                                                 </Hyperlink>
                                             </div>
                                         ) : (

src/pages/ProjectPage/scripts/RunScript/RunScriptWorkerTypes.ts

@@ -32,8 +32,14 @@ export type RunScriptAddJob = {
     runMethod: 'local' | 'aws_batch' | 'slurm'
 }
 
+export type RunScriptAddedFile = {
+    fileName: string
+    url: string
+}
+
 export type RunScriptResult = {
     jobs: RunScriptAddJob[]
+    addedFiles: RunScriptAddedFile[]
 }
 
 export {}

src/pages/ProjectPage/scripts/RunScript/runScriptWorker.ts

@@ -7,7 +7,7 @@ import {
 } from "./RunScriptWorkerTypes";
 
 const runScript = (_script: string, _files: any[]) => {
-  const result: RunScriptResult = { jobs: [] };
+  const result: RunScriptResult = { jobs: [], addedFiles: []};
   const _addJob = (a: {
     processorName: string;
     inputs: RunScriptAddJobInputFile[];
@@ -95,13 +95,22 @@ const runScript = (_script: string, _files: any[]) => {
     });
   };
 
+  const _addFile = (fileName: string, a: {url: string}) => {
+    result.addedFiles.push({fileName, url: a.url});
+  }
+
   const log: string[] = [];
   const context = {
     print: function (message: string) {
+      // make sure it is a string
+      if (typeof message !== "string") {
+        message = JSON.stringify(message);
+      }
       log.push(message);
     },
     files: _files,
     addJob: _addJob,
+    addFile: _addFile
   };
   const scriptFunction = new Function("context", _script);
   try {

src/pages/ProjectsPage/ProjectsPage.css

@@ -1,13 +1,12 @@
 .projects-page {
-    padding: 20px;
+    /* padding: 20px; */
     font-family: Arial, sans-serif;
     max-width: 1400px;
     margin: auto;
 }
 
-.projects-page h1, .projects-page h2 {
+.projects-page h3 {
     color: #333;
-    text-align: center;
 }
 
 .projects-page p {

src/pages/ProjectsPage/ProjectsPage.tsx

@@ -30,8 +30,8 @@ const ProjectsPage: FunctionComponent<Props> = ({width, height}) => {
 
     return (
         <div className="projects-page" style={{position: 'absolute', width, height, overflowY: 'auto'}}>
-            <h3>Your projects</h3>
-            <div>
+            <h3 style={{paddingLeft: 10}}>Your projects</h3>
+            <div style={{paddingLeft: 10}}>
                 <SmallIconButton icon={<Add />} onClick={handleAdd} label="Create a new project" />
                 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
                 <SmallIconButton icon={<ImportExport />} onClick={() => setRoute({page: 'dandisets'})} label="Import data from DANDI" />

src/pages/ProjectsPage/ProjectsTable.tsx

@@ -14,6 +14,10 @@ const ProjectsTable: FunctionComponent<Props> = ({admin}) => {
 
     const { setRoute } = useRoute()
 
+    const sortedProjects = useMemo(() => {
+        return projects ? projects.sort((a, b) => b.timestampCreated - a.timestampCreated) : []
+    }, [projects])
+
     if (!projects) return <div>Retrieving projects...</div>
 
     return (
@@ -30,7 +34,7 @@ const ProjectsTable: FunctionComponent<Props> = ({admin}) => {
             </thead>
             <tbody>
                 {
-                    projects.map((pr) => (
+                    sortedProjects.map((pr) => (
                         <tr key={pr.projectId}>
                             <td>
                                 <Hyperlink onClick={() => setRoute({page: 'project', projectId: pr.projectId})}>