diff --git a/python/dendro/api_helpers/routers/gui/file_routes.py b/python/dendro/api_helpers/routers/gui/file_routes.py index 981e08c..479e75a 100644 --- a/python/dendro/api_helpers/routers/gui/file_routes.py +++ b/python/dendro/api_helpers/routers/gui/file_routes.py @@ -62,7 +62,18 @@ async def set_file(project_id, file_name, data: SetFileRequest, github_access_to metadata = data.metadata is_folder = data.isFolder - assert size is not None, "size must be specified" + if size is None: + if content.startswith("url:"): + # if the content is a URL, we can get the size from the URL + headers = { + 'Accept-Encoding': 'identity' # don't accept encoding in order to get the actual size + } + from aiohttp import ClientSession + async with ClientSession() as session: + async with session.head(content[len("url:"):], headers=headers) as response: + size = int(response.headers['Content-Length']) + else: + raise Exception("size must be specified") project = await fetch_project(project_id) assert project is not None, f"No project with ID {project_id}" diff --git a/src/dbInterface/dbInterface.ts b/src/dbInterface/dbInterface.ts index b237079..3caaef4 100644 --- a/src/dbInterface/dbInterface.ts +++ b/src/dbInterface/dbInterface.ts @@ -173,20 +173,13 @@ export const fetchFile = async (projectId: string, fileName: string, auth: Auth) } const headRequest = async (url: string, headers?: any) => { - // Cannot use HEAD, because it is not allowed by CORS on DANDI AWS bucket - // let headResponse - // try { - // headResponse = await fetch(url, {method: 'HEAD'}) - // if (headResponse.status !== 200) { - // return undefined - // } - // } - // catch(err: any) { - // console.warn(`Unable to HEAD ${url}: ${err.message}`) - // return undefined - // } - // return headResponse + // It's annoying that we need to hard-code this... + // but if we are fetching from a cloudflare bucket, then we can do the normal HEAD request + if ((url.startsWith('https://lindi.neurosift.org')) || (url.startsWith('https://neurosift.org'))) { + return normalHeadRequest(url, headers) + } + // Otherwise we cannot use HEAD, because it is not allowed by CORS on DANDI AWS bucket // Instead, use aborted GET. const controller = new AbortController(); const signal = controller.signal; @@ -198,9 +191,32 @@ const headRequest = async (url: string, headers?: any) => { return response } +const normalHeadRequest = async (url: string, headers?: any) => { + try { + const response = await fetch(url, {method: 'HEAD', headers}) + if (response.status !== 200) { + return undefined + } + // display response headers + for (const [key, value] of response.headers) { + console.log(`${key}: ${value}`); + } + return response + } + catch(err: any) { + console.warn(`Unable to HEAD ${url}: ${err.message}`) + return undefined + } +} + const getSizeForRemoteFile = async (url: string): Promise => { const authorizationHeader = getAuthorizationHeaderForUrl(url) - const headers = authorizationHeader ? {Authorization: authorizationHeader} : undefined + const headers: {[key: string]: string} = authorizationHeader ? {Authorization: authorizationHeader} : {} + + // we are going to need the content-length + headers['User-Agent'] = 'Mozilla/5.0' + + // add user agent to avoid 403 const response = await headRequest(url, headers) if (!response) { throw Error(`Unable to HEAD ${url}`) @@ -222,10 +238,12 @@ const getSizeForRemoteFile = async (url: string): Promise => { export const setUrlFile = async (projectId: string, fileName: string, url: string, metadata: any, auth: Auth): Promise => { const reqUrl = `${apiBase}/api/gui/projects/${projectId}/files/${fileName}` - const size = await getSizeForRemoteFile(url) + + // the size is now computed on the server side + // const size = await getSizeForRemoteFile(url) const body = { content: `url:${url}`, - size, + // size, metadata } const response = await putRequest(reqUrl, body, auth) diff --git a/src/pages/ProjectPage/FileView/FileView.tsx b/src/pages/ProjectPage/FileView/FileView.tsx index 49b9e9c..184afa3 100644 --- a/src/pages/ProjectPage/FileView/FileView.tsx +++ b/src/pages/ProjectPage/FileView/FileView.tsx @@ -48,7 +48,7 @@ const FileViewChild: FunctionComponent = ({fileName, width, if (!files) return undefined return files.find(f => (f.fileName === fileName)) }, [files, fileName]) - if (fileName.endsWith('.nwb')) { + if ((fileName.endsWith('.nwb')) || (fileName.endsWith('.nwb.zarr.json'))) { return ( = ({fileName, additio URL: {theUrl} - {theFile && theFile.size && theFile.size < 50e6 && ( + {theFile && theFile.size && theFile.size < 50e6 ? ( <>   - )} + ) : ""} diff --git a/src/pages/ProjectPage/FileView/NwbFileView.tsx b/src/pages/ProjectPage/FileView/NwbFileView.tsx index d2d2a15..b8ef65d 100644 --- a/src/pages/ProjectPage/FileView/NwbFileView.tsx +++ b/src/pages/ProjectPage/FileView/NwbFileView.tsx @@ -198,9 +198,12 @@ const NwbFileView: FunctionComponent = ({fileName, width, height}) => { const dandiAssetPathEncoded = encodeURIComponent(metadata.dandiAssetPath) additionalQueryParams += `&dandiAssetPath=${dandiAssetPathEncoded}` } - const u = `https://flatironinstitute.github.io/neurosift/?p=/nwb&url=${nwbUrl}${additionalQueryParams}` + let u = `https://flatironinstitute.github.io/neurosift/?p=/nwb&url=${nwbUrl}${additionalQueryParams}` + if (fileName.endsWith('.json')) { + u += '&st=lindi' + } window.open(u, '_blank') - }, [nwbUrl, metadata]) + }, [nwbUrl, metadata, fileName]) useEffect(() => { if (!dandisetId) return diff --git a/src/pages/ProjectPage/scripts/RunScript/RunScriptWindow.tsx b/src/pages/ProjectPage/scripts/RunScript/RunScriptWindow.tsx index 2fb1c39..1edec6c 100644 --- a/src/pages/ProjectPage/scripts/RunScript/RunScriptWindow.tsx +++ b/src/pages/ProjectPage/scripts/RunScript/RunScriptWindow.tsx @@ -4,7 +4,7 @@ import { useProject } from "../../ProjectPageContext" import { RunScriptAddJob, RunScriptResult } from "./RunScriptWorkerTypes" import { Hyperlink } from "@fi-sci/misc" import { useGithubAuth } from "../../../../GithubAuth/useGithubAuth" -import { DendroProcessingJobDefinition, createJob } from "../../../../dbInterface/dbInterface" +import { DendroProcessingJobDefinition, createJob, setUrlFile } from "../../../../dbInterface/dbInterface" type RunScriptWindowProps = { width: number @@ -60,6 +60,18 @@ const RunScriptWindow: FunctionComponent = ({width, height return {newJobs, existingJobs} }, [jobs, result, projectJobHashes, computeResourceProcessorsByName]) + const {newFiles} = useMemo(() => { + if (!files) return {newFiles: undefined, existingFiles: undefined} + if (!result) return {newFiles: undefined, existingFiles: undefined} + const newFiles: {fileName: string, url: string}[] = [] + for (const f of result.addedFiles) { + if (!files.find(f1 => f1.fileName === f.fileName)) { + newFiles.push(f) + } + } + return {newFiles} + }, [files, result]) + const jobProblems = useMemo(() => { if (!result) return undefined if (!computeResourceProcessorsByName) return undefined @@ -81,9 +93,17 @@ const RunScriptWindow: FunctionComponent = ({width, height const auth = useGithubAuth() - const handleSubmitJobs = useCallback(async (jobsToSubmit: RunScriptAddJob[]) => { + const handleSubmit = useCallback(async () => { + const jobsToSubmit = newJobs + const filesToAdd = newFiles + if (!jobsToSubmit) return + if (!filesToAdd) return if (!auth.signedIn) return if (!files) return + for (const f of filesToAdd) { + const metaData = {} + await setUrlFile(projectId, f.fileName, f.url, metaData, auth) + } const batchId = createRandomId(8) for (const j of jobsToSubmit) { console.info(`Submitting job: ${j.processorName}`) @@ -115,8 +135,16 @@ const RunScriptWindow: FunctionComponent = ({width, height } refreshFiles() refreshJobs() - alert('Jobs submitted') - }, [auth, projectId, files, computeResourceProcessorsByName, refreshFiles, refreshJobs]) + if ((newFiles.length > 0) && (newJobs.length > 0)) { + alert('New files and jobs have been submitted') + } + else if (newFiles.length > 0) { + alert('New files have been submitted') + } + else if (newJobs.length > 0) { + alert('New jobs have been submitted') + } + }, [auth, projectId, files, computeResourceProcessorsByName, refreshFiles, refreshJobs, newJobs, newFiles]) return (
@@ -137,18 +165,18 @@ const RunScriptWindow: FunctionComponent = ({width, height {jobProblems.map((p, i) =>
{p}
)}
) : ( - newJobs && existingJobs && ( + newJobs && existingJobs && newFiles && (
-
{newJobs.length} new jobs ({newJobs.length + existingJobs.length} total)
- {newJobs.length > 0 && ( +
{newJobs.length} new jobs ({newJobs.length + existingJobs.length} total); {newFiles.length} new files
+ {newJobs.length + newFiles.length > 0 && ( saved ? (
{ - handleSubmitJobs(newJobs) + handleSubmit() }} > - {newJobs.length > 1 ? `Submit these ${newJobs.length} new jobs` : `Submit this one new job?`} + SUBMIT
) : ( diff --git a/src/pages/ProjectPage/scripts/RunScript/RunScriptWorkerTypes.ts b/src/pages/ProjectPage/scripts/RunScript/RunScriptWorkerTypes.ts index 96a6a32..c53a33b 100644 --- a/src/pages/ProjectPage/scripts/RunScript/RunScriptWorkerTypes.ts +++ b/src/pages/ProjectPage/scripts/RunScript/RunScriptWorkerTypes.ts @@ -32,8 +32,14 @@ export type RunScriptAddJob = { runMethod: 'local' | 'aws_batch' | 'slurm' } +export type RunScriptAddedFile = { + fileName: string + url: string +} + export type RunScriptResult = { jobs: RunScriptAddJob[] + addedFiles: RunScriptAddedFile[] } export {} \ No newline at end of file diff --git a/src/pages/ProjectPage/scripts/RunScript/runScriptWorker.ts b/src/pages/ProjectPage/scripts/RunScript/runScriptWorker.ts index d00c933..0cd26a2 100644 --- a/src/pages/ProjectPage/scripts/RunScript/runScriptWorker.ts +++ b/src/pages/ProjectPage/scripts/RunScript/runScriptWorker.ts @@ -7,7 +7,7 @@ import { } from "./RunScriptWorkerTypes"; const runScript = (_script: string, _files: any[]) => { - const result: RunScriptResult = { jobs: [] }; + const result: RunScriptResult = { jobs: [], addedFiles: []}; const _addJob = (a: { processorName: string; inputs: RunScriptAddJobInputFile[]; @@ -95,13 +95,22 @@ const runScript = (_script: string, _files: any[]) => { }); }; + const _addFile = (fileName: string, a: {url: string}) => { + result.addedFiles.push({fileName, url: a.url}); + } + const log: string[] = []; const context = { print: function (message: string) { + // make sure it is a string + if (typeof message !== "string") { + message = JSON.stringify(message); + } log.push(message); }, files: _files, addJob: _addJob, + addFile: _addFile }; const scriptFunction = new Function("context", _script); try { diff --git a/src/pages/ProjectsPage/ProjectsPage.css b/src/pages/ProjectsPage/ProjectsPage.css index 5fa18bf..b6d655f 100644 --- a/src/pages/ProjectsPage/ProjectsPage.css +++ b/src/pages/ProjectsPage/ProjectsPage.css @@ -1,13 +1,12 @@ .projects-page { - padding: 20px; + /* padding: 20px; */ font-family: Arial, sans-serif; max-width: 1400px; margin: auto; } -.projects-page h1, .projects-page h2 { +.projects-page h3 { color: #333; - text-align: center; } .projects-page p { diff --git a/src/pages/ProjectsPage/ProjectsPage.tsx b/src/pages/ProjectsPage/ProjectsPage.tsx index 8484f90..f12fe1e 100644 --- a/src/pages/ProjectsPage/ProjectsPage.tsx +++ b/src/pages/ProjectsPage/ProjectsPage.tsx @@ -30,8 +30,8 @@ const ProjectsPage: FunctionComponent = ({width, height}) => { return (
-

Your projects

-
+

Your projects

+
} onClick={handleAdd} label="Create a new project" />        } onClick={() => setRoute({page: 'dandisets'})} label="Import data from DANDI" /> diff --git a/src/pages/ProjectsPage/ProjectsTable.tsx b/src/pages/ProjectsPage/ProjectsTable.tsx index 7388dd8..317f7f0 100644 --- a/src/pages/ProjectsPage/ProjectsTable.tsx +++ b/src/pages/ProjectsPage/ProjectsTable.tsx @@ -14,6 +14,10 @@ const ProjectsTable: FunctionComponent = ({admin}) => { const { setRoute } = useRoute() + const sortedProjects = useMemo(() => { + return projects ? projects.sort((a, b) => b.timestampCreated - a.timestampCreated) : [] + }, [projects]) + if (!projects) return
Retrieving projects...
return ( @@ -30,7 +34,7 @@ const ProjectsTable: FunctionComponent = ({admin}) => { { - projects.map((pr) => ( + sortedProjects.map((pr) => ( setRoute({page: 'project', projectId: pr.projectId})}>