feat: add metadata and preview (#292)

* chore: upload flow uses metadata object and has preview

* chore: remove SwarmFile

* feat: upload metadata and file preview

* feat: add metadata and preview on download

* fix: package the meta and preview files

* fix: upload websites that are inside a folder (#296)

* fix: upload websites that are inside a folder

* docs: few comments to clarify what is going on

* refactor: decrease local variables and fix state order to detect websites properly

Co-authored-by: Cafe137 <aron@aronsoos.com>

src/constants.ts

@@ -0,0 +1,3 @@
+export const META_FILE_NAME = '.swarmgatewaymeta.json'
+export const PREVIEW_FILE_NAME = '.swarmgatewaypreview.jpeg'
+export const PREVIEW_DIMENSIONS = { maxWidth: 250, maxHeight: 175 }

src/pages/files/AssetPreview.tsx

@@ -1,99 +1,55 @@
 import { Box, Grid, Typography } from '@material-ui/core'
 import { Web } from '@material-ui/icons'
-import { ReactElement, useEffect, useState } from 'react'
+import { ReactElement } from 'react'
 import { File, Folder } from 'react-feather'
 import { FitImage } from '../../components/FitImage'
-import { detectIndexHtml, getAssetNameFromFiles, getHumanReadableFileSize } from '../../utils/file'
-import { SwarmFile } from '../../utils/SwarmFile'
+import { getHumanReadableFileSize } from '../../utils/file'
 import { AssetIcon } from './AssetIcon'
+import { shortenHash } from '../../utils/hash'
 
 interface Props {
-  assetName?: string
-  files: SwarmFile[]
+  previewUri?: string
+  metadata?: Metadata
 }
 
 // TODO: add optional prop for indexDocument when it is already known (e.g. downloading a manifest)
 
-export function AssetPreview({ assetName, files }: Props): ReactElement {
-  const [previewComponent, setPreviewComponent] = useState<ReactElement | undefined>(undefined)
-  const [previewUri, setPreviewUri] = useState<string | undefined>(undefined)
+export function AssetPreview({ metadata, previewUri }: Props): ReactElement | null {
+  let previewComponent = <File />
+  let type = metadata?.type
 
-  useEffect(() => {
-    if (files.length === 1) {
-      // single image
-      if (files[0].type.startsWith('image/')) {
-        files[0].arrayBuffer().then(value => {
-          const blob = new Blob([value])
-          setPreviewUri(URL.createObjectURL(blob))
-        })
-        // single non-image
-      } else {
-        setPreviewUri(undefined)
-        setPreviewComponent(<AssetIcon icon={<File />} />)
-      }
-      // collection
-    } else if (detectIndexHtml(files)) {
-      setPreviewUri(undefined)
-      setPreviewComponent(<AssetIcon icon={<Web />} />)
-    } else {
-      setPreviewUri(undefined)
-      setPreviewComponent(<AssetIcon icon={<Folder />} />)
-    }
-  }, [files])
-
-  const getPrimaryText = () => {
-    const name = getAssetNameFromFiles(files)
-
-    if (files.length === 1) {
-      return 'Filename: ' + (assetName || name)
-    }
-
-    return 'Folder name: ' + (assetName || name)
-  }
-
-  const getKind = () => {
-    if (files.length === 1) {
-      return files[0].type
-    }
-
-    if (detectIndexHtml(files)) {
-      return 'Website'
-    }
-
-    return 'Folder'
+  if (metadata?.isWebsite) {
+    previewComponent = <Web />
+    type = 'Website'
+  } else if (metadata?.type === 'folder') {
+    previewComponent = <Folder />
+    type = 'Folder'
   }
 
-  const isFolder = () => ['Folder', 'Website'].includes(getKind())
-
-  const getSize = () => {
-    const bytes = files.reduce((total, item) => total + item.size, 0)
-
-    return getHumanReadableFileSize(bytes)
-  }
-
-  const size = getSize()
-
   return (
     <Box mb={4}>
       <Box bgcolor="background.paper">
         <Grid container direction="row">
-          {previewComponent ? (
-            previewComponent
-          ) : (
+          {previewUri ? (
             <FitImage maxWidth="250px" maxHeight="175px" alt="Upload Preview" src={previewUri} />
+          ) : (
+            <AssetIcon icon={previewComponent} />
           )}
           <Box p={2}>
-            <Typography>{getPrimaryText()}</Typography>
-            <Typography>Kind: {getKind()}</Typography>
-            {size !== '0 bytes' && <Typography>Size: {size}</Typography>}
+            {metadata?.hash && <Typography>Swarm Hash: {shortenHash(metadata.hash)}</Typography>}
+            <Typography>
+              {metadata?.type === 'folder' ? 'Folder Name' : 'Filename'}: {metadata?.name}
+            </Typography>
+            <Typography>Kind: {type}</Typography>
+            {metadata?.size && <Typography>Size: {getHumanReadableFileSize(metadata.size)}</Typography>}
           </Box>
         </Grid>
       </Box>
-      {isFolder() && (
+      {metadata?.type === 'folder' && metadata.count && (
         <Box mt={0.25} p={2} bgcolor="background.paper">
           <Grid container justifyContent="space-between" alignItems="center" direction="row">
             <Typography variant="subtitle2">Folder content</Typography>
-            <Typography variant="subtitle2">{files.length} items</Typography>
+            <Typography variant="subtitle2">{metadata.count} items</Typography>
           </Grid>
         </Box>
       )}

src/pages/files/AssetSummary.tsx

@@ -4,21 +4,19 @@ import { ReactElement } from 'react'
 import { DocumentationText } from '../../components/DocumentationText'
 import ExpandableListItemKey from '../../components/ExpandableListItemKey'
 import ExpandableListItemLink from '../../components/ExpandableListItemLink'
-import { detectIndexHtml } from '../../utils/file'
-import { SwarmFile } from '../../utils/SwarmFile'
 
 interface Props {
-  files: SwarmFile[]
+  isWebsite?: boolean
   hash: string
 }
 
-export function AssetSummary({ files, hash }: Props): ReactElement {
+export function AssetSummary({ isWebsite, hash }: Props): ReactElement {
   return (
     <>
       <Box mb={4}>
         <ExpandableListItemKey label="Swarm hash" value={hash} />
         <ExpandableListItemLink label="Share on Swarm Gateway" value={`https://gateway.ethswarm.org/access/${hash}`} />
-        {detectIndexHtml(files) && (
+        {isWebsite && (
           <ExpandableListItemLink
             label="BZZ Link"
             value={`https://${swarmCid.encodeManifestReference(hash).toString()}.bzz.link`}

src/pages/files/Share.tsx

@@ -4,17 +4,16 @@ import { saveAs } from 'file-saver'
 import JSZip from 'jszip'
 import { useSnackbar } from 'notistack'
 import { ReactElement, useContext, useEffect, useState } from 'react'
-import { useParams, useNavigate } from 'react-router-dom'
+import { useNavigate, useParams } from 'react-router-dom'
 import { HistoryHeader } from '../../components/HistoryHeader'
 import { Loading } from '../../components/Loading'
 import TroubleshootConnectionCard from '../../components/TroubleshootConnectionCard'
+import config from '../../config'
+import { META_FILE_NAME, PREVIEW_FILE_NAME } from '../../constants'
 import { Context as BeeContext } from '../../providers/Bee'
 import { Context as SettingsContext } from '../../providers/Settings'
 import { ROUTES } from '../../routes'
-import { convertBeeFileToBrowserFile, convertManifestToFiles } from '../../utils/file'
-import { shortenHash } from '../../utils/hash'
 import { determineHistoryName, HISTORY_KEYS, putHistory } from '../../utils/local-storage'
-import { SwarmFile } from '../../utils/SwarmFile'
 import { AssetPreview } from './AssetPreview'
 import { AssetSummary } from './AssetSummary'
 import { DownloadActionBar } from './DownloadActionBar'
@@ -31,10 +30,11 @@ export function Share(): ReactElement {
 
   const [loading, setLoading] = useState(true)
   const [downloading, setDownloading] = useState(false)
-  const [files, setFiles] = useState<SwarmFile[]>([])
   const [swarmEntries, setSwarmEntries] = useState<Record<string, string>>({})
   const [indexDocument, setIndexDocument] = useState<string | null>(null)
   const [notFound, setNotFound] = useState(false)
+  const [preview, setPreview] = useState<string | undefined>(undefined)
+  const [metadata, setMetadata] = useState<Metadata | undefined>()
 
   async function prepare() {
     if (!beeApi || !status.all) {
@@ -51,16 +51,37 @@ export function Share(): ReactElement {
       return
     }
     const entries = await manifestJs.getHashes(reference)
-    setSwarmEntries(entries)
     const indexDocument = await manifestJs.getIndexDocumentPath(reference)
     setIndexDocument(indexDocument)
 
-    if (Object.keys(entries).length === 1) {
-      const response = await beeApi.downloadFile(reference)
-      setFiles([new SwarmFile(convertBeeFileToBrowserFile(response) as File)])
-    } else {
-      setFiles(convertManifestToFiles(entries))
+    const previewFile = entries[PREVIEW_FILE_NAME]
+
+    delete entries[META_FILE_NAME]
+    delete entries[PREVIEW_FILE_NAME]
+    setSwarmEntries(entries)
+
+    const count = Object.keys(entries).length
+
+    let metadata: Metadata | undefined = {
+      hash,
+      size: 0,
+      type: count > 1 ? 'folder' : 'unknown',
+      name: reference,
+      isWebsite: Boolean(indexDocument) && count > 1,
+      count,
     }
+
+    try {
+      const mtdt = await beeApi.downloadFile(reference, META_FILE_NAME)
+      const remoteMetadata = mtdt.data.text()
+      metadata = { ...metadata, ...(JSON.parse(remoteMetadata) as Metadata) }
+    } catch (e) {} // eslint-disable-line no-empty
+
+    if (previewFile) {
+      setPreview(`${config.BEE_API_HOST}/bzz/${reference}/${PREVIEW_FILE_NAME}`)
+    }
+
+    setMetadata(metadata)
   }
 
   function onOpen() {
@@ -109,8 +130,6 @@ export function Share(): ReactElement {
     setDownloading(false)
   }
 
-  const assetName = shortenHash(reference)
-
   if (!status.all) return <TroubleshootConnectionCard />
 
   if (loading) {
@@ -129,17 +148,17 @@ export function Share(): ReactElement {
   return (
     <>
       <Box mb={4}>
-        <AssetPreview files={files} assetName={assetName} />
+        <AssetPreview metadata={metadata} previewUri={preview} />
       </Box>
       <Box mb={4}>
-        <AssetSummary files={files} hash={reference} />
+        <AssetSummary isWebsite={metadata?.isWebsite} hash={reference} />
       </Box>
       <DownloadActionBar
         onOpen={onOpen}
         onCancel={onClose}
         onDownload={onDownload}
         onUpdateFeed={onUpdateFeed}
-        hasIndexDocument={Boolean(indexDocument && files.length > 1)}
+        hasIndexDocument={Boolean(metadata?.isWebsite)}
         loading={downloading}
       />
     </>

src/pages/files/Upload.tsx

@@ -12,7 +12,7 @@ import { Context as FileContext } from '../../providers/File'
 import { Context as SettingsContext } from '../../providers/Settings'
 import { Context as StampsContext, EnrichedPostageBatch } from '../../providers/Stamps'
 import { ROUTES } from '../../routes'
-import { detectIndexHtml, getAssetNameFromFiles } from '../../utils/file'
+import { detectIndexHtml, getAssetNameFromFiles, packageFile } from '../../utils/file'
 import { persistIdentity, updateFeed } from '../../utils/identity'
 import { HISTORY_KEYS, putHistory } from '../../utils/local-storage'
 import { FeedPasswordDialog } from '../feeds/FeedPasswordDialog'
@@ -21,6 +21,7 @@ import { PostageStampSelector } from '../stamps/PostageStampSelector'
 import { AssetPreview } from './AssetPreview'
 import { StampPreview } from './StampPreview'
 import { UploadActionBar } from './UploadActionBar'
+import { META_FILE_NAME, PREVIEW_FILE_NAME } from '../../constants'
 
 export function Upload(): ReactElement {
   const [step, setStep] = useState(0)
@@ -31,7 +32,7 @@ export function Upload(): ReactElement {
 
   const { refresh } = useContext(StampsContext)
   const { beeApi } = useContext(SettingsContext)
-  const { files, setFiles, uploadOrigin } = useContext(FileContext)
+  const { files, setFiles, uploadOrigin, metadata, previewUri, previewBlob } = useContext(FileContext)
   const { identities, setIdentities } = useContext(IdentityContext)
   const { status } = useContext(BeeContext)
 
@@ -66,16 +67,63 @@ export function Upload(): ReactElement {
   }
 
   const uploadFiles = (password?: string) => {
-    if (!beeApi || !files.length || !stamp) {
+    if (!beeApi || !files.length || !stamp || !metadata) {
       return
     }
 
-    const indexDocument = files.length === 1 ? files[0].name : detectIndexHtml(files) || undefined
+    let fls = files.map(packageFile) // Apart from packaging, this is needed to not modify the original files array as it can trigger effects
+    let indexDocument: string | undefined = undefined // This means we assume it's folder
+
+    if (files.length === 1) indexDocument = files[0].name
+    else if (files.length > 1) {
+      const idx = detectIndexHtml(files)
+
+      // This is a website
+      if (idx) {
+        // The website is in some directory, remove it
+        if (idx.commonPrefix) {
+          const substrStart = idx.commonPrefix.length
+          indexDocument = idx.indexPath.substr(substrStart)
+          fls = fls.map(f => {
+            const path = (f.path as string).substr(substrStart)
+
+            return { ...f, path, webkitRelativePath: path, fullPath: path }
+          })
+        } else {
+          // The website is not packed in a directory
+          indexDocument = idx.indexPath
+        }
+      }
+    }
+    const lastModified = files[0].lastModified
+
+    // We want to store only some metadata
+    const mtd: SwarmMetadata = {
+      name: metadata.name,
+      size: metadata.size,
+    }
+
+    // Type of the file only makes sense for a single file
+    if (files.length === 1) mtd.type = metadata.type
+
+    const metafile = new File([JSON.stringify(mtd)], META_FILE_NAME, {
+      type: 'application/json',
+      lastModified,
+    })
+    fls.push(packageFile(metafile))
+
+    if (previewBlob) {
+      const previewFile = new File([previewBlob], PREVIEW_FILE_NAME, {
+        type: 'image/jpeg',
+        lastModified,
+      })
+      fls.push(packageFile(previewFile))
+    }
 
     setUploading(true)
 
     beeApi
-      .uploadFiles(stamp.batchID, files as unknown as File[], { indexDocument })
+      .uploadFiles(stamp.batchID, fls, { indexDocument })
       .then(hash => {
         putHistory(HISTORY_KEYS.UPLOAD_HISTORY, hash.reference, getAssetNameFromFiles(files))
 
@@ -121,7 +169,7 @@ export function Upload(): ReactElement {
       <Box mb={4}>
         <ProgressIndicator steps={['Preview', 'Add postage stamp', 'Upload to node']} index={step} />
       </Box>
-      {(step === 0 || step === 2) && <AssetPreview files={files} />}
+      {(step === 0 || step === 2) && <AssetPreview metadata={metadata} previewUri={previewUri} />}
       {step === 1 && (
         <>
           <Box mb={2}>

src/pages/files/UploadArea.tsx

@@ -9,7 +9,6 @@ import { SwarmButton } from '../../components/SwarmButton'
 import { Context, UploadOrigin } from '../../providers/File'
 import { ROUTES } from '../../routes'
 import { detectIndexHtml } from '../../utils/file'
-import { SwarmFile } from '../../utils/SwarmFile'
 
 interface Props {
   uploadOrigin: UploadOrigin
@@ -99,8 +98,8 @@ export function UploadArea({ uploadOrigin, showHelp }: Props): ReactElement {
 
   const handleChange = (files?: File[]) => {
     if (files) {
-      const swarmFiles = files.map(x => new SwarmFile(x))
-      const indexDocument = files.length === 1 ? files[0].name : detectIndexHtml(swarmFiles) || undefined
+      const FilePaths = files as FilePath[]
+      const indexDocument = files.length === 1 ? files[0].name : detectIndexHtml(FilePaths) || undefined
 
       if (files.length && strictWebsiteMode && !indexDocument) {
         enqueueSnackbar('To upload a website, there must be an index.html or index.htm in the root of the folder.', {
@@ -111,7 +110,7 @@ export function UploadArea({ uploadOrigin, showHelp }: Props): ReactElement {
         return
       }
 
-      setFiles(swarmFiles)
+      setFiles(FilePaths)
 
       if (files.length) {
         setUploadOrigin(uploadOrigin)