Skip to content

Commit

Permalink
infer s3 bucket region by issuing headBucket request (#1594)
Browse files Browse the repository at this point in the history
  • Loading branch information
nl0 committed Apr 13, 2020
1 parent d7542a7 commit dcdcdd4
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 66 deletions.
2 changes: 2 additions & 0 deletions catalog/app/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import * as Intercom from 'components/Intercom'
import Layout from 'components/Layout'
import Placeholder from 'components/Placeholder'
import App from 'containers/App'
import { BucketCacheProvider } from 'containers/Bucket'
import LanguageProvider from 'containers/LanguageProvider'
import * as Auth from 'containers/Auth'
import * as Notifications from 'containers/Notifications'
Expand Down Expand Up @@ -138,6 +139,7 @@ const render = (messages) => {
AWSSigner.Provider,
Notifications.WithNotifications,
ErrorBoundary,
BucketCacheProvider,
App,
),
MOUNT_NODE,
Expand Down
59 changes: 40 additions & 19 deletions catalog/app/containers/Bucket/Bucket.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,25 @@ import Layout from 'components/Layout'
import Placeholder from 'components/Placeholder'
import { ThrowNotFound } from 'containers/NotFoundPage'
import * as AWS from 'utils/AWS'
import { useCurrentBucketConfig } from 'utils/BucketConfig'
import AsyncResult from 'utils/AsyncResult'
import Data from 'utils/Data'
import * as NamedRoutes from 'utils/NamedRoutes'
import * as RT from 'utils/reactTools'

import { displayError } from './errors'
import * as requests from './requests'

const CacheCtx = React.createContext()

export function BucketCacheProvider({ children }) {
const ref = React.useRef({})
return <CacheCtx.Provider value={ref.current}>{children}</CacheCtx.Provider>
}

function useBucketCache() {
return React.useContext(CacheCtx)
}

const mkLazy = (load) =>
RT.loadable(load, { fallback: () => <Placeholder color="text.secondary" /> })

Expand Down Expand Up @@ -74,6 +89,8 @@ const useStyles = M.makeStyles((t) => ({
function BucketLayout({ bucket, section = false, children }) {
const { urls } = NamedRoutes.use()
const classes = useStyles()
const s3req = AWS.S3.useRequest()
const cache = useBucketCache()
return (
<Layout
pre={
Expand All @@ -96,7 +113,15 @@ function BucketLayout({ bucket, section = false, children }) {
)}
</M.Tabs>
</M.AppBar>
<M.Container maxWidth="lg">{children}</M.Container>
<M.Container maxWidth="lg">
<Data fetch={requests.bucketExists} params={{ s3req, bucket, cache }}>
{AsyncResult.case({
Ok: () => children,
Err: displayError(),
_: () => <Placeholder color="text.secondary" />,
})}
</Data>
</M.Container>
</>
}
/>
Expand All @@ -110,23 +135,19 @@ export default ({
},
}) => {
const { paths } = NamedRoutes.use()
const bucketCfg = useCurrentBucketConfig()
const s3Props = bucketCfg && bucketCfg.region && { region: bucketCfg.region }
return (
<AWS.S3.Provider {...s3Props}>
<BucketLayout bucket={bucket} section={getBucketSection(paths)(location.pathname)}>
<Switch>
<Route path={paths.bucketFile} component={File} exact strict />
<Route path={paths.bucketDir} component={Dir} exact />
<Route path={paths.bucketOverview} component={Overview} exact />
<Route path={paths.bucketSearch} component={Search} exact />
<Route path={paths.bucketPackageList} component={PackageList} exact />
<Route path={paths.bucketPackageDetail} component={PackageTree} exact />
<Route path={paths.bucketPackageTree} component={PackageTree} exact />
<Route path={paths.bucketPackageRevisions} component={PackageRevisions} exact />
<Route component={ThrowNotFound} />
</Switch>
</BucketLayout>
</AWS.S3.Provider>
<BucketLayout bucket={bucket} section={getBucketSection(paths)(location.pathname)}>
<Switch>
<Route path={paths.bucketFile} component={File} exact strict />
<Route path={paths.bucketDir} component={Dir} exact />
<Route path={paths.bucketOverview} component={Overview} exact />
<Route path={paths.bucketSearch} component={Search} exact />
<Route path={paths.bucketPackageList} component={PackageList} exact />
<Route path={paths.bucketPackageDetail} component={PackageTree} exact />
<Route path={paths.bucketPackageTree} component={PackageTree} exact />
<Route path={paths.bucketPackageRevisions} component={PackageRevisions} exact />
<Route component={ThrowNotFound} />
</Switch>
</BucketLayout>
)
}
54 changes: 22 additions & 32 deletions catalog/app/containers/Bucket/Overview.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import useComponentSize from '@rehooks/component-size'

import { copyWithoutSpaces } from 'components/BreadCrumbs'
import * as Pagination from 'components/Pagination'
import Placeholder from 'components/Placeholder'
import * as Preview from 'components/Preview'
import Skeleton from 'components/Skeleton'
import StackedAreaChart from 'components/StackedAreaChart'
Expand All @@ -25,7 +24,6 @@ import Link from 'utils/StyledLink'
import { getBreadCrumbs } from 'utils/s3paths'
import { readableBytes, readableQuantity } from 'utils/string'

import { displayError } from './errors'
import * as requests from './requests'

import bg from './Overview-bg.jpg'
Expand Down Expand Up @@ -1160,35 +1158,27 @@ export default function Overview({
const overviewUrl = cfg && cfg.overviewUrl
const description = cfg && cfg.description
return (
<Data fetch={requests.bucketExists} params={{ s3req, bucket }}>
{AsyncResult.case({
Ok: () => (
<M.Box pb={{ xs: 0, sm: 4 }} mx={{ xs: -2, sm: 0 }}>
{!!cfg && (
<React.Suspense fallback={null}>
<LinkedData.BucketData bucket={cfg} />
</React.Suspense>
)}
{cfg ? (
<Head {...{ es, s3req, bucket, overviewUrl, description }} />
) : (
<M.Box
pt={2}
pb={{ xs: 2, sm: 0 }}
px={{ xs: 2, sm: 0 }}
textAlign={{ xs: 'center', sm: 'left' }}
>
<M.Typography variant="h5">{bucket}</M.Typography>
</M.Box>
)}
<Readmes {...{ s3req, bucket, overviewUrl }} />
<Imgs {...{ es, s3req, bucket, inStack, overviewUrl }} />
<Summary {...{ es, s3req, bucket, inStack, overviewUrl }} />
</M.Box>
),
Err: displayError(),
_: () => <Placeholder color="text.secondary" />,
})}
</Data>
<M.Box pb={{ xs: 0, sm: 4 }} mx={{ xs: -2, sm: 0 }}>
{!!cfg && (
<React.Suspense fallback={null}>
<LinkedData.BucketData bucket={cfg} />
</React.Suspense>
)}
{cfg ? (
<Head {...{ es, s3req, bucket, overviewUrl, description }} />
) : (
<M.Box
pt={2}
pb={{ xs: 2, sm: 0 }}
px={{ xs: 2, sm: 0 }}
textAlign={{ xs: 'center', sm: 'left' }}
>
<M.Typography variant="h5">{bucket}</M.Typography>
</M.Box>
)}
<Readmes {...{ s3req, bucket, overviewUrl }} />
<Imgs {...{ es, s3req, bucket, inStack, overviewUrl }} />
<Summary {...{ es, s3req, bucket, inStack, overviewUrl }} />
</M.Box>
)
}
3 changes: 2 additions & 1 deletion catalog/app/containers/Bucket/index.js
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
export default from './Bucket'
export * from './Bucket'
export { default } from './Bucket'
31 changes: 20 additions & 11 deletions catalog/app/containers/Bucket/requests.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { join as pathJoin } from 'path'

import S3 from 'aws-sdk/clients/s3'
import * as dateFns from 'date-fns'
import sampleSize from 'lodash/fp/sampleSize'
import * as R from 'ramda'
Expand Down Expand Up @@ -199,17 +200,25 @@ export const bucketAccessCounts = async ({

const parseDate = (d) => d && new Date(d)

export const bucketExists = ({ s3req, bucket }) =>
s3req({ bucket, operation: 'headBucket', params: { Bucket: bucket } }).catch(
catchErrors([
[
R.propEq('code', 'NotFound'),
() => {
throw new errors.NoSuchBucket()
},
],
]),
)
export function bucketExists({ s3req, bucket, cache }) {
if (S3.prototype.bucketRegionCache[bucket]) return Promise.resolve()
if (cache && cache[bucket]) return Promise.resolve()
return s3req({ bucket, operation: 'headBucket', params: { Bucket: bucket } })
.then(() => {
// eslint-disable-next-line no-param-reassign
if (cache) cache[bucket] = true
})
.catch(
catchErrors([
[
R.propEq('code', 'NotFound'),
() => {
throw new errors.NoSuchBucket()
},
],
]),
)
}

const getOverviewBucket = (url) => parseS3Url(url).bucket
const getOverviewPrefix = (url) => parseS3Url(url).key
Expand Down
9 changes: 6 additions & 3 deletions catalog/app/utils/AWS/S3.js
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,17 @@ export const useRequest = (extra) => {
...extra,
})
return React.useCallback(
({ bucket, operation, params }) => {
({ operation, params }) => {
let type = 'unsigned'
if (cfg.mode === 'LOCAL') {
type = 'signed'
} else if (authenticated) {
if (
(cfg.analyticsBucket && cfg.analyticsBucket === bucket) ||
(cfg.mode !== 'OPEN' && isInStack(bucket))
// sign if operation is not bucket-specific
// (not sure if there are any such operations that can be used from the browser)
!params.Bucket ||
(cfg.analyticsBucket && cfg.analyticsBucket === params.Bucket) ||
(cfg.mode !== 'OPEN' && isInStack(params.Bucket))
) {
type = 'signed'
}
Expand Down

0 comments on commit dcdcdd4

Please sign in to comment.