Skip to content

Commit

Permalink
Ability to upload and download from Google Cloud Storage (#1226)
Browse files Browse the repository at this point in the history 
Fixes #1147
  • Loading branch information
Peter Bengtsson committed Oct 16, 2018
1 parent 6661de1 commit 90071f0
Show file tree
Hide file tree
Showing 20 changed files with 1,768 additions and 463 deletions.
1 change: 1 addition & 0 deletions .circleci/config.yml
View file
Expand Up @@ -29,6 +29,7 @@ jobs:
command: |
docker info
cp .env-dist .env
touch google_service_account.json
# build tecken containers
make build
Expand Down
2 changes: 2 additions & 0 deletions .env-dist
View file
Expand Up @@ -19,3 +19,5 @@ AWS_SECRET_ACCESS_KEY=miniostorage
# Get these settings from Auth0 or some provider you use.
DJANGO_OIDC_RP_CLIENT_ID=mustbesomething
DJANGO_OIDC_RP_CLIENT_SECRET=mustbesomething

#GOOGLE_APPLICATION_CREDENTIALS_PATH=google_service_account.json
1 change: 1 addition & 0 deletions .gitignore
View file
Expand Up @@ -29,3 +29,4 @@ miniodata/
.pytest_cache/
.cache/
docsvenv
google_service_account.json
186 changes: 100 additions & 86 deletions frontend/src/Common.js
View file
Expand Up @@ -226,97 +226,111 @@ export const filterToQueryString = (filterObj, overrides) => {

const URLTag = url => <span className="url">{url}</span>;

export const ShowUploadMetadata = ({ upload }) => (
<table className="table is-fullwidth">
<tbody>
<tr>
<th>User</th>
<td>{upload.user.email}</td>
</tr>
<tr>
<th>Size</th>
<td>{formatFileSize(upload.size)}</td>
</tr>
<tr>
<th>Filename</th>
<td>{upload.filename}</td>
</tr>
<tr>
<th>Try Symbols</th>
<td>{upload.try_symbols ? "Yes" : "No"}</td>
</tr>
<tr>
<th>Download URL</th>
<td>
{upload.download_url ? URLTag(upload.download_url) : <i>null</i>}
</td>
</tr>
{upload.download_url && (
export const ShowUploadMetadata = ({ upload }) => {
return (
<table className="table is-fullwidth">
<tbody>
<tr>
<th>User</th>
<td>{upload.user.email}</td>
</tr>
<tr>
<th>Size</th>
<td>{formatFileSize(upload.size)}</td>
</tr>
<tr>
<th>Filename</th>
<td>{upload.filename}</td>
</tr>
<tr>
<th>Try Symbols</th>
<td>{upload.try_symbols ? "Yes" : "No"}</td>
</tr>
<tr>
<th>Download URL</th>
<td>
{upload.download_url ? URLTag(upload.download_url) : <i>null</i>}
</td>
</tr>
{upload.download_url && (
<tr>
<th>Redirect URLs</th>
<td>
{!upload.redirect_urls.length && <i>n/a</i>}
{upload.redirect_urls.length ? (
<ol start="0" className="redirect-urls">
<li>{URLTag(upload.download_url)}</li>
{upload.redirect_urls.map(url => (
<li key={url}>{URLTag(url)}</li>
))}
</ol>
) : null}
</td>
</tr>
)}
<tr>
<th>Bucket Name</th>
<td>{upload.bucket_name}</td>
</tr>
<tr>
<th>Bucket Region</th>
<td>{upload.bucket_region ? upload.bucket_region : <i>null</i>}</td>
</tr>
<tr>
<th>Bucket Endpoint URL</th>
<td>
{upload.bucket_endpoint_url ? (
upload.bucket_endpoint_url
) : (
<i>null</i>
)}
</td>
</tr>
<tr>
<th>Uploaded</th>
<td>
<DisplayDate date={upload.created_at} />
</td>
</tr>
<tr>
<th>Redirect URLs</th>
<th title="Time when its content was fully processed and uploaded, skipped or ignored">
Completed
</th>
<td>
{!upload.redirect_urls.length && <i>n/a</i>}
{upload.redirect_urls.length ? (
<ol start="0" className="redirect-urls">
<li>{URLTag(upload.download_url)}</li>
{upload.redirect_urls.map(url => (
<li key={url}>{URLTag(url)}</li>
))}
</ol>
{upload.completed_at ? (
<DisplayDate date={upload.completed_at} />
) : (
<i>
Incomplete!{" "}
<DisplayIncompleteRatio files={upload.file_uploads} />
</i>
)}
{upload.completed_at ? (
<small>
{" "}
(took{" "}
<DisplayDateDifference
from={upload.created_at}
to={upload.completed_at}
/>
)
</small>
) : null}
</td>
</tr>
)}
<tr>
<th>Bucket Name</th>
<td>{upload.bucket_name}</td>
</tr>
<tr>
<th>Bucket Region</th>
<td>{upload.bucket_region ? upload.bucket_region : <i>null</i>}</td>
</tr>
<tr>
<th>Bucket Endpoint URL</th>
<td>
{upload.bucket_endpoint_url ? (
upload.bucket_endpoint_url
) : (
<i>null</i>
)}
</td>
</tr>
<tr>
<th>Uploaded</th>
<td>
<DisplayDate date={upload.created_at} />
</td>
</tr>
<tr>
<th title="Time when its content was fully processed and uploaded, skipped or ignored">
Completed
</th>
<td>
{upload.completed_at ? (
<DisplayDate date={upload.completed_at} />
) : (
<i>Incomplete!</i>
)}
{upload.completed_at ? (
<small>
{" "}
(took{" "}
<DisplayDateDifference
from={upload.created_at}
to={upload.completed_at}
/>
)
</small>
) : null}
</td>
</tr>
</tbody>
</table>
);
</tbody>
</table>
);
};

const DisplayIncompleteRatio = ({ files }) => {
// Return a string like "56 of 103 (54%)" to mean that 56 of 103 files are complete.
const all = files.length;
const incomplete = files.filter(file => !!file.completed_at).length;
const complete = all - incomplete;
const percentage = (100 * complete) / all;
return `${complete} of ${all} (${Math.trunc(percentage)}%)`;
};

export const ShowFileMetadata = ({ file }) => (
<table className="table is-fullwidth">
Expand Down
50 changes: 50 additions & 0 deletions requirements-constraints.txt
View file
Expand Up @@ -143,3 +143,53 @@ amqp==2.3.2 \
kombu==4.2.1 \
--hash=sha256:86adec6c60f63124e2082ea8481bbe4ebe04fde8ebed32c177c7f0cd2c1c9082 \
--hash=sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af
google-cloud-core==0.28.1 \
--hash=sha256:0090df83dbc5cb2405fa90844366d13176d1c0b48181c1807ab15f53be403f73 \
--hash=sha256:89e8140a288acec20c5e56159461d3afa4073570c9758c05d4e6cb7f2f8cc440
google-resumable-media==0.3.1 \
--hash=sha256:116de90b9cd483b17c53618ee6a5a20f33e741c648140c8cc9c2141e07616ff1 \
--hash=sha256:97de518f8166d442cc0b61fab308bcd319dbb970981e667ec8ded44f5ce49836
google-api-core==1.4.1 \
--hash=sha256:67fcea8865fae31c42228bf227c90ad293b5c519a3c6fd8d8c9850bebaef59f2 \
--hash=sha256:a9ae625afd0ea5a4618604675d1fc140998c9c2b17f1d91817a7a7f5b33f7484
protobuf==3.6.1 \
--hash=sha256:10394a4d03af7060fa8a6e1cbf38cea44be1467053b0aea5bbfcb4b13c4b88c4 \
--hash=sha256:1489b376b0f364bcc6f89519718c057eb191d7ad6f1b395ffd93d1aa45587811 \
--hash=sha256:1931d8efce896981fe410c802fd66df14f9f429c32a72dd9cfeeac9815ec6444 \
--hash=sha256:196d3a80f93c537f27d2a19a4fafb826fb4c331b0b99110f985119391d170f96 \
--hash=sha256:46e34fdcc2b1f2620172d3a4885128705a4e658b9b62355ae5e98f9ea19f42c2 \
--hash=sha256:59cd75ded98094d3cf2d79e84cdb38a46e33e7441b2826f3838dcc7c07f82995 \
--hash=sha256:5ee0522eed6680bb5bac5b6d738f7b0923b3cafce8c4b1a039a6107f0841d7ed \
--hash=sha256:65917cfd5da9dfc993d5684643063318a2e875f798047911a9dd71ca066641c9 \
--hash=sha256:685bc4ec61a50f7360c9fd18e277b65db90105adbf9c79938bd315435e526b90 \
--hash=sha256:92e8418976e52201364a3174e40dc31f5fd8c147186d72380cbda54e0464ee19 \
--hash=sha256:9335f79d1940dfb9bcaf8ec881fb8ab47d7a2c721fb8b02949aab8bbf8b68625 \
--hash=sha256:a7ee3bb6de78185e5411487bef8bc1c59ebd97e47713cba3c460ef44e99b3db9 \
--hash=sha256:ceec283da2323e2431c49de58f80e1718986b79be59c266bb0509cbf90ca5b9e \
--hash=sha256:e7a5ccf56444211d79e3204b05087c1460c212a2c7d62f948b996660d0165d68 \
--hash=sha256:fcfc907746ec22716f05ea96b7f41597dfe1a1c088f861efb8a0d4f4196a6f10
googleapis-common-protos==1.5.3 \
--hash=sha256:c075eddaa2628ab519e01b7d75b76e66c40eaa50fc52758d8225f84708950ef2
google-auth==1.5.1 \
--hash=sha256:9ca363facbf2622d9ba828017536ccca2e0f58bd15e659b52f312172f8815530 \
--hash=sha256:a4cf9e803f2176b5de442763bd339b313d3f1ed3002e3e1eb6eec1d7c9bbc9b4
rsa==4.0 \
--hash=sha256:14ba45700ff1ec9eeb206a2ce76b32814958a98e372006c8fb76ba820211be66 \
--hash=sha256:1a836406405730121ae9823e19c6e806c62bbad73f890574fff50efa4122c487
cachetools==2.1.0 \
--hash=sha256:90f1d559512fc073483fe573ef5ceb39bf6ad3d39edc98dc55178a2b2b176fa3 \
--hash=sha256:d1c398969c478d336f767ba02040fa22617333293fb0b8968e79b16028dfee35
pyasn1-modules==0.2.2 \
--hash=sha256:077250b34432520430bc1c80dcbda4e354090785567c33ded35faa6df8d24753 \
--hash=sha256:0da2f947e8ad2697e86fe5fd0e55a4093a2fd79d839c9e19c34e28097db7002c \
--hash=sha256:35ff894a0b5df8e28b700126b2869c7dcfb2b2db5bc82e5d5e82547069241553 \
--hash=sha256:44688b94841349648b1e1a5a7a3d96e6596d5d4f21d0b59a82307e153c4dc74b \
--hash=sha256:833716dde880a7f2f2ccdeea9a096842626981ff2a477d8b318c0906367ac11b \
--hash=sha256:a0cf3e1842e7c60fde97cb22d275eb6f9524f5c5250489e292529de841417547 \
--hash=sha256:a38a8811ea784c0136abfdba73963876328f66172db21a05a82f9515909bfb4e \
--hash=sha256:a728bb9502d1fdc104c66f24a176b6a70a32e89d1d8a5b55c959233ed51c67be \
--hash=sha256:c30a098435ea0989c37005a971843e9d3966c7f6d056ddbf052e5061c06e3291 \
--hash=sha256:c355a45b32c5bc1d9893eceb704b0cfcd1126f91b5a7b9ee64c1c05383283381 \
--hash=sha256:e64679de1940f41ead5170fce364d54e7b9e2e862f064727b6bcb5cee753b7a2 \
--hash=sha256:ed71d20225c356881c29f0b1d7a0d6521563a389d9478e8f95d798cc5ba07b88 \
--hash=sha256:f183f0940b9f5ed2ad9d04c80cab2451440fa9af4fc959d85113fadd2e777962
3 changes: 3 additions & 0 deletions requirements.txt
View file
Expand Up @@ -261,3 +261,6 @@ pyflakes==1.6.0 \
black==18.9b0 \
--hash=sha256:817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739 \
--hash=sha256:e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5
google-cloud-storage==1.13.0 \
--hash=sha256:936c859c47f8e94fd0005e98235a10d5e75828d2c6c3a8caacae18344a572a0a \
--hash=sha256:fc32b9be41a45016ba2387e3ad23e70ccba399d626ef596409316f7cee477956

0 comments on commit 90071f0

Please sign in to comment.