Skip to content

Commit

Permalink
feat(core): add cloudflare images & stream (#4606)
Browse files Browse the repository at this point in the history 
* add cloudflare images
* add cloudflare stream
* add video.js video player
  • Loading branch information
@soyombo-baterdene
soyombo-baterdene committed Sep 5, 2023
1 parent c921114 commit 0669abf
Show file tree
Hide file tree
Showing 10 changed files with 654 additions and 63 deletions.
36 changes: 30 additions & 6 deletions packages/core-ui/src/modules/settings/general/components/GeneralSettings.tsx
View file
Expand Up @@ -221,6 +221,35 @@ class GeneralSettings extends React.Component<Props, State> {
);
}

renderCloudflare() {
const { configsMap } = this.state;

return (
<CollapseContent
title={__('Cloudflare')}
description={__('Cloudflare R2 Bucket, Images & Stream CDN configs')}
>
{this.renderItem('CLOUDFLARE_ACCOUNT_ID')}
{this.renderItem('CLOUDFLARE_API_TOKEN')}
{this.renderItem('CLOUDFLARE_ACCESS_KEY_ID')}
{this.renderItem('CLOUDFLARE_SECRET_ACCESS_KEY')}
{this.renderItem('CLOUDFLARE_BUCKET_NAME')}
{this.renderItem('CLOUDFLARE_ACCOUNT_HASH')}
<FormGroup>
<ControlLabel>{KEY_LABELS.CLOUDFLARE_USE_CDN}</ControlLabel>
<p>{__('Upload images/videos to Cloudflare cdn')}</p>
<FormControl
componentClass={'checkbox'}
checked={configsMap.CLOUDFLARE_USE_CDN}
onChange={(e: any) => {
this.onChangeConfig('CLOUDFLARE_USE_CDN', e.target.checked);
}}
/>
</FormGroup>
</CollapseContent>
);
}

render() {
const { configsMap, language } = this.state;

Expand Down Expand Up @@ -404,12 +433,7 @@ class GeneralSettings extends React.Component<Props, State> {
</FormGroup>
</CollapseContent>

<CollapseContent title={__('Cloudflare R2')}>
{this.renderItem('CLOUDFLARE_ACCESS_KEY_ID')}
{this.renderItem('CLOUDFLARE_SECRET_ACCESS_KEY')}
{this.renderItem('CLOUDFLARE_ACCOUNT_ID')}
{this.renderItem('CLOUDFLARE_BUCKET_NAME')}
</CollapseContent>
{this.renderCloudflare()}

<CollapseContent title="AWS S3">
<Info>
Expand Down
3 changes: 2 additions & 1 deletion packages/core/package.json
View file
Expand Up @@ -63,6 +63,7 @@
"ts-node": "^10.7.0",
"validator": "^9.0.0",
"xlsx-populate": "^1.20.1",
"jimp":"^0.22.10"
"jimp":"^0.22.10",
"tus-js-client": "^3.1.1"
}
}
230 changes: 196 additions & 34 deletions packages/core/src/data/utils.ts
View file
Expand Up @@ -10,7 +10,9 @@ import * as jimp from 'jimp';
import * as nodemailer from 'nodemailer';
import * as path from 'path';
import * as xlsxPopulate from 'xlsx-populate';

import * as FormData from 'form-data';
// import * as tus from 'tus-js-client';
import fetch from 'node-fetch';
import { IModels } from '../connectionResolver';
import { IUserDocument } from '../db/models/definitions/users';
import { debugBase, debugError } from '../debuggers';
Expand Down Expand Up @@ -449,7 +451,7 @@ const createCFR2 = async (models?: IModels) => {
const CLOUDFLARE_ENDPOINT = `https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`;

if (!CLOUDFLARE_ACCESS_KEY_ID || !CLOUDFLARE_SECRET_ACCESS_KEY) {
throw new Error('Cloudflare R2 Credentials are not configured');
throw new Error('Cloudflare Credentials are not configured');
}

const options: {
Expand All @@ -467,8 +469,95 @@ const createCFR2 = async (models?: IModels) => {
return new AWS.S3(options);
};

const uploadToCFImages = async (file: any, models?: IModels) => {
const CLOUDFLARE_ACCOUNT_ID = await getConfig(
'CLOUDFLARE_ACCOUNT_ID',
'',
models
);

const CLOUDFLARE_API_TOKEN = await getConfig(
'CLOUDFLARE_API_TOKEN',
'',
models
);

// const IS_PUBLIC = forcePrivate
// ? false
// : await getConfig('FILE_SYSTEM_PUBLIC', 'true', models);

const url = `https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/images/v1`;
const headers = {
Authorization: `Bearer ${CLOUDFLARE_API_TOKEN}`
};

const fileName = `${Math.random()}${file.name.replace(/ /g, '')}`;

const formData = new FormData();
formData.append('file', fs.createReadStream(file.path));
formData.append('id', fileName);

const response = await fetch(url, {
method: 'POST',
headers,
body: formData
});

const data = await response.json();

if (!data.success) {
throw new Error('Error uploading file to Cloudflare Images');
}

if (data.result.variants.length === 0) {
throw new Error('Error uploading file to Cloudflare Images');
}

return data.result.variants[0];
};

// upload file to Cloudflare stream
const uploadToCFStream = async (file: any, models?: IModels) => {
const CLOUDFLARE_ACCOUNT_ID = await getConfig(
'CLOUDFLARE_ACCOUNT_ID',
'',
models
);

const CLOUDFLARE_API_TOKEN = await getConfig(
'CLOUDFLARE_API_TOKEN',
'',
models
);

const url = `https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/stream`;
const headers = {
Authorization: `Bearer ${CLOUDFLARE_API_TOKEN}`
};

const fileName = `${Math.random()}${file.name.replace(/ /g, '')}`;

const formData = new FormData();
formData.append('file', fs.createReadStream(file.path));
formData.append('id', fileName);

const response = await fetch(url, {
method: 'POST',
headers,
body: formData
});

const data = await response.json();

if (!data.success) {
throw new Error('Error uploading file to Cloudflare Stream');
}

return data.result.playback.hls;
};

/*
* Save file to Cloudflare r2
* Save file to Cloudflare
*/

export const uploadFileCloudflare = async (
Expand All @@ -481,19 +570,43 @@ export const uploadFileCloudflare = async (
'',
models
);

const CLOUDFLARE_USE_CDN = await getConfig(
'CLOUDFLARE_USE_CDN',
'false',
models
);

const detectedType = fileType(fs.readFileSync(file.path));

if (
(CLOUDFLARE_USE_CDN === 'true' || CLOUDFLARE_USE_CDN === true) &&
detectedType &&
isImage(detectedType.mime)
) {
return uploadToCFImages(file, models);
}

if (
CLOUDFLARE_USE_CDN === 'true' ||
(CLOUDFLARE_USE_CDN === true && detectedType && isVideo(detectedType.mime))
) {
return uploadToCFStream(file, models);
}

const IS_PUBLIC = forcePrivate
? false
: await getConfig('FILE_SYSTEM_PUBLIC', 'true', models);

// initialize r2
const r2 = await createCFR2(models);

// generate unique name
const fileName = `${Math.random()}${file.name.replace(/ /g, '')}`;

// read file
const buffer = await fs.readFileSync(file.path);

// initialize r2
const r2 = await createCFR2(models);

// upload to r2
const response: any = await new Promise((resolve, reject) => {
r2.upload(
Expand Down Expand Up @@ -567,7 +680,7 @@ export const uploadFileAWS = async (
};

/*
* Delete file from Cloudflare r2
* Delete file from Cloudflare
*/
export const deleteFileCloudflare = async (
fileName: string,
Expand Down Expand Up @@ -732,6 +845,64 @@ const deleteFileGCS = async (fileName: string, models?: IModels) => {
/**
* Read file from GCS, AWS
*/

const readFromCFImages = async (key: string, models?: IModels) => {
const CLOUDFLARE_ACCOUNT_HASH = await getConfig(
'CLOUDFLARE_ACCOUNT_HASH',
'',
models
);

if (!CLOUDFLARE_ACCOUNT_HASH) {
throw new Error('Cloudflare Account Hash is not configured');
}

const url = `https://imagedelivery.net/${CLOUDFLARE_ACCOUNT_HASH}/${key}/public`;
return new Promise(resolve => {
fetch(url)
.then(res => res.buffer())
.then(buffer => resolve(buffer))
.catch(_err => {
return readFromCR2(key, models);
});
});
};

const readFromCR2 = async (key: string, models?: IModels) => {
const CLOUDFLARE_R2_BUCKET = await getConfig(
'CLOUDFLARE_BUCKET_NAME',
'',
models
);

const r2 = await createCFR2(models);

return new Promise((resolve, reject) => {
r2.getObject(
{
Bucket: CLOUDFLARE_R2_BUCKET,
Key: key
},
(error, response) => {
if (error) {
if (
error.code === 'NoSuchKey' &&
error.message.includes('key does not exist')
) {
debugBase(
`Error occurred when fetching r2 file with key: "${key}"`
);
}

return reject(error);
}

return resolve(response.Body);
}
);
});
};

export const readFileRequest = async ({
key,
subdomain,
Expand Down Expand Up @@ -815,37 +986,20 @@ export const readFileRequest = async ({
}

if (UPLOAD_SERVICE_TYPE === 'CLOUDFLARE') {
const CLOUDFLARE_R2_BUCKET = await getConfig(
'CLOUDFLARE_BUCKET_NAME',
'',
const CLOUDFLARE_USE_CDN = await getConfig(
'CLOUDFLARE_USE_CDN',
'false',
models
);
const r2 = await createCFR2(models);

return new Promise((resolve, reject) => {
r2.getObject(
{
Bucket: CLOUDFLARE_R2_BUCKET,
Key: key
},
(error, response) => {
if (error) {
if (
error.code === 'NoSuchKey' &&
error.message.includes('key does not exist')
) {
debugBase(
`Error occurred when fetching r2 file with key: "${key}"`
);
}

return reject(error);
}
if (
(CLOUDFLARE_USE_CDN === 'true' || CLOUDFLARE_USE_CDN === true) &&
isImage(key)
) {
return readFromCFImages(key, models);
}

return resolve(response.Body);
}
);
});
return readFromCR2(key, models);
}

if (UPLOAD_SERVICE_TYPE === 'local') {
Expand Down Expand Up @@ -1246,6 +1400,14 @@ export const resizeImage = async (
}
};

export const isImage = (mimeType: string) => {
return mimeType.includes('image');
};

export const isVideo = (mimeType: string) => {
return mimeType.includes('video');
};

export const getEnv = utils.getEnv;
export const paginate = utils.paginate;
export const fixDate = utils.fixDate;
Expand Down
13 changes: 8 additions & 5 deletions packages/core/src/middlewares/fileMiddleware.ts
View file
Expand Up @@ -7,7 +7,7 @@ import * as fileType from 'file-type';
import * as fs from 'fs';
import { filterXSS } from 'xss';

import { checkFile, resizeImage, uploadFile } from '../data/utils';
import { checkFile, isImage, resizeImage, uploadFile } from '../data/utils';
import { debugExternalApi } from '../debuggers';
import { getSubdomain } from '@erxes/api-utils/src/core';

Expand Down Expand Up @@ -52,10 +52,13 @@ export const uploader = async (req: any, res, next) => {
let fileResult = file;

const detectedType = fileType(fs.readFileSync(file.path));
if (detectedType && detectedType.mime.startsWith('image/')) {
if (maxHeight || maxWidth) {
fileResult = await resizeImage(file, maxWidth, maxHeight);
}

if (!detectedType) {
return res.status(500).send('File type is not recognized');
}

if (isImage(detectedType.mime) && maxHeight && maxWidth) {
fileResult = await resizeImage(file, maxWidth, maxHeight);
}

// check file ====
Expand Down
3 changes: 2 additions & 1 deletion packages/erxes-ui/package.json
View file
Expand Up @@ -35,6 +35,7 @@
"styled-components": "^3.2.6",
"styled-components-ts": "^0.0.14",
"validator": "12.1.0",
"xss": "^1.0.3"
"xss": "^1.0.3",
"video.js":"^8.5.2"
}
}

0 comments on commit 0669abf

Please sign in to comment.