From d08963a9ecd4e78149c917c92bd581241c220ccf Mon Sep 17 00:00:00 2001 From: kyusho Date: Mon, 5 Dec 2022 13:55:11 +0800 Subject: [PATCH 01/13] fix(user): main server url --- packages/rath-client/src/constants.ts | 19 +++++++++++++++++-- packages/rath-client/src/hooks/index.ts | 6 +++--- packages/rath-client/src/store/commonStore.ts | 14 +++++++------- packages/rath-client/src/store/fetch.ts | 4 ++-- packages/rath-client/src/utils/user.ts | 16 +++++++++++----- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/packages/rath-client/src/constants.ts b/packages/rath-client/src/constants.ts index 15830c81..26f0f946 100644 --- a/packages/rath-client/src/constants.ts +++ b/packages/rath-client/src/constants.ts @@ -30,8 +30,8 @@ export const EXPLORE_MODE = { export const DEMO_DATA_REQUEST_TIMEOUT = 1000 * 10; export const ENGINE_CONNECTION_STAGES: Array<{ stage: number; name: IECStatus; description?: string }> = [ - { stage: 0, name: 'client', description: 'client module importetd.' }, - { stage: 1, name: 'proxy', description: 'database proxy connector lanuched.' }, + { stage: 0, name: 'client', description: 'client module imported.' }, + { stage: 1, name: 'proxy', description: 'database proxy connector launched.' }, { stage: 2, name: 'engine', description: 'clickhouse connected.' }, ]; @@ -53,3 +53,18 @@ export const STORAGES = { CONFIG: 'config', ITERATOR_META: 'iterator_meta', } + +export enum RATH_ENV { + DEV = 'development environment', + TEST = 'test environment', + LPE = 'local preview environment', + IPE = 'integrative preview environment', + ONLINE = 'online production environment', +} + +export const RathEnv: RATH_ENV = ( + process.env.NODE_ENV === 'development' ? RATH_ENV.DEV + : process.env.NODE_ENV === 'test' ? RATH_ENV.TEST + : window.location.host.match(/^(.*\.)?kanaries\.(net|cn)$/) ? RATH_ENV.ONLINE + : window.location.host.match(/^.*kanaries\.vercel\.app$/) ? RATH_ENV.IPE : RATH_ENV.LPE +); diff --git a/packages/rath-client/src/hooks/index.ts b/packages/rath-client/src/hooks/index.ts index 1fbb90ca..2f3c56eb 100644 --- a/packages/rath-client/src/hooks/index.ts +++ b/packages/rath-client/src/hooks/index.ts @@ -3,7 +3,7 @@ import produce, { Draft } from 'immer'; import intl from 'react-intl-universal'; import { CleanMethod } from '../interfaces'; import { notify } from '../components/error'; -import { getServerUrl } from '../utils/user'; +import { getMainServerUrl } from '../utils/user'; import { request } from '../utils/request'; /** @@ -56,7 +56,7 @@ export const useCleanMethodList = function (): typeof cleanMethodList { }; async function sendCertMail(email: string) { - const url = getServerUrl('/api/sendMailCert'); + const url = getMainServerUrl('/api/sendMailCert'); // TODO: [feat] email format check const res = await request.post<{ email: string }, string>(url, { email }); if (res) { @@ -66,7 +66,7 @@ async function sendCertMail(email: string) { } async function sendCertPhone(phone: string) { - const url = getServerUrl('/api/sendPhoneCert'); + const url = getMainServerUrl('/api/sendPhoneCert'); const res = await request.post<{ phone: string }, string>(url, { phone }); if (res) { // console.log("message sent success"); diff --git a/packages/rath-client/src/store/commonStore.ts b/packages/rath-client/src/store/commonStore.ts index 15366a07..3dd41f5c 100644 --- a/packages/rath-client/src/store/commonStore.ts +++ b/packages/rath-client/src/store/commonStore.ts @@ -2,7 +2,7 @@ import { makeAutoObservable, observable, runInAction } from 'mobx'; import { Specification } from 'visual-insights'; import { COMPUTATION_ENGINE, EXPLORE_MODE, PIVOT_KEYS } from '../constants'; import { IAccessPageKeys, ITaskTestMode, IVegaSubset } from '../interfaces'; -import { getAvatarURL, getServerUrl, AVATAR_IMG_LIST, IAVATAR_TYPES } from '../utils/user'; +import { getAvatarURL, getMainServerUrl, AVATAR_IMG_LIST, IAVATAR_TYPES } from '../utils/user'; import { destroyRathWorker, initRathWorker, rathEngineService } from '../services/index'; import { transVegaSubset2Schema } from '../utils/transform'; import { notify } from '../components/error'; @@ -159,7 +159,7 @@ export class CommonStore { } public async liteAuth(certMethod: 'email' | 'phone') { - const url = getServerUrl('/api/liteAuth'); + const url = getMainServerUrl('/api/liteAuth'); const { certCode, phone, email } = this.signup; const res = await fetch(url, { method: 'POST', @@ -212,7 +212,7 @@ export class CommonStore { public async commitLogout() { try { - const url = getServerUrl('/api/logout'); + const url = getMainServerUrl('/api/logout'); const res = await fetch(url, { method: 'GET', }); @@ -237,7 +237,7 @@ export class CommonStore { public async updateAuthStatus() { try { - const url = getServerUrl('/api/loginStatus'); + const url = getMainServerUrl('/api/loginStatus'); const res = await request.get<{}, { loginStatus: boolean; userName: string }>(url); if (res.loginStatus && res.userName !== null) { runInAction(() => { @@ -254,7 +254,7 @@ export class CommonStore { } } public async getPersonalInfo() { - const url = getServerUrl('/api/ce/personal'); + const url = getMainServerUrl('/api/ce/personal'); try { const result = await request.get<{}, IUserInfo>(url); if (result !== null) { @@ -279,7 +279,7 @@ export class CommonStore { const { file } = value; const data = new FormData(); file && data.append('file', file); - const url = getServerUrl('/api/ce/avatar'); + const url = getMainServerUrl('/api/ce/avatar'); const res = await fetch(url, { method: 'POST', credentials: 'include', @@ -300,7 +300,7 @@ export class CommonStore { } public async getAvatarImgUrl() { - const url = getServerUrl('/api/ce/avatar'); + const url = getMainServerUrl('/api/ce/avatar'); try { const result = await request.get<{}, { avatarURL: string }>(url); if (result !== null) { diff --git a/packages/rath-client/src/store/fetch.ts b/packages/rath-client/src/store/fetch.ts index d88c3853..8368300e 100644 --- a/packages/rath-client/src/store/fetch.ts +++ b/packages/rath-client/src/store/fetch.ts @@ -1,8 +1,8 @@ -import { getServerUrl } from '../utils/user'; +import { getMainServerUrl } from '../utils/user'; import { ILoginForm } from './commonStore'; export async function commitLoginService(props: ILoginForm) { - const url = getServerUrl('/api/login'); + const url = getMainServerUrl('/api/login'); const res = await fetch(url, { method: 'POST', headers: { diff --git a/packages/rath-client/src/utils/user.ts b/packages/rath-client/src/utils/user.ts index 50e0144f..ccfeda70 100644 --- a/packages/rath-client/src/utils/user.ts +++ b/packages/rath-client/src/utils/user.ts @@ -1,3 +1,5 @@ +import { RathEnv, RATH_ENV } from "../constants"; + export enum IAVATAR_TYPES { gravatar = 'gravatar', default = 'default' @@ -16,12 +18,16 @@ export const AVATAR_IMG_LIST: string[] = new Array(18) export const DEFAULT_AVATAR_URL_PREFIX = 'https://foghorn-assets.s3.ap-northeast-1.amazonaws.com/avatar/'; -export function getServerUrl(path: string) { +const DEFAULT_MAIN_SERVER_HOST = RathEnv === `${ + window.location.host.match(/kanaries\.[a-z]+$/i)?.[0] ?? 'kanaries.net' +}`; + +export function getMainServerUrl(path: string) { const baseURL = new URL(window.location.href); - const DATA_SERVER_URL = - baseURL.searchParams.get('main_service') || localStorage.getItem('main_service') || window.location.href; - // const devSpecURL = new URL(w|| window.location.href) - const url = new URL(DATA_SERVER_URL); + const CONFIGURABLE_MAIN_SERVER_URL = RathEnv === RATH_ENV.ONLINE ? null + : baseURL.searchParams.get('main_service') || localStorage.getItem('main_service'); + const serverUrl = CONFIGURABLE_MAIN_SERVER_URL ?? `${baseURL.protocol}//${DEFAULT_MAIN_SERVER_HOST}`; + const url = new URL(serverUrl); url.pathname = path; return url.toString(); } From 557639d8c00ba30bbb8be7def5e74b9ee9c4cdca Mon Sep 17 00:00:00 2001 From: kyusho Date: Mon, 5 Dec 2022 14:16:54 +0800 Subject: [PATCH 02/13] fix(login): bad comparing --- packages/rath-client/src/utils/user.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rath-client/src/utils/user.ts b/packages/rath-client/src/utils/user.ts index ccfeda70..c8a200d7 100644 --- a/packages/rath-client/src/utils/user.ts +++ b/packages/rath-client/src/utils/user.ts @@ -18,7 +18,7 @@ export const AVATAR_IMG_LIST: string[] = new Array(18) export const DEFAULT_AVATAR_URL_PREFIX = 'https://foghorn-assets.s3.ap-northeast-1.amazonaws.com/avatar/'; -const DEFAULT_MAIN_SERVER_HOST = RathEnv === `${ +const DEFAULT_MAIN_SERVER_HOST = `${ window.location.host.match(/kanaries\.[a-z]+$/i)?.[0] ?? 'kanaries.net' }`; From 91411eaf09556014c8e5a67a347774883cf12d2c Mon Sep 17 00:00:00 2001 From: kyusho Date: Mon, 5 Dec 2022 20:06:16 +0800 Subject: [PATCH 03/13] feat(datasource): combine file & local history --- README.md | 4 + packages/rath-client/package.json | 1 + .../rath-client/public/locales/en-US.json | 28 +- .../rath-client/public/locales/zh-CN.json | 16 +- .../src/pages/dataSource/config.ts | 9 +- .../src/pages/dataSource/selection/file.tsx | 141 --------- .../dataSource/selection/file/file-helper.tsx | 122 ++++++++ .../dataSource/selection/file/file-upload.tsx | 286 ++++++++++++++++++ .../selection/file/get-file-icon.tsx | 11 + .../selection/file/history-list.tsx | 194 ++++++++++++ .../pages/dataSource/selection/file/index.tsx | 130 ++++++++ .../src/pages/dataSource/selection/index.tsx | 2 +- .../src/pages/dataSource/utils/index.ts | 28 +- yarn.lock | 68 ++++- 14 files changed, 871 insertions(+), 169 deletions(-) delete mode 100644 packages/rath-client/src/pages/dataSource/selection/file.tsx create mode 100644 packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx create mode 100644 packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx create mode 100644 packages/rath-client/src/pages/dataSource/selection/file/get-file-icon.tsx create mode 100644 packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx create mode 100644 packages/rath-client/src/pages/dataSource/selection/file/index.tsx diff --git a/README.md b/README.md index 4d15d00c..c2ae2d97 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,10 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . + +--- + +Branded icons are licensed under their copyright license.


diff --git a/packages/rath-client/package.json b/packages/rath-client/package.json index 7a92a21a..440a1409 100644 --- a/packages/rath-client/package.json +++ b/packages/rath-client/package.json @@ -16,6 +16,7 @@ "@emotion/styled": "^11.10.4", "@fluentui/font-icons-mdl2": "^8.4.13", "@fluentui/react": "^8.94.4", + "@fluentui/react-file-type-icons": "^8.8.3", "@fluentui/react-hooks": "^8.6.11", "@kanaries/graphic-walker": "0.2.8", "@kanaries/loa": "^0.0.16", diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index c346aeb9..26530268 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -63,7 +63,7 @@ "config": { "computationEngine": { "title": "Computation Engine", - "desc": "Define where the analysis task run. If your dataset is small (< 100MB), web worker mode is recommanded. Otherwise try clickhosue mode.", + "desc": "Define where the analysis task run. If your dataset is small (< 100MB), web worker mode is recommended. Otherwise try clickhouse mode.", "clickhouse": "ClickHouse", "webworker": "Browser(Web Worker)", "notCompatible": "Not compatible with datasource" @@ -174,8 +174,9 @@ "lackData": "Lack of Data", "lackDimension": "It seems you don't have dimensions in your dataset. (part of functions of mega-automation maybe influenced.)", "lackMeasure": "It seems you don't have measures in your dataset. (part of functions of mega-automation maybe influenced.)", - "smallSample": "The sample size is not big enough, which may influence the reliability of recommandation.", - "forceAnalysis": "Force Analysis" + "smallSample": "The sample size is not big enough, which may influence the reliability of recommendation.", + "forceAnalysis": "Force Analysis", + "upload_file_too_large": "This file is too large. Use a smaller subset or transform it into CSV file to apply sampling." }, "meta": { "uniqueValue": "Unique Value", @@ -191,7 +192,7 @@ "buttonName": "Import Data", "load": "Load Data", "type": { - "file": "File", + "file": "Local", "restful": "RESTFUL", "mysql": "MySQL", "demo": "Demo", @@ -229,11 +230,18 @@ }, "upload": { "title": "Upload Your own dataset", - "fileTypes": "csv, json are supportted.", + "fileTypes": "JSON, CSV, XLSX are supported.", "uniqueIdIssue": "Add unique ids for fields", "sampling": "Sampling", "percentSize": "sample size(rows)", - "upload": "Upload" + "upload": "Upload", + "change": "Browse", + "lastOpen": "Last opened", + "firstOpen": "First created", + "history": "Open Recent", + "new": "New file", + "preview_parsed": "Preview", + "preview_raw": "Raw" }, "exploreMode": { "title": "Explore Mode", @@ -353,7 +361,7 @@ "pin": "pin", "compare": "compare", "vizsys": { - "title": "visualization recommand system", + "title": "visualization recommend system", "lite": "Lite mode(fast)", "strict": "Strict mode" }, @@ -436,7 +444,7 @@ "$ReLU": "Projects a field by x -> max(0, x) as a new column.", "$match": "Finds the matched parts of each text content using Regular Expression as a new column.", "$replace": "Replaces the matched parts of each text content with a given string using Regular Expression as a new column.", - "$concat": "Concats text contents of all the fields as a new column.", + "$concat": "Merge two or more text contents of all the fields as a new column.", "$sigmoid": "Projects a field by x -> (1 + e^-x)^-1 as a new column.", "$boxClip": "Maps overflowing values (due to boxplot) to the domain as a new column.", "$meanClip": "Maps overflowing values (due to the given domain) to the mean of all the other values as a new column.", @@ -480,7 +488,7 @@ }, "login": { "clickLogin": "Click Login", - "haveSent": "Alredy Send", + "haveSent": "Already Sent", "signIn": "Sign In", "signOut": "Sign Out", "preferences": "Preferences", @@ -506,7 +514,7 @@ "errEmail": "not support email type" }, "password": { - "userName": "UsernNme", + "userName": "Username", "password": "Password" } } diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index f5c35f12..3d8c092d 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -174,7 +174,8 @@ "lackDimension": "数据集中缺少维度。（全自动化分析模块部分能力会受到影响）", "lackMeasure": "数据集中缺少度量，（全自动化分析模块会受到影响，可以尝试使用其他模块）", "smallSample": "数据集中样本数量低于预期，可能会对推荐结果的一般性造成影响。（小样本问题）", - "forceAnalysis": "强制分析" + "forceAnalysis": "强制分析", + "upload_file_too_large": "文件体积过大。尝试使用更少的数据，或将文件转换为 CSV 格式以使用采样功能。" }, "meta": { "title": "元数据视图", @@ -190,7 +191,7 @@ "importData": { "buttonName": "选择数据", "type": { - "file": "文件", + "file": "本地", "restful": "RESTFUL", "mysql": "MySQL", "demo": "示例数据", @@ -229,11 +230,18 @@ }, "upload": { "title": "连接你的数据集，根据需求调整以下配置", - "fileTypes": "支持csv, json文件", + "fileTypes": "支持 JSON, CSV, XLSX 文件", "uniqueIdIssue": "添加唯一标识(字段是中文字符推荐使用)", "sampling": "数据采样", "percentSize": "样本大小(行)", - "upload": "上传文件" + "upload": "上传文件", + "change": "重新选择", + "lastOpen": "上一次使用", + "firstOpen": "创建时间", + "history": "最近使用", + "new": "新的文件", + "preview_parsed": "预览", + "preview_raw": "原始内容" }, "exploreMode": { "title": "探索模式", diff --git a/packages/rath-client/src/pages/dataSource/config.ts b/packages/rath-client/src/pages/dataSource/config.ts index d8f6f951..28ad5238 100644 --- a/packages/rath-client/src/pages/dataSource/config.ts +++ b/packages/rath-client/src/pages/dataSource/config.ts @@ -7,7 +7,6 @@ export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceTyp const fileText = intl.get(`dataSource.importData.type.${IDataSourceType.FILE}`); const restfulText = intl.get(`dataSource.importData.type.${IDataSourceType.RESTFUL}`); const demoText = intl.get(`dataSource.importData.type.${IDataSourceType.DEMO}`) - const localText = intl.get('common.history'); const dbText = intl.get(`dataSource.importData.type.${IDataSourceType.DATABASE}`); const options = useMemo>(() => { @@ -15,13 +14,7 @@ export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceTyp { key: IDataSourceType.FILE, text: fileText, - iconProps: { iconName: "ExcelDocument" }, - }, - { - key: IDataSourceType.LOCAL, - text: localText, iconProps: { iconName: "FabricUserFolder" }, - disabled: false, }, { key: IDataSourceType.DEMO, @@ -51,7 +44,7 @@ export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceTyp disabled: false }, ]; - }, [fileText, restfulText, demoText, localText, dbText]); + }, [fileText, restfulText, demoText, dbText]); return options; }; diff --git a/packages/rath-client/src/pages/dataSource/selection/file.tsx b/packages/rath-client/src/pages/dataSource/selection/file.tsx deleted file mode 100644 index 00c91485..00000000 --- a/packages/rath-client/src/pages/dataSource/selection/file.tsx +++ /dev/null @@ -1,141 +0,0 @@ -import React, { useState, useRef, useMemo } from "react"; -import { ChoiceGroup, IChoiceGroupOption, SpinButton, DefaultButton, Dropdown, IDropdownOption } from '@fluentui/react'; -import { useId } from "@fluentui/react-hooks"; -import intl from "react-intl-universal"; -import { loadDataFile, SampleKey, useSampleOptions } from "../utils"; -import { dataBackup, logDataImport } from "../../../loggers/dataImport"; -import { IMuteFieldBase, IRow } from "../../../interfaces"; -interface FileDataProps { - onClose: () => void; - onStartLoading: () => void; - onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; - onDataLoading: (p: number) => void; -} -const FileData: React.FC = (props) => { - const { onClose, onDataLoaded, onStartLoading, onLoadingFailed, onDataLoading } = props; - const sampleOptions = useSampleOptions(); - const labelId = useId("labelElement"); - const [sampleMethod, setSampleMethod] = useState(SampleKey.none); - const [sampleSize, setSampleSize] = useState(500); - const fileEle = useRef(null); - const [chartset, setcharSet] = useState('utf-8'); - - const charsetOptions = useMemo(() => { - return [ - { - text: 'UTF-8', - key: 'utf-8' - }, - { - text: 'GB2312', - key: 'gb2312' - }, - { - text: 'US-ASCII', - key: 'us-ascii' - }, - { - text: 'Big5', - key: 'big5' - }, - { - text: 'Big5-HKSCS', - key: 'Big5-HKSCS' - }, - { - text: 'GB18030', - key: 'GB18030' - }, - ] - }, []) - - async function fileUploadHanlder() { - if (fileEle.current !== null && fileEle.current.files !== null) { - const file = fileEle.current.files[0]; - onStartLoading(); - try { - const { fields, dataSource } = await loadDataFile({ - file, - sampleMethod, - sampleSize, - encoding: chartset, - onLoading: onDataLoading - }); - logDataImport({ - dataType: 'File', - fields, - dataSource: dataSource.slice(0, 10), - size: dataSource.length - }); - dataBackup(file); - onDataLoaded(fields, dataSource, file.name); - } catch (error) { - onLoadingFailed(error) - } - onClose(); - } - } - - return ( -

-
-

{intl.get("dataSource.upload.fileTypes")}

-
-
- { - item && setcharSet(item.key as string) - }} - /> -
-
- { - if (option) { - setSampleMethod(option.key as SampleKey); - } - }} - ariaLabelledBy={labelId} - /> - {sampleMethod === SampleKey.reservoir && ( - { - setSampleSize(Number(value)); - }} - onIncrement={() => { - setSampleSize((v) => v + 1); - }} - onDecrement={() => { - setSampleSize((v) => Math.max(v - 1, 0)); - }} - /> - )} -
-
- - { - if (fileEle.current) { - fileEle.current.click(); - } - }} - /> -
-
- ); -}; - -export default FileData; diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx new file mode 100644 index 00000000..fbb3bab1 --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx @@ -0,0 +1,122 @@ +import intl from 'react-intl-universal'; +import { ChoiceGroup, Dropdown, SpinButton } from "@fluentui/react"; +import { FC } from "react"; +import styled from "styled-components"; +import { SampleKey, useSampleOptions } from '../../utils'; + + +export const charsetOptions = [ + { + text: 'UTF-8', + key: 'utf-8' + }, + { + text: 'GB2312', + key: 'gb2312' + }, + { + text: 'US-ASCII', + key: 'us-ascii' + }, + { + text: 'Big5', + key: 'big5' + }, + { + text: 'Big5-HKSCS', + key: 'Big5-HKSCS' + }, + { + text: 'GB18030', + key: 'GB18030' + }, +] as const; + +export type Charset = typeof charsetOptions[number]['key']; + +const Container = styled.div` + display: flex; + flex-direction: row; + flex-wrap: wrap; + align-items: center; + & label { + font-weight: 400; + margin-right: 1em; + } + & [role=radiogroup] { + display: flex; + flex-direction: row; + align-items: center; + padding: 1em 0; + > div { + display: flex; + flex-direction: row; + > * { + margin: 0; + } + } + } +`; + +export interface IFileHelperProps { + charset: Charset; + setCharset: (charset: Charset) => void; + sampleMethod: SampleKey; + setSampleMethod: (sampleMethod: SampleKey) => void; + sampleSize: number; + setSampleSize: (sampleSize: number | ((prev: number) => number)) => void; + preview: File | null; +} + +const FileHelper: FC = ({ charset, setCharset, sampleMethod, setSampleMethod, sampleSize, setSampleSize, preview }) => { + const sampleOptions = useSampleOptions(); + + return ( + + { + item && setCharset(item.key as Charset) + }} + styles={{ root: { display: 'flex', flexDirection: 'row', marginRight: '2em' }, label: { marginRight: '1em', fontWeight: 400 }, dropdown: { width: '8em' } }} + /> + {!preview || preview.name.endsWith('.csv') ? ( + <> + { + if (option) { + setSampleMethod(option.key as SampleKey); + } + }} + /> + {sampleMethod === SampleKey.reservoir && ( + { + setSampleSize(Number(value)); + }} + onIncrement={() => { + setSampleSize((v) => v + 1); + }} + onDecrement={() => { + setSampleSize((v) => Math.max(v - 1, 0)); + }} + /> + )} + + ) : null} + + ); +}; + + +export default FileHelper; diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx new file mode 100644 index 00000000..13f6093e --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx @@ -0,0 +1,286 @@ +import { ActionButton, Icon, Pivot, PivotItem, TooltipHost } from "@fluentui/react"; +import intl from 'react-intl-universal'; +import { observer } from "mobx-react-lite"; +import { FC, useCallback, useRef, useState } from "react"; +import styled from "styled-components"; +import type { loadDataFile } from "../../utils"; +import { notify } from "../../../../components/error"; +import getFileIcon from "./get-file-icon"; +import { formatSize } from "./history-list"; + + +const Container = styled.div` + display: flex; + margin: 1em 0; + height: 12em; + overflow: hidden; +`; + +const Input = styled.div` + flex-grow: 0; + flex-shrink: 0; + width: 12em; + height: 100%; + > input { + display: none; + } +`; + +const ActionGroup = styled.div` + border: 1px solid #8884; + border-left: none; + flex-grow: 1; + flex-shrink: 1; + flex-basis: 0; + display: flex; + flex-direction: column; + overflow: hidden; + > p { + font-size: 0.9rem; + margin: 0.6em 1.2em; + color: #666; + user-select: none; + } + > div { + flex-grow: 1; + flex-shrink: 1; + overflow: hidden; + display: flex; + flex-direction: column; + & [role=tab] { + margin: 0; + &, & * { + height: max-content; + min-height: unset; + line-height: 2em; + } + } + > [role=tabpanel] { + flex-grow: 1; + flex-shrink: 1; + overflow: hidden; + > div { + width: 100%; + height: 100%; + padding: 0.6em 1.2em; + overflow: auto; + } + } + } +`; + +const InputButton = styled.div` + border: 1px solid #115ea320; + width: 100%; + height: 100%; + cursor: pointer; + color: #115ea3a0; + display: flex; + align-items: center; + justify-content: center; + :hover { + background-color: #115ea308; + } + > * { + pointer-events: none; + user-select: none; + } +`; + +const FileOutput = styled.div` + border: 1px solid #8884; + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + justify-content: space-between; + overflow: hidden; + padding: 0 1em; + > .head { + display: flex; + flex-direction: column; + align-items: center; + padding: 1em; + overflow: hidden; + > i { + flex-grow: 0; + flex-shrink: 0; + width: 4em; + height: 4em; + margin: 0 0 0.4em; + } + > div { + flex-grow: 1; + flex-shrink: 1; + overflow: hidden; + display: flex; + flex-direction: column; + align-items: center; + > header { + font-size: 0.9rem; + line-height: 1.2em; + font-weight: 550; + white-space: nowrap; + color: #111; + } + > span { + font-size: 0.6rem; + line-height: 1.2em; + margin: 0.6em 0; + color: #555; + } + } + } + > .action { + display: flex; + align-items: center; + justify-content: center; + > button, > button i { + font-size: 0.8rem; + } + } +`; + +const RawArea = styled.pre` + font-size: 0.8rem; + padding: 0 1em 1em 0; +`; + +const PreviewArea = styled.table` + font-size: 0.8rem; + padding: 0 1em 1em 0; + & * { + white-space: nowrap; + } + & th { + font-weight: 550; + } + & th, & td { + padding: 0.1em 1em 0.1em 0.4em; + :nth-child(even) { + background-color: #8881; + } + } + & tr:nth-child(odd) { + background-color: #8882; + } +`; + +const MAX_UPLOAD_SIZE = 1024 * 1024 * 128 * 0.0001; + +export interface IFileUploadProps { + preview: File | null; + previewOfFile: Awaited> | null; + previewOfRaw: string | null; + onFileUpload: (file: File | null) => void; +} + +const FileUpload: FC = ({ preview, previewOfFile, previewOfRaw, onFileUpload }) => { + const fileInputRef = useRef(null); + + const handleReset = useCallback(() => { + onFileUpload(null); + fileInputRef.current?.click(); + }, [onFileUpload]); + + const handleButtonClick = useCallback(() => { + fileInputRef.current?.click(); + }, []); + + const handleUpload = useCallback(() => { + const [file] = fileInputRef.current?.files ?? []; + if (!file.name.endsWith('.csv') && file.size > MAX_UPLOAD_SIZE) { + notify({ + type: 'error', + title: 'Failed to upload', + content: intl.get('dataSource.advice.upload_file_too_large'), + }); + return onFileUpload(null); + } + onFileUpload(file ?? null); + }, [onFileUpload]); + + const [previewTab, setPreviewTab] = useState<'parsed' | 'raw'>('parsed'); + + return ( + + + + {preview ? ( + +
+ +
+
+ + {preview.name} + +
+ {formatSize(preview.size)} +
+
+
+ +
+ + ) : ( + + + + )} + + + {preview ? ( + { + item && setPreviewTab(item.props.itemKey as typeof previewTab); + }} + > + + {previewOfFile && ( + + + + {previewOfFile.fields.map(f => ( + + {f.name || f.fid} + + ))} + + {previewOfFile.dataSource.slice(0, 20).map((row, i) => ( + + {previewOfFile.fields.map(f => ( + + {JSON.stringify(row[f.fid])} + + ))} + + ))} + + + )} + + + + {previewOfRaw} + + + + ) : ( +

{intl.get("dataSource.upload.fileTypes")}

+ )} + + + ); +}; + +export default observer(FileUpload); diff --git a/packages/rath-client/src/pages/dataSource/selection/file/get-file-icon.tsx b/packages/rath-client/src/pages/dataSource/selection/file/get-file-icon.tsx new file mode 100644 index 00000000..eb51e32e --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/file/get-file-icon.tsx @@ -0,0 +1,11 @@ +import { getFileTypeIconProps, initializeFileTypeIcons } from '@fluentui/react-file-type-icons'; + + +initializeFileTypeIcons(); + +const getFileIcon = (fileName: string): string => { + const iconProps = getFileTypeIconProps({ extension: /(?<=\.)[a-z]+/i.exec(fileName)?.[0] }); + return iconProps.iconName; +}; + +export default getFileIcon; diff --git a/packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx b/packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx new file mode 100644 index 00000000..cb8c2785 --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx @@ -0,0 +1,194 @@ +import intl from 'react-intl-universal'; +import { Icon, IconButton, TooltipHost } from "@fluentui/react"; +import { observer } from "mobx-react-lite"; +import dayjs from 'dayjs'; +import { FC, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import styled from "styled-components"; +import useBoundingClientRect from "../../../../hooks/use-bounding-client-rect"; +import { deleteDataStorageById, getDataStorageById, getDataStorageList, IDBMeta } from "../../../../utils/storage"; +import type { IMuteFieldBase, IRow } from '../../../../interfaces'; +import getFileIcon from "./get-file-icon"; + + +const List = styled.div` + margin: 1em 0; + height: 24em; + overflow: hidden auto; + display: grid; + gap: 0.4em; + grid-auto-rows: max-content; +`; + +const ListItem = styled.div` + display: flex; + flex-direction: column; + overflow: hidden; + height: 100%; + padding: 0.6em 1em 1.2em; + border-radius: 2px; + position: relative; + > .head { + display: flex; + align-items: center; + > i { + flex-grow: 0; + flex-shrink: 0; + width: 2em; + height: 2em; + margin-right: 0.8em; + } + > div { + flex-grow: 1; + flex-shrink: 1; + overflow: hidden; + > header { + font-size: 0.8rem; + line-height: 1.2em; + font-weight: 550; + white-space: nowrap; + color: #111; + } + > span { + font-size: 0.6rem; + line-height: 1.2em; + color: #555; + } + } + } + .time { + font-size: 0.5rem; + color: #888; + } + > button { + position: absolute; + top: 0; + right: 0; + margin: 0; + padding: 0; + font-size: 12px; + background-color: #d13438 !important; + border-radius: 50%; + color: #fff !important; + width: 1.4em; + height: 1.4em; + i { + font-weight: 1000; + line-height: 1em; + width: 1em; + height: 1em; + transform: scale(0.5); + } + visibility: hidden; + opacity: 0.5; + :hover { + opacity: 1; + } + } + :hover { + background-color: #88888818; + > button { + visibility: visible; + } + } + cursor: pointer; +`; + +const ITEM_MIN_WIDTH = 240; +const MAX_HISTORY_SIZE = 64; +const MAX_RECENT_TIME = 1_000 * 60 * 60 * 24 * 31 * 3; // 3 months + +export function formatSize(size: number) { + if (size < 1024) { + return `${size}B`; + } + if (size < 1024 * 1024) { + return `${(size / 1024).toFixed(2)}KB`; + } + if (size < 1024 * 1024 * 1024) { + return `${(size / 1024 / 1024).toFixed(2)}MB`; + } + return `${(size / 1024 / 1024 / 1024).toFixed(2)}GB`; +} + +interface IHistoryListProps { + onClose: () => void; + onLoadingFailed: (err: any) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; +} + +const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFailed }) => { + const [localDataList, setLocalDataList] = useState([]); + + useEffect(() => { + getDataStorageList().then((dataList) => { + const recent = dataList.filter(item => Date.now() - item.editTime < MAX_RECENT_TIME); + const sorted = recent.sort((a, b) => b.editTime - a.editTime); + setLocalDataList(sorted.slice(0, MAX_HISTORY_SIZE)); + }); + }, []); + + const listRef = useRef(null); + const { width } = useBoundingClientRect(listRef, { width: true }); + const colCount = useMemo(() => Math.floor((width ?? (window.innerWidth * 0.6)) / ITEM_MIN_WIDTH), [width]); + + const handleLoadHistory = useCallback((id: string) => { + getDataStorageById(id).then(res => { + onDataLoaded(res.fields, res.dataSource, id); + onClose(); + }).catch(onLoadingFailed); + }, [onClose, onDataLoaded, onLoadingFailed]); + + const handleDeleteHistory = useCallback((id: string) => { + deleteDataStorageById(id).then(() => { + getDataStorageList().then((dataList) => { + setLocalDataList(dataList); + }); + }); + }, []); + + return ( + + {localDataList.map((file, i) => { + const ext = /(?<=\.)[^.]+$/.exec(file.name)?.[0]; + + return ( + handleLoadHistory(file.id)} + > +
+ +
+
+ + {file.name} + +
+ + {`${ext ? `${ext} - ` : ''}${formatSize(file.size)}`} + +
+
+
+

{`${intl.get('dataSource.upload.lastOpen')}: ${dayjs(file.editTime).toDate().toLocaleString()}`}

+
+ { + e.stopPropagation(); + handleDeleteHistory(file.id); + }} + /> + + ); + })} + + ); +}; + +export default observer(HistoryList); diff --git a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx new file mode 100644 index 00000000..32dc102f --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx @@ -0,0 +1,130 @@ +import { FC, useState, useRef, useCallback, useEffect } from "react"; +import { PrimaryButton } from '@fluentui/react'; +import styled from "styled-components"; +import { observer } from "mobx-react-lite"; +import intl from "react-intl-universal"; +import { loadDataFile, readRaw, SampleKey } from "../../utils"; +import { dataBackup, logDataImport } from "../../../../loggers/dataImport"; +import type { IMuteFieldBase, IRow } from "../../../../interfaces"; +import FileUpload from "./file-upload"; +import HistoryList from "./history-list"; +import FileHelper, { Charset } from "./file-helper"; + + +const Container = styled.div` + width: 55vw; + > header { + margin-top: 1.2em; + font-weight: 550; + } + > .action { + margin: 1em 0; + } +`; + +interface FileDataProps { + onClose: () => void; + onLoadingFailed: (err: any) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; + onDataLoading: (p: number) => void; + toggleLoadingAnimation: (on: boolean) => void; +} + +const FileData: FC = (props) => { + const { onClose, onDataLoaded, onLoadingFailed, onDataLoading, toggleLoadingAnimation } = props; + const [sampleMethod, setSampleMethod] = useState(SampleKey.none); + const [sampleSize, setSampleSize] = useState(500); + const [charset, setCharset] = useState('utf-8'); + + const [preview, setPreview] = useState(null); + const [previewOfRaw, setPreviewOfRaw] = useState(null); + const [previewOfFile, setPreviewOfFile] = useState> | null>(null); + + const filePreviewPendingRef = useRef>(); + + useEffect(() => { + filePreviewPendingRef.current = undefined; + if (preview) { + setPreviewOfRaw(null); + setPreviewOfFile(null); + toggleLoadingAnimation(true); + const p = Promise.all([ + readRaw(preview, charset, 1024, 32, 128), + loadDataFile({ + file: preview, + sampleMethod, + sampleSize, + encoding: charset, + onLoading: onDataLoading + }), + ]); + filePreviewPendingRef.current = p; + p.then(res => { + if (p !== filePreviewPendingRef.current) { + return; + } + setPreviewOfRaw(res[0]); + setPreviewOfFile(res[1]); + }).catch(reason => { + onLoadingFailed(reason); + }).finally(() => { + toggleLoadingAnimation(false); + }); + } else { + setPreviewOfFile(null); + } + }, [charset, onDataLoading, onLoadingFailed, preview, sampleMethod, sampleSize, toggleLoadingAnimation]); + + const handleFileLoad = useCallback((file: File | null) => { + setPreview(file); + }, []); + + const handleFileSubmit = useCallback(() => { + if (!previewOfFile || !preview) { + return; + } + const { fields, dataSource } = previewOfFile; + logDataImport({ + dataType: 'File', + fields, + dataSource: dataSource.slice(0, 10), + size: dataSource.length + }); + dataBackup(preview); + onDataLoaded(fields, dataSource, preview.name); + onClose(); + }, [onClose, onDataLoaded, preview, previewOfFile]); + + return ( + +
{intl.get('dataSource.upload.new')}
+ + + {preview ? ( + previewOfFile && ( +
+ +
+ ) + ) : ( + <> +
{intl.get('dataSource.upload.history')}
+ + + )} + + ); +}; + +export default observer(FileData); diff --git a/packages/rath-client/src/pages/dataSource/selection/index.tsx b/packages/rath-client/src/pages/dataSource/selection/index.tsx index 6a0dac72..9b0773e9 100644 --- a/packages/rath-client/src/pages/dataSource/selection/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/index.tsx @@ -35,7 +35,7 @@ const Selection: React.FC = props => { const formMap: Record = { [IDataSourceType.FILE]: ( - + ), [IDataSourceType.DEMO]: ( diff --git a/packages/rath-client/src/pages/dataSource/utils/index.ts b/packages/rath-client/src/pages/dataSource/utils/index.ts index 412ea28d..e30e618b 100644 --- a/packages/rath-client/src/pages/dataSource/utils/index.ts +++ b/packages/rath-client/src/pages/dataSource/utils/index.ts @@ -1,5 +1,5 @@ import { Sampling } from 'visual-insights'; -import { FileReader } from '@kanaries/web-data-loader' +import { FileReader as KFileReader } from '@kanaries/web-data-loader' import intl from 'react-intl-universal'; import { useMemo } from "react"; import { STORAGE_FILE_SUFFIX } from "../../../constants"; @@ -53,6 +53,28 @@ export async function transformRawDataService (rawData: IRow[]): Promise<{ } } +export const readRaw = (file: File, encoding?: string, limit?: number, rowLimit?: number, colLimit?: number): Promise => { + const fr = new FileReader(); + fr.readAsText(file, encoding); + return new Promise(resolve => { + fr.onload = () => { + let text = fr.result as string | null; + if (typeof text === 'string') { + if (limit) { + text = text.slice(0, limit); + } + if (rowLimit || colLimit) { + text = text.split('\n').slice(0, rowLimit).map(row => row.slice(0, colLimit)).join('\n'); + } + return resolve(text); + } else { + return resolve(text); + } + }; + fr.onerror = () => resolve(null); + }); +}; + interface LoadDataFileProps { file: File; sampleMethod: SampleKey; @@ -79,7 +101,7 @@ export async function loadDataFile(props: LoadDataFileProps): Promise<{ if (file.type === 'text/csv' || file.type === 'application/vnd.ms-excel') { rawData = [] if (sampleMethod === SampleKey.reservoir) { - rawData = (await FileReader.csvReader({ + rawData = (await KFileReader.csvReader({ file, encoding, config: { @@ -89,7 +111,7 @@ export async function loadDataFile(props: LoadDataFileProps): Promise<{ onLoading })) as IRow[] } else { - rawData = (await FileReader.csvReader({ + rawData = (await KFileReader.csvReader({ file, encoding, onLoading diff --git a/yarn.lock b/yarn.lock index 6e5242b8..77a211b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1812,6 +1812,14 @@ "@fluentui/set-version" "^8.2.2" tslib "^2.1.0" +"@fluentui/dom-utilities@^2.2.3": + version "2.2.3" + resolved "https://registry.npmmirror.com/@fluentui/dom-utilities/-/dom-utilities-2.2.3.tgz#0dd8b1a0ba4d75232b7fea4552bd80afbf120593" + integrity sha512-Ml/xwpTC6vb9lHHVAbSUD9jMbt9nVzV208D0FEoQn0c0+dP2vdMXSvXC/QHs/57B6JicttVQPuX6EcPwR3Mkpg== + dependencies: + "@fluentui/set-version" "^8.2.3" + tslib "^2.1.0" + "@fluentui/font-icons-mdl2@^8.4.13", "@fluentui/font-icons-mdl2@^8.5.0": version "8.5.0" resolved "https://registry.yarnpkg.com/@fluentui/font-icons-mdl2/-/font-icons-mdl2-8.5.0.tgz#b358ed57be5b52c921558be6535e70b68d4d4f1e" @@ -1848,6 +1856,23 @@ "@fluentui/set-version" "^8.2.2" tslib "^2.1.0" +"@fluentui/merge-styles@^8.5.4": + version "8.5.4" + resolved "https://registry.npmmirror.com/@fluentui/merge-styles/-/merge-styles-8.5.4.tgz#e2c487ccc6d5b8e5a9c8914a148bb55146a54c33" + integrity sha512-CeQIEcEgZu0cxqqyhJyTqySXoUL1vXfdWDJ8WMzchaNnhvOvoXISw8xXHpNXUtEn+HgPrcy9mHQwFcAc+jv3Wg== + dependencies: + "@fluentui/set-version" "^8.2.3" + tslib "^2.1.0" + +"@fluentui/react-file-type-icons@^8.8.3": + version "8.8.3" + resolved "https://registry.npmmirror.com/@fluentui/react-file-type-icons/-/react-file-type-icons-8.8.3.tgz#bf55073acc88e83f7397fd0500051e8a25ccf454" + integrity sha512-Mn8qs+5e9shop7s3hQ0CfqR6XNXVOqYsqAzQH73ZEJpgN9B5MrQ5NECYKgJei8BEbOQatD8ZCDLTjv/Bcbm/6Q== + dependencies: + "@fluentui/set-version" "^8.2.3" + "@fluentui/style-utilities" "^8.8.3" + tslib "^2.1.0" + "@fluentui/react-focus@^8.8.5": version "8.8.5" resolved "https://registry.yarnpkg.com/@fluentui/react-focus/-/react-focus-8.8.5.tgz#6003ddadc914a2eaadb9d68d5feb793e6dc09a95" @@ -1912,6 +1937,13 @@ dependencies: tslib "^2.1.0" +"@fluentui/set-version@^8.2.3": + version "8.2.3" + resolved "https://registry.npmmirror.com/@fluentui/set-version/-/set-version-8.2.3.tgz#fe9761353dbea6a02f4a140f9bc67e5dd07ca22d" + integrity sha512-/+5vrI1Bq/ZsNDEK9++RClnDOeCILS8RXxZb7OAqmOc8GnPScxKcIN8e/1bosUxTjb2EB1KbVk6XeUpk0WvQIg== + dependencies: + tslib "^2.1.0" + "@fluentui/style-utilities@^8.7.12": version "8.7.12" resolved "https://registry.yarnpkg.com/@fluentui/style-utilities/-/style-utilities-8.7.12.tgz#f8ae3c2b850f38705c0556569a0d15d2ba4e6363" @@ -1924,6 +1956,18 @@ "@microsoft/load-themed-styles" "^1.10.26" tslib "^2.1.0" +"@fluentui/style-utilities@^8.8.3": + version "8.8.3" + resolved "https://registry.npmmirror.com/@fluentui/style-utilities/-/style-utilities-8.8.3.tgz#6ab8e332482e9698a637036f438f67e13878da1a" + integrity sha512-DxcCIHnKdaBpzdIawZIMcVyn8xVbK6A37J0Q7MPdjb9VV4Nsp/ohWnM9nPrPlbB+RSsKWrIssgWJdn5yZM9Wxg== + dependencies: + "@fluentui/merge-styles" "^8.5.4" + "@fluentui/set-version" "^8.2.3" + "@fluentui/theme" "^2.6.19" + "@fluentui/utilities" "^8.13.4" + "@microsoft/load-themed-styles" "^1.10.26" + tslib "^2.1.0" + "@fluentui/theme@^2.6.16": version "2.6.16" resolved "https://registry.yarnpkg.com/@fluentui/theme/-/theme-2.6.16.tgz#a29c58bf16f765ba1b97a497e572fd1fdc469c44" @@ -1934,6 +1978,16 @@ "@fluentui/utilities" "^8.13.1" tslib "^2.1.0" +"@fluentui/theme@^2.6.19": + version "2.6.19" + resolved "https://registry.npmmirror.com/@fluentui/theme/-/theme-2.6.19.tgz#1a8254452eefcb7322ae3a5d60b466b32b30c7fb" + integrity sha512-Pk4STq3WAM3Mq4fGCBrq3F43o1u2SitO7CZ6A3/ALreaxTA1LC4bbyKQTYH3tvxapqEOYaEPYZ3dFjWjqYlFfg== + dependencies: + "@fluentui/merge-styles" "^8.5.4" + "@fluentui/set-version" "^8.2.3" + "@fluentui/utilities" "^8.13.4" + tslib "^2.1.0" + "@fluentui/utilities@^8.13.1": version "8.13.1" resolved "https://registry.yarnpkg.com/@fluentui/utilities/-/utilities-8.13.1.tgz#028fd2e93f68b5f386039970dce0ae642eb7e9da" @@ -1944,6 +1998,16 @@ "@fluentui/set-version" "^8.2.2" tslib "^2.1.0" +"@fluentui/utilities@^8.13.4": + version "8.13.4" + resolved "https://registry.npmmirror.com/@fluentui/utilities/-/utilities-8.13.4.tgz#bad157358f2dcb09acd005a1921f7cb3abfa03c6" + integrity sha512-oJ6q8BvVdr0eEG5RgI/VBtKX2JvJV2h0AUkR7FwZoT8fvUUH/iykPZO/5CAVcQDyiXj73hmBibiEGkWNFFuPfw== + dependencies: + "@fluentui/dom-utilities" "^2.2.3" + "@fluentui/merge-styles" "^8.5.4" + "@fluentui/set-version" "^8.2.3" + tslib "^2.1.0" + "@formatjs/intl-unified-numberformat@^3.2.0": version "3.3.7" resolved "https://registry.yarnpkg.com/@formatjs/intl-unified-numberformat/-/intl-unified-numberformat-3.3.7.tgz#9995a24568908188e716d81a1de5b702b2ee00e2" @@ -3386,7 +3450,7 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== -"@types/react-dom@^17.0.1", "@types/react-dom@^17.x": +"@types/react-dom@^17.0.1": version "17.0.18" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.18.tgz#8f7af38f5d9b42f79162eea7492e5a1caff70dc2" integrity sha512-rLVtIfbwyur2iFKykP2w0pl/1unw26b5td16d5xMgp7/yjTHomkyxPYChFoCr/FtEX1lN9wY6lFj1qvKdS5kDw== @@ -3417,7 +3481,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^17", "@types/react@^17.0.2", "@types/react@^17.x": +"@types/react@*", "@types/react@^17", "@types/react@^17.0.2": version "17.0.52" resolved "https://registry.npmmirror.com/@types/react/-/react-17.0.52.tgz#10d8b907b5c563ac014a541f289ae8eaa9bf2e9b" integrity sha512-vwk8QqVODi0VaZZpDXQCmEmiOuyjEFPY7Ttaw5vjM112LOq37yz1CDJGrRJwA1fYEq4Iitd5rnjd1yWAc/bT+A== From abec2b31be7165e5440b11fa7e28aa17e45f17ba Mon Sep 17 00:00:00 2001 From: kyusho Date: Mon, 5 Dec 2022 22:05:16 +0800 Subject: [PATCH 04/13] feat(datasource): support excel files --- packages/rath-client/package.json | 3 +- .../rath-client/public/locales/en-US.json | 6 +- .../rath-client/public/locales/zh-CN.json | 6 +- .../dataSource/selection/file/file-helper.tsx | 19 +++++- .../dataSource/selection/file/file-upload.tsx | 21 +++++-- .../pages/dataSource/selection/file/index.tsx | 47 ++++++++++++++- .../src/pages/dataSource/utils/index.ts | 31 ++++++++++ packages/rath-client/src/utils/fileParser.ts | 15 +++++ yarn.lock | 60 ++++++++++++++++++- 9 files changed, 194 insertions(+), 14 deletions(-) diff --git a/packages/rath-client/package.json b/packages/rath-client/package.json index 440a1409..f04c30e4 100644 --- a/packages/rath-client/package.json +++ b/packages/rath-client/package.json @@ -47,7 +47,8 @@ "vega-scenegraph": "4.10.1-kanaries-patch", "visual-insights": "^0.12.3", "web-vitals": "^0.2.4", - "worker-loader": "^3.0.7" + "worker-loader": "^3.0.7", + "xlsx": "^0.18.5" }, "devDependencies": { "@testing-library/jest-dom": "^5.11.8", diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index 26530268..f724a54c 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -230,7 +230,7 @@ }, "upload": { "title": "Upload Your own dataset", - "fileTypes": "JSON, CSV, XLSX are supported.", + "fileTypes": "JSON, CSV and Excel files are supported.", "uniqueIdIssue": "Add unique ids for fields", "sampling": "Sampling", "percentSize": "sample size(rows)", @@ -240,8 +240,10 @@ "firstOpen": "First created", "history": "Open Recent", "new": "New file", + "sheet": "Sheet", "preview_parsed": "Preview", - "preview_raw": "Raw" + "preview_raw": "Raw", + "data_is_empty": "This dataset is empty" }, "exploreMode": { "title": "Explore Mode", diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 3d8c092d..1bc26a89 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -230,7 +230,7 @@ }, "upload": { "title": "连接你的数据集，根据需求调整以下配置", - "fileTypes": "支持 JSON, CSV, XLSX 文件", + "fileTypes": "支持 JSON, CSV 及 Excel 文件", "uniqueIdIssue": "添加唯一标识(字段是中文字符推荐使用)", "sampling": "数据采样", "percentSize": "样本大小(行)", @@ -240,8 +240,10 @@ "firstOpen": "创建时间", "history": "最近使用", "new": "新的文件", + "sheet": "工作簿", "preview_parsed": "预览", - "preview_raw": "原始内容" + "preview_raw": "原始内容", + "data_is_empty": "数据集是空的。" }, "exploreMode": { "title": "探索模式", diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx index fbb3bab1..969e015c 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx @@ -1,6 +1,6 @@ import intl from 'react-intl-universal'; import { ChoiceGroup, Dropdown, SpinButton } from "@fluentui/react"; -import { FC } from "react"; +import type { FC } from "react"; import styled from "styled-components"; import { SampleKey, useSampleOptions } from '../../utils'; @@ -66,9 +66,15 @@ export interface IFileHelperProps { sampleSize: number; setSampleSize: (sampleSize: number | ((prev: number) => number)) => void; preview: File | null; + sheetNames: string[] | false; + selectedSheetIdx: number; + setSelectedSheetIdx: (selectedSheetIdx: number) => void; } -const FileHelper: FC = ({ charset, setCharset, sampleMethod, setSampleMethod, sampleSize, setSampleSize, preview }) => { +const FileHelper: FC = ({ + charset, setCharset, sampleMethod, setSampleMethod, sampleSize, setSampleSize, preview, sheetNames, + selectedSheetIdx, setSelectedSheetIdx, +}) => { const sampleOptions = useSampleOptions(); return ( @@ -114,6 +120,15 @@ const FileHelper: FC = ({ charset, setCharset, sampleMethod, s )} ) : null} + {sheetNames && ( + ({ key: `${i}`, text: name }))} + selectedKey={`${selectedSheetIdx}`} + onChange={(_, option) => option?.key && setSelectedSheetIdx(Number(option.key))} + styles={{ root: { padding: '1em 0', display: 'flex', flexDirection: 'row', marginRight: '2em' }, label: { marginRight: '1em', fontWeight: 400 }, dropdown: { width: '10em' } }} + /> + )} ); }; diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx index 13f6093e..7f026772 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx @@ -62,9 +62,14 @@ const ActionGroup = styled.div` > div { width: 100%; height: 100%; - padding: 0.6em 1.2em; overflow: auto; } + & p { + font-size: 0.8rem; + margin: 0.6em 1.2em; + color: #555; + user-select: none; + } } } `; @@ -143,12 +148,12 @@ const FileOutput = styled.div` const RawArea = styled.pre` font-size: 0.8rem; - padding: 0 1em 1em 0; + padding: 0.6em 1.2em 2em; `; const PreviewArea = styled.table` font-size: 0.8rem; - padding: 0 1em 1em 0; + padding: 0 0 1em; & * { white-space: nowrap; } @@ -180,7 +185,11 @@ const FileUpload: FC = ({ preview, previewOfFile, previewOfRaw const handleReset = useCallback(() => { onFileUpload(null); - fileInputRef.current?.click(); + if (fileInputRef.current) { + fileInputRef.current.value = ''; + fileInputRef.current.files = null; + fileInputRef.current.click(); + } }, [onFileUpload]); const handleButtonClick = useCallback(() => { @@ -246,7 +255,7 @@ const FileUpload: FC = ({ preview, previewOfFile, previewOfRaw }} > - {previewOfFile && ( + {previewOfFile ? ( @@ -267,6 +276,8 @@ const FileUpload: FC = ({ preview, previewOfFile, previewOfRaw ))} + ) : ( +

{intl.get("dataSource.upload.data_is_empty")}

)} diff --git a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx index 32dc102f..b1b72679 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx @@ -3,7 +3,7 @@ import { PrimaryButton } from '@fluentui/react'; import styled from "styled-components"; import { observer } from "mobx-react-lite"; import intl from "react-intl-universal"; -import { loadDataFile, readRaw, SampleKey } from "../../utils"; +import { isExcelFile, loadDataFile, loadExcelFile, parseExcelFile, readRaw, SampleKey } from "../../utils"; import { dataBackup, logDataImport } from "../../../../loggers/dataImport"; import type { IMuteFieldBase, IRow } from "../../../../interfaces"; import FileUpload from "./file-upload"; @@ -40,6 +40,13 @@ const FileData: FC = (props) => { const [previewOfRaw, setPreviewOfRaw] = useState(null); const [previewOfFile, setPreviewOfFile] = useState> | null>(null); + const [excelFile, setExcelFile] = useState> | false>(false); + const [selectedSheetIdx, setSelectedSheetIdx] = useState(-1); + + useEffect(() => { + setSelectedSheetIdx(-1); + }, [excelFile]); + const filePreviewPendingRef = useRef>(); useEffect(() => { @@ -47,7 +54,21 @@ const FileData: FC = (props) => { if (preview) { setPreviewOfRaw(null); setPreviewOfFile(null); + setExcelFile(false); toggleLoadingAnimation(true); + if (isExcelFile(preview)) { + const p = parseExcelFile(preview); + filePreviewPendingRef.current = p; + p.then(res => { + if (p !== filePreviewPendingRef.current) { + return; + } + setExcelFile(res); + }).finally(() => { + toggleLoadingAnimation(false); + }); + return; + } const p = Promise.all([ readRaw(preview, charset, 1024, 32, 128), loadDataFile({ @@ -75,6 +96,27 @@ const FileData: FC = (props) => { } }, [charset, onDataLoading, onLoadingFailed, preview, sampleMethod, sampleSize, toggleLoadingAnimation]); + useEffect(() => { + if (excelFile && selectedSheetIdx !== -1) { + setPreviewOfRaw(null); + setPreviewOfFile(null); + filePreviewPendingRef.current = undefined; + toggleLoadingAnimation(true); + const p = loadExcelFile(excelFile, selectedSheetIdx, charset); + filePreviewPendingRef.current = p; + p.then(res => { + if (p !== filePreviewPendingRef.current) { + return; + } + setPreviewOfFile(res); + }).catch(reason => { + onLoadingFailed(reason); + }).finally(() => { + toggleLoadingAnimation(false); + }); + } + }, [excelFile, onLoadingFailed, selectedSheetIdx, toggleLoadingAnimation, charset]); + const handleFileLoad = useCallback((file: File | null) => { setPreview(file); }, []); @@ -106,6 +148,9 @@ const FileData: FC = (props) => { sampleSize={sampleSize} setSampleSize={setSampleSize} preview={preview} + sheetNames={excelFile ? excelFile.SheetNames : false} + selectedSheetIdx={selectedSheetIdx} + setSelectedSheetIdx={setSelectedSheetIdx} /> {preview ? ( diff --git a/packages/rath-client/src/pages/dataSource/utils/index.ts b/packages/rath-client/src/pages/dataSource/utils/index.ts index e30e618b..0532b025 100644 --- a/packages/rath-client/src/pages/dataSource/utils/index.ts +++ b/packages/rath-client/src/pages/dataSource/utils/index.ts @@ -2,6 +2,7 @@ import { Sampling } from 'visual-insights'; import { FileReader as KFileReader } from '@kanaries/web-data-loader' import intl from 'react-intl-universal'; import { useMemo } from "react"; +import * as xlsx from 'xlsx'; import { STORAGE_FILE_SUFFIX } from "../../../constants"; import { FileLoader } from "../../../utils"; import { IMuteFieldBase, IRow } from "../../../interfaces"; @@ -129,6 +130,36 @@ export async function loadDataFile(props: LoadDataFileProps): Promise<{ return dataset } +export const isExcelFile = (file: File): boolean => { + return [ + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // xlsx + 'application/vnd.ms-excel.sheet.binary.macroEnabled.12', // xlsb + 'application/vnd.ms-excel', // xls + 'application/vnd.ms-excel.sheet.macroEnabled.12', // xlsm + ].includes(file.type); +}; + +export const parseExcelFile = async (file: File) => { + const content = await FileLoader.binaryLoader(file); + const data = xlsx.read(content); + return data; +}; + +export const loadExcelFile = async (data: Awaited>, sheetIdx: number, encoding: string): Promise<{ + fields: IMuteFieldBase[]; + dataSource: IRow[]; +}> => { + const sheet = data.SheetNames[sheetIdx]; + const worksheet = data.Sheets[sheet]; + const csvData = xlsx.utils.sheet_to_csv(worksheet, { skipHidden: true }); // more options available here + const csvFile = new File([new Blob([csvData], { type: 'text/plain' })], 'file.csv'); + const rawData = (await KFileReader.csvReader({ + file: csvFile, + encoding, + })) as IRow[]; + return await transformRawDataService(rawData); +}; + export async function loadRathStorageFile (file: File): Promise { // FIXME file type if (file.name.split('.').slice(-1)[0] === STORAGE_FILE_SUFFIX) { diff --git a/packages/rath-client/src/utils/fileParser.ts b/packages/rath-client/src/utils/fileParser.ts index 419b8055..ce37fb2d 100644 --- a/packages/rath-client/src/utils/fileParser.ts +++ b/packages/rath-client/src/utils/fileParser.ts @@ -34,4 +34,19 @@ export function textLoader (file: File): Promise { } reader.onerror = reject }) +} + +export function binaryLoader (file: File): Promise { + return new Promise((resolve, reject) => { + let reader = new FileReader() + reader.readAsArrayBuffer(file) + reader.onload = (ev) => { + if (ev.target) { + resolve(ev.target.result as ArrayBuffer) + } else { + reject(ev) + } + } + reader.onerror = reject + }) } \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 77a211b5..7f1215e3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3452,7 +3452,7 @@ "@types/react-dom@^17.0.1": version "17.0.18" - resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.18.tgz#8f7af38f5d9b42f79162eea7492e5a1caff70dc2" + resolved "https://registry.npmmirror.com/@types/react-dom/-/react-dom-17.0.18.tgz#8f7af38f5d9b42f79162eea7492e5a1caff70dc2" integrity sha512-rLVtIfbwyur2iFKykP2w0pl/1unw26b5td16d5xMgp7/yjTHomkyxPYChFoCr/FtEX1lN9wY6lFj1qvKdS5kDw== dependencies: "@types/react" "^17" @@ -3901,6 +3901,11 @@ adjust-sourcemap-loader@^4.0.0: loader-utils "^2.0.0" regex-parser "^2.2.11" +adler-32@~1.3.0: + version "1.3.1" + resolved "https://registry.npmmirror.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2" + integrity sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A== + agent-base@6: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" @@ -4628,6 +4633,14 @@ case-sensitive-paths-webpack-plugin@^2.4.0: resolved "https://registry.npmmirror.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4" integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== +cfb@~1.2.1: + version "1.2.2" + resolved "https://registry.npmmirror.com/cfb/-/cfb-1.2.2.tgz#94e687628c700e5155436dac05f74e08df23bc44" + integrity sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA== + dependencies: + adler-32 "~1.3.0" + crc-32 "~1.2.0" + chalk@^1.0.0, chalk@^1.1.3: version "1.1.3" resolved "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -4800,6 +4813,11 @@ codemirror@^6.0.1: "@codemirror/state" "^6.0.0" "@codemirror/view" "^6.0.0" +codepage@~1.15.0: + version "1.15.0" + resolved "https://registry.npmmirror.com/codepage/-/codepage-1.15.0.tgz#2e00519024b39424ec66eeb3ec07227e692618ab" + integrity sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA== + collect-v8-coverage@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz#cc2c8e94fc18bbdffe64d6534570c8a673b27f59" @@ -5041,6 +5059,11 @@ cosmiconfig@^7.0.0, cosmiconfig@^7.0.1: path-type "^4.0.0" yaml "^1.10.0" +crc-32@~1.2.0, crc-32@~1.2.1: + version "1.2.2" + resolved "https://registry.npmmirror.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" + integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -6676,6 +6699,11 @@ forwarded@0.2.0: resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== +frac@~1.1.2: + version "1.1.2" + resolved "https://registry.npmmirror.com/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b" + integrity sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA== + fraction.js@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-4.2.0.tgz#448e5109a313a3527f5a3ab2119ec4cf0e0e2950" @@ -11711,6 +11739,13 @@ sqlstring@^2.3.2: resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.3.tgz#2ddc21f03bce2c387ed60680e739922c65751d0c" integrity sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg== +ssf@~0.11.2: + version "0.11.2" + resolved "https://registry.npmmirror.com/ssf/-/ssf-0.11.2.tgz#0b99698b237548d088fc43cdf2b70c1a7512c06c" + integrity sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g== + dependencies: + frac "~1.1.2" + stable@^0.1.8: version "0.1.8" resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf" @@ -13303,11 +13338,21 @@ which@^2.0.1: dependencies: isexe "^2.0.0" +wmf@~1.0.1: + version "1.0.2" + resolved "https://registry.npmmirror.com/wmf/-/wmf-1.0.2.tgz#7d19d621071a08c2bdc6b7e688a9c435298cc2da" + integrity sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw== + word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== +word@~0.3.0: + version "0.3.0" + resolved "https://registry.npmmirror.com/word/-/word-0.3.0.tgz#8542157e4f8e849f4a363a288992d47612db9961" + integrity sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA== + workbox-background-sync@6.5.4: version "6.5.4" resolved "https://registry.npmmirror.com/workbox-background-sync/-/workbox-background-sync-6.5.4.tgz#3141afba3cc8aa2ae14c24d0f6811374ba8ff6a9" @@ -13535,6 +13580,19 @@ ws@^8.4.2: resolved "https://registry.npmmirror.com/ws/-/ws-8.9.0.tgz#2a994bb67144be1b53fe2d23c53c028adeb7f45e" integrity sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg== +xlsx@^0.18.5: + version "0.18.5" + resolved "https://registry.npmmirror.com/xlsx/-/xlsx-0.18.5.tgz#16711b9113c848076b8a177022799ad356eba7d0" + integrity sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ== + dependencies: + adler-32 "~1.3.0" + cfb "~1.2.1" + codepage "~1.15.0" + crc-32 "~1.2.1" + ssf "~0.11.2" + wmf "~1.0.1" + word "~0.3.0" + xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" From 9027a29a10fbfef80066421233d54b7e6172dcef Mon Sep 17 00:00:00 2001 From: kyusho Date: Tue, 6 Dec 2022 11:05:50 +0800 Subject: [PATCH 05/13] feat(datasource): support csv-like text file with customize separator --- .../rath-client/public/locales/en-US.json | 11 +- .../rath-client/public/locales/zh-CN.json | 11 +- .../dataSource/selection/file/file-helper.tsx | 105 +++++++++++++----- .../pages/dataSource/selection/file/index.tsx | 21 +++- .../src/pages/dataSource/utils/index.ts | 33 +++++- 5 files changed, 144 insertions(+), 37 deletions(-) diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index f724a54c..a5ad64aa 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -120,6 +120,7 @@ "dataView": "Table View", "statView": "Statistics View", "charset": "character encoding", + "separator": "Separator", "databaseType": "Select Database", "connectUri": "Connection URI", "databaseName": "Database", @@ -192,7 +193,7 @@ "buttonName": "Import Data", "load": "Load Data", "type": { - "file": "Local", + "file": "File", "restful": "RESTFUL", "mysql": "MySQL", "demo": "Demo", @@ -243,7 +244,13 @@ "sheet": "Sheet", "preview_parsed": "Preview", "preview_raw": "Raw", - "data_is_empty": "This dataset is empty" + "data_is_empty": "This dataset is empty", + "separator": { + "comma": "Comma", + "semicolon": "Semicolon", + "tab": "Tab", + "other": "Other..." + } }, "exploreMode": { "title": "Explore Mode", diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 1bc26a89..74832da2 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -119,6 +119,7 @@ "dataView": "数据视图", "statView": "统计视图", "charset": "字符集编码", + "separator": "分隔符", "databaseType": "选择数据库", "connectUri": "连接 URI", "databaseName": "数据库", @@ -191,7 +192,7 @@ "importData": { "buttonName": "选择数据", "type": { - "file": "本地", + "file": "文件", "restful": "RESTFUL", "mysql": "MySQL", "demo": "示例数据", @@ -243,7 +244,13 @@ "sheet": "工作簿", "preview_parsed": "预览", "preview_raw": "原始内容", - "data_is_empty": "数据集是空的。" + "data_is_empty": "数据集是空的。", + "separator": { + "comma": "逗号", + "semicolon": "分号", + "tab": "制表符", + "other": "自定义..." + } }, "exploreMode": { "title": "探索模式", diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx index 969e015c..027d7855 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx @@ -1,6 +1,6 @@ import intl from 'react-intl-universal'; -import { ChoiceGroup, Dropdown, SpinButton } from "@fluentui/react"; -import type { FC } from "react"; +import { ChoiceGroup, Dropdown, IChoiceGroupOption, SpinButton, TextField } from "@fluentui/react"; +import { FC, useMemo, useState } from "react"; import styled from "styled-components"; import { SampleKey, useSampleOptions } from '../../utils'; @@ -36,9 +36,8 @@ export type Charset = typeof charsetOptions[number]['key']; const Container = styled.div` display: flex; - flex-direction: row; - flex-wrap: wrap; - align-items: center; + flex-direction: column; + padding-top: 1em; & label { font-weight: 400; margin-right: 1em; @@ -47,7 +46,6 @@ const Container = styled.div` display: flex; flex-direction: row; align-items: center; - padding: 1em 0; > div { display: flex; flex-direction: row; @@ -56,6 +54,9 @@ const Container = styled.div` } } } + > * { + margin-bottom: 0.8em; + } `; export interface IFileHelperProps { @@ -69,13 +70,47 @@ export interface IFileHelperProps { sheetNames: string[] | false; selectedSheetIdx: number; setSelectedSheetIdx: (selectedSheetIdx: number) => void; + separator: string; + setSeparator: (separator: string) => void; } const FileHelper: FC = ({ charset, setCharset, sampleMethod, setSampleMethod, sampleSize, setSampleSize, preview, sheetNames, - selectedSheetIdx, setSelectedSheetIdx, + selectedSheetIdx, setSelectedSheetIdx, separator, setSeparator, }) => { const sampleOptions = useSampleOptions(); + const [customizeSeparator, setCustomizeSeparator] = useState(''); + + const separatorOptions = useMemo(() => { + return [ + { key: ',', text: intl.get('dataSource.upload.separator.comma') }, + { key: '\t', text: intl.get('dataSource.upload.separator.tab') }, + { key: ';', text: intl.get('dataSource.upload.separator.semicolon') }, + { + key: '', + text: intl.get('dataSource.upload.separator.other'), + onRenderField(props, defaultRenderer) { + return ( +
+ {defaultRenderer?.(props)} + { + setCustomizeSeparator(value ?? ''); + if (value) { + setSeparator(value); + } + }} + /> +
+ ); + }, + }, + ]; + }, [customizeSeparator, setSeparator]); + + const selectedSeparatorKey = separatorOptions.find(opt => opt.key === separator)?.key ?? ''; return ( @@ -89,34 +124,48 @@ const FileHelper: FC = ({ }} styles={{ root: { display: 'flex', flexDirection: 'row', marginRight: '2em' }, label: { marginRight: '1em', fontWeight: 400 }, dropdown: { width: '8em' } }} /> - {!preview || preview.name.endsWith('.csv') ? ( + {!preview || preview.type.match(/^text\/.*/) ? ( <> { if (option) { - setSampleMethod(option.key as SampleKey); + setSeparator(option.key); } }} /> - {sampleMethod === SampleKey.reservoir && ( - { - setSampleSize(Number(value)); - }} - onIncrement={() => { - setSampleSize((v) => v + 1); - }} - onDecrement={() => { - setSampleSize((v) => Math.max(v - 1, 0)); - }} - /> + {(!preview || preview.type === 'text/csv') && separator === ',' && ( + <> + { + if (option) { + setSampleMethod(option.key as SampleKey); + } + }} + /> + {sampleMethod === SampleKey.reservoir && ( + { + setSampleSize(Number(value)); + }} + onIncrement={() => { + setSampleSize((v) => v + 1); + }} + onDecrement={() => { + setSampleSize((v) => Math.max(v - 1, 0)); + }} + /> + )} + )} ) : null} diff --git a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx index b1b72679..f9380308 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx @@ -35,6 +35,20 @@ const FileData: FC = (props) => { const [sampleMethod, setSampleMethod] = useState(SampleKey.none); const [sampleSize, setSampleSize] = useState(500); const [charset, setCharset] = useState('utf-8'); + const [separator, setSeparator] = useState(','); + const [appliedSeparator, setAppliedSeparator] = useState(separator); + + useEffect(() => { + setPreviewOfRaw(null); + setPreviewOfFile(null); + setExcelFile(false); + const delay = setTimeout(() => { + setAppliedSeparator(separator); + }, 1_000); + return () => { + clearTimeout(delay); + }; + }, [separator]); const [preview, setPreview] = useState(null); const [previewOfRaw, setPreviewOfRaw] = useState(null); @@ -76,7 +90,8 @@ const FileData: FC = (props) => { sampleMethod, sampleSize, encoding: charset, - onLoading: onDataLoading + onLoading: onDataLoading, + separator: appliedSeparator, }), ]); filePreviewPendingRef.current = p; @@ -94,7 +109,7 @@ const FileData: FC = (props) => { } else { setPreviewOfFile(null); } - }, [charset, onDataLoading, onLoadingFailed, preview, sampleMethod, sampleSize, toggleLoadingAnimation]); + }, [charset, onDataLoading, onLoadingFailed, preview, sampleMethod, sampleSize, toggleLoadingAnimation, appliedSeparator]); useEffect(() => { if (excelFile && selectedSheetIdx !== -1) { @@ -151,6 +166,8 @@ const FileData: FC = (props) => { sheetNames={excelFile ? excelFile.SheetNames : false} selectedSheetIdx={selectedSheetIdx} setSelectedSheetIdx={setSelectedSheetIdx} + separator={separator} + setSeparator={setSeparator} /> {preview ? ( diff --git a/packages/rath-client/src/pages/dataSource/utils/index.ts b/packages/rath-client/src/pages/dataSource/utils/index.ts index 0532b025..8bd5a1c0 100644 --- a/packages/rath-client/src/pages/dataSource/utils/index.ts +++ b/packages/rath-client/src/pages/dataSource/utils/index.ts @@ -82,6 +82,7 @@ interface LoadDataFileProps { sampleSize?: number; encoding?: string; onLoading?: (progress: number) => void; + separator: string; } export async function loadDataFile(props: LoadDataFileProps): Promise<{ fields: IMuteFieldBase[]; @@ -92,15 +93,28 @@ export async function loadDataFile(props: LoadDataFileProps): Promise<{ sampleMethod, sampleSize = 500, encoding = 'utf-8', - onLoading + onLoading, + separator, } = props; /** * tmpFields is fields cat by specific rules, the results is not correct sometimes, waitting for human's input */ let rawData: IRow[] = [] - if (file.type === 'text/csv' || file.type === 'application/vnd.ms-excel') { - rawData = [] + if (file.type.match(/^text\/.*/)) { // csv-like text files + if (separator && separator !== ',') { + const content = (await readRaw(file, encoding) ?? ''); + const rows = content.split(/\r?\n/g).map(row => row.split(separator)); + const fields = rows[0]?.map((h, i) => ({ + fid: `col_${i}`, + name: h, + geoRole: '?', + analyticType: '?', + semanticType: '?', + })); + const dataSource = rows.slice(1).map(row => Object.fromEntries(fields.map((f, i) => [f.fid, row[i]]))); + return { fields, dataSource }; + } if (sampleMethod === SampleKey.reservoir) { rawData = (await KFileReader.csvReader({ file, @@ -123,6 +137,19 @@ export async function loadDataFile(props: LoadDataFileProps): Promise<{ if (sampleMethod === SampleKey.reservoir) { rawData = Sampling.reservoirSampling(rawData, sampleSize) } + } else if (file.type.startsWith('text/')) { + let content = (await readRaw(file, encoding) ?? ''); + if (separator && separator !== ',') { + content = content.replaceAll( + new RegExp(`\\\\{0, 2}${separator}`, 'g'), + part => `${new RegExp(`^(\\{2})?`).exec(part)?.[0] ?? ''},`, + ); + } + const translatedFile = new File([new Blob([content], { type: 'text/plain' })], 'file.csv'); + rawData = (await KFileReader.csvReader({ + file: translatedFile, + encoding, + })) as IRow[]; } else { throw new Error(`unsupported file type=${file.type} `) } From 30f3f42a2227f964eafa7efc9c6911eb3bd0ca1f Mon Sep 17 00:00:00 2001 From: kyusho Date: Tue, 6 Dec 2022 15:21:39 +0800 Subject: [PATCH 06/13] feat(datasource): store tags and source --- .../src/pages/dataSource/config.ts | 10 +- .../src/pages/dataSource/index.tsx | 8 +- .../dataSource/selection/airtable/index.tsx | 5 +- .../dataSource/selection/database/index.tsx | 5 +- .../src/pages/dataSource/selection/demo.tsx | 114 +++++-- .../dataSource/selection/file/file-upload.tsx | 4 +- .../selection/file/history-list.tsx | 194 ------------ .../pages/dataSource/selection/file/index.tsx | 20 +- .../{file => history}/get-file-icon.tsx | 0 .../selection/history/history-list.tsx | 279 ++++++++++++++++++ .../src/pages/dataSource/selection/index.tsx | 11 +- .../src/pages/dataSource/selection/local.tsx | 104 ------- .../src/pages/dataSource/selection/olap.tsx | 5 +- .../pages/dataSource/selection/restful.tsx | 5 +- packages/rath-client/src/utils/storage.ts | 60 +++- 15 files changed, 468 insertions(+), 356 deletions(-) delete mode 100644 packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx rename packages/rath-client/src/pages/dataSource/selection/{file => history}/get-file-icon.tsx (100%) create mode 100644 packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx delete mode 100644 packages/rath-client/src/pages/dataSource/selection/local.tsx diff --git a/packages/rath-client/src/pages/dataSource/config.ts b/packages/rath-client/src/pages/dataSource/config.ts index 28ad5238..e281e091 100644 --- a/packages/rath-client/src/pages/dataSource/config.ts +++ b/packages/rath-client/src/pages/dataSource/config.ts @@ -6,11 +6,17 @@ import { IDataSourceType } from "../../global"; export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceType; text: string }> { const fileText = intl.get(`dataSource.importData.type.${IDataSourceType.FILE}`); const restfulText = intl.get(`dataSource.importData.type.${IDataSourceType.RESTFUL}`); - const demoText = intl.get(`dataSource.importData.type.${IDataSourceType.DEMO}`) + const demoText = intl.get(`dataSource.importData.type.${IDataSourceType.DEMO}`); const dbText = intl.get(`dataSource.importData.type.${IDataSourceType.DATABASE}`); + const historyText = intl.get('common.history'); const options = useMemo>(() => { return [ + { + key: IDataSourceType.LOCAL, + text: historyText, + iconProps: { iconName: "History" }, + }, { key: IDataSourceType.FILE, text: fileText, @@ -44,7 +50,7 @@ export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceTyp disabled: false }, ]; - }, [fileText, restfulText, demoText, dbText]); + }, [fileText, restfulText, demoText, dbText, historyText]); return options; }; diff --git a/packages/rath-client/src/pages/dataSource/index.tsx b/packages/rath-client/src/pages/dataSource/index.tsx index 092222bd..4b3c5dbb 100644 --- a/packages/rath-client/src/pages/dataSource/index.tsx +++ b/packages/rath-client/src/pages/dataSource/index.tsx @@ -14,7 +14,7 @@ import { observer } from 'mobx-react-lite'; import { useGlobalStore } from '../../store'; import { IDataPrepProgressTag, IDataPreviewMode, IMuteFieldBase, IRow } from '../../interfaces'; import { Card } from '../../components/card'; -import { setDataStorage } from '../../utils/storage'; +import { DataSourceTag, IDBMeta, setDataStorage } from '../../utils/storage'; import { BorderCard } from '../../components/borderCard'; import DataTable from './dataTable/index'; import MetaView from './metaView/index'; @@ -73,11 +73,11 @@ const DataSourceBoard: React.FC = (props) => { }, [dataSourceStore]); const onSelectDataLoaded = useCallback( - (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => { + (fields: IMuteFieldBase[], dataSource: IRow[], name?: string, tag?: DataSourceTag | undefined, withHistory?: IDBMeta | undefined) => { dataSourceStore.loadDataWithInferMetas(dataSource, fields); - if (name) { + if (name && tag !== undefined) { dataSourceStore.setDatasetId(name); - setDataStorage(name, fields, dataSource); + setDataStorage(name, fields, dataSource, tag, withHistory); } }, [dataSourceStore] diff --git a/packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx b/packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx index 51174d5e..d49febf9 100644 --- a/packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/airtable/index.tsx @@ -4,6 +4,7 @@ import intl from 'react-intl-universal' import { logDataImport } from '../../../../loggers/dataImport'; import { IMuteFieldBase, IRow } from '../../../../interfaces'; import { transformRawDataService } from '../../utils'; +import { DataSourceTag, IDBMeta } from '../../../../utils/storage'; import { fetchAllRecordsFromAirTable } from './utils'; @@ -11,7 +12,7 @@ interface AirTableSourceProps { onClose: () => void; onStartLoading: () => void; onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: string, tag: DataSourceTag, withHistory?: IDBMeta | undefined) => void; } const AirTableSource: React.FC = (props) => { const { onClose, onDataLoaded, onLoadingFailed, onStartLoading } = props; @@ -33,7 +34,7 @@ const AirTableSource: React.FC = (props) => { .then((data) => transformRawDataService(data)) .then((ds) => { const name = `airtable-${tableName}-${viewName}`; - onDataLoaded(ds.fields, ds.dataSource, name); + onDataLoaded(ds.fields, ds.dataSource, name, DataSourceTag.AIR_TABLE); logDataImport({ dataType: 'AirTable', fields: ds.fields, diff --git a/packages/rath-client/src/pages/dataSource/selection/database/index.tsx b/packages/rath-client/src/pages/dataSource/selection/database/index.tsx index 6016b171..1561c151 100644 --- a/packages/rath-client/src/pages/dataSource/selection/database/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/database/index.tsx @@ -5,6 +5,7 @@ import type { IMuteFieldBase, IRow } from '../../../../interfaces'; import { logDataImport } from '../../../../loggers/dataImport'; import prefetch from '../../../../utils/prefetch'; import { notify } from '../../../../components/error'; +import { DataSourceTag } from '../../../../utils/storage'; import { transformRawDataService } from '../../utils'; import Progress from './progress'; import datasetOptions from './config'; @@ -59,7 +60,7 @@ export interface TableData { interface DatabaseDataProps { onClose: () => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: string, tag: DataSourceTag) => void; setLoadingAnimation: (on: boolean) => void; } @@ -203,7 +204,7 @@ const DatabaseData: React.FC = ({ onClose, onDataLoaded, setL dataSource: dataSource.slice(0, 10), size: dataSource.length, }); - onDataLoaded(fields, dataSource, name); + onDataLoaded(fields, dataSource, name, DataSourceTag.DATABASE); onClose(); } catch (error) { diff --git a/packages/rath-client/src/pages/dataSource/selection/demo.tsx b/packages/rath-client/src/pages/dataSource/selection/demo.tsx index f05e3dfa..c02f0d94 100644 --- a/packages/rath-client/src/pages/dataSource/selection/demo.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/demo.tsx @@ -1,17 +1,21 @@ -import { ChoiceGroup, DefaultButton, Label } from '@fluentui/react'; -import React, { useCallback, useState } from 'react'; +import { Icon, Label } from '@fluentui/react'; +import { FC, useCallback, useMemo, useRef } from 'react'; import { useId } from "@fluentui/react-hooks"; import intl from 'react-intl-universal'; +import styled from 'styled-components'; import { DemoDataAssets, IDemoDataKey, useDemoDataOptions } from '../config'; import { logDataImport } from '../../../loggers/dataImport'; import { IDatasetBase, IMuteFieldBase, IRow } from '../../../interfaces'; import { DEMO_DATA_REQUEST_TIMEOUT } from '../../../constants'; +import { DataSourceTag, IDBMeta } from '../../../utils/storage'; +import useBoundingClientRect from '../../../hooks/use-bounding-client-rect'; +import getFileIcon from './history/get-file-icon'; interface DemoDataProps { onClose: () => void; onStartLoading: () => void; onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: string, tag: DataSourceTag, withHistory?: IDBMeta | undefined) => void; } function valueFix (ds: IDatasetBase): IDatasetBase { @@ -52,21 +56,71 @@ function requestDemoData (dsKey: IDemoDataKey = 'CARS'): Promise { // fields: [] // } // } -} +} + +export const RathDemoVirtualExt = 'rath-demo.json'; + +const List = styled.div` + margin: 1em 0; + min-height: 8em; + max-height: 50vh; + width: 44vw; + overflow: hidden auto; + display: grid; + gap: 0.4em; + grid-auto-rows: max-content; +`; -const DemoData: React.FC = props => { +const ListItem = styled.div` + display: flex; + flex-direction: column; + overflow: hidden; + height: 100%; + padding: 1.2em 1em 1em 1.4em; + border-radius: 2px; + position: relative; + box-shadow: inset 0 0 2px #8881; + > .head { + display: flex; + align-items: flex-start; + > i { + flex-grow: 0; + flex-shrink: 0; + width: 2em; + height: 2em; + margin-right: 0.8em; + user-select: none; + } + > header { + font-size: 0.8rem; + line-height: 1.2em; + font-weight: 550; + color: #111; + flex-grow: 1; + flex-shrink: 1; + overflow: hidden; + } + } + :hover { + background-color: #8881; + } + cursor: pointer; +`; + +const ITEM_MIN_WIDTH = 240; + +const DemoData: FC = props => { const { onDataLoaded, onClose, onStartLoading, onLoadingFailed } = props; const options = useDemoDataOptions(); - const [dsKey, setDSKey] = useState('CARS'); - const loadData = useCallback(() => { + const loadDemo = useCallback((demo: typeof options[number]) => { onStartLoading(); - requestDemoData(dsKey).then(data => { + requestDemoData(demo.key).then(data => { const { dataSource, fields } = data; - onDataLoaded(fields, dataSource, 'rdemo_' + dsKey); + onDataLoaded(fields, dataSource, `${demo.text}.${RathDemoVirtualExt}`, DataSourceTag.DEMO); logDataImport({ dataType: "Demo", - name: dsKey, + name: demo.key, fields, dataSource: [], size: dataSource.length, @@ -75,24 +129,38 @@ const DemoData: React.FC = props => { onLoadingFailed(err); }) onClose(); - }, [dsKey, onDataLoaded, onClose, onStartLoading, onLoadingFailed]) + }, [onClose, onDataLoaded, onLoadingFailed, onStartLoading]); const labelId = useId('demo-ds'); + + const listRef = useRef(null); + const { width } = useBoundingClientRect(listRef, { width: true }); + const colCount = useMemo(() => Math.floor((width ?? (window.innerWidth * 0.6)) / ITEM_MIN_WIDTH), [width]); + return (
- { - if (option) { - setDSKey(option.key as IDemoDataKey); - } - }} - /> -
- -
+ + {options.map((demo, i) => { + return ( + loadDemo(demo)} + > +
+ +
+ {demo.text} +
+
+ + ); + })} +
); } diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx index 7f026772..d26d7dc1 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx @@ -5,8 +5,8 @@ import { FC, useCallback, useRef, useState } from "react"; import styled from "styled-components"; import type { loadDataFile } from "../../utils"; import { notify } from "../../../../components/error"; -import getFileIcon from "./get-file-icon"; -import { formatSize } from "./history-list"; +import getFileIcon from "../history/get-file-icon"; +import { formatSize } from "../history/history-list"; const Container = styled.div` diff --git a/packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx b/packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx deleted file mode 100644 index cb8c2785..00000000 --- a/packages/rath-client/src/pages/dataSource/selection/file/history-list.tsx +++ /dev/null @@ -1,194 +0,0 @@ -import intl from 'react-intl-universal'; -import { Icon, IconButton, TooltipHost } from "@fluentui/react"; -import { observer } from "mobx-react-lite"; -import dayjs from 'dayjs'; -import { FC, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import styled from "styled-components"; -import useBoundingClientRect from "../../../../hooks/use-bounding-client-rect"; -import { deleteDataStorageById, getDataStorageById, getDataStorageList, IDBMeta } from "../../../../utils/storage"; -import type { IMuteFieldBase, IRow } from '../../../../interfaces'; -import getFileIcon from "./get-file-icon"; - - -const List = styled.div` - margin: 1em 0; - height: 24em; - overflow: hidden auto; - display: grid; - gap: 0.4em; - grid-auto-rows: max-content; -`; - -const ListItem = styled.div` - display: flex; - flex-direction: column; - overflow: hidden; - height: 100%; - padding: 0.6em 1em 1.2em; - border-radius: 2px; - position: relative; - > .head { - display: flex; - align-items: center; - > i { - flex-grow: 0; - flex-shrink: 0; - width: 2em; - height: 2em; - margin-right: 0.8em; - } - > div { - flex-grow: 1; - flex-shrink: 1; - overflow: hidden; - > header { - font-size: 0.8rem; - line-height: 1.2em; - font-weight: 550; - white-space: nowrap; - color: #111; - } - > span { - font-size: 0.6rem; - line-height: 1.2em; - color: #555; - } - } - } - .time { - font-size: 0.5rem; - color: #888; - } - > button { - position: absolute; - top: 0; - right: 0; - margin: 0; - padding: 0; - font-size: 12px; - background-color: #d13438 !important; - border-radius: 50%; - color: #fff !important; - width: 1.4em; - height: 1.4em; - i { - font-weight: 1000; - line-height: 1em; - width: 1em; - height: 1em; - transform: scale(0.5); - } - visibility: hidden; - opacity: 0.5; - :hover { - opacity: 1; - } - } - :hover { - background-color: #88888818; - > button { - visibility: visible; - } - } - cursor: pointer; -`; - -const ITEM_MIN_WIDTH = 240; -const MAX_HISTORY_SIZE = 64; -const MAX_RECENT_TIME = 1_000 * 60 * 60 * 24 * 31 * 3; // 3 months - -export function formatSize(size: number) { - if (size < 1024) { - return `${size}B`; - } - if (size < 1024 * 1024) { - return `${(size / 1024).toFixed(2)}KB`; - } - if (size < 1024 * 1024 * 1024) { - return `${(size / 1024 / 1024).toFixed(2)}MB`; - } - return `${(size / 1024 / 1024 / 1024).toFixed(2)}GB`; -} - -interface IHistoryListProps { - onClose: () => void; - onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; -} - -const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFailed }) => { - const [localDataList, setLocalDataList] = useState([]); - - useEffect(() => { - getDataStorageList().then((dataList) => { - const recent = dataList.filter(item => Date.now() - item.editTime < MAX_RECENT_TIME); - const sorted = recent.sort((a, b) => b.editTime - a.editTime); - setLocalDataList(sorted.slice(0, MAX_HISTORY_SIZE)); - }); - }, []); - - const listRef = useRef(null); - const { width } = useBoundingClientRect(listRef, { width: true }); - const colCount = useMemo(() => Math.floor((width ?? (window.innerWidth * 0.6)) / ITEM_MIN_WIDTH), [width]); - - const handleLoadHistory = useCallback((id: string) => { - getDataStorageById(id).then(res => { - onDataLoaded(res.fields, res.dataSource, id); - onClose(); - }).catch(onLoadingFailed); - }, [onClose, onDataLoaded, onLoadingFailed]); - - const handleDeleteHistory = useCallback((id: string) => { - deleteDataStorageById(id).then(() => { - getDataStorageList().then((dataList) => { - setLocalDataList(dataList); - }); - }); - }, []); - - return ( - - {localDataList.map((file, i) => { - const ext = /(?<=\.)[^.]+$/.exec(file.name)?.[0]; - - return ( - handleLoadHistory(file.id)} - > -
- -
-
- - {file.name} - -
- - {`${ext ? `${ext} - ` : ''}${formatSize(file.size)}`} - -
-
-
-

{`${intl.get('dataSource.upload.lastOpen')}: ${dayjs(file.editTime).toDate().toLocaleString()}`}

-
- { - e.stopPropagation(); - handleDeleteHistory(file.id); - }} - /> - - ); - })} - - ); -}; - -export default observer(HistoryList); diff --git a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx index f9380308..3c903f38 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx @@ -6,8 +6,9 @@ import intl from "react-intl-universal"; import { isExcelFile, loadDataFile, loadExcelFile, parseExcelFile, readRaw, SampleKey } from "../../utils"; import { dataBackup, logDataImport } from "../../../../loggers/dataImport"; import type { IMuteFieldBase, IRow } from "../../../../interfaces"; +import { DataSourceTag, IDBMeta } from "../../../../utils/storage"; +import HistoryList from "../history/history-list"; import FileUpload from "./file-upload"; -import HistoryList from "./history-list"; import FileHelper, { Charset } from "./file-helper"; @@ -25,7 +26,7 @@ const Container = styled.div` interface FileDataProps { onClose: () => void; onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: string, tag: DataSourceTag, withHistory?: IDBMeta | undefined) => void; onDataLoading: (p: number) => void; toggleLoadingAnimation: (on: boolean) => void; } @@ -83,7 +84,7 @@ const FileData: FC = (props) => { }); return; } - const p = Promise.all([ + const p = Promise.allSettled([ readRaw(preview, charset, 1024, 32, 128), loadDataFile({ file: preview, @@ -99,8 +100,8 @@ const FileData: FC = (props) => { if (p !== filePreviewPendingRef.current) { return; } - setPreviewOfRaw(res[0]); - setPreviewOfFile(res[1]); + setPreviewOfRaw(res[0].status === 'fulfilled' ? res[0].value : null); + setPreviewOfFile(res[1].status === 'fulfilled' ? res[1].value : null); }).catch(reason => { onLoadingFailed(reason); }).finally(() => { @@ -148,7 +149,7 @@ const FileData: FC = (props) => { size: dataSource.length }); dataBackup(preview); - onDataLoaded(fields, dataSource, preview.name); + onDataLoaded(fields, dataSource, preview.name, DataSourceTag.FILE); onClose(); }, [onClose, onDataLoaded, preview, previewOfFile]); @@ -182,7 +183,12 @@ const FileData: FC = (props) => { ) : ( <>
{intl.get('dataSource.upload.history')}
- + )} diff --git a/packages/rath-client/src/pages/dataSource/selection/file/get-file-icon.tsx b/packages/rath-client/src/pages/dataSource/selection/history/get-file-icon.tsx similarity index 100% rename from packages/rath-client/src/pages/dataSource/selection/file/get-file-icon.tsx rename to packages/rath-client/src/pages/dataSource/selection/history/get-file-icon.tsx diff --git a/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx new file mode 100644 index 00000000..5a8339dc --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx @@ -0,0 +1,279 @@ +import intl from 'react-intl-universal'; +import { Icon, IconButton, TooltipHost } from "@fluentui/react"; +import { observer } from "mobx-react-lite"; +import dayjs from 'dayjs'; +import { FC, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import styled from "styled-components"; +import useBoundingClientRect from "../../../../hooks/use-bounding-client-rect"; +import { DataSourceTag, deleteDataStorageById, getDataStorageById, getDataStorageList, IDBMeta, updateDataStorageUserTagGroup, UserTagGroup, userTagGroupColors } from "../../../../utils/storage"; +import type { IMuteFieldBase, IRow } from '../../../../interfaces'; +import { RathDemoVirtualExt } from '../demo'; +import { IDataSourceType } from '../../../../global'; +import getFileIcon from './get-file-icon'; + + +const allUserTagGroups = Object.keys(userTagGroupColors) as unknown as UserTagGroup[]; + +const UserTagGroupSize = 12; +const UserTagGroupPadding = 2; + +const List = styled.div` + margin: 1em 0; + min-height: 8em; + max-height: 50vh; + max-width: 50vw; + overflow: hidden auto; + display: grid; + gap: 0.4em; + grid-auto-rows: max-content; +`; + +const ListItem = styled.div` + display: flex; + flex-direction: column; + overflow: hidden; + min-width: ${(UserTagGroupSize + UserTagGroupPadding * 2) * ((allUserTagGroups.length - 1) * 0.8 + 1) + 10}px; + height: 100%; + padding: 1.2em 1em 1em 1.4em; + border-radius: 2px; + position: relative; + box-shadow: inset 0 0 2px #8881; + > .head { + display: flex; + align-items: center; + > i { + flex-grow: 0; + flex-shrink: 0; + width: 2em; + height: 2em; + margin-right: 0.8em; + user-select: none; + } + > div { + flex-grow: 1; + flex-shrink: 1; + overflow: hidden; + > header { + font-size: 0.8rem; + line-height: 1.2em; + font-weight: 550; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; + color: #111; + } + > span { + font-size: 0.6rem; + line-height: 1.2em; + color: #555; + } + } + } + .time { + font-size: 0.5rem; + color: #888; + } + > button { + position: absolute; + top: 0; + right: 0; + margin: 0; + padding: 0; + font-size: 12px; + background-color: #d13438 !important; + border-radius: 50%; + color: #fff !important; + width: 1.2em; + height: 1.2em; + i { + font-weight: 1000; + line-height: 1em; + width: 1em; + height: 1em; + transform: scale(0.4); + } + opacity: 0.5; + :hover { + opacity: 1; + } + } + & .hover-only:not([aria-selected=true]) { + visibility: hidden; + } + :hover { + background-color: #8881; + & .hover-only { + visibility: visible; + } + } + cursor: pointer; +`; + +const UserTagGroupContainer = styled.div` + position: absolute; + left: 0; + top: 0; + display: flex; + flex-direction: row; + background-image: linear-gradient(to bottom, #8881, transparent 5px); + padding: 0 5px; + > svg { + margin: 0 ${UserTagGroupPadding}px; + cursor: pointer; + transition: transform 200ms; + transform: translateY(-67%); + opacity: 0.2; + :hover { + opacity: 0.95; + transform: translateY(-4px); + } + &[aria-selected=true] { + opacity: 1; + transform: translateY(-2px); + } + > * { + pointer-events: none; + filter: drop-shadow(0.8px 1px 0.6px #888); + } + :not(:first-child) { + margin-left: ${-0.2 * UserTagGroupSize - UserTagGroupPadding}px; + } + } +`; + +const ITEM_MIN_WIDTH = 200; +const MAX_HISTORY_SIZE = 64; +const MAX_RECENT_TIME = 1_000 * 60 * 60 * 24 * 31 * 3; // 3 months + +export function formatSize(size: number) { + if (size < 1024) { + return `${size.toFixed(2)}KB`; + } + if (size < 1024 * 1024) { + return `${(size / 1024).toFixed(2)}MB`; + } + return `${(size / 1024 / 1024).toFixed(2)}GB`; +} + +interface IHistoryListProps { + onClose: () => void; + onLoadingFailed: (err: any) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: string, tag: DataSourceTag, withHistory: IDBMeta) => void; + is?: DataSourceTag; +} + +const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFailed, is }) => { + const [localDataList, setLocalDataList] = useState([]); + const prevList = useRef(localDataList); + prevList.current = localDataList; + + const fetchDataStorageList = useCallback((sort = true) => { + getDataStorageList().then((dataList) => { + if (sort === false) { + const next = prevList.current.map(item => dataList.find(which => which.id === item.id)!).filter(Boolean); + setLocalDataList(next); + return; + } + const list = is === undefined ? dataList : dataList.filter(item => item.tag === is); + const recent = list.filter(item => item.userTagGroup !== undefined || Date.now() - item.editTime < MAX_RECENT_TIME); + const sorted = recent.sort((a, b) => b.editTime - a.editTime).sort( + (a, b) => (a.userTagGroup ?? 1023) - (b.userTagGroup ?? 1023) + ); + const tagged = sorted.filter(item => item.userTagGroup !== undefined).length; + const next = sorted.slice(0, tagged + MAX_HISTORY_SIZE); + setLocalDataList(next); + }); + }, [is]); + + useEffect(() => { + fetchDataStorageList(); + }, [fetchDataStorageList]); + + const listRef = useRef(null); + const { width } = useBoundingClientRect(listRef, { width: true }); + const colCount = useMemo(() => Math.floor((width ?? (window.innerWidth * 0.6)) / ITEM_MIN_WIDTH), [width]); + + const handleLoadHistory = useCallback((meta: IDBMeta) => { + getDataStorageById(meta.id).then(res => { + onDataLoaded(res.fields, res.dataSource, meta.name, meta.tag!, meta); + onClose(); + }).catch(onLoadingFailed); + }, [onClose, onDataLoaded, onLoadingFailed]); + + const handleDeleteHistory = useCallback((id: string) => { + deleteDataStorageById(id).then(() => { + fetchDataStorageList(); + }); + }, [fetchDataStorageList]); + + return ( + + {localDataList.map((file, i) => { + const ext = file.name.endsWith(RathDemoVirtualExt) ? RathDemoVirtualExt : /(?<=\.)[^.]+$/.exec(file.name)?.[0]; + const isRathDemo = ext === RathDemoVirtualExt; + const name = isRathDemo ? file.name.replace(new RegExp(`\\.${RathDemoVirtualExt.replaceAll(/\./g, '\\.')}$`), '') : file.name; + + return ( + handleLoadHistory(file)} + > +
+ +
+
+ + {name} + +
+ + {`${ext ? `${ + isRathDemo ? `Rath ${ + intl.get(`dataSource.importData.type.${IDataSourceType.DEMO}`) + }` : ext + } - ` : ''}${formatSize(file.size)}`} + +
+
+
+

{`${intl.get('dataSource.upload.lastOpen')}: ${dayjs(file.editTime).toDate().toLocaleString()}`}

+
+ e.stopPropagation()}> + {allUserTagGroups.map(key => { + const selected = file.userTagGroup === key; + return ( + { + updateDataStorageUserTagGroup(file.id, selected ? undefined : key); + fetchDataStorageList(false); + }} + > + + + ); + })} + + { + e.stopPropagation(); + handleDeleteHistory(file.id); + }} + /> + + ); + })} + + ); +}; + +export default observer(HistoryList); diff --git a/packages/rath-client/src/pages/dataSource/selection/index.tsx b/packages/rath-client/src/pages/dataSource/selection/index.tsx index 9b0773e9..b014fc90 100644 --- a/packages/rath-client/src/pages/dataSource/selection/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/index.tsx @@ -1,16 +1,16 @@ import React, { useState } from 'react'; import { Modal, ChoiceGroup, IconButton, ProgressIndicator } from '@fluentui/react'; -import { useId } from '@fluentui/react-hooks'; import intl from 'react-intl-universal'; import { IDataSourceType } from '../../../global'; import { IMuteFieldBase, IRow } from '../../../interfaces'; import { useDataSourceTypeOptions } from '../config'; import DataLoadingStatus from '../dataLoadingStatus'; +import type { DataSourceTag, IDBMeta } from '../../../utils/storage'; import FileData from './file'; import DemoData from './demo'; import RestfulData from './restful'; import OLAPData from './olap'; -import Local from './local'; +import HistoryList from './history/history-list'; import DatabaseData from './database/'; import AirTableSource from './airtable'; @@ -20,7 +20,7 @@ interface SelectionProps { onClose: () => void; onStartLoading: () => void; onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string, tag?: DataSourceTag | undefined, withHistory?: IDBMeta | undefined) => void; onDataLoading: (p: number) => void; setLoadingAnimation: (on: boolean) => void; } @@ -31,8 +31,6 @@ const Selection: React.FC = props => { const [dataSourceType, setDataSourceType] = useState(IDataSourceType.DEMO); const dsTypeOptions = useDataSourceTypeOptions(); - const dsTypeLabelId = useId('dataSourceType'); - const formMap: Record = { [IDataSourceType.FILE]: ( @@ -47,7 +45,7 @@ const Selection: React.FC = props => { ), [IDataSourceType.LOCAL]: ( - + ), [IDataSourceType.DATABASE]: ( @@ -72,7 +70,6 @@ const Selection: React.FC = props => { setDataSourceType(option.key as IDataSourceType); } }} - ariaLabelledBy={dsTypeLabelId} /> {loading && dataSourceType !== IDataSourceType.FILE && } {loading && dataSourceType === IDataSourceType.FILE && } diff --git a/packages/rath-client/src/pages/dataSource/selection/local.tsx b/packages/rath-client/src/pages/dataSource/selection/local.tsx deleted file mode 100644 index 09ab0bae..00000000 --- a/packages/rath-client/src/pages/dataSource/selection/local.tsx +++ /dev/null @@ -1,104 +0,0 @@ -import React, { useEffect, useMemo, useState } from 'react'; -import dayjs from 'dayjs'; -import styled from 'styled-components'; -import { IconButton } from '@fluentui/react'; -import { IMuteFieldBase, IRow } from '../../../interfaces'; -import { deleteDataStorageById, getDataStorageById, getDataStorageList, IDBMeta } from '../../../utils/storage'; - - -const LocalCont = styled.div` - .items-container{ - max-height: 500px; - overflow-y: auto; - } -` -const DataItem = styled.div` - border-radius: 3px; - border: 1px solid #ffa940; - padding: 6px; - margin: 6px 0px; - background-color: #fff7e6; - display: flex; - .desc-container{ - flex-grow: 1; - } - .button-container { - width: 120px; - display: flex; - align-items: center; - } -` - -function formatSize(size: number) { - if (size < 1024) { - return `${size}B`; - } - if (size < 1024 * 1024) { - return `${(size / 1024).toFixed(2)}KB`; - } - if (size < 1024 * 1024 * 1024) { - return `${(size / 1024 / 1024).toFixed(2)}MB`; - } - return `${(size / 1024 / 1024 / 1024).toFixed(2)}GB`; -} - -interface LocalDataProps { - onClose: () => void; - onStartLoading: () => void; - onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name?: string) => void; -} -const Local: React.FC = props => { - const { onDataLoaded, onLoadingFailed, onClose } = props; - const [localDataList, setLocalDataList] = useState([]); - useEffect(() => { - getDataStorageList().then((dataList) => { - setLocalDataList(dataList); - }) - }, []) - const totalSize = useMemo(() => { - return localDataList.reduce((acc, cur) => { - return acc + cur.size; - }, 0) - }, [localDataList]) - return -

History

-

total: {localDataList.length} datasets. {formatSize(totalSize * 1024)}

-
- { - localDataList.map(local => -
-

{local.name}

-
size: {formatSize(local.size * 1024)}
-
time: {dayjs(local.createTime).format('YYYY-MM-DD HH:mm:ss')}
-
-
- { - getDataStorageById(local.id).then(res => { - onDataLoaded(res.fields, res.dataSource, local.id); - onClose() - }).catch(onLoadingFailed) - }} - /> - { - deleteDataStorageById(local.id).then(res => { - getDataStorageList().then((dataList) => { - setLocalDataList(dataList); - }) - }) - }} - /> -
- ) - } -
- -} - -export default Local; diff --git a/packages/rath-client/src/pages/dataSource/selection/olap.tsx b/packages/rath-client/src/pages/dataSource/selection/olap.tsx index 8e5b9119..7c727237 100644 --- a/packages/rath-client/src/pages/dataSource/selection/olap.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/olap.tsx @@ -6,6 +6,7 @@ import { IMuteFieldBase, IRow } from '../../../interfaces'; import { useGlobalStore } from '../../../store'; import { logDataImport } from '../../../loggers/dataImport'; import { notify } from '../../../components/error'; +import { DataSourceTag } from '../../../utils/storage'; const StackTokens = { childrenGap: 1 @@ -17,7 +18,7 @@ const PROTOCOL_LIST: IDropdownOption[] = [ ] interface OLAPDataProps { onClose: () => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[]) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: undefined, tag: DataSourceTag) => void; } const OLAPData: React.FC = props => { @@ -47,7 +48,7 @@ const OLAPData: React.FC = props => { dataSource: data.slice(0, 10), size: data.length }); - onDataLoaded(fieldMetas, data); + onDataLoaded(fieldMetas, data, undefined, DataSourceTag.OLAP); onClose(); }) .catch((err) => { diff --git a/packages/rath-client/src/pages/dataSource/selection/restful.tsx b/packages/rath-client/src/pages/dataSource/selection/restful.tsx index fe448be8..015352e9 100644 --- a/packages/rath-client/src/pages/dataSource/selection/restful.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/restful.tsx @@ -6,6 +6,7 @@ import intl from 'react-intl-universal' import { DEMO_DATA_REQUEST_TIMEOUT } from '../../../constants'; import { IDatasetBase, IMuteFieldBase, IRow } from '../../../interfaces'; import { logDataImport } from '../../../loggers/dataImport'; +import { DataSourceTag } from '../../../utils/storage'; function requestAPIData (api: string): Promise { return new Promise((resolve, reject) => { @@ -46,7 +47,7 @@ interface RestFulProps { onClose: () => void; onStartLoading: () => void; onLoadingFailed: (err: any) => void; - onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[]) => void; + onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: undefined, tag: DataSourceTag) => void; } const RestFul: React.FC = props => { const { onClose, onStartLoading, onLoadingFailed, onDataLoaded } = props; @@ -56,7 +57,7 @@ const RestFul: React.FC = props => { onStartLoading(); requestAPIData(api).then(data => { const { dataSource, fields } = data; - onDataLoaded(fields, dataSource); + onDataLoaded(fields, dataSource, undefined, DataSourceTag.RESTFUL); logDataImport({ dataType: "Restful API", name: api, diff --git a/packages/rath-client/src/utils/storage.ts b/packages/rath-client/src/utils/storage.ts index fe18d9f9..ddab8b4a 100644 --- a/packages/rath-client/src/utils/storage.ts +++ b/packages/rath-client/src/utils/storage.ts @@ -5,15 +5,50 @@ import type { IFieldMeta, IMuteFieldBase, IRow } from '../interfaces'; import type { ICausalStoreSave } from '../store/causalStore/mainStore'; import type { CausalLinkDirection } from './resolve-causal'; + +export enum DataSourceTag { + FILE = '@file', + DEMO = '@demo', + RESTFUL = '@restful', + DATABASE = '@database', + OLAP = '@olap', + AIR_TABLE = '@air_table', +} + +export enum UserTagGroup { + Red, + Green, + Yellow, + Berry, + Peach, + DarkGreen, + Teal, + Navy, +} + +export const userTagGroupColors: Record = { + [UserTagGroup.Red]: '#d13438', + [UserTagGroup.Green]: '#13a10e', + [UserTagGroup.Yellow]: '#fde300', + [UserTagGroup.Berry]: '#c239b3', + [UserTagGroup.Peach]: '#ff8c00', + [UserTagGroup.DarkGreen]: '#063b06', + [UserTagGroup.Teal]: '#00b7c3', + [UserTagGroup.Navy]: '#0027b4', +}; + export interface IDBMeta { id: string; name: string; type: 'workspace' | 'dataset'; createTime: number; editTime: number; + /** kb */ size: number; rows?: number; fields?: IMuteFieldBase[]; + tag?: DataSourceTag; + userTagGroup?: UserTagGroup | undefined; } /** @deprecated */ @@ -100,14 +135,14 @@ export async function deleteStorageByIdInLocal (id: string) { await storages.removeItem(id) } -export async function getDataStorageList (): Promise { +export async function getDataStorageList (): Promise<(IDBMeta & { type: 'dataset' })[]> { const metas = localforage.createInstance({ name: STORAGE_INSTANCE, storeName: STORAGES.META }); const keys = await metas.keys(); const values = await Promise.all(keys.map(itemKey => metas.getItem(itemKey))) as IDBMeta[]; - return values.filter(v => v.type === 'dataset'); + return values.filter(v => v.type === 'dataset') as (IDBMeta & { type: 'dataset' })[]; } export async function getDataStorageById (id: string): Promise<{ fields: IMuteFieldBase[]; dataSource: IRow[] }> { @@ -140,7 +175,9 @@ export async function deleteDataStorageById (id: string) { await storages.removeItem(id) } -export async function setDataStorage(name: string, fields: IMuteFieldBase[], dataSource: IRow[]) { +export async function setDataStorage( + name: string, fields: IMuteFieldBase[], dataSource: IRow[], tag: DataSourceTag, withHistory?: IDBMeta, +) { const time = Date.now(); const dataString = JSON.stringify(dataSource); const metas = localforage.createInstance({ @@ -151,11 +188,12 @@ export async function setDataStorage(name: string, fields: IMuteFieldBase[], dat id: name, name, type: 'dataset', - createTime: time, + createTime: withHistory?.createTime ?? time, editTime: time, size: Math.round(dataString.length / 1024), rows: dataSource.length, - fields + fields, + tag: withHistory?.tag ?? tag, } as IDBMeta) const storages = localforage.createInstance({ name: STORAGE_INSTANCE, @@ -177,6 +215,18 @@ export async function updateDataStorageMeta(name: string, fields: IMuteFieldBase } as IDBMeta) } +export async function updateDataStorageUserTagGroup(name: string, userTagGroup: UserTagGroup | undefined) { + const metas = localforage.createInstance({ + name: STORAGE_INSTANCE, + storeName: STORAGES.META + }); + const oldMeta = await metas.getItem(name) as IDBMeta; + await metas.setItem(name, { + ...oldMeta, + userTagGroup, + } as IDBMeta) +} + export async function updateDataConfig(name: string, value: any) { const metas = localforage.createInstance({ name: STORAGE_INSTANCE, From 684663b561b86fe47d68b0f3095406ebe64dd2a8 Mon Sep 17 00:00:00 2001 From: kyusho Date: Tue, 6 Dec 2022 16:01:33 +0800 Subject: [PATCH 07/13] style(login): pivot & i18n --- .../rath-client/public/locales/en-US.json | 2 +- packages/rath-client/src/App.tsx | 41 ++------ .../rath-client/src/components/appNav.tsx | 10 +- .../src/pages/dataSource/selection/index.tsx | 2 +- .../src/pages/loginInfo/account.tsx | 53 ++++------- .../rath-client/src/pages/loginInfo/index.tsx | 95 ++++++++++++++----- 6 files changed, 112 insertions(+), 91 deletions(-) diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index a5ad64aa..af157c01 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -496,7 +496,7 @@ "shortMonths": ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] }, "login": { - "clickLogin": "Click Login", + "clickLogin": "Login", "haveSent": "Already Sent", "signIn": "Sign In", "signOut": "Sign Out", diff --git a/packages/rath-client/src/App.tsx b/packages/rath-client/src/App.tsx index b8876cea..58d37ddb 100644 --- a/packages/rath-client/src/App.tsx +++ b/packages/rath-client/src/App.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from 'react'; +import { useEffect, useState } from 'react'; import { observer } from 'mobx-react-lite'; import { Spinner, SpinnerSize } from '@fluentui/react'; import './App.css'; @@ -19,29 +19,8 @@ import Collection from './pages/collection'; import Dashboard from './pages/dashboard'; import CausalPage from './pages/causal'; import PerformanceWindow from './components/performance-window'; -import LoginInfo from './pages/loginInfo'; -import Account from './pages/loginInfo/account'; -import Setup from './pages/loginInfo/setup'; +import useHotKey from './hooks/use-hotkey'; -export enum PreferencesType { - Account = 'account', - Info = 'info', - Setting = 'setting', - Header = 'header' -} -export interface PreferencesListType { - key: PreferencesType; - name: PreferencesType; - icon: string; - element: () => JSX.Element; -} - -const preferencesList: PreferencesListType[] = [ - { key: PreferencesType.Account, name: PreferencesType.Account, icon: 'Home', element: () => }, - // { key: PreferencesType.Info, name: PreferencesType.Info, icon: 'Info', element: () => }, - // { key: PreferencesType.Header, name: PreferencesType.Header, icon: 'Contact', element: () =>
}, - { key: PreferencesType.Setting, name: PreferencesType.Setting, icon: 'Settings', element: () => }, -]; function App() { const { langStore, commonStore } = useGlobalStore(); @@ -60,6 +39,11 @@ function App() { }; }, [commonStore]); + const [showPerformanceWindow, setShowPerformanceWindow] = useState(false); + useHotKey({ + 'Control+Shift+P': () => setShowPerformanceWindow(on => !on), + }); + if (!langStore.loaded) { return (
@@ -68,20 +52,11 @@ function App() { ); } - const showPerformanceWindow = (new URL(window.location.href).searchParams.get('performance') ?? ( - JSON.stringify(process.env.NODE_ENV !== 'production') && false // temporarily banned this feature - )) === 'true'; - return (
- { - return ; - }} - preferencesList={preferencesList} - /> +
diff --git a/packages/rath-client/src/components/appNav.tsx b/packages/rath-client/src/components/appNav.tsx index 3c439112..0ce4263f 100644 --- a/packages/rath-client/src/components/appNav.tsx +++ b/packages/rath-client/src/components/appNav.tsx @@ -6,11 +6,13 @@ import styled from 'styled-components'; import { PIVOT_KEYS } from '../constants'; import { useGlobalStore } from '../store'; +import LoginInfo from '../pages/loginInfo'; import useHotKey from '../hooks/use-hotkey'; import UserSetting from './userSettings'; const NavContainer = styled.div` - height: 100%; + height: 100vh; + overflow: hidden auto; /* display: relative; */ position: relative; /* flex-direction: vertical; */ @@ -20,6 +22,9 @@ const NavContainer = styled.div` /* position: absolute; */ bottom: 0px; /* padding-left: 1em; */ + flex-grow: 0; + flex-shrink: 0; + overflow: hidden; } padding-left: 10px; .text-red { @@ -196,11 +201,12 @@ const AppNav: React.FC = (props) => { )} -
+
+
); diff --git a/packages/rath-client/src/pages/dataSource/selection/index.tsx b/packages/rath-client/src/pages/dataSource/selection/index.tsx index b014fc90..d704c46a 100644 --- a/packages/rath-client/src/pages/dataSource/selection/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/index.tsx @@ -28,7 +28,7 @@ interface SelectionProps { const Selection: React.FC = props => { const { show, onClose, onDataLoaded, loading, onStartLoading, onLoadingFailed, onDataLoading, setLoadingAnimation } = props; - const [dataSourceType, setDataSourceType] = useState(IDataSourceType.DEMO); + const [dataSourceType, setDataSourceType] = useState(IDataSourceType.LOCAL); const dsTypeOptions = useDataSourceTypeOptions(); const formMap: Record = { diff --git a/packages/rath-client/src/pages/loginInfo/account.tsx b/packages/rath-client/src/pages/loginInfo/account.tsx index 57f1a029..23306643 100644 --- a/packages/rath-client/src/pages/loginInfo/account.tsx +++ b/packages/rath-client/src/pages/loginInfo/account.tsx @@ -13,10 +13,8 @@ const AccountDiv = styled.div` width: 100%; display: flex; flex-direction: column; - align-items: center; margin-bottom: 20px; - padding-left: 10px; - padding-top: 10px; + padding-left: 2em; .label { font-weight: 600; font-size: 14px; @@ -26,20 +24,14 @@ const AccountDiv = styled.div` -webkit-font-smoothing: antialiased; } .account { + display: flex; + flex-direction: column; width: 100%; - > span { - width: 100%; + > .label { + margin-bottom: 1em; } - > span:first-child { - display: flex; - justify-content: space-between; - height: 35px; - line-height: 35px; - margin-bottom: 3px; - } - > span:last-child { - height: 35px; - line-height: 35px; + > button { + width: max-content; } } .phone { @@ -100,23 +92,20 @@ function Account() { ) : (
- - Account - {userName ? ( - { - commonStore.commitLogout() - }} - > - {intl.get('login.signOut')} - - ) : ( - [setIsLoginStatus(true)]}> - {intl.get('login.signIn')} - - )} - + Account + {userName ? ( + { + commonStore.commitLogout() + }} + > + {intl.get('login.signOut')} + + ) : ( + [setIsLoginStatus(true)]}> + {intl.get('login.signIn')} + + )} {userName && }
{userName && ( diff --git a/packages/rath-client/src/pages/loginInfo/index.tsx b/packages/rath-client/src/pages/loginInfo/index.tsx index 5b99c641..2284276a 100644 --- a/packages/rath-client/src/pages/loginInfo/index.tsx +++ b/packages/rath-client/src/pages/loginInfo/index.tsx @@ -1,22 +1,42 @@ -import { useState } from 'react'; +import { FC, useState } from 'react'; import intl from 'react-intl-universal'; import { observer } from 'mobx-react-lite'; -import { Dialog, Icon } from '@fluentui/react'; +import { Dialog, Icon, Pivot, PivotItem } from '@fluentui/react'; import styled from 'styled-components'; -import { PreferencesListType } from '../../App'; import { useGlobalStore } from '../../store'; -import LoginInfoList from './loginInfo'; -interface loginInfoProps { - preferencesList: PreferencesListType[]; +import Account from './account'; +import Setup from './setup'; + + +export enum PreferencesType { + Account = 'account', + Info = 'info', + Setting = 'setting', + Header = 'header' +} +export interface PreferencesListType { + key: PreferencesType; + name: PreferencesType; + icon: string; element: () => JSX.Element; } +const preferencesList: PreferencesListType[] = [ + { key: PreferencesType.Account, name: PreferencesType.Account, icon: 'Home', element: () => }, + // { key: PreferencesType.Info, name: PreferencesType.Info, icon: 'Info', element: () => }, + // { key: PreferencesType.Header, name: PreferencesType.Header, icon: 'Contact', element: () =>
}, + { key: PreferencesType.Setting, name: PreferencesType.Setting, icon: 'Settings', element: () => }, +]; + const LoginInfoDiv = styled.div` height: 100%; display: flex; flex-direction: column; - > div:first-child { - flex: 1; + border-top-width: 1px; + padding: 0.6em 0.8em 0.8em; + > div { + user-select: none; + cursor: pointer; } .user { white-space: nowrap; @@ -24,7 +44,7 @@ const LoginInfoDiv = styled.div` overflow-x: auto; font-size: 0.875rem; line-height: 1.25rem; - font-weight: 500; + font-weight: 400; } .user::-webkit-scrollbar { display: none; @@ -33,14 +53,6 @@ const LoginInfoDiv = styled.div` display: flex; align-items: center; } - .hidden-login { - /* flex flex-shrink-0 border-t border-indigo-800 p-4 bg-gray-700 cursor-pointer */ - display: flex; - flex-shrink: 0; - border-top-width: 1px; - padding: 1rem; - cursor: pointer; - } .user-name { /* ml-2 */ margin-left: 0.5rem; @@ -49,19 +61,50 @@ const LoginInfoDiv = styled.div` } `; -const LoginInfo = (props: loginInfoProps) => { - const { preferencesList, element } = props; +const Container = styled.div` + & .content { + display: flex; + flex-direction: row; + > [role=tablist] { + flex-grow: 0; + flex-shrink: 0; + display: flex; + flex-direction: column; + border-right: 1px solid #8888; + > [role=tab] { + margin: 0px 4px 8px 0; + ::before { + right: 0px; + width: 2px; + height: unset; + top: 2px; + bottom: 2px; + left: unset; + transition: unset; + } + } + } + > [role=tabpanel] { + flex-grow: 1; + flex-shrink: 1; + } + } +`; + +const LoginInfo: FC = () => { const { commonStore } = useGlobalStore(); const { userName, navMode, avatarUrl, info } = commonStore; const [loginHidden, setLoginHidden] = useState(true); + const [tab, setTab] = useState(PreferencesType.Account); + return ( -
{element()}
{ setLoginHidden(false); }} + role="button" + aria-haspopup > { dialogContentProps={{ title: intl.get('login.preferences') }} minWidth={550} > - + + item && setTab(item.props.itemKey as typeof tab)}> + {preferencesList.map(pref => ( + + {pref.element()} + + ))} + +
From fce1500db66ce4f6174f1cc1092b7f3b13c8d47a Mon Sep 17 00:00:00 2001 From: kyusho Date: Tue, 6 Dec 2022 16:15:18 +0800 Subject: [PATCH 08/13] fix(datasource): parsing error blocks interaction --- .../pages/dataSource/selection/file/file-upload.tsx | 12 +++++++++--- .../src/pages/dataSource/selection/file/index.tsx | 8 +++++++- .../rath-client/src/pages/loginInfo/loginInfo.tsx | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx index d26d7dc1..4b5c3878 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx @@ -1,7 +1,7 @@ import { ActionButton, Icon, Pivot, PivotItem, TooltipHost } from "@fluentui/react"; import intl from 'react-intl-universal'; import { observer } from "mobx-react-lite"; -import { FC, useCallback, useRef, useState } from "react"; +import { forwardRef, useCallback, useImperativeHandle, useRef, useState } from "react"; import styled from "styled-components"; import type { loadDataFile } from "../../utils"; import { notify } from "../../../../components/error"; @@ -180,7 +180,9 @@ export interface IFileUploadProps { onFileUpload: (file: File | null) => void; } -const FileUpload: FC = ({ preview, previewOfFile, previewOfRaw, onFileUpload }) => { +const FileUpload = forwardRef<{ reset: () => void }, IFileUploadProps>(function FileUpload ( + { preview, previewOfFile, previewOfRaw, onFileUpload }, ref +) { const fileInputRef = useRef(null); const handleReset = useCallback(() => { @@ -192,6 +194,10 @@ const FileUpload: FC = ({ preview, previewOfFile, previewOfRaw } }, [onFileUpload]); + useImperativeHandle(ref, () => ({ + reset: () => handleReset(), + })); + const handleButtonClick = useCallback(() => { fileInputRef.current?.click(); }, []); @@ -292,6 +298,6 @@ const FileUpload: FC = ({ preview, previewOfFile, previewOfRaw ); -}; +}); export default observer(FileUpload); diff --git a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx index 3c903f38..0347caff 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx @@ -64,6 +64,8 @@ const FileData: FC = (props) => { const filePreviewPendingRef = useRef>(); + const inputRef = useRef<{ reset: () => void }>(null); + useEffect(() => { filePreviewPendingRef.current = undefined; if (preview) { @@ -79,6 +81,8 @@ const FileData: FC = (props) => { return; } setExcelFile(res); + }).catch(() => { + inputRef.current?.reset(); }).finally(() => { toggleLoadingAnimation(false); }); @@ -104,6 +108,7 @@ const FileData: FC = (props) => { setPreviewOfFile(res[1].status === 'fulfilled' ? res[1].value : null); }).catch(reason => { onLoadingFailed(reason); + inputRef.current?.reset(); }).finally(() => { toggleLoadingAnimation(false); }); @@ -127,6 +132,7 @@ const FileData: FC = (props) => { setPreviewOfFile(res); }).catch(reason => { onLoadingFailed(reason); + inputRef.current?.reset(); }).finally(() => { toggleLoadingAnimation(false); }); @@ -170,7 +176,7 @@ const FileData: FC = (props) => { separator={separator} setSeparator={setSeparator} /> - + {preview ? ( previewOfFile && (
diff --git a/packages/rath-client/src/pages/loginInfo/loginInfo.tsx b/packages/rath-client/src/pages/loginInfo/loginInfo.tsx index 3afb7ecf..d24254da 100644 --- a/packages/rath-client/src/pages/loginInfo/loginInfo.tsx +++ b/packages/rath-client/src/pages/loginInfo/loginInfo.tsx @@ -2,7 +2,7 @@ import { useState } from 'react'; import { Icon } from '@fluentui/react'; import styled from 'styled-components'; import intl from 'react-intl-universal'; -import { PreferencesListType, PreferencesType } from '../../App'; +import { PreferencesListType, PreferencesType } from '.'; const LoginInfoListDiv = styled.div` height: 250px; From 79ba6b5d9c1a564f1c462f49b73c74f9054f6812 Mon Sep 17 00:00:00 2001 From: kyusho Date: Tue, 6 Dec 2022 16:49:58 +0800 Subject: [PATCH 09/13] fix(worker): worker breaks on produciton env --- packages/rath-client/src/constants.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/rath-client/src/constants.ts b/packages/rath-client/src/constants.ts index 26f0f946..6ee19591 100644 --- a/packages/rath-client/src/constants.ts +++ b/packages/rath-client/src/constants.ts @@ -62,9 +62,10 @@ export enum RATH_ENV { ONLINE = 'online production environment', } +// This file is included in Worker so never forget to check if `window` is undefined!!!!! export const RathEnv: RATH_ENV = ( process.env.NODE_ENV === 'development' ? RATH_ENV.DEV : process.env.NODE_ENV === 'test' ? RATH_ENV.TEST - : window.location.host.match(/^(.*\.)?kanaries\.(net|cn)$/) ? RATH_ENV.ONLINE - : window.location.host.match(/^.*kanaries\.vercel\.app$/) ? RATH_ENV.IPE : RATH_ENV.LPE + : globalThis.window === undefined || globalThis.window?.location.host.match(/^(.*\.)?kanaries\.(net|cn)$/) ? RATH_ENV.ONLINE + : globalThis.window?.location.host.match(/^.*kanaries\.vercel\.app$/) ? RATH_ENV.IPE : RATH_ENV.LPE ); From 76ebde06efb9317f158e380a61e9e6cf0b70023f Mon Sep 17 00:00:00 2001 From: kyusho Date: Tue, 6 Dec 2022 20:25:55 +0800 Subject: [PATCH 10/13] fix(datasource): upload size limit --- .../rath-client/public/locales/en-US.json | 11 ++++ .../rath-client/public/locales/zh-CN.json | 11 ++++ .../src/pages/dataSource/config.ts | 60 ++++++++++++------- .../src/pages/dataSource/selection/demo.tsx | 50 ++++++++++++---- .../dataSource/selection/file/file-upload.tsx | 2 +- .../pages/dataSource/selection/file/index.tsx | 4 +- .../selection/history/get-file-icon.tsx | 2 +- .../selection/history/history-list.tsx | 2 +- 8 files changed, 104 insertions(+), 38 deletions(-) diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index af157c01..c1046d32 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -139,6 +139,17 @@ "originStatTable": "Original Data Statistics", "selectionStatTable": "Selection Data Statistics" }, + "sizeInfo": "{nCols} fields, {nRows} rows", + "demoDataset": { + "CARS": { "title": "Cars", "description": "Origin, product name and physical attributes of cars." }, + "STUDENTS": { "title": "Students", "description": "" }, + "BTC_GOLD": { "title": "BTC - Gold", "description": "" }, + "CAR_SALES": { "title": "Car Sales", "description": "" }, + "COLLAGE": { "title": "Collage", "description": "" }, + "TITANIC": { "title": "Titanic", "description": "" }, + "KEPLER": { "title": "Kepler", "description": "" }, + "BIKE_SHARING_DC": { "title": "Bike Sharing", "description": "" } + }, "dbProgress": [ { "label": "Connection", diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 74832da2..9d60f147 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -138,6 +138,17 @@ "originStatTable": "原始数据统计信息", "selectionStatTable": "筛选部分统计信息" }, + "sizeInfo": "{nCols} 列 x {nRows} 行", + "demoDataset": { + "CARS": { "title": "汽车数据集", "description": "记录了汽车的型号、厂商、物理参数的数据。" }, + "STUDENTS": { "title": "学生成绩数据集", "description": "分析学生的成绩和部分其他因素等关系。" }, + "BTC_GOLD": { "title": "比特币-金价数据集", "description": "" }, + "CAR_SALES": { "title": "汽车销售数据集", "description": "" }, + "COLLAGE": { "title": "大学数据集", "description": "" }, + "TITANIC": { "title": "泰坦尼克数据集", "description": "" }, + "KEPLER": { "title": "开普勒数据集", "description": "" }, + "BIKE_SHARING_DC": { "title": "共享单车数据集", "description": "" } + }, "dbProgress": [ { "label": "建立连接", diff --git a/packages/rath-client/src/pages/dataSource/config.ts b/packages/rath-client/src/pages/dataSource/config.ts index e281e091..33be08a9 100644 --- a/packages/rath-client/src/pages/dataSource/config.ts +++ b/packages/rath-client/src/pages/dataSource/config.ts @@ -27,16 +27,17 @@ export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceTyp text: demoText, iconProps: { iconName: "FileTemplate" }, }, - { - key: IDataSourceType.RESTFUL, - text: restfulText, - iconProps: { iconName: "Cloud" }, - }, { key: IDataSourceType.DATABASE, text: dbText, iconProps: { iconName: "Database" } }, + { + key: IDataSourceType.AIRTABLE, + text: 'AirTable', + iconProps: { iconName: 'Table' }, + disabled: false + }, { key: IDataSourceType.OLAP, text: 'OLAP', @@ -44,10 +45,9 @@ export const useDataSourceTypeOptions = function (): Array<{ key: IDataSourceTyp disabled: false, }, { - key: IDataSourceType.AIRTABLE, - text: 'AirTable', - iconProps: { iconName: 'Table' }, - disabled: false + key: IDataSourceType.RESTFUL, + text: restfulText, + iconProps: { iconName: "Cloud" }, }, ]; }, [fileText, restfulText, demoText, dbText, historyText]); @@ -62,7 +62,7 @@ export const DemoDataAssets = process.env.NODE_ENV === 'production' ? { CAR_SALES: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-carsales-service.json', COLLAGE: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-collage-service.json', TITANIC: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-titanic-service.json', - KELPER: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-kelper-service.json', + KEPLER: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-kelper-service.json', BIKE_SHARING_DC: 'https://chspace.oss-cn-hongkong.aliyuncs.com/api/bike_dc-dataset-service.json' } : { // CARS: "https://chspace.oss-cn-hongkong.aliyuncs.com/api/ds-cars-service.json", @@ -75,47 +75,63 @@ export const DemoDataAssets = process.env.NODE_ENV === 'production' ? { CAR_SALES: '/datasets/ds-carsales-service.json', COLLAGE: '/datasets/ds-collage-service.json', TITANIC: '/datasets/ds-titanic-service.json', - KELPER: '/datasets/ds-kelper-service.json', + KEPLER: '/datasets/ds-kelper-service.json', BIKE_SHARING_DC: '/datasets/bike_dc-dataset-service.json' } as const; export type IDemoDataKey = keyof typeof DemoDataAssets; -export const useDemoDataOptions = function (): Array<{key: IDemoDataKey; text: string}> { - const options = useMemo>(() => { +export const useDemoDataOptions = function (): Array<{key: IDemoDataKey; text: string; nCols: number; nRows: number}> { + const options = useMemo>(() => { return [ { key: "CARS", text: "Cars", + nCols: 9, + nRows: 406, }, { key: "STUDENTS", - text: "Students' Performance" + text: "Students' Performance", + nCols: 8, + nRows: 1000, }, { key: 'BIKE_SHARING_DC', - text: 'Bike Sharing in Washington D.C.' + text: 'Bike Sharing in Washington D.C.', + nCols: 16, + nRows: 17319, }, { key: "CAR_SALES", - text: "Car Sales" + text: "Car Sales", + nCols: 16, + nRows: 157, }, { key: "COLLAGE", - text: "Collage" + text: "Collage", + nCols: 16, + nRows: 1294, }, { - key: "KELPER", - text: "NASA Kelper" + key: "KEPLER", + text: "NASA Kepler", + nCols: 44, + nRows: 9218, }, { key: 'BTC_GOLD', - text: "2022MCM Problem C: Trading Strategies" + text: "2022MCM Problem C: Trading Strategies", + nCols: 7, + nRows: 464, }, { key: "TITANIC", - text: "Titanic" - } + text: "Titanic", + nCols: 11, + nRows: 712, + }, ]; }, []); return options; diff --git a/packages/rath-client/src/pages/dataSource/selection/demo.tsx b/packages/rath-client/src/pages/dataSource/selection/demo.tsx index c02f0d94..3b2f65f9 100644 --- a/packages/rath-client/src/pages/dataSource/selection/demo.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/demo.tsx @@ -60,6 +60,15 @@ function requestDemoData (dsKey: IDemoDataKey = 'CARS'): Promise { export const RathDemoVirtualExt = 'rath-demo.json'; +const Container = styled.div` + display: flex; + flex-direction: column; + align-items: center; + > label { + width: 100%; + } +`; + const List = styled.div` margin: 1em 0; min-height: 8em; @@ -91,14 +100,25 @@ const ListItem = styled.div` margin-right: 0.8em; user-select: none; } - > header { - font-size: 0.8rem; - line-height: 1.2em; - font-weight: 550; - color: #111; + > div { flex-grow: 1; flex-shrink: 1; + flex-basis: 0; overflow: hidden; + display: flex; + flex-direction: column; + > header { + font-size: 0.8rem; + line-height: 1.2em; + font-weight: 550; + color: #111; + margin-bottom: 0.4em; + } + > span { + word-break: break-all; + line-height: 1.2em; + margin: 0.12em 0; + } } } :hover { @@ -138,7 +158,7 @@ const DemoData: FC = props => { const colCount = useMemo(() => Math.floor((width ?? (window.innerWidth * 0.6)) / ITEM_MIN_WIDTH), [width]); return ( -
+ {options.map((demo, i) => { @@ -152,16 +172,24 @@ const DemoData: FC = props => { onClick={() => loadDemo(demo)} >
- -
- {demo.text} -
+ +
+
+ {intl.get(`dataSource.demoDataset.${demo.key}.title`)} +
+ + {intl.get(`dataSource.demoDataset.${demo.key}.description`)} + + + {intl.get(`dataSource.sizeInfo`, demo)} + +
); })} -
+ ); } diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx index 4b5c3878..c6f58607 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx @@ -171,7 +171,7 @@ const PreviewArea = styled.table` } `; -const MAX_UPLOAD_SIZE = 1024 * 1024 * 128 * 0.0001; +const MAX_UPLOAD_SIZE = 1024 * 1024 * 128; export interface IFileUploadProps { preview: File | null; diff --git a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx index 0347caff..f7876f8b 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx @@ -13,7 +13,7 @@ import FileHelper, { Charset } from "./file-helper"; const Container = styled.div` - width: 55vw; + max-width: 680px; > header { margin-top: 1.2em; font-weight: 550; @@ -89,7 +89,7 @@ const FileData: FC = (props) => { return; } const p = Promise.allSettled([ - readRaw(preview, charset, 1024, 32, 128), + readRaw(preview, charset, 4096, 64, 128), loadDataFile({ file: preview, sampleMethod, diff --git a/packages/rath-client/src/pages/dataSource/selection/history/get-file-icon.tsx b/packages/rath-client/src/pages/dataSource/selection/history/get-file-icon.tsx index eb51e32e..b26bc22a 100644 --- a/packages/rath-client/src/pages/dataSource/selection/history/get-file-icon.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/history/get-file-icon.tsx @@ -4,7 +4,7 @@ import { getFileTypeIconProps, initializeFileTypeIcons } from '@fluentui/react-f initializeFileTypeIcons(); const getFileIcon = (fileName: string): string => { - const iconProps = getFileTypeIconProps({ extension: /(?<=\.)[a-z]+/i.exec(fileName)?.[0] }); + const iconProps = getFileTypeIconProps({ extension: /(?<=\.).+/i.exec(fileName)?.[0], imageFileType: 'png', size: 16 }); return iconProps.iconName; }; diff --git a/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx index 5a8339dc..a1d32404 100644 --- a/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx @@ -223,7 +223,7 @@ const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFa onClick={() => handleLoadHistory(file)} >
- +
From db4e478727b8bbc44d639c72d259c92f22405a40 Mon Sep 17 00:00:00 2001 From: kyusho Date: Wed, 7 Dec 2022 15:23:02 +0800 Subject: [PATCH 11/13] feat(datasource): search for history name --- .../selection/history/history-list.tsx | 51 ++++++++++++++++--- .../dataSource/selection/history/index.tsx | 32 ++++++++++++ .../src/pages/dataSource/selection/index.tsx | 4 +- yarn.lock | 4 +- 4 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 packages/rath-client/src/pages/dataSource/selection/history/index.tsx diff --git a/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx index a1d32404..e49d840f 100644 --- a/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx @@ -21,7 +21,7 @@ const List = styled.div` margin: 1em 0; min-height: 8em; max-height: 50vh; - max-width: 50vw; + max-width: 44vw; overflow: hidden auto; display: grid; gap: 0.4em; @@ -55,7 +55,7 @@ const ListItem = styled.div` overflow: hidden; > header { font-size: 0.8rem; - line-height: 1.2em; + line-height: 1.5em; font-weight: 550; white-space: nowrap; text-overflow: ellipsis; @@ -66,6 +66,9 @@ const ListItem = styled.div` font-size: 0.6rem; line-height: 1.2em; color: #555; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; } } } @@ -129,7 +132,12 @@ const UserTagGroupContainer = styled.div` } &[aria-selected=true] { opacity: 1; - transform: translateY(-2px); + transform: translateY(-3px); + animation: pull 400ms linear forwards; + } + &[aria-selected=false] { + transition: transform 100ms; + filter: saturate(0.75); } > * { pointer-events: none; @@ -139,6 +147,17 @@ const UserTagGroupContainer = styled.div` margin-left: ${-0.2 * UserTagGroupSize - UserTagGroupPadding}px; } } + @keyframes pull { + from { + transform: translateY(-4px); + } + 30% { + transform: translateY(-1.5px); + } + to { + transform: translateY(-3px); + } + } `; const ITEM_MIN_WIDTH = 200; @@ -155,14 +174,15 @@ export function formatSize(size: number) { return `${(size / 1024 / 1024).toFixed(2)}GB`; } -interface IHistoryListProps { +export interface IHistoryListProps { onClose: () => void; onLoadingFailed: (err: any) => void; onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: string, tag: DataSourceTag, withHistory: IDBMeta) => void; is?: DataSourceTag; + search?: string; } -const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFailed, is }) => { +const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFailed, is, search }) => { const [localDataList, setLocalDataList] = useState([]); const prevList = useRef(localDataList); prevList.current = localDataList; @@ -206,9 +226,26 @@ const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFa }); }, [fetchDataStorageList]); + const list = useMemo(() => { + if (!search) { + return localDataList; + } + return localDataList.filter(item => { + let temp = item.name.toLocaleLowerCase(); + for (const keyword of search.toLocaleLowerCase().split(/ +/)) { + const idx = temp.indexOf(keyword); + if (idx === -1) { + return false; + } + temp = temp.slice(idx); + } + return true; + }); + }, [localDataList, search]); + return ( - {localDataList.map((file, i) => { + {list.map((file, i) => { const ext = file.name.endsWith(RathDemoVirtualExt) ? RathDemoVirtualExt : /(?<=\.)[^.]+$/.exec(file.name)?.[0]; const isRathDemo = ext === RathDemoVirtualExt; const name = isRathDemo ? file.name.replace(new RegExp(`\\.${RathDemoVirtualExt.replaceAll(/\./g, '\\.')}$`), '') : file.name; @@ -248,7 +285,7 @@ const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFa return ( { updateDataStorageUserTagGroup(file.id, selected ? undefined : key); diff --git a/packages/rath-client/src/pages/dataSource/selection/history/index.tsx b/packages/rath-client/src/pages/dataSource/selection/history/index.tsx new file mode 100644 index 00000000..843c978d --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/history/index.tsx @@ -0,0 +1,32 @@ +import { TextField } from "@fluentui/react"; +import { observer } from "mobx-react-lite"; +import { FC, useState } from "react"; +import HistoryList, { IHistoryListProps } from "./history-list"; + + +const HistoryPanel: FC> = ( + { onDataLoaded, onClose, onLoadingFailed } +) => { + const [search, setSearch] = useState(''); + + return ( + <> + setSearch(value ?? '')} + /> + + + ); +}; + + +export default observer(HistoryPanel); diff --git a/packages/rath-client/src/pages/dataSource/selection/index.tsx b/packages/rath-client/src/pages/dataSource/selection/index.tsx index d704c46a..036bf95f 100644 --- a/packages/rath-client/src/pages/dataSource/selection/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/index.tsx @@ -10,7 +10,7 @@ import FileData from './file'; import DemoData from './demo'; import RestfulData from './restful'; import OLAPData from './olap'; -import HistoryList from './history/history-list'; +import HistoryPanel from './history'; import DatabaseData from './database/'; import AirTableSource from './airtable'; @@ -45,7 +45,7 @@ const Selection: React.FC = props => { ), [IDataSourceType.LOCAL]: ( - + ), [IDataSourceType.DATABASE]: ( diff --git a/yarn.lock b/yarn.lock index 7f1215e3..bb75d29a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3450,7 +3450,7 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== -"@types/react-dom@^17.0.1": +"@types/react-dom@^17.0.1", "@types/react-dom@^17.x": version "17.0.18" resolved "https://registry.npmmirror.com/@types/react-dom/-/react-dom-17.0.18.tgz#8f7af38f5d9b42f79162eea7492e5a1caff70dc2" integrity sha512-rLVtIfbwyur2iFKykP2w0pl/1unw26b5td16d5xMgp7/yjTHomkyxPYChFoCr/FtEX1lN9wY6lFj1qvKdS5kDw== @@ -3481,7 +3481,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^17", "@types/react@^17.0.2": +"@types/react@*", "@types/react@^17", "@types/react@^17.0.2", "@types/react@^17.x": version "17.0.52" resolved "https://registry.npmmirror.com/@types/react/-/react-17.0.52.tgz#10d8b907b5c563ac014a541f289ae8eaa9bf2e9b" integrity sha512-vwk8QqVODi0VaZZpDXQCmEmiOuyjEFPY7Ttaw5vjM112LOq37yz1CDJGrRJwA1fYEq4Iitd5rnjd1yWAc/bT+A== From b2abe45979fe83f63436db39ddbb068879a82f0c Mon Sep 17 00:00:00 2001 From: kyusho Date: Wed, 7 Dec 2022 16:18:52 +0800 Subject: [PATCH 12/13] feat(datasource): timeline for history --- .../rath-client/public/locales/en-US.json | 8 + .../rath-client/public/locales/zh-CN.json | 8 + .../dataSource/selection/file/file-upload.tsx | 14 +- .../selection/history/history-list-item.tsx | 232 +++++++++++++ .../selection/history/history-list.tsx | 323 ++++++------------ .../dataSource/selection/history/index.tsx | 1 + 6 files changed, 368 insertions(+), 218 deletions(-) create mode 100644 packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index c1046d32..6856f78c 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -251,6 +251,14 @@ "lastOpen": "Last opened", "firstOpen": "First created", "history": "Open Recent", + "history_time": { + "1d": "Today", + "1w": "Last weak", + "1mo": "Last month", + "3mo": "Last 3 months", + "6mo": "Last half a year", + "1yr": "Last year" + }, "new": "New file", "sheet": "Sheet", "preview_parsed": "Preview", diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 9d60f147..4a48ab09 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -251,6 +251,14 @@ "lastOpen": "上一次使用", "firstOpen": "创建时间", "history": "最近使用", + "history_time": { + "1d": "今天", + "1w": "过去一周", + "1mo": "过去一个月", + "3mo": "过去三个月", + "6mo": "过去半年", + "1yr": "一年内" + }, "new": "新的文件", "sheet": "工作簿", "preview_parsed": "预览", diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx index c6f58607..994f857c 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx @@ -6,7 +6,6 @@ import styled from "styled-components"; import type { loadDataFile } from "../../utils"; import { notify } from "../../../../components/error"; import getFileIcon from "../history/get-file-icon"; -import { formatSize } from "../history/history-list"; const Container = styled.div` @@ -173,6 +172,19 @@ const PreviewArea = styled.table` const MAX_UPLOAD_SIZE = 1024 * 1024 * 128; +function formatSize(size: number) { + if (size < 1024) { + return `${size}B`; + } + if (size < 1024 * 1024) { + return `${(size / 1024).toFixed(2)}KB`; + } + if (size < 1024 * 1024 * 1024) { + return `${(size / 1024 / 1024).toFixed(2)}MB`; + } + return `${(size / 1024 / 1024 / 1024).toFixed(2)}GB`; +} + export interface IFileUploadProps { preview: File | null; previewOfFile: Awaited> | null; diff --git a/packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx b/packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx new file mode 100644 index 00000000..23e2ecae --- /dev/null +++ b/packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx @@ -0,0 +1,232 @@ +import intl from 'react-intl-universal'; +import { Icon, IconButton, TooltipHost } from "@fluentui/react"; +import { observer } from "mobx-react-lite"; +import dayjs from 'dayjs'; +import type { FC } from "react"; +import styled from "styled-components"; +import { IDBMeta, updateDataStorageUserTagGroup, UserTagGroup, userTagGroupColors } from "../../../../utils/storage"; +import { RathDemoVirtualExt } from '../demo'; +import { IDataSourceType } from '../../../../global'; +import getFileIcon from './get-file-icon'; + + +const allUserTagGroups = Object.keys(userTagGroupColors) as unknown as UserTagGroup[]; + +const UserTagGroupSize = 12; +const UserTagGroupPadding = 2; + +const ListItem = styled.div` + display: flex; + flex-direction: column; + overflow: hidden; + min-width: ${(UserTagGroupSize + UserTagGroupPadding * 2) * ((allUserTagGroups.length - 1) * 0.8 + 1) + 10}px; + height: 100%; + padding: 1.2em 1em 1em 1.4em; + border-radius: 2px; + position: relative; + box-shadow: inset 0 0 2px #8881; + > .head { + display: flex; + align-items: center; + > i { + flex-grow: 0; + flex-shrink: 0; + width: 2em; + height: 2em; + margin-right: 0.8em; + user-select: none; + } + > div { + flex-grow: 1; + flex-shrink: 1; + overflow: hidden; + > header { + font-size: 0.8rem; + line-height: 1.5em; + font-weight: 550; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; + color: #111; + } + > span { + font-size: 0.6rem; + line-height: 1.2em; + color: #555; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; + } + } + } + .time { + font-size: 0.5rem; + color: #888; + } + > button { + position: absolute; + top: 0; + right: 0; + margin: 0; + padding: 0; + font-size: 12px; + background-color: #d13438 !important; + border-radius: 50%; + color: #fff !important; + width: 1.2em; + height: 1.2em; + i { + font-weight: 1000; + line-height: 1em; + width: 1em; + height: 1em; + transform: scale(0.4); + } + opacity: 0.5; + :hover { + opacity: 1; + } + } + & .hover-only:not([aria-selected=true]) { + visibility: hidden; + } + :hover { + background-color: #8881; + & .hover-only { + visibility: visible; + } + } + cursor: pointer; +`; + +const UserTagGroupContainer = styled.div` + position: absolute; + left: 0; + top: 0; + display: flex; + flex-direction: row; + background-image: linear-gradient(to bottom, #8881, transparent 5px); + padding: 0 5px; + > svg { + margin: 0 ${UserTagGroupPadding}px; + cursor: pointer; + transition: transform 200ms; + transform: translateY(-67%); + opacity: 0.2; + :hover { + opacity: 0.95; + transform: translateY(-4px); + } + &[aria-selected=true] { + opacity: 1; + transform: translateY(-3px); + animation: pull 400ms linear forwards; + } + &[aria-selected=false] { + transition: transform 100ms; + filter: saturate(0.75); + } + > * { + pointer-events: none; + filter: drop-shadow(0.8px 1px 0.6px #888); + } + :not(:first-child) { + margin-left: ${-0.2 * UserTagGroupSize - UserTagGroupPadding}px; + } + } + @keyframes pull { + from { + transform: translateY(-4px); + } + 30% { + transform: translateY(-1.5px); + } + to { + transform: translateY(-3px); + } + } +`; + +export function formatSize(kb: number) { + if (kb < 1024) { + return `${kb}KB`; + } + if (kb < 1024 * 1024) { + return `${(kb / 1024).toFixed(2)}MB`; + } + return `${(kb / 1024 / 1024).toFixed(2)}GB`; +} + +export interface IHistoryListItemProps { + file: IDBMeta; + rowIndex: number; + colIndex: number; + handleClick?: (item: IDBMeta) => void; + handleClearClick?: (itemId: string) => void; + handleRefresh?: () => void; +} + +const HistoryListItem: FC = ({ + file, rowIndex, colIndex, handleClick, handleClearClick, handleRefresh, +}) => { + const ext = file.name.endsWith(RathDemoVirtualExt) ? RathDemoVirtualExt : /(?<=\.)[^.]+$/.exec(file.name)?.[0]; + const isRathDemo = ext === RathDemoVirtualExt; + const name = isRathDemo ? file.name.replace(new RegExp(`\\.${RathDemoVirtualExt.replaceAll(/\./g, '\\.')}$`), '') : file.name; + + return ( + handleClick?.(file)} + > +
+ +
+
+ + {name} + +
+ + {`${ext ? `${isRathDemo ? `Rath ${intl.get(`dataSource.importData.type.${IDataSourceType.DEMO}`) + }` : ext + } - ` : ''}${formatSize(file.size)}`} + +
+
+
+

{`${intl.get('dataSource.upload.lastOpen')}: ${dayjs(file.editTime).toDate().toLocaleString()}`}

+
+ e.stopPropagation()}> + {allUserTagGroups.map(key => { + const selected = file.userTagGroup === key; + return ( + { + updateDataStorageUserTagGroup(file.id, selected ? undefined : key); + handleRefresh?.(); + }} + > + + + ); + })} + + { + e.stopPropagation(); + handleClearClick?.(file.id); + }} + /> + + ); +}; + +export default observer(HistoryListItem); diff --git a/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx index e49d840f..fa1b9caa 100644 --- a/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/history/history-list.tsx @@ -1,188 +1,71 @@ import intl from 'react-intl-universal'; -import { Icon, IconButton, TooltipHost } from "@fluentui/react"; import { observer } from "mobx-react-lite"; -import dayjs from 'dayjs'; -import { FC, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { FC, Fragment, useCallback, useEffect, useMemo, useRef, useState } from "react"; import styled from "styled-components"; import useBoundingClientRect from "../../../../hooks/use-bounding-client-rect"; -import { DataSourceTag, deleteDataStorageById, getDataStorageById, getDataStorageList, IDBMeta, updateDataStorageUserTagGroup, UserTagGroup, userTagGroupColors } from "../../../../utils/storage"; +import { DataSourceTag, deleteDataStorageById, getDataStorageById, getDataStorageList, IDBMeta } from "../../../../utils/storage"; import type { IMuteFieldBase, IRow } from '../../../../interfaces'; -import { RathDemoVirtualExt } from '../demo'; -import { IDataSourceType } from '../../../../global'; -import getFileIcon from './get-file-icon'; +import HistoryListItem from './history-list-item'; -const allUserTagGroups = Object.keys(userTagGroupColors) as unknown as UserTagGroup[]; - -const UserTagGroupSize = 12; -const UserTagGroupPadding = 2; +const Group = styled.div` + display: flex; + flex-direction: column; + max-width: 680px; + max-height: 50vh; + overflow: hidden auto; + > * { + flex-grow: 0; + flex-shrink: 0; + max-height: unset; + } +`; const List = styled.div` margin: 1em 0; min-height: 8em; max-height: 50vh; - max-width: 44vw; + max-width: 680px; overflow: hidden auto; display: grid; gap: 0.4em; grid-auto-rows: max-content; `; -const ListItem = styled.div` - display: flex; - flex-direction: column; - overflow: hidden; - min-width: ${(UserTagGroupSize + UserTagGroupPadding * 2) * ((allUserTagGroups.length - 1) * 0.8 + 1) + 10}px; - height: 100%; - padding: 1.2em 1em 1em 1.4em; - border-radius: 2px; - position: relative; - box-shadow: inset 0 0 2px #8881; - > .head { - display: flex; - align-items: center; - > i { - flex-grow: 0; - flex-shrink: 0; - width: 2em; - height: 2em; - margin-right: 0.8em; - user-select: none; - } - > div { - flex-grow: 1; - flex-shrink: 1; - overflow: hidden; - > header { - font-size: 0.8rem; - line-height: 1.5em; - font-weight: 550; - white-space: nowrap; - text-overflow: ellipsis; - overflow: hidden; - color: #111; - } - > span { - font-size: 0.6rem; - line-height: 1.2em; - color: #555; - white-space: nowrap; - text-overflow: ellipsis; - overflow: hidden; - } - } - } - .time { - font-size: 0.5rem; - color: #888; - } - > button { - position: absolute; - top: 0; - right: 0; - margin: 0; - padding: 0; - font-size: 12px; - background-color: #d13438 !important; - border-radius: 50%; - color: #fff !important; - width: 1.2em; - height: 1.2em; - i { - font-weight: 1000; - line-height: 1em; - width: 1em; - height: 1em; - transform: scale(0.4); - } - opacity: 0.5; - :hover { - opacity: 1; - } - } - & .hover-only:not([aria-selected=true]) { - visibility: hidden; - } - :hover { - background-color: #8881; - & .hover-only { - visibility: visible; - } - } - cursor: pointer; -`; - -const UserTagGroupContainer = styled.div` - position: absolute; - left: 0; - top: 0; - display: flex; - flex-direction: row; - background-image: linear-gradient(to bottom, #8881, transparent 5px); - padding: 0 5px; - > svg { - margin: 0 ${UserTagGroupPadding}px; - cursor: pointer; - transition: transform 200ms; - transform: translateY(-67%); - opacity: 0.2; - :hover { - opacity: 0.95; - transform: translateY(-4px); - } - &[aria-selected=true] { - opacity: 1; - transform: translateY(-3px); - animation: pull 400ms linear forwards; - } - &[aria-selected=false] { - transition: transform 100ms; - filter: saturate(0.75); - } - > * { - pointer-events: none; - filter: drop-shadow(0.8px 1px 0.6px #888); - } - :not(:first-child) { - margin-left: ${-0.2 * UserTagGroupSize - UserTagGroupPadding}px; - } - } - @keyframes pull { - from { - transform: translateY(-4px); - } - 30% { - transform: translateY(-1.5px); - } - to { - transform: translateY(-3px); - } - } -`; - const ITEM_MIN_WIDTH = 200; const MAX_HISTORY_SIZE = 64; -const MAX_RECENT_TIME = 1_000 * 60 * 60 * 24 * 31 * 3; // 3 months -export function formatSize(size: number) { - if (size < 1024) { - return `${size.toFixed(2)}KB`; - } - if (size < 1024 * 1024) { - return `${(size / 1024).toFixed(2)}MB`; - } - return `${(size / 1024 / 1024).toFixed(2)}GB`; +export enum HistoryRecentTag { + TODAY = '1d', + WEEK = '1w', + MONTH = '1mo', + THREE_MONTHS = '3mo', + SIX_MONTHS = '6mo', + YEAR = '1yr', } +const limitRecentTime: Record = { + [HistoryRecentTag.TODAY]: 1_000 * 60 * 60 * 24, + [HistoryRecentTag.WEEK]: 1_000 * 60 * 60 * 24 * 7, + [HistoryRecentTag.MONTH]: 1_000 * 60 * 60 * 24 * 31, + [HistoryRecentTag.THREE_MONTHS]: 1_000 * 60 * 60 * 24 * 31 * 3, + [HistoryRecentTag.SIX_MONTHS]: 1_000 * 60 * 60 * 24 * 183, + [HistoryRecentTag.YEAR]: 1_000 * 60 * 60 * 24 * 366, +}; + +const MAX_RECENT_TIME = limitRecentTime[HistoryRecentTag.YEAR]; + export interface IHistoryListProps { onClose: () => void; onLoadingFailed: (err: any) => void; onDataLoaded: (fields: IMuteFieldBase[], dataSource: IRow[], name: string, tag: DataSourceTag, withHistory: IDBMeta) => void; is?: DataSourceTag; search?: string; + /** @default false */ + groupByPeriod?: boolean; } -const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFailed, is, search }) => { +const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFailed, is, search, groupByPeriod = false }) => { const [localDataList, setLocalDataList] = useState([]); const prevList = useRef(localDataList); prevList.current = localDataList; @@ -243,73 +126,79 @@ const HistoryList: FC = ({ onDataLoaded, onClose, onLoadingFa }); }, [localDataList, search]); + const groups = useMemo<{ list: typeof list; tag: HistoryRecentTag }[]>(() => { + if (!groupByPeriod) { + return []; + } + const all = Object.keys(limitRecentTime).map<{ list: typeof list; tag: HistoryRecentTag }>(tag => { + return { + list: [], + tag: tag as HistoryRecentTag, + }; + }); + const now = Date.now(); + for (const item of list) { + for (const group of all) { + if (group.tag === HistoryRecentTag.TODAY) { + if (new Date(item.editTime).toDateString() === new Date(now).toDateString()) { + group.list.push(item); + break; + } + } else if (now - item.editTime < limitRecentTime[group.tag]) { + group.list.push(item); + break; + } + } + } + return all.filter(group => group.list.length > 0); + }, [list, groupByPeriod]); + return ( - - {list.map((file, i) => { - const ext = file.name.endsWith(RathDemoVirtualExt) ? RathDemoVirtualExt : /(?<=\.)[^.]+$/.exec(file.name)?.[0]; - const isRathDemo = ext === RathDemoVirtualExt; - const name = isRathDemo ? file.name.replace(new RegExp(`\\.${RathDemoVirtualExt.replaceAll(/\./g, '\\.')}$`), '') : file.name; + + {groups.length > 0 ? groups.map(group => { + const beginTime = new Date(group.list.at(-1)!.editTime).toLocaleDateString(); + const endTime = new Date(group.list[0].editTime).toLocaleDateString(); + const period = endTime === beginTime ? endTime : `${beginTime} - ${endTime}`; return ( - handleLoadHistory(file)} - > -
- -
-
- - {name} - -
- - {`${ext ? `${ - isRathDemo ? `Rath ${ - intl.get(`dataSource.importData.type.${IDataSourceType.DEMO}`) - }` : ext - } - ` : ''}${formatSize(file.size)}`} - -
-
-
-

{`${intl.get('dataSource.upload.lastOpen')}: ${dayjs(file.editTime).toDate().toLocaleString()}`}

-
- e.stopPropagation()}> - {allUserTagGroups.map(key => { - const selected = file.userTagGroup === key; - return ( - { - updateDataStorageUserTagGroup(file.id, selected ? undefined : key); - fetchDataStorageList(false); - }} - > - - - ); - })} - - { - e.stopPropagation(); - handleDeleteHistory(file.id); - }} - /> - + +
+ {`${intl.get(`dataSource.upload.history_time.${group.tag}`)}${ + group.tag === HistoryRecentTag.TODAY ? '' + : ` (${period})` + }`} +
+ + {group.list.map((file, i) => ( + fetchDataStorageList(false)} + /> + ))} + + ); - })} - + }) : ( + + {list.map((file, i) => ( + fetchDataStorageList(false)} + /> + ))} + + )} + ); }; diff --git a/packages/rath-client/src/pages/dataSource/selection/history/index.tsx b/packages/rath-client/src/pages/dataSource/selection/history/index.tsx index 843c978d..f83beb6d 100644 --- a/packages/rath-client/src/pages/dataSource/selection/history/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/history/index.tsx @@ -23,6 +23,7 @@ const HistoryPanel: FC ); From d5c2d4fc2e7d4cbbd8b13b9b19c69d12e058f4fa Mon Sep 17 00:00:00 2001 From: kyusho Date: Wed, 7 Dec 2022 18:13:42 +0800 Subject: [PATCH 13/13] feat(datasource): select range of excel table --- .../rath-client/public/locales/en-US.json | 5 +- .../rath-client/public/locales/zh-CN.json | 5 +- .../dataSource/selection/file/file-helper.tsx | 119 ++++++++++++++---- .../dataSource/selection/file/file-upload.tsx | 39 +++++- .../pages/dataSource/selection/file/index.tsx | 95 +++++++++++++- .../selection/history/history-list-item.tsx | 2 +- .../dataSource/selection/history/index.tsx | 5 +- .../src/pages/dataSource/utils/index.ts | 28 ++++- 8 files changed, 260 insertions(+), 38 deletions(-) diff --git a/packages/rath-client/public/locales/en-US.json b/packages/rath-client/public/locales/en-US.json index 6856f78c..23d3e279 100644 --- a/packages/rath-client/public/locales/en-US.json +++ b/packages/rath-client/public/locales/en-US.json @@ -242,10 +242,12 @@ }, "upload": { "title": "Upload Your own dataset", - "fileTypes": "JSON, CSV and Excel files are supported.", + "fileTypes": "Excel workbooks and text-based files (e.g. JSON, CSV) are supported.", "uniqueIdIssue": "Add unique ids for fields", + "show_more": "More Options", "sampling": "Sampling", "percentSize": "sample size(rows)", + "excel_range": "Range of cells", "upload": "Upload", "change": "Browse", "lastOpen": "Last opened", @@ -262,6 +264,7 @@ "new": "New file", "sheet": "Sheet", "preview_parsed": "Preview", + "preview_full": "Full", "preview_raw": "Raw", "data_is_empty": "This dataset is empty", "separator": { diff --git a/packages/rath-client/public/locales/zh-CN.json b/packages/rath-client/public/locales/zh-CN.json index 4a48ab09..88917374 100644 --- a/packages/rath-client/public/locales/zh-CN.json +++ b/packages/rath-client/public/locales/zh-CN.json @@ -242,10 +242,12 @@ }, "upload": { "title": "连接你的数据集，根据需求调整以下配置", - "fileTypes": "支持 JSON, CSV 及 Excel 文件", + "fileTypes": "支持 Excel 工作簿以及 JSON、CSV 等文本类型文件", "uniqueIdIssue": "添加唯一标识(字段是中文字符推荐使用)", + "show_more": "高级设置", "sampling": "数据采样", "percentSize": "样本大小(行)", + "excel_range": "工作簿范围", "upload": "上传文件", "change": "重新选择", "lastOpen": "上一次使用", @@ -262,6 +264,7 @@ "new": "新的文件", "sheet": "工作簿", "preview_parsed": "预览", + "preview_full": "完整内容", "preview_raw": "原始内容", "data_is_empty": "数据集是空的。", "separator": { diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx index 027d7855..756e02e6 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-helper.tsx @@ -1,7 +1,8 @@ import intl from 'react-intl-universal'; -import { ChoiceGroup, Dropdown, IChoiceGroupOption, SpinButton, TextField } from "@fluentui/react"; +import { ChoiceGroup, Dropdown, IChoiceGroupOption, Label, SpinButton, TextField } from "@fluentui/react"; import { FC, useMemo, useState } from "react"; import styled from "styled-components"; +import produce from 'immer'; import { SampleKey, useSampleOptions } from '../../utils'; @@ -57,9 +58,27 @@ const Container = styled.div` > * { margin-bottom: 0.8em; } + & .spin-group { + display: flex; + flex-direction: row; + align-items: center; + & * { + min-width: unset; + width: max-content; + height: max-content; + min-height: unset; + } + > * { + margin: 0 0.5em; + } + & input { + width: 3em; + } + } `; export interface IFileHelperProps { + showMoreConfig: boolean; charset: Charset; setCharset: (charset: Charset) => void; sampleMethod: SampleKey; @@ -67,6 +86,10 @@ export interface IFileHelperProps { sampleSize: number; setSampleSize: (sampleSize: number | ((prev: number) => number)) => void; preview: File | null; + isExcel: boolean; + excelRef: [[number, number], [number, number]]; + excelRange: [[number, number], [number, number]]; + setExcelRange: (range: [[number, number], [number, number]]) => void; sheetNames: string[] | false; selectedSheetIdx: number; setSelectedSheetIdx: (selectedSheetIdx: number) => void; @@ -75,8 +98,10 @@ export interface IFileHelperProps { } const FileHelper: FC = ({ + showMoreConfig, charset, setCharset, sampleMethod, setSampleMethod, sampleSize, setSampleSize, preview, sheetNames, selectedSheetIdx, setSelectedSheetIdx, separator, setSeparator, + isExcel, excelRef, excelRange, setExcelRange, }) => { const sampleOptions = useSampleOptions(); const [customizeSeparator, setCustomizeSeparator] = useState(''); @@ -114,28 +139,32 @@ const FileHelper: FC = ({ return ( - { - item && setCharset(item.key as Charset) - }} - styles={{ root: { display: 'flex', flexDirection: 'row', marginRight: '2em' }, label: { marginRight: '1em', fontWeight: 400 }, dropdown: { width: '8em' } }} - /> + {showMoreConfig && ( + { + item && setCharset(item.key as Charset) + }} + styles={{ root: { display: 'flex', flexDirection: 'row', marginRight: '2em' }, label: { marginRight: '1em', fontWeight: 400 }, dropdown: { width: '8em' } }} + /> + )} {!preview || preview.type.match(/^text\/.*/) ? ( <> - { - if (option) { - setSeparator(option.key); - } - }} - /> + {showMoreConfig && ( + { + if (option) { + setSeparator(option.key); + } + }} + /> + )} {(!preview || preview.type === 'text/csv') && separator === ',' && ( <> = ({ styles={{ root: { padding: '1em 0', display: 'flex', flexDirection: 'row', marginRight: '2em' }, label: { marginRight: '1em', fontWeight: 400 }, dropdown: { width: '10em' } }} /> )} + {isExcel && showMoreConfig && ( +
+ + str && setExcelRange(produce(excelRange, draft => { + draft[0][0] = Number(str); + }))} + styles={{ spinButtonWrapper: { display: 'flex', alignItems: 'center' } }} + /> + , + str && setExcelRange(produce(excelRange, draft => { + draft[0][1] = Number(str); + }))} + styles={{ spinButtonWrapper: { display: 'flex', alignItems: 'center' } }} + /> + {'-'} + str && setExcelRange(produce(excelRange, draft => { + draft[1][0] = Number(str); + }))} + styles={{ spinButtonWrapper: { display: 'flex', alignItems: 'center' } }} + /> + , + str && setExcelRange(produce(excelRange, draft => { + draft[1][1] = Number(str); + }))} + styles={{ spinButtonWrapper: { display: 'flex', alignItems: 'center' } }} + /> +
+ )} ); }; diff --git a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx index 994f857c..bcebeb32 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/file-upload.tsx @@ -1,7 +1,7 @@ import { ActionButton, Icon, Pivot, PivotItem, TooltipHost } from "@fluentui/react"; import intl from 'react-intl-universal'; import { observer } from "mobx-react-lite"; -import { forwardRef, useCallback, useImperativeHandle, useRef, useState } from "react"; +import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from "react"; import styled from "styled-components"; import type { loadDataFile } from "../../utils"; import { notify } from "../../../../components/error"; @@ -187,13 +187,14 @@ function formatSize(size: number) { export interface IFileUploadProps { preview: File | null; - previewOfFile: Awaited> | null; previewOfRaw: string | null; + previewOfFull: Awaited> | null; + previewOfFile: Awaited> | null; onFileUpload: (file: File | null) => void; } const FileUpload = forwardRef<{ reset: () => void }, IFileUploadProps>(function FileUpload ( - { preview, previewOfFile, previewOfRaw, onFileUpload }, ref + { preview, previewOfFile, previewOfFull, previewOfRaw, onFileUpload }, ref ) { const fileInputRef = useRef(null); @@ -227,7 +228,13 @@ const FileUpload = forwardRef<{ reset: () => void }, IFileUploadProps>(function onFileUpload(file ?? null); }, [onFileUpload]); - const [previewTab, setPreviewTab] = useState<'parsed' | 'raw'>('parsed'); + const [previewTab, setPreviewTab] = useState<'parsed' | 'full' | 'raw'>('parsed'); + + useEffect(() => { + if (previewTab === 'full' && !previewOfFull) { + setPreviewTab('parsed'); + } + }, [previewOfFull, previewTab]); return ( @@ -298,6 +305,30 @@ const FileUpload = forwardRef<{ reset: () => void }, IFileUploadProps>(function

{intl.get("dataSource.upload.data_is_empty")}

)} + {previewOfFull && ( + + + + + {previewOfFull.fields.map(f => ( + + {f.name || f.fid} + + ))} + + {previewOfFull.dataSource.slice(0, 20).map((row, i) => ( + + {previewOfFull.fields.map(f => ( + + {JSON.stringify(row[f.fid])} + + ))} + + ))} + + + + )} {previewOfRaw} diff --git a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx index f7876f8b..0ce8f932 100644 --- a/packages/rath-client/src/pages/dataSource/selection/file/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/file/index.tsx @@ -1,9 +1,10 @@ import { FC, useState, useRef, useCallback, useEffect } from "react"; -import { PrimaryButton } from '@fluentui/react'; +import { PrimaryButton, Toggle } from '@fluentui/react'; import styled from "styled-components"; import { observer } from "mobx-react-lite"; +import * as xlsx from 'xlsx'; import intl from "react-intl-universal"; -import { isExcelFile, loadDataFile, loadExcelFile, parseExcelFile, readRaw, SampleKey } from "../../utils"; +import { isExcelFile, loadDataFile, loadExcelFile, loadExcelRaw, parseExcelFile, readRaw, SampleKey } from "../../utils"; import { dataBackup, logDataImport } from "../../../../loggers/dataImport"; import type { IMuteFieldBase, IRow } from "../../../../interfaces"; import { DataSourceTag, IDBMeta } from "../../../../utils/storage"; @@ -17,6 +18,29 @@ const Container = styled.div` > header { margin-top: 1.2em; font-weight: 550; + &.upload { + display: flex; + flex-direction: row; + align-items: center; + > span { + flex-grow: 1; + flex-shrink: 1; + } + > div { + margin: 0; + flex-grow: 0; + flex-shrink: 0; + transform: scale(0.9); + > label { + padding-left: 0.3em; + margin: 0; + font-weight: 500; + } + > div { + transform: scale(0.8); + } + } + } } > .action { margin: 1em 0; @@ -53,10 +77,13 @@ const FileData: FC = (props) => { const [preview, setPreview] = useState(null); const [previewOfRaw, setPreviewOfRaw] = useState(null); + const [previewOfFull, setPreviewOfFull] = useState> | null>(null); const [previewOfFile, setPreviewOfFile] = useState> | null>(null); const [excelFile, setExcelFile] = useState> | false>(false); const [selectedSheetIdx, setSelectedSheetIdx] = useState(-1); + const [excelRef, setExcelRef] = useState<[[number, number], [number, number]]>([[0, 0], [0, 0]]); + const [excelRange, setExcelRange] = useState<[[number, number], [number, number]]>([[0, 0], [0, 0]]); useEffect(() => { setSelectedSheetIdx(-1); @@ -70,6 +97,7 @@ const FileData: FC = (props) => { filePreviewPendingRef.current = undefined; if (preview) { setPreviewOfRaw(null); + setPreviewOfFull(null); setPreviewOfFile(null); setExcelFile(false); toggleLoadingAnimation(true); @@ -105,6 +133,7 @@ const FileData: FC = (props) => { return; } setPreviewOfRaw(res[0].status === 'fulfilled' ? res[0].value : null); + setPreviewOfFull(null); setPreviewOfFile(res[1].status === 'fulfilled' ? res[1].value : null); }).catch(reason => { onLoadingFailed(reason); @@ -113,6 +142,8 @@ const FileData: FC = (props) => { toggleLoadingAnimation(false); }); } else { + setPreviewOfRaw(null); + setPreviewOfFull(null); setPreviewOfFile(null); } }, [charset, onDataLoading, onLoadingFailed, preview, sampleMethod, sampleSize, toggleLoadingAnimation, appliedSeparator]); @@ -120,16 +151,26 @@ const FileData: FC = (props) => { useEffect(() => { if (excelFile && selectedSheetIdx !== -1) { setPreviewOfRaw(null); + setPreviewOfFull(null); setPreviewOfFile(null); + const sheet = excelFile.Sheets[excelFile.SheetNames[selectedSheetIdx]]; + const range = sheet["!ref"] ? xlsx.utils.decode_range(sheet["!ref"]) : { s: { r: 0, c: 0 }, e: { r: 0, c: 0 } }; + const rangeRef = [[range.s.r, range.s.c], [range.e.r, range.e.c]] as [[number, number], [number, number]]; + setExcelRef(rangeRef); + setExcelRange(rangeRef); filePreviewPendingRef.current = undefined; toggleLoadingAnimation(true); - const p = loadExcelFile(excelFile, selectedSheetIdx, charset); + const p = Promise.allSettled([ + loadExcelRaw(excelFile, selectedSheetIdx, 4096, 64, 128), + loadExcelFile(excelFile, selectedSheetIdx, charset), + ] as const); filePreviewPendingRef.current = p; p.then(res => { if (p !== filePreviewPendingRef.current) { return; } - setPreviewOfFile(res); + setPreviewOfRaw(res[0].status === 'fulfilled' ? res[0].value : null); + setPreviewOfFull(res[1].status === 'fulfilled' ? res[1].value : null); }).catch(reason => { onLoadingFailed(reason); inputRef.current?.reset(); @@ -139,6 +180,26 @@ const FileData: FC = (props) => { } }, [excelFile, onLoadingFailed, selectedSheetIdx, toggleLoadingAnimation, charset]); + useEffect(() => { + if (excelFile && previewOfFull) { + setPreviewOfFile(null); + filePreviewPendingRef.current = undefined; + toggleLoadingAnimation(true); + const p = loadExcelFile(excelFile, selectedSheetIdx, charset, excelRange); + filePreviewPendingRef.current = p; + p.then(res => { + if (p !== filePreviewPendingRef.current) { + return; + } + setPreviewOfFile(res); + }).catch(reason => { + onLoadingFailed(reason); + }).finally(() => { + toggleLoadingAnimation(false); + }); + } + }, [charset, excelFile, excelRange, onLoadingFailed, previewOfFull, selectedSheetIdx, toggleLoadingAnimation]); + const handleFileLoad = useCallback((file: File | null) => { setPreview(file); }, []); @@ -159,10 +220,21 @@ const FileData: FC = (props) => { onClose(); }, [onClose, onDataLoaded, preview, previewOfFile]); + const [showMoreConfig, setShowMoreConfig] = useState(false); + return ( -
{intl.get('dataSource.upload.new')}
+
+ {intl.get('dataSource.upload.new')} + setShowMoreConfig(Boolean(checked))} + /> +
= (props) => { setSelectedSheetIdx={setSelectedSheetIdx} separator={separator} setSeparator={setSeparator} + isExcel={Boolean(excelFile)} + excelRef={excelRef} + excelRange={excelRange} + setExcelRange={setExcelRange} + /> + - {preview ? ( previewOfFile && (
diff --git a/packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx b/packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx index 23e2ecae..817c9bac 100644 --- a/packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/history/history-list-item.tsx @@ -219,7 +219,7 @@ const HistoryListItem: FC = ({ { e.stopPropagation(); handleClearClick?.(file.id); diff --git a/packages/rath-client/src/pages/dataSource/selection/history/index.tsx b/packages/rath-client/src/pages/dataSource/selection/history/index.tsx index f83beb6d..b29f78c2 100644 --- a/packages/rath-client/src/pages/dataSource/selection/history/index.tsx +++ b/packages/rath-client/src/pages/dataSource/selection/history/index.tsx @@ -1,4 +1,4 @@ -import { TextField } from "@fluentui/react"; +import { SearchBox } from "@fluentui/react"; import { observer } from "mobx-react-lite"; import { FC, useState } from "react"; import HistoryList, { IHistoryListProps } from "./history-list"; @@ -11,12 +11,13 @@ const HistoryPanel: FC - setSearch(value ?? '')} + underlined /> { return data; }; -export const loadExcelFile = async (data: Awaited>, sheetIdx: number, encoding: string): Promise<{ +export const loadExcelRaw = async (data: Awaited>, sheetIdx: number, limit?: number, rowLimit?: number, colLimit?: number): Promise => { + const sheet = data.SheetNames[sheetIdx]; + const worksheet = data.Sheets[sheet]; + const csvData = xlsx.utils.sheet_to_csv(worksheet, { skipHidden: true }); // more options available here + let text = csvData; + if (limit) { + text = text.slice(0, limit); + } + if (rowLimit || colLimit) { + text = text.split('\n').slice(0, rowLimit).map(row => row.slice(0, colLimit)).join('\n'); + } + return text; +}; + +export const loadExcelFile = async ( + data: Awaited>, sheetIdx: number, encoding: string, + range?: [[number, number], [number, number]], +): Promise<{ fields: IMuteFieldBase[]; dataSource: IRow[]; }> => { const sheet = data.SheetNames[sheetIdx]; const worksheet = data.Sheets[sheet]; - const csvData = xlsx.utils.sheet_to_csv(worksheet, { skipHidden: true }); // more options available here + const copy = range ? { ...worksheet } : worksheet; + if (range) { + copy['!ref'] = xlsx.utils.encode_range({ + s: { r: range[0][0], c: range[0][1] }, + e: { r: range[1][0], c: range[1][1] }, + }); + } + const csvData = xlsx.utils.sheet_to_csv(copy, { skipHidden: true }); // more options available here const csvFile = new File([new Blob([csvData], { type: 'text/plain' })], 'file.csv'); const rawData = (await KFileReader.csvReader({ file: csvFile,