Merge pull request #230 from Kanaries/feat-dataSource-local_xlsx-ks-1205

[feat] {datasource,user} support xlsx file; fix main service url

README.md

@@ -227,6 +227,10 @@ GNU Affero General Public License for more details.
 
 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+---
+
+Branded icons are licensed under their copyright license.
 <br>
 <p align="center">
   <br>

packages/rath-client/package.json

@@ -16,6 +16,7 @@
     "@emotion/styled": "^11.10.4",
     "@fluentui/font-icons-mdl2": "^8.4.13",
     "@fluentui/react": "^8.94.4",
+    "@fluentui/react-file-type-icons": "^8.8.3",
     "@fluentui/react-hooks": "^8.6.11",
     "@kanaries/graphic-walker": "0.2.8",
     "@kanaries/loa": "^0.0.16",
@@ -46,7 +47,8 @@
     "vega-scenegraph": "4.10.1-kanaries-patch",
     "visual-insights": "^0.12.3",
     "web-vitals": "^0.2.4",
-    "worker-loader": "^3.0.7"
+    "worker-loader": "^3.0.7",
+    "xlsx": "^0.18.5"
   },
   "devDependencies": {
     "@testing-library/jest-dom": "^5.11.8",

packages/rath-client/public/locales/en-US.json

@@ -63,7 +63,7 @@
     "config": {
         "computationEngine": {
             "title": "Computation Engine",
-            "desc": "Define where the analysis task run. If your dataset is small (< 100MB), web worker mode is recommanded. Otherwise try clickhosue mode.",
+            "desc": "Define where the analysis task run. If your dataset is small (< 100MB), web worker mode is recommended. Otherwise try clickhouse mode.",
             "clickhouse": "ClickHouse",
             "webworker": "Browser(Web Worker)",
             "notCompatible": "Not compatible with datasource"
@@ -120,6 +120,7 @@
         "dataView": "Table View",
         "statView": "Statistics View",
         "charset": "character encoding",
+        "separator": "Separator",
         "databaseType": "Select Database",
         "connectUri": "Connection URI",
         "databaseName": "Database",
@@ -138,6 +139,17 @@
             "originStatTable": "Original Data Statistics",
             "selectionStatTable": "Selection Data Statistics"
         },
+        "sizeInfo": "{nCols} fields, {nRows} rows",
+        "demoDataset": {
+            "CARS": { "title": "Cars", "description": "Origin, product name and physical attributes of cars." },
+            "STUDENTS": { "title": "Students", "description": "" },
+            "BTC_GOLD": { "title": "BTC - Gold", "description": "" },
+            "CAR_SALES": { "title": "Car Sales", "description": "" },
+            "COLLAGE": { "title": "Collage", "description": "" },
+            "TITANIC": { "title": "Titanic", "description": "" },
+            "KEPLER": { "title": "Kepler", "description": "" },
+            "BIKE_SHARING_DC": { "title": "Bike Sharing", "description": "" }
+        },
         "dbProgress": [
             {
                 "label": "Connection",
@@ -174,8 +186,9 @@
             "lackData": "Lack of Data",
             "lackDimension": "It seems you don't have dimensions in your dataset. (part of functions of mega-automation maybe influenced.)",
             "lackMeasure": "It seems you don't have measures in your dataset. (part of functions of mega-automation maybe influenced.)",
-            "smallSample": "The sample size is not big enough, which may influence the reliability of recommandation.",
-            "forceAnalysis": "Force Analysis"
+            "smallSample": "The sample size is not big enough, which may influence the reliability of recommendation.",
+            "forceAnalysis": "Force Analysis",
+            "upload_file_too_large": "This file is too large. Use a smaller subset or transform it into CSV file to apply sampling."
         },
         "meta": {
             "uniqueValue": "Unique Value",
@@ -229,11 +242,37 @@
         },
         "upload": {
             "title": "Upload Your own dataset",
-            "fileTypes": "csv, json are supportted.",
+            "fileTypes": "Excel workbooks and text-based files (e.g. JSON, CSV) are supported.",
             "uniqueIdIssue": "Add unique ids for fields",
+            "show_more": "More Options",
             "sampling": "Sampling",
             "percentSize": "sample size(rows)",
-            "upload": "Upload"
+            "excel_range": "Range of cells",
+            "upload": "Upload",
+            "change": "Browse",
+            "lastOpen": "Last opened",
+            "firstOpen": "First created",
+            "history": "Open Recent",
+            "history_time": {
+                "1d": "Today",
+                "1w": "Last weak",
+                "1mo": "Last month",
+                "3mo": "Last 3 months",
+                "6mo": "Last half a year",
+                "1yr": "Last year"
+            },
+            "new": "New file",
+            "sheet": "Sheet",
+            "preview_parsed": "Preview",
+            "preview_full": "Full",
+            "preview_raw": "Raw",
+            "data_is_empty": "This dataset is empty",
+            "separator": {
+                "comma": "Comma",
+                "semicolon": "Semicolon",
+                "tab": "Tab",
+                "other": "Other..."
+            }
         },
         "exploreMode": {
             "title": "Explore Mode",
@@ -353,7 +392,7 @@
             "pin": "pin",
             "compare": "compare",
             "vizsys": {
-                "title": "visualization recommand system",
+                "title": "visualization recommend system",
                 "lite": "Lite mode(fast)",
                 "strict": "Strict mode"
             },
@@ -436,7 +475,7 @@
             "$ReLU": "Projects a field by x -> max(0, x) as a new column.",
             "$match": "Finds the matched parts of each text content using Regular Expression as a new column.",
             "$replace": "Replaces the matched parts of each text content with a given string using Regular Expression as a new column.",
-            "$concat": "Concats text contents of all the fields as a new column.",
+            "$concat": "Merge two or more text contents of all the fields as a new column.",
             "$sigmoid": "Projects a field by x -> (1 + e^-x)^-1 as a new column.",
             "$boxClip": "Maps overflowing values (due to boxplot) to the domain as a new column.",
             "$meanClip": "Maps overflowing values (due to the given domain) to the mean of all the other values as a new column.",
@@ -479,8 +518,8 @@
         "shortMonths": ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
     },
     "login": {
-        "clickLogin": "Click Login",
-        "haveSent": "Alredy Send",
+        "clickLogin": "Login",
+        "haveSent": "Already Sent",
         "signIn": "Sign In",
         "signOut": "Sign Out",
         "preferences": "Preferences",
@@ -506,7 +545,7 @@
             "errEmail": "not support email type"
         },
         "password": {
-            "userName": "UsernNme",
+            "userName": "Username",
             "password": "Password"
         }
     }

packages/rath-client/public/locales/zh-CN.json

@@ -119,6 +119,7 @@
         "dataView": "数据视图",
         "statView": "统计视图",
         "charset": "字符集编码",
+        "separator": "分隔符",
         "databaseType": "选择数据库",
         "connectUri": "连接 URI",
         "databaseName": "数据库",
@@ -137,6 +138,17 @@
             "originStatTable": "原始数据统计信息",
             "selectionStatTable": "筛选部分统计信息"
         },
+        "sizeInfo": "{nCols} 列 x {nRows} 行",
+        "demoDataset": {
+            "CARS": { "title": "汽车数据集", "description": "记录了汽车的型号、厂商、物理参数的数据。" },
+            "STUDENTS": { "title": "学生成绩数据集", "description": "分析学生的成绩和部分其他因素等关系。" },
+            "BTC_GOLD": { "title": "比特币-金价数据集", "description": "" },
+            "CAR_SALES": { "title": "汽车销售数据集", "description": "" },
+            "COLLAGE": { "title": "大学数据集", "description": "" },
+            "TITANIC": { "title": "泰坦尼克数据集", "description": "" },
+            "KEPLER": { "title": "开普勒数据集", "description": "" },
+            "BIKE_SHARING_DC": { "title": "共享单车数据集", "description": "" }
+        },
         "dbProgress": [
             {
                 "label": "建立连接",
@@ -174,7 +186,8 @@
             "lackDimension": "数据集中缺少维度。（全自动化分析模块部分能力会受到影响）",
             "lackMeasure": "数据集中缺少度量，（全自动化分析模块会受到影响，可以尝试使用其他模块）",
             "smallSample": "数据集中样本数量低于预期，可能会对推荐结果的一般性造成影响。（小样本问题）",
-            "forceAnalysis": "强制分析"
+            "forceAnalysis": "强制分析",
+            "upload_file_too_large": "文件体积过大。尝试使用更少的数据，或将文件转换为 CSV 格式以使用采样功能。"
         },
         "meta": {
             "title": "元数据视图",
@@ -229,11 +242,37 @@
         },
         "upload": {
             "title": "连接你的数据集，根据需求调整以下配置",
-            "fileTypes": "支持csv, json文件",
+            "fileTypes": "支持 Excel 工作簿以及 JSON、CSV 等文本类型文件",
             "uniqueIdIssue": "添加唯一标识(字段是中文字符推荐使用)",
+            "show_more": "高级设置",
             "sampling": "数据采样",
             "percentSize": "样本大小(行)",
-            "upload": "上传文件"
+            "excel_range": "工作簿范围",
+            "upload": "上传文件",
+            "change": "重新选择",
+            "lastOpen": "上一次使用",
+            "firstOpen": "创建时间",
+            "history": "最近使用",
+            "history_time": {
+                "1d": "今天",
+                "1w": "过去一周",
+                "1mo": "过去一个月",
+                "3mo": "过去三个月",
+                "6mo": "过去半年",
+                "1yr": "一年内"
+            },
+            "new": "新的文件",
+            "sheet": "工作簿",
+            "preview_parsed": "预览",
+            "preview_full": "完整内容",
+            "preview_raw": "原始内容",
+            "data_is_empty": "数据集是空的。",
+            "separator": {
+                "comma": "逗号",
+                "semicolon": "分号",
+                "tab": "制表符",
+                "other": "自定义..."
+            }
         },
         "exploreMode": {
             "title": "探索模式",

packages/rath-client/src/App.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect } from 'react';
+import { useEffect, useState } from 'react';
 import { observer } from 'mobx-react-lite';
 import { Spinner, SpinnerSize } from '@fluentui/react';
 import './App.css';
@@ -19,29 +19,8 @@ import Collection from './pages/collection';
 import Dashboard from './pages/dashboard';
 import CausalPage from './pages/causal';
 import PerformanceWindow from './components/performance-window';
-import LoginInfo from './pages/loginInfo';
-import Account from './pages/loginInfo/account';
-import Setup from './pages/loginInfo/setup';
+import useHotKey from './hooks/use-hotkey';
 
-export enum PreferencesType {
-    Account = 'account',
-    Info = 'info',
-    Setting = 'setting',
-    Header = 'header'
-}
-export interface PreferencesListType {
-    key: PreferencesType;
-    name: PreferencesType;
-    icon: string;
-    element: () => JSX.Element;
-}
-
-const preferencesList: PreferencesListType[] = [
-    { key: PreferencesType.Account, name: PreferencesType.Account, icon: 'Home', element: () => <Account /> },
-    // { key: PreferencesType.Info, name: PreferencesType.Info, icon: 'Info', element: () => <Info /> },
-    // { key: PreferencesType.Header, name: PreferencesType.Header, icon: 'Contact', element: () => <Header /> },
-    { key: PreferencesType.Setting, name: PreferencesType.Setting, icon: 'Settings', element: () => <Setup /> },
-];
 
 function App() {
     const { langStore, commonStore } = useGlobalStore();
@@ -60,6 +39,11 @@ function App() {
         };
     }, [commonStore]);
 
+    const [showPerformanceWindow, setShowPerformanceWindow] = useState(false);
+    useHotKey({
+        'Control+Shift+P': () => setShowPerformanceWindow(on => !on),
+    });
+
     if (!langStore.loaded) {
         return (
             <div style={{ marginTop: '6em' }}>
@@ -68,20 +52,11 @@ function App() {
         );
     }
 
-    const showPerformanceWindow = (new URL(window.location.href).searchParams.get('performance') ?? (
-        JSON.stringify(process.env.NODE_ENV !== 'production') && false  // temporarily banned this feature
-    )) === 'true';
-
     return (
         <div>
             <div className="main-app-container">
                 <div className="main-app-nav" style={{ flexBasis: navMode === 'text' ? '220px' : '3px' }}>
-                    <LoginInfo
-                        element={() => {
-                            return <AppNav />;
-                        }}
-                        preferencesList={preferencesList}
-                    />
+                    <AppNav />
                 </div>
                 <div className="main-app-content">
                     <div className="message-container">

packages/rath-client/src/components/appNav.tsx

@@ -6,11 +6,13 @@ import styled from 'styled-components';
 
 import { PIVOT_KEYS } from '../constants';
 import { useGlobalStore } from '../store';
+import LoginInfo from '../pages/loginInfo';
 import useHotKey from '../hooks/use-hotkey';
 import UserSetting from './userSettings';
 
 const NavContainer = styled.div`
-    height: 100%;
+    height: 100vh;
+    overflow: hidden auto;
     /* display: relative; */
     position: relative;
     /* flex-direction: vertical; */
@@ -20,6 +22,9 @@ const NavContainer = styled.div`
         /* position: absolute; */
         bottom: 0px;
         /* padding-left: 1em; */
+        flex-grow: 0;
+        flex-shrink: 0;
+        overflow: hidden;
     }
     padding-left: 10px;
     .text-red {
@@ -196,11 +201,12 @@ const AppNav: React.FC<AppNavProps> = (props) => {
                     </h1>
                 )}
             </LogoBar>
-            <div className="flex-1">
+            <div style={{ flexGrow: 1, flexShrink: 1 }}>
                 <Nav selectedKey={appKey} groups={groups} />
             </div>
             <div className="nav-footer">
                 <UserSetting />
+                <LoginInfo />
             </div>
         </NavContainer>
     );

packages/rath-client/src/constants.ts

@@ -30,8 +30,8 @@ export const EXPLORE_MODE = {
 export const DEMO_DATA_REQUEST_TIMEOUT = 1000 * 10;
 
 export const ENGINE_CONNECTION_STAGES: Array<{ stage: number; name: IECStatus; description?: string }> = [
-    { stage: 0, name: 'client', description: 'client module importetd.' },
-    { stage: 1, name: 'proxy', description: 'database proxy connector lanuched.' },
+    { stage: 0, name: 'client', description: 'client module imported.' },
+    { stage: 1, name: 'proxy', description: 'database proxy connector launched.' },
     { stage: 2, name: 'engine', description: 'clickhouse connected.' },
 ];
 
@@ -53,3 +53,19 @@ export const STORAGES = {
     CONFIG: 'config',
     ITERATOR_META: 'iterator_meta',
 }
+
+export enum RATH_ENV {
+    DEV = 'development environment',
+    TEST = 'test environment',
+    LPE = 'local preview environment',
+    IPE = 'integrative preview environment',
+    ONLINE = 'online production environment',
+}
+
+// This file is included in Worker so never forget to check if `window` is undefined!!!!!
+export const RathEnv: RATH_ENV = (
+    process.env.NODE_ENV === 'development' ? RATH_ENV.DEV
+        : process.env.NODE_ENV === 'test' ? RATH_ENV.TEST
+        : globalThis.window === undefined || globalThis.window?.location.host.match(/^(.*\.)?kanaries\.(net|cn)$/) ? RATH_ENV.ONLINE
+        : globalThis.window?.location.host.match(/^.*kanaries\.vercel\.app$/) ? RATH_ENV.IPE : RATH_ENV.LPE
+);

packages/rath-client/src/hooks/index.ts

@@ -3,7 +3,7 @@ import produce, { Draft } from 'immer';
 import intl from 'react-intl-universal';
 import { CleanMethod } from '../interfaces';
 import { notify } from '../components/error';
-import { getServerUrl } from '../utils/user';
+import { getMainServerUrl } from '../utils/user';
 import { request } from '../utils/request';
 
 /**
@@ -56,7 +56,7 @@ export const useCleanMethodList = function (): typeof cleanMethodList {
 };
 
 async function sendCertMail(email: string) {
-    const url = getServerUrl('/api/sendMailCert');
+    const url = getMainServerUrl('/api/sendMailCert');
     // TODO: [feat] email format check
     const res = await request.post<{ email: string }, string>(url, { email });
     if (res) {
@@ -66,7 +66,7 @@ async function sendCertMail(email: string) {
 }
 
 async function sendCertPhone(phone: string) {
-    const url = getServerUrl('/api/sendPhoneCert');
+    const url = getMainServerUrl('/api/sendPhoneCert');
     const res = await request.post<{ phone: string }, string>(url, { phone });
     if (res) {
         // console.log("message sent success");