Display Throughput and Progress Bar in Sync UI (#19193)

* Display Throughput and Progress Bar in Sync UI * test for numberHelper * use clear button * noEstimate * remove ? * cleanup ternary * clear up styles * fixup color ref * cleanup and test math * pluralize * Custom progress bar line * remove padding on clear buttons * Tim's suggestions * size=xs * Some cleanups * Continue work (WIP) * WIP * WIP * WIP * Cleanup utils file * Cleanup i18n * Cleanup unused classes * Cleanup more leftovers * Remove debug data Co-authored-by: Tim Roes <tim@airbyte.io>
airbytehq · Dec 7, 2022 · 28f1f49 · 28f1f49
1 parent a7c4f1d
commit 28f1f49
Show file tree

Hide file tree

Showing 18 changed files with 576 additions and 18 deletions.
diff --git a/airbyte-webapp/.eslintrc.js b/airbyte-webapp/.eslintrc.js
@@ -89,6 +89,7 @@ module.exports = {
     "react/jsx-fragments": "warn",
     "react/jsx-no-useless-fragment": ["warn", { allowExpressions: true }],
     "react/self-closing-comp": "warn",
+    "react/style-prop-object": ["warn", { allow: ["FormattedNumber"] }],
     "no-restricted-imports": [
       "error",
       {

diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx
@@ -3,6 +3,8 @@ import dayjs from "dayjs";
 import React from "react";
 import { FormattedMessage, useIntl } from "react-intl";
 
+import { formatBytes } from "utils/numberHelper";
+
 import { AttemptRead, AttemptStatus } from "../../../core/request/AirbyteClient";
 import styles from "./AttemptDetails.module.scss";
 
@@ -23,20 +25,6 @@ const AttemptDetails: React.FC<AttemptDetailsProps> = ({ attempt, className, has
     return null;
   }
 
-  const formatBytes = (bytes?: number) => {
-    if (!bytes) {
-      return <FormattedMessage id="sources.countBytes" values={{ count: bytes || 0 }} />;
-    }
-
-    const k = 1024;
-    const dm = 2;
-    const sizes = ["Bytes", "KB", "MB", "GB", "TB"];
-    const i = Math.floor(Math.log(bytes) / Math.log(k));
-    const result = parseFloat((bytes / Math.pow(k, i)).toFixed(dm));
-
-    return <FormattedMessage id={`sources.count${sizes[i]}`} values={{ count: result }} />;
-  };
-
   const getFailureOrigin = (attempt: AttemptRead) => {
     const failure = getFailureFromAttempt(attempt);
     const failureOrigin = failure?.failureOrigin ?? formatMessage({ id: "errorView.unknown" });

diff --git a/airbyte-webapp/src/components/JobItem/components/MainInfo.module.scss b/airbyte-webapp/src/components/JobItem/components/MainInfo.module.scss
@@ -5,10 +5,10 @@
   cursor: pointer;
   height: unset !important;
   min-height: 70px;
+  gap: variables.$spacing-lg;
   padding: variables.$spacing-lg 0;
 
   .titleCell {
-    width: 80%;
     display: flex;
     color: colors.$dark-blue;
     min-width: 0;
@@ -39,6 +39,7 @@
     display: flex;
     justify-content: flex-end;
     align-items: center;
+    flex: 0 0 auto;
 
     .attemptCount {
       font-size: 12px;

diff --git a/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx b/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx
@@ -4,6 +4,7 @@ import classNames from "classnames";
 import React, { useMemo } from "react";
 import { FormattedDateParts, FormattedMessage, FormattedTimeParts } from "react-intl";
 
+import { JobProgress } from "components/connection/JobProgress";
 import { Cell, Row } from "components/SimpleTableComponents";
 import { StatusIcon } from "components/ui/StatusIcon";
 
@@ -87,6 +88,7 @@ const MainInfo: React.FC<MainInfoProps> = ({ job, attempts = [], isOpen, onExpan
         <div className={styles.statusIcon}>{statusIcon}</div>
         <div className={styles.justification}>
           {label}
+          {jobConfigType === "sync" && <JobProgress job={job} expanded={isOpen} />}
           {attempts.length > 0 && (
             <>
               {jobConfigType === "reset_connection" ? (

diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.module.scss b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.module.scss
@@ -0,0 +1,23 @@
+@use "scss/variables";
+@use "scss/colors";
+
+.estimationStats {
+  display: flex;
+  justify-content: space-between;
+}
+
+.estimationDetails {
+  display: flex;
+  gap: variables.$spacing-lg;
+  margin: variables.$spacing-md 0;
+
+  .icon {
+    color: colors.$grey-400;
+    margin-right: variables.$spacing-sm;
+  }
+}
+
+.streams {
+  margin: variables.$spacing-md 0 0;
+  width: 100%;
+}
diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx
@@ -0,0 +1,126 @@
+import { faDatabase, faDiagramNext } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import classNames from "classnames";
+import { FormattedMessage, useIntl } from "react-intl";
+
+import { getJobStatus } from "components/JobItem/JobItem";
+import { Text } from "components/ui/Text";
+
+import { AttemptRead, AttemptStatus, SynchronousJobRead } from "core/request/AirbyteClient";
+import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/JobsList";
+import { formatBytes } from "utils/numberHelper";
+
+import styles from "./JobProgress.module.scss";
+import { ProgressLine } from "./JobProgressLine";
+import { StreamProgress } from "./StreamProgress";
+import { progressBarCalculations } from "./utils";
+
+function isJobsWithJobs(job: JobsWithJobs | SynchronousJobRead): job is JobsWithJobs {
+  return "attempts" in job;
+}
+
+interface ProgressBarProps {
+  job: JobsWithJobs | SynchronousJobRead;
+  expanded?: boolean;
+}
+
+export const JobProgress: React.FC<ProgressBarProps> = ({ job, expanded }) => {
+  const { formatMessage, formatNumber } = useIntl();
+
+  let latestAttempt: AttemptRead | undefined;
+  if (isJobsWithJobs(job) && job.attempts) {
+    latestAttempt = job.attempts[job.attempts.length - 1];
+  }
+  if (!latestAttempt) {
+    return null;
+  }
+
+  const jobStatus = getJobStatus(job);
+  if (["failed", "succeeded", "cancelled"].includes(jobStatus)) {
+    return null;
+  }
+
+  const {
+    displayProgressBar,
+    totalPercentRecords,
+    timeRemaining,
+    numeratorBytes,
+    numeratorRecords,
+    denominatorRecords,
+    denominatorBytes,
+    elapsedTimeMS,
+  } = progressBarCalculations(latestAttempt);
+
+  let timeRemainingString = "";
+  if (elapsedTimeMS && timeRemaining) {
+    const minutesRemaining = Math.ceil(timeRemaining / 1000 / 60);
+    const hoursRemaining = Math.ceil(minutesRemaining / 60);
+    if (minutesRemaining <= 60) {
+      timeRemainingString = formatMessage({ id: "connection.progress.minutesRemaining" }, { value: minutesRemaining });
+    } else {
+      timeRemainingString = formatMessage({ id: "connection.progress.hoursRemaining" }, { value: hoursRemaining });
+    }
+  }
+
+  return (
+    <Text as="div" size="md">
+      {displayProgressBar && (
+        <ProgressLine percent={totalPercentRecords} type={jobStatus === "incomplete" ? "warning" : "default"} />
+      )}
+      {latestAttempt?.status === AttemptStatus.running && (
+        <>
+          {displayProgressBar && (
+            <div className={styles.estimationStats}>
+              <span>{timeRemaining < Infinity && timeRemaining > 0 && timeRemainingString}</span>
+              <span>{formatNumber(totalPercentRecords, { style: "percent", maximumFractionDigits: 0 })}</span>
+            </div>
+          )}
+          {expanded && (
+            <>
+              {denominatorRecords > 0 && denominatorBytes > 0 && (
+                <div className={styles.estimationDetails}>
+                  <span>
+                    <FontAwesomeIcon icon={faDiagramNext} className={styles.icon} />
+                    <FormattedMessage
+                      id="connection.progress.recordsSynced"
+                      values={{
+                        synced: numeratorRecords,
+                        total: denominatorRecords,
+                        speed: Math.round((numeratorRecords / elapsedTimeMS) * 1000),
+                      }}
+                    />
+                  </span>
+                  <span>
+                    <FontAwesomeIcon icon={faDatabase} className={styles.icon} />
+                    <FormattedMessage
+                      id="connection.progress.bytesSynced"
+                      values={{
+                        synced: formatBytes(numeratorBytes),
+                        total: formatBytes(denominatorBytes),
+                        speed: formatBytes((numeratorBytes * 1000) / elapsedTimeMS),
+                      }}
+                    />
+                  </span>
+                </div>
+              )}
+              {latestAttempt.streamStats && (
+                <div className={classNames(styles.streams)}>
+                  {latestAttempt.streamStats
+                    ?.map((stats) => ({
+                      ...stats,
+                      done: (stats.stats.recordsEmitted ?? 0) >= (stats.stats.estimatedRecords ?? Infinity),
+                    }))
+                    // Move finished streams to the end of the list
+                    .sort((a, b) => Number(a.done) - Number(b.done))
+                    .map((stream) => {
+                      return <StreamProgress stream={stream} key={stream.streamName} />;
+                    })}
+                </div>
+              )}
+            </>
+          )}
+        </>
+      )}
+    </Text>
+  );
+};
diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.module.scss b/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.module.scss
@@ -0,0 +1,24 @@
+@use "scss/colors";
+
+.lineOuter {
+  height: 5px;
+  width: 100%;
+  background-color: colors.$grey-100;
+  border-radius: 10px;
+  margin-top: 5px;
+  margin-bottom: 5px;
+}
+
+.lineInner {
+  height: 100%;
+  border-radius: 10px;
+  transition: width 5s ease-in-out;
+
+  &.warning {
+    background-color: colors.$yellow-400;
+  }
+
+  &.default {
+    background-color: colors.$blue-200;
+  }
+}
diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.tsx b/airbyte-webapp/src/components/connection/JobProgress/JobProgressLine.tsx
@@ -0,0 +1,29 @@
+// Inspired by https://dev.to/ramonak/react-how-to-create-a-custom-progress-bar-component-in-5-minutes-2lcl
+
+import classNames from "classnames";
+import { useIntl } from "react-intl";
+
+import styles from "./JobProgressLine.module.scss";
+
+interface ProgressLineProps {
+  type?: "default" | "warning";
+  percent: number;
+}
+
+export const ProgressLine: React.FC<ProgressLineProps> = ({ type = "default", percent }) => {
+  const { formatMessage } = useIntl();
+  return (
+    <div
+      className={classNames(styles.lineOuter)}
+      aria-label={formatMessage({ id: "connection.progress.percentage" }, { percent: Math.floor(percent * 100) })}
+    >
+      <div
+        style={{ width: `${percent * 100}%` }}
+        className={classNames(styles.lineInner, {
+          [styles.default]: type === "default",
+          [styles.warning]: type === "warning",
+        })}
+      />
+    </div>
+  );
+};
diff --git a/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.module.scss b/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.module.scss
@@ -0,0 +1,73 @@
+@use "scss/variables";
+@use "scss/colors";
+
+.stream {
+  display: inline-block;
+  padding: variables.$spacing-xs variables.$spacing-md;
+  background: colors.$blue-50;
+  border-radius: 16px;
+  margin-right: variables.$spacing-md;
+  margin-bottom: variables.$spacing-sm;
+  white-space: nowrap;
+  color: colors.$grey-700;
+  line-height: 16px;
+}
+
+.wrapper {
+  display: flex;
+  align-items: center;
+  gap: variables.$spacing-sm;
+  min-height: 16px + 2 * variables.$spacing-xs;
+}
+
+.progress {
+  justify-content: center;
+  align-items: center;
+  min-width: 16px;
+  margin: variables.$spacing-xs;
+  aspect-ratio: 1 / 1;
+  display: flex;
+
+  .check {
+    fill: colors.$white;
+    display: none;
+  }
+
+  .fg {
+    stroke: colors.$blue;
+  }
+
+  .bg {
+    fill: colors.$white;
+  }
+
+  &.done {
+    .bg {
+      fill: colors.$green;
+    }
+
+    .fg {
+      display: none;
+    }
+
+    .check {
+      display: block;
+    }
+  }
+}
+
+.metrics {
+  margin: variables.$spacing-md 0 0;
+  display: grid;
+  grid-template-columns: max-content auto;
+  gap: variables.$spacing-md;
+
+  dt {
+    grid-column-start: 1;
+  }
+
+  dd {
+    margin: 0;
+    grid-column-start: 2;
+  }
+}