Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CLI hints at how long a sync should take #241

Merged
merged 2 commits into from Oct 27, 2022
Merged

CLI hints at how long a sync should take #241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
42b840a
CLI hints at how long a sync should take
thomas-gerber Oct 27, 2022
69e853c
Rounding
thomas-gerber Oct 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 20 additions & 2 deletions cli/src/airbyte/airbyte-client.ts
View file
Open in desktop
@@ -1,5 +1,6 @@
import retry from 'async-retry';
import axios, {AxiosInstance} from 'axios';
import { ceil } from 'lodash';
import ProgressBar from 'progress';
import {VError} from 'verror';

Expand Down Expand Up @@ -81,9 +82,26 @@ export class Airbyte {
return response.data.job.status;
}

async triggerAndTrackSync(connectionId: string): Promise<void> {
async triggerAndTrackSync(
connectionId: string,
days: number,
entries: number
): Promise<void> {
try {
display('Syncing %s', Emoji.SYNC);
display(
'Syncing %s days of data for %s repos/projects %s',
days,
entries,
Emoji.SYNC
);
const duration_lower = ceil(entries * days) / 30;
const duration_upper = 2 * duration_lower;
display(
'Time to get that data is typically between %s and %s minutes %s',
duration_lower,
duration_upper,
Emoji.STOPWATCH
);
const job = await this.triggerSync(connectionId);

const syncBar = new ProgressBar(':bar', {
Expand Down
12 changes: 8 additions & 4 deletions cli/src/bitbucket/run.ts
View file
Open in desktop
Expand Up @@ -140,10 +140,10 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise<void> {
break;
}

try {
const workspaces = cfg.workspace;
const repos = cfg.repoList;
const workspaces = cfg.workspace;
const repos = cfg.repoList;

try {
if (!repos || repos.length === 0) {
try {
if ((await getWorkspaces(bitbucket)).length === 0) {
Expand Down Expand Up @@ -219,7 +219,11 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise<void> {
return;
}

await cfg.airbyte.triggerAndTrackSync(BITBUCKET_CONNECTION_ID);
await cfg.airbyte.triggerAndTrackSync(
BITBUCKET_CONNECTION_ID,
cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
repos?.length || 0
);
}

interface Workspace {
Expand Down
10 changes: 7 additions & 3 deletions cli/src/github/run.ts
View file
Open in desktop
Expand Up @@ -80,9 +80,9 @@ export async function runGithub(cfg: GithubConfig): Promise<void> {
startDate.getDate() - (cfg.cutoffDays || DEFAULT_CUTOFF_DAYS)
);

try {
let repos = cfg.repoList;
let repos = cfg.repoList;

try {
if (!repos || repos.length === 0) {
try {
if ((await getRepos(token)).length === 0) {
Expand Down Expand Up @@ -138,7 +138,11 @@ export async function runGithub(cfg: GithubConfig): Promise<void> {
return;
}

await cfg.airbyte.triggerAndTrackSync(GITHUB_CONNECTION_ID);
await cfg.airbyte.triggerAndTrackSync(
GITHUB_CONNECTION_ID,
cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
repos?.length || 0
);
}

async function promptForRepos(token: string): Promise<ReadonlyArray<string>> {
Expand Down
10 changes: 7 additions & 3 deletions cli/src/gitlab/run.ts
View file
Open in desktop
Expand Up @@ -91,9 +91,9 @@ export async function runGitlab(cfg: GitLabConfig): Promise<void> {
startDate.getDate() - (cfg.cutoffDays || DEFAULT_CUTOFF_DAYS)
);

try {
let projects = cfg.projectList;
let projects = cfg.projectList;

try {
if (!projects || projects.length === 0) {
try {
if ((await getProjects(token, api_url)).length === 0) {
Expand Down Expand Up @@ -146,7 +146,11 @@ export async function runGitlab(cfg: GitLabConfig): Promise<void> {
return;
}

await cfg.airbyte.triggerAndTrackSync(GITLAB_CONNECTION_ID);
await cfg.airbyte.triggerAndTrackSync(
GITLAB_CONNECTION_ID,
cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
projects?.length || 0
);
}

async function promptForProjects(
Expand Down
10 changes: 7 additions & 3 deletions cli/src/jira/run.ts
View file
Open in desktop
Expand Up @@ -107,9 +107,9 @@ export async function runJira(cfg: JiraConfig): Promise<void> {
startDate.getDate() - (cfg.cutoffDays || DEFAULT_CUTOFF_DAYS)
);

try {
let projects = cfg.projectList;
let projects = cfg.projectList;

try {
if (!projects || projects.length === 0) {
try {
if ((await getProjects(jira)).length === 0) {
Expand Down Expand Up @@ -165,7 +165,11 @@ export async function runJira(cfg: JiraConfig): Promise<void> {
return;
}

await cfg.airbyte.triggerAndTrackSync(JIRA_CONNECTION_ID);
await cfg.airbyte.triggerAndTrackSync(
JIRA_CONNECTION_ID,
cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
projects?.length || 0
);
}

async function promptForProjects(
Expand Down
1 change: 1 addition & 0 deletions cli/src/utils/index.ts
View file
Open in desktop
Expand Up @@ -12,6 +12,7 @@ export enum Emoji {
CHECK_CONNECTION = '🔍',
PROGRESS = '⏳',
EMPTY = '🪹',
STOPWATCH = '⏱',
}

function processEmoji(...args: any[]): any[] {
Expand Down
1 change: 0 additions & 1 deletion cli/test/integration-tests/integration.test.ts
View file
Open in desktop
Expand Up @@ -6,7 +6,6 @@ const EXPECTED_OUTPUT_SUCCESS_SYNC = [
'Checking connection with Airbyte \n',
'Setting up source \n',
'Setup succeeded \n',
'Syncing \n',
'Syncing succeeded \n',
];

Expand Down