Skip to content

Commit

Permalink
fix(mutagen): use shorter directories for mutagen syncs (#4867)
Browse files Browse the repository at this point in the history 
Use directory at shorter path for mutagen syncs if the project
path is longer than MUTAGEN_DATA_DIRECTORY_LENGTH_LIMIT. 
The shorter directory path is computed using hash of the path of 
.garden dir in project and appending to $HOME/.garden/mutagen.
  • Loading branch information
@shumailxyz
shumailxyz committed Jul 20, 2023
1 parent fbc2320 commit 2698410
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 43 deletions.
60 changes: 38 additions & 22 deletions core/src/mutagen.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,28 @@
*/

import AsyncLock from "async-lock"
import Bluebird from "bluebird"
import chalk from "chalk"
import { join } from "path"
import dedent from "dedent"
import EventEmitter from "events"
import { ExecaReturnValue } from "execa"
import { mkdirp, pathExists } from "fs-extra"
import hasha from "hasha"
import pRetry from "p-retry"
import { join } from "path"
import respawn from "respawn"
import split2 from "split2"
import { GARDEN_GLOBAL_PATH, MUTAGEN_DIR_NAME } from "./constants"
import { GardenBaseError } from "./exceptions"
import pMemoize from "./lib/p-memoize"
import { Log } from "./logger/log-entry"
import { PluginContext } from "./plugin-context"
import { PluginToolSpec } from "./plugin/tools"
import { PluginTool } from "./util/ext-tools"
import { registerCleanupFunction, sleep } from "./util/util"
import { GardenBaseError } from "./exceptions"
import pRetry from "p-retry"
import { syncGuideLink } from "./plugins/kubernetes/sync"
import dedent from "dedent"
import { PluginContext } from "./plugin-context"
import Bluebird from "bluebird"
import { MUTAGEN_DIR_NAME } from "./constants"
import { ExecaReturnValue } from "execa"
import EventEmitter from "events"
import split2 from "split2"
import { TypedEventEmitter } from "./util/events"
import pMemoize from "./lib/p-memoize"
import { PluginTool } from "./util/ext-tools"
import { deline } from "./util/string"
import { registerCleanupFunction, sleep } from "./util/util"
import { emitNonRepeatableWarning } from "./warnings"

const maxRestarts = 10
Expand Down Expand Up @@ -134,7 +135,6 @@ export class MutagenError extends GardenBaseError {
interface MutagenDaemonParams {
ctx: PluginContext
log: Log
dataDir?: string
}

interface MutagenMonitorParams {
Expand Down Expand Up @@ -328,10 +328,10 @@ export class Mutagen {
private configLock: AsyncLock
private monitoring: boolean

constructor({ ctx, log, dataDir }: MutagenDaemonParams) {
constructor({ ctx, log }: MutagenDaemonParams) {
this.log = log
this.configLock = new AsyncLock()
this.dataDir = dataDir || join(ctx.gardenDirPath, MUTAGEN_DIR_NAME)
this.dataDir = getMutagenDataDir(ctx.gardenDirPath, log)
this.activeSyncs = {}
this.monitoring = false

Expand Down Expand Up @@ -629,7 +629,7 @@ export class Mutagen {
cwd: this.dataDir,
args,
log: this.log,
env: getMutagenEnv({ dataDir: this.dataDir, log: this.log }),
env: getMutagenEnv(this.dataDir),
})
} catch (err) {
const unableToConnect = err.message.match(/unable to connect to daemon/)
Expand Down Expand Up @@ -778,13 +778,29 @@ export interface SyncSession {
*/
const MUTAGEN_DATA_DIRECTORY_LENGTH_LIMIT = 70

export function getMutagenEnv({ dataDir, log }: { dataDir: string; log: Log }) {
if (dataDir.length > MUTAGEN_DATA_DIRECTORY_LENGTH_LIMIT) {
emitNonRepeatableWarning(
log,
`Your Garden project path looks too long, that might cause errors while starting the syncs. Consider using a shorter path (no longer than ${MUTAGEN_DATA_DIRECTORY_LENGTH_LIMIT} characters).`
/**
* Returns mutagen data directory path based on the project dir.
* If the project path longer than `MUTAGEN_DATA_DIRECTORY_LENGTH_LIMIT`, it computes
* hash of project dir path, uses first 9 characters of hash as directory name
* and creates a directory in $HOME/.garden/mutagen.
*
* However, if the path is not longer than `MUTAGEN_DATA_DIRECTORY_LENGTH_LIMIT`, then
* it uses the ./project-root/.garden/mutagen directory.
*/
export function getMutagenDataDir(path: string, log: Log) {
if (path.length > MUTAGEN_DATA_DIRECTORY_LENGTH_LIMIT) {
const hash = hasha(path, { algorithm: "sha256" }).slice(0, 9)
const shortPath = join(GARDEN_GLOBAL_PATH, MUTAGEN_DIR_NAME, hash)
log.verbose(
`Your Garden project path looks too long, that might cause errors while starting the syncs. Garden will create a new directory to manage syncs at path: ${shortPath}.`
)
return shortPath
}
// if path is not too long, then use relative directory to the project
return join(path, MUTAGEN_DIR_NAME)
}

export function getMutagenEnv(dataDir: string) {
return {
MUTAGEN_DATA_DIRECTORY: dataDir,
}
Expand Down
20 changes: 7 additions & 13 deletions core/src/plugins/kubernetes/commands/sync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
*/

import chalk from "chalk"
import { getMutagenEnv, mutagenCliSpec, parseSyncListResult } from "../../../mutagen"
import { MUTAGEN_DIR_NAME } from "../../../constants"
import { join } from "path"
import { getMutagenDataDir, getMutagenEnv, mutagenCliSpec, parseSyncListResult } from "../../../mutagen"
import { pathExists } from "fs-extra"
import { dedent } from "../../../util/string"
import { Log } from "../../../logger/log-entry"
Expand All @@ -24,7 +22,7 @@ export const syncStatus: PluginCommand = {
title: "Get the current sync status",

handler: async ({ ctx, log }) => {
const dataDir = getMutagenDataDir(ctx.gardenDirPath)
const dataDir = getMutagenDataDir(ctx.gardenDirPath, log)
const mutagen = new PluginTool(mutagenCliSpec)

if (!(await pathExists(dataDir))) {
Expand Down Expand Up @@ -58,7 +56,7 @@ export const syncPause: PluginCommand = {
title: "Pause sync",

handler: async ({ ctx, log }) => {
const dataDir = getMutagenDataDir(ctx.gardenDirPath)
const dataDir = getMutagenDataDir(ctx.gardenDirPath, log)
const mutagen = new PluginTool(mutagenCliSpec)

if (!(await pathExists(dataDir))) {
Expand Down Expand Up @@ -87,7 +85,7 @@ export const syncPause: PluginCommand = {
await mutagen.exec({
cwd: dataDir,
log,
env: getMutagenEnv({ dataDir, log }),
env: getMutagenEnv(dataDir),
args: ["sync", "pause", sessionName],
})
}
Expand All @@ -104,7 +102,7 @@ export const syncResume: PluginCommand = {
title: "Resume sync",

handler: async ({ ctx, log }) => {
const dataDir = getMutagenDataDir(ctx.gardenDirPath)
const dataDir = getMutagenDataDir(ctx.gardenDirPath, log)
const mutagen = new PluginTool(mutagenCliSpec)

if (!(await pathExists(dataDir))) {
Expand Down Expand Up @@ -133,7 +131,7 @@ export const syncResume: PluginCommand = {
await mutagen.exec({
cwd: dataDir,
log,
env: getMutagenEnv({ dataDir, log }),
env: getMutagenEnv(dataDir),
args: ["sync", "resume", sessionName],
})
}
Expand All @@ -148,12 +146,8 @@ async function getMutagenSyncSessions({ mutagen, dataDir, log }: { mutagen: Plug
const res = await mutagen.exec({
cwd: dataDir,
log,
env: getMutagenEnv({ dataDir, log }),
env: getMutagenEnv(dataDir),
args: ["sync", "list", "--template={{ json . }}"],
})
return parseSyncListResult(res)
}

function getMutagenDataDir(gardenDirPath: string) {
return join(gardenDirPath, MUTAGEN_DIR_NAME)
}
14 changes: 6 additions & 8 deletions core/test/integ/src/plugins/kubernetes/commands/sync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,15 @@

import { expect } from "chai"
import { Garden } from "../../../../../../src/garden"
import { getContainerTestGarden } from "../container/container"
import { ConfigGraph } from "../../../../../../src/graph/config-graph"
import { Log } from "../../../../../../src/logger/log-entry"
import { getMutagenDataDir, getMutagenMonitor } from "../../../../../../src/mutagen"
import { PluginContext } from "../../../../../../src/plugin-context"
import { KubernetesProvider } from "../../../../../../src/plugins/kubernetes/config"
import { getMutagenMonitor } from "../../../../../../src/mutagen"
import { syncPause, syncResume, syncStatus } from "../../../../../../src/plugins/kubernetes/commands/sync"
import { KubernetesProvider } from "../../../../../../src/plugins/kubernetes/config"
import { DeployTask } from "../../../../../../src/tasks/deploy"
import { Log } from "../../../../../../src/logger/log-entry"
import { ConfigGraph } from "../../../../../../src/graph/config-graph"
import { join } from "path"
import { MUTAGEN_DIR_NAME } from "../../../../../../src/constants"
import { cleanProject } from "../../../../../helpers"
import { getContainerTestGarden } from "../container/container"

describe("sync plugin commands", () => {
let garden: Garden
Expand All @@ -34,7 +32,7 @@ describe("sync plugin commands", () => {
after(async () => {
if (garden) {
garden.close()
const dataDir = join(garden.gardenDirPath, MUTAGEN_DIR_NAME)
const dataDir = getMutagenDataDir(garden.gardenDirPath, garden.log)
await getMutagenMonitor({ log, dataDir }).stop()
await cleanProject(garden.gardenDirPath)
}
Expand Down

0 comments on commit 2698410

Please sign in to comment.