Skip to content

Commit

Permalink
Merge pull request #1850 from botpress/rl_ghost_ignore
Browse files Browse the repository at this point in the history 
feat(core): add .ghostignore to exclude files from tracking
  • Loading branch information
@slvnperron
slvnperron committed Jun 5, 2019
2 parents 343cb35 + f685cf1 commit 387668e
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 64 deletions.
42 changes: 24 additions & 18 deletions src/bp/core/services/ghost/disk-driver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import fse from 'fs-extra'
import glob from 'glob'
import { injectable } from 'inversify'
import _ from 'lodash'
import os from 'os'
import path from 'path'
import { VError } from 'verror'

Expand Down Expand Up @@ -64,8 +65,7 @@ export default class DiskStorageDriver implements StorageDriver {

async directoryListing(
folder: string,
exclude?: string | string[],
includeDotFiles: boolean = false
options: { excludes?: string | string[]; includeDotFiles?: boolean } = { excludes: [], includeDotFiles: false }
): Promise<string[]> {
try {
await fse.access(this.resolvePath(folder), fse.constants.R_OK)
Expand All @@ -78,13 +78,23 @@ export default class DiskStorageDriver implements StorageDriver {
throw new VError(e, `[Disk Storage] No read access to directory "${folder}"`)
}

const options = { cwd: this.resolvePath(folder), dot: includeDotFiles }
if (exclude) {
options['ignore'] = exclude
const ghostIgnorePatterns = await this._getGhostIgnorePatterns(this.resolvePath('data/.ghostignore'))
const globOptions = {
cwd: this.resolvePath(folder),
dot: options.includeDotFiles
}

// options.excludes can either be a string or an array of strings or undefined
if (Array.isArray(options.excludes)) {
globOptions['ignore'] = [...options.excludes, ...ghostIgnorePatterns]
} else if (options.excludes) {
globOptions['ignore'] = [options.excludes, ...ghostIgnorePatterns]
} else {
globOptions['ignore'] = ghostIgnorePatterns
}

try {
const files = await Promise.fromCallback<string[]>(cb => glob('**/*.*', options, cb))
const files = await Promise.fromCallback<string[]>(cb => glob('**/*.*', globOptions, cb))
return files.map(filePath => forceForwardSlashes(filePath))
} catch (e) {
return []
Expand All @@ -104,18 +114,6 @@ export default class DiskStorageDriver implements StorageDriver {
}
}

async discoverTrackableFolders(baseDir: string): Promise<string[]> {
try {
const allFiles = await this.directoryListing(baseDir, undefined, true)
const allDirectories = this._getBaseDirectories(allFiles)
const noghostFiles = allFiles.filter(x => path.basename(x).toLowerCase() === '.noghost')
const noghostDirectories = this._getBaseDirectories(noghostFiles)
return _.without(allDirectories, ...noghostDirectories)
} catch (err) {
return []
}
}

async absoluteDirectoryListing(destination: string) {
try {
const files = await Promise.fromCallback<string[]>(cb => glob('**/*.*', { cwd: destination }, cb))
Expand All @@ -133,4 +131,12 @@ export default class DiskStorageDriver implements StorageDriver {
.uniq()
.value()
}

private async _getGhostIgnorePatterns(ghostIgnorePath: string): Promise<string[]> {
if (await fse.pathExists(ghostIgnorePath)) {
const ghostIgnoreFile = await fse.readFile(ghostIgnorePath)
return ghostIgnoreFile.toString().split(/\r?\n/gi)
}
return []
}
}
5 changes: 4 additions & 1 deletion src/bp/core/services/ghost/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ export interface StorageDriver {
readFile(filePath: string): Promise<Buffer>
deleteFile(filePath: string, recordRevision: boolean): Promise<void>
deleteDir(dirPath: string): Promise<void>
directoryListing(folder: string, exclude?: string | string[], includeDotFiles?: boolean): Promise<string[]>
directoryListing(
folder: string,
options: { excludes?: string | string[]; includeDotFiles?: boolean }
): Promise<string[]>
listRevisions(pathPrefix: string): Promise<FileRevision[]>
deleteRevision(filePath: string, revision: string): Promise<void>
moveFile(fromPath: string, toPath: string): Promise<void>
Expand Down
3 changes: 0 additions & 3 deletions src/bp/core/services/ghost/service.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,6 @@ describe('Ghost Service', () => {
it('if disk is not up to date, mark as dirty and dont sync disk files', async () => {
dbDriver.listRevisions.mockReturnValue(['1', '2', '3'].map(buildRev))
diskDriver.listRevisions.mockReturnValue(['1', '2'].map(buildRev)) // missing revision "3"
diskDriver.discoverTrackableFolders.mockReturnValue(['test', '.'])

await ghost.global().sync()

Expand All @@ -181,12 +180,10 @@ describe('Ghost Service', () => {

// Make sure we haven't synced anything
expect(diskDriver.readFile).not.toHaveBeenCalled()
expect(diskDriver.directoryListing).not.toHaveBeenCalled()
expect(dbDriver.upsertFile).not.toHaveBeenCalled()
})
it('if disk is up to date, sync disk files', async () => {
dbDriver.listRevisions.mockReturnValue(['1', '2', '3'].map(buildRev))
diskDriver.discoverTrackableFolders.mockReturnValue(['.'])
diskDriver.listRevisions.mockReturnValue(['1', '2', '3'].map(buildRev)) // All synced!
diskDriver.readFile.mockReturnValueOnce('FILE A CONTENT')
diskDriver.readFile.mockReturnValueOnce('FILE D CONTENT')
Expand Down
75 changes: 33 additions & 42 deletions src/bp/core/services/ghost/service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,12 @@ export class ScopedGhostService {
this.primaryDriver = useDbDriver ? dbDriver : diskDriver
}

private normalizeFolderName(rootFolder: string) {
private _normalizeFolderName(rootFolder: string) {
return forceForwardSlashes(path.join(this.baseDir, rootFolder))
}

private normalizeFileName(rootFolder: string, file: string) {
return forceForwardSlashes(path.join(this.normalizeFolderName(rootFolder), file))
private _normalizeFileName(rootFolder: string, file: string) {
return forceForwardSlashes(path.join(this._normalizeFolderName(rootFolder), file))
}

objectCacheKey = str => `string::${str}`
Expand All @@ -164,13 +164,13 @@ export class ScopedGhostService {
}

async invalidateFile(rootFolder: string, fileName: string): Promise<void> {
const filePath = this.normalizeFileName(rootFolder, fileName)
const filePath = this._normalizeFileName(rootFolder, fileName)
await this._invalidateFile(filePath)
}

async ensureDirs(rootFolder: string, directories: string[]): Promise<void> {
if (!this.useDbDriver) {
await Promise.mapSeries(directories, d => this.diskDriver.createDir(this.normalizeFileName(rootFolder, d)))
await Promise.mapSeries(directories, d => this.diskDriver.createDir(this._normalizeFileName(rootFolder, d)))
}
}

Expand All @@ -179,7 +179,7 @@ export class ScopedGhostService {
throw new Error(`Ghost can't read or write under this scope`)
}

const fileName = this.normalizeFileName(rootFolder, file)
const fileName = this._normalizeFileName(rootFolder, file)

if (content.length > MAX_GHOST_FILE_SIZE) {
throw new Error(`The size of the file ${fileName} is over the 20mb limit`)
Expand All @@ -194,16 +194,18 @@ export class ScopedGhostService {
await Promise.all(content.map(c => this.upsertFile(rootFolder, c.name, c.content)))
}

/** All tracked directories will be synced
* Directories are tracked by default, unless a `.noghost` file is present in the directory
/**
* All tracked files will be synced.
* All files are tracked by default, unless `.ghostignore` is used to exclude them.
*/
async sync() {
if (!this.useDbDriver) {
// We don't have to sync anything as we're just using the files from disk
return
}

const paths = await this.diskDriver.discoverTrackableFolders(this.normalizeFolderName('./'))
// Get files from disk that should be ghosted
const trackedFiles = await this.diskDriver.directoryListing(this.baseDir, { includeDotFiles: true })

const diskRevs = await this.diskDriver.listRevisions(this.baseDir)
const dbRevs = await this.dbDriver.listRevisions(this.baseDir)
Expand All @@ -219,36 +221,26 @@ export class ScopedGhostService {
return
}

for (const path of paths) {
const normalizedPath = this.normalizeFolderName(path)
let currentFiles = await this.dbDriver.directoryListing(normalizedPath)
let newFiles = await this.diskDriver.directoryListing(normalizedPath, undefined, true)

if (path === './') {
currentFiles = currentFiles.filter(x => !x.includes('/'))
newFiles = newFiles.filter(x => !x.includes('/'))
}

// We delete files that have been deleted from disk
for (const file of _.difference(currentFiles, newFiles)) {
const filePath = this.normalizeFileName(path, file)
await this.dbDriver.deleteFile(filePath, false)
}
// Delete the ghosted files that has been deleted from disk
const ghostedFiles = await this.dbDriver.directoryListing(this._normalizeFolderName('./'))
const filesToDelete = _.difference(ghostedFiles, trackedFiles)
await Promise.map(filesToDelete, filePath =>
this.dbDriver.deleteFile(this._normalizeFileName('./', filePath), false)
)

// We now update files in DB by those on the disk
for (const file of newFiles) {
const filePath = this.normalizeFileName(path, file)
const content = await this.diskDriver.readFile(filePath)
await this.dbDriver.upsertFile(filePath, content, false)
}
}
// Overwrite all of the ghosted files with the tracked files
await Promise.each(trackedFiles, async file => {
const filePath = this._normalizeFileName('./', file)
const content = await this.diskDriver.readFile(filePath)
await this.dbDriver.upsertFile(filePath, content, false)
})
}

public async exportToDirectory(directory: string, exludes?: string | string[]): Promise<string[]> {
const allFiles = await this.directoryListing('./', '*.*', exludes, true)

for (const file of allFiles.filter(x => x !== 'revisions.json')) {
const content = await this.primaryDriver.readFile(this.normalizeFileName('./', file))
const content = await this.primaryDriver.readFile(this._normalizeFileName('./', file))
const outPath = path.join(directory, file)
mkdirp.sync(path.dirname(outPath))
await fse.writeFile(outPath, content)
Expand Down Expand Up @@ -307,7 +299,7 @@ export class ScopedGhostService {
throw new Error(`Ghost can't read or write under this scope`)
}

const fileName = this.normalizeFileName(rootFolder, file)
const fileName = this._normalizeFileName(rootFolder, file)
const cacheKey = this.bufferCacheKey(fileName)

if (!(await this.cache.has(cacheKey))) {
Expand All @@ -324,7 +316,7 @@ export class ScopedGhostService {
}

async readFileAsObject<T>(rootFolder: string, file: string): Promise<T> {
const fileName = this.normalizeFileName(rootFolder, file)
const fileName = this._normalizeFileName(rootFolder, file)
const cacheKey = this.objectCacheKey(fileName)

if (!(await this.cache.has(cacheKey))) {
Expand All @@ -338,7 +330,7 @@ export class ScopedGhostService {
}

async fileExists(rootFolder: string, file: string): Promise<boolean> {
const fileName = this.normalizeFileName(rootFolder, file)
const fileName = this._normalizeFileName(rootFolder, file)
try {
await this.primaryDriver.readFile(fileName)
return true
Expand All @@ -352,15 +344,15 @@ export class ScopedGhostService {
throw new Error(`Ghost can't read or write under this scope`)
}

const fileName = this.normalizeFileName(rootFolder, file)
const fileName = this._normalizeFileName(rootFolder, file)
await this.primaryDriver.deleteFile(fileName, true)
this.events.emit('changed', fileName)
await this._invalidateFile(fileName)
}

async renameFile(rootFolder: string, fromName: string, toName: string): Promise<void> {
const fromPath = this.normalizeFileName(rootFolder, fromName)
const toPath = this.normalizeFileName(rootFolder, toName)
const fromPath = this._normalizeFileName(rootFolder, fromName)
const toPath = this._normalizeFileName(rootFolder, toName)

await this.primaryDriver.moveFile(fromPath, toPath)
}
Expand All @@ -370,7 +362,7 @@ export class ScopedGhostService {
throw new Error(`Ghost can't read or write under this scope`)
}

const folderName = this.normalizeFolderName(folder)
const folderName = this._normalizeFolderName(folder)
await this.primaryDriver.deleteDir(folderName)
}

Expand All @@ -381,11 +373,10 @@ export class ScopedGhostService {
includeDotFiles?: boolean
): Promise<string[]> {
try {
const files = await this.primaryDriver.directoryListing(
this.normalizeFolderName(rootFolder),
const files = await this.primaryDriver.directoryListing(this._normalizeFolderName(rootFolder), {
excludes,
includeDotFiles
)
})

return (files || []).filter(
minimatch.filter(fileEndingPattern, { matchBase: true, nocase: true, noglobstar: false, dot: includeDotFiles })
Expand Down

0 comments on commit 387668e

Please sign in to comment.