fix: handle non utf8 files with autocrlf correctly

isomorphic-git · May 5, 2024 · 402668d · 402668d
1 parent 2d0c021
commit 402668d
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 21 deletions.
diff --git a/src/api/add.js b/src/api/add.js
@@ -1,7 +1,7 @@
 // @ts-check
 import '../typedefs.js'
 
-import { MultipleGitError } from '../errors/MultipleGitError'
+import { MultipleGitError } from '../errors/MultipleGitError.js'
 import { NotFoundError } from '../errors/NotFoundError.js'
 import { GitConfigManager } from '../managers/GitConfigManager.js'
 import { GitIgnoreManager } from '../managers/GitIgnoreManager.js'
@@ -119,20 +119,12 @@ async function addToIndex({
       }
     } else {
       const config = await GitConfigManager.get({ fs, gitdir })
-      const autocrlf = (await config.get('core.autocrlf')) || false
+      const autocrlf = await config.get('core.autocrlf')
       const object = stats.isSymbolicLink()
         ? await fs.readlink(join(dir, currentFilepath)).then(posixifyPathBuffer)
-        : await fs.read(join(dir, currentFilepath), {
-            encoding: 'utf8',
-            autocrlf,
-          })
+        : await fs.read(join(dir, currentFilepath), { autocrlf })
       if (object === null) throw new NotFoundError(currentFilepath)
-      const oid = await _writeObject({
-        fs,
-        gitdir,
-        type: 'blob',
-        object: new TextEncoder().encode(object),
-      })
+      const oid = await _writeObject({ fs, gitdir, type: 'blob', object })
       index.insert({ filepath: currentFilepath, stats, oid })
     }
   })

diff --git a/src/models/FileSystem.js b/src/models/FileSystem.js
@@ -101,8 +101,14 @@ export class FileSystem {
   async read(filepath, options = {}) {
     try {
       let buffer = await this._readFile(filepath, options)
-      if (typeof buffer === 'string' && options.autocrlf) {
-        buffer = buffer.replace(/\r\n/g, '\n')
+      if (options.autocrlf === 'true') {
+        try {
+          buffer = new TextDecoder('utf8', { fatal: true }).decode(buffer)
+          buffer = buffer.replace(/\r\n/g, '\n')
+          buffer = new TextEncoder().encode(buffer)
+        } catch (error) {
+          // non utf8 file
+        }
       }
       // Convert plain ArrayBuffers to Buffers
       if (typeof buffer !== 'string') {

diff --git a/src/models/GitWalkerFs.js b/src/models/GitWalkerFs.js
@@ -1,7 +1,7 @@
 import { GitConfigManager } from '../managers/GitConfigManager.js'
 import { GitIndexManager } from '../managers/GitIndexManager.js'
 import { compareStats } from '../utils/compareStats.js'
-import { join } from '../utils/join'
+import { join } from '../utils/join.js'
 import { normalizeStats } from '../utils/normalizeStats.js'
 import { shasum } from '../utils/shasum.js'
 
@@ -100,17 +100,14 @@ export class GitWalkerFs {
         entry._content = undefined
       } else {
         const config = await GitConfigManager.get({ fs, gitdir })
-        const autocrlf = (await config.get('core.autocrlf')) || false
-        const content = await fs.read(`${dir}/${entry._fullpath}`, {
-          encoding: 'utf8',
-          autocrlf,
-        })
+        const autocrlf = await config.get('core.autocrlf')
+        const content = await fs.read(`${dir}/${entry._fullpath}`, { autocrlf })
         // workaround for a BrowserFS edge case
         entry._actualSize = content.length
         if (entry._stat && entry._stat.size === -1) {
           entry._stat.size = entry._actualSize
         }
-        entry._content = new TextEncoder().encode(content)
+        entry._content = new Uint8Array(content)
       }
     }
     return entry._content