fix: added experimental encoding priority list (#835)

JohnstonCode · Feb 21, 2020 · 706dbc1 · 706dbc1
1 parent 609d44d
commit 706dbc1
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 6 deletions.
diff --git a/package.json b/package.json
@@ -1071,6 +1071,14 @@
                     "description": "Try the experimental encoding detection",
                     "default": false
                 },
+                "svn.experimental.encoding_priority": {
+                    "type":"array",
+                    "description": "Priority of encoding",
+                    "default": [],
+                    "examples": [
+                        ["UTF-8", "GB18030", "windows-1251"]
+                    ]
+                },
                 "svn.gravatar.icon_url": {
                     "type": "string",
                     "description": "Url for the gravitar icon using the <AUTHOR>, <AUTHOR_MD5> and <SIZE> placeholders",

diff --git a/src/encoding.ts b/src/encoding.ts
@@ -46,6 +46,10 @@ const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = {
   big5: "cp950"
 };
 
+function normaliseEncodingName(name: string): string {
+  return name.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
+}
+
 export function detectEncoding(buffer: Buffer): string | null {
   const result = detectEncodingByBOM(buffer);
 
@@ -58,9 +62,22 @@ export function detectEncoding(buffer: Buffer): string | null {
     false
   );
   if (experimental) {
-    const detected = chardet.detect(buffer);
-    if (detected) {
-      return detected.replace(/[^a-zA-Z0-9]/g, "").toLocaleLowerCase();
+    const detected = chardet.detectAll(buffer);
+    const encodingPriorities = configuration.get<string[]>(
+      "experimental.encoding_priority",
+      []
+    );
+
+    if (!detected) {
+      return null;
+    }
+
+    for (const pri of encodingPriorities) {
+      for (const det of detected) {
+        if (normaliseEncodingName(pri) === normaliseEncodingName(det.name)) {
+          return normaliseEncodingName(det.name);
+        }
+      }
     }
 
     return null;
@@ -80,9 +97,7 @@ export function detectEncoding(buffer: Buffer): string | null {
     return null;
   }
 
-  const normalizedEncodingName = encoding
-    .replace(/[^a-zA-Z0-9]/g, "")
-    .toLowerCase();
+  const normalizedEncodingName = normaliseEncodingName(encoding);
   const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName];
 
   return mapped || normalizedEncodingName;