Add means to speed up encoding detection

clade/extensions/abstract.py

@@ -219,6 +219,9 @@ def __get_cmd_chunk(self, cmds, chunk_size=1000):
 
     def parse_cmds_in_parallel(self, cmds, unwrap, total_cmds=None):
         if os.environ.get("CLADE_DEBUG"):
+            if total_cmds:
+                self.log("Parsing {} commands".format(total_cmds))
+
             for cmd in cmds:
                 unwrap(self, cmd)
             return

clade/extensions/cl.py

@@ -13,7 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import chardet
+try:
+    import cchardet as chardet
+except ImportError:
+    import chardet
+
 import os
 import re
 import subprocess
@@ -141,7 +145,11 @@ def __collect_deps(self, cmd_id, cmd):
 
         self.__preprocess_cmd(cmd)
 
-        encoding = chardet.detect(output_bytes)["encoding"]
+        if self.conf.get("CL.deps_encoding"):
+            encoding = self.conf.get("CL.deps_encoding")
+        else:
+            encoding = chardet.detect(output_bytes)["encoding"]
+
         if not encoding:
             return ""
         return output_bytes.decode(encoding)
@@ -187,7 +195,11 @@ def __preprocess_cmd(self, cmd):
 
     def __normalize_paths(self, c_file, cwd):
         rawdata = open(c_file, 'rb').read()
-        encoding = chardet.detect(rawdata)["encoding"]
+
+        if self.conf.get("CL.deps_encoding"):
+            encoding = self.conf.get("CL.deps_encoding")
+        else:
+            encoding = chardet.detect(rawdata)["encoding"]
 
         with open(c_file, "r", encoding=encoding) as c_file_fh, open(
             c_file + ".new", "w", encoding="utf-8"

clade/extensions/presets/presets.json

@@ -9,6 +9,8 @@
         "CC.ignore_cc1": true,
         "CC.filter_deps": true,
         "CC.with_system_header_files": true,
+        "CL.deps_encoding": null,
+        "CL.pre_encoding": null,
         "Compiler.store_deps": true,
         "Compiler.preprocess_cmds": false,
         "Compiler.extra_preprocessor_opts": [],