splitter: Add null split mode

CyberShadow · Nov 21, 2020 · 6634109 · 6634109
1 parent cb0855d
commit 6634109
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 1 deletion.
diff --git a/splitter.d b/splitter.d
@@ -143,12 +143,13 @@ enum Splitter
 {
 	files,     /// Load entire files only
 	lines,     /// Split by line ends
+	null_,     /// Split by the \0 (NUL) character
 	words,     /// Split by whitespace
 	D,         /// Parse D source code
 	diff,      /// Unified diffs
 	indent,    /// Indentation (Python, YAML...)
 }
-immutable string[] splitterNames = [EnumMembers!Splitter].map!(e => e.text().toLower()).array();
+immutable string[] splitterNames = [EnumMembers!Splitter].map!(e => e.text().toLower().chomp("_")).array();
 
 struct ParseRule
 {
@@ -266,6 +267,9 @@ Entity loadFile(string name, string path, ParseOptions options)
 				case Splitter.words:
 					result.children = parseToWords(result.contents);
 					return result;
+				case Splitter.null_:
+					result.children = parseToNull(result.contents);
+					return result;
 				case Splitter.D:
 				{
 					if (result.contents.startsWith("Ddoc"))
@@ -1265,6 +1269,7 @@ Entity[] parseSplit(alias fun)(string text)
 
 alias parseToWords = parseSplit!isNotAlphaNum;
 alias parseToLines = parseSplit!isNewline;
+alias parseToNull  = parseSplit!(c => c == '\0');
 
 /// Split s on end~start, preserving end and start on each chunk
 private string[] split2(string end, string start)(string s)

diff --git a/tests/null/opts.txt b/tests/null/opts.txt
@@ -0,0 +1,2 @@
+--split
+*.txt:null
diff --git a/tests/null/progress.txt b/tests/null/progress.txt
@@ -0,0 +1,50 @@
+############### ITERATION 0 ################
+============= Depth 0 =============
+[ 0.0%] Remove [] => No
+============= Depth 1 =============
+[ 0.0%] Remove [0] => No (cached)
+[ 0.0%] Concat [0] => N/A
+============= Depth 2 =============
+[ 0.0%] Remove [01] => No
+[35.5%] Remove [00] => Yes
+============= Depth 3 =============
+[ 0.0%] Remove [001] => Yes
+[ 9.5%] Remove [000] => No
+============= Depth 4 =============
+[ 0.0%] Remove [0001] => Yes
+[46.9%] Remove [0000] => No (cached)
+============= Depth 5 =============
+[ 0.0%] Remove [00001] => No
+[42.8%] Remove [00000] => Yes
+============= Depth 6 =============
+[ 0.0%] Remove [000001] => No
+[35.0%] Remove [000000] => Yes
+============= Depth 7 =============
+[ 0.0%] Remove [0000001] => Yes
+[23.0%] Remove [0000000] => No (cached)
+============= Depth 8 =============
+[ 0.0%] Remove [00000001] => Yes
+[10.0%] Remove [00000000] => No (cached)
+[10.0%] Unwrap [00000000] => No (cached)
+############### ITERATION 1 ################
+============= Depth 0 =============
+[ 0.0%] Remove [] => No (cached)
+============= Depth 1 =============
+[ 0.0%] Remove [0] => No (cached)
+[ 0.0%] Concat [0] => N/A
+============= Depth 2 =============
+[ 0.0%] Remove [00] => No (cached)
+============= Depth 3 =============
+[ 0.0%] Remove [000] => No (cached)
+============= Depth 4 =============
+[ 0.0%] Remove [0000] => No (cached)
+============= Depth 5 =============
+[ 0.0%] Remove [00000] => No (cached)
+============= Depth 6 =============
+[ 0.0%] Remove [000000] => No (cached)
+============= Depth 7 =============
+[ 0.0%] Remove [0000000] => No (cached)
+============= Depth 8 =============
+[ 0.0%] Remove [00000000] => No (cached)
+[ 0.0%] Unwrap [00000000] => No (cached)
+Done in 13 tests
diff --git a/tests/null/src.dump b/tests/null/src.dump
diff --git a/tests/null/src.result/text.txt b/tests/null/src.result/text.txt
diff --git a/tests/null/src/text.txt b/tests/null/src/text.txt
diff --git a/tests/null/test.cmd b/tests/null/test.cmd
@@ -0,0 +1 @@
+@grep -qFz ite text.txt
diff --git a/tests/null/test.sh b/tests/null/test.sh
@@ -0,0 +1 @@
+grep -qFz ite text.txt