add - doc - Added common words

--- We've added common words --- Type: add Breaking: False Doc Required: True Part: 1/1
Aptivi · Apr 19, 2024 · 081782e · 081782e
1 parent 0d00c41
commit 081782e
Show file tree

Hide file tree

Showing 8 changed files with 64 additions and 1 deletion.
diff --git a/Textify.Data.Analysis/Tools/DataInitializer.cs b/Textify.Data.Analysis/Tools/DataInitializer.cs
@@ -48,6 +48,8 @@ internal static void Initialize(DataType types)
             bool needsWordsDirty = types.HasFlag(DataType.WordsDirty);
             bool needsWordsDirtyFull = types.HasFlag(DataType.WordsDirtyFull);
             bool needsWordsJustDirty = types.HasFlag(DataType.WordsJustDirty);
+            bool needsCommonWords = types.HasFlag(DataType.CommonWords);
+            bool needsCommonWordsDirty = types.HasFlag(DataType.CommonWordsDirty);
 
             // Go through all the types
             if (needsNames)
@@ -125,6 +127,16 @@ internal static void Initialize(DataType types)
                 if (!DataTools.dataStreams.ContainsKey(nameof(WordsData.bad_words)))
                     DataTools.dataStreams.Add(nameof(WordsData.bad_words), WordsData.bad_words);
             }
+            if (needsCommonWords)
+            {
+                if (!DataTools.dataStreams.ContainsKey(nameof(WordsData.words_common_clean)))
+                    DataTools.dataStreams.Add(nameof(WordsData.words_common_clean), WordsData.words_common_clean);
+            }
+            if (needsCommonWordsDirty)
+            {
+                if (!DataTools.dataStreams.ContainsKey(nameof(WordsData.words_common)))
+                    DataTools.dataStreams.Add(nameof(WordsData.words_common), WordsData.words_common);
+            }
         }
     }
 }
diff --git a/Textify.Data.Analysis/Tools/DataType.cs b/Textify.Data.Analysis/Tools/DataType.cs
@@ -84,5 +84,13 @@ internal enum DataType
         /// Initializes the zip files containing just the offensive words (18+) for bad word filtering
         /// </summary>
         WordsJustDirty = 16384,
+        /// <summary>
+        /// Initializes the zip files containing common words list
+        /// </summary>
+        CommonWords = 32768,
+        /// <summary>
+        /// Initializes the zip files containing common words list, including the offensive words (18+)
+        /// </summary>
+        CommonWordsDirty = 65536,
     }
 }
diff --git a/Textify.Data.Analysis/Words/WordDataType.cs b/Textify.Data.Analysis/Words/WordDataType.cs
@@ -44,5 +44,13 @@ public enum WordDataType
         /// Offensive words list (18+) for bad word filtering
         /// </summary>
         BadWords,
+        /// <summary>
+        /// Common word list
+        /// </summary>
+        CommonWords,
+        /// <summary>
+        /// Common word list, including offensive words (18+)
+        /// </summary>
+        CommonWordsDirty,
     }
 }
diff --git a/Textify.Data.Analysis/Words/WordManager.cs b/Textify.Data.Analysis/Words/WordManager.cs
@@ -147,6 +147,8 @@ private static async Task<string[]> GetWordListAsync(WordDataType type)
                 type == WordDataType.WordsDirty ? (DataType.WordsDirty, "words_alpha", "words_alpha.txt") :
                 type == WordDataType.WordsDirtyFull ? (DataType.WordsDirtyFull, "words", "words.txt") :
                 type == WordDataType.BadWords ? (DataType.WordsJustDirty, "bad_words", "bad-words.txt") :
+                type == WordDataType.CommonWords ? (DataType.CommonWords, "words_common_clean", "words-common-clean.txt") :
+                type == WordDataType.CommonWordsDirty ? (DataType.CommonWordsDirty, "words_common", "words-common.txt") :
                 throw new TextifyException("Invalid word data type");
             DataInitializer.Initialize(dataType);
             var contentStream = new MemoryStream(DataTools.GetDataFrom(resourceName));

diff --git a/Textify.Data/DataRes/WordsData.Designer.cs b/Textify.Data/DataRes/WordsData.Designer.cs
diff --git a/Textify.Data/DataRes/WordsData.resx b/Textify.Data/DataRes/WordsData.resx
@@ -133,4 +133,10 @@
   <data name="bad_words" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>../../assets/WordsList/bad-words.zip;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </data>
+  <data name="words_common" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>../../assets/WordsList/words-common.zip;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </data>
+  <data name="words_common_clean" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>../../assets/WordsList/words-common-clean.zip;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </data>
 </root>
diff --git a/Textify.Demos.Offline/Fixtures/Cases/WordGet.cs b/Textify.Demos.Offline/Fixtures/Cases/WordGet.cs
@@ -33,6 +33,13 @@ public void RunFixture()
                 string word = WordManager.GetRandomWord();
                 Console.WriteLine($"Word {i + 1}: {word}");
             }
+
+            // Get 10 random common words
+            for (int i = 0; i < 10; i++)
+            {
+                string word = WordManager.GetRandomWord(WordDataType.CommonWords);
+                Console.WriteLine($"Common word {i + 1}: {word}");
+            }
         }
     }
 }
diff --git a/assets/WordsList b/assets/WordsList