Allow non-ASCII description (#62)

Add config `ascii_description`.

Resolves #61

README.md

@@ -496,6 +496,11 @@ The configuration example above changes the default configuration to show all pr
         <td>The maximum length of the project description.</td>
         <td><code>55</code></td>
     </tr>
+    <tr>
+        <td><code>ascii_description</code></td>
+        <td>If <code>True</code>, all non-ASCII characters in the project description will be removed. Useful for filtering out distractive emoji, but hurtful in non-English cases. (Note: GitHub emoji commands (e.g. <code>:smile:</code>) are always removed.)</td>
+        <td><code>True</code></td>
+    </tr>
     <tr>
         <td><code>projects_history_folder</code></td>
         <td>The folder used for storing history files (<code>csv</code> files with project metadata). If <code>null</code>, no history files will be created.</td>

src/best_of/default_config.py

@@ -64,6 +64,9 @@ def prepare_configuration(cfg: dict) -> Dict:
     if "min_description_length" not in config:
         config.min_description_length = MIN_PROJECT_DESC_LENGTH
 
+    if "ascii_description" not in config:
+        config.ascii_description = True
+
     if "hide_project_license" not in config:
         config.hide_project_license = False
 

src/best_of/generators/markdown_list.py

@@ -292,7 +292,11 @@ def generate_project_md(
             )
         )
     )
-    description = utils.process_description(project.description, desc_length)
+    description = utils.process_description(
+        project.description,
+        desc_length,
+        ascii_only=configuration.ascii_description,
+    )
 
     # target="_blank"
     if project.resource:

src/best_of/projects_collection.py

@@ -680,7 +680,7 @@ def collect_projects_info(
         if project_info.description:
             # Process description
             project_info.description = utils.process_description(
-                project_info.description, 120
+                project_info.description, 120, ascii_only=config.ascii_description
             )
 
         # Check and update the project category

src/best_of/utils.py

@@ -33,14 +33,16 @@ def remove_special_chars(text: str) -> str:
     return text.encode("ascii", "ignore").decode("ascii")
 
 
-def process_description(text: str, max_lenght: int) -> str:
+def process_description(text: str, max_length: int, *, ascii_only: bool) -> str:
     if not text:
         return ""
 
     # Remove github emoji commands
     text = re.sub(":[a-zA-Z_]*:", "", text).strip()
 
-    text = remove_special_chars(text).strip()
+    if ascii_only:
+        text = remove_special_chars(text)
+    text = text.strip()
     text = text.replace('"', "")
     text = text.replace("'", "")
     text = text.replace("<", "")
@@ -53,7 +55,7 @@ def process_description(text: str, max_lenght: int) -> str:
         # make sure that the text always ends with a dot
         text += "."
 
-    return clean_whitespaces(textwrap.shorten(text, width=max_lenght, placeholder=".."))
+    return clean_whitespaces(textwrap.shorten(text, width=max_length, placeholder=".."))
 
 
 url_validator = re.compile(