Polish library indexing

open-cogsci · May 9, 2024 · d8b4a49 · d8b4a49
1 parent a3a365f
commit d8b4a49
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 8 deletions.
diff --git a/index_library.py b/index_library.py
@@ -2,10 +2,4 @@
 import logging; logging.basicConfig(level=logging.INFO, force=True)
 
 if __name__ == '__main__':
-    # library.load_library(force_reindex=True, exclude_filter=['forum'])
-    library.load_library(force_reindex=True,
-                         cache_folder='.public-with-forum.db.cache',
-                         exclude_filter=['howtos'])
-    library.load_library(force_reindex=True,
-                         cache_folder='.public-without-forum.db.cache',
-                         exclude_filter=['howtos', 'forum'])
+    library.load_library(force_reindex=True, exclude_filter=['forum'])
diff --git a/sigmund/library.py b/sigmund/library.py
@@ -35,7 +35,7 @@ def load_library(force_reindex=False, cache_folder=config.db_cache,
             if any(f in str(src) for f in exclude_filter):
                 logger.info(f'skipping json: {src}')
                 continue            
-            loader = JSONLoader(src, jq_schema='', content_key='content',
+            loader = JSONLoader(src, jq_schema='.', content_key='content',
                                 json_lines=True,
                                 metadata_func=_extract_metadata)
             data += loader.load()