Add new data

warioddly · May 14, 2024 · 5881298 · 5881298
1 parent 89b2225
commit 5881298
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 11 deletions.
diff --git a/parser/bin/parser.dart b/parser/bin/parser.dart
@@ -49,19 +49,39 @@ void main(List<String> arguments) async {
 
 
 Future<void> _scrapy() async {
+
   print('[+] Scrapy started.');
 
   final scrapy = Scrapy();
 
-  await scrapy.parse(['', 'https://www.sanjyra.net/man/1']);
+  void bundler() {
+    Bundler()
+      ..generate(jsonEncode(scrapy.people.map((e) => e.toJson()).toList()), 'all_data.json')
+      ..generate(jsonEncode(scrapy.people.map((e) => e.toCytoscapeJson()).toList()), 'all_data_site.json');
+  }
+
+
+  ProcessSignal.sigint.watch().listen((event) {
+    bundler();
+    print('\n[+] Exiting...');
+    exit(0);
+  });
+
+  scrapy.visited.addAll({
+  'https://www.sanjyra.net/man/1',
+  'https://www.sanjyra.net/man/2',
+  'https://www.sanjyra.net/man/32252'
+  });
+
+  await scrapy.parse(['', 'https://www.sanjyra.net/man/44275']);
 
   print('\n[+] Done! Parsed ${scrapy.people.length} people.');
 
-  Bundler()
-    ..generate(jsonEncode(scrapy.people.map((e) => e.toJson())), 'all_data.json')
-    ..generate(jsonEncode(scrapy.people.map((e) => e.toCytoscapeJson())), 'all_data_site.json');
+  bundler();
 
   print('[+] Scrapy finished.');
+  exit(0);
+
 }
 
 

diff --git a/parser/lib/scrapy.dart b/parser/lib/scrapy.dart
@@ -8,7 +8,7 @@ import 'package:parser/people_model.dart';
 
 class Scrapy {
 
-  final _visited = <String>{};
+  final visited = <String>{};
   final people = <People>[];
   final bundler = Bundler();
 
@@ -69,7 +69,7 @@ class Scrapy {
 
           var url = child.children.first.attributes['href'];
 
-          if (url != null && !_visited.contains(url)) {
+          if (url != null && !visited.contains(url)) {
             urls.add(url);
           }
 
@@ -83,11 +83,11 @@ class Scrapy {
 
 
   bool _crawl(String url) {
-    if (_visited.contains(url)) {
+    if (visited.contains(url)) {
       print('[-] Already crawled: $url');
       return false;
     }
-    _visited.add(url);
+    visited.add(url);
     print('[+] Crawling $url');
     return true;
   }

diff --git a/parser/output/data.json b/parser/output/data.json
diff --git a/parser/output/куу уул/all_data.json b/parser/output/куу уул/all_data.json
diff --git a/parser/output/куу уул/all_data_site.json b/parser/output/куу уул/all_data_site.json
diff --git a/parser/output/куу уул/dynamic_all_data.json b/parser/output/куу уул/dynamic_all_data.json
diff --git a/src/assets/data/data.json b/src/assets/data/data.json
diff --git a/src/components/tree/TreeInteractiveViewer.tsx b/src/components/tree/TreeInteractiveViewer.tsx
@@ -30,7 +30,7 @@ export function TreeInteractiveViewer() {
 
     fetch('/api/get-nodes', {
       method: 'POST',
-      body: JSON.stringify({ limit: 2500 }),
+      body: JSON.stringify({ limit: 4500 }),
     }).then((res) => res.json()).then((data: TreeNodeDefinition[]) => {
       setGraph(data);
       setLoading(false);