fix: add async to osintbuddy plugins

README.md

@@ -116,7 +116,7 @@ See the [open issues](https://github.com/jerlendds/osintbuddy/issues) for a list
 
 - [x] Added a plugin system
 -  For this update you'll have to rebuild your docker containers. After that long wait you'll be rewarded with a newly released package: [osintbuddy](https://pypi.org/project/osintbuddy/)  
-    The osintbuddy package allows for transforming data through a `@transform` decorator, which is applied to methods in the OBPlugin class.
+    The osintbuddy package allows for transforming data through a `@transform` decorator, which is applied to methods in the Plugin class.
 - The transform decorator must have a label and, optionally, a [tabler-icons](https://tabler-icons.io/) icon name. You can manage node transformations and node creation easily through the API.
 -  Basically these updates make it easier to create nodes and context menu options for a nodes transform options which are mapped to plugin methods.
 - More documentation is coming soon when I have the time.

backend/.gitignore

@@ -1,2 +1,3 @@
 .env
 venv/
+backend/osintbuddy-plugins/

backend/backend/app/app/api/api_v1/endpoints/ghdb.py

@@ -1,5 +1,7 @@
 from typing import Any, List
 from datetime import datetime
+
+
 import requests
 from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy.orm import Session

backend/backend/app/app/api/api_v1/endpoints/nodes.py

@@ -1,62 +1,26 @@
 from typing import List
 from fastapi import (
     APIRouter,
-    Depends,
-    HTTPException,
     WebSocket,
     WebSocketException,
     WebSocketDisconnect
 )
 from websockets.exceptions import ConnectionClosedError
-from sqlalchemy.orm import Session
 from app.api import deps
 from app.core.logger import get_logger
-from app.llm.prompts import get_prompt_transform_options
+# from app.llm.prompts import get_prompt_transform_options
 from functools import lru_cache
-from osintbuddy.plugins import OBRegistry, discover_plugins
+from osintbuddy import Registry, discover_plugins
 from osintbuddy.errors import OBPluginError
 
 router = APIRouter(prefix='/nodes')
 
 logger = get_logger(' api_v1.endpoints.nodes ')
 
-CORE_LABELS = [
-    'GoogleSearch',
-    'GoogleCacheSearch',
-    'GoogleResult',
-    'CseSearch',
-    'CseResult',
-    'IpAddress',
-    'Email',
-    'SmtpTest',
-    'Domain',
-    'Subdomain',
-    'URL',
-    'urlscanIO',
-    'Traceroute',
-    'Geolocation',
-    'DNSRecord',
-    'Username',
-    'Profile',
-    'Person',
-    'Pastebin',
-    'Phone',
-    'Business',
-    'ImageSearch',
-    'Image',
-    'VideoSearch',
-    'Video',
-    'News',
-    'RSS',
-    'MalwareCheck',
-    'Malware',
-    'NLP',
-]
-
-
-@lru_cache(maxsize=128)
-def fetch_node_transforms(plugin_label):
-    plugin = OBRegistry.get_plugin(plugin_label=plugin_label)
+
+# @lru_cache(maxsize=128)
+async def fetch_node_transforms(plugin_label):
+    plugin = await Registry.get_plugin(plugin_label=plugin_label)
     if plugin is not None:
         labels = plugin().transform_labels
     else:
@@ -66,38 +30,34 @@ def fetch_node_transforms(plugin_label):
 
 @router.get('/refresh')
 async def refresh_plugins():
+    # fetch_node_transforms.cache_clear()
     discover_plugins('app/plugins/')
-    fetch_node_transforms.cache_clear()
-    return {'status': 'success', 'plugins': OBRegistry.ui_labels}
+    return {'status': 'success', 'plugins': Registry.ui_labels}
 
 
 @router.get('/transforms')
 async def get_node_transforms(node_type: str):
     return {
         'type': node_type,
-        'transforms': fetch_node_transforms(node_type)
+        'transforms': await fetch_node_transforms(node_type)
     }
 
 
 @router.get('/type')
 async def get_node_option(node_type: str):
-    try:
-        plugin = OBRegistry.get_plugin(plugin_label=node_type)
-        if plugin:
-            node = plugin.blueprint()
-            node['label'] = node_type
-            return node
-    except Exception as e:
-        logger.error(e)
-        raise HTTPException(status_code=422, detail='pluginNotFound')
+    plugin = await Registry.get_plugin(plugin_label=node_type)
+    if plugin:
+        node = plugin.blueprint()
+        node['label'] = node_type
+        return node
 
 
 def save_transform(results: List[dict]):
     for node in results:
         print(node)
 
 
-def get_command_type(event):
+async def get_command_type(event):
     user_event = event['action'].split(':')
     action = user_event[0]
     action_type = None
@@ -122,17 +82,19 @@ async def remove_nodes(node, action_type, send_json):
     await send_json({'action': 'remove:node', 'node': node})
 
 
-async def transform_nodes(node, action_type, send_json):
+async def nodes_transform(node, action_type, send_json):
     node_output = {}
-    plugin = OBRegistry.get_plugin(node['type'])
+    plugin = await Registry.get_plugin(node['type'])
+    print('plugin: ', plugin)
     if plugin := plugin():
+        print('plugin instance: ', plugin)
         transform_type = node['transform']
-        node_output = plugin._get_transform(
+        node_output = await plugin._get_transform(
             transform_type=transform_type,
             node=node,
             get_driver=deps.get_driver,
         )
-        if isinstance(node_output, list):
+        if type(node_output) == list:
             for output in node_output:
                 output['action'] = 'addNode'
                 output['position'] = node['position']
@@ -145,7 +107,7 @@ async def transform_nodes(node, action_type, send_json):
 
 
 async def execute_event(event, send_json):
-    action, action_type = get_command_type(event)
+    action, action_type = await get_command_type(event)
     if action == 'read':
         await read_graph(event['node'], action_type, send_json)
     if action == 'create':
@@ -155,7 +117,7 @@ async def execute_event(event, send_json):
     if action == 'delete':
         await remove_nodes(event['node'], action_type, send_json)
     if action == 'transform':
-        await transform_nodes(event['node'], action_type, send_json)
+        await nodes_transform(event['node'], action_type, send_json)
 
 
 @router.websocket('/investigation')
@@ -172,7 +134,6 @@ async def active_investigation(
                 send_json=websocket.send_json
             )
         except OBPluginError as e:
-            print(e)
             await websocket.send_json({'action': 'error', 'detail': f'{e}'})
         except WebSocketDisconnect as e:
             logger.info(e)

backend/backend/app/app/api/extractors.py

@@ -2,6 +2,7 @@
 from typing import List
 import validators
 from pydantic import EmailStr
+
 import requests
 from neo4j._sync import work
 from app.core.logger import get_logger

backend/backend/app/app/llm/prompts.py

@@ -1,5 +1,5 @@
 from langchain import PromptTemplate, FewShotPromptTemplate
-from osintbuddy.plugins import OBRegistry
+from osintbuddy import Registry
 
 template_to_osintbuddy_command = """
 Context: Your name is OSINTBuddy. You are an advanced AI OSINT investigator working on intelligence investigations
@@ -52,10 +52,10 @@
   - search_query is a required argument
 """  # noqa
 
-def get_prompt_transform_options():
+async def get_prompt_transform_options():
     data = {}
-    for label in OBRegistry.labels:
-        transforms = OBRegistry.get_plugin(label)(
+    for label in Registry.labels:
+        transforms = await Registry.get_plugin(label)(
             ).__class__.__dict__.values()
         if transforms:
             data[label] = [

backend/backend/app/app/main.py

@@ -3,7 +3,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi_cache import caches, close_caches
 from fastapi_cache.backends.redis import CACHE_KEY, RedisCacheBackend
-from osintbuddy.plugins import discover_plugins
+from osintbuddy import discover_plugins
 
 from app.api.api_v1.api import api_router
 from app.core.config import settings