89 recording stops if agent restarts (#91)

* Update api init to set url in settings

* Create status constants

* Update statuses controller show and index endpoint responses

stoobly_agent/app/api/__init__.py

@@ -4,6 +4,7 @@
 
 from http.server import HTTPServer
 
+from stoobly_agent.app.settings import Settings
 from stoobly_agent.config.constants import env_vars
 
 from .application_http_request_handler import ApplicationHTTPRequestHandler
@@ -15,8 +16,13 @@ def start_server(host, port):
 def run(**kwargs):
     ui_host = kwargs['ui_host']
     ui_port = kwargs['ui_port']
+    url = f"http://{kwargs['ui_host']}:{kwargs['ui_port']}"
 
-    os.environ[env_vars.AGENT_URL] = f"http://{kwargs['ui_host']}:{kwargs['ui_port']}"
+    os.environ[env_vars.AGENT_URL] = url
+
+    settings = Settings.instance()
+    settings.ui.url = url
+    settings.commit()
 
     print(f"UI server listening at http://{ui_host}:{ui_port}\n")
 

stoobly_agent/app/api/requests_controller.py

@@ -76,6 +76,10 @@ def __init__(self, request, response):
         if not request:
             return context.internal_error()
 
+        from stoobly_agent.lib.api.keys.project_key import LOCAL_PROJECT_ID
+        from stoobly_agent.app.proxy.utils.publish_change_service import publish_requests_modified
+        publish_requests_modified(body_params.get('project_id') or LOCAL_PROJECT_ID, sync=True)
+
         context.render(
             json = request,
             status = 200

stoobly_agent/app/api/routes.py

@@ -56,8 +56,8 @@
       [REQUEST_PATH, RequestsController.instance().get],
       [SCENARIOS_PATH, ScenariosController.instance().index],
       [re.compile('/'.join([SCENARIO_PATH.pattern, 'download$'])), ScenariosController.instance().download],
+      [STATUSES_PATH, StatusesController.instance().index],
       [SCENARIO_PATH, ScenariosController.instance().get],
-      [STATUS_PATH, StatusesController.instance().get],
   ],
   'POST': [
       ['/'.join([PROXY_PATH, 'post']), ProxyController.instance().do_POST],

stoobly_agent/app/api/statuses_controller.py

@@ -18,6 +18,27 @@ def instance(cls):
 
         return cls._instance
 
+    def index(self, context):
+        version = -1
+
+        try:
+            version = int(context.params.get('version'))
+        except ValueError as e:
+            pass
+
+        cache = Cache.instance()
+        statuses = []
+        if not version or version != cache.version:
+            statuses = cache.read_all()
+
+        context.render(
+            json = {
+                'statuses': statuses,
+                'version': cache.version,
+            },
+            status = 200
+        )
+
     # GET /api/v1/admin/statuses/:id
     def get(self, context):
         context.parse_path_params({
@@ -33,9 +54,11 @@ def get(self, context):
                 status = 204
             )
         else:
-            cache.delete(context.params.get('id'))
             context.render(
-                plain = status,
+                json = {
+                    'status': status,
+                    'version': cache.version
+                },
                 status = 200
             )
 
@@ -46,7 +69,7 @@ def update(self, context):
         })
 
         cache = Cache.instance()
-        cache.write(context.params.get('id'), context.body)
+        cache.write(context.params.get('id'), context.body.decode())
 
         context.render(
             plain = '',

stoobly_agent/app/proxy/intercept_handler.py

@@ -21,9 +21,6 @@
 # Disable proxy settings in urllib
 os.environ['no_proxy'] = '*'
 
-# Observe config for changes
-Settings.instance().watch()
-
 LOG_ID = 'InterceptHandler'
 
 def request(flow: MitmproxyHTTPFlow):

stoobly_agent/app/proxy/intercept_settings.py

@@ -57,8 +57,11 @@ def alias_resolve_strategy(self):
   def active(self):
     if self.__intercept_settings.active:
       return True
+
+    if not self.__headers:
+      return False
 
-    return self.__headers and custom_headers.PROXY_MODE in self.__headers
+    return custom_headers.PROXY_MODE in self.__headers
 
   @property
   def lifecycle_hooks_script_path(self):

stoobly_agent/app/proxy/upload/upload_request_service.py

@@ -15,7 +15,7 @@
 from stoobly_agent.lib.logger import Logger, bcolors
 from stoobly_agent.lib.orm.request import Request
 
-from ..utils.publish_change_service import publish_change
+from ..utils.publish_change_service import publish_requests_modified
 from .join_request_service import join_rewritten_request
 
 AGENT_STATUSES = {
@@ -104,7 +104,7 @@ def __upload_request_with_body_params(request_model: RequestModel, body_params:
     #    return None
 
     if request:
-        publish_change(AGENT_STATUSES['REQUESTS_MODIFIED'])
+        publish_requests_modified(body_params['project_id'])
 
     return request
 

stoobly_agent/app/proxy/utils/publish_change_service.py

@@ -1,13 +1,22 @@
 import pdb
 import threading
 
-from stoobly_agent.app.settings import Settings
+from typing import TypedDict
+
+from stoobly_agent.config.constants.statuses import REQUESTS_MODIFIED
 from stoobly_agent.lib.api.agent_api import AgentApi
-from stoobly_agent.lib.api.keys import ProjectKey
+from stoobly_agent.lib.cache import Cache
 from stoobly_agent.lib.logger import Logger
 
+class Options(TypedDict):
+  sync: bool
+
 # Announce that a new request has been created
-def publish_change(status: str):
+def publish_change(status: str, value: any, **options: Options):
+  if options.get('sync'):
+    return __publish_change_sync(status, value)
+
+  from stoobly_agent.app.settings import Settings
   settings = Settings.instance()
 
   # If ui is not active, return
@@ -20,11 +29,18 @@ def publish_change(status: str):
     Logger.instance().warn('Settings.ui.url not configured')
     return False
   else:
-    project_key = ProjectKey(settings.proxy.intercept.project_key)
-    thread = threading.Thread(target=__put_status, args=(ui_url, status, project_key.id))
+    thread = threading.Thread(target=__put_status, args=(ui_url, status, value))
     thread.start()
     return True
 
-def __put_status(ui_url, status, project_id):
+def publish_requests_modified(value, **options: Options):
+  return publish_change(REQUESTS_MODIFIED, value, **options)
+
+def __publish_change_sync(status: str, value):
+  cache = Cache.instance()
+  cache.write(status, value)
+  return True
+
+def __put_status(ui_url, status, value):
     api: AgentApi = AgentApi(ui_url)
-    api.update_status(status, project_id)
+    api.update_status(status, value)

stoobly_agent/app/settings/__init__.py

@@ -9,7 +9,7 @@
 from watchdog.events import PatternMatchingEventHandler
 from yamale import *
 
-from stoobly_agent.config.constants import env_vars
+from stoobly_agent.config.constants import env_vars, statuses
 from stoobly_agent.config.data_dir import DataDir
 from stoobly_agent.config.source_dir import SourceDir
 from stoobly_agent.lib.logger import Logger
@@ -36,6 +36,7 @@ class Settings:
     __schema_file_path = None
 
     __load_lock = False
+    __watching = False
 
     def __init__(self, **kwargs: SettingsOptions):
         if Settings.__instance:
@@ -84,6 +85,9 @@ def remote(self):
         return self.__remote_settings
 
     def watch(self):
+        if self.__watching:
+            return False
+
         patterns = ['settings.yml']
         ignore_patterns = None
         ignore_directories = False
@@ -97,6 +101,10 @@ def watch(self):
         observer.schedule(event_handler, watch_dir)
         observer.start()
 
+        self.__watching = True
+
+        return True
+
     ### Get
 
     def to_dict(self):
@@ -150,11 +158,15 @@ def __load_settings(self):
 
     def __reload_settings(self, event):
         if not self.__load_lock:
+            from stoobly_agent.app.proxy.utils.publish_change_service import publish_change
+
             self.__load_lock = True
 
             Logger.instance().info(f"{self.LOG_ID}.reload_settings")
             self.__load_settings()
 
+            publish_change(statuses.SETTINGS_MODIFIED, self.__settings)
+
         self.__load_lock = False
 
     def __validate(self):

stoobly_agent/cli.py

@@ -89,6 +89,9 @@ def init(**kwargs):
 def run(**kwargs):
     os.environ[env_vars.AGENT_PROXY_URL] = f"http://{kwargs['proxy_host']}:{kwargs['proxy_port']}"
 
+    # Observe config for changes
+    Settings.instance().watch()
+
     if not os.getenv(env_vars.LOG_LEVEL):
         os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
 

stoobly_agent/config/constants/statuses.py

@@ -0,0 +1,2 @@
+REQUESTS_MODIFIED = 'requests-modified'
+SETTINGS_MODIFIED = 'settings-modified'

stoobly_agent/lib/cache.py

@@ -1,3 +1,6 @@
+import pdb
+import random
+
 class Cache:
     _instance = None
 
@@ -6,7 +9,9 @@ def __init__(self):
             raise RuntimeError('Call instance() instead')
         else:
             self.data = {}
-
+            self.dataVersions = {}
+            self.startVersion = random.randint(1, 1000000)
+            self.version = self.startVersion
 
     @classmethod
     def instance(cls):
@@ -15,11 +20,43 @@ def instance(cls):
 
         return cls._instance
 
+    def clear(self):
+        self.data = {}
+        self.dataVersions = {}
+        self.version += 1
+
     def read(self, key):
-        return self.data.get(key)
+        data = self.data.get(key)
+
+        if data == None:
+            return None
+
+        return {
+            'name': key,
+            'value': data,
+            'version': self.dataVersions[key],
+        }
+
+    def read_all(self):
+        data = []
+
+        for key in self.data.keys():
+            datum = self.read(key)
+            if datum:
+                data.append(datum)
+
+        return data
 
     def write(self, key, val):
         self.data[key] = val
 
+        if key not in self.dataVersions:
+            self.dataVersions[key] = self.startVersion
+
+        self.dataVersions[key] += 1
+        self.version += 1
+
     def delete(self, key):
         del self.data[key]
+        del self.dataVersions[key]
+        self.version += 1