Live log streaming (#6)

* Add live log streaming endpoints

* Add JS / HTML for live log streaming, add new Command model to record command outputs

web/api/serializers.py

@@ -1079,6 +1079,13 @@ def get_engine_name(self, subscan):
 		return ''
 
 
+class CommandSerializer(serializers.ModelSerializer):
+	class Meta:
+		model = Command
+		fields = '__all__'
+		depth = 1
+
+
 class ScanHistorySerializer(serializers.ModelSerializer):
 
 	subdomain_count = serializers.SerializerMethodField('get_subdomain_count')

web/api/urls.py

@@ -6,28 +6,19 @@
 
 app_name = 'api'
 router = routers.DefaultRouter()
-
 router.register(r'listDatatableSubdomain', SubdomainDatatableViewSet)
-
 router.register(r'listTargets', ListTargetsDatatableViewSet)
-
 router.register(r'listSubdomains', SubdomainsViewSet)
-
 router.register(r'listEndpoints', EndPointViewSet)
-
 router.register(r'listDirectories', DirectoryViewSet)
-
 router.register(r'listVulnerability', VulnerabilityViewSet)
-
 router.register(r'listInterestingSubdomains', InterestingSubdomainViewSet)
-
 router.register(r'listInterestingEndpoints', InterestingEndpointViewSet)
-
 router.register(r'listSubdomainChanges', SubdomainChangesViewSet)
-
 router.register(r'listEndPointChanges', EndPointChangesViewSet)
-
 router.register(r'listIps', IpAddressViewSet)
+router.register(r'listActivityLogs', ListActivityLogsViewSet)
+router.register(r'listScanLogs', ListScanLogsViewSet)
 
 urlpatterns = [
     url('^', include(router.urls)),

web/api/views.py

@@ -401,7 +401,6 @@ def post(self, request):
 		return Response(response)
 
 
-
 class ToggleSubdomainImportantStatus(APIView):
 	def post(self, request):
 		req = self.request
@@ -419,6 +418,7 @@ def post(self, request):
 
 		return Response(response)
 
+
 class AddTarget(APIView):
 	def post(self, request):
 		req = self.request
@@ -445,6 +445,7 @@ def post(self, request):
 			'domain_id': domain.id
 		})
 
+
 class FetchSubscanResults(APIView):
 	def post(self, request):
 		req = self.request
@@ -1863,6 +1864,26 @@ def special_lookup(self, search_value):
 		return qs
 
 
+class ListActivityLogsViewSet(viewsets.ModelViewSet):
+	serializer_class = CommandSerializer
+	queryset = Command.objects.none()
+	def get_queryset(self):
+		req = self.request
+		activity_id = req.query_params.get('activity_id')
+		self.queryset = Command.objects.filter(activity__id=activity_id)
+		return self.queryset
+
+
+class ListScanLogsViewSet(viewsets.ModelViewSet):
+	serializer_class = CommandSerializer
+	queryset = Command.objects.none()
+	def get_queryset(self):
+		req = self.request
+		scan_id = req.query_params.get('scan_id')
+		self.queryset = Command.objects.filter(scan_history__id=scan_id)
+		return self.queryset
+
+
 class ListEndpoints(APIView):
 	def get(self, request, format=None):
 		req = self.request

web/reNgine/celery_custom_task.py

@@ -79,6 +79,7 @@ def __call__(self, *args, **kwargs):
 		self.domain_id = self.domain.id if self.domain else None
 		self.subdomain = self.subscan.subdomain if self.subscan else None
 		self.subdomain_id = self.subdomain.id if self.subdomain else None
+		self.activity_id = None
 
 		# Set file self.task_name if not already set
 		if not self.filename:
@@ -175,6 +176,8 @@ def create_scan_activity(self):
 			time=timezone.now(),
 			status=RUNNING_TASK,
 			celery_id=celery_id)
+		self.activity.save()
+		self.activity_id = self.activity.id
 		if self.scan:
 			self.activity.scan_of = self.scan
 			self.activity.save()

web/reNgine/common_func.py

@@ -448,9 +448,10 @@ def sanitize_url(http_url):
 # Utils #
 #-------#
 
+
 def get_random_proxy():
 	"""Get a random proxy from the list of proxies input by user in the UI.
-	
+
 	Returns:
 		str: Proxy name or '' if no proxy defined in db or use_proxy is False.
 	"""

web/reNgine/settings.py

@@ -227,7 +227,7 @@
             'format': '%(name)-10s | %(message)s'
         },
         'task': {
-            '()': lambda : RengineTaskFormatter('%(task_name)-34s | %(message)s')
+            '()': lambda : RengineTaskFormatter('%(task_name)-34s | %(levelname)s | %(message)s')
         }
     },
     'loggers': {
@@ -259,6 +259,7 @@
         },
         'reNgine.tasks': {
             'handlers': ['task'],
+            'level': 'DEBUG' if DEBUG else 'INFO',
             'propagate': False
         }
     },