Merge pull request #349 from Crystalnix/dev

Send logs to Rsyslog and minor changes with REST API
omaha-consulting · Jun 3, 2019 · 98fd1e3 · 98fd1e3
2 parents 5cd4e20 + 2336250
commit 98fd1e3
Show file tree

Hide file tree

Showing 17 changed files with 190 additions and 22 deletions.
diff --git a/Dockerfile-base-alpine b/Dockerfile-base-alpine
@@ -126,7 +126,8 @@ RUN mkdir -p $OMAHA_SERVER_PATH/requirements
 ADD Pipfile Pipfile.lock $OMAHA_SERVER_PATH/
 
 # Dependencies for Pillow
-RUN apk add jpeg-dev libjpeg
+RUN apk add jpeg-dev libjpeg \
+  && apk add rsyslog
 
 RUN \
   pip install pipenv \

diff --git a/README.md b/README.md
@@ -246,6 +246,7 @@ app:
 | LOG_NGINX_TO_FILEBEAT     | Send logs to filebeat| 'True'                     |
 | EMAIL_SENDER              | Verified SES email   |                            |
 | EMAIL_RECIPIENTS          | Feedback recepients  |                            |
+| RSYSLOG_ENABLE            | Send logs to rsyslog | ''                         |
 
 
 

diff --git a/conf/rsyslog.conf b/conf/rsyslog.conf
@@ -0,0 +1,96 @@
+# rsyslog configuration file
+#
+# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
+# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
+# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
+
+
+#### Global directives ####
+
+# Sets the directory that rsyslog uses for work files.
+$WorkDirectory /var/lib/rsyslog
+
+# Sets default permissions for all log files.
+$FileOwner root
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+
+# Check config syntax on startup and abort if unclean (default off).
+#$AbortOnUncleanConfig on
+
+# Reduce repeating messages (default off).
+#$RepeatedMsgReduction on
+
+# Include all config files in /etc/rsyslog.d/.
+include(file="/etc/rsyslog.d/*.conf" mode="optional")
+
+
+#### Modules ####
+
+# Provides --MARK-- message capability.
+module(load="immark")
+
+# Provides support for local system logging (e.g. via logger command).
+module(load="imuxsock")
+
+# Reads kernel messages.
+module(load="imklog")
+
+
+#### Rules ####
+
+# Log all kernel messages to kern.log.
+kern.*							/var/log/kern.log
+
+# Log anything (except mail) of level info or higher.
+# Don't log private authentication messages!
+# NOTE: The minus sign in front of filename disables buffer flush.
+*.info;authpriv.none;cron.none;kern.none;mail.none	-/var/log/messages
+
+# The authpriv file has restricted access.
+authpriv.*						/var/log/auth.log
+
+# Log all the mail messages in one place.
+mail.*							-/var/log/mail.log
+
+# Log cron stuff.
+cron.*							-/var/log/cron.log
+
+# Everybody gets emergency messages.
+*.emerg							:omusrmsg:*
+
+# Save boot messages also to boot.log.
+local7.*						/var/log/boot.log
+
+# Log all kernel messages to the console.
+# Logging much else clutters up the screen.
+#kern.*							/dev/console
+
+
+### Examples ####
+
+# Send all logs to remote syslog via UDP.
+# An on-disk queue is created for this action. If the remote host is
+# down, messages are spooled to disk and sent when it is up again.
+#*.* action(
+#	type="omfwd"
+#	target="192.168.0.1"
+#	port="514"
+#	protocol="udp"
+#	queue.filename="fwdRule1"  # unique name prefix for spool files
+#	queue.type="LinkedList"
+#	queue.maxDiskSpace="256m"
+#	queue.saveOnShutdown="on"
+#	action.resumeRetryCount="-1"
+#	action.resumeInterval="30"
+#)
+
+# Receive messages from remote host via UDP
+# for parameters see http://www.rsyslog.com/doc/imudp.html
+#module(load="imudp")  # needs to be done just once
+#input(
+#	type="imudp"
+#	port="514"
+#)
diff --git a/docker-compose.common.yml b/docker-compose.common.yml
@@ -34,3 +34,4 @@ web:
     ELK_HOST: ''
     ELK_PORT: ''
     FILEBEAT_DESTINATION: ''
+    RSYSLOG_ENABLE: 'True'
diff --git a/omaha_server/feedback/admin.py b/omaha_server/feedback/admin.py
@@ -65,7 +65,7 @@ class FeedbackAdmin(admin.ModelAdmin):
 
     def get_queryset(self, request):
         qs = super(FeedbackAdmin, self).get_queryset(request)
-        return qs.filter(description__startswith='BlackBox')
+        return qs.get_feedbacks()
 
 
 @admin.register(FeedbackDescription)
@@ -78,4 +78,4 @@ class FeedbackDescriptionAdmin(admin.ModelAdmin):
 
     def get_queryset(self, request):
         qs = super(FeedbackDescriptionAdmin, self).get_queryset(request)
-        return qs.exclude(description__startswith='BlackBox')
+        return qs.get_feedbacks_description()
diff --git a/omaha_server/feedback/api.py b/omaha_server/feedback/api.py
@@ -26,10 +26,16 @@
 from feedback.serializers import FeedbackSerializer
 from feedback.models import Feedback
 
-
-class FeedbackViewSet(mixins.ListModelMixin,
-                      mixins.RetrieveModelMixin,
-                      viewsets.GenericViewSet):
-    queryset = Feedback.objects.all().order_by('-id')
+class BaseFeedbackViewSet(mixins.ListModelMixin,
+                          mixins.RetrieveModelMixin,
+                          viewsets.GenericViewSet):
     serializer_class = FeedbackSerializer
     pagination_class = StandardResultsSetPagination
+
+
+class FeedbackViewSet(BaseFeedbackViewSet):
+    queryset = Feedback.objects.get_feedbacks()
+
+
+class FeedbackDescriptionViewSet(BaseFeedbackViewSet):
+    queryset = Feedback.objects.get_feedbacks_description()
diff --git a/omaha_server/feedback/factories.py b/omaha_server/feedback/factories.py
@@ -27,7 +27,7 @@ class FeedbackFactory(factory.DjangoModelFactory):
     class Meta:
         model = 'feedback.Feedback'
 
-    description = factory.Sequence(lambda n: 'Description #%s' % n)
+    description = factory.Sequence(lambda n: 'BlackBox Description #%s' % n)
     email = factory.Sequence(lambda n: 'user%s@example.com' % n)
     page_url = factory.Sequence(lambda n: 'http://url%s.com/' % n)
 
@@ -44,3 +44,7 @@ class Meta:
             'user_email': 'user@example.com',
       }
     }
+
+
+class FeedbackDescriptionFactory(FeedbackFactory):
+    description = factory.Sequence(lambda n: 'Description #%s' % n)
diff --git a/omaha_server/feedback/managers.py b/omaha_server/feedback/managers.py
@@ -29,6 +29,12 @@ def filter_by_enabled(self, *args, **kwargs):
     def get_size(self):
         return self.aggregate(size=Sum(F('screenshot_size') + F('blackbox_size') + F('system_logs_size') + F('attached_file_size')))['size'] or 0
 
+    def get_feedbacks_description(self, *args, **kwargs):
+        return self.exclude(description__startswith='BlackBox', *args, **kwargs).order_by('-id')
+
+    def get_feedbacks(self, *args, **kwargs):
+        return self.filter(description__startswith='BlackBox', *args, **kwargs).order_by('-id')
+
 class FeedbackManager(models.Manager):
     def get_queryset(self):
         return FeedbackQuerySet(self.model, using=self._db)
@@ -37,4 +43,4 @@ def __getattr__(self, name):
         if name.startswith('_'):
             raise AttributeError
         else:
-            return getattr(self.get_queryset(), name)
+            return getattr(self.get_queryset(), name)
diff --git a/omaha_server/feedback/tests/test_api.py b/omaha_server/feedback/tests/test_api.py
@@ -26,23 +26,23 @@
 from rest_framework.test import APITestCase
 
 from feedback.serializers import FeedbackSerializer
-from feedback.factories import FeedbackFactory
+from feedback.factories import FeedbackFactory, FeedbackDescriptionFactory
 
 from omaha.tests.utils import temporary_media_root
 from omaha.tests.test_api import BaseTest
 from omaha_server.utils import is_private
 
 
-class FeedbackTest(BaseTest, APITestCase):
-    url = 'feedback-list'
-    url_detail = 'feedback-detail'
-    factory = FeedbackFactory
+class FeedbackDescriptionTest(BaseTest, APITestCase):
+    url = 'api-feedback-description-list'
+    url_detail = 'api-feedback-description-detail'
+    factory = FeedbackDescriptionFactory
     serializer = FeedbackSerializer
 
     @is_private()
     @temporary_media_root(MEDIA_URL='http://cache.pack.google.com/edgedl/chrome/install/782.112/')
     def test_detail(self):
-        super(FeedbackTest, self).test_detail()
+        super(FeedbackDescriptionTest, self).test_detail()
 
     @is_private()
     @temporary_media_root(MEDIA_URL='http://cache.pack.google.com/edgedl/chrome/install/782.112/')
@@ -57,3 +57,15 @@ def test_list(self):
     def test_create(self):
         response = self.client.post(reverse(self.url), {})
         self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
+
+
+class FeedbackTest(FeedbackDescriptionTest):
+    url = 'api-feedback-list'
+    url_detail = 'api-feedback-detail'
+    factory = FeedbackFactory
+    serializer = FeedbackSerializer
+
+    @is_private()
+    @temporary_media_root(MEDIA_URL='http://cache.pack.google.com/edgedl/chrome/install/782.112/')
+    def test_detail(self):
+        super(FeedbackTest, self).test_detail()
diff --git a/omaha_server/omaha_server/settings_prod.py b/omaha_server/omaha_server/settings_prod.py
@@ -57,6 +57,7 @@
 
 FILEBEAT_HOST = os.environ.get('FILEBEAT_HOST', 'localhost')
 FILEBEAT_PORT = os.environ.get('FILEBEAT_PORT', 9021)
+RSYSLOG_ENABLE = True if os.environ.get('RSYSLOG_ENABLE', '').title() == 'True' else False
 
 INSTALLED_APPS = INSTALLED_APPS + (
     'raven.contrib.django.raven_compat',
@@ -88,6 +89,12 @@
             'level': 'DEBUG',
             'class': 'logging.StreamHandler',
             'formatter': 'verbose'
+        },
+        'rsyslog': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.SysLogHandler',
+            'formatter': 'filebeat_format',
+            'address': '/dev/log'
         }
     },
     'loggers': {
@@ -116,6 +123,16 @@
             'handlers': ['console'],
             'propagate': False,
         },
+        'celery.task': {
+            'level': 'INFO',
+            'handlers': ['console'],
+            'propagate': False,
+        },
+        'limitation': {
+            'level': 'INFO',
+            'handlers': ['console'],
+            'propagate': False,
+        }
     },
 }
 
@@ -128,3 +145,10 @@
     }
     LOGGING['root']['handlers'].append('filebeat')
     LOGGING['loggers']['django.request']['handlers'].append('filebeat')
+
+if RSYSLOG_ENABLE:
+    LOGGING['root']['handlers'].append('rsyslog')
+    LOGGING['loggers']['django.request']['handlers'].append('rsyslog')
+    LOGGING['loggers']['celery.beat']['handlers'].append('rsyslog')
+    LOGGING['loggers']['celery.task']['handlers'].append('rsyslog')
+    LOGGING['loggers']['limitation']['handlers'].append('rsyslog')
diff --git a/omaha_server/omaha_server/urls.py b/omaha_server/omaha_server/urls.py
@@ -22,7 +22,8 @@
 router.register(r'sparkle/version', sparkle.api.SparkleVersionViewSet)
 router.register(r'symbols', crash.api.SymbolsViewSet)
 router.register(r'crash_report', crash.api.CrashViewSet)
-router.register(r'feedback', feedback.api.FeedbackViewSet)
+router.register(r'feedback/description', feedback.api.FeedbackDescriptionViewSet, 'api-feedback-description')
+router.register(r'feedback', feedback.api.FeedbackViewSet, 'api-feedback')
 router.register(r'partialupdate', omaha.api.PartialUpdateViewSet)
 router.register(r'statistics/live', omaha.api.StatisticsVersionsLiveView, 'api-statistics-live')
 

diff --git a/omaha_server/sparkle/factories.py b/omaha_server/sparkle/factories.py
@@ -31,5 +31,6 @@ class Meta:
     channel = factory.lazy_attribute(lambda x: ChannelFactory())
     version = '2062.124'
     short_version = '37.0.2062.124'
+    minimum_system_version = '10.14.4'
     file = factory.django.FileField(filename='the_file.dat')
     file_size = 123
diff --git a/omaha_server/sparkle/serializers.py b/omaha_server/sparkle/serializers.py
@@ -35,12 +35,13 @@ class SparkleVersionSerializer(serializers.HyperlinkedModelSerializer):
     channel = serializers.PrimaryKeyRelatedField(queryset=Channel.objects.all())
     version = serializers.CharField()
     short_version = serializers.CharField(required=False)
+    minimum_system_version = serializers.CharField(required=False)
 
     class Meta:
         model = SparkleVersion
         fields = ('id', 'is_enabled', 'is_critical', 'app', 'channel', 'version', 'short_version',
-                  'release_notes', 'file', 'file_size', 'dsa_signature',
-                  'created', 'modified')
+                  'minimum_system_version', 'release_notes', 'file', 'file_size', 'dsa_signature',
+                  'created', 'modified',)
         read_only_fields = ('created', 'modified')
 
     def create(self, validated_data):

diff --git a/omaha_server/sparkle/tests/fixtures.py b/omaha_server/sparkle/tests/fixtures.py
@@ -28,6 +28,7 @@
 
             <item>
                 <title>chrome 782.112</title>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -52,6 +53,7 @@
 
             <item>
                 <title>chrome 782.110</title>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -64,6 +66,7 @@
             </item>
             <item>
                 <title>chrome 782.111</title>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -87,6 +90,7 @@
         <language>en</language>
             <item>
                 <title>chrome 782.111</title>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -99,6 +103,7 @@
             </item>
             <item>
                 <title>chrome 782.112</title>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -111,6 +116,7 @@
             </item>
             <item>
                 <title>chrome 782.113</title>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -135,6 +141,7 @@
 
             <item>
                 <title>chrome_dsa 782.112</title>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -160,7 +167,7 @@
 
             <item>
                 <title>chrome 782.112</title>
-                <sparkle:minimumSystemVersion>10.8.6</sparkle:minimumSystemVersion>
+                <sparkle:minimumSystemVersion>10.14.4</sparkle:minimumSystemVersion>
                 <description><![CDATA[
 
                 ]]>
@@ -173,4 +180,4 @@
             </item>
 
     </channel>
-</rss>"""
+</rss>"""