Add Bulk Import/Export Functionnality in Admin Interface

Watcher/Watcher/data_leak/admin.py

@@ -1,5 +1,7 @@
 from django.contrib import admin
-from .models import Subscriber, Alert, Keyword, PasteId
+from .models import Subscriber, Alert, Keyword
+from import_export import resources
+from import_export.admin import ImportExportModelAdmin
 
 
 def custom_titled_filter(title):
@@ -25,6 +27,9 @@ class Alert(admin.ModelAdmin):
     list_filter = ('keyword', ('status', custom_titled_filter('Active Status')))
     search_fields = ['id', 'url', 'keyword']
 
+    def has_add_permission(self, request):
+        return False
+
     def make_disable(self, request, queryset):
         rows_updated = queryset.update(status=False)
 
@@ -50,8 +55,15 @@ def make_enable(self, request, queryset):
     actions = [make_disable, make_enable]
 
 
+class KeywordResource(resources.ModelResource):
+    class Meta:
+        model = Keyword
+        exclude = ('created_at',)
+
+
 @admin.register(Keyword)
-class Keyword(admin.ModelAdmin):
+class Keyword(ImportExportModelAdmin):
     list_display = ['name', 'created_at']
     list_filter = ['created_at']
     search_fields = ['name']
+    resource_class = KeywordResource

Watcher/Watcher/data_leak/migrations/0009_auto_20201123_1812.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.1.3 on 2020-11-23 18:12
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_leak', '0008_auto_20200403_1502'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='keyword',
+            options={'ordering': ['name'], 'verbose_name_plural': 'Keywords Monitored'},
+        ),
+    ]

Watcher/Watcher/data_leak/models.py

@@ -13,6 +13,7 @@ class Keyword(models.Model):
 
     class Meta:
         ordering = ["name"]
+        verbose_name_plural = 'Keywords Monitored'
 
     def __str__(self):
         return self.name

Watcher/Watcher/dns_finder/admin.py

@@ -1,5 +1,7 @@
 from django.contrib import admin
 from .models import DnsMonitored, DnsTwisted, Alert, Subscriber
+from import_export import resources
+from import_export.admin import ImportExportModelAdmin
 
 
 def custom_titled_filter(title):
@@ -18,6 +20,9 @@ class Alert(admin.ModelAdmin):
     list_filter = ('created_at', ('status', custom_titled_filter('Active Status')))
     search_fields = ['id', 'dns_twisted']
 
+    def has_add_permission(self, request):
+        return False
+
     def make_disable(self, request, queryset):
         rows_updated = queryset.update(status=False)
 
@@ -43,11 +48,18 @@ def make_enable(self, request, queryset):
     actions = [make_disable, make_enable]
 
 
+class DnsMonitoredResource(resources.ModelResource):
+    class Meta:
+        model = DnsMonitored
+        exclude = ('created_at',)
+
+
 @admin.register(DnsMonitored)
-class DnsMonitored(admin.ModelAdmin):
+class DnsMonitored(ImportExportModelAdmin):
     list_display = ['domain_name', 'created_at']
     list_filter = ['created_at']
     search_fields = ['domain_name']
+    resource_class = DnsMonitoredResource
 
 
 @admin.register(DnsTwisted)
@@ -56,6 +68,9 @@ class DnsTwisted(admin.ModelAdmin):
     list_filter = ['created_at', 'dns_monitored', 'fuzzer']
     search_fields = ['domain_name', 'dns_monitored', 'fuzzer']
 
+    def has_add_permission(self, request):
+        return False
+
 
 @admin.register(Subscriber)
 class Subscriber(admin.ModelAdmin):

Watcher/Watcher/dns_finder/migrations/0005_auto_20201123_1812.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.1.3 on 2020-11-23 18:12
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dns_finder', '0004_subscriber'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='dnsmonitored',
+            options={'ordering': ['domain_name'], 'verbose_name': 'Corporate DNS', 'verbose_name_plural': 'Corporate DNS Assets Monitored'},
+        ),
+        migrations.AlterModelOptions(
+            name='dnstwisted',
+            options={'ordering': ['-created_at'], 'verbose_name': 'Twisted DNS', 'verbose_name_plural': 'Twisted DNS'},
+        ),
+    ]

Watcher/Watcher/dns_finder/models.py

@@ -12,7 +12,8 @@ class DnsMonitored(models.Model):
 
     class Meta:
         ordering = ["domain_name"]
-        verbose_name = 'Monitored DNS'
+        verbose_name = 'Corporate DNS'
+        verbose_name_plural = "Corporate DNS Assets Monitored"
 
     def __str__(self):
         return self.domain_name
@@ -30,6 +31,7 @@ class DnsTwisted(models.Model):
     class Meta:
         ordering = ["-created_at"]
         verbose_name = 'Twisted DNS'
+        verbose_name_plural = "Twisted DNS"
 
     def __str__(self):
         return self.domain_name

Watcher/Watcher/frontend/src/components/DataLeak/KeyWords.js

@@ -218,7 +218,7 @@ export class KeyWords extends Component {
                 <div className="row">
                     <div className="col-lg-12">
                         <div className="float-left">
-                            <h4>Keywords Monitoring</h4>
+                            <h4>Keywords Monitored</h4>
                         </div>
                         <div className="float-right mr-1 mb-2">
                             <button className="btn btn-success" onClick={() => {

Watcher/Watcher/frontend/src/components/DnsFinder/DnsMonitored.js

@@ -223,7 +223,7 @@ export class DnsMonitored extends Component {
                 <div className="row">
                     <div className="col-lg-12">
                         <div className="float-left">
-                            <h4>Corporate DNS Assets</h4>
+                            <h4>Corporate DNS Assets Monitored</h4>
                         </div>
                         <div className="float-right mr-1 mb-2">
                             <button className="btn btn-success" onClick={() => {

Watcher/Watcher/frontend/src/components/SiteMonitoring/SuspiciousSites.js

@@ -496,7 +496,7 @@ export class SuspiciousSites extends Component {
                 <div className="row">
                     <div className="col-lg-12">
                         <div className="float-left">
-                            <h4>Suspicious Websites Monitoring</h4>
+                            <h4>Suspicious Websites Monitored</h4>
                         </div>
                         <div className="float-right mr-1 mb-2">
                             <button className="btn btn-success" onClick={() => {

Watcher/Watcher/site_monitoring/admin.py

@@ -1,5 +1,7 @@
 from django.contrib import admin
 from .models import Alert, Site, Subscriber
+from import_export import resources
+from import_export.admin import ExportMixin
 
 
 def custom_titled_filter(title):
@@ -14,10 +16,15 @@ def __new__(cls, *args, **kwargs):
 
 @admin.register(Alert)
 class Alert(admin.ModelAdmin):
-    list_display = ['id', 'type', 'site', 'new_ip', 'new_ip_second', 'new_MX_records', 'new_mail_A_record_ip', 'old_ip', 'old_ip_second', 'old_MX_records', 'old_mail_A_record_ip', 'difference_score',
+    list_display = ['id', 'type', 'site', 'new_ip', 'new_ip_second', 'new_MX_records', 'new_mail_A_record_ip', 'old_ip',
+                    'old_ip_second', 'old_MX_records', 'old_mail_A_record_ip', 'difference_score',
                     'status', 'created_at']
     list_filter = ('site', ('status', custom_titled_filter('Active Status')))
-    search_fields = ['id', 'new_ip', 'new_ip_second', 'old_ip', 'old_ip_second', 'difference_score', 'new_MX_records', 'new_mail_A_record_ip', 'old_MX_records', 'old_mail_A_record_ip']
+    search_fields = ['id', 'new_ip', 'new_ip_second', 'old_ip', 'old_ip_second', 'difference_score', 'new_MX_records',
+                     'new_mail_A_record_ip', 'old_MX_records', 'old_mail_A_record_ip']
+
+    def has_add_permission(self, request):
+        return False
 
     def make_disable(self, request, queryset):
         rows_updated = queryset.update(status=False)
@@ -44,12 +51,22 @@ def make_enable(self, request, queryset):
     actions = [make_disable, make_enable]
 
 
+class SiteResource(resources.ModelResource):
+    class Meta:
+        model = Site
+        exclude = ('the_hive_case_id', 'misp_event_id', 'monitored', 'content_monitoring', 'content_fuzzy_hash', 'mail_monitoring', 'ip_monitoring')
+
+
 @admin.register(Site)
-class Site(admin.ModelAdmin):
+class Site(ExportMixin, admin.ModelAdmin):
     list_display = ['rtir', 'domain_name', 'ip', 'ip_second', 'monitored', 'web_status', 'misp_event_id',
                     'the_hive_case_id', 'created_at', 'expiry']
     list_filter = ['created_at', 'expiry', 'monitored', 'web_status']
     search_fields = ['rtir', 'domain_name', 'ip', 'ip_second', 'the_hive_case_id', 'misp_event_id']
+    resource_class = SiteResource
+
+    def has_add_permission(self, request):
+        return False
 
 
 @admin.register(Subscriber)

Watcher/Watcher/site_monitoring/migrations/0026_auto_20201123_1812.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.1.3 on 2020-11-23 18:12
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('site_monitoring', '0025_auto_20200910_1014'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='site',
+            options={'ordering': ['-rtir'], 'verbose_name': 'Website', 'verbose_name_plural': 'Suspicious Websites Monitored'},
+        ),
+    ]

Watcher/Watcher/site_monitoring/models.py

@@ -34,6 +34,8 @@ class Site(models.Model):
 
     class Meta:
         ordering = ["-rtir"]
+        verbose_name = 'Website'
+        verbose_name_plural = 'Suspicious Websites Monitored'
 
     def __str__(self):
         return self.domain_name

Watcher/Watcher/site_monitoring/serializers.py

@@ -8,7 +8,7 @@
 from .models import Alert, Site
 
 from thehive4py.api import TheHiveApi
-from thehive4py.models import CaseTask, Case
+from thehive4py.models import Case
 from .thehive import create_observables, update_observables
 
 from pymisp import ExpandedPyMISP, MISPEvent

Watcher/Watcher/threats_watcher/admin.py

@@ -2,28 +2,51 @@
 from __future__ import unicode_literals
 
 from django.contrib import admin
-from .models import Source, TrendyWord, BannedWord, PostUrl, Subscriber
+from .models import Source, TrendyWord, BannedWord, Subscriber
+from import_export import resources
+from import_export.admin import ImportExportModelAdmin, ExportMixin
+
+
+class SourceResource(resources.ModelResource):
+    class Meta:
+        model = Source
+        exclude = ('created_at',)
 
 
 @admin.register(Source)
-class SourceAdmin(admin.ModelAdmin):
+class SourceAdmin(ImportExportModelAdmin):
     list_display = ['url', 'created_at']
     list_filter = ['created_at']
     search_fields = ['url']
+    resource_class = SourceResource
+
+
+class BannedWordResource(resources.ModelResource):
+    class Meta:
+        model = BannedWord
+        exclude = ('created_at',)
 
 
 @admin.register(BannedWord)
-class BannedWordAdmin(admin.ModelAdmin):
+class BannedWordAdmin(ImportExportModelAdmin):
     list_display = ['name', 'created_at']
     list_filter = ['created_at']
     search_fields = ['name']
+    resource_class = BannedWordResource
+
+
+class TrendyWordResource(resources.ModelResource):
+    class Meta:
+        model = TrendyWord
+        exclude = ('posturls',)
 
 
 @admin.register(TrendyWord)
-class TrendyWordAdmin(admin.ModelAdmin):
+class TrendyWordAdmin(ExportMixin, admin.ModelAdmin):
     list_display = ('name', 'occurrences', 'created_at')
     list_filter = ['created_at']
     search_fields = ['name']
+    resource_class = TrendyWordResource
 
     def has_add_permission(self, request):
         return False

Watcher/Watcher/threats_watcher/migrations/0011_auto_20201123_1812.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.1.3 on 2020-11-23 18:12
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('threats_watcher', '0010_auto_20201102_1151'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='bannedword',
+            options={'verbose_name': 'block word', 'verbose_name_plural': 'Blocklist'},
+        ),
+    ]

Watcher/Watcher/threats_watcher/models.py

@@ -59,7 +59,7 @@ def cascade_delete_branch(sender, instance, **kwargs):
     Also verified if the posts is not reference for another :model:`threats_watcher.TrendyWord`.
     """
     for posturl in instance.posturls.all():
-        # Si le posturl est associé seulement à 1 ou 0 trendyword on peut le supprimer
+        # If posturl is associated to 1 or 0 trendyword, we can remove it
         if TrendyWord.objects.filter(posturls=posturl).count() <= 1:
             PostUrl.objects.get(url=posturl).delete()
 
@@ -75,7 +75,7 @@ def __str__(self):
         return self.name
 
     class Meta:
-        verbose_name = 'Blocklist'
+        verbose_name = 'block word'
         verbose_name_plural = 'Blocklist'
 
 

Watcher/Watcher/watcher/settings.py

@@ -114,6 +114,7 @@
     'knox',
     'django.contrib.admindocs',
     'accounts',
+    'import_export',
 ]
 
 REST_FRAMEWORK = {

Watcher/requirements.txt

@@ -18,4 +18,5 @@ shadow-useragent==0.0.17
 thehive4py==1.7.2
 pymisp==2.4.134
 django-auth-ldap==2.2.0
-dnstwist==20201022
+dnstwist==20201022
+django-import-export==2.4.0