Skip to content

Commit

Permalink
provide flexible logging capability
Browse files Browse the repository at this point in the history 
* introduce various loggers in settings/base.py
* differentiate log levels in dev/prod
  • Loading branch information
@chamalis
chamalis committed Jun 6, 2018
1 parent 03a5876 commit 6e174fd
Show file tree
Hide file tree
Showing 7 changed files with 194 additions and 16 deletions.
30 changes: 30 additions & 0 deletions ahmia/ahmia/settings/base.py
View file
Expand Up @@ -125,3 +125,33 @@ def my_path(*x):
'search',
'stats'
)

# Log everything to the logs directory at the top
LOGFILE_ROOT = my_path('logs')
print(LOGFILE_ROOT)

# Disable automatic default configuration process to apply our own settings
LOGGING_CONFIG = None

# Logging
LOG_LEVEL = config('LOG_LEVEL', default='INFO') # Debug, Info, Warning, Error, Critical

# Common settings ~ This dict is being updated inside dev.py / prod.py
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': "[%(asctime)s] %(levelname)s [%(name)s:%(filename)s:%(lineno)s] %(message)s",
'datefmt': "%d/%b/%Y %H:%M:%S"
},
'simple': {
'format': '%(levelname)s %(message)s'
},
},
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse',
}
},
}
71 changes: 69 additions & 2 deletions ahmia/ahmia/settings/dev.py
View file
@@ -1,3 +1,4 @@
import logging.config
import sys

from .base import *
Expand All @@ -9,13 +10,79 @@
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Database engine
'NAME': join(PROJECT_HOME, 'ahmia_db_test') # Database name
'NAME': my_path('ahmia_db_test') # Database name
}
}
else:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Database engine
'NAME': join(PROJECT_HOME, 'ahmia_db') # Database name
'NAME': my_path('ahmia_db') # Database name
}
}

# additionally to default LOGGING settings from base.py
LOGGING.update({
'handlers': {
'django_file': {
'level': config('LOG_LEVEL', default='INFO'),
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'django.log'),
'formatter': 'verbose'
},
'ahmia_file': {
'level': config('LOG_LEVEL', default='DEBUG'),
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'ahmia.log'),
'formatter': 'verbose'
},
'search_file': {
'level': config('LOG_LEVEL', default='DEBUG'),
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'search.log'),
'formatter': 'verbose'
},
'stats_file': {
'level': config('LOG_LEVEL', default='DEBUG'),
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'stats.log'),
'formatter': 'verbose'
},
'console': {
'level': config('LOG_LEVEL', default='DEBUG'),
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler',
'filters': ['require_debug_false'], # dont spam email while debugging
'formatter': 'verbose'
}
},
'loggers': {
# root is the catch-all logger
'': {
'handlers': ['django_file', 'console', 'mail_admins'],
'level': config('LOG_LEVEL', default='INFO'),
},
# our own-defined loggers
'ahmia': {
'handlers': ['ahmia_file', 'console', 'mail_admins'],
'level': config('LOG_LEVEL', default='DEBUG'),
'propagate': False
},
'search': {
'handlers': ['search_file', 'console', 'mail_admins'],
'level': config('LOG_LEVEL', default='DEBUG'),
'propagate': False
},
'stats': {
'handlers': ['stats_file', 'console', 'mail_admins'],
'level': config('LOG_LEVEL', default='DEBUG'),
'propagate': False
}
}
})

logging.config.dictConfig(LOGGING)
69 changes: 69 additions & 0 deletions ahmia/ahmia/settings/prod.py
View file
@@ -1,3 +1,5 @@
import logging.config

from .base import *


Expand All @@ -22,3 +24,70 @@
}

DEPLOYMENT_DIR = config('DEPLOYMENT_DIR', default='/usr/local/lib/ahmia-site/ahmia/')

# additionally to default LOGGING settings from base.py
LOGGING.update({
'handlers': {
'django_file': {
'level': 'ERROR',
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'django.log'),
'formatter': 'verbose'
},
'ahmia_file': {
'level': 'WARNING',
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'ahmia.log'),
'formatter': 'verbose'
},
'search_file': {
'level': 'WARNING',
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'search.log'),
'formatter': 'verbose'
},
'stats_file': {
'level': 'WARNING',
'class': 'logging.handlers.WatchedFileHandler', # log rotation with logrotate
'filename': join(LOGFILE_ROOT, 'stats.log'),
'formatter': 'verbose'
},
# in production console is usually handled by the webserver
'console': {
'level': config('LOG_LEVEL', default='INFO'),
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
'mail_admins': {
'level': 'ERROR',
'class': 'django.utils.log.AdminEmailHandler',
'filters': ['require_debug_false'], # dont spam email while debugging
'formatter': 'verbose'
}
},
'loggers': {
# root is the catch-all logger
'': {
'handlers': ['django_file', 'console', 'mail_admins'],
'level': 'WARNING',
},
# our own-defined logger
'ahmia': {
'handlers': ['ahmia_file', 'console', 'mail_admins'],
'level': config('LOG_LEVEL', default='INFO'),
'propagate': False
},
'search': {
'handlers': ['search_file', 'console', 'mail_admins'],
'level': config('LOG_LEVEL', default='INFO'),
'propagate': False
},
'stats': {
'handlers': ['stats_file', 'console', 'mail_admins'],
'level': config('LOG_LEVEL', default='INFO'),
'propagate': False
}
}
})

logging.config.dictConfig(LOGGING)
2 changes: 2 additions & 0 deletions ahmia/ahmia/utils.py
View file
Expand Up @@ -5,6 +5,8 @@

def get_elasticsearch_object():
""" Creating an elasticsearch object to query the index """
# todo move the default values to settings instead of using exception handling

try:
es_servers = settings.ELASTICSEARCH_SERVERS
es_servers = es_servers if isinstance(es_servers, list) \
Expand Down
15 changes: 10 additions & 5 deletions ahmia/ahmia/views.py
View file
Expand Up @@ -4,6 +4,7 @@
These pages does not require database connection.
"""
import hashlib
import logging
from operator import itemgetter

from django.db import IntegrityError
Expand All @@ -18,6 +19,8 @@
from .models import HiddenWebsite
from .forms import AddOnionForm, ReportOnionForm

logger = logging.getLogger(__name__)


class CoreView(TemplateView):
"""Core page of the website."""
Expand Down Expand Up @@ -71,7 +74,7 @@ class GsocView(CoreView):

class AddView(TemplateView):
"""Add form for a new .onion address."""
# todo distinguish between failure and already exists case?
# todo distinguish between ORM failure (except) case and already exists (outer else) case

form_class = AddOnionForm
template_name = "add.html"
Expand All @@ -81,15 +84,17 @@ class AddView(TemplateView):
def post(self, request):
if request.method == "POST":
domain = AddOnionForm(request.POST)
onion_url = request.POST.get('onion', '').strip()
if domain.is_valid():
onion = request.POST.get('onion', '').strip()
onion = HiddenWebsite(onion=onion)
onion = HiddenWebsite(onion=onion_url)
try:
onion.save()
except IntegrityError: # probably already exists
pass
except IntegrityError as e: # error saving to DB
logger.exception(e)
else:
return render(request, self.successpage) # redirect('/add/success')
else:
logger.info("Domain {} already exists or invalid".format(onion_url))

return render(request, self.failpage)

Expand Down
9 changes: 5 additions & 4 deletions ahmia/search/views.py
View file
Expand Up @@ -2,6 +2,7 @@
Views
Full text search views.
"""
import logging
import math
import time
from datetime import date, datetime
Expand All @@ -13,6 +14,8 @@
from ahmia.models import SearchResultsClicks
from ahmia.views import ElasticsearchBaseListView

logger = logging.getLogger(__name__)


def onion_redirect(request):
"""Add clicked information and redirect to .onion address."""
Expand All @@ -30,7 +33,7 @@ def onion_redirect(request):
_, _ = SearchResultsClicks.objects.get_or_create(
onionDomain=onion, clicked=redirect_url, searchTerm=search_term)
except Exception as error:
print("Error with redirect URL: {0}\n{1}".format(redirect_url, error))
logger.error("Error with redirect URL: {0}\n{1}".format(redirect_url, error))

message = "Redirecting to hidden service."
return redirect_page(message, 0, redirect_url)
Expand Down Expand Up @@ -154,9 +157,7 @@ def format_hits(self, hits):
for res in results:
try:
res['anchors'] = res['anchors'][0]
except KeyError:
pass
except TypeError:
except (KeyError, TypeError):
pass
res['updated_on'] = datetime.strptime(res['updated_on'],
'%Y-%m-%dT%H:%M:%S')
Expand Down
14 changes: 9 additions & 5 deletions ahmia/stats/link_graph.py
View file
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""Download data from Elasticsearch and convert it to Gexf/XML format."""
import logging
import time
from random import randint

Expand All @@ -11,15 +12,17 @@
except ImportError:
from urlparse import urlparse

logger = logging.getLogger(__name__)


def query(graph, es, color):
"""Make query and create a graph."""
# Status of doc_type in the READ onion index
index = "crawl"
index = "latest-crawl"
q = 'links.link:*'
res = es.search(index=index, q=q)
size = res['hits']['total']
print "READ Index onions size in this range is %d" % size
print("READ Index onions size in this range is %d" % size)
start = 0
limit = 1
# added = 0
Expand All @@ -29,9 +32,10 @@ def query(graph, es, color):
try:
res = es.search(index=index, from_=start, size=limit, q=q)
except Exception as e:
print e
logger.exception(e)
continue
print "range=%d-%d, hits %d" % (start, start+limit, len(res['hits']['hits']))

print("range=%d-%d, hits %d" % (start, start+limit, len(res['hits']['hits'])))
for hit in res['hits']['hits']:
item = hit["_source"]
graph.add_node(item["domain"])
Expand All @@ -57,7 +61,7 @@ def use_data(es):
graph = nx.Graph()
color = {'r': 0, 'g': 255, 'b': 0, 'a': 0.8}
query(graph, es, color)
print "The number of nodes %d" % len(graph.nodes())
print("The number of nodes %d" % len(graph.nodes()))
nx.write_gexf(graph, "onionlinks.gexf", encoding='utf-8', prettyprint=True, version='1.2draft')


Expand Down

0 comments on commit 6e174fd

Please sign in to comment.