load commands

manage.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ssbc.settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

search/admin.py

@@ -0,0 +1,6 @@
+from django.contrib import admin
+from search.models import Hash
+
+# Register your models here.
+admin.site.register(Hash)
+

search/management/commands/loadhash.py

@@ -0,0 +1,43 @@
+#coding: utf8
+from django.core.management.base import BaseCommand
+from django import db as ddb
+from search.models import Hash
+import pymongo
+
+db = pymongo.MongoClient().dht
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        Hash.objects.all().delete()
+        print 'inputing ...'
+        total = db.basic.count()
+        ii = 0
+        ready = []
+        for x in db.basic.find():
+            ii += 1
+            if ii % 10000 == 0:
+                print ii * 100 / total, '%', total - ii
+                Hash.objects.bulk_create(ready)
+                ready = []
+                ddb.reset_queries()
+
+            h = Hash(info_hash = x['info_hash'])
+            h.classified = x.get('classified', False)
+            h.tagged = x.get('tagged', False)
+
+            h.name = unicode(x.get('name',''))[:255]
+            h.category = x.get('category','')[:20]
+            h.extension = x.get('extension', '')[:20]
+            h.data_hash = x.get('data_hash', '')
+            h.comment = x.get('comment','')[:255]
+            h.creator = x.get('creator','')[:20]
+
+            h.length = x.get('length', 0)
+            h.requests = x.get('requests', 0)
+            h.source_ip = x.get('source_ip')
+            h.create_time = x.get('create_time')
+            h.last_seen = x.get('last_seen', h.create_time)
+            ready.append(h)
+
+        if ready:
+            Hash.objects.bulk_create(ready)

search/management/commands/loadlist.py

@@ -0,0 +1,41 @@
+#coding: utf8
+from django.core.management.base import BaseCommand
+from django import db as ddb
+from search.models import FileList
+import pymongo
+import json
+import binascii
+
+db = pymongo.MongoClient().dht
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        #FileList.objects.all().delete()
+        print 'inputing ...'
+        total = db.filelist.count()
+        ii = 0
+        ready = []
+        for x in db.filelist.find().skip(ii):
+            ii += 1
+            if ii % 200 == 0:
+                try:
+                    FileList.objects.bulk_create(ready)
+                except:
+                    for r in ready:
+                        try:
+                            r.save()
+                        except:
+                            import traceback
+                            traceback.print_exc()
+                ready = []
+            if ii % 10000 == 0:
+                print ii * 100 / total, '%', total - ii
+                ddb.reset_queries()
+
+            h = FileList(info_hash = binascii.hexlify(x['_id']))
+            h.file_list = json.dumps(x['files'])
+            ready.append(h)
+
+        if ready:
+            FileList.objects.bulk_create(ready)
+

search/models.py

@@ -0,0 +1,26 @@
+from django.db import models
+
+# Create your models here.
+class Hash(models.Model):
+    info_hash = models.CharField(max_length=40, unique=True)
+    category = models.CharField(max_length=20)
+    data_hash = models.CharField(max_length=32)
+    name = models.CharField(max_length=255)
+    extension = models.CharField(max_length=20)
+    classified = models.BooleanField(default=False)
+    source_ip = models.CharField(max_length=20, null=True)
+    tagged = models.BooleanField(default=False)
+    length = models.BigIntegerField()
+    create_time = models.DateTimeField()
+    last_seen = models.DateTimeField()
+    requests = models.PositiveIntegerField()
+    comment = models.CharField(max_length=255, null=True)
+    creator = models.CharField(max_length=20, null=True)
+
+    def __unicode__(self):
+        return self.name
+
+class FileList(models.Model):
+    info_hash = models.CharField(max_length=40, primary_key=True)
+    file_list = models.TextField()
+

search/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

search/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

ssbc/settings.py

@@ -0,0 +1,109 @@
+"""
+Django settings for ssbc project.
+
+Generated by 'django-admin startproject' using Django 1.8.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.8/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.8/ref/settings/
+"""
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+import os
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'fvewrf=&i9mjawldfkbxt%(oqi%3g1s=18o+n*5b-t4-k&-o=e'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = (
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'search',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+)
+
+ROOT_URLCONF = 'ssbc.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'ssbc.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'ssbc',
+        'HOST': '127.0.0.1',
+        'PORT': 3306,
+        'USER': 'root',
+        'OPTIONS': {
+           "init_command": "SET storage_engine=MYISAM",
+        }  
+    }
+}
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.8/topics/i18n/
+
+LANGUAGE_CODE = 'zh-hans'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = False
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.8/howto/static-files/
+
+STATIC_URL = '/static/'

ssbc/urls.py

@@ -0,0 +1,21 @@
+"""ssbc URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/1.8/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
+Including another URLconf
+    1. Add an import:  from blog import urls as blog_urls
+    2. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls))
+"""
+from django.conf.urls import include, url
+from django.contrib import admin
+
+urlpatterns = [
+    url(r'^admin/', include(admin.site.urls)),
+]

ssbc/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for ssbc project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ssbc.settings")
+
+application = get_wsgi_application()