From ccc0ed30e2f37e17a10ad70df050a86a833470e9 Mon Sep 17 00:00:00 2001 From: Xiaoxia Date: Sat, 9 May 2015 02:39:46 +0800 Subject: [PATCH] load commands --- manage.py | 10 +++ search/__init__.py | 0 search/admin.py | 6 ++ search/management/__init__.py | 0 search/management/commands/__init__.py | 0 search/management/commands/loadhash.py | 43 ++++++++++ search/management/commands/loadlist.py | 41 ++++++++++ search/models.py | 26 ++++++ search/tests.py | 3 + search/views.py | 3 + ssbc/__init__.py | 0 ssbc/settings.py | 109 +++++++++++++++++++++++++ ssbc/urls.py | 21 +++++ ssbc/wsgi.py | 16 ++++ 14 files changed, 278 insertions(+) create mode 100755 manage.py create mode 100644 search/__init__.py create mode 100644 search/admin.py create mode 100644 search/management/__init__.py create mode 100644 search/management/commands/__init__.py create mode 100644 search/management/commands/loadhash.py create mode 100644 search/management/commands/loadlist.py create mode 100644 search/models.py create mode 100644 search/tests.py create mode 100644 search/views.py create mode 100644 ssbc/__init__.py create mode 100644 ssbc/settings.py create mode 100644 ssbc/urls.py create mode 100644 ssbc/wsgi.py diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..bce66f5 --- /dev/null +++ b/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ssbc.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/search/__init__.py b/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/search/admin.py b/search/admin.py new file mode 100644 index 0000000..9eb6599 --- /dev/null +++ b/search/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from search.models import Hash + +# Register your models here. +admin.site.register(Hash) + diff --git a/search/management/__init__.py b/search/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/search/management/commands/__init__.py b/search/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/search/management/commands/loadhash.py b/search/management/commands/loadhash.py new file mode 100644 index 0000000..ff64ab7 --- /dev/null +++ b/search/management/commands/loadhash.py @@ -0,0 +1,43 @@ +#coding: utf8 +from django.core.management.base import BaseCommand +from django import db as ddb +from search.models import Hash +import pymongo + +db = pymongo.MongoClient().dht + +class Command(BaseCommand): + def handle(self, *args, **options): + Hash.objects.all().delete() + print 'inputing ...' + total = db.basic.count() + ii = 0 + ready = [] + for x in db.basic.find(): + ii += 1 + if ii % 10000 == 0: + print ii * 100 / total, '%', total - ii + Hash.objects.bulk_create(ready) + ready = [] + ddb.reset_queries() + + h = Hash(info_hash = x['info_hash']) + h.classified = x.get('classified', False) + h.tagged = x.get('tagged', False) + + h.name = unicode(x.get('name',''))[:255] + h.category = x.get('category','')[:20] + h.extension = x.get('extension', '')[:20] + h.data_hash = x.get('data_hash', '') + h.comment = x.get('comment','')[:255] + h.creator = x.get('creator','')[:20] + + h.length = x.get('length', 0) + h.requests = x.get('requests', 0) + h.source_ip = x.get('source_ip') + h.create_time = x.get('create_time') + h.last_seen = x.get('last_seen', h.create_time) + ready.append(h) + + if ready: + Hash.objects.bulk_create(ready) diff --git a/search/management/commands/loadlist.py b/search/management/commands/loadlist.py new file mode 100644 index 0000000..2e3d046 --- /dev/null +++ b/search/management/commands/loadlist.py @@ -0,0 +1,41 @@ +#coding: utf8 +from django.core.management.base import BaseCommand +from django import db as ddb +from search.models import FileList +import pymongo +import json +import binascii + +db = pymongo.MongoClient().dht + +class Command(BaseCommand): + def handle(self, *args, **options): + #FileList.objects.all().delete() + print 'inputing ...' + total = db.filelist.count() + ii = 0 + ready = [] + for x in db.filelist.find().skip(ii): + ii += 1 + if ii % 200 == 0: + try: + FileList.objects.bulk_create(ready) + except: + for r in ready: + try: + r.save() + except: + import traceback + traceback.print_exc() + ready = [] + if ii % 10000 == 0: + print ii * 100 / total, '%', total - ii + ddb.reset_queries() + + h = FileList(info_hash = binascii.hexlify(x['_id'])) + h.file_list = json.dumps(x['files']) + ready.append(h) + + if ready: + FileList.objects.bulk_create(ready) + diff --git a/search/models.py b/search/models.py new file mode 100644 index 0000000..ad2af21 --- /dev/null +++ b/search/models.py @@ -0,0 +1,26 @@ +from django.db import models + +# Create your models here. +class Hash(models.Model): + info_hash = models.CharField(max_length=40, unique=True) + category = models.CharField(max_length=20) + data_hash = models.CharField(max_length=32) + name = models.CharField(max_length=255) + extension = models.CharField(max_length=20) + classified = models.BooleanField(default=False) + source_ip = models.CharField(max_length=20, null=True) + tagged = models.BooleanField(default=False) + length = models.BigIntegerField() + create_time = models.DateTimeField() + last_seen = models.DateTimeField() + requests = models.PositiveIntegerField() + comment = models.CharField(max_length=255, null=True) + creator = models.CharField(max_length=20, null=True) + + def __unicode__(self): + return self.name + +class FileList(models.Model): + info_hash = models.CharField(max_length=40, primary_key=True) + file_list = models.TextField() + diff --git a/search/tests.py b/search/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/search/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/search/views.py b/search/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/search/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/ssbc/__init__.py b/ssbc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ssbc/settings.py b/ssbc/settings.py new file mode 100644 index 0000000..c0c2859 --- /dev/null +++ b/ssbc/settings.py @@ -0,0 +1,109 @@ +""" +Django settings for ssbc project. + +Generated by 'django-admin startproject' using Django 1.8.1. + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'fvewrf=&i9mjawldfkbxt%(oqi%3g1s=18o+n*5b-t4-k&-o=e' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = ( + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'search', +) + +MIDDLEWARE_CLASSES = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', +) + +ROOT_URLCONF = 'ssbc.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'ssbc.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'ssbc', + 'HOST': '127.0.0.1', + 'PORT': 3306, + 'USER': 'root', + 'OPTIONS': { + "init_command": "SET storage_engine=MYISAM", + } + } +} + + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'zh-hans' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = False + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.8/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/ssbc/urls.py b/ssbc/urls.py new file mode 100644 index 0000000..71e9c14 --- /dev/null +++ b/ssbc/urls.py @@ -0,0 +1,21 @@ +"""ssbc URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.8/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Add an import: from blog import urls as blog_urls + 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) +""" +from django.conf.urls import include, url +from django.contrib import admin + +urlpatterns = [ + url(r'^admin/', include(admin.site.urls)), +] diff --git a/ssbc/wsgi.py b/ssbc/wsgi.py new file mode 100644 index 0000000..f596e9d --- /dev/null +++ b/ssbc/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for ssbc project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ssbc.settings") + +application = get_wsgi_application()