Fix bug

AlexeyManikin · May 31, 2016 · 7ab4815 · 7ab4815
1 parent ca98f2d
commit 7ab4815
Show file tree

Hide file tree

Showing 11 changed files with 57 additions and 38 deletions.
diff --git a/.gitignore b/.gitignore
@@ -55,3 +55,6 @@ docs/_build/
 
 # PyBuilder
 target/
+
+docker/counteiner_data/download
+docker/counteiner_data/datadir
diff --git a/README.md b/README.md
@@ -36,20 +36,17 @@
 * docker-compose up -d
 
 
-
-Далее каждую ночь база данных доменов будет обновляться. На двух процессорах E5-2620v3 процесс обновления БД занимает 4-5 часов.
+Далее каждую ночь база данных доменов будет обновляться. На двух 
+процессорах E5-2620v3 с 32 гигабайтами памяти процесс обновления БД з
+анимает 11-12 часов, плюс еще несколько часов агрегирование данных.
 
 # Тестовый доступ к БД
 
- Сервер: manikin.beget.ru
-
- Порт: 3310
-
- Пользователь: amanikin_stat
-
- Пароль: openstatistic
-
- База: domain_statistic
+* Сервер: manikin.beget.ru
+* Порт: 3310
+* Пользователь: amanikin_stat
+* Пароль: openstatistic
+* База: domain_statistic
 
 Если кто не умеет пользоваться консолью, установил PhpMyAdmin: http://pma.amanikin.ru
 
@@ -59,3 +56,5 @@
 ![example](https://scontent.xx.fbcdn.net/hphotos-xpt1/t31.0-8/11779902_855515371153091_8587193411725580989_o.png)
 
 
+
+
diff --git a/classes/asInet.py b/classes/asInet.py
@@ -9,6 +9,7 @@
 import pprint
 import urllib2
 import traceback
+import time
 
 from helpers.helperUnicode import as_default_string
 from helpers.helpers import get_mysql_connection
@@ -166,6 +167,7 @@ def update_as(self, number, as_data, show_log=False):
                        'USE_FAST': 1}
         else:
             try:
+                time.sleep(1)
                 as_info = self._get_asn_description(number)
             except:
                 as_info = {'AS': number,

diff --git a/classes/statistic.py b/classes/statistic.py
@@ -7,6 +7,7 @@
 from config.main import START_YEAR, START_MONTH, START_DAY, PREFIX_LIST, MINIMUM_DOMAIN_COUNT
 import datetime
 import MySQLdb
+import pprint
 
 
 class Statistic(object):
@@ -22,7 +23,6 @@ class Statistic(object):
     # ns_domain_old_count_statistic
     # a_domain_old_count_statistic
 
-
     def __init__(self):
         """
         :return:
@@ -84,8 +84,9 @@ def update_domain_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('domain')
 
-        for prefix in PREFIX_LIST:
-            self._update_domain_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_domain_count_per_zone(date, today, prefix)
 
     def _update_as_count_per_zone(self, date, today, zone):
         """
@@ -141,8 +142,9 @@ def update_as_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('as')
 
-        for prefix in PREFIX_LIST:
-            self._update_as_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_as_count_per_zone(date, today, prefix)
 
     def _update_mx_count_per_zone(self, date, today, zone):
         """
@@ -192,8 +194,9 @@ def update_mx_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('mx')
 
-        for prefix in PREFIX_LIST:
-            self._update_mx_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_mx_count_per_zone(date, today, prefix)
 
     def _update_ns_count_per_zone(self, date, today, zone):
         """
@@ -243,8 +246,9 @@ def update_ns_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('ns')
 
-        for prefix in PREFIX_LIST:
-            self._update_ns_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_ns_count_per_zone(date, today, prefix)
 
     def _update_registrant_count_per_zone(self, date, today, zone):
         """
@@ -285,8 +289,9 @@ def update_registrant_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('registrant')
 
-        for prefix in PREFIX_LIST:
-            self._update_registrant_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_registrant_count_per_zone(date, today, prefix)
 
     def _update_a_count_per_zone(self, date, today, zone):
         """
@@ -350,8 +355,9 @@ def update_a_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('a')
 
-        for prefix in PREFIX_LIST:
-            self._update_a_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_a_count_per_zone(date, today, prefix)
 
     def _update_cname_count_per_zone(self, date, today, zone):
         """
@@ -392,8 +398,9 @@ def update_cname_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('cname')
 
-        for prefix in PREFIX_LIST:
-            self._update_cname_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_cname_count_per_zone(date, today, prefix)
 
     def _update_as_domain_old_count_per_zone(self, date, today, zone):
             """
@@ -437,8 +444,9 @@ def update_as_domain_old_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('as_domain_old')
 
-        for prefix in PREFIX_LIST:
-            self._update_as_domain_old_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_as_domain_old_count_per_zone(date, today, prefix)
 
     def _update_ns_domain_old_count_per_zone(self, date, today, zone):
         """
@@ -500,8 +508,9 @@ def update_ns_domain_old_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('ns_domain_old')
 
-        for prefix in PREFIX_LIST:
-            self._update_ns_domain_old_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_ns_domain_old_count_per_zone(date, today, prefix)
 
     def _update_a_domain_old_count_per_zone(self, date, today, zone):
         """
@@ -545,8 +554,9 @@ def update_a_domain_old_count_statistic(self):
         today = datetime.date.today()
         date = self.get_date_after_without_statistic('a_domain_old')
 
-        for prefix in PREFIX_LIST:
-            self._update_a_domain_old_count_per_zone(date, today, prefix)
+        if date is not None:
+            for prefix in PREFIX_LIST:
+                self._update_a_domain_old_count_per_zone(date, today, prefix)
 
     def update_all_statistic(self):
         """

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -2,6 +2,8 @@ mysql:
   build: mysql/
   name: "mysql_open_domain_statistic"
   mem_limit: "20G"
+  ports:
+    - "3311:33060"
   volumes:
     - counteiner_data/datadir/:/home/mysql:rw
     - mysql/config/my.cnf:/etc/mysql/my.cnf:rw

diff --git a/docker/runner/Dockerfile b/docker/runner/Dockerfile
@@ -20,6 +20,8 @@ RUN \
   pip install dnspython && \
   pip install --upgrade psutil && \
   apt-get clean && \
+  rm -rf /etc/localtime && \
+  ln -s /usr/share/zoneinfo/Europe/Moscow /etc/localtime && \
   rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/*;
 
 ADD install_minit.sh /usr/local/sbin/install_minit.sh

diff --git a/docker/runner/config/authorized_keys b/docker/runner/config/authorized_keys
@@ -1 +1,2 @@
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDkUCjIHm0gza7pDGedAL2adWDxauepelJZC5FNbVeTk7K/2yOwPRH8uu1tqV5ygMAuZk/pO+U7UKTXxoyxhNlq4EtjfqaHFIjp/y3ZjgTmHWJUItsDCGwvjFZsN8yE0hBGX7cc8Lh9/S/NJ+CPKuW8TcAUWoSxnMAq4tRCcQKc+SWglmAFyyWJRx2FGhh4yKQSUQwrT55sVz8Pl/+n5kIT9myY7U0esc/fAE8Gxuy19vxSjkvrutsBrp5mHU1jRGTd6xhkPrXNjUJse4jCy+/OzgkOC+p8gqsIxQhPAUR5d9UxuFpCdC4IbtLmovwB1BYTDwWvF8mCfcE37RAamlZp alekseymanikin@MacBook-Pro-Aleksej.local
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDkUCjIHm0gza7pDGedAL2adWDxauepelJZC5FNbVeTk7K/2yOwPRH8uu1tqV5ygMAuZk/pO+U7UKTXxoyxhNlq4EtjfqaHFIjp/y3ZjgTmHWJUItsDCGwvjFZsN8yE0hBGX7cc8Lh9/S/NJ+CPKuW8TcAUWoSxnMAq4tRCcQKc+SWglmAFyyWJRx2FGhh4yKQSUQwrT55sVz8Pl/+n5kIT9myY7U0esc/fAE8Gxuy19vxSjkvrutsBrp5mHU1jRGTd6xhkPrXNjUJse4jCy+/OzgkOC+p8gqsIxQhPAUR5d9UxuFpCdC4IbtLmovwB1BYTDwWvF8mCfcE37RAamlZp alekseymanikin@MacBook-Pro-Aleksej.local
diff --git a/docker/runner/config/root b/docker/runner/config/root
@@ -1,4 +1,4 @@
-10 1 */5 * * /usr/bin/python /home/domain_statistic/update_as_info.py >> /home/domain_statistic/download/update_as_info.log
+#10 1 */5 * * /usr/bin/python /home/domain_statistic/update_as_info.py >> /home/domain_statistic/download/update_as_info.log
 30 0 * * * /usr/bin/python /home/domain_statistic/update_domain.py -n `cat /etc/resolv.conf | awk '{print $2}'` >> /home/domain_statistic/download/update_domain.log
 30 12 * * * /usr/bin/python /home/domain_statistic/update_regru.py >> /home/domain_statistic/download/update_regru.log
 30 12 * * * /usr/bin/python /home/domain_statistic/update_statistic.py
diff --git a/migration/version4.sql b/migration/version4.sql
@@ -8,7 +8,7 @@ CREATE TABLE `as_domain_old_count_statistic` (
   KEY `date` (`date`),
   KEY `asn` (`asn`),
   KEY `asn_tld` (`asn`, `tld`),
-  KEY `date_tld` (`date`, `tld`),
+  KEY `date_tld` (`date`, `tld`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
 
 CREATE TABLE `ns_domain_old_count_statistic` (
@@ -21,7 +21,7 @@ CREATE TABLE `ns_domain_old_count_statistic` (
   KEY `date` (`date`),
   KEY `ns` (`ns`),
   KEY `ns_tld` (`ns`, `tld`),
-  KEY `date_tld` (`date`, `tld`),
+  KEY `date_tld` (`date`, `tld`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
 
 CREATE TABLE `a_domain_old_count_statistic` (
@@ -34,5 +34,5 @@ CREATE TABLE `a_domain_old_count_statistic` (
   KEY `date` (`date`),
   KEY `a` (`a`),
   KEY `a_tld` (`a`, `tld`),
-  KEY `date_tld` (`date`, `tld`),
+  KEY `date_tld` (`date`, `tld`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
diff --git a/structure.sql b/structure.sql
@@ -375,5 +375,5 @@ CREATE TABLE `a_domain_old_count_statistic` (
   KEY `date` (`date`),
   KEY `a` (`a`),
   KEY `a_tld` (`a`, `tld`),
-  KEY `date_tld` (`date`, `tld`),
+  KEY `date_tld` (`date`, `tld`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
diff --git a/update_as_info.py b/update_as_info.py
@@ -19,7 +19,7 @@
 if __name__ == "__main__":
     try:
         as_parser = AsInet()
-        as_parser.parsing_as(show_log=False)
+        as_parser.parsing_as(show_log=True)
     except Exception as e:
         print "Got an exception: %s" % e.message
         print traceback.format_exc()