new: Add CAIDA downloader

.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+env/
 build/
 develop-eggs/
 dist/
@@ -23,7 +24,6 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -45,7 +45,6 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
-.pytest_cache/
 
 # Translations
 *.mo
@@ -54,7 +53,6 @@ coverage.xml
 # Django stuff:
 *.log
 local_settings.py
-db.sqlite3
 
 # Flask stuff:
 instance/
@@ -81,14 +79,13 @@ celerybeat-schedule
 # SageMath parsed files
 *.sage.py
 
-# Environments
+# dotenv
 .env
+
+# virtualenv
 .venv
-env/
 venv/
 ENV/
-env.bak/
-venv.bak/
 
 # Spyder project settings
 .spyderproject
@@ -102,3 +99,14 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+# Redis
+*.rdb
+
+# Storage
+rawdata
+
+# ardb
+storage/ardb.pid
+storage/data
+storage/repl

bin/caida_dl.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pathlib import Path
+import logging
+from dateutil.relativedelta import relativedelta
+import asyncio
+from datetime import date
+import aiohttp
+
+from ipasnhistory.abstractmanager import AbstractManager
+from ipasnhistory.caida_downloader import CaidaDownloader
+from ipasnhistory.libs.helpers import get_homedir
+
+logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
+                    level=logging.WARNING, datefmt='%I:%M:%S')
+
+
+class CaidaManager(AbstractManager):
+
+    def __init__(self, storage_directory: Path=None, loglevel: int=logging.WARNING):
+        super().__init__(loglevel)
+        if not storage_directory:
+            self.storage_directory = get_homedir() / 'rawdata'
+        self.downloader = CaidaDownloader(self.storage_directory, loglevel)
+        # Download last year data.
+        loop = asyncio.get_event_loop()
+        last_year = date.today() - relativedelta(years=1)
+        v4 = asyncio.ensure_future(self.downloader.find_routes('v4', first_date=last_year))
+        v6 = asyncio.ensure_future(self.downloader.find_routes('v6', first_date=last_year))
+        loop.run_until_complete(asyncio.gather(v4, v6, return_exceptions=True))
+
+    def _to_run_forever(self):
+        try:
+            loop = asyncio.get_event_loop()
+            tasks = []
+            for address_family in ['v4', 'v6']:
+                task = self.downloader.download_latest(address_family)
+                tasks.append(task)
+            loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
+        except aiohttp.client_exceptions.ClientConnectorError as e:
+            self.logger.critical(f'Error while fetching a routeview file: {e}')
+
+
+if __name__ == '__main__':
+    m = CaidaManager()
+    m.run(sleep_in_sec=3600)

bin/run_backend.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from ipasnhistory.libs.helpers import get_homedir, check_running
+from subprocess import Popen
+import time
+from pathlib import Path
+
+import argparse
+
+
+def launch_cache(storage_directory: Path=None):
+    if not storage_directory:
+        storage_directory = get_homedir()
+    if not check_running('cache'):
+        Popen(["./run_redis.sh"], cwd=(storage_directory / 'cache'))
+
+
+def shutdown_cache(storage_directory: Path=None):
+    if not storage_directory:
+        storage_directory = get_homedir()
+    Popen(["./shutdown_redis.sh"], cwd=(storage_directory / 'cache'))
+
+
+def launch_storage(storage_directory: Path=None):
+    if not storage_directory:
+        storage_directory = get_homedir()
+    if not check_running('storage'):
+        Popen(["./run_ardb.sh"], cwd=(storage_directory / 'storage'))
+
+
+def shutdown_storage(storage_directory: Path=None):
+    if not storage_directory:
+        storage_directory = get_homedir()
+    Popen(["./shutdown_ardb.sh"], cwd=(storage_directory / 'storage'))
+
+
+def launch_all():
+    launch_cache()
+    launch_storage()
+
+
+def check_all(stop=False):
+    backends = [['cache', False], ['storage', False]]
+    while True:
+        for b in backends:
+            try:
+                b[1] = check_running(b[0])
+            except Exception:
+                b[1] = False
+        if stop:
+            if not any(b[1] for b in backends):
+                break
+        else:
+            if all(b[1] for b in backends):
+                break
+        for b in backends:
+            if not stop and not b[1]:
+                print(f"Waiting on {b[0]}")
+            if stop and b[1]:
+                print(f"Waiting on {b[0]}")
+        time.sleep(1)
+
+
+def stop_all():
+    shutdown_cache()
+    shutdown_storage()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Manage backend DBs.')
+    parser.add_argument("--start", action='store_true', default=False, help="Start all")
+    parser.add_argument("--stop", action='store_true', default=False, help="Stop all")
+    parser.add_argument("--status", action='store_true', default=True, help="Show status")
+    args = parser.parse_args()
+
+    if args.start:
+        launch_all()
+    if args.stop:
+        stop_all()
+    if not args.stop and args.status:
+        check_all()

bin/shutdown.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from ipasnhistory.libs.helpers import is_running, get_socket_path
+import time
+from redis import StrictRedis
+
+if __name__ == '__main__':
+    r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
+    r.set('shutdown', 1)
+    while True:
+        running = is_running()
+        print(running)
+        if not running:
+            break
+        time.sleep(10)

bin/start.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from subprocess import Popen
+from ipasnhistory.libs.helpers import get_homedir
+
+
+if __name__ == '__main__':
+    # Just fail if the env isn't set.
+    get_homedir()
+    p = Popen(['run_backend.py', '--start'])
+    p.wait()
+    Popen(['caida_dl.py'])

bin/stop.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from subprocess import Popen
+from ipasnhistory.libs.helpers import get_homedir
+
+if __name__ == '__main__':
+    get_homedir()
+    p = Popen(['shutdown.py'])
+    p.wait()
+    Popen(['run_backend.py', '--stop'])