update project

CrawlerAndValidator/CymonCrawler.py

@@ -6,30 +6,31 @@
 
 
 def crawl(days, tags, adrtypes):
-        for day in days:
-            for tag in tags:
-                for adrtype in adrtypes:
-                    getCymonPage(str(day), tag, adrtype, '10', '0')
-                    pass
+    print 'Crawling from cymon.io...'
+    print 'j'
+    for day in days:
+        for tag in tags:
+            for adrtype in adrtypes:
+                getCymonPage(str(day), tag, adrtype, '100000', '0')
+
 
-def getCymonPage( day, tag, adrtype, limit, offset):        
-        print 'Crawling from cymon.io...'
+def getCymonPage( day, tag, adrtype, limit, offset):                
         url = 'https://cymon.io/api/nexus/v1/blacklist/'
         url += adrtype + '/' + tag
         url += '/?days=' + day
         url += '&limit=' + limit
         url += '&offset=' + offset
-
-        print url
+        print 'URL: ', url
         r = requests.get(url)
         raw = json.loads(r.text)
-        result = raw['results']        
-        print 'Count : ', len(result)
+        result = raw['results']                
         for item in result:
+            print '.'
+
             ShowProgress.show(result.index(item) + 1, len(result))
             if not ('addr' in item.keys()):
                 item['addr'] = item['name']
             DBHelper.updateAddress({'address': item['addr'], 
                                     'address_type': adrtype})
-
-                                    
+            print '\b'
+

CrawlerAndValidator/DBHelper.py

@@ -20,8 +20,7 @@
 db = client['CymonAV']
 collection = db['addresses']
 
-def updateAddress(item):
-    print 'Updating item ', item['address']
+def updateAddress(item):    
     row = collection.find_one({'address': item['address']})
 
 
@@ -41,7 +40,6 @@ def updateAddress(item):
     noDup = {v['value']:v for v in item['detections']}.values()
     collection.update({'address' : item['address']},
                       {"$set":{'detections': noDup}})
-    print 'New size : ', len(item['detections'])
 
 def getAddresses():
     result = []

CrawlerAndValidator/DailyCrawler.py

@@ -5,12 +5,20 @@
 
 
 def crawl(delta_time=1):
-#     cymondelta = min(4, 1 + delta_time)
-#     CymonCrawler.crawl(range(1,), ['malware'], ['ip', 'domain'])
+    crawlCymon(delta_time)
+    crawAlienVault()
+    validateAll()
 
+
+def crawlCymon(delta_time):
+    cymondelta = min(4, 1 + delta_time)
+    CymonCrawler.crawl(range(1,), ['malware'], ['ip', 'domain'])    
+
+def crawAlienVault():
     today = datetime.datetime.today().strftime("%Y-%m-%d")
-    AVcrawler.crawl(today)
+    AVcrawler.crawl(today)    
 
-    VirusTotalValidater.validateAll()
+def validateAll():
+    VirusTotalValidater.validateAll()    
 
 crawl()

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import setup
 
-setup(name='Crawler and Validator',
+setup(name='CrawlerAndValidator',
       version='1.0',
       description='Crawler and Validator to work with informations from AllienVault and Cymon',
       author='BigForce',