Merge pull request #185 from FuelRats/drillsqueak

Mecha 2.3.0.3

ratlib/api/names.py

@@ -2,6 +2,7 @@
 import ratlib.api
 import ratlib.api.http
 import functools
+from sopel.module import NOLIMIT
 
 urljoin = ratlib.api.http.urljoin
 savedratids = {}
@@ -213,6 +214,7 @@ def guarded(bot, trigger, *args, **kwargs):
             if getPrivLevel(trigger)<6:
                 if message and not callable(message):
                     bot.say(message)
+                    return NOLIMIT
             else:
                 return function(bot, trigger, *args, **kwargs)
         return guarded
@@ -230,6 +232,7 @@ def guarded(bot, trigger, *args, **kwargs):
             if getPrivLevel(trigger)<5:
                 if message and not callable(message):
                     bot.say(message)
+                    return NOLIMIT
             else:
                 return function(bot, trigger, *args, **kwargs)
         return guarded
@@ -247,6 +250,7 @@ def guarded(bot, trigger, *args, **kwargs):
             if getPrivLevel(trigger)<4:
                 if message and not callable(message):
                     bot.say(message)
+                    return NOLIMIT
             else:
                 return function(bot, trigger, *args, **kwargs)
         return guarded
@@ -264,6 +268,7 @@ def guarded(bot, trigger, *args, **kwargs):
             if getPrivLevel(trigger)<3:
                 if message and not callable(message):
                     bot.say(message)
+                    return NOLIMIT
             else:
                 return function(bot, trigger, *args, **kwargs)
         return guarded
@@ -281,6 +286,7 @@ def guarded(bot, trigger, *args, **kwargs):
             if getPrivLevel(trigger)<2:
                 if message and not callable(message):
                     bot.say(message)
+                    return NOLIMIT
             else:
                 return function(bot, trigger, *args, **kwargs)
         return guarded
@@ -298,6 +304,7 @@ def guarded(bot, trigger, *args, **kwargs):
             if getPrivLevel(trigger)<1:
                 if message and not callable(message):
                     bot.say(message)
+                    return NOLIMIT
             else:
                 return function(bot, trigger, *args, **kwargs)
         return guarded
@@ -315,6 +322,7 @@ def guarded(bot, trigger, *args, **kwargs):
             if getPrivLevel(trigger)<0:
                 if message and not callable(message):
                     bot.say(message)
+                    return NOLIMIT
             else:
                 return function(bot, trigger, *args, **kwargs)
         return guarded
@@ -330,6 +338,8 @@ def getPrivLevel(trigger):
         return 9
     if trigger.admin:
         return 8
+    elif str(trigger.host).endswith('techrat.fuelrats.com'):
+        return privlevels.get('techrat.fuelrats.com')
     else:
         for key in privlevels.keys():
             if str(trigger.host).endswith(key):

ratlib/literalstatement.py

@@ -0,0 +1,48 @@
+# BIG thanks to stackoverflow. Use ONLY for DEBUGGING!
+# Source: http://stackoverflow.com/questions/5631078/sqlalchemy-print-the-actual-query
+from sqlalchemy.engine.default import DefaultDialect
+from sqlalchemy.sql.sqltypes import String, DateTime, NullType
+
+PY3 = str is not bytes
+text = str
+int_type = int
+str_type = str
+
+
+class StringLiteral(String):
+    """Teach SA how to literalize various things."""
+    def literal_processor(self, dialect):
+        super_processor = super(StringLiteral, self).literal_processor(dialect)
+
+        def process(value):
+            if isinstance(value, int_type):
+                return text(value)
+            if not isinstance(value, str_type):
+                value = text(value)
+            result = super_processor(value)
+            if isinstance(result, bytes):
+                result = result.decode(dialect.encoding)
+            return result
+        return process
+
+
+class LiteralDialect(DefaultDialect):
+    colspecs = {
+        # prevent various encoding explosions
+        String: StringLiteral,
+        # teach SA about how to literalize a datetime
+        DateTime: StringLiteral,
+        # don't format py2 long integers to NULL
+        NullType: StringLiteral,
+    }
+
+
+def literalquery(statement):
+    """NOTE: This is entirely insecure. DO NOT execute the resulting strings."""
+    import sqlalchemy.orm
+    if isinstance(statement, sqlalchemy.orm.Query):
+        statement = statement.statement
+    return statement.compile(
+        dialect=LiteralDialect(),
+        compile_kwargs={'literal_binds': True},
+    ).string

ratlib/sopel.py

@@ -61,6 +61,7 @@ class RatbotConfigurationSection(StaticSection):
     websocketport = types.ValidatedAttribute('websocketport', str, default='9000')
     shortenerurl = types.ValidatedAttribute('shortenerurl', str, default='')
     shortenertoken = types.ValidatedAttribute('shortenertoken', str, default='asdf')
+    debug_channel = types.ValidatedAttribute('debug_channel', str, default='#mechadeploy')
 
 def parameterize(params=None, usage=None, split=re.compile(r'\s+').split):
     """
@@ -198,6 +199,7 @@ def configure(config):
     config.ratbot.configure_setting('websocketport', "The port for the Websocket to listen on")
     config.ratbot.configure_setting('shortenerurl', "The url for the shortener to listen on")
     config.ratbot.configure_setting('shortenertoken', "The Auth token the shortener should use")
+    config.ratbot.configure_setting('debug_channel', "Channel for debug output")
 
 def setup(bot):
     """

ratlib/starsystem.py

@@ -20,6 +20,7 @@
 import requests
 import sqlalchemy as sa
 from sqlalchemy import sql, orm, schema
+from sqlalchemy import *
 from ratlib.db import get_status, get_session, with_session, Starsystem, StarsystemPrefix
 from ratlib.bloom import BloomFilter
 
@@ -95,9 +96,9 @@ def _refresh_database(bot, force=False, callback=None, background=False, db=None
     start = time()
     print('Starting refresh at '+str(start))
     edsm_url = bot.config.ratbot.edsm_url or "http://edsm.net/api-v1/systems?coords=1"
+    chunked = bot.config.ratbot.chunked_systems == 'True'
     status = get_status(db)
-    print('status: '+str(status))
-    edsm_maxage = float(bot.config.ratbot.edsm_maxage) or 60*12*12
+    edsm_maxage = float(bot.config.ratbot.edsm_maxage) or 604800 # Once per week = 604800 seconds
     if not (
         force or
         not status.starsystem_refreshed or
@@ -122,7 +123,18 @@ def _refresh_database(bot, force=False, callback=None, background=False, db=None
     if req.status_code != 200:
         print('ERROR When calling EDSM - Status code was '+str(req.status_code))
         return
-    print('Fetch done!')
+    if chunked:
+        data = []
+        response = req.json()
+        for part in response:
+            part = part.get('SectorName')
+            print('requesting from: '+edsm_url[0:edsm_url.rfind('/')+1] + str(part))
+            partreq = requests.get(edsm_url[0:edsm_url.rfind('/')+1] + str(part))
+            partdata = partreq.json()
+            data.extend(partdata)
+    else:
+        data = req.json()
+    print('Fetch done, Code was 200, data loaded into var!')
     fetch_end = time()
     # with open('run/systems.json') as f:
     #     import json
@@ -150,25 +162,32 @@ def _format_system(s):
 
     print('loading data into db....')
     load_start = time()
+    datatotal = len(data)
+    dataremaining = datatotal
+    halfway = False
+    print('data length: '+str(dataremaining))
     for chunk in chunkify(data, 5000):
         db.bulk_insert_mappings(Starsystem, [_format_system(s) for s in chunk])
-        print(ct)
+        dataremaining -= 5000
+        if (not halfway and dataremaining < datatotal/2):
+            halfway = True
+            print('Half way done with inserts...')
+    print('Done with chunkified stuff, deleting data var to free up mem. Analyzing stuff.')
     del data
     db.connection().execute("ANALYZE " + Starsystem.__tablename__)
-    print('done loading!')
+    print('Done loading!')
     load_end = time()
 
     stats_start = time()
     # Pass 2: Calculate statistics.
     # 2A: Quick insert of prefixes for single-name systems
-    print('line 162')
+    print('Executing against Database...')
     db.connection().execute(
         sql.insert(StarsystemPrefix).from_select(
             (StarsystemPrefix.first_word, StarsystemPrefix.word_ct),
             db.query(Starsystem.name_lower, Starsystem.word_ct).filter(Starsystem.word_ct == 1).distinct()
         )
     )
-    print('line 169')
     def _gen():
         for s in (
             db.query(Starsystem)
@@ -179,12 +198,10 @@ def _gen():
             yield (first_word, s.word_ct), words, s
 
     ct = 0
-    print('for chunk line 180')
+    print('Adding Prefixes to db...')
     for chunk in chunkify(itertools.groupby(_gen(), operator.itemgetter(0)), 100):
-        print('ct: '+str(ct))
         for (first_word, word_ct), group in chunk:
             ct += 1
-            print('ct in loop: '+str(ct))
             const_words = None
             for _, words, system in group:
                 if const_words is None:
@@ -197,12 +214,12 @@ def _gen():
             prefix = StarsystemPrefix(
                 first_word=first_word, word_ct=word_ct, const_words=" ".join(const_words)
             )
-            print('prefix: '+str(prefix))
+            # print('prefix: '+str(prefix))
             db.add(prefix)
         # print(ct)
-        print('db.flush for ct '+str(ct))
+        # print('db.flush for ct '+str(ct))
         db.flush()
-    print('db. connection().execute line 200')
+    print('Prefixes added and database flushed. Executing more stuff against database...')
     db.connection().execute(
         sql.update(
             Starsystem, values={
@@ -231,12 +248,11 @@ def _gen():
         ) AS t
         WHERE t.id=starsystem_prefix.id
         """.format(sp=StarsystemPrefix.__tablename__, s=Starsystem.__tablename__)
-    print('db. connection().execute line 211, exestring: '+str(exestring))
+    print('Executing yet more stuff...')
 
     db.connection().execute(
         exestring
     )
-    print('Done with stats.')
     stats_end = time()
 
     # Update refresh time
@@ -357,3 +373,34 @@ def scan_for_systems(bot, line, min_ratio=0.05, min_length=6):
         return set(results.values())
     finally:
         db.rollback()
+
+@with_session
+def getSystemFromDB(bot, db=None, sysname="fuelum"):
+    # Create query for system
+    print('searching for '+str(sysname))
+    systems = db.query(Starsystem).filter(Starsystem.name_lower == str(sysname).lower())
+    # CAUTION! Debug ONLY!
+    # statement = systems.statement
+    # from ratlib.literalstatement import literalquery
+    # print("Statement: "+str(literalquery(statement)))
+    # convert found systems to dict
+    result = [u.__dict__ for u in systems.all()]
+    for res in result:
+        # return first element from dict (should never be longer than 1)
+        print('Returning '+str(res)+' for system '+sysname)
+        return res
+    print('No system found for '+str(sysname))
+    return None
+
+@with_session
+def getSystemsInBox(bot, x1, y1, z1, x2, y2, z2, db=None):
+    # Create query to get systems in the given box
+    systems = db.query(Starsystem).filter(Starsystem.x.between(x1, x2), Starsystem.y.between(y1, y2), Starsystem.z.between(z1, z2))
+    # CAUTION! Debug ONLY!
+    # statement = systems.statement
+    # from ratlib.literalstatement import literalquery
+    # print("Statement: "+str(literalquery(statement)))
+    # convert query result to dict
+    result = [u.__dict__ for u in systems.all()]
+
+    return result

requirements.txt

@@ -9,3 +9,4 @@ autobahn[twisted]
 twisted
 pyopenssl
 service_identity
+numpy