generate.py

#  BEGIN LICENSE BLOCK 
#  Version: MPL 1.1/GPL 2.0/LGPL 2.1
# 
#  The contents of this file are subject to the Mozilla Public License Version
#  1.1 (the "License"); you may not use this file except in compliance with
#  the License. You may obtain a copy of the License at
#  http://www.mozilla.org/MPL/
# 
#  Software distributed under the License is distributed on an "AS IS" basis,
#  WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
#  for the specific language governing rights and limitations under the
#  License.
# 
#  The Original Code is Places Test code.
# 
#  The Initial Developer of the Original Code is Mozilla Corp.
#  Portions created by the Initial Developer are Copyright (C) 2009
#  the Initial Developer. All Rights Reserved.
# 
#  Contributor(s):
#  David Dahl <ddahl@mozilla.com>
# 
#  Alternatively, the contents of this file may be used under the terms of
#  either the GNU General Public License Version 2 or later (the "GPL"), or
#  the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
#  in which case the provisions of the GPL or the LGPL are applicable instead
#  of those above. If you wish to allow use of your version of this file only
#  under the terms of either the GPL or the LGPL, and not to allow others to
#  use your version of this file under the terms of the MPL, indicate your
#  decision by deleting the provisions above and replace them with the notice
#  and other provisions required by the GPL or the LGPL. If you do not delete
#  the provisions above, a recipient may use your version of this file under
#  the terms of any one of the MPL, the GPL or the LGPL.
# 
#  END LICENSE BLOCK 

"""
using the 'Places generator'
----------------------------

Requirements: django.db - you should have a full install of django 1.0.2 release. We are using django solely for it's ORM and 'manage.py inspectdb' utility which reverse-engineers a db schema into django Model classes.

You must have django and the 'places' python module in your PYTHONPATH

in my ~/.zshrc I have:

export PYTHONPATH=$PYTHONPATH:~/code/python:~/code/mozilla-central/mozilla/toolkit/components/places/tests/generator/places

inside ~/code/mozilla-central/mozilla/toolkit/components/places/tests/generator is the places/ Django 'project'

You must set 2 ENV VARS: 

1) PLACES_DB_PATH
 e.g.: export PLACES_DB_PATH=~/code/python/places/places.sqlite

2) DJANGO_SETTINGS_MODULE
 e.g.: export DJANGO_SETTINGS_MODULE=places.settings

Generate Script Docs:

python places/builddb/generate.py --help

================================================
This will pump your places.sqlite with a bunch of 
ficticious places, history and bookmarks based on the current 'max' aggregation at https://places-stats.mozilla.com/
"""

import os
from optparse import OptionParser
import random
import urllib2
import random
import time
from copy import deepcopy
from math import ceil, floor

from places import uuid 
from places.builddb.models import *
from places.builddb.http import get_web_data

"""
Places data generator. Create as many places/history/bookmarks as needed for testing Places queries for performance.

Tunable parameters: absolute # visits, bookmarks, age: 
create random dates up to today, 

"""

DEBUG = True
PERCENT_PLACES_TYPED = 0.3
TWENTY_FOUR_HOURS = (60 * 60 * 24 * 1000 * 1000)

from django.db import connection, transaction

class DateForward(object):
    """
    go through the places db and increment the dates by one day
    moz_annos.dateAdded
    moz_annos.lastModified
    moz_annos.expiration
    moz_bookmarks.dateAdded
    moz_bookmarks.lastModified
    moz_favicons.expiration
    moz_historyvisits.visit_date
    moz_items_annos.dateAdded
    moz_items_annos.expiration
    moz_items_annos.lastModified
    
    """
    def __init__(self):
        pass

    def table_column(self):
        """
        keeps track of the tables/columns that need to be updated
        """
        table_cols = {'moz_annos':['dateadded', 'lastmodified', 'expiration',],
                      'moz_bookmarks': ['dateadded', 'lastmodified',],
                      'moz_favicons': ['expiration'],
                      'moz_historyvisits': ['visit_date'],
                      'moz_items_annos': ['dateadded', 'lastmodified', 'expiration',]}
        return table_cols

    def place_id(self):
        sql = """select id from moz_places where title = 'Places Testing Placeholder'"""
        cursor = connection.cursor()
        rows = cursor.execute(sql, [])
        id = rows.fetchall()[0][0]
        return id

    def last_update(self):
        """
        get the last update date
        """
        id = self.place_id()
        sql = """SELECT visit_date FROM moz_historyvisits WHERE place_id = %s ORDER BY id DESC""" % id
        cursor = connection.cursor()
        rows = cursor.execute(sql)
        _rows= rows.fetchall()
        return _rows[0][0]

    def increment_dates(self):
        """
        update the dates
        """
        last_update_seconds = ((self.last_update() / 1000) / 1000)
        now = int(time.time())
        self.now_micro_secs = ((now * 1000) * 1000)
        
        if DEBUG:
            print "last update microseconds: %s" % self.last_update()
            print "now: %s" %  self.now_micro_secs
        time_diff = (now - last_update_seconds)
        self.time_diff = ((time_diff * 1000) * 1000)

        if DEBUG:
            print "\nTIME DIFF (microseconds): %s\n\n" % str(self.time_diff)

        if DEBUG:
            print "Incrementing History"
        self.increment_history()
        if DEBUG:
            print "Incrementing Bookmarks"
        self.increment_bookmarks()
        if DEBUG:
            print "Incrementing Favicons"
        self.increment_favicons()
        if DEBUG:
            print "Incrementing Annos"
        self.increment_annos()
        if DEBUG:
            print "Incrementing Items Annos"
        self.increment_items_annos()
        if DEBUG:
            print "VACUUM/REINDEX"
        self.vacuum_and_reindex()
        
        last_time_run()

    def increment_history(self):

        self.increment_query('moz_historyvisits', 'visit_date', restrict_update=True)

    def increment_bookmarks(self):

        self.increment_query('moz_bookmarks', 'dateadded')
        self.increment_query('moz_bookmarks', 'lastmodified')
            
    def date_missing(self, _date):
        """
        check to see if the date is None or 0, if so, return True
        """
        if _date == 0 or _date is None or _date == '':
            return True
        else:
            return False

    def increment_query(self, table_name, col_name, restrict_update=False):
        """
        shorter, faster updates of the date column
        """
        if restrict_update:
            # get the place_id
            place_id = self.place_id()
            
        try:
            sql = "UPDATE %s SET %s " % (table_name, col_name,)
            sql = sql + " = %s + %s"

            if restrict_update:
                where = " WHERE %s < %s AND place_id != %s AND %s != ''" % (col_name, self.now_micro_secs, place_id, col_name,)
            else:
                where = " WHERE %s < %s AND %s != ''" % \
                    (col_name, self.now_micro_secs, col_name, )

            sql = sql + where

            debug_sql = sql % (col_name, self.time_diff,)
            if DEBUG:
                print debug_sql
            cursor = connection.cursor()
            cursor.execute(debug_sql)
            cursor.close()
        except Exception, e:
            print """************ ERROR *******************"""
            print """***%s""" % e
            raise
            
    def increment_favicons(self):
        """
        uses a simpler approach via a cursor 
        """        
        self.increment_query('moz_favicons', 'expiration')


    def increment_annos(self):

        self.increment_query('moz_annos', 'dateadded')
        self.increment_query('moz_annos', 'lastmodified')
        self.increment_query('moz_annos', 'expiration')

    def increment_items_annos(self):
        
        self.increment_query('moz_items_annos', 'dateadded')
        self.increment_query('moz_items_annos', 'lastmodified')
        self.increment_query('moz_items_annos', 'expiration')            

    def vacuum_and_reindex(self):
        try:
            v_sql = "VACUUM;"
            r_sql = "REINDEX;"

            cursor = connection.cursor()
            cursor.execute(v_sql)
            cursor.execute(r_sql)
            cursor.close()
        except Exception, e:
            print """************ ERROR *******************"""
            print """***%s""" % e
            raise

def right_now():
    """
    right now! in microseconds
    """
    return time.time() * 1000 * 1000

def six_months_ago():
    """
    time in microseconds six months ago (roughly)
    """
    six_months = (60*60*24*180*1000*1000)
    n = right_now()
    smo = n - six_months 
    return  smo
    
def random_date():
    """
    a random date (in microseconds since epoch) up to six months ago
    """
    six_months = six_months_ago()
    now = right_now()
    
    s = random.randint(int(six_months), int(now))
    return s

def alpha():
    return map(chr, range(97, 123))

ALPHA = alpha()

def numbers():
    return [0,1,2,3,4,5,6,7,8,9]

def new_favicon():
    """
    create a new, unique favicon 
    """
    guid = uuid.generate()
    url = 'http://tld.tld/%s.ico' % guid
    favicon = MozFavicons.objects.create(
        url=url,
        data='wootuyiuyeryuwyerywueyurywueyruuewryueyuryuewyurkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjjsdhjshjdhjshdjhjsdhjshjkdhjshjdhjshkjdhsjkhdkjhsjdhjshjhdd;sklfnlk;dnfjnklwmnkldjfklhadlfgjkhrnfjkgb;jlshgn;ruignifng;irgn;srhg;rnv;irn;',
        mime_type='image/png',
        expiration=random_date())
    return favicon

def reverse_host(url):
    """
    reverse the host
    """
    parts = urllib2.urlparse.urlparse(url)
    ru = "".join(reversed(parts[1])) + '.'
    return ru

def url_parts(): 
    """
    return a dictionary like: {'proto':'http'
                               'host':'www',
                               'domain':'foo',
                               'tld':'com'}
    """
    protocol = ['https','http','ftp']
    host_len = random.randint(4,26)
    host = "".join(random.sample(ALPHA,host_len))
    domain_len = random.randint(2,26)
    domain = "".join(random.sample(ALPHA,domain_len))
    tld_len = random.randint(2,3)
    tld = "".join(random.sample(ALPHA,tld_len))
    proto_idx = random.randint(0, 2)
    proto = protocol[proto_idx]
    return {'proto':proto,'host':host,'domain':domain,'tld':tld}

def new_title():
    """
    return a random title
    """
    title_len = random.randint(3,26)
    title = "".join(random.sample(ALPHA,title_len))
    return title

def keyword_gen(n):
    kw_list, kw_count = make_keywords(n)
    def gen():
        for obj in kw_list:
            yield obj
    return gen()

def make_keywords(n):
    """
    make num_keywords 
    """
    kw_list = []
    i = 0
    while n > 0:
        keyword_len = random.randint(3,18)
        kw = "".join(random.sample(ALPHA,keyword_len))
        keyword = MozKeywords(keyword=kw)
        keyword.save()
        k = MozKeywords.objects.get(keyword=kw)
        kw_list.append(k)
        n = n - 1
        i = i + 1

    return (kw_list,i)

def expo(MEAN,round_ceil='ceil'):
    """
    returns a value for a exponential dustribution
    """
    lambd = 1.0 / MEAN # the lambda value for the random generator
    val = random.expovariate(lambd)
    if round_ceil == 'ceil':
        val = int(ceil(val))
    val = int(round(val))
    return val

def test_expo(num,MEAN):
    """
    test the curvyness
    """
    f = []
    while num > 0:
        f.append(expo(MEAN))
        num = num -1
    return f
        
def frecency():
    """
    calculate an exponential frecency value
    """
    MEAN = 146 # 146 is our mean value, because ddahl's is 146
    lambd = 1.0 / MEAN # the lambda value for the random generator
    val = random.expovariate(lambd)
    if val < 1:
        # flip a coin: 
        if random.randint(0, 1) == 0:
            return 0
        else:
            return -1
    f_ceil = ceil(val)
    return f_ceil
    

def test_frecency(num):
    frec_list = []
    while num > 0:
        frec_list.append(frecency())
        num = num - 1
    frec_list.sort()
    return frec_list

def flip_coin():
    return random.randint(0,1)

def slant(rows,heads):
    """
    slant the coin toss to a percentage
    """
    results = []
    num_heads = round(int(rows * heads))
    num_tails = rows - num_heads
    while num_heads > 0:
        results.append(1)
        num_heads = num_heads - 1
    while num_tails > 0:
        results.append(0)
        num_tails = num_tails - 1
    return results

def flip_coin_slant(rows,heads):
    results = slant(rows,heads)
    def fcs():
        for r in results:
            yield r
    return fcs

def last_time_run():
    """
    Use get_or_create to make/get a url that keeps track of when this operation was last run on the database 
    returns date in microseconds of when this place was last updated
    """
    my_url = "http://mozilla.com/places_test_placeholder"
    my_title = "Places Testing Placeholder"

    p, created = place = MozPlaces.objects.get_or_create(url=my_url,title=my_title,rev_host=reverse_host(my_url),visit_count=1,hidden=0,typed=1,frecency=-1)    
    sql = """select id from moz_places where title = 'Places Testing Placeholder'"""
    cursor = connection.cursor()
    rows = cursor.execute(sql)
    id = rows.fetchall()[0][0]
    # insert the latest record into historyvisits
    sql = """INSERT INTO moz_historyvisits (from_visit, place_id, visit_date, visit_type, session ) VALUES (0, %s, %s, 1, 0 )""" % (int(id), right_now(),)
    cursor.execute(sql)
    transaction.commit_unless_managed()

    sql = """SELECT visit_date FROM moz_historyvisits WHERE place_id = %s ORDER BY id DESC""" % id
    rows = cursor.execute(sql)
    _rows = rows.fetchall()
    visit_date = _rows[0][0]
    return visit_date

def new_place(places_typed_gen):
    """
    create a new place
    """
    guid = uuid.generate()
    parts = url_parts()
    my_url = '%s://%s.%s.%s/%s' % (parts['proto'],
                                   parts['host'],
                                   parts['domain'],
                                   parts['tld'],
                                   guid)
    if DEBUG:
        print my_url
    my_title = new_title()
    frec = frecency()
    try:
        place = MozPlaces(
            url=my_url,
            title=my_title,
            rev_host=reverse_host(my_url),
            visit_count=expo(8), # using 8 for MEAN for the fun of it for now 
            hidden=0,
            typed=places_typed_gen.next(),
            favicon=new_favicon(),
            frecency=frec)
        place.save()
        p = MozPlaces.objects.filter(rev_host=place.rev_host)[0]
        return p

    except Exception, e:
        print e
        raise

def new_inputhistory(place):
    """
    create an inputhistory record for the place object passed in
    """
    input_len = random.randint(3,20)
    ih = "".join(random.sample(ALPHA,input_len))
    ih_multi = random.randint(2,9)
    uc = ih_multi * 0.9
    inp_hist = MozInputhistory(place_id=place.id,input=ih,use_count=uc)
    inp_hist.save()
    return inp_hist

def new_history(place,qty=5):
    """
    create a history record
    """
    rndm_date = random_date()

    try:
        while qty > 0:
            history = MozHistoryvisits.objects.create(
                from_visit=0, # other visit this is related to/from/etc
                place=place,
                visit_date=rndm_date,
                visit_type=1, 
                session=0
                )
            qty = qty -1 
    except:
        print "ERROR: cannot create historyvisit for %s" % str(place.id)
        raise

def new_bookmark(place,kw_gen):
    """
    make a bookmark item
    """
    try:
        k_obj = kw_gen.next()
    except Exception, e:
        print str(e)
        k_obj = None

    try:
        bookmark = MozBookmarks.objects.create(
            _type=1, # XXX: what should this be?
            fk=place.id,
            parent=5, # XXX: what is unfiled bookmarks id?
            position=-1, # XXX: what should this be?
            title=place.title,
            folder_type=1, # XXX: what should this be?
            dateadded=random_date(),
            lastmodified=random_date(),
            keyword=k_obj
            )
        return bookmark
    except Exception, e:
        print str(e)
        print "Err: Could not create bookmark from place: %s" % place.id

def parse_places_stats_obj(obj,source_idx):
    """
    Parse and massage the input for the places generator 
    """
    class Options(object):
        def __init__(self,stats_obj,source_idx):
            # XXX: check names in json obj
            self.source = 'places_stats_site'
            self.path = None
            self.source_idx = source_idx
            src_obj = stats_obj[source_idx]
            self.n_places = int(float(stats_obj[source_idx]['moz_places_cnt']))
            self.h_mult = int(float(stats_obj[source_idx]['moz_historyvisits_cnt']) / float(stats_obj[source_idx]['moz_places_cnt']))
            self.b_mod = int(float(stats_obj[source_idx]['moz_places_cnt']) / float(stats_obj[source_idx]['moz_bookmarks_cnt']))
            kw_count = int(float(stats_obj[source_idx]['moz_keywords_cnt']))
            bm_count = int(float(stats_obj[source_idx]['moz_bookmarks_cnt']))
            self.n_keywords = kw_count
            self.n_inputhistory = \
                int(float(stats_obj[source_idx]['moz_inputhistory_cnt']))
            if DEBUG:
                print "########################################################"
                print "Creating %s Places" % self.n_places
                print "Creating about %s History Visits" % stats_obj[source_idx]['moz_historyvisits_cnt']
                print "Creating about %s Bookmarks" % stats_obj[source_idx]['moz_bookmarks_cnt']
                print "Creating %s Keywords" % self.n_keywords 
                print "Creating %s Input History Records" % self.n_inputhistory 
                print "########################################################"
            else:
                print "Starting Places DB Generation"
            time.sleep(3)

    options = Options(obj,source_idx)
    return options

def main(source,
         source_idx,
         num_places,
         bookmarks_mod,
         history_mult,
         n_keywords,
         n_inputhistory,
         path,
         debug):
    """
    configure and run the whole operation of creating data in places.sqlite
    """
    DEBUG = False
    if debug:
        DEBUG = True
    if path:
        os.environ['PLACES_DB_PATH'] = path
    counter = 1
    if DEBUG:
        print "Creating %s Places" % num_places  

    PERCENT_PLACES_TYPED = 0.3

    kw_gen = keyword_gen(n_keywords)
    places_typed_gen = flip_coin_slant(num_places,PERCENT_PLACES_TYPED)()
    while num_places > 0:
        if DEBUG:
            print 'Place #%s created' % counter
        place = new_place(places_typed_gen) # create a place
        if n_inputhistory > 0:
            inp_hist = new_inputhistory(place)
            n_inputhistory = n_inputhistory - 1
        history = new_history(place) # create 5 history records
        if num_places % bookmarks_mod == 0: 
            nb = new_bookmark(place,kw_gen) # create a bookmark 
        num_places = num_places - 1
        counter = counter + 1

    # update the time this was last run. The value is stored in a 
    # moz_historyvisits row as 'visit_date' for the place with the 
    # uri of "http://mozilla.com/places_test_placeholder'

    if DEBUG:
        print "Updating last time run timestamp"
    try:
        last_time_run()
    except Exception, e:
        print e
        raise

if __name__ == '__main__':
    
    parser = OptionParser()
    parser.add_option("-p", "--path", dest="path", default=None,
                      help="Path to SQLite db", metavar="PATH")
    parser.add_option("-s", "--source", 
                      dest="source", 
                      default='places_stats_site',
                      help="'made_up' data or 'places_stats_site' data?", 
                      metavar="SOURCE")
    parser.add_option("-i", "--source_idx", dest="source_idx", 
                      default='max',
                      help="Source object index: avg, min, max", 
                      metavar="SOURCE_IDX")
    parser.add_option("-n", "--num-places", dest="n_places", default=300,
                      help="Number of Places to create", 
                      metavar="NUM_PLACES")
    parser.add_option("-b", "--bookmarks-modulus", dest="b_mod", default=4,
                      help="Bookmarks Modulus, the number of places for which a single bookmark is created", 
                      metavar="BOOKMARKS_MODULUS")
    parser.add_option("-k", "--num-keywords", dest="n_keywords", default=15,
                      help="Number of total Keywords", 
                      metavar="KEYWORDS")
    parser.add_option("-t", "--input-history", dest="n_inputhistory", default=50,
                      help="Number of Input History rows", 
                      metavar="INPUT_HISTORY")
    parser.add_option("-m", "--history-mult", dest="h_mult", default=5,
                      help="The number of history items to create for each place", metavar="HISTORY_MULT")
    parser.add_option("-v", action="store_true", dest="debug", 
                      default=True, help="Turn DEBUG on")
    parser.add_option("-q", action="store_false", dest="debug", 
                      default=False, help="Turn DEBUG off")

    (options, args) = parser.parse_args()
    
    if options.source == 'places_stats_site':
        cmd_line_options = options
        stats_obj = get_web_data()
        options = parse_places_stats_obj(stats_obj,
                                         cmd_line_options.source_idx)
        options.path = cmd_line_options.path
        options.debug = cmd_line_options.debug

    KEYWORD_COUNT = 0
    KEYWORD_IDS = []

    main(options.source,
         options.source_idx,
         options.n_places,
         int(options.b_mod),
         int(options.h_mult),
         options.n_keywords,
         options.n_inputhistory,
         options.path,
         options.debug)