uci_006_internet_advertisements.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import urllib.request
import io
import pandas # install pandas by "pip install pandas", or install Anaconda distribution (https://www.anaconda.com/)

# Warning: the data processing techniques shown below are just for concept explanation, which are not best-proctices

# data set repository
# https://archive.ics.uci.edu/ml/datasets/Internet+Advertisements

# if the file is on your local device, change url_data_train into local file path, e.g., '‪D:\local_file.data'
url_data_train = 'https://archive.ics.uci.edu/ml/machine-learning-databases/internet_ads/ad.data'

def download_file(url):
    resp = urllib.request.urlopen(url)
    if resp.status != 200:
        resp.close()
        raise ValueError('Error: {0}'.format(resp.reason))

    print('\rStarted', end = '\r')
    content_length = resp.getheader('Content-Length')
    if content_length is None:
        content_length = '(total: unknown)'
    else:
        content_length = int(content_length)
        if content_length < 1024:
            content_length_str = '(total %.0f Bytes)' % content_length
        elif content_length < 1024 * 1024:
            content_length_str = '(total %.0f KB)' % (content_length / 1024)
        else:
            content_length_str = '(total %.1f MB)' % (content_length / 1024 / 1024)

    total = bytes()
    while not resp.isclosed():
        total += resp.read(10 * 1024)
        if len(total) < 1024:
            print(('\rDownloaded: %.0f Bytes ' % len(total)) + content_length_str + '  ', end = '\r')
        if len(total) < 1024 * 1024:
            print(('\rDownloaded: %.0f KB ' % (len(total) / 1024)) + content_length_str + '  ', end = '\r')
        else:
            print(('\rDownloaded: %.1f MB ' % (len(total) / 1024 / 1024)) + content_length_str + '  ', end = '\r')

    print()
    return io.BytesIO(total)

# download data from UCI Machine Learning Repository
data_train = download_file(url_data_train) if url_data_train.startswith('http') else url_data_train

# ad_or_nonad is the original target variable, which will be converted into 0 or 1 later
columns = [
    'height',
    'width',
    'aratio',
    'local',
    'url*images+buttons',
    'url*likesbooks.com',
    'url*www.slake.com',
    'url*hydrogeologist',
    'url*oso',
    'url*media',
    'url*peace+images',
    'url*blipverts',
    'url*tkaine+kats',
    'url*labyrinth',
    'url*advertising+blipverts',
    'url*images+oso',
    'url*area51+corridor',
    'url*ran+gifs',
    'url*express-scripts.com',
    'url*off',
    'url*cnet',
    'url*time+1998',
    'url*josefina3',
    'url*truluck.com',
    'url*clawnext+gif',
    'url*autopen.com',
    'url*tvgen.com',
    'url*pixs',
    'url*heartland+5309',
    'url*meadows+9196',
    'url*blue',
    'url*ad+gif',
    'url*area51',
    'url*www.internauts.ca',
    'url*afn.org',
    'url*ran.org',
    'url*shareware.com',
    'url*baons+images',
    'url*area51+labyrinth',
    'url*pics',
    'url*image+navigate',
    'url*pool+images',
    'url*carousel.org',
    'url*members.accessus.net',
    'url*cmp+flowsoft010',
    'url*smallb+gif',
    'url*www.FlowSoft.com',
    'url*kandinsky',
    'url*csuhayward.edu',
    'url*www.yahoo.co.uk',
    'url*ibitexas.com',
    'url*www.haestad.com',
    'url*antiwar.com',
    'url*www.monmouth.com',
    'url*infoserver.etl.vt.edu',
    'url*geoguideii+tour',
    'url*heartland+pointe',
    'url*romancebooks+pix',
    'url*468x60',
    'url*ads.switchboard.com',
    'url*keith+dumble',
    'url*cqi.com',
    'url*images+ixfolder',
    'url*chat+b',
    'url*images+geoguideii',
    'url*preferences.com',
    'url*icsi.com',
    'url*pacific.net.sg',
    'url*monmouth.com',
    'url*wired.com',
    'url*go2net',
    'url*smiley+gif',
    'url*navigate+b',
    'url*sunsite.unc.edu',
    'url*www.pacific.net.sg',
    'url*home+gif',
    'url*library',
    'url*puc.edu',
    'url*internauts.ca',
    'url*www.worlddrugstore.com',
    'url*www.ibitexas.com',
    'url*ucsd.edu',
    'url*pitcairn+smallb',
    'url*kats',
    'url*cjackson+kandinsky',
    'url*tkaine+rings',
    'url*any+time',
    'url*FlowSoft.com',
    'url*geoguideii+nh',
    'url*charlie',
    'url*heartland+meadows',
    'url*home.netscape.com',
    'url*9439+butt2',
    'url*catringn+gif',
    'url*martnet',
    'url*doubleclick.net',
    'url*lycos.de',
    'url*catringn',
    'url*catring1',
    'url*fools+pixs',
    'url*logos',
    'url*private',
    'url*ng+spacedesc',
    'url*larx.com',
    'url*site',
    'url*pointe+1154',
    'url*banners',
    'url*pixs+smiley',
    'url*www.123greetings.com',
    'url*www.inwap.com',
    'url*cjackson',
    'url*labyrinth+9439',
    'url*generated+images',
    'url*image+ng',
    'url*lolnad+private',
    'url*library+pitcairn',
    'url*www.mei.co.jp',
    'url*april',
    'url*derived',
    'url*smiley',
    'url*home',
    'url*jsd',
    'url*sonic.net',
    'url*send+b',
    'url*website.ora.com',
    'url*pawbutton+gif',
    'url*victoria+pharmacy',
    'url*keith',
    'url*pages',
    'url*worlddrugstore.com',
    'url*geoguideii+email',
    'url*www.icgnet.com',
    'url*forum',
    'url*www.afn.org',
    'url*time',
    'url*www.ran.org',
    'url*main',
    'url*members+keith',
    'url*geoguideii+pages',
    'url*banner',
    'url*stinky+gra',
    'url*tkaine+buttons',
    'url*victoria',
    'url*kbell',
    'url*www.truluck.com',
    'url*img',
    'url*pics+cats',
    'url*alley',
    'url*claw1',
    'url*pharmacy+gif',
    'url*www.autopen.com',
    'url*images+img',
    'url*fbox.vt.edu',
    'url*unc.edu',
    'url*ics',
    'url*times',
    'url*forums',
    'url*chat',
    'url*images.go2net.com',
    'url*users.aol.com',
    'url*malek',
    'url*jsd+how',
    'url*www.cqi.com',
    'url*advertising+graphics',
    'url*pawbutton',
    'url*pictures',
    'url*grouper.com',
    'url*ads+media',
    'url*www.martnet.com',
    'url*set+butt',
    'url*how',
    'url*www.wednet.com',
    'url*www.globec.com.au',
    'url*athens',
    'url*info',
    'url*www.ireland-today.ie',
    'url*grn+bull',
    'url*sjsu.edu',
    'url*www.antiwar.com',
    'url*www.puc.edu',
    'url*sponsor',
    'url*www.news.com',
    'url*flowsoft010',
    'url*memberbanners',
    'url*red+ball',
    'url*ads.msn.com',
    'url*icons',
    'url*images',
    'url*access.ch',
    'url*irish+times',
    'url*media+images',
    'url*www.adamspharmacy.com',
    'url*geocities.com',
    'url*www.uk.lycos.de',
    'url*copymat.com',
    'url*hem1.passagen.se',
    'url*meadows',
    'url*grn',
    'url*gra',
    'url*vt.edu',
    'url*www.express-scripts.com',
    'url*imgs',
    'url*personal',
    'url*mindlink.net',
    'url*send',
    'url*pages+b',
    'url*accessus.net',
    'url*www.finest.tm.fr',
    'url*geoguideii+search',
    'url*polypkem',
    'url*fools',
    'url*www.larx.com',
    'url*marketplace',
    'url*wco.com',
    'url*off+gif',
    'url*tkaine+bars',
    'url*icgnet.com',
    'url*www.autotown.com',
    'url*generated',
    'url*9196+pawbutton',
    'url*geoguideii+chat',
    'url*www.vrn.de',
    'url*claw1+gif',
    'url*theinternetadvantage.com',
    'url*runofsite',
    'url*romancebooks',
    'url*foxy.net',
    'url*francois',
    'url*links',
    'url*athens+forum',
    'url*autotown.com',
    'url*logo',
    'url*members',
    'url*auto',
    'url*afn47757',
    'url*gifs',
    'url*esi',
    'url*hollywood',
    'url*forum+8078',
    'url*dept',
    'url*adamspharmacy.com',
    'url*valley',
    'url*www.pacificrim.net',
    'url*about',
    'url*www.likesbooks.com',
    'url*bossintl.com',
    'url*images+home',
    'url*wednet.com',
    'url*news.com',
    'url*arvann+images',
    'url*users',
    'url*netscape.com',
    'url*athens+8774',
    'url*pictures+gc',
    'url*butt',
    'url*e+gif',
    'url*sunsetstrip+alley',
    'url*geoguideii+send',
    'url*catring1+gif',
    'url*ball+gif',
    'url*bull',
    'url*paul.spu.edu',
    'url*rank',
    'url*2meta.com',
    'url*w+gif',
    'url*aol.com',
    'url*lg+gif',
    'url*irish-times.com',
    'url*irish',
    'url*ad.doubleclick.net',
    'url*ad.preferences.com',
    'url*bars',
    'url*forums+b',
    'url*gc+icon',
    'url*www.geocities.com',
    'url*www.carousel.org',
    'url*ball',
    'url*spu.edu',
    'url*com',
    'url*cnm',
    'url*htm+images',
    'url*htm+cmp',
    'url*cmp',
    'url*link+pics',
    'url*mindspring.com',
    'url*geoguideii+logo',
    'url*psu.edu',
    'url*go2net+ads',
    'url*www.grouper.com',
    'url*butt2',
    'url*cat',
    'url*www.dreamn.com',
    'url*vbtn+gif',
    'url*switchboard.com',
    'url*logo+b',
    'url*assets+images',
    'url*martnet.com',
    'url*buttons',
    'url*thejeep.com',
    'url*auto+generated',
    'url*search+b',
    'url*search',
    'url*pharmacy',
    'url*assets+auto',
    'url*static.wired.com',
    'url*cats',
    'url*bull+gif',
    'url*memberbanners+live',
    'url*pitcairn',
    'url*www.access.ch',
    'url*adv+images',
    'url*navigate',
    'url*runofsite+any',
    'url*dejay',
    'url*bishop',
    'url*email+b',
    'url*www.mcs.csuhayward.edu',
    'url*bin',
    'url*paris',
    'url*1154+graypaw',
    'url*stinky',
    'url*img+ads',
    'url*bds',
    'url*www.wco.com',
    'url*esi+image',
    'url*vrn.de',
    'url*www.toyotaofroswell.com',
    'url*www.copymat.com',
    'url*www.psnw.com',
    'url*go2net.com',
    'url*advertising',
    'url*slake.com',
    'url*www.theinternetadvantage.com',
    'url*baons',
    'url*geoguideii+rank',
    'url*butt2+gif',
    'url*tkaine',
    'url*art',
    'url*lovisa1',
    'url*pool',
    'url*valley+2647',
    'url*any',
    'url*corridor',
    'url*top',
    'url*rank+b',
    'url*pacificrim.net',
    'url*image',
    'url*heartland+valley',
    'url*tii',
    'url*123greetings.com',
    'url*adv',
    'url*ads',
    'url*hydrogeologist+imgs',
    'url*smallb',
    'url*members.tripod.com',
    'url*www.shareware.com',
    'url*images+ads',
    'url*banner+gif',
    'url*vbtn',
    'url*globec.com.au',
    'url*www.news-observer.com',
    'url*pointe',
    'url*mandypaul',
    'url*bds+s',
    'url*www.ems.psu.edu',
    'url*valley+2539',
    'url*members.aol.com',
    'url*yahoo.co.uk',
    'url*malek+small',
    'url*button',
    'url*msn.com',
    'url*graypaw+gif',
    'url*live',
    'url*haestad.com',
    'url*sanjosesabercats.com',
    'url*icon+gif',
    'url*april+fools',
    'url*assets',
    'url*set',
    'url*link',
    'url*wc',
    'url*uk',
    'url*sj',
    'url*images+set',
    'url*www.bossintl.com',
    'url*nh',
    'url*ng',
    'url*counter',
    'url*lg',
    'url*spacedesc',
    'url*bishop+red',
    'url*it',
    'url*ie',
    'url*rings',
    'url*gc',
    'url*peace',
    'url*dreamn.com',
    'url*ar',
    'url*ad',
    'url*thumb',
    'url*clawnext',
    'url*dumble',
    'url*kandinsky+t',
    'url*email',
    'url*martnet+images',
    'url*uk+gif',
    'url*red',
    'url*geoguideii+forums',
    'url*www.2meta.com',
    'url*ran',
    'url*ukonline.co.uk',
    'url*passagen.se',
    'url*psnw.com',
    'url*how+htm',
    'url*tm.fr',
    'url*sunsetstrip',
    'url*ora.com',
    'url*logo+gif',
    'url*marketing',
    'url*toyotaofroswell.com',
    'url*heartland',
    'url*ixfolder+gif',
    'url*mcet.edu',
    'url*enchantedforest',
    'url*alley+6750',
    'url*news-observer.com',
    'url*ireland-today.ie',
    'url*times+graphics',
    'url*madamecleo',
    'url*b+gif',
    'url*www.irish-times.com',
    'url*tripod.com',
    'url*ad+icons',
    'url*button+gif',
    'url*graypaw',
    'url*excite.com',
    'url*web.ukonline.co.uk',
    'url*polypkem+grn',
    'url*gort.ucsd.edu',
    'url*lolnad',
    'url*graphics+ad',
    'url*icon',
    'url*falcon.sonic.net',
    'url*picts',
    'url*phoenix.mcet.edu',
    'url*pix+b',
    'url*www.icsi.com',
    'url*ixfolder',
    'url*inwap.com',
    'url*www.cob.sjsu.edu',
    'url*pix',
    'url*www.thejeep.com',
    'url*tour',
    'url*www.sanjosesabercats.com',
    'url*mei.co.jp',
    'url*geoguideii',
    'url*small',
    'url*graphics',
    'url*kbell+charlie',
    'url*zueri.ch',
    'url*arvann',
    'origurl*labyrinth',
    'origurl*puc.edu',
    'origurl*charlie+charlie',
    'origurl*hevern+psychref',
    'origurl*and',
    'origurl*pterry+htm',
    'origurl*bishop',
    'origurl*ora.com',
    'origurl*www.nyx.net',
    'origurl*www.yahoo.co.uk',
    'origurl*www.truluck.com',
    'origurl*target+ion',
    'origurl*tkaine+sydkat',
    'origurl*meadows+9196',
    'origurl*top',
    'origurl*members.xoom.com',
    'origurl*0+rpp',
    'origurl*www.thriveonline.com',
    'origurl*adamspharmacy.com',
    'origurl*icsi.com',
    'origurl*tkaine+stvkat',
    'origurl*shareware.com',
    'origurl*e+html',
    'origurl*www.123greetings.com',
    'origurl*lofts+7119',
    'origurl*likesbooks.com',
    'origurl*gort.ucsd.edu',
    'origurl*users.aol.com',
    'origurl*www.innotts.co.uk',
    'origurl*roll.com',
    'origurl*web.ukonline.co.uk',
    'origurl*events',
    'origurl*www.2meta.com',
    'origurl*chuck+html',
    'origurl*paris',
    'origurl*chapel',
    'origurl*tii',
    'origurl*how+title',
    'origurl*www.news-observer.com',
    'origurl*alley',
    'origurl*april+fools',
    'origurl*carousel',
    'origurl*www.haestad.com',
    'origurl*title+html',
    'origurl*lemoyne.edu',
    'origurl*plains+5257',
    'origurl*psychref',
    'origurl*jun',
    'origurl*www.interchg.ubc.ca',
    'origurl*ukonline.co.uk',
    'origurl*www.lycos.co.uk',
    'origurl*lolnad+private',
    'origurl*ibitexas.com',
    'origurl*wco.com',
    'origurl*jsd',
    'origurl*www.rock.n.roll.com',
    'origurl*antiwar.com',
    'origurl*target',
    'origurl*phoenix.mcet.edu',
    'origurl*index',
    'origurl*1154+index',
    'origurl*pharmacy',
    'origurl*coltrane',
    'origurl*website.ora.com',
    'origurl*index+htm',
    'origurl*dept',
    'origurl*area51+labyrinth',
    'origurl*irish-times.com',
    'origurl*mandypaul',
    'origurl*pointe',
    'origurl*plato+htm',
    'origurl*arvann',
    'origurl*ion+0',
    'origurl*chapel+safehaven',
    'origurl*worldvillage.com',
    'origurl*nf+htm',
    'origurl*www.martnet.com',
    'origurl*hem1.passagen.se',
    'origurl*worlddrugstore.com',
    'origurl*www.globec.com.au',
    'origurl*athens+8774',
    'origurl*times+paper',
    'origurl*actup',
    'origurl*inwap',
    'origurl*main',
    'origurl*www.larx.com',
    'origurl*dumble+contents',
    'origurl*geocities.com',
    'origurl*4301+misfits2',
    'origurl*kattsida',
    'origurl*members.aol.com',
    'origurl*spu.edu',
    'origurl*sunsite.unc.edu',
    'origurl*valley+2647',
    'origurl*josefina3',
    'origurl*tkaine',
    'origurl*bonchin+cat',
    'origurl*chopin+htm',
    'origurl*punkie',
    'origurl*ubc.ca',
    'origurl*links+nf',
    'origurl*misfits2',
    'origurl*area51+vault',
    'origurl*kitty',
    'origurl*plato',
    'origurl*hollywood+7735',
    'origurl*link',
    'origurl*www.monmouth.com',
    'origurl*hist',
    'origurl*members.tripod.com',
    'origurl*www.ibitexas.com',
    'origurl*cgidir+dll',
    'origurl*heartland',
    'origurl*pages+catscats',
    'origurl*tkaine+linda',
    'origurl*crawler',
    'origurl*paul.spu.edu',
    'origurl*bishop+pterry',
    'origurl*carousel.org',
    'origurl*romance',
    'origurl*nyx.net',
    'origurl*hollywood',
    'origurl*simonc',
    'origurl*sox',
    'origurl*www.cob.sjsu.edu',
    'origurl*timeout+5',
    'origurl*leonora',
    'origurl*fh-rpl.de',
    'origurl*vt.edu',
    'origurl*ucsd.edu',
    'origurl*www.carousel.org',
    'origurl*www.antiwar.com',
    'origurl*FlowSoft.com',
    'origurl*kbell+charlie',
    'origurl*www.likesbooks.com',
    'origurl*labyrinth+9439',
    'origurl*www.icsi.com',
    'origurl*events+aprilfoolsday',
    'origurl*tii+tii',
    'origurl*athens+forum',
    'origurl*library',
    'origurl*valley',
    'origurl*ics+ikenobo',
    'origurl*unc.edu',
    'origurl*kandin',
    'origurl*bordeaux',
    'origurl*charlie',
    'origurl*6750+paws',
    'origurl*cccats',
    'origurl*linda+htm',
    'origurl*2647+coltrane',
    'origurl*vault+3440',
    'origurl*victoria+pharmacy',
    'origurl*wv',
    'origurl*kandin+html',
    'origurl*soho+lofts',
    'origurl*sunsetstrip+alley',
    'origurl*slake.com',
    'origurl*autotown.com',
    'origurl*www.sanjosesabercats.com',
    'origurl*infoserver.etl.vt.edu',
    'origurl*kushmerick',
    'origurl*www.toyotaofroswell.com',
    'origurl*turbo',
    'origurl*sw',
    'origurl*hills',
    'origurl*st',
    'origurl*mogwhi',
    'origurl*sj',
    'origurl*pointe+1154',
    'origurl*monmouth.com',
    'origurl*mei.co.jp',
    'origurl*sonic.net',
    'origurl*safehaven+index',
    'origurl*simonc+cybermog',
    'origurl*messier',
    'origurl*cybercats',
    'origurl*search',
    'origurl*heartland+5309',
    'origurl*psnw.com',
    'origurl*keith+dumble',
    'origurl*links',
    'origurl*nav',
    'origurl*nf',
    'origurl*sunsetstrip',
    'origurl*kandinsky',
    'origurl*meadows+3727',
    'origurl*www.afn.org',
    'origurl*southbeach',
    'origurl*pacific.net.sg',
    'origurl*heartland+pointe',
    'origurl*valley+2539',
    'origurl*catscats+html',
    'origurl*horn+html',
    'origurl*area51',
    'origurl*pages',
    'origurl*chopin',
    'origurl*it',
    'origurl*bonchin',
    'origurl*www.worldvillage.com',
    'origurl*00+html',
    'origurl*vrn.de',
    'origurl*home.netscape.com',
    'origurl*simstory+html',
    'origurl*biopic+htm',
    'origurl*dll',
    'origurl*cmeese+malibu',
    'origurl*ics',
    'origurl*linda',
    'origurl*fr',
    'origurl*private+biopic',
    'origurl*corridor+4590',
    'origurl*leonora+html',
    'origurl*dejay',
    'origurl*aprilfoolsday',
    'origurl*fools',
    'origurl*biopic',
    'origurl*lofts',
    'origurl*heartland+plains',
    'origurl*passagen.se',
    'origurl*djang',
    'origurl*5257+timstory',
    'origurl*5+hpe',
    'origurl*cybermog+htm',
    'origurl*area51+corridor',
    'origurl*yosemite+4301',
    'origurl*wosc+html',
    'origurl*rpp',
    'origurl*personal',
    'origurl*express-scripts.com',
    'origurl*madamecleo',
    'origurl*malibu',
    'origurl*sjsu.edu',
    'origurl*news',
    'origurl*www.ems.psu.edu',
    'origurl*7735+mcp',
    'origurl*sox+htm',
    'origurl*mcet.edu',
    'origurl*punkie+htm',
    'origurl*midnite',
    'origurl*foxy.net',
    'origurl*maple.lemoyne.edu',
    'origurl*times',
    'origurl*news.com',
    'origurl*4010+index',
    'origurl*www.slake.com',
    'origurl*larx.com',
    'origurl*www.FlowSoft.com',
    'origurl*paper',
    'origurl*members+keith',
    'origurl*paper+1998',
    'origurl*inwap.com',
    'origurl*www.tkz.fh-rpl.de',
    'origurl*members.accessus.net',
    'origurl*kattsida+html',
    'origurl*cjackson+kandinsky',
    'origurl*main+htm',
    'origurl*www.mei.co.jp',
    'origurl*www.adamspharmacy.com',
    'origurl*20+timeout',
    'origurl*corridor+5139',
    'origurl*falcon.sonic.net',
    'origurl*lycos.co.uk',
    'origurl*ran',
    'origurl*afn47757+midnite',
    'origurl*www.vrn.de',
    'origurl*5309+index',
    'origurl*stvkat+htm',
    'origurl*hpe',
    'origurl*wv+square',
    'origurl*keith',
    'origurl*www.pacificrim.net',
    'origurl*how',
    'origurl*www.metacrawler.com',
    'origurl*pacificrim.net',
    'origurl*5257+simstory',
    'origurl*cmeese',
    'origurl*metacrawler.com',
    'origurl*catscats',
    'origurl*wosc',
    'origurl*xoom.com',
    'origurl*innotts.co.uk',
    'origurl*paris+3963',
    'origurl*general+kushmerick',
    'origurl*mindlink.net',
    'origurl*cybermog',
    'origurl*kerouac+htm',
    'origurl*internauts.ca',
    'origurl*mcp',
    'origurl*martnet.com',
    'origurl*toyotaofroswell.com',
    'origurl*msheryl',
    'origurl*vault',
    'origurl*malibu+html',
    'origurl*jsd+how',
    'origurl*polypkem+index',
    'origurl*corridor',
    'origurl*zoe+html',
    'origurl*afn.org',
    'origurl*theinternetadvantage.com',
    'origurl*coltrane+htm',
    'origurl*timeout',
    'origurl*8078+home',
    'origurl*turbo+html',
    'origurl*heartland+meadows',
    'origurl*news-observer.com',
    'origurl*www.psnw.com',
    'origurl*contents+html',
    'origurl*tkaine+mogwhi',
    'origurl*stanford.edu',
    'origurl*www.bossintl.com',
    'origurl*general',
    'origurl*switchboard.com',
    'origurl*cgidir',
    'origurl*access.ch',
    'origurl*sydkat+htm',
    'origurl*hotsprings',
    'origurl*baons',
    'origurl*peace',
    'origurl*2647+chopin',
    'origurl*www.worlddrugstore.com',
    'origurl*actup+index',
    'origurl*bin+cgidir',
    'origurl*www.internauts.ca',
    'origurl*library+pitcairn',
    'origurl*inwap+cats',
    'origurl*lolnad',
    'origurl*pterry',
    'origurl*lovisa1',
    'origurl*www.dreamn.com',
    'origurl*polypkem',
    'origurl*tm.fr',
    'origurl*athens',
    'origurl*copymat.com',
    'origurl*soho',
    'origurl*victoria',
    'origurl*bossintl.com',
    'origurl*westhollywood',
    'origurl*private',
    'origurl*cat',
    'origurl*cats+htm',
    'origurl*malek+kandin',
    'origurl*yosemite',
    'origurl*www.inwap.com',
    'origurl*afn47757+plato',
    'origurl*timstory',
    'origurl*excite.com',
    'origurl*forum+8078',
    'origurl*cats',
    'origurl*kitty+and',
    'origurl*dreamn.com',
    'origurl*irish+times',
    'origurl*crawler+general',
    'origurl*bordeaux+actup',
    'origurl*hotsprings+4527',
    'origurl*kbell',
    'origurl*stinky+cats',
    'origurl*plains',
    'origurl*netscape.com',
    'origurl*csuhayward.edu',
    'origurl*safehaven',
    'origurl*tripod.com',
    'origurl*alley+6750',
    'origurl*sanjosesabercats.com',
    'origurl*www.news.com',
    'origurl*mogwhi+htm',
    'origurl*www.theinternetadvantage.com',
    'origurl*thejeep.com',
    'origurl*3963+leonora',
    'origurl*www.wednet.com',
    'origurl*autopen.com',
    'origurl*charlie+html',
    'origurl*forum',
    'origurl*kaitlin',
    'origurl*april',
    'origurl*grouper.com',
    'origurl*www.ran.org',
    'origurl*sydkat',
    'origurl*heartland+6712',
    'origurl*rpp+20',
    'origurl*2647+kaitlin',
    'origurl*psychref+html',
    'origurl*josefina3+horn',
    'origurl*malek',
    'origurl*stvkat',
    'origurl*timstory+html',
    'origurl*corp',
    'origurl*story',
    'origurl*thriveonline.com',
    'origurl*bds+punkie',
    'origurl*zoe',
    'origurl*kerouac',
    'origurl*123greetings.com',
    'origurl*2647+kerouac',
    'origurl*www.access.ch',
    'origurl*www.express-scripts.com',
    'origurl*paws',
    'origurl*wednet.com',
    'origurl*francois',
    'origurl*zdnet.com',
    'origurl*simstory',
    'origurl*www.puc.edu',
    'origurl*www.irish-times.com',
    'origurl*www.copymat.com',
    'origurl*cjackson',
    'origurl*kaitlin+htm',
    'origurl*tori+html',
    'origurl*southbeach+7597',
    'origurl*home+html',
    'origurl*hpe+10',
    'origurl*pad+htm',
    'origurl*psu.edu',
    'origurl*www.shareware.com',
    'origurl*bin',
    'origurl*arvann+pages',
    'origurl*tori',
    'origurl*dumble',
    'origurl*www.icgnet.com',
    'origurl*romancebooks',
    'origurl*3727+turbo',
    'origurl*football',
    'origurl*msheryl+djang',
    'origurl*zueri.ch',
    'origurl*aol.com',
    'origurl*bds',
    'origurl*cat+html',
    'origurl*and+tori',
    'origurl*hollywood+9662',
    'origurl*djang+html',
    'origurl*yahoo.co.uk',
    'origurl*truluck.com',
    'origurl*shtml',
    'origurl*link+chuck',
    'origurl*members',
    'origurl*www.thejeep.com',
    'origurl*2meta.com',
    'origurl*cats+html',
    'origurl*www.autopen.com',
    'origurl*hevern',
    'origurl*ikenobo',
    'origurl*accessus.net',
    'origurl*www.wco.com',
    'origurl*contents',
    'origurl*pitcairn',
    'origurl*westhollywood+4010',
    'origurl*messier+html',
    'origurl*icgnet.com',
    'origurl*mandypaul+main',
    'origurl*links+carousel',
    'origurl*square+chapel',
    'origurl*www.grouper.com',
    'origurl*6712+cats',
    'origurl*www.MeissnerChevrolet.autotown.com',
    'origurl*globec.com.au',
    'origurl*meadows',
    'origurl*afn47757',
    'origurl*www.geocities.com',
    'origurl*haestad.com',
    'origurl*midnite+htm',
    'origurl*tkaine+sox',
    'origurl*lovisa1+kattsida',
    'origurl*fr+bordeaux',
    'origurl*mindspring.com',
    'origurl*7597+kitty',
    'origurl*wired.com',
    'origurl*paws+pad',
    'origurl*irish',
    'origurl*square',
    'origurl*pad',
    'origurl*method',
    'origurl*www.mcs.csuhayward.edu',
    'origurl*www.pacific.net.sg',
    'origurl*chuck',
    'origurl*ran.org',
    'origurl*misfits2+htm',
    'origurl*stinky',
    'origurl*horn',
    'origurl*heartland+valley',
    'origurl*heartland+hills',
    'origurl*www.finest.tm.fr',
    'origurl*carousel+links',
    'origurl*kushmerick+method',
    'origurl*txt',
    'origurl*method+0',
    'origurl*title',
    'origurl*fbox.vt.edu',
    'origurl*index+html',
    'origurl*asp',
    'origurl*mcp+htm',
    'origurl*home',
    'origurl*netcenter',
    'origurl*hydrogeologist',
    'origurl*2539+index',
    'origurl*teams',
    'origurl*ion',
    'ancurl*search+direct',
    'ancurl*likesbooks.com',
    'ancurl*mirror',
    'ancurl*www.slake.com',
    'ancurl*cnet+cat',
    'ancurl*hydrogeologist',
    'ancurl*geoguide',
    'ancurl*2fcrawler',
    'ancurl*clawring+htm',
    'ancurl*tkaine+kats',
    'ancurl*labyrinth',
    'ancurl*clickthru+clickid',
    'ancurl*redirect',
    'ancurl*com+ads',
    'ancurl*area51+corridor',
    'ancurl*express-scripts.com',
    'ancurl*cnet',
    'ancurl*time+1998',
    'ancurl*josefina3',
    'ancurl*truluck.com',
    'ancurl*cgiadman',
    'ancurl*linkpg+htm',
    'ancurl*geoguide+rank',
    'ancurl*area51',
    'ancurl*www.internauts.ca',
    'ancurl*afn.org',
    'ancurl*www.microsoft.com',
    'ancurl*nph',
    'ancurl*ran.org',
    'ancurl*comprod+mirror',
    'ancurl*shareware.com',
    'ancurl*jump',
    'ancurl*area51+labyrinth',
    'ancurl*pics',
    'ancurl*nsads.hotwired.com',
    'ancurl*new',
    'ancurl*net',
    'ancurl*relocate+ad',
    'ancurl*sendform',
    'ancurl*carousel.org',
    'ancurl*members.accessus.net',
    'ancurl*home+htm',
    'ancurl*index+html',
    'ancurl*www.FlowSoft.com',
    'ancurl*kandinsky',
    'ancurl*csuhayward.edu',
    'ancurl*www.yahoo.co.uk',
    'ancurl*ibitexas.com',
    'ancurl*www.monmouth.com',
    'ancurl*infoserver.etl.vt.edu',
    'ancurl*heartland+pointe',
    'ancurl*msn',
    'ancurl*468x60',
    'ancurl*bin+click',
    'ancurl*general',
    'ancurl*keith+dumble',
    'ancurl*cqi.com',
    'ancurl*heartland+plains',
    'ancurl*redir',
    'ancurl*webring.org',
    'ancurl*uk+sv',
    'ancurl*mem',
    'ancurl*next',
    'ancurl*pacific.net.sg',
    'ancurl*monmouth.com',
    'ancurl*news',
    'ancurl*adclick',
    'ancurl*default',
    'ancurl*exe+cid',
    'ancurl*claw',
    'ancurl*bin+webring',
    'ancurl*emailbox',
    'ancurl*sunsite.unc.edu',
    'ancurl*exec+obidos',
    'ancurl*obidos',
    'ancurl*www.pacific.net.sg',
    'ancurl*library',
    'ancurl*puc.edu',
    'ancurl*internauts.ca',
    'ancurl*n+a',
    'ancurl*www.ibitexas.com',
    'ancurl*ucsd.edu',
    'ancurl*kats',
    'ancurl*cjackson+kandinsky',
    'ancurl*clickid',
    'ancurl*any+time',
    'ancurl*1+cid',
    'ancurl*ring',
    'ancurl*FlowSoft.com',
    'ancurl*nph+adclick',
    'ancurl*adclick+exe',
    'ancurl*event',
    'ancurl*general+site',
    'ancurl*2fclick',
    'ancurl*home.netscape.com',
    'ancurl*bin+homestead',
    'ancurl*click+runid',
    'ancurl*martnet',
    'ancurl*doubleclick.net',
    'ancurl*st+uk',
    'ancurl*catring+html',
    'ancurl*ng+spacedesc',
    'ancurl*familyid',
    'ancurl*yahoo',
    'ancurl*site',
    'ancurl*redirect+dll',
    'ancurl*pointe+1154',
    'ancurl*home+html',
    'ancurl*www.123greetings.com',
    'ancurl*cat+general',
    'ancurl*www.inwap.com',
    'ancurl*cjackson',
    'ancurl*labyrinth+9439',
    'ancurl*library+pitcairn',
    'ancurl*www.mei.co.jp',
    'ancurl*april',
    'ancurl*comprod',
    'ancurl*ukie',
    'ancurl*home',
    'ancurl*www.lycos.co.uk',
    'ancurl*jsd',
    'ancurl*ans+entry',
    'ancurl*keith',
    'ancurl*www.icgnet.com',
    'ancurl*forum',
    'ancurl*redirect+cgi',
    'ancurl*www.afn.org',
    'ancurl*geoguide+emailme',
    'ancurl*time',
    'ancurl*www.ran.org',
    'ancurl*main',
    'ancurl*members+keith',
    'ancurl*http+ad',
    'ancurl*2f+2fwww',
    'ancurl*ad+bin',
    'ancurl*www.amazon.com',
    'ancurl*4+mem',
    'ancurl*www.truluck.com',
    'ancurl*bridalinfo',
    'ancurl*pics+cats',
    'ancurl*direct',
    'ancurl*image+http',
    'ancurl*fbox.vt.edu',
    'ancurl*unc.edu',
    'ancurl*d+ukie',
    'ancurl*forums',
    'ancurl*co+uk',
    'ancurl*e+html',
    'ancurl*inwap+cats',
    'ancurl*malek',
    'ancurl*jsd+how',
    'ancurl*www.cqi.com',
    'ancurl*redirect+http',
    'ancurl*name',
    'ancurl*1+d',
    'ancurl*kat001',
    'ancurl*magic+adclick',
    'ancurl*grouper.com',
    'ancurl*readersndex+com',
    'ancurl*excite+468x60',
    'ancurl*click+ng',
    'ancurl*www.martnet.com',
    'ancurl*http+2f',
    'ancurl*direct+topic',
    'ancurl*how',
    'ancurl*www.wednet.com',
    'ancurl*www.globec.com.au',
    'ancurl*athens',
    'ancurl*info',
    'ancurl*schwab',
    'ancurl*sjsu.edu',
    'ancurl*www.puc.edu',
    'ancurl*a+uk',
    'ancurl*www.news.com',
    'ancurl*2fclick+ng',
    'ancurl*ads.msn.com',
    'ancurl*images',
    'ancurl*access.ch',
    'ancurl*home+ring',
    'ancurl*plug.go2net.com',
    'ancurl*geocities.com',
    'ancurl*geoguide+tour',
    'ancurl*copymat.com',
    'ancurl*www.webring.org',
    'ancurl*groupid+1',
    'ancurl*gif+pk',
    'ancurl*dll+cid',
    'ancurl*adid',
    'ancurl*vt.edu',
    'ancurl*worlddrug.com',
    'ancurl*edition',
    'ancurl*www.express-scripts.com',
    'ancurl*personal',
    'ancurl*gid',
    'ancurl*bin+geoguide',
    'ancurl*mindlink.net',
    'ancurl*accessus.net',
    'ancurl*type',
    'ancurl*ng+type',
    'ancurl*polypkem',
    'ancurl*fools',
    'ancurl*bin+cgiadman',
    'ancurl*marketplace',
    'ancurl*wco.com',
    'ancurl*links+html',
    'ancurl*tkaine+bars',
    'ancurl*icgnet.com',
    'ancurl*www.autotown.com',
    'ancurl*adcount+plx',
    'ancurl*1+familyid',
    'ancurl*yahoo.com',
    'ancurl*page',
    'ancurl*theinternetadvantage.com',
    'ancurl*runofsite',
    'ancurl*romancebooks',
    'ancurl*francois',
    'ancurl*links',
    'ancurl*athens+forum',
    'ancurl*site+sw',
    'ancurl*autotown.com',
    'ancurl*amazon.com',
    'ancurl*site+ne',
    'ancurl*qual',
    'ancurl*members',
    'ancurl*exe',
    'ancurl*pratchett',
    'ancurl*afn47757',
    'ancurl*esi',
    'ancurl*1154+linkpg',
    'ancurl*desc',
    'ancurl*forum+8078',
    'ancurl*topic',
    'ancurl*bin+acc',
    'ancurl*valley',
    'ancurl*delv',
    'ancurl*family',
    'ancurl*lspace.org',
    'ancurl*www.pacificrim.net',
    'ancurl*about',
    'ancurl*webring+ring',
    'ancurl*www.likesbooks.com',
    'ancurl*bossintl.com',
    'ancurl*runid',
    'ancurl*wednet.com',
    'ancurl*news.com',
    'ancurl*lycos.co.uk',
    'ancurl*arvann+images',
    'ancurl*526+redirect',
    'ancurl*ads+msn',
    'ancurl*netscape.com',
    'ancurl*athens+8774',
    'ancurl*click+profileid',
    'ancurl*qual+mc',
    'ancurl*homestead+ans',
    'ancurl*paul.spu.edu',
    'ancurl*dll',
    'ancurl*readersndex',
    'ancurl*dir',
    'ancurl*rank',
    'ancurl*2meta.com',
    'ancurl*Lpage.com',
    'ancurl*emailme',
    'ancurl*event+ng',
    'ancurl*3a+2fcrawler',
    'ancurl*cgiadman+dll',
    'ancurl*aol.com',
    'ancurl*lg+gif',
    'ancurl*magic',
    'ancurl*bin+ad',
    'ancurl*www.worlddrug.com',
    'ancurl*ring+midnight',
    'ancurl*irish-times.com',
    'ancurl*download',
    'ancurl*http+www',
    'ancurl*homepage',
    'ancurl*midnight+id',
    'ancurl*ad.doubleclick.net',
    'ancurl*exec',
    'ancurl*mem+1',
    'ancurl*msn+com',
    'ancurl*preferences',
    'ancurl*contact',
    'ancurl*bars',
    'ancurl*products',
    'ancurl*www.geocities.com',
    'ancurl*www.carousel.org',
    'ancurl*spu.edu',
    'ancurl*com',
    'ancurl*cnm',
    'ancurl*webring+home',
    'ancurl*link+pics',
    'ancurl*cid',
    'ancurl*psu.edu',
    'ancurl*ads+redirect',
    'ancurl*cfm',
    'ancurl*www.grouper.com',
    'ancurl*member',
    'ancurl*index',
    'ancurl*cat',
    'ancurl*www.dreamn.com',
    'ancurl*afn47757+clawring',
    'ancurl*switchboard.com',
    'ancurl*ad+preferences',
    'ancurl*inwap',
    'ancurl*adcount',
    'ancurl*martnet.com',
    'ancurl*thejeep.com',
    'ancurl*search',
    'ancurl*shtml',
    'ancurl*www+yahoo',
    'ancurl*desc+4',
    'ancurl*cats',
    'ancurl*pitcairn',
    'ancurl*www.access.ch',
    'ancurl*runofsite+any',
    'ancurl*url',
    'ancurl*dejay',
    'ancurl*gallery',
    'ancurl*bishop',
    'ancurl*features+forums',
    'ancurl*area',
    'ancurl*www.mcs.csuhayward.edu',
    'ancurl*bin',
    'ancurl*bds',
    'ancurl*www.wco.com',
    'ancurl*www.toyotaofroswell.com',
    'ancurl*www.copymat.com',
    'ancurl*go2net.com',
    'ancurl*delv+dir',
    'ancurl*slake.com',
    'ancurl*www+readersndex',
    'ancurl*www.theinternetadvantage.com',
    'ancurl*edition+cnet',
    'ancurl*tkaine',
    'ancurl*catring',
    'ancurl*asp',
    'ancurl*art',
    'ancurl*relocate',
    'ancurl*468x60+runofsite',
    'ancurl*valley+2647',
    'ancurl*any',
    'ancurl*ans',
    'ancurl*corridor',
    'ancurl*amp',
    'ancurl*top',
    'ancurl*pacificrim.net',
    'ancurl*image',
    'ancurl*heartland+valley',
    'ancurl*tii',
    'ancurl*123greetings.com',
    'ancurl*ads',
    'ancurl*www.yahoo.com',
    'ancurl*cuteftp',
    'ancurl*acc',
    'ancurl*bin+search',
    'ancurl*www.shareware.com',
    'ancurl*dir+qual',
    'ancurl*globec.com.au',
    'ancurl*www.news-observer.com',
    'ancurl*index+htm',
    'ancurl*xmission.com',
    'ancurl*ukie+s',
    'ancurl*629+redirect',
    'ancurl*pointe',
    'ancurl*www.ems.psu.edu',
    'ancurl*http+ads',
    'ancurl*members.aol.com',
    'ancurl*www.xmission.com',
    'ancurl*yahoo.co.uk',
    'ancurl*ring+kat001',
    'ancurl*msn.com',
    'ancurl*groupid',
    'ancurl*geoguide+geoguide',
    'ancurl*sanjosesabercats.com',
    'ancurl*excite',
    'ancurl*april+fools',
    'ancurl*slagen',
    'ancurl*link',
    'ancurl*tour+member',
    'ancurl*uk',
    'ancurl*people',
    'ancurl*sw',
    'ancurl*sv',
    'ancurl*st',
    'ancurl*sj',
    'ancurl*pl',
    'ancurl*pk',
    'ancurl*www.bossintl.com',
    'ancurl*ng',
    'ancurl*ne',
    'ancurl*mc',
    'ancurl*lg',
    'ancurl*spacedesc',
    'ancurl*it',
    'ancurl*ie',
    'ancurl*id',
    'ancurl*go',
    'ancurl*gg',
    'ancurl*homestead',
    'ancurl*bin+nph',
    'ancurl*peace',
    'ancurl*t+s',
    'ancurl*de',
    'ancurl*co',
    'ancurl*dreamn.com',
    'ancurl*plx+http',
    'ancurl*ad',
    'ancurl*rnd',
    'ancurl*dumble',
    'ancurl*kandinsky+p',
    'ancurl*linkpg',
    'ancurl*webring',
    'ancurl*ref',
    'ancurl*www.2meta.com',
    'ancurl*ran',
    'ancurl*ukonline.co.uk',
    'ancurl*www.Lpage.com',
    'ancurl*2fwww',
    'ancurl*ring+claw',
    'ancurl*midnight',
    'ancurl*hotwired.com',
    'ancurl*how+htm',
    'ancurl*tm.fr',
    'ancurl*ora.com',
    'ancurl*marketing',
    'ancurl*toyotaofroswell.com',
    'ancurl*heartland',
    'ancurl*s+b',
    'ancurl*mcet.edu',
    'ancurl*clickthru',
    'ancurl*enchantedforest',
    'ancurl*clawring',
    'ancurl*news-observer.com',
    'ancurl*plains',
    'ancurl*entry',
    'ancurl*acc+clickthru',
    'ancurl*3a',
    'ancurl*www.irish-times.com',
    'ancurl*2f',
    'ancurl*dll+type',
    'ancurl*soho',
    'ancurl*features',
    'ancurl*adclick+id',
    'ancurl*click',
    'ancurl*preferences+com',
    'ancurl*claw+id',
    'ancurl*url+http',
    'ancurl*microsoft.com',
    'ancurl*excite.com',
    'ancurl*web.ukonline.co.uk',
    'ancurl*gort.ucsd.edu',
    'ancurl*com+home',
    'ancurl*2fcgi',
    'ancurl*pub',
    'ancurl*bin+adcount',
    'ancurl*mc+3a',
    'ancurl*phoenix.mcet.edu',
    'ancurl*plx',
    'ancurl*inwap.com',
    'ancurl*www.cob.sjsu.edu',
    'ancurl*www.thejeep.com',
    'ancurl*tour',
    'ancurl*www.sanjosesabercats.com',
    'ancurl*mei.co.jp',
    'ancurl*geoguide+sendform',
    'ancurl*services',
    'ancurl*type+click',
    'ancurl*tagvalues',
    'ancurl*profileid',
    'ancurl*zueri.ch',
    'ancurl*arvann',
    'ancurl*tagvalues+13',
    'alt*your',
    'alt*and',
    'alt*top',
    'alt*all',
    'alt*email',
    'alt*out',
    'alt*our',
    'alt*here',
    'alt*the',
    'alt*download',
    'alt*netscape',
    'alt*what',
    'alt*explorer',
    'alt*more',
    'alt*for+a',
    'alt*take+a',
    'alt*kitty',
    'alt*visit+our',
    'alt*link',
    'alt*wacky',
    'alt*like',
    'alt*heartland',
    'alt*visit+the',
    'alt*romance',
    'alt*online',
    'alt*products',
    'alt*now',
    'alt*click+here',
    'alt*pages+like',
    'alt*rank',
    'alt*information',
    'alt*us',
    'alt*send+this',
    'alt*up',
    'alt*to',
    'alt*chat',
    'alt*new',
    'alt*net',
    'alt*my+site',
    'alt*search',
    'alt*this',
    'alt*on',
    'alt*of',
    'alt*links',
    'alt*bytes',
    'alt*my',
    'alt*me',
    'alt*picture',
    'alt*club',
    'alt*pages',
    'alt*this+page',
    'alt*in',
    'alt*on+the',
    'alt*guestbook',
    'alt*by',
    'alt*click',
    'alt*at',
    'alt*ad',
    'alt*mine',
    'alt*amazon+com',
    'alt*web',
    'alt*next',
    'alt*news',
    'alt*info',
    'alt*join',
    'alt*of+the',
    'alt*book',
    'alt*amazon',
    'alt*forums',
    'alt*back',
    'alt*next+site',
    'alt*a+tour',
    'alt*com',
    'alt*microsoft',
    'alt*geocities',
    'alt*ring',
    'alt*cat',
    'alt*send',
    'alt*internet+explorer',
    'alt*banner',
    'alt*home+page',
    'alt*internet',
    'alt*find',
    'alt*cool',
    'alt*my+guestbook',
    'alt*here+for',
    'alt*with',
    'alt*get',
    'alt*take',
    'alt*tour',
    'alt*site',
    'alt*to+visit',
    'alt*rank+my',
    'alt*from',
    'alt*page',
    'alt*graphic',
    'alt*like+mine',
    'alt*email+me',
    'alt*visit',
    'alt*free',
    'alt*the+kat',
    'alt*award',
    'alt*services',
    'alt*about',
    'alt*for',
    'alt*here+to',
    'alt*network',
    'alt*you',
    'alt*logo',
    'alt*home',
    'alt*kat',
    'caption*and',
    'caption*home+page',
    'caption*click+here',
    'caption*the',
    'caption*pratchett',
    'caption*here+for',
    'caption*site',
    'caption*page',
    'caption*to',
    'caption*of',
    'caption*home',
    'caption*my',
    'caption*your',
    'caption*in',
    'caption*bytes',
    'caption*here',
    'caption*click',
    'caption*for',
    'caption*you',
    'ad_or_nonad']

# convert flat file into pandas dataframe
df_train = pandas.read_csv(data_train, header = None, names = columns, index_col = False,
    skipinitialspace = True, low_memory = False, na_values = '?')

# the target variable, 0 = nonad and 1 = ad
df_train['ad_or_nonad'] = df_train['ad_or_nonad'].apply(lambda x: 1 if x == 'ad.' else 0)

# insert target_ad_or_nonad as the first column, and drop ad_or_nonad
df_train.insert(0, 'target_ad_or_nonad', df_train['ad_or_nonad'])
df_train = df_train.drop('ad_or_nonad', axis = 1)

# save the dataframe as CSV file, you can zip it, upload it to t1modeler.com, and build a model
df_train.to_csv('uci_006_internet_advertisements.csv', index = False)