diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..df9efad --- /dev/null +++ b/.gitignore @@ -0,0 +1,116 @@ +# Initially taken from Github's Python gitignore file + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100755 index 0000000..124b4b3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# How to Contribute + +BERT needs to maintain permanent compatibility with the pre-trained model files, +so we do not plan to make any major changes to this library (other than what was +promised in the README). However, we can accept small patches related to +re-factoring and documentation. To submit contributes, there are just a few +small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). diff --git a/__init__.py b/__init__.py new file mode 100755 index 0000000..deb5958 --- /dev/null +++ b/__init__.py @@ -0,0 +1,14 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/data/dev.tsv b/data/dev.tsv new file mode 100644 index 0000000..8afa81f --- /dev/null +++ b/data/dev.tsv @@ -0,0 +1,201 @@ +CleanedDescription MainIndustry +service industrial steam cleaning patio deck cleaning restoration "Professional, Scientific, and Technical Services" +aaa rent eventsprovides wide range indoor outdoor party rental equipment table chair tent canopy lighting equipment great eventwe providefirstclass party rental service glendale santa clarita burbank pasadena caour party rentalsare always wellmaintained highest quality rest assured every piece party rental equipment best condition party eventwe work within budget best customer service available areaourprofessional team dedicated provide best rental service los angeles make event memorable "Professional, Scientific, and Technical Services" +aaa riteway chemical company located deerfield ave deerfield beach florida united state "Professional, Scientific, and Technical Services" +general info aaa security inc conveniently located becker drive roanoke rapid nc offer latest burglar alarm security system fire alarm home business specialize closed circuit television access control device serving area since pride great customer service whether need security system large plant burglar alarm system lake house company highly trained experienced staff assist obtaining best system need also offer hour monitoring low per month call u today talk helping home business vehicle secure time look forward serving "Professional, Scientific, and Technical Services" +aaa solar screen accounting company located n 92nd ave omaha nebraska united state "Professional, Scientific, and Technical Services" +aaa solution improves financial process address cost quality issue time starting value adding customer service necessary supporting service design lean process delivering service consider standardization automation consolidation outsourcing aaa solution provides project management project office support occasionally strengthens management team reducing avoiding expensive backlog "Professional, Scientific, and Technical Services" +aaa star bail bond legal service company located congress ave west palm beach fl united state "Professional, Scientific, and Technical Services" +aaa tidy bag located wanniassa australia company working general business business activity "Professional, Scientific, and Technical Services" +aaa trailer repair marketing advertising company located lincoln ave tacoma washington united state "Professional, Scientific, and Technical Services" +aaaa wildlife control located indooroopilly australia company working general business business activity "Professional, Scientific, and Technical Services" +founded privately owned aaai group one australia s leading specialist executive search firm client range global leader smes entrepreneurial start ups dedicated team comprises state director industry sector leader director senior associate associate consultant extensive specialist industry knowledgewith office sydney brisbane melbourne perth middle east able offer national international executive search executive recruitment service goal partner client candidate sourcing senior management industry specialist executive career engineering logistics supply chain oil gas defence aerospace mining buildingconstruction energy water waste water environmental manufacturing "Professional, Scientific, and Technical Services" +service financial servicestaxes "Professional, Scientific, and Technical Services" +aacpa inc accounting company located canal st new york new york united state "Professional, Scientific, and Technical Services" +aadnesen system graphic design company based mckiernan dr folsom california united state "Professional, Scientific, and Technical Services" +aadobe animal hospital human resource company located richmond ave staten island new york united state "Professional, Scientific, and Technical Services" +aaa vent ltd family owned run specialist commercial kitchen cleansing organisation based south coast england specialise largely total grease removal extraction system ranging small large resource vast wealth experience include carrying work multiple cafe uno chiquito s frankie benny s burger king uk holland including schiphol airport site well independent restaurant catering establishmentsour total coverage approach seek maximise safety system increase longevity equipment ensure value money also consult throughout process ensure complete picture formed location subcontracted others cover system deep cleaning well know restaurant chain seen u travel scotland northern ireland wale virtually corner england base south coast quality cornerstone organisation believe literally go company hesitation showing people work getting client inspect fact promote often amazed achievefor aaa standard cleanliness let greasebusters aaa vent ltd turn system grime slime total shine "Professional, Scientific, and Technical Services" +provide honest answer honest solution honest pricing practice customer know real way build solid loyal customer base call aadvantage pest control today free estimate "Professional, Scientific, and Technical Services" +aaen performance inc writing editing company located sheridan rd kenosha wisconsin united state "Professional, Scientific, and Technical Services" +onestop shop mobile tour vendorbringing idea life providing peace mindover mobile tour yearaardvark passionate team designer doer meticulous builder focused planner avid problem solver year experience experiential marketing industry though largescale complex experiential mobile tour forte support mobile promotion shape size 1week tour 5year tour everything inbetweenflexibility meet fastlane approachcall u need mobile touring expert ll work team along every step way get need done neurotic attention detail solutionsbasedproblemsolvingattitude make u stand rest core value s right customer else ve never experienced true vendor ve experienced working aardvarkhere s mean onestop shopwhether need aardvark shop road swiss army knife service invaluable custom tool set experiential mobile marketing tour provide resource need need push buttonprogram budget consultation idea s going costpitch assistance let u help win rfplarge inventory promotional vehicle save fortune repurposecustom vehicle fabrication complex build fanatical attention detail vehicle graphic hd graphic printing installationtour personnel driver dependable workforcerouting assistance ll make routing make sensefleet management insurance complete dot management million insurancepermits compliant tour surprisesvehicle storage warehousing fulfillment secure storage fulfillment touring itemswhen done right call aardvark "Professional, Scientific, and Technical Services" +aardvark home inspection marketing advertising company based loveless dr ne rockford michigan united state "Professional, Scientific, and Technical Services" +textile apparel manufacturing qa logistics service industrial consultancy setup cotton polycotton blended synthetic yarns2 real estate building design construction repair building maintenance hygiene service hygiene janitorial chemical tropical fruit sugar coffee Consumer Durables & Apparel +aardvark book depot consumer good company located po box shorewood wisconsin united state Consumer Durables & Apparel +aai foster grant consumer good company located 5th ave new york new york united state Consumer Durables & Apparel +aardvark supply consumer good company located po box midland texas united state Consumer Durables & Apparel +priyanka eponymous jewelry label designer priyanka luganiborn entrepreneurial family international upbringing priyanka inspired translate global travel brand age 18priyanka jewelry fuse eastern western influence create beautiful statement piece rich indian heritage visible complexity color combination stone timeless design priyanka often us charm woven silkcord bracelet represent spiritual happiness protection luckeach piece exquisite design attention detail individual stone presented ideal unique setting priyanka take holistic approach running business combining natural creativity cool business head strong social conscience priyanka stockist united kingdom include match shop bluebird wardrobe couture lab genevieve harvey nichols internationally label stocked australia asia france germany greece ireland italy holland norway russia south korea spain switzerland ukraine united state Consumer Durables & Apparel +aad inc consumer good company based pennsville pdrcktwn r pedricktown new jersey united state Consumer Durables & Apparel +service list servicespopular service Consumer Durables & Apparel +introduce ourselve leading manufacturer exporter jute bag cotton bag type bag shopping promotional wine beach gift etc assure good quality bag competitive price manufacturing unit Consumer Durables & Apparel +mission aaron barrada inc pottiestickers rewardemstickers help child learn new task manner promotes independence enhances selfesteem help make life easier mother childcaregivers commitment mission mean develop quality product reasonable price goal Consumer Durables & Apparel +aahana decor lead interior designer palvi virmani involved manufacturing modular custom made kitchen door shutter wardrobe home office furniture fixture panel retail outlet vanity shutter mdf grill wave board decorative vineers related Consumer Durables & Apparel +founded aalfs manufacturing private label leader apparel manufacturing three generation family leadership produce quality denim twill garment operation united state mexico nicaragua information please contact u 7122521877partial list customer jc penney arizona jean companytarget mossimo denim supplylevi dockerspolo ralph laurenlucky brandharley davidsonrocky mountainzumiezll beanlands endgapjkl Consumer Durables & Apparel +aakar scientific pvt ltd iso certified company one reputed reliable laboratory furniture manufacturer supplier providing complete laboratory solution provide complete laboratory solution furnishing lab effective economic way aakar supply extensive range laboratory furniture variety safety fume hood laboratory equipment s Consumer Durables & Apparel +aamelia s mater flower located brisbane australia company working art craft florist business activity Consumer Durables & Apparel +aaron direct paper forest product company located w germantown pike plymouth meeting pennsylvania united state Consumer Durables & Apparel +aalsmeer floral studio located bundoora australia company working art craft florist business activity Consumer Durables & Apparel +hotfrog u provides information regarding aaron lelah jeweler la vega nv aaron lelah jeweler located w flamingo beltway grand canyon dr suit la vega nv provides jewelry service contact visiting website http wwwaaronlelahjewelerscom Consumer Durables & Apparel +aaron s jeweler located national highway la vale maryland full service jewelry store specializing jewelry sale custom jewelry design manufacturing jewelry repair aaron s jeweler owned operated aaron brotemarkle graduate gemologistjeweler gemological institute america santa monica california Consumer Durables & Apparel +aardvark swim sport designed logo printed swim suit speedo usa olympic team canadian olympic team Consumer Durables & Apparel +western pennsylvania s largest oriental area rug washer one stop shop oriental area rug service including oriental area rug washing rug repair new antique rug sale rug padding aaron s rug care handeld major restoration antique heirloom well stitching corner macine made kitchen rug leader aaron stewardson studied oriental rug joined society like hajji baba textile museum type backround allows u focus conservation well cosmetic team understands value customer alwayse put customer need ahead thats service count visit website find free pickup delivery ship rug u Consumer Durables & Apparel +aaron gun sporting good company located rr box 112a greensburg kansa united state Consumer Durables & Apparel +aaa ohio motorist assn insurance company located cleveland ohio united state Finance and Insurance +aaa texas member club affiliated american automobile association aaa national federation serf member state texas exception texarkana Finance and Insurance +service insurancemotor vehicle Finance and Insurance +aadi inc financial service company located south ave north lima oh united state Finance and Insurance +aadvantage mortgage financial service company located w sam houston pkwy houston texas united state Finance and Insurance +aag insurance enterprise insurance company located bel air south pkwy bel air maryland united state Finance and Insurance +aanig inc insurance company located primera blvd ste lake mary fl united state Finance and Insurance +aana insurance svc insurance company located prospect ave park ridge il united state Finance and Insurance +aarya innovation leading fast growing marketing research financial research analytics company headquarters faridabad haryana india s member aarya india group includes three company aarya packaging aarya innovation amogh creation add value client real value develop learning culture foster innovation also contribute academia policy maker developed sustainable working environment good fair u client empower customer market large independent analysis benchmark tool help lender borrower issuer investor regulator market intermediary make betterinformed investment business decision offering allow market market participant become transparent efficient mitigating managing risk taking pricing decision generating revenue reducing time market enhancing return helping shape public policy infrastructure emerging market help catalyse economic growth development country Finance and Insurance +ab business group investment management company based colonial heritage dr nashville tennessee united state Finance and Insurance +aau karate insurance company located ne 177th st shoreline washington united state Finance and Insurance +service financial servicesinvestment Finance and Insurance +abacus bookeeping financial service company located viewside dr north potomac maryland united state Finance and Insurance +hotfrog u provides information regarding abacus financial group inc los angeles ca abacus financial group inc located victoria ave los angeles ca provides financial service service contact visiting website http abacusfinancialgroupcom Finance and Insurance +abacus equity partner venture capital company located madison ave new york new york united state Finance and Insurance +abacus financial service located 12a saturn place north harbour new zealand company working financial activity business activity Finance and Insurance +abacus investment located whitaker st te aroha waikato new zealand company working investment company financial activity business activity Finance and Insurance +abacus lending corp financial service company located wynkoop st denver colorado united state Finance and Insurance +abacus specialty inc insurance company located somers rd hampden massachusetts united state Finance and Insurance +abba insurance agency insurance company located wilson woodard rd eastman georgia united state Finance and Insurance +aah pet computer software company located aspen dr lewisville texas united state Software & Services +aagna websoft software development company specializing whole range custom software development web design web development successful onshore offshore business model web design web development software development erp crm billing software hrms mobile application development android blackberry iphone io window Software & Services +long history proven track record success aama silicon valley s leading business network promoting success technology enterprise pacific rim deep root wide reach local silicon valley community greater asia aama today worldwide network representing well company aama member diverse influential sector commerce industry reaching wide across entire span asia pacific rim aama boast strong lineup various program ranging monthly speaker series meeting premiere annual connect conference attracts participant year china aama chapter beijing shanghai Software & Services +aamra network limited formerly global online service limited last decade consistently provided customer stateoftheart communication solution client able rely ability provide stable consistent connectivity solution using state art backbone infrastructure ensured client minimal worry come dependability communication turn ensured u enviable list bluechip customer internet related value added service critical input business corporate bangladesh one obvious choiceaamraaamra network limited employ employee diverse skill set expertise year experience anl able accurately ass constantly changing customer requirement offering extensive affordable service available anl place due importance quickly adopting new technology investing budget r company also strives maintain international standard product service anl upholding iso90012008 certification last yearsthe company one concern aamra group company group diversified investment ict textile lifestyle sector bangladesh starting group presently employ individual Software & Services +aandrajax canadian information technology consulting firm predominantly focused niche expertise area drawn many year expertise management associate consultant servicing banking financial service telecommunication healthcare education retail government besides critical technology solution backdrop north american offshore setting spinoff successful canadian branch niche global software solution firm delivered credible solution valued canadian u client decade post eranow management team one half decade industry backed unmatched technical expertise insight maintained highest level compliance quality go changing time technology combined knowledge investment backed year r led creation lab center excellence produced innovative solution led way enhancing business customer delightwe dedicated helping client grow combined businessit knowledge thoughtful innovation global scale year ve built expertise run broad deep across almost every major industry today enables u anticipate client requirement fulfill demand emerges Software & Services +small town pawn shop offering deal quality used merchandise sale cash valuable via consignment collateral loan outright purchase everyday low price dvd s video game fine gold jewelry electronics power tool musical instrument computer Software & Services +aarsh infotech expert offshore web design development company india welcome outsource web design development mobile app development seo service cost effective solution Software & Services +aarya technology limited computer software company located flat cooper court axe street barking essex united kingdom Software & Services +unstructured peertopeer network formed node randomly form connection easy build b optimized locally c highly robust large number peer frequently join leave networkhowever design cause limitationsa bandwidth floodingb high signaling trafficc high cpu memory usaged unpredictable responseaarzoo potentially solved problem creating classification method peer tier method flexible scalable matrixour first solution performance tuned robust peer peer broadcasting software data vault used secure content ensure compliance copyright intellectual right Software & Services +aba group inc provides global network industry expert automotive aerospace oil gas power industry partnering customer deliver guaranteed superior client result additionally support strategic marketing campaign sale initiative provide new customer introduction provide service Software & Services +aba intl computer software company located orange st brooklyn new york united state Software & Services +ab computer system inc information technology service company located broadhollow rd melville new york united state Software & Services +aaworldsalescom computer software company located hampshire dr south bend united state Software & Services +abacus epos provides transaction solution software equipment help customer receive payment correct efficient mannerour product range includes basic cash register touch screen service point mobile ordering device cashless transaction system pride finding solution problem others failed utilising innovative developer experienced engineersour system provide business ability achieve higher service productivity labour saving reduction operator errorsother benefit include easy product price change detailed management information stock control management integrated stock ordering system integrated account customer loyalty marketing initiative cashless payment prevention theft breakage fraudour aim provide customer smooth profitable operation using latest technology whatever need abacus solution Software & Services +abacus solution inc developer saturn integrated enterprise etrm system front mid back office decision support capability additional unique tool offer solution key business functionssaturn enterprise business management system integrates multiple corporate application facilitates improved decision help analyze corporate functional risk revenue cost profit portfolio plant trade asset using modern technology saturn help user realise significant benefit gain unique sustainable competitive advantage Software & Services +ab technology inc computer software company based drawer shrewsbury shrewsbury new jersey united state Software & Services +abaris delivers fully integrated oracle solution accelerate growth streamline operation create new level customer value build effective business solution drive business performance create competitive advantageabaris specializes oracle r12 implementation upgrade complete project engagement staff augmentation Software & Services +abbey inn hotel located buckfast road tq11 0ea buckfastleigh united kingdom company working computer software solution computer service business activity Software & Services +abc manufacturing inc computer software company located e 4th st 2b loveland co united state Software & Services +abc softwork s mission make supply chain analytics driver creating outstanding result believe people key change develop software service challenge common practice status quowe strive keep software service simple possible successfully developed one world s simple powerful analytical toolsour platform highly relevant supply chain professional company itemsabc softwork privately held headquartered copenhagenfor danish visitor view short presentation og basic concept http wwwslidesharenetabcsoftworkabcanalysenintroduktiontildobbeltabc Software & Services +aaa protection inc electrical electronic manufacturing company located w market st greensboro north carolina united state Technology Hardware & Equipment +goallineca number one name sport administration software goalline specializes building web based tool enhancing sport organization Technology Hardware & Equipment +aaaittech provides full complement call center contact center software consulting service tailored individual need client experience track record successful project result key reason aaaittech established reputation leading call center consulting firm following representative list Technology Hardware & Equipment +distributor consumer electronics Technology Hardware & Equipment +aaci located pasay city philippine company working electrical good air conditioning construction business activity question please contact u Technology Hardware & Equipment +private advertising publication distribution industry provides high quality lower cost direct door distribution service alternative advertiser seeking improve profit Technology Hardware & Equipment +aadi semicon global technology consulting company strong focus providing endtoend engineering service segment semiconductor industry team technically strong highly experienced professional strive provide best inclass solution customer special emphasis quality schedule Technology Hardware & Equipment +aadya switch gear pvt ltd take opportunity introduce respected low voltage switchboard manufacturer engaged business providing total solution cutomized low voltage switchboard control automation Technology Hardware & Equipment +aarkay sign located bangalore india company working advertising sign electrical equipment electrical good business activity Technology Hardware & Equipment +aaction service electrical electronic manufacturing company located santos rd fort myers beach fl united state Technology Hardware & Equipment +strong option server drive memory motherboards networking new used server pc drive pcmemory brand visit web site wwwaajalliancebiz forinventory Technology Hardware & Equipment +aaron associate ct inc computer networking company located w main st waterbury connecticut united state Technology Hardware & Equipment +al ansari power technology india private limited aapt india part four decade old al ansari group company executing electrical project past five year across various location indiathe al ansari group company est muscat oman employee al ansari evolved dynamic leading group sultanate oman group currently spread activity across electromechanical work civil construction landscaping irrigation system including marketing high quality building service product subsidiary across middle east indiafor past five year aapt india executed order various segment various reputed corporate government organization like dmrc tyco electronics indospace keihin automotive enerparc tata power solar xl energy precot meredian air liquide wipro infrastructure group saint gobain asian ppg wipro technology shimzu corporation ztt cable wipro infotech micro lab jipmer accenture kapil group transit orient devolopment etc Technology Hardware & Equipment +aaron air conditioning svc electrical electronic manufacturing company located oak tree dr ste b houston texas united state Technology Hardware & Equipment +aaryan computer computer hardware company located holworthy way sacramento california united state Technology Hardware & Equipment +ab audio electrical electronic manufacturing company based canby ave reseda california united state Technology Hardware & Equipment +abacus print located greenford united kingdom company working computer consumables printer supply business activity Technology Hardware & Equipment +abacus llc online retailer discount quality printer ink toner cartridge well office home business supply accessory founded abacus established top merchant printer ink accessory market ebay amazon website operation abacus llc based phoenix arizona european warehouse united kingdom customer service headquarters novi sad serbia Technology Hardware & Equipment +abaton located warszawa poland company working security service electrical good business activity Technology Hardware & Equipment +measurement made easy goal make instrumentation analyzer technology selection purchasing installation operation ownership easy focus businesswe located main part country auckland northland mid south north island south island locally accessible help solve measurementrelated pain pointshttp wwwabbconzmeasurement abb wwwabbcom leader power automation technology enable utility industry transport infrastructure customer improve performance lowering environmental impact abb group company operates roughly country employ people Technology Hardware & Equipment +aaa flooring inc locally owned registered contractor serving residential commercial customer throughout greater phoenix area give u call today find customer keep coming back driven quality product highly skilled trained staff truly love stuff driven desire meet expectation exceed Construction +aaa pro plumbing rooter plumbing company based san jose california specialize complete residential commercial industrial plumbing service repair specialize lot new plumbing technology including state art trenchless sewer replacement innovative sewer video camera inspection leak detection well solar tankless water heater repair replacement installation one source plumbing need Construction +aaa restoration company onestop singlesource solution homeowner disaster aaa restoration company minorityowned business serving client metro atlanta metro nashville county using methodology created year reconstruction industry expertise experience aaa restoration company help property owner restore home businessesaaa restoration company proud member following organization georgia chamber commercemetro atlanta chamber commercefayette chamber commercehenry county chamber commerceria restoration industry associationscrt society cleaning restoration techniciansinstitute inspection cleanind certification clean trusthenry council quality growth Construction +provide complete window door solution siding roofing remodeling project stand commitment deliver quality product superior service absolute customer satisfactionwe offer standard custom product provide flexibility infinite possibility featuring extensive assortment quality brand accommodate project design aspect budgetswhether re building remodeling simply replacing window door team provide fact guidance required make important investment decision knowledgeable sale consultant inhouse installation technician competitively priced product industry leading warranty simply ca nt go wrong please feel free call arrange detailed obligation quote Construction +aaa southern state cleaning construction company located po box cochran georgia united state Construction +service storage servicesgeneral storagespeciality storage Construction +land surveyor civil engineer Construction +aaaa quality garage door co offer garage door repair installation antioch ca residential commercial purpose visit u today Construction +aad design multi disciplinary building design firm provide architectural design town planning expertise heritage advice manner building project based paddington brisbane work includes type development home addition renovation commercial hirise experience broad Construction +aadvance contracting construction company located dhemecourt st new orleans louisiana united state Construction +aaa wysong glass co glass ceramic concrete company based 5044b coosaw creek blvd north charleston sc united state Construction +aardvark drilling inc provides high quality drilling service throughout bermuda southern ontario eastern canada done since drilling contractor provide equipment expertise mineral exploration well abandonment hydrogeological geotechnical environmental drilling many project terrain service aardvark drilling owns operates fleet drill rig employ people serf engineering planning firm mining company exploration group public private sector governmentaardvark drilling integrated management approach encompassing entire team management approach strive innovative drilling service solution client reliable crew equipment employee accountability safe work environment employee subcontractor client representativesaardvark drilling licensed well contractor ministry environment employ certified cmt installers licensed well technician assistant well tech first line supervisor blow prevention employee whmis propane handling fire extinguisher fall arrest first aid trained Construction +downer grove electrical service provider provides electrical troubleshooting service service panel upgrade service code violation punch change service downer grove surrounding area Construction +aardvark electric located guildford australia company working engineering business activity Construction +aardvark welding co construction company located pine st elizabeth new jersey united state Construction +aargee professionalism innovation technical excellenceaargee skilled service professional expertise electrical sanitary fire fighting work multistoried building hotel industrial plant repeat business get client power customer confidence integrated planning contracting experience total quality unconditional system highcaliber professional power aargee customer satisfaction provider low cost engineering solution infrastructure sector Construction +offer consultancy outsourcing service business growth Construction +aaction electric construction company located northwestern hwy farmington hill michigan united state Construction +leading melbourne shopfitting company offering fit advice product knowledge industry retail interior design trend latest shopfitting news helpful hint information shopfitting assist increase foot traffic boost sale build customer acquisitionaafs shopfitting client offered unique opportunity oversee transformation space real time online client portal image uploaded secure account via website http aafsshopfittingcomau throughout build offering personalised experience like otherwith almost year experience commercial construction team friendly qualified shop fitter utilize quality joinery produce work contributing longevity performed combining skilled experienced professional firstclass machinery ensures joinery manufactured pace precision image completed fitout project detailed case study check aafs shopfitting s portfolio http aafsshopfittingcomauportfolio Construction +aag construction inc construction company located po box phoenix arizona united state Construction +aaa wholesale transmission automotive company located n dixie hwy fort lauderdale fl united state Automobiles & Components +aafjes car system gedreven door zijn ervaring de automotive en passie voor alles wat met de auto te maken heeft paul aafjes gestart met aafjes car systemswerkend voor particulier en al freelance inbouw monteur voor diverse grote leveranciers van track trace systemengespecialiseerd de inbouw en het onderhoud van diverse elektrische installaties voertuigen waaronder voertuigvolgsystementrack tracefleetmanagement systemenalarmencarkitsnavigatiesystemen multimediasystemen daarnaast het motto kan niet bestaat niet en er meer mogelijk dan dat u denktdus voor vragen offertes inhuur een kop koffie kunt u mij bereiken op0623916818of kijk op wwwincarsystemsnl Automobiles & Components +founded aarco transmission auto repair rich background san antonio aarco automotive provides san antonio surrounding community complete automotive maintenance repair service including transmission rebuilds replacement brake work engine repair oil change tuneups air conditioning battery experienced trained certified technician make sure get work need done right timely manner addition wide range service provide one town offer year mile nationwide warrantywe conveniently located ih north new braunfels exit access road new braunfels walter drop car take advantage complementary shuttle service dedicated technician complete free thorough diagnostic vehicleaarco transmission auto repair offer free obligation diagnostic vehicle come door diagnostic process call road lift one important step repairing vehicle provides customer clear answer vehicle problem technician able provide reliable estimate cost repair Automobiles & Components +aaron executive livery set standard since houston area limousine chauffeured transportation industry houston texas Automobiles & Components +aaron auto automotive company located finch dr ste longview washington united state Automobiles & Components +hotfrog u provides information regarding aaron auto service hanover township pa aaron auto service located sans souci pkwy provides automotive maintenance repair service contact Automobiles & Components +independent mobile autolocksmiths service offering mobile key cutting including loststolen key carry large stock key blade covering majority market carry range remote key also would impossible carry stock every vehicle case order vehicle transponder programming since late s almost car key chip inside speaks computer vehicle engine required making key otherwise key turn ignition engine start small number vehicle still made dont chip almost prevents wouldbe car thief simply forcing ignition screwdriver hotwiring car even nonremote key transponder vehicle entry loststolenlocked key tool gain entry nondestructively car may involve picking lock method note thing seen movie like slimjims tennis ball worklock repairreplacement remote key likely dont use door lock cause seize strip repair different key door ignition change use keyremote control repair lot people problem remote could simple cosmetic issue button rubber perished sometimes button work case repair install new button onto remote much cover majority car make model constantly expanding ability recently added ability bmw mini s soon hope add ability mercedes benz vehicle situation try resolve first visitcontact u quotation Automobiles & Components +ab pneumatics ltd uk ireland leading producer seat airsprings product shockabsorber sleeve donut rubber spring easy see expert vibration control product tested iso standard proven track record field pride qualitybased moira 6000sq ft manufacturing facility enables fast delivery product shipped world Automobiles & Components +hotfrog u provides information regarding abc auto repair newport news va abc auto repair located jefferson ave newport news va provides auto repair service contact Automobiles & Components +abc auto service located hiraokacho japan company working vehicle service business activity Automobiles & Components +abc motor market wide range nissan passenger commercial vehicle well nissan diesel truck tractor bus abc motor market leader past consecutive year abc autotech ltdabc autotech ltd sole distributor fiat alfa romeo porsche vehicle ginza motorsginza motor dealer second hand vehicle imported directly japan singapore abc coach work ltdabc coach work ltd member company abc group categorised pioneer field luxury coach building abc coach work business manufacturing body bus since factory registered office company located le guibies pailles among diversified product service supply conditioning registered body cooling unit air conditioning bus distribution installation tail lift market second hand imported truck also import sell fully built busescoaches yutong brand quikfix car centrequikfix car centre brings new dimension automotive service house one roof tyre centre equipped latest wheel balancing alignment facility also provides complete solution regarding maintenance brand model vehicle service philosophy car centre deliver consistent superior standard service competitive pricing abc marketingabc marketing ltd specializes importation distribution european asian high quality tyre affordable price like vredestein hankook apollo ornet abc marketing ltd also exclusive distributor armor industry leader car care appearance product Automobiles & Components +abba driving school automotive company located ne 9th ave gainesville florida united state Automobiles & Components +abc motor official renault dacia representative estonia established operate biggest renault dacia sale service centre tallinn official car partner estonian football association estonian song dance celebration foundation tallinn town theatre tallinn philarmonic society birgitta festival estonian rollerskating association many event organization Automobiles & Components +heart nissanwe nissan car dealer since address purchase maintenance nissan flevoland since started new bussines zwollewe nissan people first hour know brand well Automobiles & Components +abdoo wrecking automotive company located sand rd fremont ohio united state Automobiles & Components +aberdeen sport recreation automotive company located east dr aberdeen south dakota united state Automobiles & Components +abik located aleppo syria company working vehicle service business activity Automobiles & Components +abilene used car sale inc automotive company located 11th st abilene texas united state Automobiles & Components +abel notary service llc familyowned operated business offering notary license vehicle service york lancaster county Automobiles & Components +absolute auto sale automotive company located w oglethorpe blvd albany georgia united state Automobiles & Components +provide fund management service fixed income equityfocused portfolions corporate client pension fund insurance company individual company high networth individual well individual retail market offer service manage client portfolio form discretionary fund advisory fund depending Educational Services +year aaa tutorial provided highest standard vce tutor melbourne wide enabling student learn comfort home trusted tutoring service registered member australian tutoring association convenient payasyougo system contract advance payment tutor Educational Services +hotfrog u provides information regarding aardvark pizza killington vt aardvark pizza located e mountain rd provides social food dining restaurant service contact Educational Services +aarnun galleryfine picture framing year exceptional resource fine art distinctive custom framing Educational Services +aamc training group australianbased registered training organisation rto number operating nationally internationally operation located strategically india indonesia philippinespoints difference experience knowledge understanding vision industry support put u ahead rest offer flexible option learning caters busy lifestyle businessindividual training need support learning objective planning principal australian international corporate client strategic partnership nationally internationally enhance broaden service offering clientswhether looking first job promotion career change need extra knowledge skill aamc training help aamc training work closely industry employer group ensure training relevant high standardwe offer training following industry accounting bookkeeping banking finance business management financial planning insurance finance broking training assessment professional development property service antimoney laundering work health safety retail sale marketing mentoring coaching tourism travel hospitality transport logistics information technology bpo cultural english economics Educational Services +aardvark child care education management company located genie st se salem oregon united state Educational Services +abacus computer training located rickmansworth united kingdom company working secretarial service training business activity Educational Services +abacus training college computer training company based auckland new zealand specialist training mous computer course Educational Services +specialist company offering high end training consultancy variety sector including health social care leisure security business leadership management teaching training offered accredited simply inhouse bespoked specific need client client base range private public sector includes small business multi national benchmark set director company instructor figureheadswe also operational arm supply close protection personnel corporate security finite number client taken job job basis subject certain condition met within operational criterion client matter small large treated highest level privacy professionalism pride sharp end s belong Educational Services +abacus tutor located rickmansworth united kingdom company working education business activity Educational Services +abay orthodontics higher education company located e douglas ave wichita kansa united state Educational Services +abbey montessori marin accepting enrollment infant toddler preschooler Educational Services +aiken bamberg barnwell edgefield regional library system Educational Services +abbeville presbyterian church higher education company located n main st abbeville south carolina united state Educational Services +abbey boutique education management company located e cherry st rahway new jersey united state Educational Services +committed equality opportunity service provided customer aspect employment warmly welcome application sector community recruitment policy procedure practice enable applicant considered merit ability job make reasonable adjustment line Educational Services +abc barn child apos place education management company located fury ln la mesa california united state Educational Services +abbotsholme school located rocester st14 5bs uttoxeter united kingdom company working school business activity Educational Services +abbeville memorial library library company located kirkland st abbeville alabama united state Educational Services +abbey gate college located chester united kingdom company working school business activity Educational Services +aabsolute painting environmental service company located old cheney rd lincoln nebraska united state Energy and Environmental +aardvark pest control based wigan lancashire expert wildlife management fumigation flea bed bug cockroach ant mouse rat mole wasp nest removal service provide throughout northwest england rsph bpca qualified basisprompt member crru wildlife aware accredited Energy and Environmental +aaron chiropractic clinic oil energy company located stellhorn rd fort wayne united state Energy and Environmental +largest collection historic cigarette lighter presently display xinhai electric co ltd ningbo chinaone foundation civilization know ability make transport fire ever since cavedwelling ancestor rubbed stick stone together humankind pursued quest efficient reliable firemaking device quest inspired innovative invention history collection dutch lighter museum foundation trace fascinating journey fire steel ancient world tinderbox seventeenth century modern cigarette lighter carefully collected astonishing variety designsthe numerous unique distinctive whimsical lighter chart voyage lighter important scientific discovery treasured collector s item along way learn history pipe popularization smoking growth today s global marketplace well significant event wondrous invention played role history lighter Energy and Environmental +manchester electrical contractor eca approved electrical contractor electrical installation service inspection testing electrical maintenance emergency lighting system company manchester north west Energy and Environmental +aaron landscaping environmental service company located little eagle creek westfield indiana united state Energy and Environmental +aaxon lab inc oil energy company located augusta dr fort worth texas united state Energy and Environmental +ab resource ab fast growing privatelyheld company actively engaged natural gas oil exploration development production appalachian basin company s prime marcellus shale acreage pennsylvania west virginia retains potential horizontal well location company s strategic direction Energy and Environmental +abalon exterminating co environmental service company located park ave new york new york united state Energy and Environmental +world s leading manufacturer industrial oil skimmer coolant maintenance product Energy and Environmental +abbey appliance oil energy company located independence missouri united state Energy and Environmental +great quality great service great price tool equipment supply need Energy and Environmental +abb technologybased provider power automation product system solution service building core strength technology leadership pioneering spirit sustainable approach business abb help customer become profitable lowering environmental impact abb operation nigeria headquartered lagos branch abuja porthatcourt abb operates around country employ approximately people truly multinational organization global headquarters zurich switzerland abbng incorporated registered corporate affair commission federal republic nigeria 24th february company registration rc Energy and Environmental +general info dependable home heating oil available Energy and Environmental +abbottstown paradise joint environmental service company located w water st abbottstown pennsylvania united state Energy and Environmental +abesim expert soil risk managementwe want expert indepth market knowledge pragmatic solution soil pollution deal itour methodology follows law procedure standard reference guide ovam bim owd specific belgian region flanders wallonia brusselscapital france polandabesim reached process quality management compliance iso standard site engineering service consulting concerning environmental diagnostics soil soil decontamination geotechnics soil structure interactionabesim competent site investigation measurement campaign characterize state environment indicate source spread potential target pollution realization actualization concept investigation environmental vulnerability assessment state environment management plan analysis residual risk view potential action concerning use soil state environment abesim equipment necessary investigate potentially contaminated site soil equipment accomplish mission three hydraulic drilling machine mounted 4x4 truck rotary drilling solid hollow auger drilling percussion drilling percussion gouge drilling several van manual drilling water sampling Energy and Environmental +ability tree svc environmental service company based dhanmandi cir tampa florida united state Energy and Environmental +abl landscaping inc oil energy company located elmgrove st farmington hill michigan united state Energy and Environmental +abode plumbing located queanbeyan australia company working oil gas company business activity Energy and Environmental +plumbing located warner bay australia company working oil gas company business activity Energy and Environmental +aaa radiator food beverage company located beech ave melrose massachusetts united state Food and Beverages +aaha impex one leader amongst manufacturer exporter wide range premium quality cooking spice cooking masala indian pickle food product offering includes indian pickle cooking spice cooking masala medicinal herb naturally dried vegetable gravy chutnies also major exporter branded fm Food and Beverages +aaron vending food beverage company located beachview dr se smyrna georgia united state Food and Beverages +ab farm located kampung terang malaysia company working food retailer food product business activity visit company website information u http abfarmblogspotcom question please contact u Food and Beverages +aba food specialises quality african food product steadily gaining popularity mainstream market throughout britainthe company produce corn dough cassava dough ogi spicy sauce sheto snapped customer moment delivered retail outlet aba food product added advantage free artificial preservative colouring Food and Beverages +abacus consulting llc develops custom software improve use time matter billing matter nearly window application little single hour time may save hundred hour staff time write software automate repetitive action enhance functionality integrate application parse convert data Food and Beverages +ab restaurant largest restaurant company kazakhstan comprising establishment various price range dining format group specialises international cuisine restaurant serving kazakh dumpling traditional dish caucasus homecooked italian meal german beer french pastry first restaurant chain opened early askar baitassov launched managing company ab restaurant vision creating chain stylish eatery across kazakhstan would make cultural social impact city life since foundation managing company baitassov overseen rapid expansion ab restaurant today restaurant distinct brand share commitment fine cuisine convivial atmosphere stunning internal decor working ab restaurant mean joining team strong professional becoming valued member guarantee career growth free training every employee moreover committed high standard corporate culture since hard work u part impeccable service customer care build good reputation company Food and Beverages +abad enterprise food production company located colcord ave waco texas united state Food and Beverages +abali foreign trade co young energetic company focused producing trading high quality olive oil olimare brand turkeyour headquarters located istanbul production facility settled city antakya antioch southern anatolia product pure olive oil natural state pressed prepared without treatment process combine benefit traditional methodology upmost contemporary concern food hygiene nutritional value production packaged various sized glass bottle aimed local international market particularly spot authentic natural homemade product favoredour olive grove located southern anatolia mediterranean coast olive cultivation go back thousand year location advantage procure one best olive oil produced mediterranean region expert selection olive oil bottled consumption Food and Beverages +abbey security inc restaurant company located white st springfield massachusetts united state Food and Beverages +breakfast lunch dinner riverfront view Food and Beverages +abc bakery supply food beverage company located nw 1st ave miami florida united state Food and Beverages +abbondanza pizzeria pride preparing delicious meal every customer every time ingredient fresh staff friendly courteous convenience offer dinein pickup delivery service check menu place order let u show customer satisfaction number Food and Beverages +abbott farm inc food beverage company located e 160th ave keenesburg colorado united state Food and Beverages +abc liquor inc information technology service company located n federal hwy boca raton florida united state Food and Beverages +abc refreshment customer service number one priority ability create customized break room solution includes stateoftheart vending machine office coffee equipment water cooler necessary break room supply Food and Beverages +abc restaurant located vietnam company working restaurant business activity Food and Beverages +abc restaurant supply located northbridge australia company working general business business activity Food and Beverages +somers home somers hour tree service tree removal stump grinding Food and Beverages +abela co located dubai uae company working catering business activity visit company website information u http wwwabelaandcocom question please contact u Food and Beverages \ No newline at end of file diff --git a/data/train.tsv b/data/train.tsv new file mode 100644 index 0000000..e73a35b --- /dev/null +++ b/data/train.tsv @@ -0,0 +1,1001 @@ +CleanedDescription MainIndustry +beta highly motivated group advertising marketing medium communication people beta fast experienced effective idea create manifest believe advertising solution mean one believe successful marketing energy teamwork continuous testing improvement "Professional, Scientific, and Technical Services" +business service located romsey united kingdom company working audit accounting business activity "Professional, Scientific, and Technical Services" +big online result small medium business strategic online marketing service consulting local business wherever website s great know even best looking website going nt know market promote online prettiest site world one ever see work closely understand market ideal customer want people know much much know goal either handle online marketing promotion simply give guidance idea team member take ball run choose deliver call email today schedule free consultation "Professional, Scientific, and Technical Services" +always willing help var way hesitate contact u marketing idea marketing need best fulfill every time value added distributor s job provide var tool necessary facilitate daily business task following tool help "Professional, Scientific, and Technical Services" +therapy staffing therapist recruiting staffing company locally owned operated state california place qualified physical therapist occupational therapist speechlanguage pathologist physical therapy assistant certified occupational therapy assistant within type health care setting throughout california therapy staffing inc part alphavista group one largest therapy staffing company based california specialize providing permanent longterm contract temptoperm perdiem placement personnel every segment healthcare industry therapy staffing highly professional organization familylike culture excellent clinical support learn train offer industrybest takehome pay comprehensive benefit package signon bonus relocation incentive company paid medical dental life insurance extensive family health option travel r "Professional, Scientific, and Technical Services" +12hoist4u specializes staffing recruitment service market hoisting lifting heavy lift specialized transport competent experienced personnel "Professional, Scientific, and Technical Services" +1680pr thriving independent communication consultancy significant global footprint highly specialized firm 1680pr provides advice particular field public relation strategic digital marketing positioning branding imaging department state government head music film television personality entertainment industry executive native american tribe native enterprise nonprofit organization entrepreneur continue benefit expertise consultancy native americanowned company firm respective sister division serve truly diverse constituency warner music nashville multiple grammyaward winning 1hit songwriter lionesque fashion laura small woman innovate mobile american express cnn aflac npr best buy duke university bowdoin mcguirewoods thunderbird school global management examinercom presidential candidatenew mexico governor bill richardson reuters blo "Professional, Scientific, and Technical Services" +1mb corporation offer full suite technical consulting service knowledgeable consultant quickly resolve complex issue advise business using industry best practice offer technical consulting sized business big small expertise project based application development computer related advisory servi "Professional, Scientific, and Technical Services" +1myroi shall strategize deliver best possible result business shortest possible timeframe client allocated account manager drive result well main point contact answer query help build understanding strategy implemented attitude towards clie "Professional, Scientific, and Technical Services" +1nteractivesolutionz social medium digital marketing firm based dallas texas founded entrepreneur advertising enthusiast preston pennington preston witnessed rise social medium digital marketing avid observer participant decided make 1nteractivesolutionz industry leader every day leverage h "Professional, Scientific, and Technical Services" +1st call recruitment agency mrcsa rcsa accredited professional recruiter new zealand wide including tauranga rotorua bay plenty waikato auckland finding employment solution industry sector specialist recruiter focused personnel recruitment hr consulting veterinary job office labour technical specialist management ceo position seek award winning recruiter look labour hire temping locum contract permanent executive leasing skill testing behavioural profiling reference checking offer excellent guarantee ensure get right staff "Professional, Scientific, and Technical Services" +1st translation enviable team top legal financial linguist expert field due standard consistency work conscientiousness professionalism team established relationship trust respect leading international institution bank law practice "Professional, Scientific, and Technical Services" +company overview 1stopfotocom located stoughton division natural color imaging year natural color imaging serviced need professional photographer country 1stopfotocom outstanding selection product service reasonable price many order ship within twentyfour hour mission statement remains deliver product service reasonable price short amount time level quality better expected 1stopfotocom work ordering 1stopfotocom easy free downloadable roe remote order entry software digilabs album software fast easy use two free software downloads offer anyone create professional looking photograph album press printed product "Professional, Scientific, and Technical Services" +nonstop translation language guaranteed ontime delivery certified translation interpreting get instant quote www1x1forditoirodahuen language albanian arabic armenian basque belarusian bosnian brazilian portuguese bulgarian catalan chinese croatian czech danish dutch english esperanto estonian finnish flemish french georgian german greek hebrew hindi hungarian icelandic irish gaelic italian japanese kirghiz korean latin latvian lithuanian macedonian maltese mandarin moldovan mongolian norwegian persian farsi polish portuguese romanian russian serbian serbocroatian slovak slovene spanish swedish tagalog thai tibetan turkish turkmen ukrainian uzbek vietnamese subject area legal finance information technology music tendering tourism electricity education official document chemistry business engineering pharmaceutical science agriculture mechanical engineering environ "Professional, Scientific, and Technical Services" +heart located warszawa poland company working advertising business activity question please contact u "Professional, Scientific, and Technical Services" +medium located delhi india company working consultant business activity "Professional, Scientific, and Technical Services" +21st century law firm professionally managed corporate law firm based delhi expertise corporate litigation supreme court litigation delhi high court litigation arbitration negotiation contract joint venture foreign direct investment fdi competition law mining law company incorporation succession case consumer "Professional, Scientific, and Technical Services" +excellent full service laser engraving shop provide custom marking light industrial award trophy personalized gift item name tag b2b signage promotional item would love help ever project currently working exceeds expectation "Professional, Scientific, and Technical Services" +24hourtek inc provides support business san francisco bay area offer flat fee support plan cover server workstation mobile phone everything please contact u anytime information "Professional, Scientific, and Technical Services" +brix serve client food wine industry exclusively know business bring experience expertise internet mobile technology latest marketing technique develop brand increase sale "Professional, Scientific, and Technical Services" +marketing everyday marketing firm story born desire create effective valuable brand experience client independent creatively driven agency thrive delivering work produce result listen collaborate initiate strategic plan result memorable resultsdriven concept marketing may wear cowboy boot rebuke traditional title yet value principal strong innovative original thought work push creative envelope benefiting client come rely u experience count show professionalism thoroughness execute project "Professional, Scientific, and Technical Services" +graphic design website design logo design corporate identity packaging design brochure design exhibition graphic design simon bruce martin barlow lisburn northern ireland "Professional, Scientific, and Technical Services" +2b graphic design located greenfields st ives pe27 5hb st ives united kingdom company working advertising graphic design web design graphic design business activity "Professional, Scientific, and Technical Services" +europages european b2b platform language million registered company mainly manufacturer service provider wholesaler distributor every month million unique visitor world use platform search product service supplier "Professional, Scientific, and Technical Services" +2nd city resourcing ltd independently owned niche recruiter marketing digital communication staff recruit interim manager contractor permanent staff philosophy built one core principal recruit attitude train skill great attitude heart everything "Professional, Scientific, and Technical Services" +hotfrog u provides information regarding 2nd insight inc alameda ca 2nd insight inc located island dr ste alameda ca provides lawyer service contact visiting website http 2ndinsightcom "Professional, Scientific, and Technical Services" +graphic located lonsdale road barnes sw13 9qb united kingdom company working advertising business activity "Professional, Scientific, and Technical Services" +welcome 2k printing promotion 2k business growing business personalized marketing strategy state art printing method unique hand approach customer service take business band sport team organization event new level professionalism growth call use live chat feature speak directly one experienced marketing agent handle project account start finish browsing idea try promotional search engine unique customizable item right one always click away 2k know especially today s economy marketing branding become even important ever marketing specialist created one stop shop everything business need put logo make site offer everything business need graphic "Professional, Scientific, and Technical Services" +europages european b2b platform language million registered company mainly manufacturer service provider wholesaler distributor every month million unique visitor world use platform search product service supplier "Professional, Scientific, and Technical Services" +leading los angeles interactive agency offering digital strategy design technology medium service integrated digital campaign leverage web mobile ecommerce search engine optimization social medium insight integrated campaign produce since pleasure augmenting digital roi wide variety consumer brand agency record label public figure approach approach providing integrated digital service leverage multidisciplinary resource branding digital design development creative marketing social medium business management strategic balance allows u effectively provide resultdriven digital creative solution online broadcast branding mobile print packaging campaign enable talent corporate brand strategize produce distribute monetize digital initiative value authenticity vision unique digital strategy creative solut "Professional, Scientific, and Technical Services" +pea baby product marketing virtual baby product incubator agency focus helping inventor distributor marketing expert get product market help mom dad baby created group mom dad made career big agency wellknown brand believe experience track record highly qualified partner give u edge others whether starting need prototyping patent research seasoned product want give new life help building brand marketing many service help take conception registrymust help every step main area focus public relation branding identity development social medium management web service online marketing sale channel development advertising "Professional, Scientific, and Technical Services" +3 located jelczlaskowice hirszfelda poland company working general business business activity "Professional, Scientific, and Technical Services" +award winning boutique fullservice advertising agency "Professional, Scientific, and Technical Services" +marketing located charlotte street eden terrace new zealand company working advertising marketing business activity visit company website information u http www360marketingconz "Professional, Scientific, and Technical Services" +360fusion multimedia development company middlesbrough create digital content web specialise virtual tour panorama also offer range web design service "Professional, Scientific, and Technical Services" +digital located stanhope bridge bldg main rd claremont south africa company working general business business activity visit company website information u http wwwsport365coza question please contact u "Professional, Scientific, and Technical Services" +3ca limited 3ca privacy policy developed extension commitment combine highestquality product service highest level integrity dealing client partner policy designed assist understanding collect use safeguard personal information provide u "Professional, Scientific, and Technical Services" +3d digital technology firm seeking generate capital continue research development palmsized adapter connects hdmi input 2d projector create 3d experience 3d digital comprised seasoned team high tech industry veteran average year industry experience launching product fortune company microsoft intel successful innovative track record gotomarket launch 3d graphic technology online 3d film production distribution stereoscopic glass production distribution large scale home entertainment system 3d digital poised bring another successful product work right box "Professional, Scientific, and Technical Services" +3d engineer set one primary aim prove modern computer technique could utilised save car restorer enthusiast manufacturer considerable time money aggravation whilst improving quality final product bracket whole car body want replace traditional craftsmanship rat "Professional, Scientific, and Technical Services" +3d model club supply model many industry including sign architecture woodworker jewelry jewellery company designer contact u today special one kind custom modelling need aim achieve simple let client experience wow time efficient cost effect "Professional, Scientific, and Technical Services" +3encores creates engaging original web graphic design stimulating content innovative social medium campaign interactive multimedia 3encores keep away conveyor belt premade template offer top line unique design connect fan experience expertise put u par large big city web design firm offer integrity commitment personal touch expect creative design studio whether need start small build established brand would like create new image help "Professional, Scientific, and Technical Services" +vol inspirerende 3dprinting verhalen aansprekende 3d print design 3d printer nieuws en creatieve en innovatieve voorbeelden 3d printen "Professional, Scientific, and Technical Services" +europages european b2b platform language million registered company mainly manufacturer service provider wholesaler distributor every month million unique visitor world use platform search product service supplier "Professional, Scientific, and Technical Services" +3 lift ficont industry beijing engineering service co ltd auxiliary company ficont industry beijing co ltd earliest one engages blade engineer service tubular tower clean provides maintenance service blade inspection maintaining repair clean tubular clean painting remedy custom 3 lift "Professional, Scientific, and Technical Services" +take care marketing one stop business card website social medium logo design know running business tough time smaller budget limited staff never enough hour day usually push marketing side get later tomorrow eventually push get running plan easy execute maximizes marketing dollar give return investment help know every business need little different nt outline cookie cutter package offer client custom fit everything budget need help reach goal listen learn business know one know business better ca nt wait learn everything know understand business entail ca nt market product service best "Professional, Scientific, and Technical Services" +specialize designing building installing highly customized three dimensional 3d environment face face live event includes trade show exhibit corporate showroom user group meeting retail showroom outdoor pavilion sale meeting rental exhibit road show mobile trailer retail environment special event popup store ultimately would like reframe opportunity exhibit event industry help connect qualified customersleads improve hit rate help track provide metric prove success program short maximize value marketing dollar spent mission answer question better anyone else industry pursuant end created proprietary step process generate exposure attract qualified buyer close business "Professional, Scientific, and Technical Services" +3d project located johannesburg south africa company working general business business activity question please contact u "Professional, Scientific, and Technical Services" +3rdrail demand gen agency one goal making look good agency ob shiny award re figuring squeeze every ounce effectiveness budget beat last month s roi fill sale pipeline keep filled "Professional, Scientific, and Technical Services" +corner international located semouha original st mostafa kamel st flat semouha alexandria egypt company working overseas business import export agent business activity "Professional, Scientific, and Technical Services" +u located dubai uae company working others business activity question please contact u "Professional, Scientific, and Technical Services" +located prestigious bay street heart downtown toronto bay centre offer furnished executive team office short term lease meeting room fully equipped business centre cafe lounge kitchen complementary tea coffee receptioist answer call greet guest mail telephone answering day office provide everything need bring laptop whether establishing business presence toronto opening branch office simply need office occasional meeting room bay centre answer unlike managed office bay unbranded name address organization listed building lobby directory prominently displayed office smart alternative challenging time long term commitment fund expensive office equipment furniture experienced adminstrative staff need "Professional, Scientific, and Technical Services" +40offsetcom solution printing need provide today s business exceptional quality printing using revolutionary technology reduce turnaround time important specialize reducing printing cost focus fullyautomated process start finish allows u cut cost corner highly successful company understand success must based saving valuable time money provide easiest comprehensive online ordering tool printing need waiting standing line local printer click mouse re way discovering easy convenient online color printing try 40offsetcom start saving every order look forward serving "Professional, Scientific, and Technical Services" +service offset printing desktop publishing full serve color copying print demand typesetting graphic "Professional, Scientific, and Technical Services" +buck located norman lane johnsonville wellington wellington region new zealand company working retail service restaurant business activity visit company website information u http 5xconz "Professional, Scientific, and Technical Services" +star cleaner located london united kingdom company working others dry cleaner business activity "Professional, Scientific, and Technical Services" +4exmilitary established niche recruitment consultancy internet job board aimed assisting military leaver exmilitary personnel finding second career remaining employment throughout working life match offer member candidate vacancy nationally internationally membe "Professional, Scientific, and Technical Services" +4sight canadian consulting firm work company want business first nation "Professional, Scientific, and Technical Services" +5rb leading set medium entertainment barrister proven trackrecord area medium law including defamation privacy confidence data protection harassment medium regulation intellectual property sport human right member set write contribute leading textbook field gatley libel "Professional, Scientific, and Technical Services" +staff friendly experienced creative "Professional, Scientific, and Technical Services" +64bit computing located swansea united kingdom company working consultant computer repair business activity "Professional, Scientific, and Technical Services" +enterprise located dhaka bangladesh company working general business business activity "Professional, Scientific, and Technical Services" +impression ltd located carlson court putney bridge road sw15 2nq united kingdom company working advertising design consultant business activity "Professional, Scientific, and Technical Services" +printing located jakarta indonesia company working printing business activity "Professional, Scientific, and Technical Services" +magnetic metal screened vinyl lettered sign banner screen printing compact disk cd duplication printing service flexographic industrial printing service rotogravure industrial printing service "Professional, Scientific, and Technical Services" +insulation located glenidan collinstown westmeath ireland company working consultant business activity "Professional, Scientific, and Technical Services" +pallet located birmingham united kingdom company working retail service packaging solution business activity "Professional, Scientific, and Technical Services" +design located medan indonesia company working consultant business activity "Professional, Scientific, and Technical Services" +c located hong kong hong kong company working shopping centre business activity "Professional, Scientific, and Technical Services" +b consultant located barn london road cb11 4jl saffron walden united kingdom company working consultant business activity "Professional, Scientific, and Technical Services" +label located kempton park south africa company working printing business activity "Professional, Scientific, and Technical Services" +j contractor located felgrove street glendene auckland new zealand company working general business business activity "Professional, Scientific, and Technical Services" +f enterprise located taman sri setia tlk intan perak malaysia malaysia company working general business business activity "Professional, Scientific, and Technical Services" +g cash register serving northwest ohio since g cash register authorized dealer future po touch screen software sam4s cash register service equipment sell stock full supply ribbon paper cash register po printer "Professional, Scientific, and Technical Services" +j heard located willow way pl14 4su liskeard united kingdom company working audit accounting business activity "Professional, Scientific, and Technical Services" +europages european b2b platform language million registered company mainly manufacturer service provider wholesaler distributor every month million unique visitor world use platform search product service supplier "Professional, Scientific, and Technical Services" +chimney sweep located rosehill garden wd5 0hf abbot langley united kingdom company working others business activity "Professional, Scientific, and Technical Services" +endless amount choice today international ready show knitwear knitted need new delhi area source dynamically generated "Professional, Scientific, and Technical Services" +label located manchester united kingdom company working others business activity "Professional, Scientific, and Technical Services" +security located plymouth united kingdom company working security business activity "Professional, Scientific, and Technical Services" +n marketing located orrell mount l20 6ns bootle merseyside united kingdom company working advertising advertising equipment business activity "Professional, Scientific, and Technical Services" +r partner provides strategic customer acquisition retention service consumer businesseswe help grow customer database costeffectively keep customer buying proven technique direct online marketing help understand profitable customer keep engaged business contact u today info acquireretaincom "Professional, Scientific, and Technical Services" +hotfrog u provides information regarding consulting la vega nv consulting located w lake mead blvd la vega nv provides business consulting service contact "Professional, Scientific, and Technical Services" +r cleaning service reliable professional familyrun cleaning business working client ranging toyota garage bridgend county council business pride high standard unlike many cleaning business use temporary worker untrained staff r stringent 10step recruitment process make recruiting right people hard receive service r cleaning know staff sent premise highly vetted professionalthe company grown quickly member staff counting director aim double size expanding number contract service maintaining high standardsthe company beginning rayner davy md set ironing business advertise put handwritten sign window grew reputation year sold original business set r cleaning later joined husband ashley ve expanded business staff growing "Professional, Scientific, and Technical Services" +ice co logistics supply chain company based spring hill florida united state "Professional, Scientific, and Technical Services" +landscaping located nunawading australia company working general business business activity "Professional, Scientific, and Technical Services" +beach rental accounting company based se 17th ave portland oregon united state "Professional, Scientific, and Technical Services" +diesel repair accounting company located old ridgeway rd bremen ga united state "Professional, Scientific, and Technical Services" +security system located petersfield united kingdom company working locksmith security business activity "Professional, Scientific, and Technical Services" +nail located heilongjiang province dong da zhi street nangang district harbin china company working general business hairdresser home garden business activity "Professional, Scientific, and Technical Services" +a1 sign customer a1 sign business since owner charles labor started producing sign home chippewa avenue south st louis business grew need space business moved south st louis current location corner russell boulevard gravois "Professional, Scientific, and Technical Services" +2nd chance bail bond mission bring new dynamic bail bonding business incomparable approach excellence service anticipation client s every need offer fast quality reputable service client 2nd chance bail bond available execute bond united state keep local state national issue important always changing industry handson company helping start finish business conveniently located within walking distance cobb county jail fulton county jail 2nd chance bail bond live person office handle question inquiry open hour day day week commitment free advice never charge consulting fee ll help difficult process give u call need usa 2nd chance bail bond opened 2nd location atlanta ga fulton countyservicing atlanta alpharetta roswell sandy spring john creek milton east point everyone deserves 2nd chance "Professional, Scientific, and Technical Services" +portable toilet marketing advertising company based baldwin height cir howard ohio united state "Professional, Scientific, and Technical Services" +quality appliance human resource company located e fairhaven ave burlington washington united state "Professional, Scientific, and Technical Services" +studio located sandy lane farm cv32 6rw royal leamington spa united kingdom company working advertising graphic design web design graphic design business activity "Professional, Scientific, and Technical Services" +europages european b2b platform language million registered company mainly manufacturer service provider wholesaler distributor every month million unique visitor world use platform search product service supplier "Professional, Scientific, and Technical Services" +employment recruitment service "Professional, Scientific, and Technical Services" +consulting ltd located birmingham united kingdom company working consultant business activity "Professional, Scientific, and Technical Services" +looking improve online presence traffic lead conversion whether re base lancaster pa across country leverage experience internet expertise jump ahead pack since sky medium providing online solution client internet marketing web design development need create search engine friendly website provide house search engine optimization confusion company born internet marketing firm nt print nt radio television focus exclusively client internet marketing success across america customer service dying art firm industry think service dirty word re different trust sky medium built philosophy providing great customer service success success deliver best work provide positive return investment w "Professional, Scientific, and Technical Services" +offer wide range service people want set already established business planning expand "Professional, Scientific, and Technical Services" +degree online marketing digital medium specialist company one leading south african medium placement online marketing agency degree analyse plan implement best medium strategy business due year experience know effective come online marketing expertise passion help u come creative effective online marketing plan medium placement strategy also formed strategic alliance contact within ever changing online arena enables u keep date latest online marketing trend contact also help u provide client best possible online advertising real estate best possible price "Professional, Scientific, and Technical Services" +international inc located levittown puerto rico company working clothing accessory shoe business activity Consumer Durables & Apparel +plus located alexandria egypt company working clothing accessory business activity Consumer Durables & Apparel +19th gallery located bangkok thailand company working furniture business activity Consumer Durables & Apparel +1st gear cycle bike shop full workshop facility alton hampshire stock wide range bike accessory cyclist road bike mountain bike bmx leisure cycle family authorised dealer leading bike brand felt trek giant mongoose range bmx bike cater f Consumer Durables & Apparel +1st national pawn pawn way serving north carolina triad since gold buyer pro specialize jewelry especially buying gold type general electronics goal give best value whether sale loan customer Consumer Durables & Apparel +friend gift unique gift store includes fine kameleon jewelry novo bead heartstrings personalized gift baby gift keep coming back always something new Consumer Durables & Apparel +optician located tralee ireland company working art craft glass art optician business activity Consumer Durables & Apparel +25karatscom established team expert many year experience jewelry industry ecommerce simple goal providing best online jewelry shopping experience customer controlling sourcing manufacturing product distribution 25karats bring fine jewelry public exceptional price raising quality bar competition 25karats office heart manhattan s diamond district allows immediate access best resource jewelry industry locally worldwide Consumer Durables & Apparel +2d drafting service located stokeontrent united kingdom company working art craft business activity Consumer Durables & Apparel +2nd chance boutique located enugu nigeria company working clothing accessory boutique business activity Consumer Durables & Apparel +2nd love cosmetic creation 2nd generation cosmetic company based sunny los angeles california currently creating developing quality cosmetic innovation style offer growing line product suit skin type age website featuring baked powder collection soft sheer powder provide flawless coverage intense color enjoy Consumer Durables & Apparel +2nd nature located lagos nigeria company working clothing accessory shoe bag kid kid shoe business activity Consumer Durables & Apparel +2nd season llc highquality gently seasoned clothing entire family mid high end consignment boutique located downtown greenfield clothing shoe handbag coat etc men woman child many item brand new also carry brighton jewelry handbag shoe consigned many brand vera bradley coach ann taylor ralph lauren aeropostale abercrombie fitch name probably Consumer Durables & Apparel +dimension located singapore singapore company working furniture business activity Consumer Durables & Apparel +founded 2k develops publishes interactive entertainment globally console system handheld gaming system personal computer including smartphones tablet delivered physical retail digital download online platform cloud streaming service 2k publishes title today s popular gaming genre Consumer Durables & Apparel +antique located london united kingdom company working art craft antique business activity Consumer Durables & Apparel +3p enterprise ltd distributor premium branded high quality pet accessory sourced around world establisted business focus sourcing supplying functional practical stylish branded accessory uk trade market company specialises supplying innovative cat dog product many u Consumer Durables & Apparel +located city road ec1v 1jh united kingdom company working hobby model shop business activity Consumer Durables & Apparel +paw tail located cork ireland company working pet animal business activity Consumer Durables & Apparel +paw grooming located macroom ireland company working pet animal business activity Consumer Durables & Apparel +sport located amman jordan company working sport sporting good business activity Consumer Durables & Apparel +40notes perfume luxury niche fragrance company specializes rare exquisite ingredient creating limitededition collection oneofakind perfume creation smoothest perfumery one experience 40notes described elegant oil based perfume concentrate artistically blended presented made highest concentration natural blend Consumer Durables & Apparel +buy furniture homeware lighting decor south africa s first online furniture decor store enjoy worryfree shopping free delivery Consumer Durables & Apparel +carton supply located harlow united kingdom company working paper product business activity Consumer Durables & Apparel +diamond located beirut lebanon company working jewellery business activity question please contact u Consumer Durables & Apparel +b sign located thatcham united kingdom company working art craft business activity Consumer Durables & Apparel +h located south korea clothes shoe bag wholesaler mainly eastern asia market Consumer Durables & Apparel +j fabric located banbury united kingdom company working fashion fabric business activity Consumer Durables & Apparel +co located hong kong hong kong company working book business activity Consumer Durables & Apparel +sign located barn c new barnes mill cottonmill la al1 2ha united kingdom company working art craft business activity Consumer Durables & Apparel +glass located oruanui street taupo new zealand company working art craft glass art business activity Consumer Durables & Apparel +window cleaning located wellingborough united kingdom company working cleaning business activity Consumer Durables & Apparel +pawn shop paper forest product company based college ave ste elberton ga united state Consumer Durables & Apparel +z located pattaya thailand company working clothing accessory business activity Consumer Durables & Apparel +embroidery apparel fashion company located cherrywood ter hayesville north carolina united state Consumer Durables & Apparel +engraving located glen waverley australia company working art craft business activity Consumer Durables & Apparel +furniture located london united kingdom company working art craft antique business activity Consumer Durables & Apparel +stop cosmetic company based david elder rd griffin ga united state Consumer Durables & Apparel +z emblem apparel fashion company located tomahawk dr burlington new jersey united state Consumer Durables & Apparel +understand frustrating cell phone smart phone computer ipod game system nt work way expect challenge anyone keep device fully protected countless threat surface every day 1800fixcom want share personalized solution everyone one device goal become valuable trusted partner effectively manage tech device making sure tech work work better buy take tradeins sell repair major type device cell phone smart phone iphones laptop netbooks desktop computer gamaing console portable gaming system digital music systemsmp3 player flat screen tv s etc commitment technology customer service significant investment technology allows customer place service request click well provide u valuable feedback hav Consumer Durables & Apparel +retail sporting good firearm ammunition gunsmith service coming fall lane state art indoor shooting range handgun rifle shotgun located downtown zion since firearm stock consignment welcome custom order specialty Consumer Durables & Apparel +amigo located higuey dominican republic company working gift gift shop business activity Consumer Durables & Apparel +1st class cleaning service provide comprehensive office cleaning solution including thorough medical clean ensure home office kept highest standard also offer ground maintenance give business premise appealing look potential customer Consumer Durables & Apparel +way located cairo egypt company working car part accessory business activity Consumer Durables & Apparel +21st century furniture located amman jordan company working furniture business activity Consumer Durables & Apparel +21st century good located south park loop new zealand company working communication radio communication camping outdoors clothing accessory bag vehicle service business activity visit company website information u http 21stcenturygoodscom question please contact u Consumer Durables & Apparel +construction located cere clse portland mitchell plain south africa company working art craft artist business activity question please contact u Consumer Durables & Apparel +located bangkok thailand company working car part accessory business activity Consumer Durables & Apparel +3j fashion wear located dhaka bangladesh company working clothing accessory business activity Consumer Durables & Apparel +dog located salisbury united kingdom company working pet animal business activity Consumer Durables & Apparel +kid located cairo egypt company working kid kid furniture toy shop business activity Consumer Durables & Apparel +located rijeka croatia company working home garden business activity Consumer Durables & Apparel +3v underwear manufacture underwear high quality ecofriendly facrics bamboo micro modal company global market leading starter clothing fashion industry value sustainability environment high functionality design sophisticated clothing primarily prefered welldressed fashion focussed Consumer Durables & Apparel +4u ltd located hong kong hong kong company working clothing accessory business activity Consumer Durables & Apparel +4luggagecom internet authorized dealer luggage travel gear accessory travel pillow headphone carry brand hartmann briggs riley heys eagle creek ogio oakley nike adidas samsonite monster sennheiser bose shure Consumer Durables & Apparel +star enterprise working agent providing service produce kind apparel factory per requirement buyer also offer compatable price respective garment quality controller team capable enough find quality problem inline inspection rectify avoid disaster come forward merchandise passing final inspection know well time delivery half competition retailer thats also take care shipment buyer agreed want say giving u order headace consignment inhoused become also contact 5starenterprises2008 gmailcom asgher sayani star enterprise Consumer Durables & Apparel +500cosmetics spanish company founded intention serving people need different area health beauty strength holding company extensive experience online sector distribution group mercantis created 500cosmetics powered natural logistics numerous product launc Consumer Durables & Apparel +international inc private company company involve manufacture wood household furniture Consumer Durables & Apparel +school supply store Consumer Durables & Apparel +hotfrog u provides information regarding futon port jefferson ny futon located main st port jefferson ny provides furniture service contact visiting website http 8futonscom Consumer Durables & Apparel +retail store selling kitchenware homeware hardware stationery souvenires general merchandise Consumer Durables & Apparel +trading located damascus syria company working kid toy shop business activity question please contact u Consumer Durables & Apparel +e located samarinda indonesia company working art craft glass art business activity Consumer Durables & Apparel +c antique located bristol united kingdom company working art craft clock watch repair jewellery watch clock business activity Consumer Durables & Apparel +j davidson located aberdeen united kingdom company working art craft artist business activity Consumer Durables & Apparel +h fashion located main st anby jamaica company working clothing accessory business activity Consumer Durables & Apparel +eventually everyone need adult productsmail order adult supply productservice time come call l supply l supply serving auckland providing excellent adult productsmail order adult supply service need source dynamically generated Consumer Durables & Apparel +sport north american distributor rrd roberto ricci design windsurf standup paddle gear rrd one largest producer windsurf board world produce complete line windsurf boom mast accessory extensive line windsurf sail well comprehensive line sup standup paddle board accessory introduceda sport also distributes ocean sunglass full line watersports sport fashion kid sunglass Consumer Durables & Apparel +propane consumer good company based texas ave bryan texas united state Consumer Durables & Apparel +taxidermy art craft company based w genesee st syracuse new york united state Consumer Durables & Apparel +upholstery located glasgow united kingdom company working art craft antique business activity Consumer Durables & Apparel +basket located gulf view pk la romain trinidad tobago company working gift gift shop business activity Consumer Durables & Apparel +wholesale bulk organic product morocco cosmetic argan oil prickly pear oil ghassoul black cumin oil saffron Consumer Durables & Apparel +acdc power supply adapter electronic device adapter high quality regulated power supply stable enough sensitive electronics ideal lcd screen digital medium audio video application scanner hard drive device requires high quality stable dc power day shipping order pm eastern time order ship day stand behind product one year warranty unit Consumer Durables & Apparel +find perfect gift idea match personality 15gifts introduces new unique approach buying gift online Consumer Durables & Apparel +plus located high st new zealand company working leisure gift business activity Consumer Durables & Apparel +1st avenue located london united kingdom company working clothing accessory clothes shop lady clothing real estate business activity Consumer Durables & Apparel +infinity located 34b high street pe21 8sp boston united kingdom company working fashion body piercing beauty professional tattooists business activity Consumer Durables & Apparel +company provides anything everything could possibly want within sport volleyball volleyball headquarters mission provide customer best service possible provide team uniform jersey short sock kneepads shoe warmups ankle guard ball equipment coach apparel casual wear gift much Consumer Durables & Apparel +cool located bangkok thailand company working clothing accessory business activity Consumer Durables & Apparel +specializing western hand tooled leather handbag american west western purse 22fabu western tooled leather backpack western briefcase cell phone case wallet cosmetic s case shaving kit day planner duffel bag rodeo bag wheeled luggage american west tooled leather throw pillow picture frame country decor bo Consumer Durables & Apparel +office furniture passionate sourcing highest quality office furniture make sure client receive exceptional service exemplary first rate worked hard build close relationship leading furniture manufacturer allowing u combine experience expertise along team highly qualified specialist collectively year experience well acquainted servicing top distinguished business company southern california addition strong tie modern aesthetic well dedication good interior design ensure client project endresult nothing short sublime Consumer Durables & Apparel +service quality used furniture home accessory Consumer Durables & Apparel +2entertain located oslo norway company working art craft business activity Consumer Durables & Apparel +mission operate premier professional audio video technology store dedicated providing highest quality customer service superior technology solution vision single technology source professional audio video engineer designer technician integrator consultant entrepreneur extensive working knowledge audio video industry talented team 3dhd gear year combined experience broadcast television worked stage studio audio design av presentation engineering layout installation equipment operation prepost production handson technical knowledge come directly involvement creating thousand hour television programming Consumer Durables & Apparel +seven retail location california mainly wholesale two factory china distribute human virgin hair world see quality hair please visit website video testing product beauty supply product difference quality drastic looking best quality longestlasting hair competitive price ask question thank Consumer Durables & Apparel +season located latakia syria company working clothing accessory business activity Consumer Durables & Apparel +star located aleppo syria company working clothing accessory business activity Consumer Durables & Apparel +established mother daughter team hilary mcmurray shelley martin 4beauty group leading manufacturer exporter beauty product domestic international market expanding portfolio beauty brand includes awardwinning professional tanning range heshi highly popular retail tanning brand tantastic Consumer Durables & Apparel +framing store former name spectrum framing providing client high quality frame competitive price since offer custom framing fine art mirror poster etc service include custom framing matting dry mounting canvas stretching replacing glass poster sale guarantee best quality affordable price first class service confidentiality satisfaction work done premise one hour framing service available turnaround time one week Consumer Durables & Apparel +manufacture woman miss junior athletic clothing manufacture embroidered emblem safety chain roller chain proof coil chain sash chain jack chain coil chain ball chain chain link magnetic sign neon sign traffic sign directional sign point purchase sign pennant flag accessory wooden sign Consumer Durables & Apparel +located san lorenzo puerto rico company working clothing accessory business activity Consumer Durables & Apparel +4ever young located sheraton mall barbados company working clothing accessory business activity Consumer Durables & Apparel +star located alexandria egypt company working clothing accessory shoe kid kid shoe business activity Consumer Durables & Apparel +life located malang indonesia company working pet animal business activity Consumer Durables & Apparel +west located cairo egypt company working clothing accessory business activity Consumer Durables & Apparel +small family run business carry variety handcrafted country everyday seasonal decor including candle bowl filler handmade organic natural goat s milk soap dead sea salt bath tea sachet aromatherapy soy candle potpourri small business advantage provide personal customer service speedy delivery retail location website particiapte local craft show festival year trying expand handmade soap portion business offering variety bath product small inn hotel b b business Consumer Durables & Apparel +re 99labels give lip smacking deal ca nt resist make smart brand shopping simple Consumer Durables & Apparel +store located quezon city philippine company working gift childrens gift kid baby clothing kid clothing kid shoe business activity Consumer Durables & Apparel +concept 9five started year share namesake first product offering however manifested launch summerspring collection taking inspiration classic era skateboarding along counter culture supported 9five draw vibe style skateboarding obsessed crate digging niche thrived golden era lucky enough part certainly see parallel core design well drive modernize influence look past s current Consumer Durables & Apparel +hotfrog u provides information regarding f f provides insurance service contact visiting website http wwwcafenevescominsights slightly warped mind guy named rich Finance and Insurance +e financial capital market company based glenridge connector ne atlanta georgia united state Finance and Insurance +ford insurance insurance company located holland st b houston texas united state Finance and Insurance +financial service institute fsi founded clear mission ensure individual access competent affordable financial advice product service delivered growing network independent financial advisor independent financial service firm Finance and Insurance +grooming place banking company located pacific blvd se albany oregon united state Finance and Insurance +ah brook co offer fixed rate service enable recover monies customer unpaid invoice often case go way provide good service supply product customer hold back payment late payment frustrating critical cash flow business even best Finance and Insurance +j capital financial service company located baywick dr spring texas united state Finance and Insurance +loan financial service company located de la cruz blvd santa clara california united state Finance and Insurance +hotfrog u provides information regarding investment los angeles ca investment located burbank blvd provides financial financial planning investment service contact Finance and Insurance +l hofaker inc insurance company located maplewood ave albany new york united state Finance and Insurance +capital electrical electronic manufacturing company located 5th ave new york new york united state Finance and Insurance +mark entertainment financial service company based wilshire blvd beverly hill california united state Finance and Insurance +p schweitzer insurance agency inc independent insurance agency established provide property casualty solution personal commercial clientel focus reviewing current policy coverage address gap coverage also maintaining lower rate coverage need focus maintain best coverage best price client requoting policy renewal available carrier s promise also specialize policy agent trouble writing due carrier restriction independent agency able meet client need matter circumstance may bewe also additional service available client though sister company p schweitzer tax service including full service tax accounting firm provide innovative solution individual small business addition professional income tax preparation service provide expert payroll accounting service incorporation llc partnership formation service free notary servicewe currently service kentucky ohio Finance and Insurance +plus marketing insurance company based w irlo bronson meml hwy kissimmee florida united state Finance and Insurance +truck equipment inc financial service company based w silver spring rd milwaukee wisconsin united state Finance and Insurance +ap capital partner private equity firm make longterm investment exceptional enterprise Finance and Insurance +ok mini storage insurance company based n collins blvd covington louisiana united state Finance and Insurance +p solution inc financial service company located nw 53rd st miami florida united state Finance and Insurance +plus service amp insurance company located silver ridge dr houston texas united state Finance and Insurance +r management service inc mental health care company located meridian ave n seattle washington united state Finance and Insurance +rental insurance company based kennedy intl airpr bldg jamaica new york united state Finance and Insurance +rent insurance company located grass valley rd winnemucca nevada united state Finance and Insurance +sunrise insurance financial service company based palm ave hialeah florida united state Finance and Insurance +signature bail bond financial service company located andrew ave fort lauderdale fl united state Finance and Insurance +z bail bond inc insurance company located palmetto st mobile alabama united state Finance and Insurance +visit z investment read find resource many hot investment topic get question investing basic answered right way Finance and Insurance +team auto sale financial service company based n burton rd orange texas united state Finance and Insurance +touch class detailing banking company located e king ave kingsland georgia united state Finance and Insurance +west enterprise insurance company based churchill ct imperial missouri united state Finance and Insurance +yard art insurance company based tresa dr indianapolis indiana united state Finance and Insurance +hotfrog u provides information regarding a1 payday loan everett wa a1 payday loan located rainier ave provides financial service contact Finance and Insurance +american insurance insurance company based ridge rd cleveland ohio united state Finance and Insurance +american insurance agency insurance company based po box cuyahoga fall ohio united state Finance and Insurance +r property financial service company located airport rd ste warwick rhode island united state Finance and Insurance +b insurance located sandringham drive bl8 4dj bury united kingdom company working financial activity business activity Finance and Insurance +hotfrog u provides information regarding benefit source dallas tx benefit source located knoll trail dr ste dallas tx provides insurance service contact visiting website http wwwabenefitsourcebiz Finance and Insurance +c b office service located torquay united kingdom company working financial activity business activity Finance and Insurance +better insurance school insurance company located glenridge dr ne atlanta georgia united state Finance and Insurance +get better mortgage inc offer personalized service professional ethical knowledgeable robert distefano mark tamburro combine year accumulated experience knowledge mortgage financial service field primary function work arranging mortgage ensure p Finance and Insurance +boutique financial service company located metro dr spartanburg south carolina united state Finance and Insurance +c e engineering insurance located birmingham united kingdom company working others financial activity business activity Finance and Insurance +cellular internet company located racine ave chicago illinois united state Finance and Insurance +hotfrog u provides information regarding choice funding choice funding provides financial service service contact 2370registered mortgage broker new york pennsylvania loan provided 3rd party lender Finance and Insurance +investor group inc help canadian plan future providing quality financial planning advice product network consultant Finance and Insurance +personal commercial insurance financial product h rist insurance agency turner fall massachusetts Finance and Insurance +g welding insurance company based lewis st worcester massachusetts united state Finance and Insurance +home loan inc financial service company located e cesar e chavez ave los angeles california united state Finance and Insurance +grain hope foundation insurance company located po box alpharetta georgia united state Finance and Insurance +aea federal credit union established serving yuma lapaz county arizona half century aea fcu notforprofit financial cooperative dedicated people helping people membership aea fcu mean becoming part credit union family member Finance and Insurance +dance class financial service company based main st cheshire connecticut united state Finance and Insurance +fitting solution financial service company based doby peak dr la vega nevada united state Finance and Insurance +people insurance agency serve people pennsylvania developed solid reputation customer care personal attention detail broad array insurance solution focused budget need Finance and Insurance +j etheredge company insurance company located highway ste houston texas united state Finance and Insurance +k wenban located cirencester united kingdom company working financial activity business activity Finance and Insurance +l brown high school financial service company located e 1st st kannapolis north carolina united state Finance and Insurance +support basis used external expert value option stock transaction asset sale disposal entity carve out let team quickly evaluate financial positioning develop plan action Finance and Insurance +new look interior banking company located rancho del mar way north la vega nv united state Finance and Insurance +peaceful solution insurance company located n rancho dr ste la vega nevada united state Finance and Insurance +one insurance store insurance company located w washington st ava missouri united state Finance and Insurance +p international marketing investment management company located de longpre ave west hollywood ca united state Finance and Insurance +plus insurance service ctr insurance company based sw railroad ave c hammond louisiana united state Finance and Insurance +personal touch class financial service company located lacy ave streamwood illinois united state Finance and Insurance +r insurance insurance company located po box matthew north carolina united state Finance and Insurance +salt group financial service company based sandy pond rd concord massachusetts united state Finance and Insurance +2nd chance bail bond located within walking distance cobb county fulton county jail providing bail bond city nation wide Finance and Insurance +sacco company supplying religious gift church supply religious community since Finance and Insurance +act auxilliary financial intermediation Finance and Insurance +w manufacturing financial service company based wohlert st angola indiana united state Finance and Insurance +v insurance svc financial service company based w avenue m4 palmdale california united state Finance and Insurance +view banking company based 53rd st omaha nebraska united state Finance and Insurance +z insurance insurance company located kenmore blvd akron ohio united state Finance and Insurance +full service independent insurance agency business insurance including general liability property bond commercial auto commercial personal umbrella personal homeowner insurance personal auto insurancewe multiple carrier meet insurance need Finance and Insurance +hotfrog u provides information regarding a1 financial inc los angeles ca a1 financial inc located sepulveda blvd los angeles ca provides financial service service contact visiting website http wwwcheckngocom Finance and Insurance +het ofonds waterschappen opgericht opgericht door de sociale partner de sector waterschappen doel van het fonds om een bijdrage te leveren aan goed werknemer en werkgeverschap de sector dit vertaalt zich aantrekkelijke dynamische en creatieve organisaties met een gezond arbeidsklimaat en met wendbare en weerbare medewerkers het fonds opereert al een een projectorganisatie met een jaarlijks activiteitenprogramma de instrumenten die het fonds inzet zijn onderzoek kennisontwikkeling instrumentontwikkeling en kennisdeling het fonds er voor alle organisaties en werknemers op wie de cao waterschappen saw van toepassing en aan de waterschappen gelieerde instellingen die deze cao vrijwillig volgen voor de sector en door de sector het uitgangspunt bij de werkzaamheden van het ofonds waterschappen dit vertaalt zich een actieve betrokkenheid van professional uit de waterschapsorganisaties bij de activiteiten van het fonds Finance and Insurance +service financial servicesaccountingtaxes Finance and Insurance +adependable insurance agency serving la vega area since founding june product range auto insurance health insurance great rate available riskswe offer full range insurance product auto insurance home insurance business insurance health insurance adependable insurance agency treat client level respect caring give friend come customer service agency go far remember name say look forward servicing insurance need Finance and Insurance +p orleans risk management help company develop implement successful risk management program business insurance employer sponsored group medical planswe particular concentration collaborating mid sized company believe type company many issue larger company financial resource human capital dedicate concern regarding risk insurance employee benefit proven method involves diagnostic risk assessment short long term goal dialogue development strategic plan meet goal implementation plan followed meeting throughout year result evaluated necessary modified Finance and Insurance +a1 insurance solution financial service company based wheeler oak dr brandon florida united state Finance and Insurance +a1 capital provides finance business investing equipment technology vehicle type whether brand new start established successful company help also provide vehicle finance leasing private customer aim ensure customer get right financial package need u Finance and Insurance +a1 payday advance banking company located ashley river rd charleston south carolina united state Finance and Insurance +aa insurance world established current president principal agent andrew alexander locally owned operated miramar fl insurance agency re proud help member community find insurance plan suit lifestyle need every stage life year experience helping wide variety client Finance and Insurance +merchant banking service business energy gas consulting office business security automationmerchant banking service anovia accepts major payment type point sale environment location time s simplywe work existing equipment provide new equipment turning smart phone tablet ecommerce solutionour strength technology saving unlike merchant service provider bypass high cost marketing advertising pas saving youat anovia work give best rate possible ll receive competitive easy understand rate upfront cost hidden fee fact confident ll save money u guarantee reduce overall monthly cost ll give gift card question ask simply allowing u quote business qualify submit price request acn independent business owner online store provide current complete merchant statement http sudburywatertowershopacnrepcomsmallbusinessmerchantservicesaspxlongterm relationship important u s addition saving money offer best class service personalized supportgetting started fast seamless let u handle payment processing need concentration matter building business Finance and Insurance +hotfrog u provides information regarding a1 investigation harrison ar a1 investigation located ridgepark rd provides financial financial planning investment service contact Finance and Insurance +aaa bookkeeping solution financial service company located grouse dr caldwell idaho united state Finance and Insurance +a1 financial financial service company located po box los angeles california united state Finance and Insurance +aa carolina mortgage banking company based n college ave newton north carolina united state Finance and Insurance +aaa enterprise located mukim sungai pinang balik pulau pulau pinang malaysia malaysia company working general business business activity Finance and Insurance +monetize ppp trade Finance and Insurance +service chain link chain link fencing commercial estimate fence fence fencing free estimate guard rail industrial residential residential building residential service temporary fence Finance and Insurance +aaa mortgage service operating mortgage industry since staff combined industry experience excess year dedicated providing client professional efficient home loan service save client time money service designed help client negotiate many avai Finance and Insurance +absalom jones chapel investment bankingventure company located fair st sw atlanta georgia united state Finance and Insurance +adbic industrial development investment company created facilitate expansion sustainable industrial activity abu dhabi base metal aluminum steel copper petrochemical sector established adbic wholly owned subsidiary abu dhabi government s general holding corporation current investmen Finance and Insurance +strive help business owner develop solid foundation business strategy successful work improve sale productivity organization company culture employee relation absolute value financial consultant saudi consultancy corporation operates provide financial advisory service min Finance and Insurance +absolution pilate studio financial service company located wolf ln pelham new york united state Finance and Insurance +acacia smart venture financial company mexico like financial institution acacia smart venture relationship client utmost importance strive extension business associate confide reach financial goalswe help company successful providing support offer considerable level personalized service unique among mexican financial institution help company successful providing financing associate upper management available help business success help company successful providing knowledge aware market serve knowledge experience industry mean work someone understands unique characteristic company industry financing need trust acacia smart venture finance company need bridge loan financing construction getting raw material inventory working capital contact u analysis credit approval fast efficientwe customized loan addition traditional business real estate loan acacia smart venture offer special loan variety industry including health technology education energy many others microcredit offer cost interestfree financing accessible term knowledge help start new business help family limited economic resource help get povertyaccounting legal advice offer knowledge make best decision businessjoint ventureacacia smart venture provides seed venture growthstage funding best company around world focus early stage vecause s greatest impact entrepeneurs Finance and Insurance +abu dhabi capital management adcm alternative investment firm based abu dhabi invests private equity real estate public equity debt adcm currently active financial service real estate sector uae london Finance and Insurance +year experience financial service gilberta dorsey abundant life investment provides investing retirement stock market advice quincy illinois entire tristate area Finance and Insurance +ac tek inc financial service company located sandpiper dr houston texas united state Finance and Insurance +abundant life deliverance financial service company located shalimar st miramar florida united state Finance and Insurance +almost year alexander calin associate aca provided design recordkeeping administration consulting service small company retirement plan client range size one fifty employee frequently goal employer plan sponsor maximize benefit principal valuable employee minimizing much possible cost benefit rank file participant many time accomplished proper plan design creates various class employee different level benefit class may even excluded participation altogether thus reducing significantly cost maintaining plan main niche design plan maximize benefit owner key employeeswe striving provide highly professional efficient affordable service client please explore web site learn company service offer wwwpensionlacom forget check instant defned benefit calculator maximum tax deductible contribution Finance and Insurance +technology located singapore singapore company working computer service business activity Software & Services +10biosystems provides scientific software product technology consulting strategic solution life science including academic research institution biotechnology biofuel medical device healthcare pharmaceutical company consulting practice scientific data management bioinformatics chemiinformatics laboratory automation workflow custom software development Software & Services +11giraffes digital signage software company software service saas model provide tool enabling brand communicate customer guest employee business model subscription based recurring revenue model utilizes single technology platform 11giraffes delivers digital signage audio music messaging 11giraffes client may utilize content source agency advertiser self created andor system template software architecture provides automated tool create schedule traffic present monitor digital medium backend software architecture includes webbased tool hundred template business connect brand consumer market solution objective digital network enhance customer communication education entertainment 11giraffes reduces cost ownership provides disruptive single platform technology deliver video Software & Services +17a4 llc emessaging compliance firm focus software company established address expanding regulatory requirement imposed financial institution 17a4 s approach bridge compliance legal information technology management offer comprehensive compliance solution regulated content includes email instant messaging data stream trading platform website social medium database archive ediscovery 17a4 s software product used extensively institution regulatory internal policy requiring emessaging retention edisco division 17a4 handle ediscovery service including litigation production support 17a4 based new york city operates remote office nationwide Software & Services +1techno ltd reseller comprehensive solution center graitec advance steel highend structural steel modeling detailing fabrication software used construction various industry headquartered montreal canada 1techno office personnel distributed across north america mexico chile rapid expansio Software & Services +headquartered austin since office washington dc 21ct reward curious innovator creator analytics product anyone seek shed light intelligence hidden within data create investigative analytics product way user think look find innovative product service used detect neutralize healthcare fraud target eradicate network security attack reward curiosity 21ctcom Software & Services +develop test deploy custom webenabled software make point learn nature business order design webenabled software specifically meet business need help client leverage digital technology improve business core example Software & Services +founded computer scientist focused decision support software technology r today remain company software engineer computer scientist focus complex data integration decision ambiguity problem innovative software r part business mutually exclusive fact data integration decision support technology started concept developed talented r team create myriad new way solve different data integration decision ambiguity problem government commercial customer small business innovative research sbir award belt since testament r team s creative innovative prowess Software & Services +2morrow s mission help million improve life use evidencebased behavioral change program creating distributing leadingedge program based work respected researcher thought leader 2morrow delivers program directly hand people whenever wherever need via smartphone program supplement effort coach counselor health care professional providing support visit used alone people might otherwise access behavioral focused help 2morrow also provides proprietary backend system offer corporate organizational coaching account distribution management well access data used improve program monitor engagement level research smartquit 2morrow collaborated internationally renowned cancer research institute address nicotine addiction mobile smoking Software & Services +provide fast predictable costeffective deployment highvalue service unique combination technology people process guarantee success technology world class solution 3cnet partnered leading oems technology solution provider world back ample support vie Software & Services +3d printing model buy sell 3d printing model design plan print build using 3d printer Software & Services +3p technology danish software engineering company specialises developing automation control platform dc plc scada system company s establishment denmark since remained one soughtafter system provider country since catered numerous local overseas clien Software & Services +making multiprocessing simple sole focus 3l since formed painfree movement task one processing element another essence 3l expertise 3l work close collaboration tool provided device manufacturer 3rd party specialist optimisation efficiency verification know Software & Services +3p development software development company focused web mobile application development offer client combination strategy creativity cuttingedge technology maintaining openminded dynamic customercentric approach delivering best solution Software & Services +offer broad range solution microsoft outlook window file beyond synchronization sharing tool microsoft outlook exchange server backup migration software microsoft outlook exchange server duplicate management software microsoft outlook importexport plugins ical vcard converter pst Software & Services +engineering service aeronautical engineering electrical power transmission engineering professional engineering service mechanical engineering electrical electronic engineering chemical engineering oil gas engineering mining engineering ocean engineering transportation engineering Software & Services +4th quarter technology top real estate developer erp software provider 4qt s top real estate erp software provides builder top real estate crm construction erp commercial lease management facility management software 4qtcom committed creating reliable cost effective software solution assist customer reac Software & Services +7solutions india leading cad engineering service provider firm based india 7solutions india caters steel detailing rebar detailing precast detailing requirement worldwide client 7solutions india created one stop solution service line structural engineering service need client established Software & Services +would like request information inquiry would love hear please contact u question email u contact 8thbasecom please visit contact page http www8thbasecomindexphp routeinformationcreator form_id46 Software & Services +meet bug software testing Software & Services +10clouds amazing place lot great innovative development achieved diverse rich portfolio project sport community elite athlete tool helping teacher create assignment revolutionary way mobile application used ass hazardous work task australian desert Software & Services +12d solution pty ltd australian software developer specialising civil engineering surveying application established 12d solution sell support principal product suite 12d model direct distributor around world 12d solution client include civil water engineering consultant environment Software & Services +telecommunication premier network engineering firm year extensive experience providing support telecommunication networking banking financial institution telecom carrier isp s cloud computing service provider data center noc tier 1234lce technical support problem determination customer support critical network telecommunication infrastructure device mission telecommunication delivers top notch service helping organization washington dc metro area greater new york city area south florida area reduce operational cost increase revenue improve profitability visibility staying within budget service engineering network design implementation network implementation operation network command center data center help desk network supporttroubleshooting network reengineering technical support technical support continuity business Software & Services +muebles Software & Services +21st century system located sheffield united kingdom company working computer software solution computer service business activity Software & Services +2nd floor computer inc full service networking computer maintenance repair company based camrose ab year computer servicing experience team dedicated technician help build personal business networking system maintain current network also offer wide range service Software & Services +computer onestop technology shop position complete technology solution firm client whatever need solution implement focus important thing computer repair virus removal data recovery hardware diagnosis hardware operating system upgrade mac pc service support specialty sale new used computer component associated technology intelligent effective service virtually technology use today computer offer free diagnosis take time right first time rate service best town computer corporation employ fully qualified technician enable u accountable work radio dispatched technician faster service computer technician bonded insured would love hear feel free call make comment send u email Software & Services +360works georgia llc company prometheus system consulting llc dba 360works established april originally founded macintosh support consulting company shifted focus filemaker consulting custom database development filemaker pro desktop application http wwwfilemakercom java webobjects web application http wwwapplecomwebobjects 360works platinum member filemaker business allance currently employ ten people seven certified filemaker developer company president jesse barnum invited present 360works product several filemaker developer group around country also lead atlanta filemaker developer group meet monthly addition filemaker skill strong java programming background five staff java expertise including gui applet development swingawt webhtmlxml developm Software & Services +3c technology ltd established since head office based colchester essex also large sale office hemel hempstead hertfordshire customer ukwide deliver computer system solution service type business throughout country alliance industry leader Software & Services +3d network located singapore singapore company working computer service business activity visit company website information u http wwwplanetoneasiacom Software & Services +front network located newbury united kingdom company working computer software solution computer service business activity Software & Services +4c s located maidenhead united kingdom company working computer software solution computer service business activity Software & Services +whol computer product service Software & Services +5d innovation developed suite tool let conversation data literally talk data technology allows business professional alike get immediate answer complex question strategic level prerequisite knowledge programming statistic high performance computing machine learning explore data revolutionary strategic analytics user interface form conversation ask question word back instinct intuition right analytical technique act extract value data way never possible Software & Services +5dt fifth dimension technology located de havilland crescent persequor park south africa company working computer software solution business activity visit company website information u http www5dtcom Software & Services +big date analytics software solution retailer Software & Services +software agency specializing web mobile consulting solution midsized business client hire solve complex business problem otherwise shackle business internet application often core client business easy use work great phone tablet desktop create custom application amazing business speak business speak tech bridge gap two providing technologically creative solution give client competitive advantage Software & Services +custom computer programing internet intranet server application development service mainframe software application design personal computer pc application design system integration design Software & Services +11th hour developed sophisticated customised database u track compare sale department past year love simplicity allows u accurately predict future sale required staff level enables u continue give great customer service without wage cost blow Software & Services +123compliance corporation world s first fully cloud mobileenabled enterprise quality management software company 123compliance offer fully validated cfr part 11compliant application provide complaint handling adverse event reporting quality management device tracking training management document management built Software & Services +onecall provides access service allows pbx communicate outside world dont pbx worry partnership pbx service provider even throw kitchen sink outbound traffic designed plan around nonsense policy hold near dear system ensures receive ultimate flexibility best price inbound traffic include competitive wholesale voip rate incoming call different type incoming call plan subscribe onecall customer try best help save dids state art system allows purchase manage dids ability purchase single dids well block Software & Services +u computer located hawkes farm stambourne road cb10 2qs saffron walden essex united kingdom company working computer software solution computer service business activity Software & Services +247meeting provides web audio video conferencing software solution arrange meeting online fraction cost Software & Services +2nd byte computer located weymouth united kingdom company working computer software solution computer service business activity Software & Services +mission 2g robotics provide best subsea imaging measurement technology dedicated advanced research development engineering innovation production cuttingedge solution foundation 2g robotics wealth engineering expertise development innovative reliable system Software & Services +2is inc founded rapidly growing software service engineering company provides affordable supply support innovation department defense dod industrial partner 2is recognized inc magazine four consecutive year one america s fastestgrowing privately held usbased company 2is inc three business line software service part sourcing engineering technical service ets 2is software service develops decision support tool provide independent verification validation supply chain pricing decision using erp legacy data application web based saas tool easy access meet department defense information assurance requirement part sourcing business focus winning successfully delivering awarded repair part contract cooperation integrated manufacturing network ets team work closely defe Software & Services +2sys software company specialising developing webhosted software business across number sector many advantage webhosted approach including computer organisation connected internet access system need server premise run system nt need Software & Services +365scores cross platform app enables anyone freedom create sport channel customised provide live score news world s medium video highlight statistic fixture standing tweet lot world sport fan want keep updated also part community 365scores brings two together connecting fan around world virtual stadium 365scores cover aspect sport platform providing comprehensive content thousand source around world supporting different sport type language mission 365scores continue fuel passion sport fan providing best experience possible 365scores workplace perfect place continue goal harness love sport technology exciting multicultural vibe continue build create vision Software & Services +re seeking answer web development using python s coffee cup rapid software development ecommerce real estate crm others industry using python django google app engine much Software & Services +develop social network mobile enabled web application paperless eform solution interactive multimedia emagazines variety industry design develop net web application mobile web web service soa framework using microsoft net framework c programming language design distributed application using cloud technology utilizing mvc jquery mobile build unified web mobile web application pc tablet smart phone design develop epublishing tool automation version control repository workflow solution adobe indesign c provide plugins widget highly interactive multimedia electronic magazine experience Software & Services +3kv specialised distributor since foundation wehave become leading specialised distributor wireless andwired infrastructure security solution make networksfaster intelligent secure subsidiary ofhpi ag hoechst procurement international stock exchange codecew3 frankfurt stock exchange leading stock exchange listedindustrial service provider strategic procurement managementsince tnd ag specialised distributor server andstorage also belongs hpi group perfectlycomplementary portfolio offer comprehensiveinfrastructure solution see http www3kvdeencompanyhtml sthashrvddadg8dpuf Software & Services +try understand business use expertise help maximize contact client management boost productivity translate complex business requirement seamless workflow enhance increase user adoption provide tool gather use retain crucial customerintelligence information ou Software & Services +long standing microsoft solution partner 3si established reputation total service extensive product portfolio provides solution aspect client need including web enablement back office system customer relationship management membership event committee cpd product sale much suc Software & Services +4cast s mission provide powerful high quality flexible project management tool 4cast project software evolved year director chris curnow worked closely project manager site across australia internationally proud bring product see today powerful comprehensive project mana Software & Services +3vector system premier infrastructure solution consulting organization providing consulting engineering staffing integration service enabling organization transform datacenter operation strategically leverage power cloud computing raise bar leveraging investment deliver like big consulting name low cost flexible stay focused core competency deliver best client virtualization cloud computing staffing managed service application development integration storage information management network infrastructure telephony unified communication management Software & Services +6kare global enterprise provides software solution leading company great variety sector finance manufacturing legal hr operation aim simplify business process meantime improve quality create value innovation growth since day founded creative young Software & Services +teksolutions focus client understand importance time strive maintain relationship based accountability description extensive array hardware software based solution vary simple pc troubleshooting includes virus malware detection removal full blown data driven web application ecommerce site goal assist customer use technology building loyal customer following service offering website development management network design implementation maintenance virus detection recovery managed service wait system failure equipment break call technician call put hold given time tech come take look issue managed service know problem problem occurs quickly start working without leaving commission l Software & Services +import autoworks computer software company located francisco blvd e san rafael california united state Software & Services +b mobility computer software company based kennedy blvd northfield ohio united state Software & Services +ab technology one premier software consulting company offering high end solution empower company success approach project leveraging expertise technology enterprise application integration web service enterprise portal technology aligned strong process enable u execute project conceptualization implementation effectivelya b technology belives simplicity design use information system running business complicated enough system nt part cause let u help process ability understand business challenge implement practice best match need help cut cost reduce risk increase business profit Software & Services +computer located seremban malaysia company working computer service business activity Software & Services +computer clinic located eagle vale australia company working computer software solution computer hardware business activity Software & Services +diver world computer software company based lone oak ct charlotte north carolina united state Software & Services +e information svc computer software company based oakland california united state Software & Services +friendly inter face computer software company based hallmark new braunfels texas united state Software & Services +great good place book computer game company located la salle ave oakland california united state Software & Services +gem software incorporated computer software company located sw shady ct portland oregon united state Software & Services +software inc computer software company located montgomery rd cincinnati ohio united state Software & Services +j builder computer software company based se emi st clackamas oregon united state Software & Services +lowly apprentice production computer software company located la place ct carlsbad california united state Software & Services +n communication located springwood australia company working computer software solution computer hardware business activity Software & Services +p c c located harrow united kingdom company working computer software solution computer service business activity Software & Services +founded p system established provider computer technology service reputation built simple philosophy providing personalised service responsive cost effective professional mission apply quality technology backed high degree service support throughout Software & Services +prompt corporation computer software company located grandview cir honey brook pennsylvania united state Software & Services +l computer service located rugby united kingdom company working computer software solution computer service business activity Software & Services +reason celebrate computer software company located stone edge ct marietta georgia united state Software & Services +team tech svc computer software company based main st tom river new jersey united state Software & Services +tech computing computer software company based curcombe st hartford connecticut united state Software & Services +pc service based marsden west yorkshire 07889765200computer laptop repair upgrade part software virus removal trouble shootingnetwork setuptrouble shooting tuition advice supportnew used computer laptop sale tablet sale printer ink salescall out home businesscollectiondelivery service ask detailscddvdblueraygame disk cleaningcontact day week 8am 8pm office hour monfri use mobile hoursany enquiry welcome domestic business customersany service listed please ask may able helpover year experience trade Software & Services +l computer ltd located litherland crescent haresfinch wa11 9ar st helen merseyside united kingdom company working consultant computer software solution computer repair computer service business activity Software & Services +data deduplication technique reducing amount storage space organization need save data organization storage system contain duplicate copy many piece data example file may saved several different place different user two file nt identical may still include much data deduplication eliminates extra copy saving one copy data replacing copy pointer lead back original copy company frequently use deduplication backup disaster recovery application used free space primary storage wellin simplest form deduplication take place file level eliminates duplicate copy file kind deduplication sometimes called filelevel deduplication single instance storage si deduplication also take place block level eliminating duplicated block data occur nonidentical file blocklevel deduplication free space si particular type known variable block variable length deduplication become popular often phrase data deduplication used synonym blocklevel variable length deduplication Software & Services +a1 technology new york headquartered software development firm software developer a1 technology two state art development center owns manages center camera provide visibility client resource assigned client visit center encouraged Software & Services +b c computer located banbridge ireland company working computer service business activity Software & Services +run two sister beautiful mess woman s lifestyle company focused creating happiness every day homemade lifestyle blogon abeautifulmesscom share home decor project recipe craft well bit life missouri applast year launched beautiful mess photo editing app paid app store nearly week straight customize photo reflect personality apps come future shopat shopabeautifulmesscom offer digital product including class photoshop action lightroom presets Software & Services +better world social enterprise whose mission make uplifting game apps brighten world goal everything dogood fun change world re committed creating awesome stuff designed purpose making difference optimism altruism charity flagship product Software & Services +crowd monster award winning video game development studio headquartered barcelona spain focus create high quality video game help client expand brand wide range platform perform assembling team top industry talent artist designer writer harness full entertainment potential property work withwe deliver time highest standard quality using agile development method practice including continuous integration weekly status meeting optimal communication Software & Services +different design studio computer software company based e romie ln salina california united state Software & Services +e software ltd located derby united kingdom company working computer software solution computer service business activity Software & Services +g computer located coburg australia company working computer software solution computer hardware business activity Software & Services +geek wheel provides expert support home business dfw metroplex voted denton s best computer repair store open day week Software & Services +professional tutoring provides live oneonone online tutoring academic discipline well inperson instruction sat act prep course standardized test preparation program unparalleled particularly private school college admission prep course fcat sat act Software & Services +p uk ltd located unit riverside indust est river way se10 0bh united kingdom company working computer software solution computer service business activity Software & Services +p associate located edinburgh united kingdom company working computer software solution computer service business activity Software & Services +p consulting inc computer software company located e waterloo rd akron ohio united state Software & Services +tech hire information technology service company located north st fairhaven massachusetts united state Software & Services +touch stone computer software company located nw 12th ter pompano beach florida united state Software & Services +computer service formerly mobiletek consulting founded jason houghton quickly became recognized leader client service industry technical skill often trump people skill jason team continually strive raise bar important part client service understanding customer concer Software & Services +byte advice inc computer software company located po box macon ga united state Software & Services +c power system located suite mount osborne business centre mount osborne indust s71 1hh barnsley south yorkshire united kingdom company working computer software solution computer service business activity Software & Services +c worldwide located angel gate city rd ec1v 2pt united kingdom company working computer software solution computer service business activity Software & Services +expansion business towards electronic gadget outsourcing service towards marketing product started chain business quality employeeslooking forward marketing wholesaler distribution company ready business u always think giving 1st impression quality aspect second none Technology Hardware & Equipment +welcome 1ct hard drive specialist buy world sell every continent repair drive major original equipment manufacturer manufacturer world recover data drive anywhere world Technology Hardware & Equipment +3cs managed service provider offering managed service small medium sized business nt think afford full time staff 3cs take care department 3cs manage system providing antivirus malware protection system update computer support help desk support well active directory service computer server backup s low monthly fee service offered arenetworkpersonal computer security active directory design migration exchange design migration system architecture system security upgrade product evaluation department analysis system process procedure implementation administration system internet security firewall av id system s backup recovery penetration testing various project Technology Hardware & Equipment +3stars located new zealand new zealand company working computer service electrical good computer business activity visit company website information u http 3starsconz Technology Hardware & Equipment +let u design circuitry lay pcb writeoptimize microcontroller code otherwise help embedded electronics requirement circuit llc founded two south dakota school mine technology alumnus andrew electrical engineer ed computer scientist Technology Hardware & Equipment +r computer service located shak shak complex hastings main road barbados company working electrical good computer business activity question please contact u Technology Hardware & Equipment +alarm system electrical electronic manufacturing company based wilshire dr belle mead new jersey united state Technology Hardware & Equipment +copier computer networking company based w 20th ave ste 102a hialeah florida united state Technology Hardware & Equipment +promote located london united kingdom company working computer consumables printer supply business activity Technology Hardware & Equipment +brazilian company headquarter sao paulobrazil branch miamiusa bangaloreindia product development center company founded today important presence customersdistributors country like paraguay argentina uruguay peru colombia venezuela mexico spain others dedicated develop manufacture electronic product software segment like electronic security alarm panel gsmgprs communicator tracking device etc remote monitoring telemetry m2m machinetomachine communication Technology Hardware & Equipment +since 4c electronics developed kind electronic product using various technology technique based stocktonontees north east england vast majority work past year designing developing electronics product customer check project page see Technology Hardware & Equipment +4wave provides plasma processing equipment thin film coating milling service meet challenging vacuum equipment thin film processing need using atomic layer processing capability 4wave also offer multilayer device fabrication miniature optical component based sterling virginia minute washington dulles airport 4wave serf client worldwide Technology Hardware & Equipment +communication located hawally kuwait company working mobile phone shop mobile phone business activity Technology Hardware & Equipment +c sale located marabella trinidad tobago company working electrical good computer business activity Technology Hardware & Equipment +service tire clutch battery engine rebuilt timing belt valve job alternator tuneups brake electrical engine transmission fuel injection computer diagnostics service repair foreign domestic vehicle Technology Hardware & Equipment +b electric located chesterfield pk d15 ireland company working general business business activity Technology Hardware & Equipment +l computer servicing central okanagan region british columbia canada since strived provide excellent customer service support microsoft certified trainer technician help solve complicated problem goal partner client help achieve business goal ever competitive world help gain technological edge need succeedwe sell top quality computer desktop server notebook accessory certified repair technician microsoft system engineer help build safe secure productive network certified trainer quickbooks accountedge accounting software help setup implementation accounting payroll software need Technology Hardware & Equipment +whol ret computer electronic equipment Technology Hardware & Equipment +tv inc electrical electronic manufacturing company based allison ave panama city florida united state Technology Hardware & Equipment +1st line support located bournemouth united kingdom company working computer repair electrical good computer business activity Technology Hardware & Equipment +technology specializes service sale critical power equipment data center telecommunication facility operation mission critical power requirement provide nationwide solution ups uninterruptible power system battery dc plant power distribution equipment service capability include preventive maintenance emergency service battery replacement major ups manufacturer including liebert powerware mge apc others supply new refurbished rental ups equipment installation deinstallation relocation upgrade project technology headquartered near atlanta georgia facility industry leading includes ups system spare part technical training ups battery test refurb facility also stock extensive inventory ups battery respond emergency x would expect Technology Hardware & Equipment +aspire distinguished provider computer hard ware peripheral locally internationally 2b computer mission offer reliable highquality product offer high level practical experience knowhow client must know working u professional way business exist attrac Technology Hardware & Equipment +tier located sandton south africa company working networking cabling fibre optic business activity Technology Hardware & Equipment +service alternator engine light starter transmission Technology Hardware & Equipment +power system provides high quality engineering service area electrical power system analysis power distribution year experience industrial cogeneration utility power system renewable energy Technology Hardware & Equipment +r service located bayamon puerto rico company working electrical good business activity Technology Hardware & Equipment +phillips painting computer networking company based hickory knl cumming ga united state Technology Hardware & Equipment +electric inc electrical electronic manufacturing company located n 21st st lincoln nebraska united state Technology Hardware & Equipment +co located hong kong hong kong company working electrical good business activity Technology Hardware & Equipment +accurate accounting ser computer hardware company based elder ln franklin park illinois united state Technology Hardware & Equipment +whol heating unit air conditioning unit generator electrical hardware supply intelligent building installation ibi electrical safety device accessory electrical variable speed drive Technology Hardware & Equipment +power generator electrical electronic manufacturing company based ne 20th ave ste miami florida united state Technology Hardware & Equipment +refrigeration located kingsgrove australia company working electrical good air conditioning business activity Technology Hardware & Equipment +b b located homs syria company working energy supplier electricity supplier business activity Technology Hardware & Equipment +b c electrical service electrical electronic manufacturing company based lakeshore rd cicero new york united state Technology Hardware & Equipment +better computer co located north bondi australia company working general business business activity Technology Hardware & Equipment +c lighting design electrical electronic manufacturing company located main st coolidge arizona united state Technology Hardware & Equipment +c r system inc located bayamon puerto rico company working electrical good air conditioning business activity Technology Hardware & Equipment +dream deferred inc computer network security company located crenshaw blvd apt los angeles california united state Technology Hardware & Equipment +digital house production computer hardware company based linda vista rd san diego california united state Technology Hardware & Equipment +g manufacturing co electrical electronic manufacturing company based houston texas united state Technology Hardware & Equipment +h communication inc computer network security company based soapstone cir raleigh north carolina united state Technology Hardware & Equipment +great way go computer hardware company based lobelia rd nw albuquerque new mexico united state Technology Hardware & Equipment +j ranch electrical electronic manufacturing company located carrisa hwy santa margarita california united state Technology Hardware & Equipment +j drilling consumer electronics company based sunrise cir austin texas united state Technology Hardware & Equipment +k technical service electrical electronic manufacturing company located w 2nd st chester pennsylvania united state Technology Hardware & Equipment +electric co electrical electronic manufacturing company located w main st louisville kentucky united state Technology Hardware & Equipment +k located aleppo syria company working mobile phone shop mobile phone business activity Technology Hardware & Equipment +new dimension lighting electrical electronic manufacturing company located e higgins rd elk grove vlg illinois united state Technology Hardware & Equipment +anpowell electrical niceic registered electrician worcester worcestershire carrying electrical work specialising energy saving installationsbreakdowns minor work rewire fuse board replacement consumer unit light fitting testing inspection maintenance electrical installation portable appliance testing emergency lighting cat cabling rointe heating landlord certificatesinfo solcarbcouk 426821niceic registered Technology Hardware & Equipment +specialist car connectivity car stereo installation car audio repair home audio repairsservicingplease call u enquirycontact detail phone 6486mail334 beaufort stperth wa 6000address334 beaufort stperth wa Technology Hardware & Equipment +net located bangkok thailand company working electrical good computer business activity Technology Hardware & Equipment +p co located hong kong hong kong company working electrical good computer business activity Technology Hardware & Equipment +hotfrog u provides information regarding p instrument windsor ca p instrument located shiloh rd provides manufacturing service contact Technology Hardware & Equipment +p e inc electrical electronic manufacturing company located n palafox st pensacola florida united state Technology Hardware & Equipment +customer servicewe stop offering low price aim total businesstocustomer buinesstobusiness satisfaction want customer feel confident buying u specialized dedicated customer service team available proven track record serving many happy customer experience knowhow give customer best buying experience available completely satisfied purchase best correct issue operation capability cover favorable policy affiliate discount instant safe delivery privacy protection wellrounded customer support able supply low price establish relationship manufacturer allows continually access latest unique bestpriced product product quality wholesale product brand new originalfactory sealed manufacturer included accessory boxsame local store promise provide customer support weekday offer customer lowest wholesale price streamline buying paying process deliver good customer world speed precision ensure excellent quality product Technology Hardware & Equipment +toner cartridge hp canon brother lexmark samsung dell womanowned toner cartridge Technology Hardware & Equipment +print shop located kingston ho boothley rd fy1 3rs blackpool united kingdom company working computer consumables printer supply business activity Technology Hardware & Equipment +source electrical electronic manufacturing company located aragon cir buena park california united state Technology Hardware & Equipment +superior electric llc electrical electronic manufacturing company located mid town ct hendersonville tennessee united state Technology Hardware & Equipment +component inc electrical electronic manufacturing company located hein dr garner north carolina united state Technology Hardware & Equipment +z auto sound consumer electronics company based newfield ave hartford connecticut united state Technology Hardware & Equipment +wire guy computer networking company located san francisco california united state Technology Hardware & Equipment +dealing trading mobile phonesselling world especially dubaihkg singapore market Technology Hardware & Equipment +corporation indent project participates behalf principal international tender floated government organisation public private sector around world field energy telecommunication infrastructure civil aviation oil gas steel chemical industrial plant procurement electrical equipment substation transmission line thermal hydroelectric power plant equipment oil gas telecommunication scada railway refinery pipeline metal petroleum product fertilizer chemical barter counter trade construction dam canal airport highway industrial project Technology Hardware & Equipment +n smoking solution inc offer unique electroniccigarette brand come various flavor nicotine level eg high medium low zero whether prefer strong tobacco taste menthol mildness perhaps cigar always find need storesthe owner nickolas martinez amber stancato know product first hand exsmokers conventional tobacco current user n electronic cigarette honest successful people well since built business employ people serf customer nationwidewe proudly help local community assembling managing product midwest united state n smoking solution inc offer electronic cigarette accessory impeccable quality right price throughout year company famous excellent customer service expertise wide selection productsa n smoking solution inc active advocate conventional tobacco smoking sure product help american live better life motto save money change humanity Technology Hardware & Equipment +a1 office equipment located san juan puerto rico company working computer consumables printer supply business activity Technology Hardware & Equipment +baisitai computer accessory co ltd important player onto chinese computer accessory market company founded specialized manufacture wide variety computer connecting cable usb cable vga cable hdmi cable lan cable company headed famous shenzhen special economic zone china covering area square meter employing peopleby virtue excellent product advanced technology constant innovation strong management company registered continuous business expansion term revenue geographical coveragethe company develops produce high quality wire cable certified international quality management system iso baisitai product engineered r center thoroughly tested conformity international standard Technology Hardware & Equipment +american electric electrical electronic manufacturing company based parfet st littleton co united state Technology Hardware & Equipment +b w airconditioning located bassendean australia company working electrical good air conditioning business activity visit company website information u http wwwabwaircomau Technology Hardware & Equipment +help small medium business virtualize infrastructure making business mobile secure scalable eliminating capital expenditure itour completecloud solution avatara llc helped customer across country reduce cost enhance security disaster recovery increase efficiency performance eliminate costly hardware purchase Technology Hardware & Equipment +cummings electrical electronic manufacturing company located sandy hook rd street maryland united state Technology Hardware & Equipment +c electronics located coleraine ireland company working electrical good computer business activity visit company website information u http wwwacelectronicscouk Technology Hardware & Equipment +c group semiconductor company located 19th ave se ste everett washington united state Technology Hardware & Equipment +g air conditioning located san juan puerto rico company working electrical good air conditioning business activity Technology Hardware & Equipment +g custom consumer electronics company based jewel cave rd se albuquerque new mexico united state Technology Hardware & Equipment +k computer located leopold st iv12 4be nairn nairnshire united kingdom company working computer software solution computer service business activity Technology Hardware & Equipment +k computer computer network security company located phoenix arizona united state Technology Hardware & Equipment +europages european b2b platform language million registered company mainly manufacturer service provider wholesaler distributor every month million unique visitor world use platform search product service supplier Technology Hardware & Equipment +l electric inc electrical electronic manufacturing company located plum st brooksville florida united state Technology Hardware & Equipment +p oil inc computer networking company based 65th st brooklyn new york united state Technology Hardware & Equipment +niche time computer network security company based watson st syracuse new york united state Technology Hardware & Equipment +prompt computer computer network security company located highway 1d belle chasse louisiana united state Technology Hardware & Equipment +print located newport united kingdom company working computer consumables printer supply business activity Technology Hardware & Equipment +quality business system electrical electronic manufacturing company based delsea dr franklinville new jersey united state Technology Hardware & Equipment +rapid pc inc computer network security company located military trl lake worth fl united state Technology Hardware & Equipment +source technology consumer electronics company based n pastoria ave sunnyvale california united state Technology Hardware & Equipment +z autoparts limited electrical electronic manufacturing company located colmore circus queensway birmingham united kingdom Technology Hardware & Equipment +z welding electrical electronic manufacturing company located se 121st pl ocklawaha florida united state Technology Hardware & Equipment +z computing solution semiconductor company located n arbor ave queen creek arizona united state Technology Hardware & Equipment +v computer computer hardware company located tuolumne rd b sonora california united state Technology Hardware & Equipment +v technology intl inc computer hardware company located clayton st watertown massachusetts united state Technology Hardware & Equipment +z electric co electrical electronic manufacturing company located weaver rd west milford nj united state Technology Hardware & Equipment +j product solution prototype plus fast growing consumer electronics product prototyping company place highest value excellence customer service quality speed working relationshipswe help product developer designer turn idea 3d form touched held manipulated prototype near representation real thing whether manufacturing engineer industrial designer help bring idea near lifecall u arrange visit production facility meet people make solution happen youthe company started art villa focus producing urethane casting prototype us since evolved onestop service center help product developer designer tell story pitch idea confirm manufacturing activity make real partsaj helped company large small startup design studio major corporation develop product line build business Technology Hardware & Equipment +re america s vehicle safety consultant vehicle safety technology vst make driver smarter saferwe strongly recommend hand car key employee teen driver consults u expert show driver safe save money gas reduce repair costsfleetsour vst system modernize fleet reduce operating cost obtain full roi le monthsave gas repair collision cost yes percent repair repair nt need implement patented vst methodfor detail order technology fill contact form visit wwwmysamaritannetparentsprotect child save money vehicle crash number one killer teen ahead gun violence disease drug use use vst method make sure nt become statisticfor detail order technology fill contact form visit wwwmysamaritannet Technology Hardware & Equipment +hotfrog u provides information regarding computer fix orlando fl computer fix located orlando fl provides security service contact visiting website http wwwfixmefastcom Technology Hardware & Equipment +alaser high precision laser cutting service specializing ultra thin film foil application service wide range industry common thread thin material high precision high quality laser system built maintain tolerance better making u ideal uv ir source Technology Hardware & Equipment +b refrigeration company electrical electronic manufacturing company located w warren st dearborn height michigan united state Technology Hardware & Equipment +b c located bangkok thailand company working electrical good business activity Technology Hardware & Equipment +b c engineering located liverpool united kingdom company working art craft antique business activity Technology Hardware & Equipment +thinking installing outdoor synthetic grass residential commercial area 1st impression synthetic lawn green remodeling yard beautiful low maintenance usable landscape space easy artificial grass perfect solution making yard place beauty without hassle go along maintaining natural lawn synthetic lawn installed around tree rock bush hillside yard furniture much today s synthetic grass closely resembles natural grass product professional level installation people hard time discerning difference artificial grass natural grass enjoy sitting lawn without itching grass stain installing artificial lawn kind environment save serious money use simple formula average bay area residential lawn cost per square foot professional Construction +21st century lift take enormous pride dedication excellent service total customer satisfaction ethos allowed u establish one nation s leading lift installation maintenance company originally lift service company supporting major lift manufacturer year grow Construction +2b designed provides pioneering elegant lighting solution expert residential lighting designer innovation provide stylish architectural lighting design image show method use convey concept client Construction +2plan certainly fall category spent considerable time money setting infrastructure support exact requirement arguably highest profile company recent year success strategy depends clear vision ability execute highest standard Construction +3cl worldwide industrial refrigeration air conditioning design build engineer consultant installation service maintenance contractor Construction +welcome 3dreid awardwinning architecture practice established people across uk studio staff malaysia studio provide full architectural service masterplanning interior design reputation built upon providing creative intelligent architectural solution meet operational brief without compromise seek proud everything concept delivery success practice quality project rely solely quality people employ encourage people move across organisation work different colleague different project share knowledge resource idea believe improves skill expertise enriches work produce client everybody 3dreid exceptionally good enjoy process making good architecture Construction +3twenty modular design manufacture modular workforce housing office product exploration mining oil gas industry durable 3twenty s module built steel instead wood frame truly survive harsh industrial environment stackable 3twenty s module engineered stack allowing vertical expansion transportable 3twenty s module meant move whether truck train barge whatever equipment field comfortable 3twenty focus smart design attention detail provide comfortable living work space Construction +3j ryan inc founded provide comprehensive construction fabrication maintenance instrumentation electrical service privately held full service contracting company corporate office fabrication facility conveniently located pasadena texas near houston ship channel extensive fabrication capability consist inhouse specialty service including welding pipe tank vessel structural steel skid duct work fabrication multifaceted operation offer service site well pride ability meet perform construction maintenance need customer operate certified coded fabrication facility uphold highest level quality project accordance applicable code standard Construction +star located cairo egypt company working engineering business activity Construction +north premier building company offering consulting development designbuild construction property management general contracting mission continue develop one truly transparent building business practice client treated like partner involved entire process description north builder owned operated sean burke president northern heritage builder year sean awarded prism award best custom home within career managed million dollar construction project east coast operated construction company seven different state developed property far north maine far south turk caicos island british west indie Construction +custom electronics installation tv installation home theater home automation security surveillance installation service new york new jersey area appointment call estimate free charge install flat screen tv various environment tv fireplace patio commercial residential Construction +5th wall design llc offer custom coffered wood ceiling provide beautiful raised panel effect ceiling coffered meaning sunken panel typically seen shape square rectangle ceiling 5th wall design coffered ceiling system includes solid wood grid raised panel l u beam built pr Construction +distributor certified trainerand installer granicrete international system revolutionary concrete overlay system adheres surface look function real stone marble granite decorative concrete fraction cost product line includes counter top overlay interior flooring exterior flooring epoxy flooring 3d application wall system outdoor kitchen barbecue surface Construction +c ltd located george town cayman island company working construction business activity Construction +electric inc licensed electrical contracting company electric founded company commercial industrial installation always known service electric s goal always provide quality installation fair price management team average year electrical construction experience electric also service department handle service need pride professional personal service ability partner commercial industrial client project Construction +building svc construction company located 44th st sw c wyoming michigan united state Construction +home remodeling amp constr construction company located terremont cir norcross ga united state Construction +driver ed building material company based main st hamilton ohio united state Construction +flooring construction company located holton ave fort wayne united state Construction +quality concrete construction company located rolling hill dr meridian idaho united state Construction +gutter construction company located miller dr greensboro north carolina united state Construction +2nd look construction company located 2nd st atwood kansa united state Construction +american roofing co construction company located mountain ave monrovia california united state Construction +north shore s premier hardscape contractor Construction +service paving contractor driveway contractor concrete contractor specializing concrete driveway replacement since replaced driveway bringing year concrete construction experience every jobsite Construction +driveway etc construction company located la paloma dr houston texas united state Construction +excavation construction company located fm melissa texas united state Construction +floor galore construction company based macarthur blvd oakland california united state Construction +quality redimix construction company located highway socorro new mexico united state Construction +z plumbing amp heating construction company located canyon rd e puyallup washington united state Construction +service home contractorsinteriorlandscaping fencingconstructionexteriorgeneral Construction +family owned operated company providing home inspection testing people buying real estate also offer radon water quality testing well mold testinginspections massachusetts licensed home inspector pesticide mold certified massachusetts construction supervisor member nachi internachi npma nepma service provided termite carpenter ant treatment well termite damage repair expanded include duct cleaning hvac system dryer vent cleaning Construction +1st choice builder established year company completed project project include custom home multi family apartment commercial building type home remodeling repair ultimately goal create high quality home commercial building appropriate setting Construction +design located quezon city philippine company working interior design business activity Construction +quiet revolution taking place roofing industry 3c network brings together roofing contractor insurance company homeowner building owner ensure roof siding gutter replacement repair done right right away 3c network contractor committed highest standard using roofing industry s accurate measuring estimating tool Construction +3d facility support specialise engineering solution facility management commercial industrial building size client engage u deliver planned maintenance support programme bespoke need business priority business ethos deliver quality dependable trustwort Construction +since season air conditioning heating inc family keeping central florida comfortable fullservice florida statecertified class air conditioning repair contractor offer emergency service planned maintenance sale installation design engineering whether s routine ac maintenance air conditioning rep Construction +7star event management stand designed built unique speak identity build express vision bold statement ground stand design quote instantly assembling spot backed year experience uae architect team Construction +b restoration offer complete range roofing repair construction service fully able handle roofing job matter size commercial residential type general roofing b restoration filing insurance claim let u handle detail roof replacement Construction +b timber engineering based heart suffolk serving east anglia beyond quality engineered timber product able offer full design manufacture supply roof truss easi joistsall manufactured item delivered fleet transport Construction +b maintenance located nassau bahamas company working cleaning equipment service building material construction service construction business activity Construction +h commercial office cleaning inc formed provide specialized service building maintenance related service common wealth pennsylvania within greater pittsburgh area minority majority woman owned company look provide superior service seeking empower strenthen entrepenuerial spirit minority community sothwestern pennsylvania beyond may turn support empower family community large Construction +leading fabricator granite marble quartz prestigious building contractor developer also member public throughout lower mainland british columbia canadawith sf fabrication space year experience industry latest technology cnc qualified craftsman specializing customized stone product produce best quality product second none marketplace Construction +residential commercial roof coloradoa h roofing installs type roofing asphalt shingle commercial flat roof work includes small repair job residential way large commercial projectsroofing type include asphalt shingle wood shake tile metal exposed fastener standing seam designer stone coated metal including custom metal application custom metal flashing flat roof epdm tpo modified buildup tar gravel roof inspection roof certification install top quality material roofing project house inspector ensure top quality job every project Construction +skyline roofing specializes many type residential roof installation roof repair price believe fair affordable pride year experience roofing industry past work year gained knowledge base pas customer recommending application material appropriate customer roofing project along expert advice ice damage prevention proper ventilationin addition year experience pride superior workmanship fair business practice highest level customer service honest ethical way business Construction +k builder located dublin ireland company working industrial service building material construction equipment construction equipment construction construction business activity Construction +since l exterior setting apart company quality workmanship honest trustworthy craftsman dedication putting customer first l exterior pride quality craftsmanship relationship forge customer entrust work highly skilled respectful employee coupled using quality material allows u meet goal customer satisfaction nt believe u ask around majority work come customer referral large scale marketing campaign family owned company customer assured owner involved phase job negotiating term condition completing finishing touch project nt let family owned root fool l exterior capacity accomplish job scope ranging large commercial job small residential repair Construction +local family owned company fixing basement macomb oakland wayne county year pride customer service fixing basement foundation leak clean quick manner meet beat written estimate basement waterproofing company offer lifetime guarantee work life houseour motto fix right first time Construction +established fencing company ltd specialise supplying erecting type fencing gate trained fence erectors hold cscs card minimum requirement demonstrating level health safety fencing industry competence carry work number local authority housing association buildingconstruction organisation school also several industrial domestic property around greater london home county south east england Construction +service commercial commercial service composition shingle metal roofing mobile home pvc roofing residential residential service roofing tile Construction +repair construction company located georgia ave silver spring maryland united state Construction +provide onestop shopping asphalt parking lot need contact u today quote Construction +interior construction company based station rd quakertown pennsylvania united state Construction +a1 painting performance film year experience insured licensed bonded offer interiorexterior painting service specialty finish window tinting film commercial residential installation seattle area western washington lead safe certified Construction +pool service building material company based ruby st redwood city california united state Construction +excavating construction company located birch st fall river massachusetts united state Construction +construction llc construction company located seaway rd gulfport m united state Construction +team architect designer practicing architecture urbanism team collaborator context new paradigm professional practice pronounced oneoneten attempt address emerging challenge time urban place regional landscape community unified philosophy approach explore new theme architectural theory research form collaboration armature built unbuilt work emerging force architecture canada work founded belief architecture profound powerful role shaping experience world growing collective work pursues opportunity collaborate expert relevant discipline enable architecture enrich reinterpret contribute transformative cultural exchange realize potential unique project project attempt situate new experience within urban rural Construction +1st place building service provide extensive building maintenance service commercial market specialise personal oneonone service commitment start finish deliver project time within budget highest quality building standard complete satisfaction tradespeople highly e Construction +3d company inc heavy highway civil earthwork general contractor diversity help u meet need customer construct engineer solution variety problem customer face daytoday basis work indiana department transportation private owner contractor local government municipality private engineering firm combined background experience well year dot prequalified indiana certified disadvantaged business enterprise dbe company company headquartered muncie indiana work business size public private 3d company inc belief hard work pay good business relationship key completing quality job Construction +3d air sale ltd uk uk distributor mitsubishi heavy industry air conditioning system mitsubishi heavy industry one world s largest manufacturer air conditioning system long standing reputation exceptional reliability quality engineering evident product whether Construction +3rd dimension inc founded unique vision specializing museum industry endeavored expand creative horizon architect designer wide variety art production talented staff combine latest technology fabrication traditional scenic treatment Construction +committed incorporating sustainable construction project believe sustainability mean meeting need today without compromising ability future generation meet need careful design sustainable construction aim achieve principle applying method use le virgin material Construction +minute team also offer full architectural interior landscape design service offering continuity consistency inception completion minute makeover team prepare sketch proposal extending renovating property carefully consider need occupant functionality space de Construction +lighting located colombo sri lanka company working home garden construction business activity Construction +carry landscaping stone supply need landscape chip sand gravel decorative boulder pavers block whether homeowner small walkway project contractor large culvert material needed get job doneour service excellent price competitive product highquality Construction +h electrical service based north west operate area united kingdom carry aspect electrical work design installation commercial industrial large domestic project currently completing project matalan newbury high wycombe process introducing energy saving system help reduce carbon footprint client work american embassy londoncity syncclement dickenssainsburystescogalliford trychubb fire securityprotec fire systemsnational tyresspeedy hireenterprise car hirewalkibooths supermarketsassociated british port Construction +j ryan mechanical service building mechanical service company based godstone surrey project across london south east englandthe company specialises supply installation building mechanical service high specification residential commercial development Construction +contractor supply fit type timber window door throughout central belt scotland specialising sash case type window listed building conservation area customer include housing association commercial building office many residential private client Construction +cabinet refacing construction company based front st lisle illinois united state Construction +maintenance amp repair construction company located knoxville tennessee united state Construction +plastering construction company located coast hwy oceanside california united state Construction +glass repair replacement service olney illinois auto glass home glass business glass service Construction +wastewater svc construction company based hayes rd silsbee texas united state Construction +waterproofing construction company based rustic ln lothian maryland united state Construction +drilling construction company located pearl jackson jackson ohio united state Construction +floor covering construction company located centre st west roxbury united state Construction +foundation construction company based ne 225th ave brush prairie washington united state Construction +plumbing company construction company located catskill dr old hickory tennessee united state Construction +arbor tree service construction company based farm market rd kalispell montana united state Construction +associate located pune india company working construction service business activity Construction +frame civil engineering company based w appleton ave milwaukee wisconsin united state Construction +tire construction company based thorpe ave orange city florida united state Construction +land survey civil engineering company based karen way nw salem oregon united state Construction +national roofing inc construction company located clay ave waco texas united state Construction +stucco glass ceramic concrete company based westbriar san antonio texas united state Construction +coring inc construction company located w front st hutto texas united state Construction +blacktop construction company based melissa terry ln joplin missouri united state Construction +marketing inc building material company located willow ave elkins park pennsylvania united state Construction +redi mix construction company based po box chippewa fall wisconsin united state Construction +pool construction company based purick st blue point new york united state Construction +asphalt paving inc construction company located alabama ave paterson new jersey united state Construction +assoc located pune india company working construction service business activity Construction +fire system inc construction company located scarlett ct dublin california united state Construction +home improvement construction company located park rd charlton massachusetts united state Construction +pavers civil engineering company located e prince rd landrum south carolina united state Construction +aamerican roofing inc understand need true professional field roofing remodeling service key component satisfying customer timely work done competitive price professional roofing technician commit contract honor professional code complete job highest quality integrity craftsmanship service continue build business reputation Construction +people madison wisconsin surrounding area come rely aa quality welding mfg last year commercial residential welding project locally owned operated aa quality welding mfg provides standard welding service decorative iron work home business licensed insured w Construction +complete golf course material supplier whether building maintaining golf course make job easier let u supply Construction +approved air pty ltd located salisbury australia company working oil gas company business activity Construction +21st century trucking located kingston jamaica company working vehicle service business activity Automobiles & Components +star auto located beirut lebanon company working vehicle service business activity question please contact u Automobiles & Components +service auto repairsexteriorinteriorgeneral Automobiles & Components +garage located phnom penh cambodia company working vehicle service business activity visit company website information u http wwwiknowcomkh919garage question please contact u Automobiles & Components +general info pay high price new auto part find everything need auto salvage car stock foreign domestic looking get rid old junker also buy car truck matter condition carry part vehicle get new inventory added daily call u today find looking Automobiles & Components +service auto repairsheating coolingbrakestransmission drivetrainsteering suspensiongeneraltireselectrical electronicengine exhaust system Automobiles & Components +auto glass located 34a passagefort dr ptmr jamaica company working car part accessory business activity Automobiles & Components +hotfrog u provides information regarding j auto repair danbury ct j auto repair located chestnut street ext danbury ct provides auto repair service contact Automobiles & Components +hotfrog u provides information regarding h auto repair hawaiian garden ca h auto repair located centralia rd hawaiian garden ca provides auto repair service contact Automobiles & Components +j unique position bodyshop mechanical workshop one site give u advantage ensuring vehicle need catered one place whether need major accident repair small repair bumper mot service advice car fully trained staff help least know man Automobiles & Components +currently work dealership referral gsa ndot metropolitan police department clark county insurance company provide best auto body paint service certification knowledge equipment put back factory spec separate building aluminum repair provide free estimate known dedication customer service best body shop la vega Automobiles & Components +transmission automotive company located n 1st st artesia new mexico united state Automobiles & Components +auto recycling automotive company located narrow rd painesville ohio united state Automobiles & Components +hotfrog u provides information regarding stop auto repair wyoming mi stop auto repair located 28th st sw provides automotive maintenance repair service contact Automobiles & Components +service auto repairsengine exhaust systemexteriorinteriorbrakes Automobiles & Components +service auto repairsexteriorinterior Automobiles & Components +hotfrog u provides information regarding hour auto repair new york ny hour auto repair located park ave provides automotive maintenance repair service contact Automobiles & Components +europages european b2b platform language million registered company mainly manufacturer service provider wholesaler distributor every month million unique visitor world use platform search product service supplier Automobiles & Components +hotfrog u provides information regarding star auto care mentor oh star auto care located twinbrook rd mentor oh provides auto repair service contact Automobiles & Components +hotfrog u provides information regarding star auto repair santa clara ca star auto repair located el camino real santa clara ca provides auto repair service contact Automobiles & Components +service auto repairsexteriorinteriorgeneral Automobiles & Components +b auto located hunter street pobox trinidad tobago company working farming agriculture industrial service construction service vehicle service environmental service business activity Automobiles & Components +b motor located lloydsville yala jamaica company working vehicle service business activity Automobiles & Components +car part located crowthorne united kingdom company working others business activity Automobiles & Components +j enterprise located kingston jamaica company working automotive car part accessory vehicle service business activity Automobiles & Components +machine shop located hector urdaneta santurce puerto rico company working vehicle service business activity Automobiles & Components +family owned operated since rv center founded rochester michigan tiny office attached gas station small display area indoor service bay fifty year later third generation part business dealership expanded nearly acre outdoor display area stateofthe art indoor showroom large service department dedicated reconditioning building fullystocked partsaccessories store second location added midland mia rv center sell variety fifth wheel travel trailer hybrid trailer toy hauler truck camper motor home diesel pusher motor home travel trailer rental available auburn hill location rv center part route rv dealer network elite group dealer chosen based reputation providing excellent customer service membership route rv dealer network entitles rv center customer priority rv service part accessory discount nearly location around country Automobiles & Components +allen otani ralph olmos opened auto repair shop hermosa beach factory trained master service technician year professional experience r german motor hermosa beach premier dealership alternative service maintenance repair bmw mini cooper mercedes benz vehicle without paying dealership priceswe take time explain work need done provide accurate written itemized estimate service repair part surprise using factory proven merchandise high quality distributed reputable firmswe provide unbeatable service excellent review Automobiles & Components +professional r custom tinting trained certified serve window tinting need quickly affordably world class customer service attitude r custom tinting us llumar window film nation s largest manufacturer window film llumar window film come lifetime nationwide warranty provide customer best window film best pricewe specialize auto tinting make model vehicle highest quality window film great customer service offer window tinting variety different transparency work performed highly skilled certified tint technician clean controlled indoor environmentwe also offer complete auto detailing service need car cleaned bring shop ll take care auto detailing always recommended sell car detailing great way make car feel new expert custom window tinting service llumar window film prevents harmful uv ray heat coming car help improve fuel economy reducing need airconditioning tinting also stop interior damage sun help resale value car want help car look best keep comfortable looking best window tinting company gulf breeze give u call stop today let u tint car mechanicsa r custom tinting also repair vehicle rob r custom tinting twenty five year experience mechanic paint body repair problem vehicle give u call schedule time rob take look itflorida registration mv88117 Automobiles & Components +car wash automotive company based wayne st milledgeville ga united state Automobiles & Components +auto maintenance amp repair automotive company located massillon rd canton ohio united state Automobiles & Components +auto supply automotive company located 28th st springfield oregon united state Automobiles & Components +1imports automotive company based quesada ave san francisco california united state Automobiles & Components +a1 tire auto complete full service automotive repair facility located factory rd serving crystal lake cary algonquin lake hill oakwood hill area year Automobiles & Components +located gananoque island rv centre sell sevices rv s trailer part accessory exceptional professional customer service large selection quality product fair pricing creating incomparable purchase service experience u island rv centre concerned sincerely earning business making customer life provide huge selection fair competitive price experience easy straight forward approach taking risk buying Automobiles & Components +wholesale price doityourselfers car part resellers professional mechanic auto part catalog inventory updated real time online part order shipped day u destination 123autopartscom value relationship referral partner online field Automobiles & Components +motor located cairo egypt company working vehicle sale business activity Automobiles & Components +founded formally reorganized 4x4 utv accessory llc offer professional grade aftermarket gear part cab enclosure accessory utility vehicle allterrain vehicle 4x4 offroad vehicle golf cart bad boy buggy specializing aftermarket u made accessory polaris ranger rzr kawasaki mule trans profxt teryx arctic cat prowler yamaha rhino viking honda big red pioneer john deere gator bobcat many maintain leading retail ecommerce site vast dealership network close relationship many manufacturer industry Automobiles & Components +trailer sale service part trailer sale include flatbed enclose auto utility cargo landscape heavy equipment snowmobile dump motorcycle atv construction plus consider leading trailer dealership nw specializing custom trailer Automobiles & Components +b auto part located portmore ln ptmr jamaica company working vehicle sale vehicle service business activity Automobiles & Components +auto part located st joseph trinidad tobago company working vehicle service business activity Automobiles & Components +hotfrog u provides information regarding k automotive cedar city ut k automotive located northfield rd cedar city ut provides auto repair service contact Automobiles & Components +a1 limousine inc people going place personal business travel Automobiles & Components +auto corporation automotive company located w lake st elmhurst illinois united state Automobiles & Components +service auto repairsheating coolingsteering suspensiongeneraltireselectrical electronicengine exhaust system Automobiles & Components +sale automotive company based n western ave chicago illinois united state Automobiles & Components +auto automotive company located brighton michigan united state Automobiles & Components +auto cleaning automotive company based scale st reidsville north carolina united state Automobiles & Components +autobody inc automotive company located n andreasen dr c escondido california united state Automobiles & Components +detail automotive company located n stanislaus st stockton california united state Automobiles & Components +automart located lot leader plaza mdvl jamaica company working vehicle sale vehicle service business activity Automobiles & Components +motor inc automotive company based state st clarksdale mississippi united state Automobiles & Components +radiator service located ascot vale australia company working automotive business activity Automobiles & Components +hotfrog u provides information regarding transmission arlington tx transmission located cooper st provides automotive maintenance repair service contact Automobiles & Components +h enterprise automotive company located joy rd detroit michigan united state Automobiles & Components +auto detail svc automotive company based w 90th pl denver co united state Automobiles & Components +b motor located dhaka bangladesh company working vehicle service business activity Automobiles & Components +berger precision worldwide leader precision machined ready assemble component assembly specializing automotive transportation medical dental commercial aerospace power tool market location usa canada north american headquarters germany worldwide headquarters china poland privately owned company founded employee worldwide nonunion plant either vda ts16949 certified pride partnering customer development stage overcome technical cost constraint making recommendation reduce cost improve quality main area expertise manufacturing highly complex precision turned component tight tolerance small medium high volume capability many additional operation turning milling broaching grinding honing many type deburring process currently produce high precision machined component assembly various size volume Automobiles & Components +general info better way wholesale auto family owned operated dealership located naugatuck ct since provided top quality preowned vehicle wholesale price superior customer service stock hand picked car truck suv s whatever re looking re sure best price around s sell car per year nt know another independent dealer say Automobiles & Components +supply personable non pressured service specific requirement sourced order required c car established family run business supplying vehicle countrywide workshop croham road crowborough offer quality servicing make reasonable price along mot testing tyre diagnostics Automobiles & Components +c motor automotive company based white st haverhill massachusetts united state Automobiles & Components +colorado spring co new larry h miller collision center sell service collision vehicle greater colorado spring area Automobiles & Components +crivelli buick inc automotive company located po box reno pennsylvania united state Automobiles & Components +j leather inc automotive company based broadway los angeles california united state Automobiles & Components +hotfrog u provides information regarding auto repair fort pierce fl auto repair located n 39th st fort pierce fl provides auto repair service contact Automobiles & Components +aone tire sale service provides quality tire auto repair dothan al call visit u today Automobiles & Components +one truck automotive company located fletcher broome rd monroe north carolina united state Automobiles & Components +p auto part automotive company located 7th st mendota california united state Automobiles & Components +quality auto sale inc located eubank blvd ne albuquerque nm offer quality used car sale Automobiles & Components +palm coast auto rental inc automotive company based n state st bunnell florida united state Automobiles & Components +rental center automotive company located w walnut st chillicothe illinois united state Automobiles & Components +plus auto collision amp towing automotive company located church st laurens south carolina united state Automobiles & Components +plus auto glass automotive company based loveland st metairie louisiana united state Automobiles & Components +plus motor located nairobi kenya company working vehicle sale business activity Automobiles & Components +p engineering located bangkok thailand company working vehicle service business activity Automobiles & Components +motor automotive company located morton st elkhart kansa united state Automobiles & Components +z driving school automotive company based frontage rd moorhead minnesota united state Automobiles & Components +hotfrog u provides information regarding z auto repair pensacola fl z auto repair located w gadsden st pensacola fl provides auto repair service contact Automobiles & Components +z auto automotive company located nashua new hampshire united state Automobiles & Components +general info make friend accident welcome touch class auto body battle ground washington local top quality auto body repair shop business year specializing insured noninsured collision repair pleased provide battleground area free estimate windshield glass replacement rental car assistance complete refinishing icar certified insurance claim welcome guaranteed warranty every repair fix scratch dent rust water leak collision damage break crack failed repair attempt peeling clearcoat whether want full body refinish paint job need scratch disappear got fender bender happy take look damage give free estimate addition competitive pricing touch class auto body also offer superior quality guarantee warranty work also sell replacement oem aftermarket auto body part course able paint part match vehicle installed onbudget friendly pricing payment optionsdo need repair performed pay pocket sure afford love help way often find way lower price repair something better afford without sacrificing quality repair utilize tactic finding used replacement part like new condition local junkyard repairing instead replacing part possible communication key quick repair superior quality safetywe understand want top quality repair vehicle sure safe reliable sometimes may need done quickly touch class auto body always try keep customer updated status vehicle exactly including part delay estimated comp Automobiles & Components +town automotive towing automotive company based jonesboro rd se atlanta georgia united state Automobiles & Components +service truck dealersvan dealer Automobiles & Components +universal auto body amp paint automotive company located normandie ave gardena california united state Automobiles & Components +family owned operated company established responsible providing outstanding auto collision repair painting ever since specialty area custom paint matching frame work body repair majority customer repeat customer wreckamended past customer keep coming back particularly value honesty integrity outstanding workmanship offer fair price accept insurance work insurance insurance company best interest mind work hard work insurance company get vehicle back preaccident condition using oem part substandard part result diminished auto valuewe service type vehicle including limited car truck motorcycle rv boat trailer perform repair matter big small tiny scratch major accident burned lightbulb rebuilding motor custom paint match specialist well frame machine expert also offer welding service boat motor hood painting general auto repair service onestop shop Automobiles & Components +k finishing team dedicated exceeding customer expectation delivering quality product first time every time supplier choice employer choice keep door open maintain communication level team without get third party involved working Automobiles & Components +topper denver s leading truck topper truck accessory provider colorado offer best brand industry leer snugtop ranch fiberglass noone denver offer better selection choose fromwe also specialize aftermarket addition make truck look exactly like want tocheck u either one two convenient location Automobiles & Components +auto glass located aurora co offer top quality material workmanship complete warranty insurance company billing year experience call auto glass auto glass repair auto glass replacement headlight restoration auto home glass scratch removal side mirror replacement rating bbb troubleshooter website year shop aurora bring car mobile work denver metro area would love opportunity make sure windshield replacement done right way Automobiles & Components +a1 auto supply located manila philippine company working farming tractor part spare car part accessory transport business activity question please contact u Automobiles & Components +a1 clutch brake located se 9th ct bahamas company working automotive business activity Automobiles & Components +hotfrog u provides information regarding a1 foreign auto repair mountain view ca a1 foreign auto repair located w el camino real mountain view ca provides auto repair service contact Automobiles & Components +need expert automotive repair car leasing fast call auto service come automotive repair car leasing service new plymouth area auto service handle need source dynamically generated Automobiles & Components +b ford automotive company located 7321a mansfield hwy kennedale texas united state Automobiles & Components +c import located carr tsalcedo tenares dominican republic company working vehicle sale business activity Automobiles & Components +hotfrog u provides information regarding e auto repair holland mi e auto repair located highland ave provides automotive maintenance repair service contact Automobiles & Components +g auto body automotive company based georgia ave west palm beach fl united state Automobiles & Components +florida direct automotive company located old u fort myers fl united state Automobiles & Components +aj foyt racing competes verizon indycar series founder aj foyt first 4time winner indianapolis driver win indy daytona hour lemans Automobiles & Components +j tucker supply automotive company located cochran dr greenville south carolina united state Automobiles & Components +lube automotive company located fort mill hwy fort mill south carolina united state Automobiles & Components +k auto inc automotive company located w main st washington united state Automobiles & Components +1to1 training system unique method training working hand hand industry develop highly skilled practitioner hairdressing beauty industry aim provide learning environment provides realistic experience learning within workplace many trainer employer salon owner currently work within industry experience knowledge best equips trainee right attitude skill preparation employment training hub found various australian state brisbane launceston Educational Services +scuba diving course r1790 save r600 scuba diving course includes course material use scuba gear incl wetsuits etc scuba diving lesson whole family well scuba diving equipment scuba diving trip scuba club Educational Services +one hertfordshire s finest creative digital training studio established year st albans year limited company offer variety training course web design including wordpress html seo video editing final cut pro x adobe premiere pro adobe effect scriptwriting film comedy fil Educational Services +decade 21st century way make teach understand experience support art constant flux order take first step towards addressing change art sector essential field tap ingenuity youngest professional ironically due low pay unreliable career path frustrating glass ceiling lack agency autonomy art sector currently experiencing exodus gen xers millennials need insight 20under40 brings voice young art leader margin forefront cultural dialogue first project kind 20under40 began call chapter proposal attracted submission generated buzz throughout internet selected author within anthology represent diverse array art professional including teaching artist administrator dancer filmmaker musician screenwriter Educational Services +stage right productions2nd street theater exists continue cultivate performing art central oregon providing venue resource education central oregon actor playwright director producer stage film pursue creative endeavor stage right production also produce several theatrical event year form play concert movie screening class workshop adult child ticket sale important keeping stage light s enough need support community inkind financial contribution stage right production nonprofit organization c status pending Educational Services +5th wall design delivers range fun dynamic learning development programme deliver unique online training room work closely produce learning experience engaging fastpaced relevant time delivering real sustained change return investment get touch Educational Services +hotfrog u provides information regarding driving school san francisco ca driving school located broderick st apt san francisco ca provides school service contact Educational Services +jewelry professional training company located broadway new york new york united state Educational Services +1st approach transport service located peterborough united kingdom company working training business activity Educational Services +fine art gallery painting sculpture fine gemstone jewelry lapidary work oil acrylic watercolor sumie Educational Services +360learning come need deploy elearning strategy one solution let team create deploy etrainings minute enjoy use first user friendly elearning software manager smoothly interacts team driving success setup fee obligation stop month 360learningtm web based requires configuration ready use even big company monitoring etrainings impact key part successful elearning strategy 360learning offer easy understand realtime figure chart participation rate user involvment success rate customizable report 360learningtm serving company size form world top small business Educational Services +4life college sure friendly professional handson trainer deliver positive welcoming learning environment one goal mind make sure leave skill qualification give confidence excel chosen career purposeful practical training along relevant Educational Services +leader grant education since 4grantsnet created nevada s 1st distance ed course grant industry s 1st gpc exam prep course online web s longest running grant writing course featuring stateoftheart audio video offered since u college new grant education product include 3day grant conference create grant readiness 3ring binder 12month mentoring program gpc exam prep home study course paid internship webinar facilitator certification program gpcs cfres funder research 5webinar series letter proposal writing 4webinar series turnkey service include virtual nonprofit management funder research grant readiness grant writing grant management sustainability planning faculty grant professional certified gpc Educational Services +51creditcom provides advanced outstanding training environment strategy cultivating talented staff realized synchronous developing staff company working financial industry 51creditcom excellent choice first step path financial expert Educational Services +key provides kindermusik germanborn music organization program targeted newborn sevenyearolds parent involved intimately class musical development child fun activity based around music key www88keysca five scholarship award child underser Educational Services +training industry Educational Services +located warszawa poland company working training business activity question please contact u Educational Services +pro cleaning svc inc education management company based allegiance dr north la vega nv united state Educational Services +one located dorking united kingdom company working training business activity Educational Services +way store located vega baja puerto rico company working education business activity Educational Services +360see exhibit fine art functional art furniture design object driven sustainable minded medium choice philosophy result gallery s curatorial process evident rotating exhibition represented artist designer showing work share modern aesthetic sustainable minded conceptual medium element 360see position intersection strong challenging well executed fine functional art sustainable minded medium choice philosophy defining premier gallery kind Educational Services +3d coaching valuesbased business least work done fee basis addition profit 3d coaching ltd given away year working nondirective way organisation many year claire pedrick founding director christian belief strongly influenced devel Educational Services +mission three fold create manage charter alternativeeducation private school urban youth deliver cuttingedge solution urban attitude problem rampant today s hip hop culture train professional effectively relate educate generation student currently oversee three school york pennsylvania school challenge academy alternative education middle high school impact academy elementary risk private school new hope academy public charter school public charter school serving grade also oversee uicf urban impact community foundation operates nonprofit organization raise fund scholarship student well urban renewal project york city working create educational opportunity safe passage youth neighbor allow positive development birth college career new groun Educational Services +77academy offer range graduate corporate adhoc training program graduate training 10week fulltime training course graduate want enter new medium industry looking practical handson experience facebook training full halfday training course introduce key marketing tool availabl Educational Services +contact 9th chakra carl fisher fl get tarot card reading acquire metaphysical supply balance chakras Educational Services +provide traffic safety driver education adult beginner expert teen year old individual corporation want learn accident avoidance technique instructor licensed bonded professional receive three time training required secretary state teac Educational Services +plumbing education management company located po box altus arkansas united state Educational Services +b c playschool located manchester united kingdom company working child s service education business activity Educational Services +brighter rainbow learning education management company located boston rd billerica massachusetts united state Educational Services +bit paradise education management company located homewood dr san jose california united state Educational Services +general info c jones high school offer range educational program student grade academic curriculum includes study english mathematics science history social study school also offer class fine art foreign language health physical education additionally offer gifted talented program includes advanced placement independent study mentorship c jones high school athletics department offer various sport activity football basketball cross country soccer softball baseball wrestling library feature collection reference material directory book catalog addition school parentteacher association c jones high school located beeville texas part beeville independent school district Educational Services +conscious choice hypnosis professional training company based severn maryland united state Educational Services +driving advantage education management company located main st dallas georgia united state Educational Services +fantis parochial school education management company located state st brooklyn new york united state Educational Services +h e c education management company located w 12th st rolla missouri united state Educational Services +great start preschool education management company located wilmington pike dayton ohio united state Educational Services +take stress schoolwork turn nt know imagine spending le time working child s homework time thing lovepicture celebrating report card sending happy confident child school every morningdoes sound good m help name thandiwe helpful tutor take headache advancing child s academic career help see child succeedi tutor student currently enrolled math science english course winter park public library also work adult preparing certification entrance examslearn website wwwahelpfultutorcomsee brighter future child today call schedule free consultation Educational Services +general info carry lodge dutch oven camp chef dutch oven maca dutch oven dutch oven cookbook dutch oven accessory dutch oven table dutch oven cooking class camping gadget volcano stove fire starter fire pan insect repellent lantern camp stove hot dog fork flashlight sleeping bag wall tent backpacking tent leatherman tool pie iron bbq wall tent plenty making everybody deal happy camper Educational Services +training consulting located singapore singapore company working training business activity visit company website information u http wwwmmapplicationscom question please contact u Educational Services +kid place drop child higher education company based route cranberry twp pennsylvania united state Educational Services +jump ahead education management company located woodglen dr oakley california united state Educational Services +l concrete construction education management company located po box seagraves texas united state Educational Services +e zion church inc education management company located james st goldsboro north carolina united state Educational Services +receiving referral bringing child home guide though korean chinese thai colombian international adoption process Educational Services +new horizon counseling education management company located hill rd parsippany new jersey united state Educational Services +martin lerner md higher education company located pierce st beverly hill michigan united state Educational Services +mother touch preschool e learning company based houston town st maryville tennessee united state Educational Services +mystical creation professional training company located fm rd 163rd cleveland texas united state Educational Services +leap ahead education management company located hotel cir san diego california united state Educational Services +p brewer high school education management company located eva rd somerville alabama united state Educational Services +p training located rugby united kingdom company working training business activity Educational Services +professional inspection svc professional training company based east petersburg pa united state Educational Services +q located guildford united kingdom company working training business activity Educational Services +place ponder higher education company based 2nd st hamilton montana united state Educational Services +plus driving academy e learning company based n lebanon st lebanon united state Educational Services +pet sitting service jacksonville florida Educational Services +plus prep school education management company located w parker rd plano texas united state Educational Services +parent resource professional training company located po box mercer island washington united state Educational Services +picket fence professional training company located main st sumner washington united state Educational Services +plus driving school located paranaque city philippine company working training education school business activity Educational Services +plus math tutoring education management company located hollywood florida united state Educational Services +plus math tutor education management company located elizabeth colorado united state Educational Services +hotfrog u provides information regarding safeway driving school bronx ny safeway driving school located white plain rd bronx ny provides school service contact visiting website http asafewaydrivingschoolcom Educational Services +social ignition working reduce stigma shame criminal record effort area twofold first founder sonja skvarla developed curriculum entrepreneurship taught inside outside prison series participant enroll long haul program consists individual small group coaching toward specific goal second almost important folding people criminal record business professional one network bringing people class prison otherwise group meet common ground begin see people instead merely representative label Educational Services +step preschool education management company located brewery rd cross plain wisconsin united state Educational Services +atti training company within mission statement passion enable individual enterprise across south africa achieve greater success providing knowledge skill solution service pioneering effort usage appropriate technology Educational Services +smart start daycare education management company located washington st sterling nebraska united state Educational Services +special place preschool education management company located 158th st se snohomish washington united state Educational Services +thru z best place quality san antonio pediatric care knowledgeable professional staff care child every step way Educational Services +z expedition duke edinburgh s award approved activity provider aap specialise delivering dofe training expedition offer tailor made dofe programme school group open expedition programme individual bronze silver gold walk cycle canoe expedition choice Educational Services +team group promotes practical scientific course student variety course available interest procedure teaching follows authentic global approach montessori reggio emilia gamification method company emphasizes best quality possible lecture follows providing equipment facility technology fully trained teacher Educational Services +g mobile piano school stretching boundary definition piano pedagogya g understands benefit learning music hope make music lesson routine canadianswith g expect traditional teaching method mixed creative innovation memory last wwwatogcawwwfacebookcomatogpianoteaching Educational Services +teacher touch education management company located catherine way escalon california united state Educational Services +z preschool education management company based bonham st paris texas united state Educational Services +tri county driving school higher education company based e main st ste dundee illinois united state Educational Services +training company provides complete classroom online food safety training unique group instructor comprised local chef restaurant manager caterer peer best understand specific challenge today s busy food industry get food safety training exam done hour offer hour online training hour group training also offer training s exam spanish please visit website information calendar class near http wwwatrainingcompanycom Educational Services +z achievement academy education management company based u highway englishtown new jersey united state Educational Services +year production served northwestern student body diverse entertaining programming today continues enrich northwestern experience bringing quality concert interesting informative speaker blockbuster film campus giving member experience planning execution largescale event Educational Services +coding institute provides educational career based training individual interested becoming furthering career certified medical coder student success rate continues increase grow provide positive reinforcement ensure every student learns regardless previous experience level addition instruction expert instructor year coding experience also offer many financial option available tuition cost addition providing educational service coding institute also delivers consulting service focused healthcare operation improvment revenue cycle management specialized expert consultant assist healthcare provider facility medical equipment vendor health care payer interested improving revenue compliance billing coding operational processesvisit u online wwwapluscodinginstitutecomclass size kept low order ensure student get attention need seat reserved firstcomefirstserve basis therefore recommended interested attendee submit registration soon possible guarantee seat Educational Services +a1 driving school located p box p c al hail north oman company working driving school business activity Educational Services +aacademics dedicated success student ever since opened business goal provide highest quality educational assistance affordable price proud say aacademics subsidiary aa tutor service hundred student age throughout san diego ar Educational Services +hotfrog u provides information regarding county pasadena md county located pasadena pasadena md provides furniture service contact Educational Services +aaa self storage higher education company based po box siloam spring ar united state Educational Services +beautiful design higher education company located judy ct macomb michigan united state Educational Services +better pet professional training company located queenston rd cleveland ohio united state Educational Services +basic cremation higher education company located wadsworth blvd lakewood colorado united state Educational Services +better future wholesale company located se division st portland oregon united state Educational Services +air conditioning training school training hvac technician enty level job Educational Services +child place child care ctr education management company located noble st smithfield north carolina united state Educational Services +licensed qualified teacher prepare child school scholarship may available variety nursery school program call today Educational Services +club driving school inc education management company located 43rd st long island city new york united state Educational Services +creative beginning preschool provides gymnastics music cooking fallbrook ca area call Educational Services +child day education management company located po box uvalde texas united state Educational Services +child first step daycare education management company located ogontz ave philadelphia pennsylvania united state Educational Services +based south east nothumberland computer tutor offer oneoff tailormade basic course 50sas well serving domestic customer comuter tutor also offer medium production workshop school community group organisation Educational Services +hotfrog u provides information regarding g cox middle school winterville nc g cox middle school located church st winterville nc provides school service contact Educational Services +g tv repair education management company located n 59th ave glendale arizona united state Educational Services +helping heart education management company based chinowth st visalia california united state Educational Services +kidz academy preschool education management company located w murray utah united state Educational Services +look optical higher education company located sheridan blvd arvada colorado united state Educational Services +4r energy promotes issue address environmental socioeconomics energy benefit waste management public health benefit mobilizing stakeholder facilitating supporting future bioenergy project undertaken alleviating pollution contributes towards soil conservation climatic change company currently complete Energy and Environmental +50lar inc mission drive widespread solar adoption design install finance solar system residential industrial commercial nonprofit client 50lar inc deliver best value educating client designing custom match solar system specific energy requirement determining solar production specific location assessing impact switching standard plan time use billing 50lar inc allows client effectively buy low sell high securing premium solar electricity production buying power utility company offpeak hour nonprofit client developed way establish new llcs benefit federal tax credit depreciation still allow nonprofit reduce electric bill commercial client develop complete sustainability plan including solar fuel cell biogas genus Energy and Environmental +environmental environmental service company located w 24th st norfolk virginia united state Energy and Environmental +1st coast recycling state art scrap recycling facility opened march growing strong recognition southeast ever since buy sell secondary metal scrap quickly processing necessary spec included anything aluminum sink car company motto safety customer employee business Energy and Environmental +2co first cc company backed private equity finance investment business model people whose knowledge resource created exceptional capability 2co turn co threat environment valuable resource reversing effect take captured co2 revitalise exis Energy and Environmental +350green installs charging station infrastructure electric car specialize level public charger provide electric vehicle owner convenient charging 350green locates fast charging station parking lot grocery store shopping center city garage thus providing practical solution driver without garage level secure outlet designated residential multi family unit parking garage large corporate entity hotel anywhere ev spend hour 350green working partner supporting initial nissan electric car large metropolitan city across north america government funding incentive reduce equipment installation cost allows installation charging infrastructure additional cost host 350green installing charging infrastructure grid state pennsylvania chicago metro area well san francisco surrounding bay area por Energy and Environmental +inc develops innovative solution food processing life cycle believe proactive management residual food manufacturing opportunity exist recovery repurposing feeding livestock high quality feed recovering plastic packaging understand goal waste reduction reuse Energy and Environmental +1300rubbish pty ltd www1300rubbishcomau first australian company provide personal rubbish removal service home business 1300rubbish offer superior convenient rubbish removal service load item clean taking away customer nt lift finger service eliminates hassle trip Energy and Environmental +1st waste management located bournemouth united kingdom company working environmental service business activity Energy and Environmental +3r group ltd new zealand business delivering sustainable commercially driven environmental waste solution offer specialist advice product stewardship resource recovery system waste management recycling ewaste recycling solution Energy and Environmental +4t consultant pty ltd environmental consultancy based emerald queensland environmental consulting commitment assist stakeholder use natural resource sustainably possible provide environmental monitoring service help client obtain production little effect environment Energy and Environmental +plumbing heating ac oil energy company based oak st west hempstead ny united state Energy and Environmental +c exterminating co environmental service company based po box meriden connecticut united state Energy and Environmental +general info landscaping construction design excavating landscaping turn residential commercial property dream estate quality landscaping topnotch excavating Energy and Environmental +green touch environmental service company located pine st ste r watsonville california united state Energy and Environmental +healthy touch environmental service company located terry rd louisville kentucky united state Energy and Environmental +kleen sweep environmental service company located elmwood ct hephzibah ga united state Energy and Environmental +lighter touch environmental service company located pearl st bethesda maryland united state Energy and Environmental +raymond fastener india private limited oil energy company located gat 2768b nighoje chakan pune maharashtra india Energy and Environmental +z carpet cleaning environmental service company located cape horn dr henderson nevada united state Energy and Environmental +village preschool oil energy company based strack rd houston texas united state Energy and Environmental +work art landscape oil energy company located w 111th st minneapolis minnesota united state Energy and Environmental +cosmas caltan represent seller petroleum product stone bring buyer seller mandate together matching needsa global enterprise represents stone company china quarry factory help promote product granite marble sandstone limestone roman column fountain table chair bust statue fireplace flower pot vas gazebo statue base feng sui ball floor inlay construction building material like paving stone kerb stone building blockscosmas caltan act facilitator mandate end seller low cost allocation oil russia nigerian light crude oilwe process negotiating suppleirs biodiesel represent serve buyer better Energy and Environmental +help farmer feed world raise healthy animal protect environment nutritional innovation controlled quality product industrial process make major actor improving food chain protecting animal human environment Energy and Environmental +ac electric company inc south carolina corporation specializing electrical contracting company s office located hollywood south carolina employ approximately employee hold unlimited electrical contractor license state south carolina georgia north carolina alabama key personnel include carl bec Energy and Environmental +aa waste management oil energy company based county road fulton missouri united state Energy and Environmental +advanced pumping environmental service company based ne east fork dr yacolt washington united state Energy and Environmental +environmental service provide expertise asbestos abatement waste disposal asbestos survey thermal insulation client include council education body various industrial providersa environmental hold newly hse granted three year asbestos license running march Energy and Environmental +b cooling heating renewables environment company based hudson st orlando florida united state Energy and Environmental +b engineer located kolkata india company working electrical service contractor business activity Energy and Environmental +b septic environmental service company based woodland ridge dr orange texas united state Energy and Environmental +c management group inc oil energy company based e boone ave ste spokane washington united state Energy and Environmental +cleaner world carpet clean environmental service company located e mountain st kernersville north carolina united state Energy and Environmental +c exploration llc oil energy company located augusta dr ste houston texas united state Energy and Environmental +e robison co inc oil energy company based w 151st st eudora kansa united state Energy and Environmental +greener image landscaping environmental service company located green way st petersburg florida united state Energy and Environmental +g oil energy company located w 110th st new york new york united state Energy and Environmental +j heating inc oil energy company based james st franklin square new york united state Energy and Environmental +light remember oil energy company based kirkchapel st spring texas united state Energy and Environmental +little landscaping inc environmental service company located stanley rd bald knob arkansas united state Energy and Environmental +j blosenski inc environmental service company located po box elverson pennsylvania united state Energy and Environmental +new world pest control environmental service company located n kendall dr miami florida united state Energy and Environmental +service retaining wall Energy and Environmental +spence consulting service oil energy company located russell cir plano texas united state Energy and Environmental +vasquez roofing inc oil energy company based sonoma way pinole california united state Energy and Environmental +z tree service environmental service company based po box saint ignatius montana united state Energy and Environmental +carbon asset co ltd ukregistered firm specializing development highquality emission reduction project sale cer vers vcssa committed serving client utmost integrity transparency maintains close relationship regulatory agency project stakeholder throughout project development process team highly trained professional provides advice guidance project creation production pertinent documentation efficient realization final delivery greenhouse gas emission reduction Energy and Environmental +a1 petroleum inc associated company trusted partner supply aviation fuel additive many year slimline dynamic organization allows smooth quick delivery wide range aviation fuel throughout europewe pay special attention providing high quality fuel attractive price pride excellent value money Energy and Environmental +aaqua provides quality european equipment service support equipment affordable reliable importantly highly flexible range treatment storage drinking water wastewater design deliver solution product individual staff refugee camp field hospital municipal preparedness larger urban population aaqua fulfill client need year practical experience production equipment operation maintenance equipment real world situation Energy and Environmental +learning assisting providing positive solution alok product incorporated leader designing building construction seal flexible connector underground stormwater wastewater conveyance system capable providing watertight solution pipe diameter ranging plastic steel iron clay concrete pipe since everchanging demand underground sewer construction led alok develop offer wide range new product include flexible pipetomanhole connector application plastic insert seal manhole entry joint forming manhole inverts corrosion resistant lining lubricant sealant beginning new century provided alok opportunity create watertight storm water system contribute preserving environment improving country s infrastructure future generation support talented human reso Energy and Environmental +e plumbing heating oil energy company located 36th ave w alexandria minnesota united state Energy and Environmental +friend bride environmental service company based brenner pas tallahassee florida united state Energy and Environmental +partner energy efficiency project energy recovery product efficiency energy consumptionefficient manpower planningincreasing productivityheat recovery waste waterheat recovery flue gaswaste water color removalwet scrabber Energy and Environmental +cleaning svc oil energy company based franklin ave troy illinois united state Energy and Environmental +one septic environmental service company based pipestone rd eau claire michigan united state Energy and Environmental +r resource inc oil energy company based spring glen rd jacksonville florida united state Energy and Environmental +rocha grew one project established portugal become international organization coordinating work eight country several new initiative group preparing join network international team established assisting nineteen national organization team Energy and Environmental +cash junk car vehicle accident recovery impound secure large lot recycling center Energy and Environmental +simpler place time environmental service company based arrow blvd apt fontana california united state Energy and Environmental +team environmental environmental service company based nw 15th ct pembroke pine florida united state Energy and Environmental +steam carpet care environmental service company located jumilla ave woodland hill california united state Energy and Environmental +touch nature inc environmental service company located hawes cir riverton utah united state Energy and Environmental +top notch tree service inc environmental service company located e grant st orlando florida united state Energy and Environmental +tak pest control inc environmental service company based essex ct seabrook texas united state Energy and Environmental +w landscape inc environmental service company located redland rd rockville maryland united state Energy and Environmental +full service environmentally safe pest control includes using line product called plus natural enzyme product naturally eliminate mosquito flea tick bed bug fruit drain fly spider mite many bug also offer wood destroying insect report termite inspection see sister company website wwwplusnaturalenzymescom full product description instruction angie s list super service award Energy and Environmental +a1 oil gas company subsidiary punjtan energy based houston texas usa heart world capital oil gas equipment capable supply type oilfield equipment octg pipe specialized incomplete land drilling rig swamp posted barge rig jack offshore drilling group company also capability extend financing drilling production contract gas pipeline oil refinery lng projectsthe following item carry could useful day day operation bop annular double ram single complete stack power swivel top drive ranging 150tons 750tons travelling blockhook spider elevator mud pump caterpillar engine rig power gen set fishing handling tool mud logging system spare part equipment full range control valve wellhead safety equipment pumping jack octg supply grade size drill pipe drill collar line pipe casing tubing kelly pup joint sucker rod etc Energy and Environmental +company fastest growing epc business enterprise transmission distribution sectorthe group focus business areas1 power sectorepc part transmission distribution2 facility management servicesservices operation maintenance existing infrstructure3 solid waste management4 cogeneration power project Energy and Environmental +aabsolute painting environmental service company located old cheney rd lincoln nebraska united state Energy and Environmental +aardvark pest control based wigan lancashire expert wildlife management fumigation flea bed bug cockroach ant mouse rat mole wasp nest removal service provide throughout northwest england rsph bpca qualified basisprompt member crru wildlife aware accredited Energy and Environmental +aaron chiropractic clinic oil energy company located stellhorn rd fort wayne united state Energy and Environmental +largest collection historic cigarette lighter presently display xinhai electric co ltd ningbo chinaone foundation civilization know ability make transport fire ever since cavedwelling ancestor rubbed stick stone together humankind pursued quest efficient reliable firemaking device quest inspired innovative invention history collection dutch lighter museum foundation trace fascinating journey fire steel ancient world tinderbox seventeenth century modern cigarette lighter carefully collected astonishing variety designsthe numerous unique distinctive whimsical lighter chart voyage lighter important scientific discovery treasured collector s item along way learn history pipe popularization smoking growth today s global marketplace well significant event wondrous invention played role history lighter Energy and Environmental +manchester electrical contractor eca approved electrical contractor electrical installation service inspection testing electrical maintenance emergency lighting system company manchester north west Energy and Environmental +aaron landscaping environmental service company located little eagle creek westfield indiana united state Energy and Environmental +aaxon lab inc oil energy company located augusta dr fort worth texas united state Energy and Environmental +ab resource ab fast growing privatelyheld company actively engaged natural gas oil exploration development production appalachian basin company s prime marcellus shale acreage pennsylvania west virginia retains potential horizontal well location company s strategic direction Energy and Environmental +abalon exterminating co environmental service company located park ave new york new york united state Energy and Environmental +world s leading manufacturer industrial oil skimmer coolant maintenance product Energy and Environmental +abbey appliance oil energy company located independence missouri united state Energy and Environmental +great quality great service great price tool equipment supply need Energy and Environmental +abb technologybased provider power automation product system solution service building core strength technology leadership pioneering spirit sustainable approach business abb help customer become profitable lowering environmental impact abb operation nigeria headquartered lagos branch abuja porthatcourt abb operates around country employ approximately people truly multinational organization global headquarters zurich switzerland abbng incorporated registered corporate affair commission federal republic nigeria 24th february company registration rc Energy and Environmental +general info dependable home heating oil available Energy and Environmental +abbottstown paradise joint environmental service company located w water st abbottstown pennsylvania united state Energy and Environmental +abesim expert soil risk managementwe want expert indepth market knowledge pragmatic solution soil pollution deal itour methodology follows law procedure standard reference guide ovam bim owd specific belgian region flanders wallonia brusselscapital france polandabesim reached process quality management compliance iso standard site engineering service consulting concerning environmental diagnostics soil soil decontamination geotechnics soil structure interactionabesim competent site investigation measurement campaign characterize state environment indicate source spread potential target pollution realization actualization concept investigation environmental vulnerability assessment state environment management plan analysis residual risk view potential action concerning use soil state environment abesim equipment necessary investigate potentially contaminated site soil equipment accomplish mission three hydraulic drilling machine mounted 4x4 truck rotary drilling solid hollow auger drilling percussion drilling percussion gouge drilling several van manual drilling water sampling Energy and Environmental +ability tree svc environmental service company based dhanmandi cir tampa florida united state Energy and Environmental +abl landscaping inc oil energy company located elmgrove st farmington hill michigan united state Energy and Environmental +abode plumbing located queanbeyan australia company working oil gas company business activity Energy and Environmental +plumbing located warner bay australia company working oil gas company business activity Energy and Environmental +production oil energy company based n raymond ave pasadena california united state Energy and Environmental +tree svc environmental service company located rock spring rd apopka florida united state Energy and Environmental +absolute electric located dalkey ireland company working electrical service business activity Energy and Environmental +absolute green energy corporation focused part solution energy independence environmental responsibility provide range service help comercial industrial residential facility reduce energy cost carbon footprint application energy optimizing technology renewable energy system process aimed fast thorough analysis situation cost effective implementation offer site assessment provide renewable energy system option budget provide turnkey project management service large commercial facility licensed electrician plumber offer training equipment let get business quickly competentlyfor anyone interested renewable energy system offer short day seminar aimed educating public technology cost incentive Energy and Environmental +absolute pest management llc environmental service company located manchester rd 11a derry new hampshire united state Energy and Environmental +company focus raw material biodiesel production energy source lng Energy and Environmental +a farr electrical helping hobart property owner business electrical work year team qualified electrician apprentice assist electrical matter fully staffed office assist enquiry Energy and Environmental +asap electrical located wamberal australia company working electrical service business activity Energy and Environmental +ax located al khobar saudi arabia company working oil gas company business activity Energy and Environmental +a1 electrical service located morphett vale australia company working electrical service business activity Energy and Environmental +aaa electrical located enniscorthy ireland company working electrical service business activity Energy and Environmental +skip hire alternative load rubbish take away Energy and Environmental +houston s newest microbrewery located springchampions area s easy start oilfield move brewing industry like founder s gallon barrel crude oil gallon barrel beer see Food and Beverages +rabbit brings food back root providing authentic simple treat delectable granola granola bar source ingredient small farmer committed sustainability product certified organic ccof give bar made kid need Food and Beverages +cafe located dammam saudi arabia company working food retailer coffee shop business activity Food and Beverages +1st stop located london united kingdom company working wine beer wine beer wholesale business activity Food and Beverages +restaurant located tangerang indonesia company working restaurant business activity Food and Beverages +dog australian fast food franchise located heart brisbane gold coast also coming city serving mouthwatering succulent hotdog vegan friendly sausage visit u get dog yah Food and Beverages +s cafe located cairo egypt company working restaurant italian restaurant business activity Food and Beverages +degree wood fired pizzeria idea born shared passion high quality food building community around positive dining experience owner priscilla roundeau infuses belief protecting environment every aspect running degree chef david smith reflects share priscilla s belief creates dish highlight new hampshire s local farm fresh ingredient degree make food different high quality ingredient environment prepared live fire influence dish prepared add unique quality searing heat smoke Food and Beverages +five location katy sugar land richmond wharton houston serving best burger fry area Food and Beverages +bottle specialty beer store brew including beer washington microbrewery handcrafted beer around usa imported beer nearly country mix custom 6pack bottle choose door cold beer door hard cider plus shelf beer mead handcrafted fruit soda beer virtually always available others constantly change due seasonal rotating limited release fact nearly beer made way onto store s book ll opportunity discover unique brew upon visiting bottle federal way wash bottle also large selection beer glases tshirts hat breweriana beer store gift card stop store wednesday eve pm sample two beer featuring owner weekly pick visit web site www99bottlesnet tasting calendar special guest visiting sto Food and Beverages +supermarket located birmingham united kingdom company working food retailer food market business activity Food and Beverages +hotfrog u provides information regarding b food los angeles ca b food located e olympic blvd provides social food dining restaurant seafood service contact Food and Beverages +e marketing ltd located kingston jamaica company working food retailer health food fitness dietician business activity Food and Beverages +j mini market located mitcham united kingdom company working food retailer food market business activity Food and Beverages +j wholesale located grey street ne24 blyth united kingdom company working food retailer fruit importer food product business activity Food and Beverages +meat located walsall united kingdom company working food retailer butcher business activity Food and Beverages +cake located leyland lane pr25 1tb leyland united kingdom company working food retailer sweet shop business activity Food and Beverages +food located sheffield united kingdom company working food retailer fishmonger business activity Food and Beverages +cue barbecue owner operated barbecue restaurant location milton cumming georgia everything homemade even bread pickle offer catering service anywhere metro atlanta area franchise available interested bos experience restaurant industry give u call Food and Beverages +sign wine sign wine currently sold online 12signswinecom select retailer san diego area finch s wine bar sea rocket bistro vin de syrah boney s bayside market west cedar market splash sign created san diego local marie fox wine sourced boutique vineyard califor Food and Beverages +restaurant located hanoi vietnam company working restaurant business activity Food and Beverages +service restaurant Food and Beverages +bottle shop located singapore singapore company working wine beer whisky spirit business activity visit company website information u http www1855thebottleshopcom Food and Beverages +perfection located place de plancenoit 6a plancenoit belgium company working restaurant business activity visit company website information u http www2perfectionbe question please contact u Food and Beverages +full service restaurant bakery deli featuring relaxed casual dining atmosphere restaurant ranked restaurant hospitality s top independently owned restaurant restaurant offered smokefree dining item menu sure please appetite age deli serf breakfast Food and Beverages +west club historical building providing classic feel old new york complete crystal chandelier high ceiling art deco touch club centrally located 5th avenue across rockefeller center radio city music hall sq ft meeting event space optimal setting executive meeting private function corporate seminar board meeting elegant reception wedding special event fabulous convenient location midtown manhattan Food and Beverages +monkey located phuket thailand company working wine beer business activity Food and Beverages +hotfrog u provides information regarding bistro duncan ok bistro located n highway provides social food dining restaurant pizza social food dining restaurant barbecue social food dining restaurant fast food service contact Food and Beverages +3d wine formed started hobby many first vineyard discovered numerous visit france year personal friendship winemaker fun visiting vineyard enabled u create still unique business today year experience Food and Beverages +43rd street deli breakfast house served breakfast lunch gainesville fl community past year locally owned deli known nationally serf homemade food friend family enjoy serve breakfast lunch day diverse breakfast menu northern florida w Food and Beverages +cafe located hanoi vietnam company working cafe business activity Food and Beverages +great tasting steak mouth watering appetizer matter occasion enjoy causally premium experience fork restaurant thunder bay Food and Beverages +north chicago area providing quality food including bugers chicken ice cream Food and Beverages +5th avenue cafe located jakarta indonesia company working cafe business activity Food and Beverages +star restaurant located ho chi minh city vietnam company working restaurant business activity question please contact u Food and Beverages +cafe located hanoi vietnam company working cafe business activity Food and Beverages +restaurant located singapore singapore company working restaurant chinese restaurant business activity Food and Beverages +meat located newtownards united kingdom company working food retailer butcher business activity Food and Beverages +b wholesale located whitegate inner prom fy8 1dw lytham st anne united kingdom company working catering catering supply business activity Food and Beverages +b service located quito ecuador company working food retailer food product business activity Food and Beverages +j produce located leicester united kingdom company working food retailer fruit importer food product business activity Food and Beverages +mini market located carr km bo montellanos morovis puerto rico company working food retailer food market business activity Food and Beverages +r located jakarta indonesia company working food retailer food market business activity Food and Beverages +carpet cleaner restaurant company based park circle dr swainsboro georgia united state Food and Beverages +w restaurant located bekasi indonesia company working restaurant business activity Food and Beverages +vending co food beverage company located se 14th ave portland oregon united state Food and Beverages +chef brings table selection authentic sicilian recipe made fresh ingredient imported island sun authentic sicilian food brought table authentic sicilian plate feature siclian signature dish something every palate taste salad pasta eggplant original norma rigatoni vegetarian seared mediterranean sardine gluttonous special sweet sour tuna filet healthy chef also present rich variety special dish paired one exceptional sicilian wine simplicity preparation freshness ingredient make dining experience cacio e vino unforgettable hope food able take tasty multisensorial journey sicily s flavor scent Food and Beverages +point located diepestraat leefdaal belgium company working restaurant business activity visit company website information u http www2thepointbe question please contact u Food and Beverages +windsor st offer picture perfect setting wedding reception located burbage beautiful village close hinckley warwickshire every room within windsor st decorated extremely high standard food world offer individualised completely bespoke wedding service clien Food and Beverages +season restaurant located dhaka bangladesh company working restaurant chinese restaurant business activity Food and Beverages +hotfrog u provides information regarding 4food bronx ny 4food located lincoln ave provides social food dining restaurant service contact Food and Beverages +tay street restaurant perth eat scotland gold award holder using produce sourced locally fine supplier little afield combined strong emphasis seasonality tay street restaurant situated heart historic perth adjacent river tay Food and Beverages +copper caboose opened door corner 4th street boston avenue lubbock texas founded john frank fitzpatrick frank smith first restaurant lubbock texas serve fajitas serving chicken beef fajitas added many traditional american authentic mexican dish prepared lupe gonzales 50th street caboose opened august corner 50th street slide road designed managing partner phil bradley owner bob jordan twist enhance proven caboose concept midway game redemption center stocked full toy novelty giant stuffed animal help enhance fun atmosphere serving fine mexican dish new twist george amayathe 50th street caboose also party room available conduct meeting throw birthday party team party view sporting event private gettogether friend Food and Beverages +cafe located quezon city philippine company working restaurant business activity Food and Beverages +till late located hounslow united kingdom company working food retailer food market business activity Food and Beverages +restaurant located hong kong hong kong company working restaurant business activity Food and Beverages +bottle winery vineyard located four mile southwest hayfield iowa heart midwest family owned run farmstyle farm style winery established mark newman come enjoy beautiful view complementary wine tasting take tour vineyard winery temperature controlled wine cave location also perfect outdoor wedding reunion graduation party event Food and Beverages +depot located semarang indonesia company working restaurant business activity Food and Beverages +supermarket located 168a168b sutcliffe avenue dn33 1ha grimsby north lincolnshire united kingdom company working food retailer food market business activity Food and Beverages +trading located singapore singapore company working food retailer food product business activity Food and Beverages +import llc americanbrazilian partnership committed introducing american wine consumer best wine brazil mission leading importer quality brazilian product governing policy business respect fairness honesty believe quality excellent service value partner consumer lastly believe relationship important measure success Food and Beverages +z medical supply food production company based main st danbury connecticut united state Food and Beverages +alcoholic anonymous located ll61 6pa llanfairpwllgwyngyll isle anglesey united kingdom company working wine beer business activity Food and Beverages +meat inc food production company located 111th st sw lakewood washington united state Food and Beverages +repair restaurant company located endicott ln houston texas united state Food and Beverages +america cargo svc food production company based n sam houston pkwy e houston texas united state Food and Beverages +bellagio restaurant company located central ave campbell california united state Food and Beverages +b mini market food beverage company located fulton st brooklyn new york united state Food and Beverages +better cheesecake food production company based n mcmullen booth rd clearwater florida united state Food and Beverages +brooklyn pizzaria food beverage company located mission gorge rd san diego california united state Food and Beverages +c legg food beverage company based po box calera alabama united state Food and Beverages +cake reason located noranda australia company working food retailer food product business activity Food and Beverages +chau located girrawheen australia company working food retailer butcher business activity Food and Beverages +coleman located melton mowbray united kingdom company working food retailer sweet shop business activity Food and Beverages +complete service restaurant company located brevard rd asheville north carolina united state Food and Beverages +cut deli food beverage company located otterhole rd west milford nj united state Food and Beverages +delicate touch located freeport bahamas company working restaurant business activity Food and Beverages +dish dash catering food beverage company located alpharetta hwy alpharetta georgia united state Food and Beverages +buy flower local florist salem va flower shoppe main provide floral gift need salem va Food and Beverages +j produce food production company based central ave los angeles california united state Food and Beverages +j cafe located dandenong australia company working cafe business activity Food and Beverages +j s located islington australia company working food retailer food market business activity Food and Beverages +j s bakery located healesville australia company working food retailer baker business activity Food and Beverages +j s cafe located balaclava australia company working cafe business activity Food and Beverages +liquor store food beverage company located e 88th ave abc denver co united state Food and Beverages +hotfrog u provides information regarding bakery sun city az bakery located w bell rd provides social food dining bakerie service contact Food and Beverages +mustard seed located chiang mai thailand company working food retailer coffee shop business activity Food and Beverages +la heart catering food beverage company located benson ln fort bragg california united state Food and Beverages +oseguera co food production company located rogge rd salina california united state Food and Beverages +ap deli started chicago s south side 107th michigan little corner food liquorgrocery store sold corned beef sandwich side Food and Beverages +hotfrog u provides information regarding pizza heart nederland tx pizza heart located highway provides social food dining restaurant pizza service contact Food and Beverages +piece past restaurant company located main st white bluff tennessee united state Food and Beverages +sweet connection design develop display table occasion currently design tasty eye catching display event wedding reception engagement party birthday party prom shower corporate event private party display customized individual preference need popular display candy sweet bar popcorn bar snack bar cereal bar much Food and Beverages +spot tail salmon guide restaurant company located magnolia blvd w seattle washington united state Food and Beverages +view menu hour coupon taste chicago located n hollywood way burbank ca order taste chicago delivery online phone mypizzacom Food and Beverages +spoonful sugar located high street auckland new zealand company working pub club social club restaurant business activity Food and Beverages +z irrigation food beverage company located n sea mecox rd southampton new york united state Food and Beverages +taste remember food beverage company located n altadena ave royal oak michigan united state Food and Beverages +make tavolastore source gourmet food professionalquality cookware choose small kitchen appliance cooking utensil decor match cooking entertaining style wwwatavolastorecomkitchen accessory welcome join tavola email list registering individual company reseller hear latest product bargain cooking classesinquire private cooking lesson showcase unique selection cookware ceramic knife comfort home Food and Beverages +taste tuscany food beverage company located n bridge st bridgewater new jersey united state Food and Beverages \ No newline at end of file diff --git a/download_glue.py b/download_glue.py new file mode 100644 index 0000000..107935d --- /dev/null +++ b/download_glue.py @@ -0,0 +1,142 @@ +''' Script for downloading all GLUE data. +Note: for legal reasons, we are unable to host MRPC. +You can either use the version hosted by the SentEval team, which is already tokenized, +or you can download the original data from (https://download.microsoft.com/download/D/4/6/D46FF87A-F6B9-4252-AA8B-3604ED519838/MSRParaphraseCorpus.msi) and extract the data from it manually. +For Windows users, you can run the .msi file. For Mac and Linux users, consider an external library such as 'cabextract' (see below for an example). +You should then rename and place specific files in a folder (see below for an example). +mkdir MRPC +cabextract MSRParaphraseCorpus.msi -d MRPC +cat MRPC/_2DEC3DBE877E4DB192D17C0256E90F1D | tr -d $'\r' > MRPC/msr_paraphrase_train.txt +cat MRPC/_D7B391F9EAFF4B1B8BCE8F21B20B1B61 | tr -d $'\r' > MRPC/msr_paraphrase_test.txt +rm MRPC/_* +rm MSRParaphraseCorpus.msi +''' + +import os +import sys +import shutil +import argparse +import tempfile +import urllib.request +import zipfile + +TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "SNLI", "QNLI", "RTE", "WNLI", "diagnostic"] +TASK2PATH = { + "CoLA": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4', + "SST": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', + "MRPC": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc', + "QQP": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5', + "STS": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5', + "MNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce', + "SNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df', + "QNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLI.zip?alt=media&token=c24cad61-f2df-4f04-9ab6-aa576fa829d0', + "RTE": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb', + "WNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf', + "diagnostic": 'https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D'} + +MRPC_TRAIN = 'https://s3.amazonaws.com/senteval/senteval_data/msr_paraphrase_train.txt' +MRPC_TEST = 'https://s3.amazonaws.com/senteval/senteval_data/msr_paraphrase_test.txt' + + +def download_and_extract(task, data_dir): + print("Downloading and extracting %s..." % task) + data_file = "%s.zip" % task + urllib.request.urlretrieve(TASK2PATH[task], data_file) + with zipfile.ZipFile(data_file) as zip_ref: + zip_ref.extractall(data_dir) + os.remove(data_file) + print("\tCompleted!") + + +def format_mrpc(data_dir, path_to_data): + print("Processing MRPC...") + mrpc_dir = os.path.join(data_dir, "MRPC") + if not os.path.isdir(mrpc_dir): + os.mkdir(mrpc_dir) + if path_to_data: + mrpc_train_file = os.path.join(path_to_data, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(path_to_data, "msr_paraphrase_test.txt") + else: + mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt") + urllib.request.urlretrieve(MRPC_TRAIN, mrpc_train_file) + urllib.request.urlretrieve(MRPC_TEST, mrpc_test_file) + assert os.path.isfile(mrpc_train_file), "Train data not found at %s" % mrpc_train_file + assert os.path.isfile(mrpc_test_file), "Test data not found at %s" % mrpc_test_file + urllib.request.urlretrieve(TASK2PATH["MRPC"], os.path.join(mrpc_dir, "dev_ids.tsv")) + + dev_ids = [] + with open(os.path.join(mrpc_dir, "dev_ids.tsv")) as ids_fh: + for row in ids_fh: + dev_ids.append(row.strip().split('\t')) + + with open(mrpc_train_file, encoding="utf8") as data_fh, \ + open(os.path.join(mrpc_dir, "train.tsv"), 'w') as train_fh, \ + open(os.path.join(mrpc_dir, "dev.tsv"), 'w') as dev_fh: + header = data_fh.readline() + train_fh.write(header) + dev_fh.write(header) + for row in data_fh: + label, id1, id2, s1, s2 = row.strip().split('\t') + if [id1, id2] in dev_ids: + dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + else: + train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + + with open(mrpc_test_file) as data_fh, \ + open(os.path.join(mrpc_dir, "test.tsv"), 'w') as test_fh: + header = data_fh.readline() + test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n") + for idx, row in enumerate(data_fh): + label, id1, id2, s1, s2 = row.strip().split('\t') + test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2)) + print("\tCompleted!") + + +def download_diagnostic(data_dir): + print("Downloading and extracting diagnostic...") + if not os.path.isdir(os.path.join(data_dir, "diagnostic")): + os.mkdir(os.path.join(data_dir, "diagnostic")) + data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv") + urllib.request.urlretrieve(TASK2PATH["diagnostic"], data_file) + print("\tCompleted!") + return + + +def get_tasks(task_names): + task_names = task_names.split(',') + if "all" in task_names: + tasks = TASKS + else: + tasks = [] + for task_name in task_names: + assert task_name in TASKS, "Task %s not found!" % task_name + tasks.append(task_name) + return tasks + + +def main(arguments): + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', help='directory to save data to', type=str, default='glue_data') + parser.add_argument('--tasks', help='tasks to download data for as a comma separated string', + type=str, default='all') + parser.add_argument('--path_to_mrpc', + help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt', + type=str, default='') + args = parser.parse_args(arguments) + + if not os.path.isdir(args.data_dir): + os.mkdir(args.data_dir) + tasks = get_tasks(args.tasks) + + for task in tasks: + if task == 'MRPC': + format_mrpc(args.data_dir, args.path_to_mrpc) + elif task == 'diagnostic': + download_diagnostic(args.data_dir) + else: + download_and_extract(task, args.data_dir) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/modeling.py b/modeling.py new file mode 100644 index 0000000..dde9af1 --- /dev/null +++ b/modeling.py @@ -0,0 +1,988 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The main BERT model and related functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import copy +import json +import math +import re +import six +import tensorflow as tf + + +class BertConfig(object): + """Configuration for `BertModel`.""" + + def __init__(self, + vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02): + """Constructs BertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = BertConfig(vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with tf.gfile.GFile(json_file, "r") as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +class BertModel(object): + """BERT model ("Bidirectional Embedding Representations from a Transformer"). + + Example usage: + + ```python + # Already been converted into WordPiece token ids + input_ids = tf.constant([[31, 51, 99], [15, 5, 0]]) + input_mask = tf.constant([[1, 1, 1], [1, 1, 0]]) + token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]]) + + config = modeling.BertConfig(vocab_size=32000, hidden_size=512, + num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024) + + model = modeling.BertModel(config=config, is_training=True, + input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids) + + label_embeddings = tf.get_variable(...) + pooled_output = model.get_pooled_output() + logits = tf.matmul(pooled_output, label_embeddings) + ... + ``` + """ + + def __init__(self, + config, + is_training, + input_ids, + input_mask=None, + token_type_ids=None, + use_one_hot_embeddings=True, + scope=None): + """Constructor for BertModel. + + Args: + config: `BertConfig` instance. + is_training: bool. rue for training model, false for eval model. Controls + whether dropout will be applied. + input_ids: int32 Tensor of shape [batch_size, seq_length]. + input_mask: (optional) int32 Tensor of shape [batch_size, seq_length]. + token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. + use_one_hot_embeddings: (optional) bool. Whether to use one-hot word + embeddings or tf.embedding_lookup() for the word embeddings. On the TPU, + it is must faster if this is True, on the CPU or GPU, it is faster if + this is False. + scope: (optional) variable scope. Defaults to "bert". + + Raises: + ValueError: The config is invalid or one of the input tensor shapes + is invalid. + """ + config = copy.deepcopy(config) + if not is_training: + config.hidden_dropout_prob = 0.0 + config.attention_probs_dropout_prob = 0.0 + + input_shape = get_shape_list(input_ids, expected_rank=2) + batch_size = input_shape[0] + seq_length = input_shape[1] + + if input_mask is None: + input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32) + + if token_type_ids is None: + token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32) + + with tf.variable_scope(scope, default_name="bert"): + with tf.variable_scope("embeddings"): + # Perform embedding lookup on the word ids. + (self.embedding_output, self.embedding_table) = embedding_lookup( + input_ids=input_ids, + vocab_size=config.vocab_size, + embedding_size=config.hidden_size, + initializer_range=config.initializer_range, + word_embedding_name="word_embeddings", + use_one_hot_embeddings=use_one_hot_embeddings) + + # Add positional embeddings and token type embeddings, then layer + # normalize and perform dropout. + self.embedding_output = embedding_postprocessor( + input_tensor=self.embedding_output, + use_token_type=True, + token_type_ids=token_type_ids, + token_type_vocab_size=config.type_vocab_size, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=config.initializer_range, + max_position_embeddings=config.max_position_embeddings, + dropout_prob=config.hidden_dropout_prob) + + with tf.variable_scope("encoder"): + # This converts a 2D mask of shape [batch_size, seq_length] to a 3D + # mask of shape [batch_size, seq_length, seq_length] which is used + # for the attention scores. + attention_mask = create_attention_mask_from_input_mask( + input_ids, input_mask) + + # Run the stacked transformer. + # `sequence_output` shape = [batch_size, seq_length, hidden_size]. + self.all_encoder_layers = transformer_model( + input_tensor=self.embedding_output, + attention_mask=attention_mask, + hidden_size=config.hidden_size, + num_hidden_layers=config.num_hidden_layers, + num_attention_heads=config.num_attention_heads, + intermediate_size=config.intermediate_size, + intermediate_act_fn=get_activation(config.hidden_act), + hidden_dropout_prob=config.hidden_dropout_prob, + attention_probs_dropout_prob=config.attention_probs_dropout_prob, + initializer_range=config.initializer_range, + do_return_all_layers=True) + + self.sequence_output = self.all_encoder_layers[-1] + # The "pooler" converts the encoded sequence tensor of shape + # [batch_size, seq_length, hidden_size] to a tensor of shape + # [batch_size, hidden_size]. This is necessary for segment-level + # (or segment-pair-level) classification tasks where we need a fixed + # dimensional representation of the segment. + with tf.variable_scope("pooler"): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. We assume that this has been pre-trained + first_token_tensor = tf.squeeze(self.sequence_output[:, 0:1, :], axis=1) + self.pooled_output = tf.layers.dense( + first_token_tensor, + config.hidden_size, + activation=tf.tanh, + kernel_initializer=create_initializer(config.initializer_range)) + + def get_pooled_output(self): + return self.pooled_output + + def get_sequence_output(self): + """Gets final hidden layer of encoder. + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size] corresponding + to the final hidden of the transformer encoder. + """ + return self.sequence_output + + def get_all_encoder_layers(self): + return self.all_encoder_layers + + def get_embedding_output(self): + """Gets output of the embedding lookup (i.e., input to the transformer). + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size] corresponding + to the output of the embedding layer, after summing the word + embeddings with the positional embeddings and the token type embeddings, + then performing layer normalization. This is the input to the transformer. + """ + return self.embedding_output + + def get_embedding_table(self): + return self.embedding_table + + +def gelu(input_tensor): + """Gaussian Error Linear Unit. + + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + + Args: + input_tensor: float Tensor to perform activation. + + Returns: + `input_tensor` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.erf(input_tensor / tf.sqrt(2.0))) + return input_tensor * cdf + + +def get_activation(activation_string): + """Maps a string to a Python function, e.g., "relu" => `tf.nn.relu`. + + Args: + activation_string: String name of the activation function. + + Returns: + A Python function corresponding to the activation function. If + `activation_string` is None, empty, or "linear", this will return None. + If `activation_string` is not a string, it will return `activation_string`. + + Raises: + ValueError: The `activation_string` does not correspond to a known + activation. + """ + + # We assume that anything that"s not a string is already an activation + # function, so we just return it. + if not isinstance(activation_string, six.string_types): + return activation_string + + if not activation_string: + return None + + act = activation_string.lower() + if act == "linear": + return None + elif act == "relu": + return tf.nn.relu + elif act == "gelu": + return gelu + elif act == "tanh": + return tf.tanh + else: + raise ValueError("Unsupported activation: %s" % act) + + +def get_assignment_map_from_checkpoint(tvars, init_checkpoint): + """Compute the union of the current variables and checkpoint variables.""" + assignment_map = {} + initialized_variable_names = {} + + name_to_variable = collections.OrderedDict() + for var in tvars: + name = var.name + m = re.match("^(.*):\\d+$", name) + if m is not None: + name = m.group(1) + name_to_variable[name] = var + + init_vars = tf.train.list_variables(init_checkpoint) + + assignment_map = collections.OrderedDict() + for x in init_vars: + (name, var) = (x[0], x[1]) + if name not in name_to_variable: + continue + assignment_map[name] = name + initialized_variable_names[name] = 1 + initialized_variable_names[name + ":0"] = 1 + + return (assignment_map, initialized_variable_names) + + +def dropout(input_tensor, dropout_prob): + """Perform dropout. + + Args: + input_tensor: float Tensor. + dropout_prob: Python float. The probability of dropping out a value (NOT of + *keeping* a dimension as in `tf.nn.dropout`). + + Returns: + A version of `input_tensor` with dropout applied. + """ + if dropout_prob is None or dropout_prob == 0.0: + return input_tensor + + output = tf.nn.dropout(input_tensor, 1.0 - dropout_prob) + return output + + +def layer_norm(input_tensor, name=None): + """Run layer normalization on the last dimension of the tensor.""" + return tf.contrib.layers.layer_norm( + inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) + + +def layer_norm_and_dropout(input_tensor, dropout_prob, name=None): + """Runs layer normalization followed by dropout.""" + output_tensor = layer_norm(input_tensor, name) + output_tensor = dropout(output_tensor, dropout_prob) + return output_tensor + + +def create_initializer(initializer_range=0.02): + """Creates a `truncated_normal_initializer` with the given range.""" + return tf.truncated_normal_initializer(stddev=initializer_range) + + +def embedding_lookup(input_ids, + vocab_size, + embedding_size=128, + initializer_range=0.02, + word_embedding_name="word_embeddings", + use_one_hot_embeddings=False): + """Looks up words embeddings for id tensor. + + Args: + input_ids: int32 Tensor of shape [batch_size, seq_length] containing word + ids. + vocab_size: int. Size of the embedding vocabulary. + embedding_size: int. Width of the word embeddings. + initializer_range: float. Embedding initialization range. + word_embedding_name: string. Name of the embedding table. + use_one_hot_embeddings: bool. If True, use one-hot method for word + embeddings. If False, use `tf.nn.embedding_lookup()`. One hot is better + for TPUs. + + Returns: + float Tensor of shape [batch_size, seq_length, embedding_size]. + """ + # This function assumes that the input is of shape [batch_size, seq_length, + # num_inputs]. + # + # If the input is a 2D tensor of shape [batch_size, seq_length], we + # reshape to [batch_size, seq_length, 1]. + if input_ids.shape.ndims == 2: + input_ids = tf.expand_dims(input_ids, axis=[-1]) + + embedding_table = tf.get_variable( + name=word_embedding_name, + shape=[vocab_size, embedding_size], + initializer=create_initializer(initializer_range)) + + if use_one_hot_embeddings: + flat_input_ids = tf.reshape(input_ids, [-1]) + one_hot_input_ids = tf.one_hot(flat_input_ids, depth=vocab_size) + output = tf.matmul(one_hot_input_ids, embedding_table) + else: + output = tf.nn.embedding_lookup(embedding_table, input_ids) + + input_shape = get_shape_list(input_ids) + + output = tf.reshape(output, + input_shape[0:-1] + [input_shape[-1] * embedding_size]) + return (output, embedding_table) + + +def embedding_postprocessor(input_tensor, + use_token_type=False, + token_type_ids=None, + token_type_vocab_size=16, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1): + """Performs various post-processing on a word embedding tensor. + + Args: + input_tensor: float Tensor of shape [batch_size, seq_length, + embedding_size]. + use_token_type: bool. Whether to add embeddings for `token_type_ids`. + token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. + Must be specified if `use_token_type` is True. + token_type_vocab_size: int. The vocabulary size of `token_type_ids`. + token_type_embedding_name: string. The name of the embedding table variable + for token type ids. + use_position_embeddings: bool. Whether to add position embeddings for the + position of each token in the sequence. + position_embedding_name: string. The name of the embedding table variable + for positional embeddings. + initializer_range: float. Range of the weight initialization. + max_position_embeddings: int. Maximum sequence length that might ever be + used with this model. This can be longer than the sequence length of + input_tensor, but cannot be shorter. + dropout_prob: float. Dropout probability applied to the final output tensor. + + Returns: + float tensor with same shape as `input_tensor`. + + Raises: + ValueError: One of the tensor shapes or input values is invalid. + """ + input_shape = get_shape_list(input_tensor, expected_rank=3) + batch_size = input_shape[0] + seq_length = input_shape[1] + width = input_shape[2] + + output = input_tensor + + if use_token_type: + if token_type_ids is None: + raise ValueError("`token_type_ids` must be specified if" + "`use_token_type` is True.") + token_type_table = tf.get_variable( + name=token_type_embedding_name, + shape=[token_type_vocab_size, width], + initializer=create_initializer(initializer_range)) + # This vocab will be small so we always do one-hot here, since it is always + # faster for a small vocabulary. + flat_token_type_ids = tf.reshape(token_type_ids, [-1]) + one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) + token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) + token_type_embeddings = tf.reshape(token_type_embeddings, + [batch_size, seq_length, width]) + output += token_type_embeddings + + if use_position_embeddings: + assert_op = tf.assert_less_equal(seq_length, max_position_embeddings) + with tf.control_dependencies([assert_op]): + full_position_embeddings = tf.get_variable( + name=position_embedding_name, + shape=[max_position_embeddings, width], + initializer=create_initializer(initializer_range)) + # Since the position embedding table is a learned variable, we create it + # using a (long) sequence length `max_position_embeddings`. The actual + # sequence length might be shorter than this, for faster training of + # tasks that do not have long sequences. + # + # So `full_position_embeddings` is effectively an embedding table + # for position [0, 1, 2, ..., max_position_embeddings-1], and the current + # sequence has positions [0, 1, 2, ... seq_length-1], so we can just + # perform a slice. + position_embeddings = tf.slice(full_position_embeddings, [0, 0], + [seq_length, -1]) + num_dims = len(output.shape.as_list()) + + # Only the last two dimensions are relevant (`seq_length` and `width`), so + # we broadcast among the first dimensions, which is typically just + # the batch size. + position_broadcast_shape = [] + for _ in range(num_dims - 2): + position_broadcast_shape.append(1) + position_broadcast_shape.extend([seq_length, width]) + position_embeddings = tf.reshape(position_embeddings, + position_broadcast_shape) + output += position_embeddings + + output = layer_norm_and_dropout(output, dropout_prob) + return output + + +def create_attention_mask_from_input_mask(from_tensor, to_mask): + """Create 3D attention mask from a 2D tensor mask. + + Args: + from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...]. + to_mask: int32 Tensor of shape [batch_size, to_seq_length]. + + Returns: + float Tensor of shape [batch_size, from_seq_length, to_seq_length]. + """ + from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) + batch_size = from_shape[0] + from_seq_length = from_shape[1] + + to_shape = get_shape_list(to_mask, expected_rank=2) + to_seq_length = to_shape[1] + + to_mask = tf.cast( + tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) + + # We don't assume that `from_tensor` is a mask (although it could be). We + # don't actually care if we attend *from* padding tokens (only *to* padding) + # tokens so we create a tensor of all ones. + # + # `broadcast_ones` = [batch_size, from_seq_length, 1] + broadcast_ones = tf.ones( + shape=[batch_size, from_seq_length, 1], dtype=tf.float32) + + # Here we broadcast along two dimensions to create the mask. + mask = broadcast_ones * to_mask + + return mask + + +def attention_layer(from_tensor, + to_tensor, + attention_mask=None, + num_attention_heads=1, + size_per_head=512, + query_act=None, + key_act=None, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None): + """Performs multi-headed attention from `from_tensor` to `to_tensor`. + + This is an implementation of multi-headed attention based on "Attention + is all you Need". If `from_tensor` and `to_tensor` are the same, then + this is self-attention. Each timestep in `from_tensor` attends to the + corresponding sequence in `to_tensor`, and returns a fixed-with vector. + + This function first projects `from_tensor` into a "query" tensor and + `to_tensor` into "key" and "value" tensors. These are (effectively) a list + of tensors of length `num_attention_heads`, where each tensor is of shape + [batch_size, seq_length, size_per_head]. + + Then, the query and key tensors are dot-producted and scaled. These are + softmaxed to obtain attention probabilities. The value tensors are then + interpolated by these probabilities, then concatenated back to a single + tensor and returned. + + In practice, the multi-headed attention are done with transposes and + reshapes rather than actual separate tensors. + + Args: + from_tensor: float Tensor of shape [batch_size, from_seq_length, + from_width]. + to_tensor: float Tensor of shape [batch_size, to_seq_length, to_width]. + attention_mask: (optional) int32 Tensor of shape [batch_size, + from_seq_length, to_seq_length]. The values should be 1 or 0. The + attention scores will effectively be set to -infinity for any positions in + the mask that are 0, and will be unchanged for positions that are 1. + num_attention_heads: int. Number of attention heads. + size_per_head: int. Size of each attention head. + query_act: (optional) Activation function for the query transform. + key_act: (optional) Activation function for the key transform. + value_act: (optional) Activation function for the value transform. + attention_probs_dropout_prob: (optional) float. Dropout probability of the + attention probabilities. + initializer_range: float. Range of the weight initializer. + do_return_2d_tensor: bool. If True, the output will be of shape [batch_size + * from_seq_length, num_attention_heads * size_per_head]. If False, the + output will be of shape [batch_size, from_seq_length, num_attention_heads + * size_per_head]. + batch_size: (Optional) int. If the input is 2D, this might be the batch size + of the 3D version of the `from_tensor` and `to_tensor`. + from_seq_length: (Optional) If the input is 2D, this might be the seq length + of the 3D version of the `from_tensor`. + to_seq_length: (Optional) If the input is 2D, this might be the seq length + of the 3D version of the `to_tensor`. + + Returns: + float Tensor of shape [batch_size, from_seq_length, + num_attention_heads * size_per_head]. (If `do_return_2d_tensor` is + true, this will be of shape [batch_size * from_seq_length, + num_attention_heads * size_per_head]). + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + + def transpose_for_scores(input_tensor, batch_size, num_attention_heads, + seq_length, width): + output_tensor = tf.reshape( + input_tensor, [batch_size, seq_length, num_attention_heads, width]) + + output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) + return output_tensor + + from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) + to_shape = get_shape_list(to_tensor, expected_rank=[2, 3]) + + if len(from_shape) != len(to_shape): + raise ValueError( + "The rank of `from_tensor` must match the rank of `to_tensor`.") + + if len(from_shape) == 3: + batch_size = from_shape[0] + from_seq_length = from_shape[1] + to_seq_length = to_shape[1] + elif len(from_shape) == 2: + if (batch_size is None or from_seq_length is None or to_seq_length is None): + raise ValueError( + "When passing in rank 2 tensors to attention_layer, the values " + "for `batch_size`, `from_seq_length`, and `to_seq_length` " + "must all be specified.") + + # Scalar dimensions referenced here: + # B = batch size (number of sequences) + # F = `from_tensor` sequence length + # T = `to_tensor` sequence length + # N = `num_attention_heads` + # H = `size_per_head` + + from_tensor_2d = reshape_to_matrix(from_tensor) + to_tensor_2d = reshape_to_matrix(to_tensor) + + # `query_layer` = [B*F, N*H] + query_layer = tf.layers.dense( + from_tensor_2d, + num_attention_heads * size_per_head, + activation=query_act, + name="query", + kernel_initializer=create_initializer(initializer_range)) + + # `key_layer` = [B*T, N*H] + key_layer = tf.layers.dense( + to_tensor_2d, + num_attention_heads * size_per_head, + activation=key_act, + name="key", + kernel_initializer=create_initializer(initializer_range)) + + # `value_layer` = [B*T, N*H] + value_layer = tf.layers.dense( + to_tensor_2d, + num_attention_heads * size_per_head, + activation=value_act, + name="value", + kernel_initializer=create_initializer(initializer_range)) + + # `query_layer` = [B, N, F, H] + query_layer = transpose_for_scores(query_layer, batch_size, + num_attention_heads, from_seq_length, + size_per_head) + + # `key_layer` = [B, N, T, H] + key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, + to_seq_length, size_per_head) + + # Take the dot product between "query" and "key" to get the raw + # attention scores. + # `attention_scores` = [B, N, F, T] + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + attention_scores = tf.multiply(attention_scores, + 1.0 / math.sqrt(float(size_per_head))) + + if attention_mask is not None: + # `attention_mask` = [B, 1, F, T] + attention_mask = tf.expand_dims(attention_mask, axis=[1]) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + adder = (1.0 - tf.cast(attention_mask, tf.float32)) * -10000.0 + + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + attention_scores += adder + + # Normalize the attention scores to probabilities. + # `attention_probs` = [B, N, F, T] + attention_probs = tf.nn.softmax(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = dropout(attention_probs, attention_probs_dropout_prob) + + # `value_layer` = [B, T, N, H] + value_layer = tf.reshape( + value_layer, + [batch_size, to_seq_length, num_attention_heads, size_per_head]) + + # `value_layer` = [B, N, T, H] + value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) + + # `context_layer` = [B, N, F, H] + context_layer = tf.matmul(attention_probs, value_layer) + + # `context_layer` = [B, F, N, H] + context_layer = tf.transpose(context_layer, [0, 2, 1, 3]) + + if do_return_2d_tensor: + # `context_layer` = [B*F, N*V] + context_layer = tf.reshape( + context_layer, + [batch_size * from_seq_length, num_attention_heads * size_per_head]) + else: + # `context_layer` = [B, F, N*V] + context_layer = tf.reshape( + context_layer, + [batch_size, from_seq_length, num_attention_heads * size_per_head]) + + return context_layer + + +def transformer_model(input_tensor, + attention_mask=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + do_return_all_layers=False): + """Multi-headed, multi-layer Transformer from "Attention is All You Need". + + This is almost an exact implementation of the original Transformer encoder. + + See the original paper: + https://arxiv.org/abs/1706.03762 + + Also see: + https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py + + Args: + input_tensor: float Tensor of shape [batch_size, seq_length, hidden_size]. + attention_mask: (optional) int32 Tensor of shape [batch_size, seq_length, + seq_length], with 1 for positions that can be attended to and 0 in + positions that should not be. + hidden_size: int. Hidden size of the Transformer. + num_hidden_layers: int. Number of layers (blocks) in the Transformer. + num_attention_heads: int. Number of attention heads in the Transformer. + intermediate_size: int. The size of the "intermediate" (a.k.a., feed + forward) layer. + intermediate_act_fn: function. The non-linear activation function to apply + to the output of the intermediate/feed-forward layer. + hidden_dropout_prob: float. Dropout probability for the hidden layers. + attention_probs_dropout_prob: float. Dropout probability of the attention + probabilities. + initializer_range: float. Range of the initializer (stddev of truncated + normal). + do_return_all_layers: Whether to also return all layers or just the final + layer. + + Returns: + float Tensor of shape [batch_size, seq_length, hidden_size], the final + hidden layer of the Transformer. + + Raises: + ValueError: A Tensor shape or parameter is invalid. + """ + if hidden_size % num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (hidden_size, num_attention_heads)) + + attention_head_size = int(hidden_size / num_attention_heads) + input_shape = get_shape_list(input_tensor, expected_rank=3) + batch_size = input_shape[0] + seq_length = input_shape[1] + input_width = input_shape[2] + + # The Transformer performs sum residuals on all layers so the input needs + # to be the same as the hidden size. + if input_width != hidden_size: + raise ValueError("The width of the input tensor (%d) != hidden size (%d)" % + (input_width, hidden_size)) + + # We keep the representation as a 2D tensor to avoid re-shaping it back and + # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on + # the GPU/CPU but may not be free on the TPU, so we want to minimize them to + # help the optimizer. + prev_output = reshape_to_matrix(input_tensor) + + all_layer_outputs = [] + for layer_idx in range(num_hidden_layers): + with tf.variable_scope("layer_%d" % layer_idx): + layer_input = prev_output + + with tf.variable_scope("attention"): + attention_heads = [] + with tf.variable_scope("self"): + attention_head = attention_layer( + from_tensor=layer_input, + to_tensor=layer_input, + attention_mask=attention_mask, + num_attention_heads=num_attention_heads, + size_per_head=attention_head_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + batch_size=batch_size, + from_seq_length=seq_length, + to_seq_length=seq_length) + attention_heads.append(attention_head) + + attention_output = None + if len(attention_heads) == 1: + attention_output = attention_heads[0] + else: + # In the case where we have other sequences, we just concatenate + # them to the self-attention head before the projection. + attention_output = tf.concat(attention_heads, axis=-1) + + # Run a linear projection of `hidden_size` then add a residual + # with `layer_input`. + with tf.variable_scope("output"): + attention_output = tf.layers.dense( + attention_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range)) + attention_output = dropout(attention_output, hidden_dropout_prob) + attention_output = layer_norm(attention_output + layer_input) + + # The activation is only applied to the "intermediate" hidden layer. + with tf.variable_scope("intermediate"): + intermediate_output = tf.layers.dense( + attention_output, + intermediate_size, + activation=intermediate_act_fn, + kernel_initializer=create_initializer(initializer_range)) + + # Down-project back to `hidden_size` then add the residual. + with tf.variable_scope("output"): + layer_output = tf.layers.dense( + intermediate_output, + hidden_size, + kernel_initializer=create_initializer(initializer_range)) + layer_output = dropout(layer_output, hidden_dropout_prob) + layer_output = layer_norm(layer_output + attention_output) + prev_output = layer_output + all_layer_outputs.append(layer_output) + + if do_return_all_layers: + final_outputs = [] + for layer_output in all_layer_outputs: + final_output = reshape_from_matrix(layer_output, input_shape) + final_outputs.append(final_output) + return final_outputs + else: + final_output = reshape_from_matrix(prev_output, input_shape) + return final_output + + +def get_shape_list(tensor, expected_rank=None, name=None): + """Returns a list of the shape of tensor, preferring static dimensions. + + Args: + tensor: A tf.Tensor object to find the shape of. + expected_rank: (optional) int. The expected rank of `tensor`. If this is + specified and the `tensor` has a different rank, and exception will be + thrown. + name: Optional name of the tensor for the error message. + + Returns: + A list of dimensions of the shape of tensor. All static dimensions will + be returned as python integers, and dynamic dimensions will be returned + as tf.Tensor scalars. + """ + if name is None: + name = tensor.name + + if expected_rank is not None: + assert_rank(tensor, expected_rank, name) + + shape = tensor.shape.as_list() + + non_static_indexes = [] + for (index, dim) in enumerate(shape): + if dim is None: + non_static_indexes.append(index) + + if not non_static_indexes: + return shape + + dyn_shape = tf.shape(tensor) + for index in non_static_indexes: + shape[index] = dyn_shape[index] + return shape + + +def reshape_to_matrix(input_tensor): + """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" + ndims = input_tensor.shape.ndims + if ndims < 2: + raise ValueError("Input tensor must have at least rank 2. Shape = %s" % + (input_tensor.shape)) + if ndims == 2: + return input_tensor + + width = input_tensor.shape[-1] + output_tensor = tf.reshape(input_tensor, [-1, width]) + return output_tensor + + +def reshape_from_matrix(output_tensor, orig_shape_list): + """Reshapes a rank 2 tensor back to its original rank >= 2 tensor.""" + if len(orig_shape_list) == 2: + return output_tensor + + output_shape = get_shape_list(output_tensor) + + orig_dims = orig_shape_list[0:-1] + width = output_shape[-1] + + return tf.reshape(output_tensor, orig_dims + [width]) + + +def assert_rank(tensor, expected_rank, name=None): + """Raises an exception if the tensor rank is not of the expected rank. + + Args: + tensor: A tf.Tensor to check the rank of. + expected_rank: Python integer or list of integers, expected rank. + name: Optional name of the tensor for the error message. + + Raises: + ValueError: If the expected shape doesn't match the actual shape. + """ + if name is None: + name = tensor.name + + expected_rank_dict = {} + if isinstance(expected_rank, six.integer_types): + expected_rank_dict[expected_rank] = True + else: + for x in expected_rank: + expected_rank_dict[x] = True + + actual_rank = tensor.shape.ndims + if actual_rank not in expected_rank_dict: + scope_name = tf.get_variable_scope().name + raise ValueError( + "For the tensor `%s` in scope `%s`, the actual rank " + "`%d` (shape = %s) is not equal to the expected rank `%s`" % + (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) diff --git a/modeling_test.py b/modeling_test.py new file mode 100755 index 0000000..bed462f --- /dev/null +++ b/modeling_test.py @@ -0,0 +1,275 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import json +import random +import re + +import modeling +import six +import tensorflow as tf + + +class BertModelTest(tf.test.TestCase): + class BertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + scope=None): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.scope = scope + + def create_model(self): + input_ids = BertModelTest.ids_tensor([self.batch_size, self.seq_length], + self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = BertModelTest.ids_tensor( + [self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = BertModelTest.ids_tensor( + [self.batch_size, self.seq_length], self.type_vocab_size) + + config = modeling.BertConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + model = modeling.BertModel( + config=config, + is_training=self.is_training, + input_ids=input_ids, + input_mask=input_mask, + token_type_ids=token_type_ids, + scope=self.scope) + + outputs = { + "embedding_output": model.get_embedding_output(), + "sequence_output": model.get_sequence_output(), + "pooled_output": model.get_pooled_output(), + "all_encoder_layers": model.get_all_encoder_layers(), + } + return outputs + + def check_output(self, result): + self.parent.assertAllEqual( + result["embedding_output"].shape, + [self.batch_size, self.seq_length, self.hidden_size]) + + self.parent.assertAllEqual( + result["sequence_output"].shape, + [self.batch_size, self.seq_length, self.hidden_size]) + + self.parent.assertAllEqual(result["pooled_output"].shape, + [self.batch_size, self.hidden_size]) + + def test_default(self): + self.run_tester(BertModelTest.BertModelTester(self)) + + def test_config_to_json_string(self): + config = modeling.BertConfig(vocab_size=99, hidden_size=37) + obj = json.loads(config.to_json_string()) + self.assertEqual(obj["vocab_size"], 99) + self.assertEqual(obj["hidden_size"], 37) + + def run_tester(self, tester): + with self.test_session() as sess: + ops = tester.create_model() + init_op = tf.group(tf.global_variables_initializer(), + tf.local_variables_initializer()) + sess.run(init_op) + output_result = sess.run(ops) + tester.check_output(output_result) + + self.assert_all_tensors_reachable(sess, [init_op, ops]) + + @classmethod + def ids_tensor(cls, shape, vocab_size, rng=None, name=None): + """Creates a random int32 tensor of the shape within the vocab size.""" + if rng is None: + rng = random.Random() + + total_dims = 1 + for dim in shape: + total_dims *= dim + + values = [] + for _ in range(total_dims): + values.append(rng.randint(0, vocab_size - 1)) + + return tf.constant(value=values, dtype=tf.int32, shape=shape, name=name) + + def assert_all_tensors_reachable(self, sess, outputs): + """Checks that all the tensors in the graph are reachable from outputs.""" + graph = sess.graph + + ignore_strings = [ + "^.*/dilation_rate$", + "^.*/Tensordot/concat$", + "^.*/Tensordot/concat/axis$", + "^testing/.*$", + ] + + ignore_regexes = [re.compile(x) for x in ignore_strings] + + unreachable = self.get_unreachable_ops(graph, outputs) + filtered_unreachable = [] + for x in unreachable: + do_ignore = False + for r in ignore_regexes: + m = r.match(x.name) + if m is not None: + do_ignore = True + if do_ignore: + continue + filtered_unreachable.append(x) + unreachable = filtered_unreachable + + self.assertEqual( + len(unreachable), 0, "The following ops are unreachable: %s" % + (" ".join([x.name for x in unreachable]))) + + @classmethod + def get_unreachable_ops(cls, graph, outputs): + """Finds all of the tensors in graph that are unreachable from outputs.""" + outputs = cls.flatten_recursive(outputs) + output_to_op = collections.defaultdict(list) + op_to_all = collections.defaultdict(list) + assign_out_to_in = collections.defaultdict(list) + + for op in graph.get_operations(): + for x in op.inputs: + op_to_all[op.name].append(x.name) + for y in op.outputs: + output_to_op[y.name].append(op.name) + op_to_all[op.name].append(y.name) + if str(op.type) == "Assign": + for y in op.outputs: + for x in op.inputs: + assign_out_to_in[y.name].append(x.name) + + assign_groups = collections.defaultdict(list) + for out_name in assign_out_to_in.keys(): + name_group = assign_out_to_in[out_name] + for n1 in name_group: + assign_groups[n1].append(out_name) + for n2 in name_group: + if n1 != n2: + assign_groups[n1].append(n2) + + seen_tensors = {} + stack = [x.name for x in outputs] + while stack: + name = stack.pop() + if name in seen_tensors: + continue + seen_tensors[name] = True + + if name in output_to_op: + for op_name in output_to_op[name]: + if op_name in op_to_all: + for input_name in op_to_all[op_name]: + if input_name not in stack: + stack.append(input_name) + + expanded_names = [] + if name in assign_groups: + for assign_name in assign_groups[name]: + expanded_names.append(assign_name) + + for expanded_name in expanded_names: + if expanded_name not in stack: + stack.append(expanded_name) + + unreachable_ops = [] + for op in graph.get_operations(): + is_unreachable = False + all_names = [x.name for x in op.inputs] + [x.name for x in op.outputs] + for name in all_names: + if name not in seen_tensors: + is_unreachable = True + if is_unreachable: + unreachable_ops.append(op) + return unreachable_ops + + @classmethod + def flatten_recursive(cls, item): + """Flattens (potentially nested) a tuple/dictionary/list to a list.""" + output = [] + if isinstance(item, list): + output.extend(item) + elif isinstance(item, tuple): + output.extend(list(item)) + elif isinstance(item, dict): + for (_, v) in six.iteritems(item): + output.append(v) + else: + return [item] + + flat_output = [] + for x in output: + flat_output.extend(cls.flatten_recursive(x)) + return flat_output + + +if __name__ == "__main__": + tf.test.main() diff --git a/optimization.py b/optimization.py new file mode 100755 index 0000000..72dcd76 --- /dev/null +++ b/optimization.py @@ -0,0 +1,171 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions and classes related to optimization (weight updates).""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re +import tensorflow as tf + + +def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): + """Creates an optimizer training op.""" + global_step = tf.train.get_or_create_global_step() + + learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) + + # Implements linear decay of the learning rate. + learning_rate = tf.train.polynomial_decay( + learning_rate, + global_step, + num_train_steps, + end_learning_rate=0.0, + power=1.0, + cycle=False) + + # Implements linear warmup. I.e., if global_step < num_warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + if num_warmup_steps: + global_steps_int = tf.cast(global_step, tf.int32) + warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) + + global_steps_float = tf.cast(global_steps_int, tf.float32) + warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) + + warmup_percent_done = global_steps_float / warmup_steps_float + warmup_learning_rate = init_lr * warmup_percent_done + + is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) + learning_rate = ( + (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) + + # It is recommended that you use this optimizer for fine tuning, since this + # is how the model was trained (note that the Adam m/v variables are NOT + # loaded from init_checkpoint.) + optimizer = AdamWeightDecayOptimizer( + learning_rate=learning_rate, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) + + if use_tpu: + optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) + + tvars = tf.trainable_variables() + grads = tf.gradients(loss, tvars) + + # This is how the model was pre-trained. + (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) + + train_op = optimizer.apply_gradients( + zip(grads, tvars), global_step=global_step) + + new_global_step = global_step + 1 + train_op = tf.group(train_op, [global_step.assign(new_global_step)]) + return train_op + + +class AdamWeightDecayOptimizer(tf.train.Optimizer): + """A basic Adam optimizer that includes "correct" L2 weight decay.""" + + def __init__(self, + learning_rate, + weight_decay_rate=0.0, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=None, + name="AdamWeightDecayOptimizer"): + """Constructs a AdamWeightDecayOptimizer.""" + super(AdamWeightDecayOptimizer, self).__init__(False, name) + + self.learning_rate = learning_rate + self.weight_decay_rate = weight_decay_rate + self.beta_1 = beta_1 + self.beta_2 = beta_2 + self.epsilon = epsilon + self.exclude_from_weight_decay = exclude_from_weight_decay + + def apply_gradients(self, grads_and_vars, global_step=None, name=None): + """See base class.""" + assignments = [] + for (grad, param) in grads_and_vars: + if grad is None or param is None: + continue + + param_name = self._get_variable_name(param.name) + + m = tf.get_variable( + name=param_name + "/adam_m", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + v = tf.get_variable( + name=param_name + "/adam_v", + shape=param.shape.as_list(), + dtype=tf.float32, + trainable=False, + initializer=tf.zeros_initializer()) + + # Standard Adam update. + next_m = ( + tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) + next_v = ( + tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, + tf.square(grad))) + + update = next_m / (tf.sqrt(next_v) + self.epsilon) + + # Just adding the square of the weights to the loss function is *not* + # the correct way of using L2 regularization/weight decay with Adam, + # since that will interact with the m and v parameters in strange ways. + # + # Instead we want ot decay the weights in a manner that doesn't interact + # with the m/v parameters. This is equivalent to adding the square + # of the weights to the loss with plain (non-momentum) SGD. + if self._do_use_weight_decay(param_name): + update += self.weight_decay_rate * param + + update_with_lr = self.learning_rate * update + + next_param = param - update_with_lr + + assignments.extend( + [param.assign(next_param), + m.assign(next_m), + v.assign(next_v)]) + return tf.group(*assignments, name=name) + + def _do_use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if not self.weight_decay_rate: + return False + if self.exclude_from_weight_decay: + for r in self.exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True + + def _get_variable_name(self, param_name): + """Get the variable name from the tensor name.""" + m = re.match("^(.*):\\d+$", param_name) + if m is not None: + param_name = m.group(1) + return param_name diff --git a/optimization_test.py b/optimization_test.py new file mode 100755 index 0000000..266ecec --- /dev/null +++ b/optimization_test.py @@ -0,0 +1,48 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import optimization +import tensorflow as tf + + +class OptimizationTest(tf.test.TestCase): + + def test_adam(self): + with self.test_session() as sess: + w = tf.get_variable( + "w", + shape=[3], + initializer=tf.constant_initializer([0.1, -0.2, -0.1])) + x = tf.constant([0.4, 0.2, -0.5]) + loss = tf.reduce_mean(tf.square(x - w)) + tvars = tf.trainable_variables() + grads = tf.gradients(loss, tvars) + global_step = tf.train.get_or_create_global_step() + optimizer = optimization.AdamWeightDecayOptimizer(learning_rate=0.2) + train_op = optimizer.apply_gradients(zip(grads, tvars), global_step) + init_op = tf.group(tf.global_variables_initializer(), + tf.local_variables_initializer()) + sess.run(init_op) + for _ in range(100): + sess.run(train_op) + w_np = sess.run(w) + self.assertAllClose(w_np.flat, [0.4, 0.2, -0.5], rtol=1e-2, atol=1e-2) + + +if __name__ == "__main__": + tf.test.main() diff --git a/run_classifier.py b/run_classifier.py new file mode 100644 index 0000000..67a1fe3 --- /dev/null +++ b/run_classifier.py @@ -0,0 +1,928 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# %% +# Version 2 added predict function + +# %% +"""BERT finetuning runner.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import csv +import os +import modeling +import optimization +import tokenization +import tensorflow as tf +import platform +import numpy as np +from sklearn.metrics import classification_report, confusion_matrix +import pickle + +flags = tf.flags + +FLAGS = flags.FLAGS + +# %% +## Required parameters +flags.DEFINE_string( + "data_dir", None, + "The input data dir. Should contain the .tsv files (or other data files) " + "for the task.") + +flags.DEFINE_string( + "bert_config_file", None + "bert_config.json", + "The config json file corresponding to the pre-trained BERT model. " + "This specifies the model architecture.") + +flags.DEFINE_string("task_name", "MRPC", "The name of the task to train.") + +flags.DEFINE_string("vocab_file", None + "vocab.txt", + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_string( + "output_dir", None, + "The output directory where the model checkpoints will be written.") + +## Other parameters + +flags.DEFINE_string( + "init_checkpoint", None + "bert_model.ckpt", + "Initial checkpoint (usually from a pre-trained BERT model).") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sequence length after WordPiece tokenization. " + "Sequences longer than this will be truncated, and sequences shorter " + "than this will be padded.") + +flags.DEFINE_bool("do_train", True, "Whether to run training.") + +flags.DEFINE_bool("do_eval", True, "Whether to run eval on the dev set.") + +flags.DEFINE_bool( + "do_predict", False, + "Whether to run the model in inference mode on the test set.") + +flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.") + +flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.") + +flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predict.") + +flags.DEFINE_float("learning_rate", 2e-5, "The initial learning rate for Adam.") + +flags.DEFINE_float("num_train_epochs", 10.0, + "Total number of training epochs to perform.") + +flags.DEFINE_float( + "warmup_proportion", 0.1, + "Proportion of training to perform linear learning rate warmup for. " + "E.g., 0.1 = 10% of training.") + +flags.DEFINE_integer("save_checkpoints_steps", 50, + "How often to save the model checkpoint.") + +flags.DEFINE_integer("iterations_per_loop", 50, + "How many steps to make in each estimator call.") + +flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") + +tf.flags.DEFINE_string( + "tpu_name", None, + "The Cloud TPU to use for training. This should be either the name " + "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " + "url.") + +tf.flags.DEFINE_string( + "tpu_zone", None, + "[Optional] GCE zone where the Cloud TPU is located in. If not " + "specified, we will attempt to automatically detect the GCE project from " + "metadata.") + +tf.flags.DEFINE_string( + "gcp_project", None, + "[Optional] Project name for the Cloud TPU-enabled project. If not " + "specified, we will attempt to automatically detect the GCE project from " + "metadata.") + +tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") + +flags.DEFINE_integer( + "num_tpu_cores", 8, + "Only used if `use_tpu` is True. Total number of TPU cores to use.") + + +class InputExample(object): + """A single training/test example for simple sequence classification.""" + + def __init__(self, guid, text_a, text_b=None, label=None): + """Constructs a InputExample. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, input_ids, input_mask, segment_ids, label_id): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + # added base class function for get_results + def get_results(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with tf.gfile.Open(input_file, "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), + "dev_matched") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test_matched.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, tokenization.convert_to_unicode(line[0])) + text_a = tokenization.convert_to_unicode(line[8]) + text_b = tokenization.convert_to_unicode(line[9]) + if set_type == "test": + label = "contradiction" + else: + label = tokenization.convert_to_unicode(line[-1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MrpcProcessor(DataProcessor): + """Processor for the MRPC data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "test") + + def get_labels(self, labels): + """See base class.""" + return set(labels) + + # new function get the result tsv + def get_results(self, data_dir): + """See base class.""" + return self._read_tsv(os.path.join(data_dir, "test_results.tsv")) + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + labels = [] + labels_test = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + + # tokenization is based on vocab file + text_a = tokenization.convert_to_unicode(line[0]) + label = tokenization.convert_to_unicode(line[1]) + labels.append(label) + + if set_type == "test": + label = "0" + labels_test.append(label) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + + return examples, labels, labels_test + + +class ColaProcessor(DataProcessor): + """Processor for the CoLA data set (GLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + # Only the test set has a header + if set_type == "test" and i == 0: + continue + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = tokenization.convert_to_unicode(line[1]) + label = "0" + else: + text_a = tokenization.convert_to_unicode(line[3]) + label = tokenization.convert_to_unicode(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +def convert_single_example(ex_index, example, label_list, max_seq_length, + tokenizer): + """Converts a single `InputExample` into a single `InputFeatures`.""" + label_map = {} + for (i, label) in enumerate(label_list): + label_map[label] = i + + tokens_a = tokenizer.tokenize(example.text_a) + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + # The convention in BERT is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in tokens_a: + tokens.append(token) + segment_ids.append(0) + tokens.append("[SEP]") + segment_ids.append(0) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + label_id = label_map[example.label] + if ex_index < 5: + tf.logging.info("*** Example ***") + tf.logging.info("guid: %s" % (example.guid)) + tf.logging.info("tokens: %s" % " ".join( + [tokenization.printable_text(x) for x in tokens])) + tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + tf.logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + tf.logging.info("label: %s (id = %d)" % (example.label, label_id)) + + feature = InputFeatures( + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id) + return feature, label_map + + +def file_based_convert_examples_to_features( + examples, label_list, max_seq_length, tokenizer, output_file): + """Convert a set of `InputExample`s to a TFRecord file.""" + + writer = tf.python_io.TFRecordWriter(output_file) + + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + tf.logging.info("Writing example %d of %d" % (ex_index, len(examples))) + + feature, label_map = convert_single_example(ex_index, example, label_list, + max_seq_length, tokenizer) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + features["label_ids"] = create_int_feature([feature.label_id]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + return label_map + + +def file_based_input_fn_builder(input_file, seq_length, is_training, + drop_remainder): + """Creates an `input_fn` closure to be passed to TPUEstimator.""" + + name_to_features = { + "input_ids": tf.FixedLenFeature([seq_length], tf.int64), + "input_mask": tf.FixedLenFeature([seq_length], tf.int64), + "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), + "label_ids": tf.FixedLenFeature([], tf.int64), + } + + def _decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + example = tf.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.to_int32(t) + example[name] = t + + return example + + def input_fn(params): + """The actual input function.""" + batch_size = params["batch_size"] + + # For training, we want a lot of parallel reading and shuffling. + # For eval, we want no shuffling and parallel reading doesn't matter. + d = tf.data.TFRecordDataset(input_file) + if is_training: + d = d.repeat() + d = d.shuffle(buffer_size=100) + + d = d.apply( + tf.contrib.data.map_and_batch( + lambda record: _decode_record(record, name_to_features), + batch_size=batch_size, + drop_remainder=drop_remainder)) + + return d + + return input_fn + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, + labels, num_labels, use_one_hot_embeddings): + """Creates a classification model.""" + model = modeling.BertModel( + config=bert_config, + is_training=is_training, + input_ids=input_ids, + input_mask=input_mask, + token_type_ids=segment_ids, + use_one_hot_embeddings=use_one_hot_embeddings) + + # %% Added layers for specific tasks + + # In the demo, we are doing a simple classification task on the entire + # segment. + # + # If you want to use the token-level output, use model.get_sequence_output() + # instead. + + # 768 hidden size + # 128 seq length + # 32 batch size + output_layer = model.get_pooled_output() + + hidden_size = output_layer.shape[-1].value + + output_weights = tf.get_variable( + "output_weights", [num_labels, hidden_size], + initializer=tf.truncated_normal_initializer(stddev=0.02)) + + output_bias = tf.get_variable( + "output_bias", [num_labels], initializer=tf.zeros_initializer()) + + with tf.variable_scope("loss"): + if is_training: + # I.e., 0.1 dropout + output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) + + logits = tf.matmul(output_layer, output_weights, transpose_b=True) + logits = tf.nn.bias_add(logits, output_bias) + probabilities = tf.nn.softmax(logits, axis=-1) + log_probs = tf.nn.log_softmax(logits, axis=-1) + + # correct_predictions = tf.equal(log_probs, labels) + # accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy") + + # size becomes binary 32*2 + # labels is the actual label value + one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) + + per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) + loss = tf.reduce_mean(per_example_loss) + + return (loss, per_example_loss, logits, probabilities) + + +def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate, + num_train_steps, num_warmup_steps, use_tpu, + use_one_hot_embeddings): + """Returns `model_fn` closure for TPUEstimator.""" + + def model_fn(features, labels, mode, params): # pylint: disable=unused-argument + """The `model_fn` for TPUEstimator.""" + + tf.logging.info("*** Features ***") + for name in sorted(features.keys()): + tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) + + input_ids = features["input_ids"] + input_mask = features["input_mask"] + segment_ids = features["segment_ids"] + label_ids = features["label_ids"] + + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + + (total_loss, per_example_loss, logits, probabilities) = create_model( + bert_config, is_training, input_ids, input_mask, segment_ids, label_ids, + num_labels, use_one_hot_embeddings) + + tvars = tf.trainable_variables() + initialized_variable_names = {} + scaffold_fn = None + if init_checkpoint: + (assignment_map, initialized_variable_names + ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) + if use_tpu: + + def tpu_scaffold(): + tf.train.init_from_checkpoint(init_checkpoint, assignment_map) + return tf.train.Scaffold() + + scaffold_fn = tpu_scaffold + else: + tf.train.init_from_checkpoint(init_checkpoint, assignment_map) + + tf.logging.info("**** Trainable Variables ****") + for var in tvars: + init_string = "" + if var.name in initialized_variable_names: + init_string = ", *INIT_FROM_CKPT*" + tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, + init_string) + + output_spec = None + if mode == tf.estimator.ModeKeys.TRAIN: + predictions = tf.argmax(logits, axis=-1, output_type=tf.int32) + + accuracy = tf.metrics.accuracy(label_ids, predictions) + tf.summary.scalar('accuracy', accuracy[1]) + + train_op = optimization.create_optimizer( + total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu) + + output_spec = tf.contrib.tpu.TPUEstimatorSpec( + mode=mode, + loss=total_loss, + train_op=train_op, + scaffold_fn=scaffold_fn) + + elif mode == tf.estimator.ModeKeys.EVAL: + + def metric_fn(per_example_loss, label_ids, logits): + predictions = tf.argmax(logits, axis=-1, output_type=tf.int32) + + accuracy = tf.metrics.accuracy(label_ids, predictions) + loss = tf.metrics.mean(per_example_loss) + return { + "eval_accuracy": accuracy, + "eval_loss": loss, + } + + eval_metrics = (metric_fn, [per_example_loss, label_ids, logits]) + output_spec = tf.contrib.tpu.TPUEstimatorSpec( + mode=mode, + loss=total_loss, + eval_metrics=eval_metrics, + scaffold_fn=scaffold_fn) + else: + output_spec = tf.contrib.tpu.TPUEstimatorSpec( + mode=mode, predictions=probabilities, scaffold_fn=scaffold_fn) + return output_spec + + return model_fn + + +# This function is not used by this file but is still used by the Colab and +# people who depend on it. +def input_fn_builder(features, seq_length, is_training, drop_remainder): + """Creates an `input_fn` closure to be passed to TPUEstimator.""" + + all_input_ids = [] + all_input_mask = [] + all_segment_ids = [] + all_label_ids = [] + + # each sample has a corresponding feature dict + # contains input id: token ids + # input mask: whether or not value is zero padding + # segment id: mask showing first part or second part of sequence + # label value + # just unpacks different types to a combined list + for feature in features: + all_input_ids.append(feature.input_ids) + all_input_mask.append(feature.input_mask) + all_segment_ids.append(feature.segment_ids) + all_label_ids.append(feature.label_id) + + def input_fn(params): + """The actual input function.""" + batch_size = params["batch_size"] + + num_examples = len(features) + + # This is for demo purposes and does NOT scale to large data sets. We do + # not use Dataset.from_generator() because that uses tf.py_func which is + # not TPU compatible. The right way to load data is with TFRecordReader. + d = tf.data.Dataset.from_tensor_slices({ + "input_ids": + tf.constant( + all_input_ids, shape=[num_examples, seq_length], + dtype=tf.int32), + "input_mask": + tf.constant( + all_input_mask, + shape=[num_examples, seq_length], + dtype=tf.int32), + "segment_ids": + tf.constant( + all_segment_ids, + shape=[num_examples, seq_length], + dtype=tf.int32), + "label_ids": + tf.constant(all_label_ids, shape=[num_examples], dtype=tf.int32), + }) + + if is_training: + d = d.repeat() + d = d.shuffle(buffer_size=100) + + d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder) + return d + + return input_fn + + +# This function is not used by this file but is still used by the Colab and +# people who depend on it. +def convert_examples_to_features(examples, label_list, max_seq_length, + tokenizer): + """Convert a set of `InputExample`s to a list of `InputFeatures`.""" + + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + tf.logging.info("Writing example %d of %d" % (ex_index, len(examples))) + + feature = convert_single_example(ex_index, example, label_list, + max_seq_length, tokenizer) + + features.append(feature) + return features + + +def main(_): + tf.logging.set_verbosity(tf.logging.INFO) + + processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mrpc": MrpcProcessor, + } + + if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict: + raise ValueError( + "At least one of `do_train`, `do_eval` or `do_predict' must be True.") + + bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) + + if FLAGS.max_seq_length > bert_config.max_position_embeddings: + raise ValueError( + "Cannot use sequence length %d because the BERT model " + "was only trained up to sequence length %d" % + (FLAGS.max_seq_length, bert_config.max_position_embeddings)) + + tf.gfile.MakeDirs(FLAGS.output_dir) + + task_name = FLAGS.task_name.lower() + + if task_name not in processors: + raise ValueError("Task not found: %s" % (task_name)) + + processor = processors[task_name]() + + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + + tpu_cluster_resolver = None + if FLAGS.use_tpu and FLAGS.tpu_name: + tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( + FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) + + is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 + run_config = tf.contrib.tpu.RunConfig( + cluster=tpu_cluster_resolver, + master=FLAGS.master, + model_dir=FLAGS.output_dir, + save_checkpoints_steps=FLAGS.save_checkpoints_steps, + tpu_config=tf.contrib.tpu.TPUConfig( + iterations_per_loop=FLAGS.iterations_per_loop, + num_shards=FLAGS.num_tpu_cores, + per_host_input_for_training=is_per_host)) + + train_examples = None + num_train_steps = None + num_warmup_steps = None + + # moved outside of the for loop + train_examples, train_labels, temp = processor.get_train_examples(FLAGS.data_dir) + + if FLAGS.do_train: + num_train_steps = int( + len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) + num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) + + label_list = processor.get_labels(train_labels) + + model_fn = model_fn_builder( + bert_config=bert_config, + + # value is 2 + num_labels=len(label_list), + init_checkpoint=FLAGS.init_checkpoint, + learning_rate=FLAGS.learning_rate, + num_train_steps=num_train_steps, + num_warmup_steps=num_warmup_steps, + use_tpu=FLAGS.use_tpu, + use_one_hot_embeddings=FLAGS.use_tpu) + + # If TPU is not available, this will fall back to normal Estimator on CPU + # or GPU. + estimator = tf.contrib.tpu.TPUEstimator( + use_tpu=FLAGS.use_tpu, + model_fn=model_fn, + config=run_config, + train_batch_size=FLAGS.train_batch_size, + eval_batch_size=FLAGS.eval_batch_size, + predict_batch_size=FLAGS.predict_batch_size) + + if FLAGS.do_train: + train_file = os.path.join(FLAGS.output_dir, "train.tf_record") + + # example to features and then serialise to string and put into file + label_map_train = file_based_convert_examples_to_features( + train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) + + tf.logging.info("***** Running training *****") + tf.logging.info(" Num examples = %d", len(train_examples)) + tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) + tf.logging.info(" Num steps = %d", num_train_steps) + train_input_fn = file_based_input_fn_builder( + input_file=train_file, + seq_length=FLAGS.max_seq_length, + is_training=True, + drop_remainder=True) + + # Start training + estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) + + if FLAGS.do_eval: + eval_examples, eval_labels, temp = processor.get_dev_examples(FLAGS.data_dir) + eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") + label_map_dev = file_based_convert_examples_to_features( + eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) + + tf.logging.info("***** Running evaluation *****") + tf.logging.info(" Num examples = %d", len(eval_examples)) + tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) + + # This tells the estimator to run through the entire set. + eval_steps = None + # However, if running eval on the TPU, you will need to specify the + # number of steps. + if FLAGS.use_tpu: + # Eval will be slightly WRONG on the TPU because it will truncate + # the last batch. + eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size) + + eval_drop_remainder = True if FLAGS.use_tpu else False + eval_input_fn = file_based_input_fn_builder( + input_file=eval_file, + seq_length=FLAGS.max_seq_length, + is_training=False, + drop_remainder=eval_drop_remainder) + + result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) + + output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") + with tf.gfile.GFile(output_eval_file, "w") as writer: + tf.logging.info("***** Eval results *****") + for key in sorted(result.keys()): + tf.logging.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + if FLAGS.do_predict: + predict_examples, truth_labels, placeholder_labels = processor.get_test_examples(FLAGS.data_dir) + predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") + + label_list_test = processor.get_labels(placeholder_labels) + + # example to features and then serialise to string and put into file + label_map_test = file_based_convert_examples_to_features(predict_examples, label_list_test, + FLAGS.max_seq_length, tokenizer, + predict_file) + + tf.logging.info("***** Running prediction*****") + tf.logging.info(" Num examples = %d", len(predict_examples)) + tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) + + if FLAGS.use_tpu: + # Warning: According to tpu_estimator.py Prediction on TPU is an + # experimental feature and hence not supported here + raise ValueError("Prediction in TPU not supported") + + predict_drop_remainder = True if FLAGS.use_tpu else False + predict_input_fn = file_based_input_fn_builder( + input_file=predict_file, + seq_length=FLAGS.max_seq_length, + is_training=False, + drop_remainder=predict_drop_remainder) + + result = estimator.predict(input_fn=predict_input_fn) + + output_predict_file = os.path.join(FLAGS.output_dir, "test_results.tsv") + with tf.gfile.GFile(output_predict_file, "w") as writer: + tf.logging.info("***** Predict results *****") + for prediction in result: + output_line = "\t".join( + str(class_probability) for class_probability in prediction) + "\n" + writer.write(output_line) + + +if __name__ == "__main__": + flags.mark_flag_as_required("data_dir") + flags.mark_flag_as_required("task_name") + flags.mark_flag_as_required("vocab_file") + flags.mark_flag_as_required("bert_config_file") + flags.mark_flag_as_required("output_dir") + tf.app.run() diff --git a/tokenization.py b/tokenization.py new file mode 100755 index 0000000..a24ba8d --- /dev/null +++ b/tokenization.py @@ -0,0 +1,292 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import unicodedata +import six +import tensorflow as tf + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with tf.gfile.GFile(vocab_file, "r") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_tokens_to_ids(vocab, tokens): + """Converts a sequence of tokens into ids using the vocab.""" + ids = [] + for token in tokens: + ids.append(vocab[token]) + return ids + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a peice of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_tokens_to_ids(self.vocab, tokens) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/tokenization_test.py b/tokenization_test.py new file mode 100755 index 0000000..9a6e769 --- /dev/null +++ b/tokenization_test.py @@ -0,0 +1,125 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import tempfile + +import tokenization +import tensorflow as tf + + +class TokenizationTest(tf.test.TestCase): + + def test_full_tokenizer(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing", "," + ] + with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + + vocab_file = vocab_writer.name + + tokenizer = tokenization.FullTokenizer(vocab_file) + os.unlink(vocab_file) + + tokens = tokenizer.tokenize(u"UNwant\u00E9d,running") + self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) + + self.assertAllEqual( + tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) + + def test_basic_tokenizer_lower(self): + tokenizer = tokenization.BasicTokenizer(do_lower_case=True) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["hello", "!", "how", "are", "you", "?"]) + self.assertAllEqual(tokenizer.tokenize(u"H\u00E9llo"), ["hello"]) + + def test_basic_tokenizer_no_lower(self): + tokenizer = tokenization.BasicTokenizer(do_lower_case=False) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["HeLLo", "!", "how", "Are", "yoU", "?"]) + + def test_wordpiece_tokenizer(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing" + ] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = tokenization.WordpieceTokenizer(vocab=vocab) + + self.assertAllEqual(tokenizer.tokenize(""), []) + + self.assertAllEqual( + tokenizer.tokenize("unwanted running"), + ["un", "##want", "##ed", "runn", "##ing"]) + + self.assertAllEqual( + tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) + + def test_convert_tokens_to_ids(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing" + ] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + + self.assertAllEqual( + tokenization.convert_tokens_to_ids( + vocab, ["un", "##want", "##ed", "runn", "##ing"]), [7, 4, 5, 8, 9]) + + def test_is_whitespace(self): + self.assertTrue(tokenization._is_whitespace(u" ")) + self.assertTrue(tokenization._is_whitespace(u"\t")) + self.assertTrue(tokenization._is_whitespace(u"\r")) + self.assertTrue(tokenization._is_whitespace(u"\n")) + self.assertTrue(tokenization._is_whitespace(u"\u00A0")) + + self.assertFalse(tokenization._is_whitespace(u"A")) + self.assertFalse(tokenization._is_whitespace(u"-")) + + def test_is_control(self): + self.assertTrue(tokenization._is_control(u"\u0005")) + + self.assertFalse(tokenization._is_control(u"A")) + self.assertFalse(tokenization._is_control(u" ")) + self.assertFalse(tokenization._is_control(u"\t")) + self.assertFalse(tokenization._is_control(u"\r")) + + def test_is_punctuation(self): + self.assertTrue(tokenization._is_punctuation(u"-")) + self.assertTrue(tokenization._is_punctuation(u"$")) + self.assertTrue(tokenization._is_punctuation(u"`")) + self.assertTrue(tokenization._is_punctuation(u".")) + + self.assertFalse(tokenization._is_punctuation(u"A")) + self.assertFalse(tokenization._is_punctuation(u" ")) + + +if __name__ == "__main__": + tf.test.main()