From 543466b64e6460b2781177a9ec44ca6f04a93bed Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 14:39:37 -0700 Subject: [PATCH 01/67] Added initial code of Conformer. Signed-off-by: Vahid --- .../configs/bpe_vocabs/1k/vocab.txt | 1024 +++ .../configs/bpe_vocabs/2k/vocab.txt | 2048 +++++ .../configs/bpe_vocabs/4k/vocab.txt | 4096 +++++++++ .../configs/bpe_vocabs/8k/vocab.txt | 8192 +++++++++++++++++ .../configs/conformer/conformer_bpe.yaml | 166 + .../configs/conformer/conformer_char.yaml | 163 + examples/asr/speech_to_text.py | 11 + examples/asr/speech_to_text_bpe.py | 5 + nemo/collections/asr/modules/__init__.py | 16 + nemo/collections/asr/modules/activations.py | 27 + .../asr/modules/conformer_encoder.py | 337 + .../asr/modules/conformer_modules.py | 86 + nemo/collections/asr/modules/lstm_decoder.py | 98 + .../asr/modules/multi_head_attention.py | 483 + nemo/collections/asr/modules/subsampling.py | 142 + 15 files changed, 16894 insertions(+) create mode 100644 examples/asr/experimental/configs/bpe_vocabs/1k/vocab.txt create mode 100644 examples/asr/experimental/configs/bpe_vocabs/2k/vocab.txt create mode 100644 examples/asr/experimental/configs/bpe_vocabs/4k/vocab.txt create mode 100644 examples/asr/experimental/configs/bpe_vocabs/8k/vocab.txt create mode 100644 examples/asr/experimental/configs/conformer/conformer_bpe.yaml create mode 100644 examples/asr/experimental/configs/conformer/conformer_char.yaml create mode 100644 nemo/collections/asr/modules/activations.py create mode 100644 nemo/collections/asr/modules/conformer_encoder.py create mode 100644 nemo/collections/asr/modules/conformer_modules.py create mode 100644 nemo/collections/asr/modules/lstm_decoder.py create mode 100644 nemo/collections/asr/modules/multi_head_attention.py create mode 100644 nemo/collections/asr/modules/subsampling.py diff --git a/examples/asr/experimental/configs/bpe_vocabs/1k/vocab.txt b/examples/asr/experimental/configs/bpe_vocabs/1k/vocab.txt new file mode 100644 index 000000000000..e0e2d5c408f1 --- /dev/null +++ b/examples/asr/experimental/configs/bpe_vocabs/1k/vocab.txt @@ -0,0 +1,1024 @@ +[PAD] +[UNK] +[CLS] +[SEP] +[MASK] +' +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +##o +##l +##i +##a +##n +##t +##r +##e +##u +##s +##y +##c +##b +##g +##m +##d +##w +##v +##k +##h +##p +##f +##x +##q +##j +##z +th +the +##er +##nd +##in +##ed +##ou +##at +##en +and +##or +##es +to +of +##on +##is +##ing +##ar +##it +##as +##an +##ll +in +##re +wh +he +##om +be +ha +##le +##ot +##ow +##ic +##ut +it +##ld +that +sh +##ly +was +##gh +##id +##se +on +##ve +##et +##ent +you +##im +##st +##ce +##ith +for +##ir +##ion +as +##ay +his +##al +with +##ur +st +##ter +her +##ver +##ch +re +##ad +##ght +an +not +is +had +sa +at +##oo +##her +but +##ould +she +##ct +##ess +fr +se +##am +pr +we +him +so +##ill +##ain +su +##ight +me +##ere +##all +##ke +##il +de +my +they +all +##est +ch +##ich +ne +by +have +##ome +##pp +##ck +li +this +do +##ore +con +##us +which +##ri +##ul +up +said +from +##th +##ard +##ge +##ra +or +one +no +ex +were +there +##os +##pe +man +##ble +who +al +##ant +##ie +##ous +##ust +##ation +fe +##and +them +##ers +##lf +when +##ind +##ast +go +##ood +kn +##un +##our +##ist +##em +are +##out +would +##art +what +le +their +##ong +en +if +un +##if +com +##ound +ab +out +wor +##ol +will +int +br +##ak +mis +##ate +##op +##um +##ough +##la +##itt +ar +##ook +##ish +##one +##ap +been +##own +##ame +then +##ven +##self +##ive +tr +ro +##ig +fa +##reat +af +sp +qu +##ery +##ity +##ort +##ost +tw +ag +could +am +##el +##ake +any +##ure +some +your +can +more +tim +##ep +##nder +ever +##ence +##ther +##fe +into +##ry +##ose +now +##au +##ag +cl +very +##qu +##ved +bo +other +did +##ance +than +##ittle +little +##ine +##ies +##oun +look +##way +##urn +like +##ite +gr +##ear +##ach +lo +##ass +##other +##able +##ought +know +time +im +us +co +##fore +##ell +how +##hing +##cc +ad +day +##ack +about +##ade +##ice +see +over +##ci +##ide +##pt +dis +##ink +too +##wn +great +after +per +pro +again +##ment +upon +hand +has +##ree +##ked +##ather +only +car +##tain +##ally +str +mar +po +fl +well +should +##ab +##pl +say +good +##ings +two +our +thr +##ud +##ving +##oss +down +##ty +##ire +pe +before +##ves +app +##ile +every +its +pl +cr +old +off +made +##ick +##ied +long +bl +##age +##ber +where +##ress +fir +##ang +mu +never +must +sm +##ful +such +##ull +##ions +##ft +pre +came +##ro +miss +##ward +##een +##ild +##orn +##ue +even +under +these +##ord +come +part +fo +sc +##ated +rem +ind +##ness +bec +##per +may +much +think +##ious +way +mister +##ount +##led +let +ey +##ath +thought +##ect +back +spe +gl +##ady +bet +##ff +##ans +res +just +first +##ise +here +ho +own +des +##ried +ob +##ces +mo +pla +##ced +cont +##ary +##ered +went +himself +men +##me +##co +##int +might +##ily +seem +##ip +beg +##oth +##ouse +##ave +wat +comp +sw +through +##igh +away +em +get +head +don +imp +##llow +bel +##ened +without +##iz +life +most +pass +ear +make +cons +som +turn +##ac +##iv +shall +##ng +pres +acc +those +eyes +house +att +somet +jo +still +night +call +##red +op +##ew +##ens +wom +last +##ign +tell +##ause +nothing +##ark +sl +new +##ction +la +##iff +take +being +comm +many +##les +ser +word +inter +##atter +wr +##ons +ret +found +##are +##ase +while +##ory +##ix +saw +put +year +##iend +##ople +##dy +young +far +##aking +once +friend +people +##air +##ether +work +##ors +##ible +room +##ently +three +##ted +yet +##pect +##ever +count +same +father +right +child +cour +##ne +place +another +##ition +want +though +##less +##ince +king +##ian +happ +heart +end +gra +nat +face +##thing +love +took +dist +arm +##ace +inst +rep +##alk +##man +##ult +char +nor +te +ple +set +##cl +light +looked +##gg +missus +asked +mind +yes +supp +wa +ll +left +##cess +##ting +ac +##ub +##augh +##ents +##te +##ished +pers +##ways +things +sir +mom +##irl +near +##sw +##ations +sat +find +##der +##sel +##ia +world +##ower +##rew +##ject +##fect +give +cap +##av +why +mother +gu +gen +##ict +always +##so +##ling +answ +##pped +soon +##iver +##den +form +heard +war +thing +something +seemed +sub +god +door +girl +##ange +##ph +##und +wind +because +diff +mon +going +##ertain +##de +told +home +ke +##orm +got +##ither +el +##ret +##ock +sur +high +##ched +eng +gi +ma +ph +##ank +hands +moment +woman +follow +##land +dr +ve +vo +feel +##aut +##ning +poss +ev +##ical +##lt +##ple +boy +return +rest +reg +knew +each +inf +conf +oh +sil +##ner +kind +cle +##ush +hel +rel +##orr +tra +##ual +ass +wo +##ars +##cept +belie +bre +certain +against +water +##uck +exp +hard +prop +years +del +unt +quite +beh +side +##iness +##ined +appear +four +few +##ool +done +better +##ited +half +min +##ail +tre +ref +also +##cy +##oub +##vent +began +##most +##ows +ter +##ully +##act +enough +pur +##ised +dra +act +dear +having +seen +gener +lady +hum +##aster +##ared +##aken +##ash +##oke +hour +inde +##ps +##ying +cur +called +##ier +##ason +does +whole +##uth +morn +turned +pleas +##ial +gave +betw +##ense +occ +##aps +sou +course +##ib +ins +##gether +##ream +##ired +both +add +poor +col +read +bla +##ten +##selves +ste +among +between +keep +##ular +beaut +matter +care +morning +white +##ained +sure +name +##ged +toward +##ute +##owed +small +whom +par +exc +sk +pa +lar +felt +##ks +##ating +##osed +##bo +myself +open +six +es +##ru +herself +however +##xt +##ond +hund +stood +hundred +##plied +sent +inc +round +##tered +sim +##ife +##ates +##ters +days +almost +point +##eng +gre +eight +words +hor +fam +stand +land +##dden +##ung +mean +sun +##ird +##haps +##iet +perhaps +##ures +art +sle +ed +##iss +best +dark +sudden +hear +bu +replied +bar +voice +met +##ton +che +anything +till +underst +##ren +##its +until +black +##oud +looking +cried +##ned +suff +aw +##ins +prin +red +##amp +less +lay +next +law +##end +power +##vel +fact +##not +brought +##up +##ately +##enty +country +help +##als +sn +quest +wonder +use +vis +laugh +##med +##fully +together +air +unc +need +adv +wish +##ove +person +indeed +since +contin +##oney +gent +present +har +forg +full +run +rather +ra +##ok +cond +ide +sy +##eter +##rib +feet +fin +fall +##cei +five +mil +talk +##ater +bed +##ured +doct +##sh +reach +interest +lord +gone +##tle +death +consid +##fort +serv +wife +doub +adm +##ief +##pr +##ene +taken diff --git a/examples/asr/experimental/configs/bpe_vocabs/2k/vocab.txt b/examples/asr/experimental/configs/bpe_vocabs/2k/vocab.txt new file mode 100644 index 000000000000..08203cc74baa --- /dev/null +++ b/examples/asr/experimental/configs/bpe_vocabs/2k/vocab.txt @@ -0,0 +1,2048 @@ +[PAD] +[UNK] +[CLS] +[SEP] +[MASK] +' +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +##l +##a +##u +##d +##i +##n +##s +##e +##h +##y +##o +##c +##k +##r +##t +##m +##v +##b +##g +##p +##f +##w +##z +##j +##q +##x +th +the +##er +##nd +##in +##ed +##ou +##at +##en +and +##or +##es +to +of +##on +##is +##ing +##ar +##it +##as +##an +##ll +in +##re +wh +he +##om +be +ha +##le +##ot +##ow +##ic +##ut +it +##ld +that +sh +##ly +was +##gh +##id +##se +on +##ve +##et +##ent +you +##im +##st +##ce +##ith +for +##ir +##ion +as +##ay +his +##al +with +##ur +st +##ter +her +##ver +##ch +re +##ad +##ght +an +not +is +had +sa +at +##oo +##her +but +##ould +she +##ct +##ess +fr +se +##am +pr +we +him +so +##ill +##ain +su +##ight +me +##ere +##all +##ke +##il +de +my +they +all +##est +ch +##ich +ne +by +have +##ome +##pp +##ck +li +this +do +##ore +con +##us +which +##ri +##ul +up +said +from +##th +##ard +##ge +##ra +or +one +no +ex +were +there +##os +##pe +man +##ble +who +al +##ant +##ie +##ous +##ust +##ation +fe +##and +them +##ers +##lf +when +##ind +##ast +go +##ood +kn +##un +##our +##ist +##em +are +##out +would +##art +what +le +their +##ong +en +if +un +##if +com +##ound +ab +out +wor +##ol +will +int +br +##ak +mis +##ate +##op +##um +##ough +##la +##itt +ar +##ook +##ish +##one +##ap +been +##own +##ame +then +##ven +##self +##ive +tr +ro +##ig +fa +##reat +af +sp +qu +##ery +##ity +##ort +##ost +tw +ag +could +am +##el +##ake +any +##ure +some +your +can +more +tim +##ep +##nder +ever +##ence +##ther +##fe +into +##ry +##ose +now +##au +##ag +cl +very +##qu +##ved +bo +other +did +##ance +than +##ittle +little +##ine +##ies +##oun +look +##way +##urn +like +##ite +gr +##ear +##ach +lo +##ass +##other +##able +##ought +know +time +im +us +co +##fore +##ell +how +##hing +##cc +ad +day +##ack +about +##ade +##ice +see +over +##ci +##ide +##pt +dis +##ink +too +##wn +great +after +per +pro +again +##ment +upon +hand +has +##ree +##ked +##ather +only +car +##tain +##ally +str +mar +po +fl +well +should +##ab +##pl +say +good +##ings +two +our +thr +##ud +##ving +##oss +down +##ty +##ire +pe +before +##ves +app +##ile +every +its +pl +cr +old +off +made +##ick +##ied +long +bl +##age +##ber +where +##ress +fir +##ang +mu +never +must +sm +##ful +such +##ull +##ions +##ft +pre +came +##ro +miss +##ward +##een +##ild +##orn +##ue +even +under +these +##ord +come +part +fo +sc +##ated +rem +ind +##ness +bec +##per +may +much +think +##ious +way +mister +##ount +##led +let +ey +##ath +thought +##ect +back +spe +gl +##ady +bet +##ff +##ans +res +just +first +##ise +here +ho +own +des +##ried +ob +##ces +mo +pla +##ced +cont +##ary +##ered +went +himself +men +##me +##co +##int +might +##ily +seem +##ip +beg +##oth +##ouse +##ave +wat +comp +sw +through +##igh +away +em +get +head +don +imp +##llow +bel +##ened +without +##iz +life +most +pass +ear +make +cons +som +turn +##ac +##iv +shall +##ng +pres +acc +those +eyes +house +att +somet +jo +still +night +call +##red +op +##ew +##ens +wom +last +##ign +tell +##ause +nothing +##ark +sl +new +##ction +la +##iff +take +being +comm +many +##les +ser +word +inter +##atter +wr +##ons +ret +found +##are +##ase +while +##ory +##ix +saw +put +year +##iend +##ople +##dy +young +far +##aking +once +friend +people +##air +##ether +work +##ors +##ible +room +##ently +three +##ted +yet +##pect +##ever +count +same +father +right +child +cour +##ne +place +another +##ition +want +though +##less +##ince +king +##ian +happ +heart +end +gra +nat +face +##thing +love +took +dist +arm +##ace +inst +rep +##alk +##man +##ult +char +nor +te +ple +set +##cl +light +looked +##gg +missus +asked +mind +yes +supp +wa +ll +left +##cess +##ting +ac +##ub +##augh +##ents +##te +##ished +pers +##ways +things +sir +mom +##irl +near +##sw +##ations +sat +find +##der +##sel +##ia +world +##ower +##rew +##ject +##fect +give +cap +##av +why +mother +gu +gen +##ict +always +##so +##ling +answ +##pped +soon +##iver +##den +form +heard +war +thing +something +seemed +sub +god +door +girl +##ange +##ph +##und +wind +because +diff +mon +going +##ertain +##de +told +home +ke +##orm +got +##ither +el +##ret +##ock +sur +high +##ched +eng +gi +ma +ph +##ank +hands +moment +woman +follow +##land +dr +ve +vo +feel +##aut +##ning +poss +ev +##ical +##lt +##ple +boy +return +rest +reg +knew +each +inf +conf +oh +sil +##ner +kind +cle +##ush +hel +rel +##orr +tra +##ual +ass +wo +##ars +##cept +belie +bre +certain +against +water +##uck +exp +hard +prop +years +del +unt +quite +beh +side +##iness +##ined +appear +four +few +##ool +done +better +##ited +half +min +##ail +tre +ref +also +##cy +##oub +##vent +began +##most +##ows +ter +##ully +##act +enough +pur +##ised +dra +act +dear +having +seen +gener +lady +hum +##aster +##ared +##aken +##ash +##oke +hour +inde +##ps +##ying +cur +called +##ier +##ason +does +whole +##uth +morn +turned +pleas +##ial +gave +betw +##ense +occ +##aps +sou +course +##ib +ins +##gether +##ream +##ired +both +add +poor +col +read +bla +##ten +##selves +ste +among +between +keep +##ular +beaut +matter +care +morning +white +##ained +sure +name +##ged +toward +##ute +##owed +small +whom +par +exc +sk +pa +lar +felt +##ks +##ating +##osed +##bo +myself +open +six +es +##ru +herself +however +##xt +##ond +hund +stood +hundred +##plied +sent +inc +round +##tered +sim +##ife +##ates +##ters +days +almost +point +##eng +gre +eight +words +hor +fam +stand +land +##dden +##ung +mean +sun +##ird +##haps +##iet +perhaps +##ures +art +sle +ed +##iss +best +dark +sudden +hear +bu +replied +bar +voice +met +##ton +che +anything +till +underst +##ren +##its +until +black +##oud +looking +cried +##ned +suff +aw +##ins +prin +red +##amp +less +lay +next +law +##end +power +##vel +fact +##not +brought +##up +##ately +##enty +country +help +##als +sn +quest +wonder +use +vis +laugh +##med +##fully +together +air +unc +need +adv +wish +##ove +person +indeed +since +contin +##oney +gent +present +har +forg +full +run +rather +ra +##ok +cond +ide +sy +##eter +##rib +feet +fin +fall +##cei +five +mil +talk +##ater +bed +##ured +doct +##sh +reach +interest +lord +gone +##tle +death +consid +##fort +serv +wife +doub +adm +##ief +##pr +##ene +taken +others +hope +cannot +speak +themselves +along +ext +dire +bes +ent +master +##od +##ued +gentle +##lic +##uring +fire +large +pol +##ention +##oy +thus +##iled +sec +##ets +play +real +prom +##ments +##ield +bus +differ +##ison +thou +##ember +##ead +##ches +reason +thous +##iting +brother +##og +##akes +thousand +money +dep +answered +true +##unt +children +behind +sound +##ants +##ably +wood +##used +whose +ele +twenty +mer +rec +believe +inv +hon +book +dec +opp +hold +##ude +fair +passed +sch +##ided +##ities +##ength +##cious +sing +##cover +##ys +nature +##ering +alone +dead +##app +sever +cat +close +gold +start +hur +fur +##ont +doctor +ask +son +ground +pat +sea +strong +leave +compan +##illed +##ept +##cond +##ides +obs +fell +##body +doubt +sign +earth +remember +fear +given +##ances +therefore +ir +##oc +##ually +deep +##iful +##ressed +body +often +##du +pap +##ane +short +second +tem +##owing +##ceed +stre +fort +##mb +##ale +horse +mad +chr +order +const +ten +clear +##irc +fif +eas +##vers +bur +really +spir +##ceived +##read +##laim +ill +##osition +bra +understand +list +abs +spr +##az +pain +question +##anger +everything +aff +circ +wall +coming +##cted +##ching +##ready +above +already +least +expl +##esh +step +used +prince +itself +##ister +necess +case +around +tri +##hn +soul +##ized +suddenly +lad +evening +general +mag +num +##imes +known +quick +captain +##ific +whether +lear +within +live +times +expect +state +friends +swe +##gn +bring +##alked +sort +##men +women +table +meet +john +sum +returned +held +dri +big +walk +##eth +says +lea +##cer +obser +else +mid +during +##ident +mus +##ape +beautiful +ord +either +town +lost +sight +sleep +##vern +means +foot +cut +cal +kept +##ety +##gar +##ience +prof +fellow +dream +mist +past +became +pret +bad +##ages +stra +making +##ught +##ey +##ution +##lish +object +est +##ts +towards +dev +human +manner +strange +low +##old +##ott +##ient +##gs +##ines +ann +##here +ang +daugh +##oon +city +rich +sometimes +##aught +##airs +rose +dem +##ility +ready +ru +##cing +##itch +pal +road +ran +corn +thy +cold +##ians +##iew +##aring +acr +bit +won +didn +christ +squ +truth +##mer +##orrow +##nded +thir +soci +perfect +subject +##br +ship +mur +answer +pie +disc +arr +free +business +river +##arr +cast +family +##ountain +hus +jud +letter +##ony +hair +arms +##oken +##ote +become +sit +ut +ah +##ingly +##ouble +val +##aint +natur +possible +several +fine +##ffect +lead +express +sus +glad +arri +blood +##itting +##rie +quiet +able +##ert +idea +##iously +##itted +##bed +charac +##ah +begin +chur +story +eye +##band +##ative +consider +across +##li +except +equ +tou +##dle +cent +##isf +sto +partic +diffic +window +surpr +prov +grand +##use +flo +pen +ca +govern +wild +soft +cross +different +effect +##ased +def +longer +discover +followed +sold +thee +husband +fre +##antly +##by +exper +##ards +pub +cor +enter +win +suppose +##teen +respect +hast +##ices +proper +sal +comple +show +heav +##cul +happy +creat +taking +##essed +stay +requ +appro +spoke +daughter +wee +##une +##lly +##alth +anim +din +green +bird +mere +##ny +cra +gard +##empt +##ms +draw +reached +##ov +appeared +cover +##ley +exist +##ug +##ination +pretty +hea +opened +##anged +slow +##ize +imag +saying +##utes +common +tom +mor +coll +ap +occas +##ames +##ench +mountain +satisf +sense +succ +fav +##oura +##ising +rock +accord +scar +break +month +continued +stri +contr +church +tree +carried +spirit +##outh +cry +##ump +wanted +certainly +secret +command +##be +dest +##lled +move +lu +street +account +ought +prep +sens +##ats +##ement +purp +##ots +##riage +tear +esc +##owers +breat +dro +fan +character +##res +##ime +decl +##medi +stud +school +##itude +sla +quar +heaven +feeling +##son +sad +regard +bright +worth +##ney +dut +hot +hung +##laimed +##ric +disp +trans +delight +reco +##iling +rid +##eric +blue +hill +cloth +div +mot +disapp +##idence +produ +sweet +success +##ury +fast +english +ye +dress +port +neither +court +var +seven +##amed +##ains +princess +lived +view +##tern +##ushed +immedi +self +chief +hours +mill +exam +##wh +ce +tried +##day +entered +##ask +eat +carry +import +bear +danger +##ledge +offic +du +cause +none +forward +##umb +uncle +number +det +further +##gr +pay +##gin +added +eff +particular +deal +queen +prot +led +front +acqu +pray +##ored +happened +alth +fight +walked +later +##bit +joy +intell +##ster +comfort +prob +maj +##ening +public +bene +##uss +##ship +although +fig +direct +##iar +##ger +bat +mouth +##ality +mem +seems +sho +##side +##iam +sol +yourself +fore +##to +doing +living +desp +##oof +##pend +field +received +shad +bey +beyond +phil +line +visit +##inct +party +garden +hall +vi +##uff +americ +##ism +mat +gentleman +burn +##ested +laid +##irt +##raid +occasion +entire +age +sister +repe +##ously +prison +accom +whis +nearly +##ndon +trees +##cri +ge +eighteen +send +##wards +early +breath +lab +forth +##ior +try +##ones +med +terri +da +##ilt +princi +enem +thinking +##where +chance +minutes +cre +anx +mary +##anced +pict +wait +vill +afraid +crow +smile +##els +england +late +pleasure +bank +##bert +##ming +faith +super +news +fle +seeing +rob +neigh +##oint +##iment +hat +auth +bill +plain +trouble +silence +aunt +exclaimed +shar +coun +er +##gy +miles +##ply +glass +drew +neighb +mine +pract +heavy +standing +sevent +change +necessary +chap +##pping +purpose +concer +length +inqu +bell +##iest +natural +deter +##icked +hardly +top +caught +##ues +strength +##ives +##ounded +##aw +##ession +circum +fing +stopped +av +opin +##ament +priv +instead +##ences +##pper +grew +loved +island +knight +ago +inn +peace +watch +##inary +##iced +third +beauty +##uct +emb +conv +approach +determ +ver +##ler +##ining +##hood +##king +convers +straight +besides +position +thirty +##iter +den +attention +south +##ager +##ically +hist +##eless +comes +##aged +clock +force +plan +sitting +please +dog +whatever +##ugg +inform +hop +chair +lie +##form +build +bab +##ma +##ustom +je +girls +rom +french +struck +pull +wrong +knowledge +grace +##oom +scarce +resol +thoughts +attempt +fifty +rap +box +getting +company +crowd +##tained +##asp +##ils +class +die +drink +dinner +exact +enj +thick +saint +slight +save +maid +immediately +born +main +##ius +##crib +rev +remark +ast +##oose +north +##ists +tears +treat +##usion +meant +boys +fat +##ls +slowly +author +incl +lim +died +infl +fool +fish +sugg +post +accept +##ern +##elve +##ceive +##cient +stone +boat +paper +fright +conse +jour +ty +enc +quickly +youth +##ging +van +vict +##ified +belong +warm +##par +fix +imposs +forgot +beside +margar +leg +tone +camp +bound +makes +margaret +##mon +brown +##bs +##val +moon +##rupt +gather +lips +placed +circumst +food +filled +##iers +##ift +desire +##ee +london +distance +forest +floor +obl +##itions +spring +jack +viol +##ics +pock +following +remained +ger +grow +snow +government +ball +hors +##ented +##aded +broken +laughed +##ann +##oured +refle +safe +##ries +hen +##ively +##itten +depart +neck +easy +thank +##dom +##ham +##ope +knows +waiting +remain +hid +fresh +und +anne +dread +ash +##arch +result +proceed +gaz +##uc +shout +##dd +started +gall diff --git a/examples/asr/experimental/configs/bpe_vocabs/4k/vocab.txt b/examples/asr/experimental/configs/bpe_vocabs/4k/vocab.txt new file mode 100644 index 000000000000..d65cec73c945 --- /dev/null +++ b/examples/asr/experimental/configs/bpe_vocabs/4k/vocab.txt @@ -0,0 +1,4096 @@ +[PAD] +[UNK] +[CLS] +[SEP] +[MASK] +' +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +##e +##s +##c +##r +##i +##p +##t +##v +##a +##n +##d +##b +##u +##l +##y +##o +##g +##w +##h +##m +##x +##k +##f +##z +##q +##j +th +the +##er +##nd +##in +##ed +##ou +##at +##en +and +##or +##es +to +of +##on +##is +##ing +##ar +##it +##as +##an +##ll +in +##re +wh +he +##om +be +ha +##le +##ot +##ow +##ic +##ut +it +##ld +that +sh +##ly +was +##gh +##id +##se +on +##ve +##et +##ent +you +##im +##st +##ce +##ith +for +##ir +##ion +as +##ay +his +##al +with +##ur +st +##ter +her +##ver +##ch +re +##ad +##ght +an +not +is +had +sa +at +##oo +##her +but +##ould +she +##ct +##ess +fr +se +##am +pr +we +him +so +##ill +##ain +su +##ight +me +##ere +##all +##ke +##il +de +my +they +all +##est +ch +##ich +ne +by +have +##ome +##pp +##ck +li +this +do +##ore +con +##us +which +##ri +##ul +up +said +from +##th +##ard +##ge +##ra +or +one +no +ex +were +there +##os +##pe +man +##ble +who +al +##ant +##ie +##ous +##ust +##ation +fe +##and +them +##ers +##lf +when +##ind +##ast +go +##ood +kn +##un +##our +##ist +##em +are +##out +would +##art +what +le +their +##ong +en +if +un +##if +com +##ound +ab +out +wor +##ol +will +int +br +##ak +mis +##ate +##op +##um +##ough +##la +##itt +ar +##ook +##ish +##one +##ap +been +##own +##ame +then +##ven +##self +##ive +tr +ro +##ig +fa +##reat +af +sp +qu +##ery +##ity +##ort +##ost +tw +ag +could +am +##el +##ake +any +##ure +some +your +can +more +tim +##ep +##nder +ever +##ence +##ther +##fe +into +##ry +##ose +now +##au +##ag +cl +very +##qu +##ved +bo +other +did +##ance +than +##ittle +little +##ine +##ies +##oun +look +##way +##urn +like +##ite +gr +##ear +##ach +lo +##ass +##other +##able +##ought +know +time +im +us +co +##fore +##ell +how +##hing +##cc +ad +day +##ack +about +##ade +##ice +see +over +##ci +##ide +##pt +dis +##ink +too +##wn +great +after +per +pro +again +##ment +upon +hand +has +##ree +##ked +##ather +only +car +##tain +##ally +str +mar +po +fl +well +should +##ab +##pl +say +good +##ings +two +our +thr +##ud +##ving +##oss +down +##ty +##ire +pe +before +##ves +app +##ile +every +its +pl +cr +old +off +made +##ick +##ied +long +bl +##age +##ber +where +##ress +fir +##ang +mu +never +must +sm +##ful +such +##ull +##ions +##ft +pre +came +##ro +miss +##ward +##een +##ild +##orn +##ue +even +under +these +##ord +come +part +fo +sc +##ated +rem +ind +##ness +bec +##per +may +much +think +##ious +way +mister +##ount +##led +let +ey +##ath +thought +##ect +back +spe +gl +##ady +bet +##ff +##ans +res +just +first +##ise +here +ho +own +des +##ried +ob +##ces +mo +pla +##ced +cont +##ary +##ered +went +himself +men +##me +##co +##int +might +##ily +seem +##ip +beg +##oth +##ouse +##ave +wat +comp +sw +through +##igh +away +em +get +head +don +imp +##llow +bel +##ened +without +##iz +life +most +pass +ear +make +cons +som +turn +##ac +##iv +shall +##ng +pres +acc +those +eyes +house +att +somet +jo +still +night +call +##red +op +##ew +##ens +wom +last +##ign +tell +##ause +nothing +##ark +sl +new +##ction +la +##iff +take +being +comm +many +##les +ser +word +inter +##atter +wr +##ons +ret +found +##are +##ase +while +##ory +##ix +saw +put +year +##iend +##ople +##dy +young +far +##aking +once +friend +people +##air +##ether +work +##ors +##ible +room +##ently +three +##ted +yet +##pect +##ever +count +same +father +right +child +cour +##ne +place +another +##ition +want +though +##less +##ince +king +##ian +happ +heart +end +gra +nat +face +##thing +love +took +dist +arm +##ace +inst +rep +##alk +##man +##ult +char +nor +te +ple +set +##cl +light +looked +##gg +missus +asked +mind +yes +supp +wa +ll +left +##cess +##ting +ac +##ub +##augh +##ents +##te +##ished +pers +##ways +things +sir +mom +##irl +near +##sw +##ations +sat +find +##der +##sel +##ia +world +##ower +##rew +##ject +##fect +give +cap +##av +why +mother +gu +gen +##ict +always +##so +##ling +answ +##pped +soon +##iver +##den +form +heard +war +thing +something +seemed +sub +god +door +girl +##ange +##ph +##und +wind +because +diff +mon +going +##ertain +##de +told +home +ke +##orm +got +##ither +el +##ret +##ock +sur +high +##ched +eng +gi +ma +ph +##ank +hands +moment +woman +follow +##land +dr +ve +vo +feel +##aut +##ning +poss +ev +##ical +##lt +##ple +boy +return +rest +reg +knew +each +inf +conf +oh +sil +##ner +kind +cle +##ush +hel +rel +##orr +tra +##ual +ass +wo +##ars +##cept +belie +bre +certain +against +water +##uck +exp +hard +prop +years +del +unt +quite +beh +side +##iness +##ined +appear +four +few +##ool +done +better +##ited +half +min +##ail +tre +ref +also +##cy +##oub +##vent +began +##most +##ows +ter +##ully +##act +enough +pur +##ised +dra +act +dear +having +seen +gener +lady +hum +##aster +##ared +##aken +##ash +##oke +hour +inde +##ps +##ying +cur +called +##ier +##ason +does +whole +##uth +morn +turned +pleas +##ial +gave +betw +##ense +occ +##aps +sou +course +##ib +ins +##gether +##ream +##ired +both +add +poor +col +read +bla +##ten +##selves +ste +among +between +keep +##ular +beaut +matter +care +morning +white +##ained +sure +name +##ged +toward +##ute +##owed +small +whom +par +exc +sk +pa +lar +felt +##ks +##ating +##osed +##bo +myself +open +six +es +##ru +herself +however +##xt +##ond +hund +stood +hundred +##plied +sent +inc +round +##tered +sim +##ife +##ates +##ters +days +almost +point +##eng +gre +eight +words +hor +fam +stand +land +##dden +##ung +mean +sun +##ird +##haps +##iet +perhaps +##ures +art +sle +ed +##iss +best +dark +sudden +hear +bu +replied +bar +voice +met +##ton +che +anything +till +underst +##ren +##its +until +black +##oud +looking +cried +##ned +suff +aw +##ins +prin +red +##amp +less +lay +next +law +##end +power +##vel +fact +##not +brought +##up +##ately +##enty +country +help +##als +sn +quest +wonder +use +vis +laugh +##med +##fully +together +air +unc +need +adv +wish +##ove +person +indeed +since +contin +##oney +gent +present +har +forg +full +run +rather +ra +##ok +cond +ide +sy +##eter +##rib +feet +fin +fall +##cei +five +mil +talk +##ater +bed +##ured +doct +##sh +reach +interest +lord +gone +##tle +death +consid +##fort +serv +wife +doub +adm +##ief +##pr +##ene +taken +others +hope +cannot +speak +themselves +along +ext +dire +bes +ent +master +##od +##ued +gentle +##lic +##uring +fire +large +pol +##ention +##oy +thus +##iled +sec +##ets +play +real +prom +##ments +##ield +bus +differ +##ison +thou +##ember +##ead +##ches +reason +thous +##iting +brother +##og +##akes +thousand +money +dep +answered +true +##unt +children +behind +sound +##ants +##ably +wood +##used +whose +ele +twenty +mer +rec +believe +inv +hon +book +dec +opp +hold +##ude +fair +passed +sch +##ided +##ities +##ength +##cious +sing +##cover +##ys +nature +##ering +alone +dead +##app +sever +cat +close +gold +start +hur +fur +##ont +doctor +ask +son +ground +pat +sea +strong +leave +compan +##illed +##ept +##cond +##ides +obs +fell +##body +doubt +sign +earth +remember +fear +given +##ances +therefore +ir +##oc +##ually +deep +##iful +##ressed +body +often +##du +pap +##ane +short +second +tem +##owing +##ceed +stre +fort +##mb +##ale +horse +mad +chr +order +const +ten +clear +##irc +fif +eas +##vers +bur +really +spir +##ceived +##read +##laim +ill +##osition +bra +understand +list +abs +spr +##az +pain +question +##anger +everything +aff +circ +wall +coming +##cted +##ching +##ready +above +already +least +expl +##esh +step +used +prince +itself +##ister +necess +case +around +tri +##hn +soul +##ized +suddenly +lad +evening +general +mag +num +##imes +known +quick +captain +##ific +whether +lear +within +live +times +expect +state +friends +swe +##gn +bring +##alked +sort +##men +women +table +meet +john +sum +returned +held +dri +big +walk +##eth +says +lea +##cer +obser +else +mid +during +##ident +mus +##ape +beautiful +ord +either +town +lost +sight +sleep +##vern +means +foot +cut +cal +kept +##ety +##gar +##ience +prof +fellow +dream +mist +past +became +pret +bad +##ages +stra +making +##ught +##ey +##ution +##lish +object +est +##ts +towards +dev +human +manner +strange +low +##old +##ott +##ient +##gs +##ines +ann +##here +ang +daugh +##oon +city +rich +sometimes +##aught +##airs +rose +dem +##ility +ready +ru +##cing +##itch +pal +road +ran +corn +thy +cold +##ians +##iew +##aring +acr +bit +won +didn +christ +squ +truth +##mer +##orrow +##nded +thir +soci +perfect +subject +##br +ship +mur +answer +pie +disc +arr +free +business +river +##arr +cast +family +##ountain +hus +jud +letter +##ony +hair +arms +##oken +##ote +become +sit +ut +ah +##ingly +##ouble +val +##aint +natur +possible +several +fine +##ffect +lead +express +sus +glad +arri +blood +##itting +##rie +quiet +able +##ert +idea +##iously +##itted +##bed +charac +##ah +begin +chur +story +eye +##band +##ative +consider +across +##li +except +equ +tou +##dle +cent +##isf +sto +partic +diffic +window +surpr +prov +grand +##use +flo +pen +ca +govern +wild +soft +cross +different +effect +##ased +def +longer +discover +followed +sold +thee +husband +fre +##antly +##by +exper +##ards +pub +cor +enter +win +suppose +##teen +respect +hast +##ices +proper +sal +comple +show +heav +##cul +happy +creat +taking +##essed +stay +requ +appro +spoke +daughter +wee +##une +##lly +##alth +anim +din +green +bird +mere +##ny +cra +gard +##empt +##ms +draw +reached +##ov +appeared +cover +##ley +exist +##ug +##ination +pretty +hea +opened +##anged +slow +##ize +imag +saying +##utes +common +tom +mor +coll +ap +occas +##ames +##ench +mountain +satisf +sense +succ +fav +##oura +##ising +rock +accord +scar +break +month +continued +stri +contr +church +tree +carried +spirit +##outh +cry +##ump +wanted +certainly +secret +command +##be +dest +##lled +move +lu +street +account +ought +prep +sens +##ats +##ement +purp +##ots +##riage +tear +esc +##owers +breat +dro +fan +character +##res +##ime +decl +##medi +stud +school +##itude +sla +quar +heaven +feeling +##son +sad +regard +bright +worth +##ney +dut +hot +hung +##laimed +##ric +disp +trans +delight +reco +##iling +rid +##eric +blue +hill +cloth +div +mot +disapp +##idence +produ +sweet +success +##ury +fast +english +ye +dress +port +neither +court +var +seven +##amed +##ains +princess +lived +view +##tern +##ushed +immedi +self +chief +hours +mill +exam +##wh +ce +tried +##day +entered +##ask +eat +carry +import +bear +danger +##ledge +offic +du +cause +none +forward +##umb +uncle +number +det +further +##gr +pay +##gin +added +eff +particular +deal +queen +prot +led +front +acqu +pray +##ored +happened +alth +fight +walked +later +##bit +joy +intell +##ster +comfort +prob +maj +##ening +public +bene +##uss +##ship +although +fig +direct +##iar +##ger +bat +mouth +##ality +mem +seems +sho +##side +##iam +sol +yourself +fore +##to +doing +living +desp +##oof +##pend +field +received +shad +bey +beyond +phil +line +visit +##inct +party +garden +hall +vi +##uff +americ +##ism +mat +gentleman +burn +##ested +laid +##irt +##raid +occasion +entire +age +sister +repe +##ously +prison +accom +whis +nearly +##ndon +trees +##cri +ge +eighteen +send +##wards +early +breath +lab +forth +##ior +try +##ones +med +terri +da +##ilt +princi +enem +thinking +##where +chance +minutes +cre +anx +mary +##anced +pict +wait +vill +afraid +crow +smile +##els +england +late +pleasure +bank +##bert +##ming +faith +super +news +fle +seeing +rob +neigh +##oint +##iment +hat +auth +bill +plain +trouble +silence +aunt +exclaimed +shar +coun +er +##gy +miles +##ply +glass +drew +neighb +mine +pract +heavy +standing +sevent +change +necessary +chap +##pping +purpose +concer +length +inqu +bell +##iest +natural +deter +##icked +hardly +top +caught +##ues +strength +##ives +##ounded +##aw +##ession +circum +fing +stopped +av +opin +##ament +priv +instead +##ences +##pper +grew +loved +island +knight +ago +inn +peace +watch +##inary +##iced +third +beauty +##uct +emb +conv +approach +determ +ver +##ler +##ining +##hood +##king +convers +straight +besides +position +thirty +##iter +den +attention +south +##ager +##ically +hist +##eless +comes +##aged +clock +force +plan +sitting +please +dog +whatever +##ugg +inform +hop +chair +lie +##form +build +bab +##ma +##ustom +je +girls +rom +french +struck +pull +wrong +knowledge +grace +##oom +scarce +resol +thoughts +attempt +fifty +rap +box +getting +company +crowd +##tained +##asp +##ils +class +die +drink +dinner +exact +enj +thick +saint +slight +save +maid +immediately +born +main +##crib +##ius +rev +remark +ast +##oose +north +##ists +tears +treat +##usion +meant +boys +fat +##ls +slowly +author +incl +lim +died +infl +fool +fish +sugg +post +accept +##ern +##elve +##ceive +##cient +stone +boat +paper +fright +conse +jour +ty +enc +quickly +youth +##ging +van +vict +##ified +belong +warm +##par +fix +imposs +forgot +beside +margar +leg +tone +camp +bound +makes +margaret +##mon +brown +##bs +##val +moon +##rupt +gather +lips +placed +circumst +food +filled +##iers +##ift +desire +##ee +london +distance +forest +floor +obl +##itions +spring +jack +viol +##ics +pock +following +remained +ger +grow +snow +government +ball +hors +##ented +##aded +broken +laughed +##ann +##oured +refle +safe +##ries +hen +##ively +##itten +depart +neck +easy +thank +##dom +##ope +##ham +knows +waiting +remain +hid +fresh +und +anne +dread +ash +##arch +result +proceed +gaz +##uc +shout +##dd +started +gall +complete +whisper +wide +horses +##isted +##cially +##ges +service +piece +##eness +forget +palace +guard +doll +##erv +talking +yell +##cial +lift +fun +grave +week +impossible +effort +##nding +imm +army +difficult +##let +##reme +stop +mess +narr +deg +incre +corner +society +path +weak +shut +hy +teach +cloud +##ency +##ivid +demand +##ended +##unk +nine +recogn +##cell +##fer +turning +sky +ori +##asure +##olog +opinion +sin +beat +single +dw +fly +lang +mass +outside +kiss +trust +occup +##uit +below +appearance +silver +bro +west +aftern +gun +couldn +ant +conn +showed +conversation +running +mention +nin +greater +music +##ases +##ises +##fection +mark +espe +##ror +##ories +une +spot +##ourage +peter +##ork +##anch +presence +winter +battle +hun +morrow +probably +##ection +clothes +fash +compl +wished +appar +married +arrived +suffic +eager +former +giving +pop +##pic +sand +usual +relig +simple +sym +##itation +neg +sor +moved +months +speaking +silent +cab +pun +expression +covered +afternoon +pet +##aped +occur +states +##rief +admir +##light +easily +##unity +phys +tender +beginning +duty +chapter +influ +##zz +instant +polit +enjoy +sick +tur +stream +figure +##ald +##ract +jew +vol +minute +pleasant +scarcely +conscious +terrible +kill +concl +pale +raised +##oop +fashion +thor +##yal +##ze +twelve +leaving +##ature +mel +fut +threw +flowers +attack +trying +##cked +sword +tall +marry +expected +according +##uel +forty +stick +temper +iron +oblig +train +rapid +ladies +learn +especially +##ipped +##orted +isn +##amb +evil +bread +learned +##orth +office +surprise +write +purs +written +killed +star +consequ +exh +places +condition +direction +##jo +entirely +action +##ding +##wise +##ibly +##mit +happiness +id +decided +golden +langu +##uous +note +bott +unless +books +fal +##aled +wonderful +shook +##ounds +##ech +##onel +perfectly +geor +sail +##ume +broad +closed +##atic +latter +steps +utter +merely +history +##gan +##osing +wise +wouldn +understood +middle +prev +enemy +sole +gar +simply +tast +shot +board +ring +sav +bow +##io +reply +##gress +changed +excell +grass +finally +##ital +suc +summer +sharp +fortune +experience +succeed +##uted +##retched +journey +observed +##osp +afterwards +##fast +##ec +earn +persons +lean +broke +deser +fit +mir +marriage +##acy +repres +bitter +lying +cou +rain +trave +listen +situ +##anc +##cience +##ected +origin +wit +##olute +##oses +calm +lives +escape +beneath +##ouses +clim +hes +ven +repeated +pocket +##lling +liber +oper +tail +whe +passion +dick +mut +becom +dar +allowed +famil +ow +##ivers +##mn +gate +spite +health +##str +##ission +ign +cand +habit +empl +strugg +firm +sorry +##under +madame +furn +sigh +future +##umed +nice +separ +presently +circumstances +##reh +##ples +sett +loud +worse +corr +wand +spread +letters +magn +disg +passing +pleased +darkness +##come +civ +##cies +tea +apart +disappear +important +##erable +legs +nation +delic +dressed +game +extra +walls +dry +virt +dim +rub +##idently +absolute +blind +discovered +exactly +dam +##oved +unh +cost +arch +employ +##velop +cous +beast +opport +hu +dropped +pack +subst +smiled +chee +resp +promise +##urs +host +creature +yellow +cla +notice +priest +knowing +heads +##esty +cook +shore +##ishment +individ +existence +determined +##lend +storm +seat +ears +##ours +mount +ur +##ina +conce +##anging +fixed +press +wear +major +ves +pros +##iod +motion +empt +leaves +bottom +arg +##iety +##sy +##enth +nobody +pick +##amber +##que +bag +intellig +stir +assist +dom +advant +tired +vent +##ument +##ashed +dan +##action +wrote +considered +houses +suit +cheer +clean +fla +lou +##path +##rel +cho +castle +frequ +drawing +luck +##cent +##pan +##idge +##ulty +ones +noble +splend +honor +paid +speech +estab +faint +##ires +individual +vall +##inite +##rodu +birds +allow +impress +jane +##my +song +touch +various +moder +believed +##istr +mod +somewh +period +shadow +affect +mal +extreme +judge +village +royal +somewhat +lower +##go +##zen +agree +remembered +aston +declared +##ays +ham +parts +colonel +##cil +breakfast +shr +tow +drop +surely +##ayed +##uded +scene +describ +comb +greatest +influence +custom +animal +##itary +sake +perform +soldiers +##iny +ancient +##ests +evidently +drawn +looks +ways +harm +destro +revol +##nes +##ator +##anted +picture +##ption +likely +laws +holding +difficulty +courage +inj +prevent +mort +troub +burst +angry +proud +spoken +##ron +##head +##ision +attend +watched +east +horr +pot +##na +agre +private +##bing +shop +jul +finished +anxious +##cise +fifteen +social +dism +wine +ideas +murder +mater +george +space +scr +inside +gentlemen +civil +##iently +formed +fol +fail +constit +lot +goes +thin +surf +servant +ourselves +fallen +fancy +henry +##ium +kne +advent +carriage +baby +tong +everybody +commun +##itive +ine +waited +grou +##gged +vain +##arked +##otion +favor +bal +speed +guess +narrow +windows +carefully +ice +jim +noise +meth +hero +william +pecul +promised +walking +forgotten +obliged +earnest +lose +imper +vessel +grad +glance +shoulder +animals +noticed +knock +peculiar +##cks +quietly +cup +ta +serious +tremb +generally +american +france +exer +##iction +property +stranger +shoulders +##osity +sympath +bod +flor +cow +dust +affection +conduct +##rop +##raph +higher +handsome +##iant +##nday +recover +wel +poet +pure +chamber +distinct +gh +mam +pier +grown +##uly +rough +##enced +desce +murm +##stem +personal +##osite +offered +##onsie +edge +built +whispered +skin +pieces +##arily +##itated +suffer +butter +faces +spent +##ables +interrupt +##how +##isters +develop +unk +fond +coat +touched +hol +jump +pi +##ingu +wasn +surprised +race +##af +##uce +##oted +dish +##ob +smo +educ +brave +fer +##over +weather +eld +wealth +system +german +false +##yn +##aked +cousin +##key +ben +prepared +##wood +fingers +surr +intellect +##ipp +##ites +opportunity +introdu +lovely +rag +crown +matters +companion +alar +innoc +ris +sorrow +##hern +quarter +band +lake +mix +meaning +mountains +memory +dor +##ring +##leep +##ctions +frank +finding +mach +charge +pan +gray +avo +inh +grat +greatly +dare +asleep +offer +dav +alto +##ounced +opposite +##ett +##unch +breast +bare +reading +altogether +writing +##pected +degree +exec +##fortun +hath +##eal +feelings +rail +##night +honour +fate +##attered +meeting +##ilar +alive +hide +cruel +blow +hurt +loss +defe +thrown +caused +##oned +seek +serve +profess +duke +bent +##kes +united +kingdom +situation +empty +wis +liked +swift +opening +servants +bush +suspic +freed +##ointed +surface +questions +##aff +curious +ess +accompan +christian +fill +wid +paris +satisfied +ere +pity +sides +##nds +equal +height +##osop +ordered +depend +grey +listened +##pet +##eer +rejo +capt +##ibility +despair +mart +happen +hurried +dollars +lin +loc +language +unf +yours +supposed +laughing +settled +talked +rode +perm +distingu +hurry +lifted +destroy +value +taste +##ores +##ners +square +vast +render +labor +rul +roof +stock +telling +papa +study +##ara +pin +rising +tal +sufficient +due +rise +##eds +forced +philosop +nose +sixty +egg +stat +fault +##asc +station +##ille +distur +regular +kitch +special +honest +building +shape +season +pride +smiling +orders +accompl +sacri +usually +watching +impro +##ushing +passage +valley +material +stairs +remarked +frightened +woods +wed +catch +mistress +directly +##he +prete +gloom +lines +justice +doesn +suggest +diam +growing +bless +##like +eur +##erence +ordinary +gathered +tit +threat +weeks +method +ale +ange +seized +search +discuss +ble +##ify +ignor +deb +##eping +supper +##uses +##ral +##bur +##chief +march +bold +putting +explain +##ik +nearer +york +affairs +contrary +##yr +##ishing +moving +streets +bir +##icated +wash +kitchen +bas +cit +##eder +imagine +prefer +possess +##rance +deliver +##imate +receive +monsie +por +twice +monsieur +##min +hole +sacrif +##hip +rever +reck +process +favour +dign +##aly +europe +unw +working +heat +##antic +cler +proved +##ades +pra +##lin +arth +lod +prove +##ception +pulled +pit +##ette +authority +jer +##ole +haven +movement +##tering +##usted +engaged +brothers +advantage +ber +##lished +arthur +stones +farm +difference +aside +hence +possession +observ +##etic +exte +soldier +hills +fortun +##uls +instance +ren +##iding +holy +color +flew +rate +wal +doors +trib +busy +lat +address +familiar +weight +played +##set +flower +aware +solemn +di +liter +test +indian +emper +distant +interesting +bull +thorough +ol +wore +worked +amb +explained +excellent +splendid +tongue +named +pard +shame +sam +franc +moments +trium +wil +##eav +myster +seated +spect +instantly +unkn +endeav +similar +naturally +triumph +##cr +measure +dogs +amount +elect +officer +##itable +uns +necessity +rank +##theless +brit +confidence +vir +##illa +##ulous +nevertheless +agreed +hearing +##ancho +takes +##ental +univers +fier +earl +keeping +drive +produced +aud +aug +cott +names +##con +disappeared +##agn +throw +president +gods +magic +prisoner +represent +unknown +##por +##ague +terror +contain +bri +smoke +support +detail +wicked +flat +yester +artic +dull +falling +##vey +worthy +liberty +##ulation +design +wants +evidence +companions +spirits +bid +##bers +coast +thro +mighty +particularly +witness +##ky +clever +sought +span +rare +officers +##lv +##ford +##zy +yesterday +powers +yield +organ +amaz +pointed +claim +content +possibly +cool +hom +shock +preser +terms +dwell +persu +drove +ceased +acquaint +occurred +lies +gri +gree +tor +otherwise +finger +emperor +bask +stars +knees +pair +paused +##ruction +failed +required +unhapp +diamond +aut +rat +ali +double +forms +gives +##race +bob +eliz +travel +carrying +gle +##iles +teeth +freedom +shown +forb +fruit +##umn +waters +hearts +ener +pill +##soci +example +upper +milk +affair +softly +moral +foreign +charles +excit +huge +##oes +possessed +cases +fet +##arther +counten +report +principle +inhab +proof +intended +satisfaction +shouted +farther +group +der +##asm +unfortun +unto +singing +arrange +religion +rocks +seventeen +james +works +buy +america +succeeded +rooms +events +leading +majesty +gently +dance +tele +##using +##ached +ga +##father +key +mentioned +seventy +ros +suffering +record +cabin +invent +diss +##ival +demanded +excitement +associ +progress +##angers +##road +curi +##erous +rule +vig +clearly +sheep +regret +neighbour +##iance +instinct +awful +advice +ban +fully +sen +##ena +##mend +papers +hidden +smooth +birth +hither +##bor +stuff +impat +calling +fourth +dreadful +ain +grief +pos +##pap +##ese +brill +powerful +presented +fairy +shoot +##ker +proc +joined +afford +##iments +shel +considerable +obey +interv +voices +interested +card +conqu +cred +needed +cord +pounds +wag +advanced +undert +resolved +##istic +excited +paul +bath +asse +conditions +pictures +##gen +shining +sunday +served +steam +police +sprang +jes +salt +fields +bore +##sie +indians +fierce +ride +nodd +aband +desired +edward +importance +information +hosp +memb +perceived +task +yard +##mond +crit +control +##ternal +fold +##ensive +sooner +merch +dur +suffered +absolutely +citiz +##isc +iss +##illy +completely +cart +rad +##ois +##hold +share +willing +devil +ships +imagination +superior +ep +##ancy +feather +env +anybody +stead +dawn +##asped +occupied +curiosity +fox +yo +##inal +##asant +##asons +##ams +myst +smith +##room +type +apparently +couple +extraord +welcome +bay +daily +modern +dying +##pir +feat +accident +##ension +countenance +##iot +##llen +##ortion +crime +mult +##get +reb +alas +refused +inquir +hate +whenever +thanks +regarded +suggested +##ulf +actually +pardon +size +##gment +cult +flesh +hotel +bodies +shell +##ishop +truly +betr +##uced +##with +rapidly +odd +##alous +club +horror +mile +san +sour +flight +crossed +slave +professor +oce +rushed +excl +worst +##ization +event +corre +science +##sed +##ainted +stepped +midst +foolish +##ha +##thus +brain +##acle +cottage +buck +brief +gain +tight +expressed +drag +equally +education +august +dun +nay +##com +##abel +tempt +picked +taught +elsie +lands +driven +political +resist +resem +instr +cell +lack +barb +dangerous +ken +keen +##hy +store +dozen +trade +points +ninet +sty +nob +##ability +crim +interp +relations +delighted +members +sisters +##ially +belief +asking +anger +park +wet +plat +relief +##ification +hunting +alex +cath +##aries +meat +obst +behold +mistake +inquired +remarkable +cop +nerv +##now +careful +cere +wounded +##ading +enemies +gradually +pu +##for +thorn +worn +interrupted +stup +silk +sixteen +##racted +flour +sylv +lone +nap +##kins +##asing +depth +dreams +##esp +kindly +##reg +##acity +eleven +invol +severe +##ocr +oak +wings +hal +##lace +begun +distr +##while +benef +moreover +exped +independ +artagn +buried +approached +artagnan +saved +sex +species +harry +physical +thunder +glory +creatures +newspap +sang +##kin +plenty +useful +shoes +hoped +frequently +##vert +saf +princip +##oe +chest +bran +##ication +roll +lover +famous +recoll +queer +grim +charming +aim +loose +nur +ath +indiff +consciousness +mamma +enthus +##ips +slept +fighting +enthusi +dig +##acc +##mas +##ids +stage +anyone +thrust +desper +drunk +lamp +##rant +bew +##stone +stern +evident +meanwhile +forgive +accepted +ocean +bot +tot +wondered +playing +hyp +hale +knife +closely +meas +proceeded +message +mour +##lied +sell +union +##ustomed +exceed +precious +century +unex +kat +##oops +##aming +everywhere +minds +extraordinary +arose +capital +entrance +recall +burning +magnific +##stand +plant +neighbor +assemb +lest +##orious +##uments +colle +virtue +retreat +##iana +capable +assured +##ograph +constant +governor +increased +horn +rush +removed +facts +absence +explan +ack +mud +fever +somebody +awa +admit +jealous +kissed +popular +hut +##pelled +grant +indign +friendship +sympathy +ug +##iable +thom +alice +level +objects +pressed +qual +begged +hind +highest +clouds +ghost +acknow +##oused +emp +wishes +##oking +strike +becomes +trembling +##gu +kindness +accordingly +throat +stretched +frag +confess +grandfather +wheel +bon +russ +deck +choose +eighty +##pher +miserable +contempt +##igned +military +basket +##nal +blank +listening +ahead +fro +##elt +larger +divine +##cription +tar +##iber +stories +reve +##izing +excuse +innocent +aid +et +hadn +remind +slaves +ara +##stairs +reward +manage +smell +throughout +waves +judgment +arrival +choice +unhappy +##astic +curr +advance +informed +impression +mysterious +acquaintance +ly +notes +comr +indust +ended +lights +nurse +bought +wra +##itor +spo +marked +scream +uneas +delicate +weary +##estic +join +prompt +experi +hungry +flying +pow +bridge +##aker +##umber +visible +understanding +crying +avoid +tis +stiff +sounds +bowed +caut +goods +david +stupid +##ancing +unable +bos +spark +amid +somewhere +infinite +ashamed +dignity +gay +vic +##ka +hollow +emotion +parents +admitted +hint +##pse +temple +comfortable +intelligence +virgin +bearing +##ldom +hers +##abeth +remains +contem +settle +immense +cher +weap +##ulated +lighted +baron +##gypt +elder +thoroughly +egypt +##ilst +##anges +friendly +rome +meal +intention +##achel +returning +billy +convin +ox +price +sup +rope +project +escaped +hopes +##ology +elizabeth +safety +wound +unus +##onscious +minister +horri +helped +secure +plainly +seldom +thinks +fellows +mood +pushed +exhib +##inging +fairly +pear +##irm +maiden +##mother +##ora +##orous +appears +cheeks +ease +skill +extent +practice +religious +becoming +features +pink +tied +##ential +whence +somehow +displ +greet +faithful +concerned +bishop +##isher +gro +eagerly +eating +waste +fem +bride +##ceiving +concerning +original +accustomed diff --git a/examples/asr/experimental/configs/bpe_vocabs/8k/vocab.txt b/examples/asr/experimental/configs/bpe_vocabs/8k/vocab.txt new file mode 100644 index 000000000000..378e9cebe711 --- /dev/null +++ b/examples/asr/experimental/configs/bpe_vocabs/8k/vocab.txt @@ -0,0 +1,8192 @@ +[PAD] +[UNK] +[CLS] +[SEP] +[MASK] +' +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +##n +##g +##e +##r +##l +##t +##i +##a +##m +##p +##v +##o +##w +##d +##u +##s +##b +##k +##c +##h +##f +##y +##x +##z +##q +##j +th +the +##er +##nd +##in +##ed +##ou +##at +##en +and +##or +##es +to +of +##on +##is +##ing +##ar +##it +##as +##an +##ll +in +##re +wh +he +##om +be +ha +##le +##ot +##ow +##ic +##ut +it +##ld +that +sh +##ly +was +##gh +##id +##se +on +##ve +##et +##ent +you +##im +##st +##ce +##ith +for +##ir +##ion +as +##ay +his +##al +with +##ur +st +##ter +her +##ver +##ch +re +##ad +##ght +an +not +is +had +sa +at +##oo +##her +but +##ould +she +##ct +##ess +fr +se +##am +pr +we +him +so +##ill +##ain +su +##ight +me +##ere +##all +##ke +##il +de +my +they +all +##est +ch +##ich +ne +by +have +##ome +##pp +##ck +li +this +do +##ore +con +##us +which +##ri +##ul +up +said +from +##th +##ard +##ge +##ra +or +one +no +ex +were +there +##os +##pe +man +##ble +who +al +##ant +##ie +##ous +##ust +##ation +fe +##and +them +##ers +##lf +when +##ind +##ast +go +##ood +kn +##un +##our +##ist +##em +are +##out +would +##art +what +le +their +##ong +en +if +un +##if +com +##ound +ab +out +wor +##ol +will +int +br +##ak +mis +##ate +##op +##um +##ough +##la +##itt +ar +##ook +##ish +##one +##ap +been +##own +##ame +then +##ven +##self +##ive +tr +ro +##ig +fa +##reat +af +sp +qu +##ery +##ity +##ort +##ost +tw +ag +could +am +##el +##ake +any +##ure +some +your +can +more +tim +##ep +##nder +ever +##ence +##ther +##fe +into +##ry +##ose +now +##au +##ag +cl +very +##qu +##ved +bo +other +did +##ance +than +##ittle +little +##ine +##ies +##oun +look +##way +##urn +like +##ite +gr +##ear +##ach +lo +##ass +##other +##able +##ought +know +time +im +us +co +##fore +##ell +how +##hing +##cc +ad +day +##ack +about +##ade +##ice +see +over +##ci +##ide +##pt +dis +##ink +too +##wn +great +after +per +pro +again +##ment +upon +hand +has +##ree +##ked +##ather +only +car +##tain +##ally +str +mar +po +fl +well +should +##ab +##pl +say +good +##ings +two +our +thr +##ud +##ving +##oss +down +##ty +##ire +pe +before +##ves +app +##ile +every +its +pl +cr +old +off +made +##ick +##ied +long +bl +##age +##ber +where +##ress +fir +##ang +mu +never +must +sm +##ful +such +##ull +##ions +##ft +pre +came +##ro +miss +##ward +##een +##ild +##orn +##ue +even +under +these +##ord +come +part +fo +sc +##ated +rem +ind +##ness +bec +##per +may +much +think +##ious +way +mister +##ount +##led +let +ey +##ath +thought +##ect +back +spe +gl +##ady +bet +##ff +##ans +res +just +first +##ise +here +ho +own +des +##ried +ob +##ces +mo +pla +##ced +cont +##ary +##ered +went +himself +men +##me +##co +##int +might +##ily +seem +##ip +beg +##oth +##ouse +##ave +wat +comp +sw +through +##igh +away +em +get +head +don +imp +##llow +bel +##ened +without +##iz +life +most +pass +ear +make +cons +som +turn +##ac +##iv +shall +##ng +pres +acc +those +eyes +house +att +somet +jo +still +night +call +##red +op +##ew +##ens +wom +last +##ign +tell +##ause +nothing +##ark +sl +new +##ction +la +##iff +take +being +comm +many +##les +ser +word +inter +##atter +wr +##ons +ret +found +##are +##ase +while +##ory +##ix +saw +put +year +##iend +##ople +##dy +young +far +##aking +once +friend +people +##air +##ether +work +##ors +##ible +room +##ently +three +##ted +yet +##pect +##ever +count +same +father +right +child +cour +##ne +place +another +##ition +want +though +##less +##ince +king +##ian +happ +heart +end +gra +nat +face +##thing +love +took +dist +arm +##ace +inst +rep +##alk +##man +##ult +char +nor +te +ple +set +##cl +light +looked +##gg +missus +asked +mind +yes +supp +wa +ll +left +##cess +##ting +ac +##ub +##augh +##ents +##te +##ished +pers +##ways +things +sir +mom +##irl +near +##sw +##ations +sat +find +##der +##sel +##ia +world +##ower +##rew +##ject +##fect +give +cap +##av +why +mother +gu +gen +##ict +always +##so +##ling +answ +##pped +soon +##iver +##den +form +heard +war +thing +something +seemed +sub +god +door +girl +##ange +##ph +##und +wind +because +diff +mon +going +##ertain +##de +told +home +ke +##orm +got +##ither +el +##ret +##ock +sur +high +##ched +eng +gi +ma +ph +##ank +hands +moment +woman +follow +##land +dr +ve +vo +feel +##aut +##ning +poss +ev +##ical +##lt +##ple +boy +return +rest +reg +knew +each +inf +conf +oh +sil +##ner +kind +cle +##ush +hel +rel +##orr +tra +##ual +ass +wo +##ars +##cept +belie +bre +certain +against +water +##uck +exp +hard +prop +years +del +unt +quite +beh +side +##iness +##ined +appear +four +few +##ool +done +better +##ited +half +min +##ail +tre +ref +also +##cy +##oub +##vent +began +##most +##ows +ter +##ully +##act +enough +pur +##ised +dra +act +dear +having +seen +gener +lady +hum +##aster +##ared +##aken +##ash +##oke +hour +inde +##ps +##ying +cur +called +##ier +##ason +does +whole +##uth +morn +turned +pleas +##ial +gave +betw +##ense +occ +##aps +sou +course +##ib +ins +##gether +##ream +##ired +both +add +poor +col +read +bla +##ten +##selves +ste +among +between +keep +##ular +beaut +matter +care +morning +white +##ained +sure +name +##ged +toward +##ute +##owed +small +whom +par +exc +sk +pa +lar +felt +##ks +##ating +##osed +##bo +myself +open +six +es +##ru +herself +however +##xt +##ond +hund +stood +hundred +##plied +sent +inc +round +##tered +sim +##ife +##ates +##ters +days +almost +point +##eng +gre +eight +words +hor +fam +stand +land +##dden +##ung +mean +sun +##ird +##haps +##iet +perhaps +##ures +art +sle +ed +##iss +best +dark +sudden +hear +bu +replied +bar +voice +met +##ton +che +anything +till +underst +##ren +##its +until +black +##oud +looking +cried +##ned +suff +aw +##ins +prin +red +##amp +less +lay +next +law +##end +power +##vel +fact +##not +brought +##up +##ately +##enty +country +help +##als +sn +quest +wonder +use +vis +laugh +##med +##fully +together +air +unc +need +adv +wish +##ove +person +indeed +since +contin +##oney +gent +present +har +forg +full +run +rather +ra +##ok +cond +ide +sy +##eter +##rib +feet +fin +fall +##cei +five +mil +talk +##ater +bed +##ured +doct +##sh +reach +interest +lord +gone +##tle +death +consid +##fort +serv +wife +doub +adm +##ief +##pr +##ene +taken +others +hope +cannot +speak +themselves +along +ext +dire +bes +ent +master +##od +##ued +gentle +##lic +##uring +fire +large +pol +##ention +##oy +thus +##iled +sec +##ets +play +real +prom +##ments +##ield +bus +differ +##ison +thou +##ember +##ead +##ches +reason +thous +##iting +brother +##og +##akes +thousand +money +dep +answered +true +##unt +children +behind +sound +##ants +##ably +wood +##used +whose +ele +twenty +mer +rec +believe +inv +hon +book +dec +opp +hold +##ude +fair +passed +sch +##ided +##ities +##ength +##cious +sing +##cover +##ys +nature +##ering +alone +dead +##app +sever +cat +close +gold +start +hur +fur +##ont +doctor +ask +son +ground +pat +sea +strong +leave +compan +##illed +##ept +##cond +##ides +obs +fell +##body +doubt +sign +earth +remember +fear +given +##ances +therefore +ir +##oc +##ually +deep +##iful +##ressed +body +often +##du +pap +##ane +short +second +tem +##owing +##ceed +stre +fort +##mb +##ale +horse +mad +chr +order +const +ten +clear +##irc +fif +eas +##vers +bur +really +spir +##ceived +##read +##laim +ill +##osition +bra +understand +list +abs +spr +##az +pain +question +##anger +everything +aff +circ +wall +coming +##cted +##ching +##ready +above +already +least +expl +##esh +step +used +prince +itself +##ister +necess +case +around +tri +##hn +soul +##ized +suddenly +lad +evening +general +mag +num +##imes +known +quick +captain +##ific +whether +lear +within +live +times +expect +state +friends +swe +##gn +bring +##alked +sort +##men +women +table +meet +john +sum +returned +held +dri +big +walk +##eth +says +lea +##cer +obser +else +mid +during +##ident +mus +##ape +beautiful +ord +either +town +lost +sight +sleep +##vern +means +foot +cut +cal +kept +##ety +##gar +##ience +prof +fellow +dream +mist +past +became +pret +bad +##ages +stra +making +##ught +##ey +##ution +##lish +object +est +##ts +towards +dev +human +manner +strange +low +##old +##ott +##ient +##gs +##ines +ann +##here +ang +daugh +##oon +city +rich +sometimes +##aught +##airs +rose +dem +##ility +ready +ru +##cing +##itch +pal +road +ran +corn +thy +cold +##ians +##iew +##aring +acr +bit +won +didn +christ +squ +truth +##mer +##orrow +##nded +thir +soci +perfect +subject +##br +ship +mur +answer +pie +disc +arr +free +business +river +##arr +cast +family +##ountain +hus +jud +letter +##ony +hair +arms +##oken +##ote +become +sit +ut +ah +##ingly +##ouble +val +##aint +natur +possible +several +fine +##ffect +lead +express +sus +glad +arri +blood +##itting +##rie +quiet +able +##ert +idea +##iously +##itted +##bed +charac +##ah +begin +chur +story +eye +##band +##ative +consider +across +##li +except +equ +tou +##dle +cent +##isf +sto +partic +diffic +window +surpr +prov +grand +##use +flo +pen +ca +govern +wild +soft +cross +different +effect +##ased +def +longer +discover +followed +sold +thee +husband +fre +##antly +##by +exper +##ards +pub +cor +enter +win +suppose +##teen +respect +hast +##ices +proper +sal +comple +show +heav +##cul +happy +creat +taking +##essed +stay +requ +appro +spoke +daughter +wee +##une +##lly +##alth +anim +din +green +bird +mere +##ny +cra +gard +##empt +##ms +draw +reached +##ov +appeared +cover +##ley +exist +##ug +##ination +pretty +hea +opened +##anged +slow +##ize +imag +saying +##utes +common +tom +mor +coll +ap +occas +##ames +##ench +mountain +satisf +sense +succ +fav +##oura +##ising +rock +accord +scar +break +month +continued +stri +contr +church +tree +carried +spirit +##outh +cry +##ump +wanted +certainly +secret +command +##be +dest +##lled +move +lu +street +account +ought +prep +sens +##ats +##ement +purp +##ots +##riage +tear +esc +##owers +breat +dro +fan +character +##res +##ime +decl +##medi +stud +school +##itude +sla +quar +heaven +feeling +##son +sad +regard +bright +worth +##ney +dut +hot +hung +##laimed +##ric +disp +trans +delight +reco +##iling +rid +##eric +blue +hill +cloth +div +mot +disapp +##idence +produ +sweet +success +##ury +fast +english +ye +dress +port +neither +court +var +seven +##amed +##ains +princess +lived +view +##tern +##ushed +immedi +self +chief +hours +mill +exam +##wh +ce +tried +##day +entered +##ask +eat +carry +import +bear +danger +##ledge +offic +du +cause +none +forward +##umb +uncle +number +det +further +##gr +pay +##gin +added +eff +particular +deal +queen +prot +led +front +acqu +pray +##ored +happened +alth +fight +walked +later +##bit +joy +intell +##ster +comfort +prob +maj +##ening +public +bene +##uss +##ship +although +fig +direct +##ger +##iar +bat +mouth +##ality +mem +seems +sho +##side +##iam +sol +yourself +fore +##to +doing +living +desp +##oof +##pend +field +received +shad +bey +beyond +phil +line +visit +##inct +party +garden +hall +vi +##uff +americ +##ism +mat +gentleman +burn +##ested +laid +##irt +##raid +occasion +entire +age +sister +repe +##ously +prison +accom +whis +nearly +##ndon +trees +##cri +ge +eighteen +send +##wards +early +breath +lab +forth +##ior +try +##ones +med +terri +da +##ilt +princi +enem +thinking +##where +chance +minutes +cre +anx +mary +##anced +pict +wait +vill +afraid +crow +smile +##els +england +late +pleasure +bank +##bert +##ming +faith +super +news +fle +seeing +rob +neigh +##oint +##iment +hat +auth +bill +plain +trouble +silence +aunt +exclaimed +shar +coun +er +##gy +miles +##ply +glass +drew +neighb +mine +pract +heavy +standing +sevent +change +necessary +chap +##pping +purpose +concer +length +inqu +bell +##iest +natural +deter +##icked +hardly +top +caught +##ues +strength +##ives +##ounded +##aw +##ession +circum +fing +stopped +av +opin +##ament +priv +instead +##ences +##pper +grew +loved +island +knight +ago +inn +peace +watch +##inary +##iced +third +beauty +##uct +emb +conv +approach +determ +ver +##ler +##ining +##hood +##king +convers +straight +besides +position +thirty +##iter +den +attention +south +##ager +##ically +hist +##eless +comes +##aged +clock +force +plan +sitting +please +dog +whatever +##ugg +inform +hop +chair +lie +##form +build +bab +##ma +##ustom +je +girls +rom +french +struck +pull +wrong +knowledge +grace +##oom +scarce +resol +thoughts +attempt +fifty +rap +box +getting +company +crowd +##tained +##asp +##ils +class +die +drink +dinner +exact +enj +thick +saint +slight +save +maid +immediately +born +main +##ius +##crib +rev +remark +ast +##oose +north +##ists +tears +treat +##usion +meant +boys +fat +##ls +slowly +author +incl +lim +died +infl +fool +fish +sugg +post +accept +##ern +##elve +##ceive +##cient +stone +boat +paper +fright +conse +jour +ty +enc +quickly +youth +##ging +van +vict +##ified +belong +warm +##par +fix +imposs +forgot +beside +margar +leg +tone +camp +bound +makes +margaret +##mon +brown +##bs +##val +moon +##rupt +gather +lips +placed +circumst +food +filled +##iers +##ift +desire +##ee +london +distance +forest +floor +obl +##itions +spring +jack +viol +##ics +pock +following +remained +ger +grow +snow +government +ball +hors +##ented +##aded +broken +laughed +##ann +##oured +refle +safe +##ries +hen +##ively +##itten +depart +neck +easy +thank +##ope +##dom +##ham +knows +waiting +remain +hid +fresh +und +anne +dread +ash +##arch +result +proceed +gaz +##uc +shout +##dd +started +gall +complete +whisper +wide +horses +##isted +##cially +##ges +service +piece +##eness +forget +palace +guard +doll +##erv +talking +yell +##cial +lift +fun +grave +week +impossible +effort +##nding +imm +army +difficult +##let +##reme +stop +mess +narr +deg +incre +corner +society +path +weak +shut +hy +teach +cloud +##ency +##ivid +demand +##ended +##unk +nine +recogn +##cell +##fer +turning +sky +ori +##asure +##olog +opinion +sin +beat +single +dw +fly +lang +mass +outside +kiss +trust +occup +##uit +below +appearance +silver +bro +west +aftern +gun +couldn +ant +conn +showed +conversation +running +mention +nin +greater +music +##ases +##ises +##fection +mark +espe +##ror +##ories +une +spot +##ourage +peter +##ork +##anch +presence +winter +battle +hun +morrow +probably +##ection +clothes +fash +compl +wished +appar +married +arrived +suffic +eager +former +giving +pop +##pic +sand +usual +relig +simple +sym +##itation +neg +sor +moved +months +speaking +silent +cab +pun +expression +covered +afternoon +pet +##aped +occur +states +##rief +admir +##light +easily +##unity +phys +tender +beginning +duty +chapter +influ +##zz +instant +polit +enjoy +sick +tur +stream +figure +##ald +##ract +jew +vol +minute +pleasant +scarcely +conscious +terrible +kill +concl +pale +raised +##oop +fashion +thor +##yal +##ze +twelve +leaving +##ature +mel +fut +threw +flowers +attack +trying +##cked +sword +tall +marry +expected +according +##uel +forty +stick +temper +iron +oblig +train +rapid +ladies +learn +especially +##ipped +##orted +isn +##amb +evil +bread +learned +##orth +office +surprise +write +purs +written +killed +star +consequ +exh +places +condition +direction +##jo +entirely +action +##ding +##wise +##ibly +##mit +happiness +id +decided +golden +langu +##uous +note +bott +unless +books +fal +##aled +wonderful +shook +##ounds +##ech +##onel +perfectly +geor +sail +##ume +broad +closed +##atic +latter +steps +utter +merely +history +##gan +##osing +wise +wouldn +understood +middle +prev +enemy +sole +gar +simply +tast +shot +board +ring +sav +bow +##io +reply +##gress +changed +excell +grass +finally +##ital +suc +summer +sharp +fortune +experience +succeed +##uted +##retched +journey +observed +##osp +afterwards +##fast +##ec +earn +persons +lean +broke +deser +fit +mir +marriage +##acy +repres +bitter +lying +cou +rain +trave +listen +situ +##anc +##cience +##ected +origin +wit +##olute +##oses +calm +lives +escape +beneath +##ouses +clim +hes +ven +repeated +pocket +##lling +liber +oper +tail +whe +passion +dick +mut +becom +dar +allowed +famil +ow +##ivers +##mn +gate +spite +health +##str +##ission +ign +cand +habit +empl +strugg +firm +sorry +##under +madame +furn +sigh +future +##umed +nice +separ +presently +circumstances +##reh +##ples +sett +loud +worse +corr +wand +spread +letters +magn +disg +passing +pleased +darkness +##come +civ +##cies +tea +apart +disappear +important +##erable +legs +nation +delic +dressed +game +extra +walls +dry +virt +dim +rub +##idently +absolute +blind +discovered +exactly +dam +##oved +unh +cost +arch +employ +##velop +cous +beast +opport +hu +dropped +pack +subst +smiled +chee +resp +promise +##urs +host +creature +yellow +cla +notice +priest +knowing +heads +##esty +cook +shore +##ishment +individ +existence +determined +##lend +storm +seat +ears +##ours +mount +ur +##ina +conce +##anging +fixed +press +wear +major +ves +pros +##iod +motion +empt +leaves +bottom +arg +##iety +##sy +##enth +nobody +pick +##amber +##que +bag +intellig +stir +assist +dom +advant +tired +vent +##ument +##ashed +dan +##action +wrote +considered +houses +suit +cheer +clean +fla +lou +##path +##rel +cho +castle +frequ +drawing +luck +##pan +##cent +##idge +##ulty +ones +noble +splend +honor +paid +speech +estab +faint +##ires +individual +vall +##inite +##rodu +birds +allow +impress +jane +##my +song +touch +various +moder +believed +##istr +mod +somewh +period +shadow +affect +mal +extreme +judge +village +royal +somewhat +lower +##go +##zen +agree +remembered +aston +declared +##ays +ham +parts +colonel +##cil +breakfast +shr +tow +drop +surely +##ayed +##uded +scene +describ +comb +greatest +influence +custom +animal +##itary +sake +perform +soldiers +##iny +ancient +##ests +evidently +drawn +looks +ways +harm +destro +revol +##nes +##ator +##anted +picture +##ption +likely +laws +holding +difficulty +courage +inj +prevent +mort +troub +burst +angry +proud +spoken +##ron +##head +##ision +attend +watched +east +horr +pot +##na +agre +private +##bing +shop +jul +finished +anxious +##cise +fifteen +social +dism +wine +ideas +murder +mater +george +space +scr +inside +gentlemen +civil +##iently +formed +fol +fail +constit +lot +goes +thin +surf +servant +ourselves +fallen +fancy +henry +##ium +kne +advent +carriage +baby +tong +everybody +commun +##itive +ine +waited +grou +##gged +vain +##arked +##otion +favor +bal +speed +guess +narrow +windows +carefully +ice +jim +noise +meth +hero +william +pecul +promised +walking +forgotten +obliged +earnest +lose +imper +vessel +grad +glance +shoulder +animals +noticed +knock +peculiar +##cks +quietly +cup +ta +serious +tremb +generally +american +france +exer +##iction +property +stranger +shoulders +##osity +sympath +bod +flor +cow +dust +affection +conduct +##rop +##raph +higher +handsome +##iant +##nday +recover +wel +poet +pure +chamber +distinct +gh +mam +pier +grown +##uly +rough +##enced +desce +murm +##stem +personal +##osite +offered +##onsie +edge +built +whispered +skin +pieces +##arily +##itated +suffer +butter +faces +spent +##ables +interrupt +##how +##isters +develop +unk +fond +coat +touched +hol +jump +pi +##ingu +wasn +surprised +race +##af +##uce +##oted +dish +##ob +smo +educ +brave +fer +##over +weather +eld +wealth +system +german +false +##yn +##aked +cousin +##key +ben +prepared +##wood +fingers +surr +intellect +##ipp +##ites +opportunity +introdu +lovely +rag +crown +matters +companion +alar +innoc +ris +sorrow +##hern +quarter +band +lake +mix +meaning +mountains +memory +dor +##ring +##leep +##ctions +frank +finding +mach +charge +pan +gray +avo +inh +grat +greatly +dare +asleep +offer +dav +alto +##ounced +opposite +##ett +##unch +breast +bare +reading +altogether +writing +##pected +degree +exec +##fortun +hath +##eal +feelings +rail +##night +honour +fate +##attered +meeting +##ilar +alive +hide +cruel +blow +hurt +loss +defe +thrown +caused +##oned +seek +serve +profess +duke +bent +##kes +united +kingdom +situation +empty +wis +liked +swift +opening +servants +bush +suspic +freed +##ointed +surface +questions +##aff +curious +ess +accompan +christian +fill +wid +paris +satisfied +ere +pity +sides +##nds +equal +height +##osop +ordered +depend +grey +listened +##pet +##eer +rejo +capt +##ibility +despair +mart +happen +hurried +dollars +lin +loc +language +unf +yours +supposed +laughing +settled +talked +rode +perm +distingu +hurry +lifted +destroy +value +taste +##ores +##ners +square +vast +render +labor +rul +roof +stock +telling +papa +study +##ara +pin +rising +tal +sufficient +due +rise +##eds +forced +philosop +nose +sixty +egg +stat +fault +##asc +station +##ille +distur +regular +kitch +special +honest +building +shape +season +pride +smiling +orders +accompl +sacri +usually +watching +impro +##ushing +passage +valley +material +stairs +remarked +frightened +woods +wed +catch +mistress +directly +##he +prete +gloom +lines +justice +doesn +suggest +diam +growing +bless +##like +eur +##erence +ordinary +gathered +tit +threat +weeks +method +ale +ange +seized +search +discuss +ble +##ify +ignor +deb +##eping +supper +##uses +##ral +##bur +##chief +march +bold +putting +explain +##ik +nearer +york +affairs +contrary +##yr +##ishing +moving +streets +bir +##icated +wash +kitchen +bas +cit +##eder +imagine +prefer +possess +##rance +deliver +##imate +receive +monsie +por +twice +monsieur +##min +hole +sacrif +##hip +rever +reck +process +favour +dign +##aly +europe +unw +working +heat +##antic +cler +proved +##ades +pra +##lin +arth +lod +prove +##ception +pulled +pit +##ette +authority +jer +##ole +haven +movement +##tering +##usted +engaged +brothers +advantage +ber +##lished +arthur +stones +farm +difference +aside +hence +possession +observ +##etic +exte +soldier +hills +fortun +##uls +instance +ren +##iding +holy +color +flew +rate +wal +doors +trib +busy +lat +address +familiar +weight +played +##set +flower +aware +solemn +di +liter +test +indian +emper +distant +interesting +bull +thorough +ol +wore +worked +amb +explained +excellent +splendid +tongue +named +pard +shame +sam +franc +moments +trium +wil +##eav +myster +seated +spect +instantly +unkn +endeav +similar +naturally +triumph +##cr +measure +dogs +amount +elect +officer +##itable +uns +necessity +rank +##theless +brit +confidence +vir +##illa +##ulous +nevertheless +agreed +hearing +##ancho +takes +##ental +univers +fier +earl +keeping +drive +produced +aud +aug +cott +names +##con +disappeared +##agn +throw +president +gods +magic +prisoner +represent +unknown +##por +##ague +terror +contain +bri +smoke +support +detail +wicked +flat +yester +artic +dull +falling +##vey +worthy +liberty +##ulation +design +wants +evidence +companions +spirits +bid +##bers +coast +thro +mighty +particularly +witness +##ky +clever +sought +span +rare +officers +##lv +##ford +##zy +yesterday +powers +yield +organ +amaz +pointed +claim +content +possibly +cool +hom +shock +preser +terms +dwell +persu +drove +ceased +acquaint +occurred +lies +gri +gree +tor +otherwise +finger +emperor +bask +stars +knees +pair +paused +##ruction +failed +required +unhapp +diamond +aut +rat +ali +double +forms +gives +##race +bob +eliz +travel +carrying +gle +##iles +teeth +freedom +shown +forb +fruit +##umn +waters +hearts +ener +pill +##soci +example +upper +milk +affair +softly +moral +foreign +charles +excit +huge +##oes +possessed +cases +fet +##arther +counten +report +principle +inhab +proof +intended +satisfaction +shouted +farther +group +der +##asm +unfortun +unto +singing +arrange +religion +rocks +seventeen +james +works +buy +america +succeeded +rooms +events +leading +majesty +gently +dance +tele +##using +##ached +ga +##father +key +mentioned +seventy +ros +suffering +record +cabin +invent +diss +##ival +demanded +excitement +associ +progress +##angers +##road +curi +##erous +rule +vig +clearly +sheep +regret +neighbour +##iance +instinct +awful +advice +ban +fully +sen +##ena +##mend +papers +hidden +smooth +birth +hither +##bor +stuff +impat +calling +fourth +dreadful +ain +grief +pos +##pap +##ese +brill +powerful +presented +fairy +shoot +##ker +proc +joined +afford +##iments +shel +considerable +obey +interv +voices +interested +card +conqu +cred +needed +cord +pounds +wag +advanced +undert +resolved +##istic +excited +paul +bath +asse +conditions +pictures +##gen +shining +sunday +served +steam +police +sprang +jes +salt +fields +bore +##sie +indians +fierce +ride +nodd +aband +desired +edward +importance +information +hosp +memb +perceived +task +yard +##mond +crit +control +##ternal +fold +##ensive +sooner +merch +dur +suffered +absolutely +citiz +##isc +iss +##illy +completely +cart +rad +##ois +##hold +share +willing +devil +ships +imagination +superior +ep +##ancy +feather +env +anybody +stead +dawn +##asped +occupied +curiosity +fox +yo +##inal +##asant +##asons +##ams +myst +smith +##room +type +apparently +couple +extraord +welcome +bay +daily +modern +dying +##pir +feat +accident +##ension +countenance +##iot +##llen +##ortion +crime +mult +##get +reb +alas +refused +inquir +hate +whenever +thanks +regarded +suggested +##ulf +actually +pardon +size +##gment +cult +flesh +hotel +bodies +shell +##ishop +truly +betr +##with +##uced +rapidly +odd +##alous +club +horror +mile +san +sour +flight +crossed +slave +professor +oce +rushed +excl +worst +##ization +event +corre +science +##sed +##ainted +stepped +midst +foolish +##ha +##thus +brain +##acle +cottage +buck +brief +gain +tight +expressed +drag +equally +education +august +dun +nay +##com +##abel +tempt +picked +taught +elsie +lands +driven +political +resist +resem +instr +cell +lack +barb +dangerous +ken +keen +##hy +store +dozen +trade +points +ninet +sty +nob +##ability +crim +interp +relations +delighted +members +sisters +##ially +belief +asking +anger +park +wet +plat +relief +##ification +hunting +alex +cath +##aries +meat +obst +behold +mistake +inquired +remarkable +cop +nerv +##now +careful +cere +wounded +##ading +enemies +gradually +pu +##for +thorn +worn +interrupted +stup +silk +sixteen +##racted +flour +sylv +lone +nap +##kins +##asing +depth +dreams +##esp +kindly +##reg +##acity +eleven +invol +severe +##ocr +oak +wings +hal +##lace +begun +distr +##while +benef +moreover +exped +independ +artagn +buried +approached +artagnan +saved +sex +species +harry +physical +thunder +glory +creatures +newspap +sang +##kin +plenty +useful +shoes +hoped +frequently +##vert +saf +princip +##oe +chest +bran +##ication +roll +lover +famous +recoll +queer +grim +charming +aim +loose +nur +ath +indiff +consciousness +mamma +enthus +##ips +slept +fighting +enthusi +dig +##acc +##mas +##ids +stage +anyone +thrust +desper +drunk +lamp +##rant +bew +##stone +stern +evident +meanwhile +forgive +accepted +ocean +bot +tot +wondered +playing +hyp +hale +knife +closely +meas +proceeded +message +mour +##lied +sell +union +##ustomed +exceed +precious +century +unex +kat +##oops +##aming +everywhere +minds +extraordinary +arose +capital +entrance +recall +burning +magnific +##stand +plant +neighbor +assemb +lest +##orious +##uments +colle +virtue +retreat +##iana +capable +assured +##ograph +constant +governor +increased +horn +rush +removed +facts +absence +explan +ack +mud +fever +somebody +awa +admit +jealous +kissed +popular +hut +##pelled +grant +indign +friendship +sympathy +ug +##iable +thom +alice +level +objects +pressed +qual +begged +hind +highest +clouds +ghost +acknow +##oused +emp +wishes +##oking +strike +becomes +trembling +##gu +kindness +accordingly +throat +stretched +frag +confess +grandfather +wheel +bon +russ +deck +choose +eighty +##pher +miserable +contempt +##igned +military +basket +##nal +blank +listening +ahead +fro +##elt +larger +divine +##cription +tar +##iber +stories +reve +##izing +excuse +innocent +aid +et +hadn +remind +slaves +ara +##stairs +reward +manage +smell +throughout +waves +judgment +arrival +choice +unhappy +##astic +curr +advance +informed +impression +mysterious +acquaintance +ly +notes +comr +indust +ended +lights +nurse +bought +wra +##itor +spo +marked +scream +uneas +delicate +weary +##estic +join +prompt +experi +hungry +flying +pow +bridge +##aker +##umber +visible +understanding +crying +avoid +tis +stiff +sounds +bowed +caut +goods +david +stupid +##ancing +unable +bos +spark +amid +somewhere +infinite +ashamed +dignity +gay +vic +##ka +hollow +emotion +parents +admitted +hint +##pse +temple +comfortable +intelligence +virgin +bearing +##ldom +hers +##abeth +remains +contem +settle +immense +cher +weap +##ulated +lighted +baron +##gypt +elder +thoroughly +egypt +##ilst +##anges +friendly +rome +meal +intention +##achel +returning +billy +convin +ox +price +sup +rope +project +escaped +hopes +##ology +elizabeth +safety +wound +unus +##onscious +minister +horri +helped +secure +plainly +seldom +thinks +fellows +mood +pushed +exhib +##inging +fairly +pear +##irm +maiden +##mother +##ora +##orous +appears +cheeks +ease +skill +extent +practice +religious +becoming +features +pink +tied +##ential +whence +somehow +displ +greet +faithful +concerned +bishop +##isher +gro +eagerly +eating +waste +fem +bride +##ceiving +concerning +original +accustomed +recom +patient +arranged +##ointment +##ville +##iture +ate +wholly +older +mont +colour +provided +eggs +##ration +partly +prospect +##rees +fee +fled +laughter +branches +##right +whilst +slipped +violent +inhabit +sons +##bb +##yer +engage +uncom +deeply +substance +tale +tiny +yards +##icks +hastily +vague +hell +amuse +wolf +hans +##illing +supply +constantly +christmas +million +mental +wes +washing +log +corresp +nodded +correspond +rust +##oms +obtain +unconscious +prepar +roman +struggle +established +##ws +bee +hunt +##very +signs +uttered +constitution +cave +crew +##well +##reci +quant +##aud +image +merry +treated +efforts +lonely +nut +spare +glanced +portion +##ax +withd +prim +mead +feast +cliff +emer +proportion +consideration +haste +gaze +savage +resemb +##ffe +coach +conscience +mercy +exha +consent +##bits +##ators +##urd +adop +amongst +tower +hanging +rab +##oul +compar +##lessly +circle +entertain +brilliant +gained +row +troops +repro +dared +lion +ming +scot +joe +helen +bringing +anxiety +consequence +establish +fought +sout +##eem +sha +##chan +pred +scra +glim +victory +##field +obv +winds +restr +bol +##ode +##lect +##ourse +handed +indul +patience +jose +rolled +##udd +setting +profession +sylvia +##utions +feared +clo +boots +forehead +principles +rig +sink +##ture +purch +gazed +employed +murmured +##fl +##ashing +##ural +exch +trad +active +bottle +rage +##aval +treasure +invest +lux +protect +encoun +hang +sank +beard +##vere +loving +obtained +native +instru +waist +relation +discovery +nervous +tort +trail +distinguished +ach +pig +stared +print +misery +##cip +guil +warri +jumped +sac +swim +approaching +suspicion +iv +rum +managed +match +guilty +sar +wretched +bree +kings +theory +sher +shone +showing +##atherine +throne +robin +##utation +addressed +protest +admiration +troubled +ugly +flung +subs +relie +thousands +lan +od +wooden +member +sacrifice +apr +##aling +##urity +##acles +farmer +needs +condem +bade +reasons +swall +shadows +dancing +nineteen +style +manif +grate +appreh +desert +engine +terr +wisdom +jesus +pir +##ols +preced +interests +trial +nest +unl +phr +voy +illust +##iven +sear +worship +ay +adam +principal +##more +spend +stands +respons +hit +ner +whist +flash +kinds +require +described +angel +precise +proposed +produce +utterly +##ulse +novel +blame +credit +ja +pause +##ses +##osen +household +armed +follows +##upon +ninety +unb +block +##ivity +refuse +musc +sensible +national +cam +grie +##othing +reven +##plic +delightful +melancho +extremely +melancholy +lunch +enorm +spec +slightly +recognized +mystery +sacred +compreh +distress +useless +trif +philip +mounted +energy +explanation +pour +##xed +##atory +haw +chosen +everyone +##umbled +solid +correct +appreci +resumed +alexand +aven +ech +wing +intense +highly +schem +lucy +departure +agreeable +exercise +ward +dece +max +request +unexpected +joseph +cas +nort +##col +convent +leap +attract +refer +victim +lit +##ndy +forces +sport +drift +sighed +wedding +british +attitude +vision +pipe +southern +manners +flood +commission +guide +observe +concern +affected +##fall +##onstr +stret +poison +directed +medic +gest +younger +rested +confusion +parli +continue +absor +centre +##conom +horrible +bark +gown +bye +aloud +feed +careless +profound +cheerful +torn +orn +unp +sentence +mistaken +figures +accompanied +catherine +econom +##rated +##lot +##uing +shaking +atm +##ella +council +ends +knights +atmosp +shade +brow +refr +sentiment +subjects +duties +recovered +##oll +locked +composed +swept +reality +singular +assistance +louis +##vereign +wake +sovereign +loves +absurd +souls +riding +immediate +labour +connection +cheek +magnificent +prud +sugar +adj +dise +plans +leaning +surrounded +ling +proble +strict +##uding +##ailed +awake +praise +convinced +##zing +rele +frame +trot +breaking +curtain +stayed +craw +divided +inclined +previous +##edy +stair +sees +pron +glo +##board +complex +prayer +pierre +unfortunate +genius +banks +increase +sufficiently +revolution +aust +countess +sleeping +devoted +utmost +market +bosom +##ev +##alt +handker +admin +midnight +collect +senses +connected +ident +permitted +fil +gift +theat +brid +faced +desir +horiz +college +handkerchief +nurs +##isions +reader +polic +announced +nons +##ales +##ounce +fleet +ragged +##ped +##icate +coffe +parties +reser +nights +delay +sounded +merchant +cities +##zed +##owned +appointed +instit +striking +guns +astonishment +parliament +pil +page +##atched +##itate +##iform +plate +straw +thirst +##eld +asce +ruin +negro +inhabitants +rib +##inate +addition +argument +tad +##ox +##arf +##igg +scen +guests +wondering +acquainted +ax +nich +intent +##pless +treas +destroyed +coffee +##inent +tos +##lebr +sob +gets +assure +raise +valu +sharply +privile +prosper +alarm +machine +##gue +##can +##fied +appeal +deliber +reflect +mich +lock +afterward +guest +conclusion +risk +ignorant +pian +##bury +##ghter +##uries +soil +tend +commanded +practical +easter +fisher +##sey +arist +strain +beheld +quarters +degrees +nonsense +mc +offe +mechan +##keep +doubtless +violence +folk +bul +ral +##rate +##liness +##lected +##char +##ilities +contained +impr +leaf +celebr +reflection +concluded +district +scient +intend +scorn +steep +crack +muttered +darling +adventure +##win +andrew +chim +quality +polly +travell +universal +cig +dir +##atives +##uten +##cher +subm +##aces +##omp +##ulted +leis +##quis +emph +job +surround +lev +overt +retired +irr +extended +puzz +##bour +##kind +libr +spl +realized +causes +punishment +physic +leisure +##isp +shake +charm +belonged +bones +range +##iation +##mber +##pent +##nders +prec +proph +invited +detect +troubles +katy +june +tables +##non +hay +##icip +ages +anywhere +maybe +##ffin +properly +local +improve +atmosphere +rang +shed +compass +lieuten +leaned +firmly +aver +nations +##arse +##add +wept +chin +conven +uniform +julia +##amm +##unted +commer +track +obsc +sorts +difficulties +intellectual +introduced +bushes +rout +##mith +##iday +##elia +throwing +plun +humble +polite +numerous +movements +successful +protection +candle +separate +rendered +thomas +enormous +repub +sunsh +descended +unusual +fid +##ived +shows +##inder +robert +cattle +crept +deserted +astonished +arab +lap +##udes +nearest +ideal +entering +##cript +##standing +##uf +clos +ours +wherever +visited +calcul +base +gates +##sing +stamp +official +northern +##agger +##enger +painful +burned +republic +lately +ital +actions +final +startled +sensation +fatal +sore +##olic +appet +bloom +stronger +numbers +gratitude +female +western +sed +##lest +stoop +fourteen +madam +learning +forbid +citizens +rude +vers +compelled +abroad +##teenth +##arrass +reckon +francis +##house +##aine +changes +mankind +contempl +thread +stout +alt +perpet +cared +plants +weakness +apartment +accomplished +title +dart +habits +##rier +scattered +parl +lawyer +trip +hunger +prisoners +details +interview +bord +rays +##oqu +##rey +fears +##ems +rod +obed +jean +gloomy +empire +investig +grasp +##elle +richard +##atively +fanci +embarrass +slightest +climbed +coo +signal +painted +vanished +ralph +lieutenant +witch +##onies +isabel +##itty +likewise +contri +hearted +giant +helpless +actual +badly +effects +beaten +knocked +diana +mild +ray +perse +carpet +term +steady +conceal +supreme +kate +oil +rif +ruth +##inated +aspect +veget +esteem +grandmother +inex +hitherto +cultiv +source +gla +##ico +marilla +description +swear +attended +committed +countries +infer +trem +irish +squire +hastened +eaten +resolution +current +pist +pitch +##eenth +##bly +neat +##ulate +ens +ard +##icken +resc +guards +warning +spiritual +disposition +examined +results +consequently +widow +##port +abrupt +fifth +exception +occasionally +instrument +sunshine +poured +psy +##urance +anna +penet +spanish +voyage +staring +steel +check +created +ridic +pursued +shelter +qualities +psych +cries +honey +rachel +tent +flame +beloved +smaller +occasions +departed +hesitated +goodness +driving +complain +furniture +folks +execution +quantity +fulf +lamb +sof +intr +##umes +purple +##engers +leader +destruction +crowded +reflected +caval +##geon +counter +observation +alike +##acious +##ads +##unning +outl +string +desk +attached +chris +disposed +borne +safely +##uits +unm +disci +tells +continually +wandered +electric +bern +##lie +##bre +##cts +excess +unple +seriously +barn +fancied +gil +mos +wave +##enom +chain +liqu +##rily +parted +mode +committ +domestic +##vation +##otes +forever +abst +happens +venture +flag +eternal +mas +silly +##cere +##oring +##icing +intim +##quet +drank +incident +craft +operation +florence +valuable +dang +##born +##verty +##pended +trick +poverty +dearest +oliver +steadily +thornton +dil +ned +pant +beings +##ama +##olas +tristr +crist +##ewell +imagined +heavens +tristram +cy +cot +vac +##gl +##bow +##ken +##att +##cture +leather +mission +calls +separated +beasts +piano +library +swell +phenom +readily +annoy +enjoyed +##burg +walter +expedition +ju +mouse +nav +someone +seeking +eyed +glit +signific +meantime +lincol +disturbed +powder +lincoln +##bon +acts +##liest +cigar +hated +##inity +staff +marsh +##fulness +entreat +suspect +ventured +failure +philosopher +audience +pine +##ault +##uch +##ki +tones +attent +related +assumed +hundreds +washington +beach +carol +masters +dressing +repeat +tax +greek +lam +pace +##rous +##ipping +##iece +##aks +quiver +consist +farewell +insisted +humanity +heavily +sne +##fty +generous +chiefly +development +manifest +sult +granted +preach +strongly +islands +classes +dus +##pes +tomb +region +burden +summon +sadly +saddle +counsel +drinking +disturb +desperate +##hen +heap +##ola +brush +gracious +remaining +sailor +reproach +gas +pony +##worth +ton +abund +applied +families +echo +##eral +attrib +infant +dragged +attempted +anticip +describe +delivered +groan +italian +fund +##cast +##rem +heed +shrie +weeping +chuck +##eps +permit +holl +trace +propos +daughters +disappointed +##ground +probable +sincere +concealed +virginia +##isl +apple +headed +satur +reaching +services +legisl +circumstance +dwarf +noon +laun +colored +coloured +simon +suspected +fastened +abandoned +beth +anch +prize +shouldn +rights +dining +##ourable +disappointment +directions +histor +growth +acknowled +shy +weigh +chose +chann +strangers +lightly +doctr +estate +definite +wandering +dirty +##uge +gover +grav +lightning +decor +towns +freely +motive +exhausted +jen +pars +##inations +##itating +stuck +wearing +brook +perceive +flow +propri +opposition +quarrel +essential +platform +newspaper +enthusiasm +cunning +oath +pool +whit +##verse +leaped +glorious +emily +##umbling +mortal +exerc +zeal +##wick +sailed +convey +cru +series +monte +##arsh +cutting +views +washed +brig +achie +pound +outer +##umble +clay +turns +finds +trap +assert +fatig +bitterly +rules +gal +gif +##ea +##ipe +ench +cock +overcome +disag +marble +gazing +anti +mixed +critic +hed +hook +##dered +toby +inher +branch +desires +acted +falls +article +##net +##rim +##work +##inction +whirl +##ica +notion +##ails +slee +shortly +variety +dragon +indiffer +grateful +##iveness +nicholas +gor +non +##smith +knee +misfort +##ression +smoking +consult +endure +maintain +belongs +fetch +experienced +misfortune +thither +beating +timid +owner +slip +silently +deny +hiding +disgu +ambition +mingled +##keeper +apt +tick +boast +boats +flut +slain +objection +duch +pockets +volume +respond +frequent +disease +wre +##osure +contrib +writ +reputation +fearful +roses +##yard +##iel +trim +afric +metal +##ribly +recon +selfish +apparent +trembled +doroth +swiftly +wire +##uine +column +incon +invitation +formerly +fed +jun +ped +thine +##ares +italy +frog +pointing +poetry +suggestion +depths +bench +gw +jar +tin +shakes +sheet +mischief +lesson +treatment +fasc +##ums +perple +flock +fired +mock +represented +descend +threatened +horizon +unpleasant +duck +##jud +##atch +##asper +shield +consol +monarch +philosophy +bewild +closer +bernard +bert +toler +rear +rolling +tremend +reform +wishing +touching +fountain +##gest +inner +grin +coal +community +##asha +cats +millions +vessels +earlier +hospital +barbar +jan +oct +writer +uncertain +##etera +fury +enterpr +commander +encoura +majority +dismiss +pretended +ignorance +wagon +mac +nan +os +##ean +herbert +albert +intimate +marian +blessed +confused +contrast +##grat +hatred +articles +precisely +committee +ot +button +confident +thirteen +purposes +effic +conviction +furnished +benefit +harsh +pup +##mony +##otions +orch +##olate +boil +cro +smart +missed +warn +monster +male +reduced +beds +starting +luc +acquired +mirror +autumn +gesture +batter +daz +sway +vivid +heels +sust +deem +worry +thence +slender +eleg +johnson +visitor +motionless +neighbourhood +ceremony +maxim +sultan +cream +lament +fores +stable +anth +marve +apply +##coat +regiment +incred +cheap +absent +venge +discussion +renew +scientific +jap +pac +noted +deed +decision +ruff +roads +characters +tyr +teacher +stirred +yonder +feeble +scheme +problem +bun +date +eve +##ih +reign +##ifying +schol +disgrace +priests +##withstanding +fest +##isk +heir +notwithstanding +china +mans +marks +swung +consum +veh +opinions +francs +cristo +fant +lust +rh +stom +##olen +pole +fires +prey +insult +naked +torr +##ds +##case +lily +depri +portra +embra +victor +urged +pressure +absorbed +cetera +##anqu +##lla +impl +specul +compet +##acon +contrad +declare +owing +shooting +duchess +##lies +inch +##gging +phot +magistr +inev +multitude +recollect +idle +attain +shepher +unre +compared +interf +grounds +independent +stomach +ela +fitted +seas +prose +##akened +trunk +persist +possibility +##chester +element +necessarily +modest +distinctly +clerk +wrapped +medicine +tun +shiver +marvel +peasant +applic +surre +veil +deeper +earnestly +puff +porth +tide +##lict +##ino +tob +##itory +##iring +flatter +spell +##actory +cheese +considering +susan +experiment +gardens +secretary +batt +visitors +fashioned +statement +##ocratic +jaw +wives +asc +##inkled +norm +activity +tempor +central +tenderness +jewels +performed +clergy +rabbit +bud +rud +##lers +##leness +##we +##uish +fac +begins +insist +landl +landed +finish +arrive +provisions +sending +coward +tub +##nel +##gie +##eys +ingen +fred +##rand +agony +dischar +decide +respectable +wheels +japan +dale +pall +##upt +inches +sober +pepper +score +harr +collar +##ropri +withdra +##sters +##agg +repent +##iverse +stole +gravely +uneasy +##uary +##isy +##erson +flashed +prejud +happily +guessed +feels +irres +limbs +teaching +climb +fortunate +obeyed +puzzled +porthos +wol +##ael +##mes +##ata +##isible +inse +##rage +alter +remote +wrath +distinction +awk +solitary +buildings +moonlight +persuaded +scotland +revenge +eastern +ire +losing +champ +upstairs +aged +##ache +insp +raising +lively +prayers +cloak +fow +doom +altern +##apt +coin +childhood +heartily +merit +intelligent +drops +##rence +##raction +##peare +anyhow +##agon +advert +impulse +acting +merc +constru +belonging +pursuit +murderer +linen +saturday +vengeance +awkward +bund +chase +##ifully +marched +precip +passions +breed +cease +shouting +permission +containing +preserved +scenes +##rels +##emon +peril +peas +contents +capacity +surv +regarding +blew +napole +aly +damp +eag +moss +rot +##back +##inth +##anz +reasonable +elements +dutch +literature +abandon +abruptly +disagree +##off +passengers +supported +##ieth +buff +bedroom +county +gathering +russian +##lars +chall +##opes +using +wanting +norman +career +awoke +depos +strangely +stopping +preparing +slavery +pulling +ashes +germany +##guer +theatre +bor +##ike +##ule +anyway +folly +reference +prepare +wander +louise +developed +distinguish +inquiry +warriors +indifferent +cake +danced +fence +faded +jest +rav +tour +conc +agent +clut +whereas +##rov +contented +presents +princes +pains +crossing +examination +encourage +domin +favorite +revealed +engagement +breeze +bind +kitty +tip +ult +##cest +sale +##ational +brings +gran +attracted +confeder +breathing +messenger +dimin +ba +chairs +twil +lofty +daylight +marguer +rivers +attacked +plunged +appetite +marguerite +lane +rent +tap +##nat +##mo +shirt +external +india +repair +artist +musk +gladly +wilder +cryst +calmly +blessing +measures +##place +##ura +congress +concent +serge +restless +refuge +footst +gallant +gallop +gallery +funny +hunter +owe +folded +display +condemned +surrender +twilight +today +whither +receiving +emotions +expense +dreamed +songs +weapons +ornament +gifts +ireland +##oying +shir +##osha +##ooked +quoth +handle +plot +waved +neighbors +##ological +exceedingly +relieved +administr +dot +doth +pile +wilt +##inter +##irmed +weep +encl +trump +thanked +##ensible +reluct +keeps +arrow +floating +jeff +sailors +virtues +arrangement +industry +alyosha +cyr +tie +tang +vul +brut +stroke +contract +doorway +satisfy +scarlet +##whel +tales +amazement +catholic +napoleon +roused +tig +spain +##elly +genuine +resting +laying +garments +murmur +indignation +ecc +whip +stated +mex +manuf +lovers +longed +resour +natasha +henri +feathers +mourn +adopted +##ada +contact +cleared +parad +security +summ +centuries +error +increasing +masses +struggling +alarmed +physician +tremendous +cub +ded +sul +##pel +unlike +tranqu +overwhel +perfection +flushed +longing +exposed +inspect +patch +##oonful +examine +burnt +remarks +diamonds +exchange +tobacc +fain +##oot +##uously +haz +exert +roast +##cca +##inkle +commenced +lords +expecting +demonstr +penny +##ovitch +communication +ske +sancho +##eus +##ucted +##oration +solic +##ante +clothing +disord +joke +annie +center +existed +civilization +lodge +cardinal +emphas +rifle +bone +tore +velve +sauce +deer +##aph +perf +remem +remove +retorted +secured +matth +frightful +crimson +promptly +shakespeare +shepherd +bible +daring +reject +frost +conception +comment +accur +##ensity +snake +recent +driver +anxiously +jackson +consequences +intervals +comrades +pronounced +cotton +rival +thief +##itz +shaw +fog +distrib +asser +brand +ruined +romans +recognize +population +enjoyment +homes +nancy +wret +##lo +##iated +##just +##inous +##inger +##uting +##amation +restored +sunk +transl +bells +porter +meadow +convention +bess +les +mould +exquis +unse +ability +##ensions +income +preparations +jaco +danglars +dried +goss +##jean +tops +##usions +enl +spin +coffin +scre +indicated +moist +attach +avail +lucky +defend +issue +glimpse +##plicity +dorothy +bars +stu +##ria +##izes +purse +barg +finest +invisible +opposed +palm +funeral +comrade +obvious +bust +cree +cape +fame +loo +wick +##eman +outward +perman +charity +##oubted +realize +sensitive +mixture +porch +torment +positive +issued +weapon +refresh +bald +##iate +##vering +atl +tramp +glow +charged +behav +keeper +streams +detective +sofa +goose +mul +##of +heath +sew +##ulating +goat +pluck +lawn +minor +awakened +paint +corners +valjean +glasses +hideous +martin +reminded +manufact +garr +##tte +##aign +sept +allah +##outs +roots +quix +##inking +peep +advised +murdered +completed +undoubted +advantages +eldest +rejoined +favourite +exquisite +gig +##lon +##orable +##itudes +##idents +##usive +##olent +##ints +arts +temp +defence +succession +loudly +methods +conquer +betrayed +july +push +rushing +##low +##ider +sooth +soph +roar +quit +deadly +patri +solitude +despite +##oiselle +newspapers +indifference +privilege +eth +##ari +shaped +##sell +deceived +##pled +minded +steal +landing +conducted +hopeless +vanity +trusted +suitable +holes +wilson +amusement +tobacco +jos +tum +##ram +##lance +##esy +##aments +##rious +root +loaded +images +contro +emma +passes +retain +brandon +boxes +disgust +individuals +essence +copy +chimney +cure +kid +##lets +freder +bred +trained +indic +lav +provin +apolog +travelling +preserve +jealousy +uncomfort +mate +naught +congrat +##tha +##abled +apples +whereupon +youngest +confined +sunlight +signed +tomorrow +flee +determination +planet +fishing +excellency +sarah +cards +divers +pages +##iency +##inery +changing +appe +bloss +wherefore +natured +monk +eloqu +edith +accounts +inclination +forests +groups +depends +shells +betray +resistance +responded +rand +pope +lessons +stephen +musical +arrested +miller +paying +##cribed +including +politics +idiot +innocence +accomplish +debt +hospit +ivan +obedience +au +dre +om +umb +wip +##eous +##ients +shag +##lyle +ancest +thereby +copper +carlyle +throng +letting +resent +belt +fortnight +devotion +published +department +hesitation +rubbed +pillow +lantern +celebrated +dumb +kit +##nie +##alled +fathers +admired +characteristic +punish +angels +##vest +##any +inward +stim +deeds +dost +##emoiselle +arise +##igs +load +thoughtfully +subd +mademoiselle +prayed +resolute +literary +autom +citizen +inferior +matthew +jacob +jones +nic +patter +##oked +##ston +stolen +chances +clar +loyal +disple +appropri +conflict +##bridge +englishman +limit +packed +adventures +dwelling +rigid +pistol +disagreeable +gn +lil +sire +##eur +##rid +bees +##leton +##utor +##laced +##aping +mars +plains +##rog +parting +limited +legal +ashore +jews +prevented +ratt +beef +michael +holland +quixote +dad +sie +##omin +bears +alm +facing +boiled +grows +pond +flames +scare +relative +treach +easier +ladder +practically +hoping +romantic +campaign +sticks +mutual +corrid +tossed +ascertain +##ana +##ills +whoever +universe +blows +lasted +drowned +parish +standard +sunset +elsewhere +angrily +province +authorities +risen +teleph +involunt +nerves +rescue +gang +pip +til +##aves +##itively +bending +reception +orange +##olence +clasped +bond +grove +ade +interior +dram +##gram +immort +dishes +preferred +procession +yielded +##piration +landsc +criminal +ridiculous +net +##ai +##inence +##ulations +unjust +##law +adorn +wherein +scale +seconds +collected +graceful +warmth +##after +assembled +searching +average +dusk +bost +##rom +##omon +beck +bron +aren +offering +impressed +slid +wax +generation +holds +studied +denied +avoided +pretend +instruments +staircase +administration +etern +hull +ink +rein +cannon +boot +resign +counted +sixth +##okes +illness +transfer +err +captured +rosy +instructions +glaci +map +##gent +##enn +shru +##ctive +deaf +likes +miser +##eling +curiously +artific +furious +illum +apost +fanny +previously +operations +impatient +total +assembly +encounter +april +imprison +channel +vulgar +faster +hood +##eon +##oen +##icted +frown +seize +##arts +abb +brass +amused +overl +swar +jolly +mail +followers +teleg +##oman +shalt +stores +##racy +leon +brisk +##aging +namely +sweep +floated +exists +commonly +satisfactory +training +candid +faintly +erect +tribe +unfortunately +prudence +doctrine +boston +kar +revel +anton +prow +spur +##ago +##agues +crou +blown +supposing +regions +assume +extrem +balls +occupation +archite +combat +jimmy +machinery +railway +fortunes +acknowledged +velvet +oy +tid +##eg +##orter +neph +reported +richmond +corpor +americans +embro +convict +dwelt +latin +phrase +raggedy +summoned +dangers +pave +##aved +##och +##anim +##sequ +##entine +seats +conceived +brood +aroused +##udder +glen +helm +woe +proposition +continent +speaker +doubtful +##cules +pressing +balance +fortunately +neighborhood +surrounding +footsteps +cush +rings +ted +vest +##iour +##mans +##encies +rela +doc +appla +prem +##izz +regards +confession +questioned +profit +solomon +verse +wheat +horrid +railroad +recalled +recommend +fatigue +frederick +dine +shudder +felic +struct +thresh +spear +glowing +##anks +##rup +helping +runs +johnny +summit +seventh +deserve +spani +rarely +gilbert +wilderness +september +nephew +dug +##idable +upward +##ander +tram +downstairs +residence +##ubb +warned +propose +sentiments +motor +maintained +compliment +furnish +assistant +rebell +pirates +travellers +discipl +eagle +##natural +tumult +shaggy +telegraph +bou +dashed +##forth +shores +upright +##berry +##fts +moan +repose +monstr +phoen +assurance +advancing +breathe +inquire +demands +healthy +solemnly +swallowed +undoubtedly +rasc +##olet +sails +lik +monkey +terribly +pleasures +eighth +sensations +medical +lifting +delicious +imperfect +accompany +##force +intentions +##rane +toil +##iser +##anor +boiling +##enses +instances +vex +believing +humour +secretly +breathed +creation +killing +swords +injured +addressing +acknowledge +specim +portrait +bargain +creek +threshold +fi +gos +jury +link +##urer +analy +cha +nove +##ops +canv +hasn +missing +thoughtful +slope +submit +springs +schools +violently +performance +affectionate +spectacle +envy +levin +convenient +vine +woo +##iving +##uity +##cle +##fic +ing +wasted +##celot +butt +franz +brains +blast +impart +founded +survey +bloody +cheerfully +vigor +positively +curtains +launcelot +cose +mast +naut +##ya +##itute +liver +altered +brick +overhead +##athan +owned +supplied +incap +simplicity +raw +oppress +dropping +jewel +travelled +stirring +hurrying +persuade +displayed +territory +avenue +altar +africa +cu +jail +##hand +shif +rect +theirs +ponder +backward +harder +merr +catching +observing +penn +coron +requires +forgetting +injury +magician +strictly +complexion +prosperity +dense +hush +rice +##ek +##ray +##onic +##arn +##ita +saving +friday +prime +##keys +trous +fawn +agitation +monday +consented +guilt +subl +dreaming +anguish +ruins +heavenly +turkey +potat +searched +offended +compassion +persec +gossip +landscape +jason +kin +##lock +haired +section +dew +##ements +enqu +willingly +spots +prod +poets +planted +offence +lace +serpent +confessed +##ceptible +humor +recently +##ocracy +expectation +churches +transform +ridge +creep +embrace +sympt +neglected +politeness +tribes +establishment +purchase +bapt +##enched +lip +lif +disd +flies +partner +conspic +traged +decent +irre +painting +announce +leads +journal +bags +speedily +impatience +impatiently +swimming +engineer +precaut +wreck +dazz +normal +uncomfortable +cosette +sins +##fold +bea +recei +butler +conceive +##apes +grain +humil +sket +covering +motives +connect +punished +struggled +dishon +cruelty +testim +ranks +steamer +kenned +independence +frozen +alexander +anchor +gorge +bother +jam +mity +pad +stall +hercules +##umps +ambit +natives +checked +speaks +leaders +##ovna +division +protested +shoe +limits +venus +endeavour +##regular +comprehend +challen +wretch +kennedy +cents +kick +tu +##oard +##adel +alight +locks +slay +forming +confirmed +adding +irregular +staying +function +admiral +cellar +cautiously +obviously +hedge +peasants +bret +eust +wy +##rical +##atically +##orse +stove +prost +chill +##rig +spar +twisted +coarse +poem +blowing +scott +waving +keys +##grim +superst +impressions +interpret +sheriff +mitya +feder +lur +##aur +##course +##aration +bend +sailing +prol +##ula +knot +robe +amiable +flowing +beggar +judged +temperature +moderate +##etta +derived +association +greeted +cens +gasped +##burn +##fish +##esis +tools +shrew +rebe +sem +prair +##alla +##ilus +##rian +yourselves +nowhere +carr +consisted +nighting +foundation +elbow +highness +walks +riches +cavern +memories +bills +romance +boldly +withdrew +torture +magistrate +cough +cash +hare +ost +rows +wounds +##nic +##elf +##ober +##chio +seal +marius +##angle +justified +mob +certainty +planned +liberal +vigorous +reserve +pilgrim +obscure +achieve +mansion +prodig +mun +na +vap +vote +##greg +##don +##bar +##hin +chau +aboard +twas +muse +retire +##lands +arrest +slau +attempts +tenderly +couch +passionate +employment +flora +hurriedly +unwilling +bidding +mourning +encountered +perplex +inspired +rhy +rational +##gers +seed +exagger +feature +carved +induced +graham +armies +woke +steward +lads +providence +proceeding +decidedly +##burgh +arrangements +ceremon +halted +glittering +proposal +bewildered +enterprise +elegant +embraced +bundle +umbre +embroid +fare +##nut +##ta +##mie +##verted +sheer +brist +pled +mule +glancing +consists +intercourse +breth +##pson +##ibilities +smallest +needle +lordship +madness +judges +collection +rocky +gleam +tightly +temptation +caution +verses +application +rebecca +sorr +viz +##ban +sullen +lend +clam +##roll +seeming +accused +##ucky +raoul +corpse +morgan +destiny +neglect +bowl +welcom +remembr +brethren +lid +vice +##ique +##pers +##wart +##ded +##asy +##anny +##che +attained +allan +chatter +crushed +descent +endless +assum +##ashes +pleasing +colors +symb +doubts +clearing +peaceful +balloon +combined +fiercely +meadows +improvement +perpetual +photograph +prosec +bows +pav +##sis +thumb +conj +offices +smiles +backs +mostly +drum +curse +admire +remembering +respects +harmony +dismay +observations +energ +recollection +hypnot +referred +commerce +propriet +inconven +november +pul +##itious +shan +shaken +stagg +ree +brute +agric +discer +scold +swam +supplies +mothers +sna +entit +reasoning +estr +ceiling +hugh +sufferings +dissol +severely +cliffs +sinking +stretch +bart +foul +ram +##cow +haught +fors +##alf +##ieu +abode +clad +bonds +flu +mustn +remedy +interval +soup +##licted +eleph +penc +caes +crazy +secrets +terrified +prevail +picking +educated +filling +basin +russia +marvell +landlord +irresist +inspector +disorder +nautilus +dict +lent +nom +wages +##uz +##nde +reass +fruits +##allel +dod +stricken +foe +indef +incom +awe +syn +introduce +claims +inquiries +measured +preparation +gloves +policy +tendency +prophet +moscow +disguise +ec +lun +##awn +##asion +enable +cares +##ilet +##ounted +godd +hearth +advise +harmon +maggie +commands +melted +davy +britain +duration +restrain +tradition +luxury +sergeant +bite +##za +##inating +##ulars +upset +dislike +prel +laura +tread +bestowed +awaken +arrows +dresses +violet +medium +jacket +deserved +extrav +wealthy +awaiting +trousers +kent +lions +math +wont +##uated +thie +##chet +##raham +brows +clung +##assed +peer +fogg +impuls +formal +monot +existing +neighbours +enjoying +guarded +eagerness +materials +procure +awaited +december +treasures +parlour +renewed +tiger +extravag +diver +eh +lemon +##ingham +##imer +stub +abraham +closing +##cian +placing +plague +slipp +answering +largest +editor +polished +patron +entertained +superint +robbed +julius +representative +unconsciously +commercial +paradise +oven +tenth +##oons +haul +##cel +##esses +enabled +amusing +clover +slumber +serving +text +subtle +drake +prett +equality +hasty +recognition +savages +louder +comparison +axe +route +bough +dy +ell +jasper +mol +og +pag +##oh +##hov +##inge +##ington +whim +shine +sack +##usal +##lac +facul +##ques +##assad +perish +##udge +blush +answers +beautifully +plays +squir +sandy +admirable +released +encouraged +faculty +bits +dame +fern +lic +pud +pushing +wits +##lor +##dles +knelt +amend +contest +##mended +wrought +ministers +promises +election +inval +bigger +dispute +occasional +occupy +climbing +release +tremble +conspicuous +##aid +##hire +##ania +chat +conject +##arded +lee +discl +firing +scand +mayor +composition +##neys +errand +recognised +sickness +experiences +navig +inevitable +corridor +telephone +fiery +nightingale +cows +gab +games +ket +##loc +##mouth +##vous +##hib +##ican +stained +select +##antial +##ourt +amelia +childish +hearty +formidable +personally +doubted +provide +student +ringing +situated +faults +ambassad +reports +pearl +pearls +reserved +resources +gulf +nigh +vale +##tur +##iary +##yl +##lessness +behalf +visits +largely +transport +narrative +prevailed +contains +preceding +cavalry +spargo +cel +rash +sid +vib +wer +##master +sund +##rial +##acks +flowed +remorse +swinging +restore +dearly +discourse +peters +turtle +hostile +purely +unworth +cordial +adopt +printed +responsible +responsibility +muscular +economic +traveller +phenomena +gravity +october +pavement +likeness +disdain +federal +ellen +nails +psmith +rus +##ody +halt +stool +chamb +orph +unne +cove +crumb +scent +indisp +workmen +wars +drown +veins +paces +raft +milton +##inese +valentine +included +thankful +dolls +apartments +cooking +arguments +invention +horns +professional +stooped +echoed +dismissed +deemed +interfere +japanese +superintend +bunch +dix +oars +##row +##rating +##line +##ien +##atus +##adem +martha +poly +##oths +attorn +##arks +colon +skil +surprising +sweetness +rubbing +heroes +reverence +lodging +undertake +fragments +gigantic +remembrance +cakes +##xter +selected +chem +chop +bru +greatness +access +housekeeper +winding +appearances +edison +eleanor +fairies +estim +destined +rider +executed +tempted +screamed +sparkling +globe +forbidden +deprived +elabor +permanent +snapped +caesar +unnecess +dum +fitting +nun +waking +##foot +##zil +##arian +heel +shelf +defect +##iforn +ark +clust +grasped +perch +blade +droop +colours +sunny +slew +harness +californ +squee +wildly +convuls +learnt +condemn +leslie +structure +monstrous +prolong +mask +##itors +hem +thereupon +pencil +benev +reflections +guardian +candles +bodily +conquered +sanct +uncommon +apron +pouring +weighed +jaws +withdraw +discipline +eug +lots +##ji +##atisf +##omb +stif +suited +degr +nois +leagues +plough +scri +induce +betty +charlot +nora +tracks +patiently +earthly +effective +ville +villain +dreaded +paths +roughly +durham +spoil +amidst +stamped +contrived +patriot +breton +dub +gy +moses +pang +sant +wig +##ainder +##irs +##ilton +chicken +##riages +org +nod +##anda +trop +appointment +compare +passages +pleaded +persisted +##retion +raven +lowest +lowered +democratic +product +tailor +holiday +threatening +##ikov +briefly +drunken +resemblance +muscles +deliberately diff --git a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml new file mode 100644 index 000000000000..107168fb6cbe --- /dev/null +++ b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml @@ -0,0 +1,166 @@ +name: &name "Conformer-BPE" + +model: + sample_rate: &sample_rate 16000 + log_predictions: true + load_weights_from_checkpoint: null + ctc_reduction: 'mean_batch' + + train_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 16 + trim_silence: false + max_duration: 16.7 + min_duration: 0.1 + shuffle: true + is_tarred: false + tarred_audio_filepaths: null + num_workers: 4 + pin_memory: false + use_start_end_token: true + + validation_ds: + manifest_filepath: ??? + sample_rate: 16000 + batch_size: 16 + shuffle: false + num_workers: 4 + pin_memory: false + use_start_end_token: true + + test_ds: + manifest_filepath: null + sample_rate: 16000 + batch_size: 16 + shuffle: false + num_workers: 4 + pin_memory: false + use_start_end_token: true + + tokenizer: + dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe) + type: ??? # Can be either bpe or wpe + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: *sample_rate + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: &n_mels 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + pad_to: 16 + stft_conv: false + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + # SpecAug params + freq_masks: 2 # set to zero to disable the SpecAug augmentation + time_masks: 2 # set to zero to disable the SpecAug augmentation + freq_width: 27 + time_width: 100 + # Cut-off params + rect_masks: 0 # set to zero to disable the cut-off augmentation + rect_time: 120 + rect_freq: 50 + + + encoder: + _target_: nemo.collections.asr.modules.ConformerEncoder + feat_in: *n_mels + feat_out: -1 # you may set it if you need different output size other than the default d_model + n_layers: 16 + d_model: 256 + + # Sub-sampling params + subsampling: vggnet # vggnet or striding + subsampling_factor: 4 # must be a multiply of 2 + subsampling_conv_channels: 64 # set to -1 to make it equal to the d_model + + # Feed forward module's params + ff_expansion_factor: 4 + + # Multi-headed Attention Module's params + self_attention_model: rel_pos # rel_pos, abs_pos, rel_pos_old, rel_pos_old_xl + n_heads: 4 + xscaling: true + + # Convolution module's params + conv_kernel_size: 31 + + ### regularization + dropout: 0.1 + dropout_emb: 0.1 + dropout_in: 0.1 + dropout_att: 0.0 + + decoder: + _target_: nemo.collections.asr.modules.LSTMDecoder + feat_in: null + num_classes: -1 # filled with vocabulary size from tokenizer at runtime + vocabulary: [] # filled with vocabulary from tokenizer at runtime + lstm_hidden_size: 640 + bidirectional: False + num_layers: 1 + + optim: + name: novograd + lr: 0.01 + + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # scheduler config override + warmup_steps: 4000 + warmup_ratio: null + min_lr: 1e-9 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + num_nodes: 1 + max_epochs: 100 + max_steps: null # computed at runtime if not set + val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + distributed_backend: ddp + accumulate_grad_batches: 2 + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + row_log_interval: 1 # Interval of logging. + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: true + create_checkpoint_callback: true + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null + resume_if_exists: false + resume_ignore_no_checkpoint: false + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/examples/asr/experimental/configs/conformer/conformer_char.yaml b/examples/asr/experimental/configs/conformer/conformer_char.yaml new file mode 100644 index 000000000000..24959752e20b --- /dev/null +++ b/examples/asr/experimental/configs/conformer/conformer_char.yaml @@ -0,0 +1,163 @@ +name: &name "Conformer-char" + +model: + sample_rate: &sample_rate 16000 + labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] + log_predictions: true + load_weights_from_checkpoint: null + ctc_reduction: 'mean_batch' + + train_ds: + manifest_filepath: ??? + labels: *labels + sample_rate: 16000 + batch_size: 16 + trim_silence: false + max_duration: 16.7 + min_duration: 0.1 + shuffle: true + is_tarred: false + tarred_audio_filepaths: null + num_workers: 4 + pin_memory: true + + validation_ds: + manifest_filepath: ??? + labels: *labels + sample_rate: 16000 + batch_size: 16 + shuffle: false + num_workers: 4 + pin_memory: true + + test_ds: + manifest_filepath: null + labels: *labels + sample_rate: 16000 + batch_size: 16 + shuffle: false + num_workers: 4 + pin_memory: true + + preprocessor: + _target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor + sample_rate: *sample_rate + normalize: "per_feature" + window_size: 0.025 + window_stride: 0.01 + window: "hann" + features: &n_mels 80 + n_fft: 512 + frame_splicing: 1 + dither: 0.00001 + pad_to: 16 + stft_conv: false + + spec_augment: + _target_: nemo.collections.asr.modules.SpectrogramAugmentation + # SpecAug params + freq_masks: 2 # set to zero to disable the SpecAug augmentation + time_masks: 2 # set to zero to disable the SpecAug augmentation + freq_width: 27 + time_width: 100 + # Cut-off params + rect_masks: 0 # set to zero to disable the cut-off augmentation + rect_time: 120 + rect_freq: 50 + + encoder: + _target_: nemo.collections.asr.modules.ConformerEncoder + feat_in: *n_mels + feat_out: -1 # you may set it if you need different output size other than the default d_model + n_layers: 16 + d_model: 256 + + # Sub-sampling params + subsampling: vggnet # vggnet or striding + subsampling_factor: 4 # must be a multiply of 2 + subsampling_conv_channels: 64 # set to -1 to make it equal to the d_model + + # Feed forward module's params + ff_expansion_factor: 4 + + # Multi-headed Attention Module's params + self_attention_model: rel_pos # rel_pos, abs_pos, rel_pos_old, rel_pos_old_xl + n_heads: 4 + xscaling: true + + # Convolution module's params + conv_kernel_size: 31 + + ### regularization + dropout: 0.1 + dropout_emb: 0.1 + dropout_in: 0.1 + dropout_att: 0.0 + + decoder: + _target_: nemo.collections.asr.modules.LSTMDecoder + feat_in: null + num_classes: 28 + vocabulary: *labels + lstm_hidden_size: 640 + bidirectional: False + num_layers: 1 + + optim: + name: novograd + lr: 0.01 + + # optimizer arguments + betas: [0.8, 0.5] + weight_decay: 0.001 + + # scheduler setup + sched: + name: CosineAnnealing + + # scheduler config override + warmup_steps: 1000 + warmup_ratio: null + min_lr: 1e-9 + last_epoch: -1 + +trainer: + gpus: 0 # number of gpus + num_nodes: 1 + max_epochs: 100 + max_steps: null # computed at runtime if not set + val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + distributed_backend: ddp + accumulate_grad_batches: 2 + gradient_clip_val: 0.0 + amp_level: O0 # O1/O2 for mixed precision + precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. + row_log_interval: 1 # Interval of logging. + resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. + num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + sync_batchnorm: true + + checkpoint_callback: false # Provided by exp_manager + logger: false # Provided by exp_manager + + +exp_manager: + exp_dir: null + name: *name + create_tensorboard_logger: true + create_checkpoint_callback: true + create_wandb_logger: false + wandb_logger_kwargs: + name: null + project: null + resume_if_exists: false + resume_ignore_no_checkpoint: false + +hydra: + run: + dir: . + job_logging: + root: + handlers: null diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index d5636f41d6e0..c77d5cf015f7 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -16,6 +16,7 @@ from nemo.collections.asr.models import EncDecCTCModel from nemo.core.config import hydra_runner +from nemo.utils import logging from nemo.utils.exp_manager import exp_manager @@ -69,8 +70,18 @@ def main(cfg): exp_manager(trainer, cfg.get("exp_manager", None)) asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) + if cfg.model.get("load_from_checkpoint", None): + logging.info(f"Loading checkpoint '{cfg.model.load_from_checkpoint}' ...") + asr_model.update_weights(checkpoint_path=cfg.model.load_weights_from_checkpoint) + trainer.fit(asr_model) + if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: + gpu = 1 if cfg.trainer.gpus != 0 else 0 + trainer = pl.Trainer(gpus=gpu) + if asr_model.prepare_test(trainer): + trainer.test(asr_model) + if __name__ == '__main__': main() # noqa pylint: disable=no-value-for-parameter diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index e49e794e7c3b..98bbc03fcf39 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -59,8 +59,13 @@ def main(cfg): logging.info(f'Hydra config: {cfg.pretty()}') trainer = pl.Trainer(**cfg.trainer) exp_manager(trainer, cfg.get("exp_manager", None)) + asr_model = EncDecCTCModelBPE(cfg=cfg.model, trainer=trainer) + if cfg.model.get("load_from_checkpoint", None): + logging.info(f"Loading checkpoint '{cfg.model.load_from_checkpoint}' ...") + asr_model.update_weights(checkpoint_path=cfg.model.load_weights_from_checkpoint) + trainer.fit(asr_model) if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: diff --git a/nemo/collections/asr/modules/__init__.py b/nemo/collections/asr/modules/__init__.py index 1de8c01c5da8..68b972cdb243 100644 --- a/nemo/collections/asr/modules/__init__.py +++ b/nemo/collections/asr/modules/__init__.py @@ -24,3 +24,19 @@ ConvASREncoder, SpeakerDecoder, ) + +from nemo.collections.asr.modules.conformer_encoder import ConformerEncoder +from nemo.collections.asr.modules.lstm_decoder import LSTMDecoder + +from nemo.collections.asr.modules.activations import Swish +from nemo.collections.asr.modules.conformer_modules import ConformerConvolution, ConformerFeedForward + +from nemo.collections.asr.modules.subsampling import ConvSubsampling +from nemo.collections.asr.modules.multi_head_attention import ( + MultiHeadAttention, + RelPositionMultiHeadAttention, + RelPositionMultiHeadAttention_old, + RelPositionalEncoding_old, + RelPositionalEncoding, + PositionalEncoding, +) \ No newline at end of file diff --git a/nemo/collections/asr/modules/activations.py b/nemo/collections/asr/modules/activations.py new file mode 100644 index 000000000000..627eef295717 --- /dev/null +++ b/nemo/collections/asr/modules/activations.py @@ -0,0 +1,27 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn + +__all__ = ['Swish'] + + +class Swish(nn.Module): + """ + Swish activation function introduced in 'https://arxiv.org/abs/1710.05941' + """ + + def forward(self, x): + return x * torch.sigmoid(x) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py new file mode 100644 index 000000000000..60866dbbc4be --- /dev/null +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -0,0 +1,337 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from collections import OrderedDict + +import torch +import torch.nn as nn +from torch.nn.modules import LayerNorm + +from nemo.collections.asr.modules.conformer_modules import ConformerConvolution, ConformerFeedForward +from nemo.collections.asr.modules.multi_head_attention import ( + MultiHeadAttention, + PositionalEncoding, + RelPositionalEncoding, + RelPositionalEncoding_old, + RelPositionMultiHeadAttention, + RelPositionMultiHeadAttention_old, +) +from nemo.collections.asr.modules.subsampling import ConvSubsampling +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.classes.module import NeuralModule +from nemo.core.neural_types import AcousticEncodedRepresentation, LengthsType, NeuralType, SpectrogramType + +__all__ = ['ConformerEncoder'] + + +class ConformerEncoder(NeuralModule, Exportable): + """ + The encoder for ASR model of Conformer. + Based on this paper: + https://arxiv.org/abs/2005.08100 + """ + + def _prepare_for_export(self): + Exportable._prepare_for_export(self) + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + input_example = torch.randn(16, self.__feat_in, 256).to(next(self.parameters()).device) + return tuple([input_example]) + + @property + def disabled_deployment_input_names(self): + """Implement this method to return a set of input names disabled for export""" + return set(["length"]) + + @property + def disabled_deployment_output_names(self): + """Implement this method to return a set of output names disabled for export""" + return set(["encoded_lengths"]) + + def save_to(self, save_path: str): + pass + + @classmethod + def restore_from(cls, restore_path: str): + pass + + @property + def input_types(self): + """Returns definitions of module input ports. + """ + return OrderedDict( + { + "audio_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()), + "length": NeuralType(tuple('B'), LengthsType()), + } + ) + + @property + def output_types(self): + """Returns definitions of module output ports. + """ + return OrderedDict( + { + "outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()), + "encoded_lengths": NeuralType(tuple('B'), LengthsType()), + } + ) + + def __init__( + self, + feat_in, + n_layers, + d_model, + feat_out=0, + subsampling='vggnet', + subsampling_factor=4, + subsampling_conv_channels=64, + ff_expansion_factor=4, + self_attention_model='rel_pos', + n_heads=4, + xscaling=True, + conv_kernel_size=31, + dropout=0.1, + dropout_emb=0.1, + dropout_in=0.1, + dropout_att=0.0, + ): + super().__init__() + + d_ff = d_model * ff_expansion_factor + self.d_model = d_model + self.scale = math.sqrt(self.d_model) + + if xscaling: + self.xscale = math.sqrt(d_model) + else: + self.xscale = None + + if subsampling: + self.pre_encode = ConvSubsampling( + subsampling=subsampling, + subsampling_factor=subsampling_factor, + idim=feat_in, + odim=d_model, + conv_channels=subsampling_conv_channels, + activation=nn.ReLU(), + ) + self.feat_out = d_model + else: + self.feat_out = d_model + self.pre_encode = nn.Linear(feat_in, d_model) + + self.u_bias = None + self.v_bias = None + + if 'old' in self_attention_model: + self.pos_enc = RelPositionalEncoding_old( + d_model=d_model, dropout=dropout, dropout_emb_rate=dropout_emb, xscale=self.xscale + ) + if "xl" in self_attention_model: + self.u_bias = nn.Parameter(torch.Tensor(n_heads, d_model // n_heads)) + self.v_bias = nn.Parameter(torch.Tensor(n_heads, d_model // n_heads)) + else: + if self_attention_model == "rel_pos": + self.pos_enc = RelPositionalEncoding( + d_model=d_model, dropout_rate=dropout_in, dropout_emb_rate=dropout_emb, xscale=self.xscale + ) + elif self_attention_model == "abs_pos": + self.pos_enc = PositionalEncoding( + d_model=d_model, dropout_rate=dropout_in, max_len=6000, reverse=False, xscale=self.xscale + ) + else: + raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") + + self.layers = nn.ModuleList() + for i in range(n_layers): + layer = ConformerEncoderBlock( + d_model=d_model, + d_ff=d_ff, + conv_kernel_size=conv_kernel_size, + self_attention_model=self_attention_model, + n_heads=n_heads, + dropout=dropout, + dropout_att=dropout_att, + ) + self.layers.append(layer) + + if feat_out > 0 and feat_out != self.output_dim: + self.out_proj = nn.Linear(self.feat_out, feat_out) + self.feat_out = feat_out + else: + self.out_proj = None + self.feat_out = d_model + + @typecheck() + def forward(self, audio_signal, length): + audio_signal = torch.transpose(audio_signal, 1, 2) + + if isinstance(self.pre_encode, ConvSubsampling): + audio_signal, length = self.pre_encode(audio_signal, length) + else: + audio_signal = self.embed(audio_signal) + + audio_signal, pos_emb = self.pos_enc(audio_signal) + bs, xmax, idim = audio_signal.size() + + # Create the self-attention and padding masks + pad_mask = self.make_pad_mask(length, max_time=xmax, device=audio_signal.device) + xx_mask = pad_mask.unsqueeze(1).repeat([1, xmax, 1]) + xx_mask = xx_mask & xx_mask.transpose(1, 2) + pad_mask = (~pad_mask).unsqueeze(2) + + for lth, layer in enumerate(self.layers): + audio_signal = layer( + x=audio_signal, + att_mask=xx_mask, + pos_emb=pos_emb, + u_bias=self.u_bias, + v_bias=self.v_bias, + pad_mask=pad_mask, + ) + + if self.out_proj is not None: + audio_signal = self.out_proj(audio_signal) + + audio_signal = torch.transpose(audio_signal, 1, 2) + return audio_signal, length + + @staticmethod + def make_pad_mask(seq_lens, max_time, device=None): + """Make masking for padding. + Args: + seq_lens (IntTensor): `[B]` + device_id (int): + Returns: + mask (IntTensor): `[B, T]` + """ + bs = seq_lens.size(0) + seq_range = torch.arange(0, max_time, dtype=torch.int32) + seq_range_expand = seq_range.unsqueeze(0).expand(bs, max_time) + seq_lens = seq_lens.type(seq_range_expand.dtype).to(seq_range_expand.device) + seq_length_expand = seq_lens.unsqueeze(-1) + mask = seq_range_expand < seq_length_expand + + if device: + mask = mask.to(device) + return mask + + +class ConformerEncoderBlock(torch.nn.Module): + """A single block of the Conformer encoder. + + Args: + d_model (int): input dimension of MultiheadAttentionMechanism and PositionwiseFeedForward + d_ff (int): hidden dimension of PositionwiseFeedForward + n_heads (int): number of heads for multi-head attention + conv_kernel_size (int): kernel size for depthwise convolution in convolution module + dropout (float): dropout probabilities for linear layers + dropout_att (float): dropout probabilities for attention distributions + """ + + def __init__( + self, d_model, d_ff, conv_kernel_size, self_attention_model, n_heads, dropout, dropout_att + ): + super(ConformerEncoderBlock, self).__init__() + + self.self_attention_model = self_attention_model + self.n_heads = n_heads + self.fc_factor = 0.5 + + # first feed forward module + self.norm_feed_forward1 = LayerNorm(d_model) + self.feed_forward1 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + + # convolution module + self.norm_conv = LayerNorm(d_model) + self.conv = ConformerConvolution(d_model=d_model, kernel_size=conv_kernel_size) + + # multi-headed self-attention module + self.norm_self_att = LayerNorm(d_model) + if self_attention_model == 'rel_pos': + self.self_attn = RelPositionMultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) + elif self_attention_model == 'abs_pos': + self.self_attn = MultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) + elif 'rel_pos_old' in self_attention_model: + self.self_attn = RelPositionMultiHeadAttention_old( + kdim=d_model, + qdim=d_model, + adim=d_model, + odim=d_model, + n_heads=n_heads, + dropout=dropout_att, + bias=False, + param_init='xavier_uniform', + xl_like=True if 'xl' in self_attention_model else False, + ) + else: + raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") + + # second feed forward module + self.norm_feed_forward2 = LayerNorm(d_model) + self.feed_forward2 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + + self.dropout = nn.Dropout(dropout) + self.norm_out = LayerNorm(d_model) + + def forward(self, x, att_mask=None, pos_emb=None, u_bias=None, v_bias=None, pad_mask=None): + """ + Args: + x (FloatTensor): `[B, T, d_model]` + att_mask (ByteTensor): `[B, T, T]` + pos_emb (LongTensor): `[L, 1, d_model]` + u (FloatTensor): global parameter for relative positional embedding + v (FloatTensor): global parameter for relative positional embedding + Returns: + xs (FloatTensor): `[B, T, d_model]` + xx_aws (FloatTensor): `[B, H, T, T]` + + """ + residual = x + x = self.norm_feed_forward1(x) + x = self.feed_forward1(x) + x = self.fc_factor * self.dropout(x) + residual + + residual = x + x = self.norm_self_att(x) + if self.self_attention_model == 'rel_pos': + x = self.self_attn(query=x, key=x, value=x, pos_emb=pos_emb, mask=att_mask) + elif self.self_attention_model == 'abs_pos': + x = self.self_attn(query=x, key=x, value=x, mask=att_mask) + elif 'old' in self.self_attention_model: + x = self.self_attn(query=x, key=x, pos_emb=pos_emb, mask=att_mask, u_bias=u_bias, v_bias=v_bias) + else: + x = None + x = self.dropout(x) + residual + + residual = x + x = self.norm_conv(x) + x = self.conv(x) + x = self.dropout(x) + residual + + residual = x + x = self.norm_feed_forward2(x) + x = self.feed_forward2(x) + x = self.fc_factor * self.dropout(x) + residual + + x = self.norm_out(x) + return x diff --git a/nemo/collections/asr/modules/conformer_modules.py b/nemo/collections/asr/modules/conformer_modules.py new file mode 100644 index 000000000000..5662ddbfe439 --- /dev/null +++ b/nemo/collections/asr/modules/conformer_modules.py @@ -0,0 +1,86 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch.nn as nn + +from nemo.collections.asr.modules.activations import Swish + +__all__ = ['ConformerConvolution', 'ConformerFeedForward'] + + +class ConformerConvolution(nn.Module): + """The convolution module for the Conformer model. + Args: + d_model (int): hidden dimension + kernel_size (int): kernel size for depthwise convolution + """ + + def __init__(self, d_model, kernel_size): + super(ConformerConvolution, self).__init__() + assert (kernel_size - 1) % 2 == 0 + self.d_model = d_model + + self.pointwise_conv1 = nn.Conv1d( + in_channels=d_model, out_channels=d_model * 2, kernel_size=1, stride=1, padding=0, bias=True + ) + self.depthwise_conv = nn.Conv1d( + in_channels=d_model, + out_channels=d_model, + kernel_size=kernel_size, + stride=1, + padding=(kernel_size - 1) // 2, + groups=d_model, + bias=True, + ) + self.batch_norm = nn.BatchNorm1d(d_model) + self.activation = Swish() + self.pointwise_conv2 = nn.Conv1d( + in_channels=d_model, out_channels=d_model, kernel_size=1, stride=1, padding=0, bias=True + ) + + def forward(self, x): + x = x.transpose(1, 2) + x = self.pointwise_conv1(x) + + x = nn.functional.glu(x, dim=1) + + x = self.depthwise_conv(x) + + x = self.batch_norm(x) + x = self.activation(x) + + x = self.pointwise_conv2(x) + x = x.transpose(1, 2) + return x + + +class ConformerFeedForward(nn.Module): + """ + feed-forward module of Conformer model. + """ + + def __init__(self, d_model, d_ff, dropout, activation=Swish()): + super(ConformerFeedForward, self).__init__() + self.linear1 = nn.Linear(d_model, d_ff) + self.activation = activation + self.dropout = nn.Dropout(p=dropout) + self.linear2 = nn.Linear(d_ff, d_model) + + def forward(self, x): + x = self.linear1(x) + x = self.activation(x) + x = self.dropout(x) + x = self.linear2(x) + return x diff --git a/nemo/collections/asr/modules/lstm_decoder.py b/nemo/collections/asr/modules/lstm_decoder.py new file mode 100644 index 000000000000..149d9b1fb9fa --- /dev/null +++ b/nemo/collections/asr/modules/lstm_decoder.py @@ -0,0 +1,98 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict + +import torch +import torch.nn as nn + +from nemo.core.classes.common import typecheck +from nemo.core.classes.exportable import Exportable +from nemo.core.classes.module import NeuralModule +from nemo.core.neural_types import AcousticEncodedRepresentation, LogprobsType, NeuralType + +__all__ = ['LSTMDecoder'] + + +class LSTMDecoder(NeuralModule, Exportable): + """ + Simple LSTM Decoder for use with CTC-based ASR models + """ + + def save_to(self, save_path: str): + pass + + @classmethod + def restore_from(cls, restore_path: str): + pass + + @property + def input_types(self): + return OrderedDict({"encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation())}) + + @property + def output_types(self): + return OrderedDict({"logprobs": NeuralType(('B', 'T', 'D'), LogprobsType())}) + + def __init__(self, feat_in, num_classes, lstm_hidden_size, vocabulary=None, bidirectional=False, num_layers=1): + super().__init__() + + if vocabulary is not None: + if num_classes != len(vocabulary): + raise ValueError( + f"If vocabulary is specified, it's length should be equal to the num_classes. " + f"Instead got: num_classes={num_classes} and len(vocabulary)={len(vocabulary)}" + ) + self.__vocabulary = vocabulary + self._feat_in = feat_in + # Add 1 for blank char + self._num_classes = num_classes + 1 + + self.lstm_layer = nn.LSTM( + input_size=feat_in, + hidden_size=lstm_hidden_size, + num_layers=num_layers, + batch_first=True, + bidirectional=bidirectional, + ) + self.linear_layer = torch.nn.Linear(in_features=lstm_hidden_size, out_features=self._num_classes) + + @typecheck() + def forward(self, encoder_output): + output = encoder_output.transpose(1, 2) + output, _ = self.lstm_layer(output) + output = self.linear_layer(output) + return torch.nn.functional.log_softmax(output, dim=-1) + + def input_example(self): + """ + Generates input examples for tracing etc. + Returns: + A tuple of input examples. + """ + bs = 8 + seq = 64 + input_example = torch.randn(bs, self._feat_in, seq).to(next(self.parameters()).device) + return tuple([input_example]) + + def _prepare_for_export(self): + Exportable._prepare_for_export(self) + + @property + def vocabulary(self): + return self.__vocabulary + + @property + def num_classes_with_blank(self): + return self._num_classes diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/modules/multi_head_attention.py new file mode 100644 index 000000000000..44211d4b3680 --- /dev/null +++ b/nemo/collections/asr/modules/multi_head_attention.py @@ -0,0 +1,483 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# Copyright 2020 Hirofumi Inaguma(Kyoto University) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Part of this code is adopted from https://github.com/espnet/espnet and https://github.com/hirofumi0810/neural_sp/ +""" + +import math + +import numpy as np +import torch +import torch.nn as nn + +from nemo.utils import logging + +__all__ = [ + 'RelPositionMultiHeadAttention_old', + 'RelPositionMultiHeadAttention', + 'RelPositionalEncoding_old', + 'RelPositionalEncoding', + 'PositionalEncoding', +] + + +class MultiHeadAttention(nn.Module): + """Multi-Head Attention layer. + :param int n_head: the number of head s + :param int n_feat: the number of features + :param float dropout_rate: dropout rate + """ + + def __init__(self, n_head, n_feat, dropout_rate): + """Construct an MultiHeadedAttention object.""" + super(MultiHeadAttention, self).__init__() + assert n_feat % n_head == 0 + # We assume d_v always equals d_k + self.d_k = n_feat // n_head + self.h = n_head + self.linear_q = nn.Linear(n_feat, n_feat) + self.linear_k = nn.Linear(n_feat, n_feat) + self.linear_v = nn.Linear(n_feat, n_feat) + self.linear_out = nn.Linear(n_feat, n_feat) + self.attn = None + self.dropout = nn.Dropout(p=dropout_rate) + + def forward_qkv(self, query, key, value): + """Transform query, key and value. + :param torch.Tensor query: (batch, time1, size) + :param torch.Tensor key: (batch, time2, size) + :param torch.Tensor value: (batch, time2, size) + :return torch.Tensor transformed query, key and value + """ + n_batch = query.size(0) + q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) + k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) + v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) + q = q.transpose(1, 2) # (batch, head, time1, d_k) + k = k.transpose(1, 2) # (batch, head, time2, d_k) + v = v.transpose(1, 2) # (batch, head, time2, d_k) + + return q, k, v + + def forward_attention(self, value, scores, mask): + """Compute attention context vector. + :param torch.Tensor value: (batch, time2, size) + :param torch.Tensor scores: (batch, time1, time2) + :param torch.Tensor mask: (batch, time1, time2) + :return torch.Tensor transformed `value` (batch, time2, d_model) + weighted by the attention score (batch, time1, time2) + """ + n_batch = value.size(0) + if mask is not None: + mask = mask.unsqueeze(1).eq(0) # (batch, 1, time1, time2) + min_value = float(np.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min) + scores = scores.masked_fill(mask, min_value) + self.attn = torch.softmax(scores, dim=-1).masked_fill(mask, 0.0) # (batch, head, time1, time2) + else: + self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) + + p_attn = self.dropout(self.attn) + x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) + x = x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) # (batch, time1, d_model) + + return self.linear_out(x) # (batch, time1, d_model) + + def forward(self, query, key, value, mask): + """Compute 'Scaled Dot Product Attention'. + :param torch.Tensor query: (batch, time1, size) + :param torch.Tensor key: (batch, time2, size) + :param torch.Tensor value: (batch, time2, size) + :param torch.Tensor mask: (batch, time1, time2) + :param torch.nn.Dropout dropout: + :return torch.Tensor: attentined and transformed `value` (batch, time1, d_model) + weighted by the query dot key attention (batch, head, time1, time2) + """ + q, k, v = self.forward_qkv(query, key, value) + scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) + return self.forward_attention(v, scores, mask) + + +class RelPositionMultiHeadAttention(MultiHeadAttention): + """Multi-Head Attention layer with relative position encoding. + Paper: https://arxiv.org/abs/1901.02860 + :param int n_head: the number of head s + :param int n_feat: the number of features + :param float dropout_rate: dropout rate + """ + + def __init__(self, n_head, n_feat, dropout_rate): + """Construct an RelPositionMultiHeadedAttention object.""" + super().__init__(n_head, n_feat, dropout_rate) + # linear transformation for positional ecoding + self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) + # these two learnable bias are used in matrix c and matrix d + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) + self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) + torch.nn.init.xavier_uniform_(self.pos_bias_u) + torch.nn.init.xavier_uniform_(self.pos_bias_v) + + def rel_shift(self, x, zero_triu=False): + """Compute relative positinal encoding. + :param torch.Tensor x: (batch, time, size) + :param bool zero_triu: return the lower triangular part of the matrix + """ + zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) + x_padded = torch.cat([zero_pad, x], dim=-1) + + x_padded = x_padded.view(*x.size()[:2], x.size(3) + 1, x.size(2)) + x = x_padded[:, :, 1:].view_as(x) + + if zero_triu: + ones = torch.ones((x.size(2), x.size(3))) + x = x * torch.tril(ones, x.size(3) - x.size(2))[None, None, :, :] + + return x + + def forward(self, query, key, value, mask, pos_emb): + """Compute 'Scaled Dot Product Attention' with rel. positional encoding. + :param torch.Tensor query: (batch, time1, size) + :param torch.Tensor key: (batch, time2, size) + :param torch.Tensor value: (batch, time2, size) + :param torch.Tensor pos_emb: (batch, time1, size) + :param torch.Tensor mask: (batch, time1, time2) + :param torch.nn.Dropout dropout: + :return torch.Tensor: attentined and transformed `value` (batch, time1, d_model) + weighted by the query dot key attention (batch, head, time1, time2) + """ + q, k, v = self.forward_qkv(query, key, value) + q = q.transpose(1, 2) # (batch, time1, head, d_k) + + n_batch_pos = pos_emb.size(0) + p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) + p = p.transpose(1, 2) # (batch, head, time1, d_k) + + # (batch, head, time1, d_k) + q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) + # (batch, head, time1, d_k) + q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) + + # compute attention score + # first compute matrix a and matrix c + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + # (batch, head, time1, time2) + matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) + + # compute matrix b and matrix d + # (batch, head, time1, time2) + matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) + matrix_bd = self.rel_shift(matrix_bd) + + scores = (matrix_ac + matrix_bd) / math.sqrt(self.d_k) # (batch, head, time1, time2) + + return self.forward_attention(v, scores, mask) + + +class PositionalEncoding(torch.nn.Module): + """Positional encoding. + :param int d_model: embedding dim + :param float dropout_rate: dropout rate + :param int max_len: maximum input length + :param reverse: whether to reverse the input position + """ + + def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False, xscale=None): + """Construct an PositionalEncoding object.""" + super(PositionalEncoding, self).__init__() + self.d_model = d_model + self.reverse = reverse + self.xscale = xscale # math.sqrt(self.d_model) + self.dropout = torch.nn.Dropout(p=dropout_rate) + self.pe = None + self.extend_pe(torch.tensor(0.0).expand(1, max_len)) + + def extend_pe(self, x): + """Reset the positional encodings.""" + if self.pe is not None: + if self.pe.size(1) >= x.size(1): + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(dtype=x.dtype, device=x.device) + return + pe = torch.zeros(x.size(1), self.d_model) + if self.reverse: + position = torch.arange(x.size(1) - 1, -1, -1.0, dtype=torch.float32).unsqueeze(1) + else: + position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, self.d_model, 2, dtype=torch.float32) * -(math.log(10000.0) / self.d_model) + ) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.pe = pe.to(device=x.device, dtype=x.dtype) + + def forward(self, x: torch.Tensor): + """Add positional encoding. + Args: + x (torch.Tensor): Input. Its shape is (batch, time, ...) + Returns: + torch.Tensor: Encoded tensor. Its shape is (batch, time, ...) + """ + self.extend_pe(x) + if self.xscale: + x = x * self.xscale + x = x + self.pe[:, : x.size(1)] + return self.dropout(x), None + + +class RelPositionalEncoding(PositionalEncoding): + """Relitive positional encoding module. + See : Appendix B in https://arxiv.org/abs/1901.02860 + :param int d_model: embedding dim + :param float dropout_rate: dropout rate + :param int max_len: maximum input length + """ + + def __init__(self, d_model, dropout_rate, max_len=5000, dropout_emb_rate=0.0, xscale=None): + """Initialize class. + :param int d_model: embedding dim + :param float dropout_rate: dropout rate + :param int max_len: maximum input length + """ + super().__init__(d_model, dropout_rate, max_len, reverse=True, xscale=xscale) + + if dropout_emb_rate > 0: + self.dropout_emb = nn.Dropout(dropout_emb_rate) + else: + self.dropout_emb = None + + def forward(self, x): + """Compute positional encoding. + Args: + x (torch.Tensor): Input. Its shape is (batch, time, ...) + Returns: + torch.Tensor: x. Its shape is (batch, time, ...) + torch.Tensor: pos_emb. Its shape is (1, time, ...) + """ + self.extend_pe(x) + if self.xscale: + x = x * self.xscale + pos_emb = self.pe[:, : x.size(1)] + if self.dropout_emb: + pos_emb = self.dropout_emb(pos_emb) + return self.dropout(x), pos_emb + + +class RelPositionMultiHeadAttention_old(nn.Module): + """Relative multi-head attention layer for TransformerXL. + Args: + kdim (int): dimension of key + qdim (int): dimension of query + adim: (int) dimension of attention space + odim: (int) dimension of output + n_heads (int): number of heads + dropout (float): dropout probability for attenion weights + dropout_head (float): HeadDrop probability + bias (bool): use bias term in linear layers + param_init (str): parameter initialization method + xl_like (bool): use TransformerXL like relative positional encoding. + Otherwise, use relative positional encoding like Shaw et al. 2018 + """ + + def __init__( + self, kdim, qdim, adim, odim, n_heads, dropout, dropout_head=0.0, bias=False, param_init='', xl_like=False + ): + + super().__init__() + + assert adim % n_heads == 0 + self.d_k = adim // n_heads + self.n_heads = n_heads + self.scale = math.sqrt(self.d_k) + self.xl_like = xl_like + + self.dropout_attn = nn.Dropout(p=dropout) + self.dropout_head = dropout_head + + assert kdim == qdim + # NOTE: relative attention is supprted for self-attention only + self.w_key = nn.Linear(kdim, adim, bias=bias) + self.w_value = nn.Linear(kdim, adim, bias=bias) + self.w_query = nn.Linear(qdim, adim, bias=bias) + self.w_out = nn.Linear(adim, odim, bias=bias) + + if xl_like: + self.w_pos = nn.Linear(qdim, adim, bias=bias) + + if param_init == 'xavier_uniform': + self.reset_parameters(bias) + else: + logging.info('Parameter initialization for RelativeMultiheadAttentionMechanism skipped.') + + def reset_parameters(self, bias): + """Initialize parameters with Xavier uniform distribution.""" + # NOTE: see https://github.com/pytorch/fairseq/blob/master/fairseq/modules/multihead_attention.py + nn.init.xavier_uniform_(self.w_key.weight, gain=1 / math.sqrt(2)) + nn.init.xavier_uniform_(self.w_value.weight, gain=1 / math.sqrt(2)) + nn.init.xavier_uniform_(self.w_query.weight, gain=1 / math.sqrt(2)) + if bias: + nn.init.constant_(self.w_key.bias, 0.0) + nn.init.constant_(self.w_value.bias, 0.0) + nn.init.constant_(self.w_query.bias, 0.0) + + nn.init.xavier_uniform_(self.w_out.weight) + if bias: + nn.init.constant_(self.w_out.bias, 0.0) + + if self.xl_like: + nn.init.xavier_uniform_(self.w_pos.weight) + if bias: + nn.init.constant_(self.w_pos.bias, 0.0) + + def _rel_shift(self, xs): + """Calculate relative positional attention efficiently. + Args: + xs (FloatTensor): `[B, qlen, klen, H]` + Returns: + xs_shifted (FloatTensor): `[B, qlen, klen, H]` + """ + bs, qlen, klen, n_heads = xs.size() + # `[qlen, klen, B, H]` -> `[B, qlen, klen, H]` + xs = xs.permute(1, 2, 0, 3).contiguous().view(qlen, klen, bs * n_heads) + + zero_pad = xs.new_zeros((qlen, 1, bs * n_heads)) + xs_shifted = torch.cat([zero_pad, xs], dim=1).view(klen + 1, qlen, bs * n_heads)[1:].view_as(xs) + return xs_shifted.view(qlen, klen, bs, n_heads).permute(2, 0, 1, 3) + + def forward(self, query, key, pos_emb, mask, u_bias=None, v_bias=None): + """Forward pass. + Args: + cat (FloatTensor): `[B, mlen+qlen, kdim]` + mask (ByteTensor): `[B, qlen, mlen+qlen]` + pos_emb (LongTensor): `[qlen, 1, d_model]` + u_bias (nn.Parameter): `[H, d_k]` + v_bias (nn.Parameter): `[H, d_k]` + Returns: + cv (FloatTensor): `[B, qlen, vdim]` + aw (FloatTensor): `[B, H, qlen, mlen+qlen]` + """ + bs, qlen = query.size()[:2] + mlen = key.size(1) - qlen + # NOTE: cat already includes memory, i.e., klen=mlen+qlen + + if mask is not None: + mask = mask.unsqueeze(3).repeat([1, 1, 1, self.n_heads]) + assert mask.size() == (bs, qlen, mlen + qlen, self.n_heads), ( + mask.size(), + (bs, qlen, mlen + qlen, self.n_heads), + ) + + k = self.w_key(key).view(bs, -1, self.n_heads, self.d_k) # `[B, mlen+qlen, H, d_k]` + v = self.w_value(key).view(bs, -1, self.n_heads, self.d_k) # `[B, mlen+qlen, H, d_k]` + q = self.w_query(key[:, -qlen:]).view(bs, -1, self.n_heads, self.d_k) # `[B, qlen, H, d_k]` + + if self.xl_like: + _pos_embs = self.w_pos(pos_emb) + else: + _pos_embs = self.w_value(pos_emb) + _pos_embs = _pos_embs.view(-1, self.n_heads, self.d_k) # `[mlen+qlen, H, d_k]` + + # content-based attention term: (a) + (c) + if u_bias is not None: + assert self.xl_like + AC = torch.einsum("bihd,bjhd->bijh", ((q + u_bias[None, None]), k)) # `[B, qlen, mlen+qlen, H]` + else: + AC = torch.einsum("bihd,bjhd->bijh", (q, k)) # `[B, qlen, mlen+qlen, H]` + + # position-based attention term: (b) + (d) + if v_bias is not None: + assert self.xl_like + BD = torch.einsum("bihd,jhd->bijh", ((q + v_bias[None, None]), _pos_embs)) # `[B, qlen, mlen+qlen, H]` + else: + BD = torch.einsum("bihd,jhd->bijh", (q, _pos_embs)) # `[B, qlen, mlen+qlen, H]` + + # Compute positional attention efficiently + BD = self._rel_shift(BD) + + # the attention is the sum of content-based and position-based attention + e = (AC + BD) / self.scale # `[B, qlen, mlen+qlen, H]` + + # Compute attention weights + if mask is not None: + NEG_INF = float(np.finfo(torch.tensor(0, dtype=e.dtype).numpy().dtype).min) + e = e.masked_fill_(mask == 0, NEG_INF) # `[B, qlen, mlen+qlen, H]` + aw = torch.softmax(e, dim=2) + aw = self.dropout_attn(aw) # `[B, qlen, mlen+qlen, H]` + + # mask out each head independently (HeadDrop) + if self.dropout_head > 0 and self.training: + aw_masked = aw.clone() + aw_masked = aw_masked.permute(0, 3, 1, 2) + aw_masked = self.dropout_head(aw_masked, self.n_heads, self.dropout_head) # `[B, H, qlen, klen]` + aw_masked = aw_masked.permute(0, 2, 3, 1) + + cv = torch.einsum("bijh,bjhd->bihd", (aw, v)) # `[B, qlen, H, d_k]` + cv = cv.contiguous().view(bs, -1, self.n_heads * self.d_k) # `[B, qlen, H * d_k]` + cv = self.w_out(cv) + # aw = aw.permute(0, 3, 1, 2) # `[B, H, qlen, mlen+qlen]` + + return cv # , aw + + +class RelPositionalEncoding_old(nn.Module): + def __init__(self, d_model, dropout, dropout_emb_rate=0.0, xscale=None): + """Positional embedding for TransformerXL.""" + super().__init__() + self.d_model = d_model + self.xscale = xscale + inv_freq = 1 / (10000 ** (torch.arange(0.0, d_model, 2.0) / d_model)) + self.register_buffer("inv_freq", inv_freq) + + self.dropout = nn.Dropout(p=dropout) + + if dropout_emb_rate > 0: + self.dropout_emb = nn.Dropout(dropout_emb_rate) + else: + self.dropout_emb = None + + self.xscale = 1 # math.sqrt(self.d_model) + + def forward(self, xs, mlen=0, clamp_len=-1, zero_center_offset=False): + """Forward pass. + Args: + xs (FloatTensor): `[B, L, d_model]` + mlen (int); length of memory + clamp_len (int): + zero_center_offset (bool): + Returns: + pos_emb (LongTensor): `[L, 1, d_model]` + """ + if zero_center_offset: + pos_idxs = torch.arange(mlen - 1, -xs.size(1) - 1, -1.0, dtype=torch.float, device=xs.device) + else: + pos_idxs = torch.arange(mlen + xs.size(1) - 1, -1, -1.0, dtype=torch.float, device=xs.device) + + # truncate by maximum length + if clamp_len > 0: + pos_idxs.clamp_(max=clamp_len) + + # outer product + sinusoid_inp = torch.einsum("i,j->ij", pos_idxs, self.inv_freq) + pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1) + + if self.dropout_emb: + pos_emb = self.dropout_emb(pos_emb) + if self.xscale: + xs = self.xscale * xs + + return self.dropout(xs), pos_emb.unsqueeze(1) diff --git a/nemo/collections/asr/modules/subsampling.py b/nemo/collections/asr/modules/subsampling.py new file mode 100644 index 000000000000..2d1d5e07c320 --- /dev/null +++ b/nemo/collections/asr/modules/subsampling.py @@ -0,0 +1,142 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +import torch.nn as nn + + +class ConvSubsampling(torch.nn.Module): + """Convolutional 2D subsampling (to 1/4 length). + + :param int idim: input dim + :param int odim: output dim + :param str activation: activation functions + :param flaot dropout_rate: dropout rate + + """ + + def __init__(self, subsampling, subsampling_factor, idim, odim, conv_channels=-1, activation=nn.ReLU()): + super(ConvSubsampling, self).__init__() + if conv_channels <= 0: + conv_channels = odim + self._subsampling = subsampling + + if subsampling_factor % 2 != 0: + raise ValueError("Sampling factor should be a multiply of 2!") + self._sampling_num = int(math.log(subsampling_factor, 2)) + + in_channels = 1 + layers = [] + if subsampling == 'vggnet': + self._padding = 0 + self._stride = 2 + self._kernel_size = 2 + self._ceil_mode = True # TODO: is False better? + + for i in range(self._sampling_num): + layers.append( + torch.nn.Conv2d( + in_channels=in_channels, out_channels=conv_channels, kernel_size=3, stride=1, padding=1 + ) + ) + layers.append(activation) + layers.append( + torch.nn.Conv2d( + in_channels=conv_channels, out_channels=conv_channels, kernel_size=3, stride=1, padding=1 + ) + ) + layers.append(activation) + layers.append( + torch.nn.MaxPool2d( + kernel_size=self._kernel_size, + stride=self._stride, + padding=self._padding, + ceil_mode=self._ceil_mode, + ) + ) + in_channels = conv_channels + elif subsampling == 'striding': + self._padding = 1 # TODO: is 0 better? + self._stride = 2 + self._kernel_size = 3 + self._ceil_mode = False + + for i in range(self._sampling_num): + layers.append( + torch.nn.Conv2d( + in_channels=in_channels, + out_channels=conv_channels, + kernel_size=self._kernel_size, + stride=self._stride, + padding=self._padding, + ) + ) + layers.append(activation) + in_channels = conv_channels + else: + raise ValueError(f"Not valid sub-sampling: {subsampling}!") + + in_length = idim + for i in range(self._sampling_num): + out_length = calc_length( + length=int(in_length), + padding=self._padding, + kernel_size=self._kernel_size, + stride=self._stride, + ceil_mode=self._ceil_mode, + ) + in_length = out_length + + self.out = torch.nn.Linear(conv_channels * out_length, odim) + self.conv = torch.nn.Sequential(*layers) + + def forward(self, x, lengths): + """Subsample x. + + :param torch.Tensor x: input tensor + :param torch.Tensor x_mask: input mask + :return: subsampled x and mask + :rtype Tuple[torch.Tensor or Tuple[torch.Tensor, torch.Tensor], torch.Tensor] + """ + x = x.unsqueeze(1) # (b, c, t, f) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + + # TODO: improve the performance of here + new_lengths = lengths + for i in range(self._sampling_num): + new_lengths = [ + calc_length( + length=int(length), + padding=self._padding, + kernel_size=self._kernel_size, + stride=self._stride, + ceil_mode=self._ceil_mode, + ) + for length in new_lengths + ] + + new_lengths = torch.IntTensor(new_lengths).to(lengths.device) + return x, new_lengths + + +def calc_length(length, padding, kernel_size, stride, ceil_mode): + if ceil_mode: + length = math.ceil((length + (2 * padding) - (kernel_size - 1) - 1) / float(stride) + 1) + else: + length = math.floor((length + (2 * padding) - (kernel_size - 1) - 1) / float(stride) + 1) + return length From 99e92332f4d7f51cdc9ece85a77dd7ed7b21a0d4 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 15:43:47 -0700 Subject: [PATCH 02/67] Added initial code of Conformer. Signed-off-by: Vahid --- .../conf/text_classification_config.yaml | 2 +- .../data/import_datasets.py | 10 +-- nemo/collections/asr/data/audio_to_text.py | 18 +++-- nemo/collections/asr/losses/__init__.py | 3 + nemo/collections/asr/losses/ctc.py | 13 ++- nemo/collections/asr/metrics/wer.py | 2 +- nemo/collections/asr/metrics/wer_bpe.py | 2 +- nemo/collections/asr/models/ctc_bpe_models.py | 6 +- nemo/collections/asr/models/ctc_models.py | 14 +++- nemo/collections/asr/parts/features.py | 31 +++---- nemo/core/classes/modelPT.py | 16 ++++ nemo/core/config/__init__.py | 1 + nemo/core/config/schedulers.py | 11 +++ nemo/core/optim/__init__.py | 1 + nemo/core/optim/lr_scheduler.py | 81 +++++++++++++++++-- 15 files changed, 168 insertions(+), 43 deletions(-) diff --git a/examples/nlp/text_classification/conf/text_classification_config.yaml b/examples/nlp/text_classification/conf/text_classification_config.yaml index 62c77984c106..b91f0d0de4a3 100644 --- a/examples/nlp/text_classification/conf/text_classification_config.yaml +++ b/examples/nlp/text_classification/conf/text_classification_config.yaml @@ -15,7 +15,7 @@ # Config file for text classification with pre-trained BERT models trainer: - gpus: 1 # number of gpus (0 for CPU), or list of the GPUs to use e.g. [0, 1] + gpus: 1 # number of GPUs (0 for CPU), or list of the GPUs to use e.g. [0, 1] num_nodes: 1 max_epochs: 100 max_steps: null # precedence over max_epochs diff --git a/examples/nlp/text_classification/data/import_datasets.py b/examples/nlp/text_classification/data/import_datasets.py index 76d08c0db675..95db24cea7d7 100644 --- a/examples/nlp/text_classification/data/import_datasets.py +++ b/examples/nlp/text_classification/data/import_datasets.py @@ -21,7 +21,8 @@ --source_data_dir "./thucnews_orig_data/" \ --target_data_dir "./thucnews/" -It reads the data from "source_data_dir" folder, processes and converts the data into NeMo's format. Then writes the results into "target_data_dir" folder. +It reads the data from "source_data_dir" folder, processes and converts the data into NeMo's format. +Then writes the results into "target_data_dir" folder. """ import argparse @@ -40,7 +41,8 @@ def process_imdb(infold, outfold, uncased, modes=['train', 'test']): link = 'https://ai.stanford.edu/~amaas/data/sentiment/' raise ValueError( f'Data not found at {infold}. ' - f'Please download IMDB reviews dataset from {link} and extract it into the folder specified by source_data_dir argument.' + f'Please download IMDB reviews dataset from {link} and ' + f'extract it into the folder specified by source_data_dir argument.' ) logging.info(f'Processing IMDB dataset and store at {outfold}') @@ -153,9 +155,7 @@ def process_thucnews(infold, outfold): if __name__ == "__main__": # Parse the command-line arguments. parser = argparse.ArgumentParser(description="Process and convert datasets into NeMo\'s format.") - parser.add_argument( - "--dataset_name", required=True, type=str, choices=['imdb', 'thucnews', 'chemprot'], - ) + parser.add_argument("--dataset_name", required=True, type=str, choices=['imdb', 'thucnews', 'chemprot']) parser.add_argument( "--source_data_dir", required=True, type=str, help='The path to the folder containing the dataset files.' ) diff --git a/nemo/collections/asr/data/audio_to_text.py b/nemo/collections/asr/data/audio_to_text.py index c60b385b0c55..f742341ef3c3 100644 --- a/nemo/collections/asr/data/audio_to_text.py +++ b/nemo/collections/asr/data/audio_to_text.py @@ -165,7 +165,7 @@ def __getitem__(self, index): offset = 0 features = self.featurizer.process( - sample.audio_file, offset=offset, duration=sample.duration, trim=self.trim, orig_sr=sample.orig_sr, + sample.audio_file, offset=offset, duration=sample.duration, trim=self.trim, orig_sr=sample.orig_sr ) f, fl = features, torch.tensor(features.shape[0]).long() else: @@ -270,7 +270,7 @@ def __init__( self.labels = labels parser = parsers.make_parser( - labels=labels, name=parser, unk_id=unk_index, blank_id=blank_index, do_normalize=normalize, + labels=labels, name=parser, unk_id=unk_index, blank_id=blank_index, do_normalize=normalize ) super().__init__( @@ -474,13 +474,14 @@ def __init__( trim: bool = False, load_audio: bool = True, add_misc: bool = False, + use_start_end_token: bool = True, ): - if hasattr(tokenizer, 'bos_token'): + if use_start_end_token and hasattr(tokenizer, 'bos_token'): bos_id = tokenizer.bos_id else: bos_id = None - if hasattr(tokenizer, 'eos_token'): + if use_start_end_token and hasattr(tokenizer, 'eos_token'): eos_id = tokenizer.eos_id else: eos_id = None @@ -553,7 +554,7 @@ def __init__( load_audio=True, ): self.collection = collections.ASRSpeechLabel( - manifests_files=manifest_filepath.split(','), min_duration=min_duration, max_duration=max_duration, + manifests_files=manifest_filepath.split(','), min_duration=min_duration, max_duration=max_duration ) self.featurizer = featurizer @@ -577,7 +578,7 @@ def __getitem__(self, index): offset = 0 features = self.featurizer.process( - sample.audio_file, offset=offset, duration=sample.duration, trim=self.trim, + sample.audio_file, offset=offset, duration=sample.duration, trim=self.trim ) f, fl = features, torch.tensor(features.shape[0]).long() else: @@ -1029,13 +1030,14 @@ def __init__( add_misc: bool = False, global_rank: int = 0, world_size: int = 0, + use_start_end_token: bool = True, ): - if hasattr(tokenizer, 'bos_token'): + if use_start_end_token and hasattr(tokenizer, 'bos_token'): bos_id = tokenizer.bos_id else: bos_id = None - if hasattr(tokenizer, 'eos_token'): + if use_start_end_token and hasattr(tokenizer, 'eos_token'): eos_id = tokenizer.eos_id else: eos_id = None diff --git a/nemo/collections/asr/losses/__init__.py b/nemo/collections/asr/losses/__init__.py index 9e3250071955..aa0a5ac45449 100644 --- a/nemo/collections/asr/losses/__init__.py +++ b/nemo/collections/asr/losses/__init__.py @@ -11,3 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from nemo.collections.asr.losses.ctc import CTCLoss +from nemo.collections.asr.losses.angularloss import AngularSoftmaxLoss diff --git a/nemo/collections/asr/losses/ctc.py b/nemo/collections/asr/losses/ctc.py index dfd70100386a..e20c554d7262 100644 --- a/nemo/collections/asr/losses/ctc.py +++ b/nemo/collections/asr/losses/ctc.py @@ -44,10 +44,16 @@ def output_types(self): """ return {"loss": NeuralType(elements_type=LossType())} - def __init__(self, num_classes, zero_infinity=False): + def __init__(self, num_classes, zero_infinity=False, reduction='mean_batch'): self._blank = num_classes # Don't forget to properly call base constructor - super().__init__(blank=self._blank, reduction='none', zero_infinity=zero_infinity) + if reduction == 'mean_batch': + ctc_reduction = 'none' + self._apply_batch_mean = True + elif reduction in ['sum', 'mean', 'none']: + ctc_reduction = reduction + self._apply_batch_mean = False + super().__init__(blank=self._blank, reduction=ctc_reduction, zero_infinity=zero_infinity) @typecheck() def forward(self, log_probs, targets, input_lengths, target_lengths): @@ -61,7 +67,8 @@ def forward(self, log_probs, targets, input_lengths, target_lengths): loss = super().forward( log_probs=log_probs, targets=targets, input_lengths=input_lengths, target_lengths=target_lengths ) - loss = torch.mean(loss) + if self._apply_batch_mean: + loss = torch.mean(loss) return loss diff --git a/nemo/collections/asr/metrics/wer.py b/nemo/collections/asr/metrics/wer.py index b949baabe21f..5403cd94ced1 100644 --- a/nemo/collections/asr/metrics/wer.py +++ b/nemo/collections/asr/metrics/wer.py @@ -158,7 +158,7 @@ def update(self, predictions: torch.Tensor, targets: torch.Tensor, target_length if self.log_prediction: logging.info(f"\n") logging.info(f"reference:{references[0]}") - logging.info(f"decoded :{hypotheses[0]}") + logging.info(f"predicted:{hypotheses[0]}") for h, r in zip(hypotheses, references): if self.use_cer: diff --git a/nemo/collections/asr/metrics/wer_bpe.py b/nemo/collections/asr/metrics/wer_bpe.py index f8f7d24541b4..ab7bc345328c 100644 --- a/nemo/collections/asr/metrics/wer_bpe.py +++ b/nemo/collections/asr/metrics/wer_bpe.py @@ -121,7 +121,7 @@ def update(self, predictions: torch.Tensor, targets: torch.Tensor, target_length if self.log_prediction: logging.info(f"\n") logging.info(f"reference:{references[0]}") - logging.info(f"decoded :{hypotheses[0]}") + logging.info(f"predicted:{hypotheses[0]}") for h, r in zip(hypotheses, references): if self.use_cer: diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index 3ef02e27281b..1ee033766961 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -82,7 +82,7 @@ def __init__(self, cfg: DictConfig, trainer=None): # Setup metric objects self._wer = WERBPE( - tokenizer=self.tokenizer, batch_dim_index=0, use_cer=False, ctc_decode=True, dist_sync_on_step=True, + tokenizer=self.tokenizer, batch_dim_index=0, use_cer=False, ctc_decode=True, dist_sync_on_step=True, log_prediction=self._cfg.get("log_prediction", False) ) def _setup_tokenizer(self): @@ -175,6 +175,7 @@ def _setup_dataloader_from_config(self, config: Optional[Dict]): add_misc=config.get('add_misc', False), global_rank=self.global_rank, world_size=self.world_size, + use_start_end_token=config.get('use_start_end_token', True), ) shuffle = False else: @@ -194,6 +195,7 @@ def _setup_dataloader_from_config(self, config: Optional[Dict]): trim=config.get('trim_silence', True), load_audio=config.get('load_audio', True), add_misc=config.get('add_misc', False), + use_start_end_token=config.get('use_start_end_token', True), ) return torch.utils.data.DataLoader( @@ -300,7 +302,7 @@ def change_vocabulary(self, new_tokenizer_dir: str, new_tokenizer_type: str): self.decoder = EncDecCTCModelBPE.from_config_dict(decoder_config) del self.loss self.loss = CTCLoss(num_classes=self.decoder.num_classes_with_blank - 1, zero_infinity=True) - self._wer = WERBPE(tokenizer=self.tokenizer, batch_dim_index=0, use_cer=False, ctc_decode=True) + self._wer = WERBPE(tokenizer=self.tokenizer, batch_dim_index=0, use_cer=False, ctc_decode=True, log_prediction=self._cfg.get("log_prediction", False)) # Update config OmegaConf.set_struct(self._cfg.decoder, False) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 64c7665a6bbe..906bf6d592ad 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -96,8 +96,18 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): super().__init__(cfg=cfg, trainer=trainer) self.preprocessor = EncDecCTCModel.from_config_dict(self._cfg.preprocessor) self.encoder = EncDecCTCModel.from_config_dict(self._cfg.encoder) + + if not self._cfg.decoder.feat_in and hasattr(self.encoder, 'feat_out'): + self._cfg.decoder.feat_in = self.encoder.feat_out + else: + logging.error("param feat_in of the decoder' config is not set!") + self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) - self.loss = CTCLoss(num_classes=self.decoder.num_classes_with_blank - 1, zero_infinity=True) + + self.loss = CTCLoss( + num_classes=self.decoder.num_classes_with_blank - 1, zero_infinity=True, reduction=self._cfg.ctc_reduction + ) + if hasattr(self._cfg, 'spec_augment') and self._cfg.spec_augment is not None: self.spec_augmentation = EncDecCTCModel.from_config_dict(self._cfg.spec_augment) else: @@ -110,6 +120,7 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): use_cer=False, ctc_decode=True, dist_sync_on_step=True, + log_prediction=self._cfg.get("log_prediction", False), ) @torch.no_grad() @@ -205,6 +216,7 @@ def change_vocabulary(self, new_vocabulary: List[str]): use_cer=False, ctc_decode=True, dist_sync_on_step=True, + log_prediction=self._cfg.get("log_prediction", False), ) # Update config diff --git a/nemo/collections/asr/parts/features.py b/nemo/collections/asr/parts/features.py index cd4a8f489fde..e9491df38201 100644 --- a/nemo/collections/asr/parts/features.py +++ b/nemo/collections/asr/parts/features.py @@ -285,7 +285,7 @@ def __init__( highfreq = highfreq or sample_rate / 2 filterbanks = torch.tensor( - librosa.filters.mel(sample_rate, self.n_fft, n_mels=nfilt, fmin=lowfreq, fmax=highfreq), dtype=torch.float, + librosa.filters.mel(sample_rate, self.n_fft, n_mels=nfilt, fmin=lowfreq, fmax=highfreq), dtype=torch.float ).unsqueeze(0) self.register_buffer("fb", filterbanks) @@ -357,6 +357,21 @@ def forward(self, x, seq_len): # dot with filterbank energies x = torch.matmul(self.fb.to(x.dtype), x) + # mask to zero any values beyond seq_len in batch, pad to multiple of + # `pad_to` (for efficiency) + max_len = x.size(-1) + mask = torch.arange(max_len).to(x.device) + mask = mask.expand(x.size(0), max_len) >= seq_len.unsqueeze(1) + x = x.masked_fill(mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value) + del mask + pad_to = self.pad_to + if pad_to == "max": + x = nn.functional.pad(x, (0, self.max_length - x.size(-1)), value=self.pad_value) + elif pad_to > 0: + pad_amt = x.size(-1) % pad_to + if pad_amt != 0: + x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value) + # log features if required if self.log: if self.log_zero_guard_type == "add": @@ -374,18 +389,4 @@ def forward(self, x, seq_len): if self.normalize: x = normalize_batch(x, seq_len, normalize_type=self.normalize) - # mask to zero any values beyond seq_len in batch, pad to multiple of - # `pad_to` (for efficiency) - max_len = x.size(-1) - mask = torch.arange(max_len).to(x.device) - mask = mask.expand(x.size(0), max_len) >= seq_len.unsqueeze(1) - x = x.masked_fill(mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value) - del mask - pad_to = self.pad_to - if pad_to == "max": - x = nn.functional.pad(x, (0, self.max_length - x.size(-1)), value=self.pad_value) - elif pad_to > 0: - pad_amt = x.size(-1) % pad_to - if pad_amt != 0: - x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value) return x, seq_len diff --git a/nemo/core/classes/modelPT.py b/nemo/core/classes/modelPT.py index c58adf9770dc..c97740d2321c 100644 --- a/nemo/core/classes/modelPT.py +++ b/nemo/core/classes/modelPT.py @@ -1005,3 +1005,19 @@ def __is_model_being_restored() -> bool: def __set_model_restore_state(is_being_restored: bool): global _MODEL_IS_RESTORED _MODEL_IS_RESTORED = is_being_restored + + def update_weights(self, checkpoint_path): + if os.path.exists(checkpoint_path): + raise FileExistsError(f"Could not find checkpoint file '{checkpoint_path}'!") + checkpoint = torch.load(checkpoint_path) + model_dict = self.state_dict() + for weights_name, weights in model_dict.items(): + if weights_name not in checkpoint['state_dict']: + logging.warning(f"Weights {weights_name} missing from the checkpoint!") + for weights_name, weights in checkpoint['state_dict'].items(): + if weights_name not in model_dict: + logging.warning(f"Weights {weights_name} missing from the model while exists in the checkpoint!") + pretrained_dict = {k: v for k, v in checkpoint['state_dict'].items() if k in model_dict} + model_dict.update(pretrained_dict) + self.load_state_dict(model_dict) + return self diff --git a/nemo/core/config/__init__.py b/nemo/core/config/__init__.py index a92557c44cec..0e192771be3c 100644 --- a/nemo/core/config/__init__.py +++ b/nemo/core/config/__init__.py @@ -31,6 +31,7 @@ from nemo.core.config.pytorch_lightning import TrainerConfig from nemo.core.config.schedulers import ( CosineAnnealingParams, + NoamAnnealingParams, InverseSquareRootAnnealingParams, PolynomialDecayAnnealingParams, PolynomialHoldDecayAnnealingParams, diff --git a/nemo/core/config/schedulers.py b/nemo/core/config/schedulers.py index 884808a38636..629a1fb9fcef 100644 --- a/nemo/core/config/schedulers.py +++ b/nemo/core/config/schedulers.py @@ -80,6 +80,16 @@ class CosineAnnealingParams(WarmupSchedulerParams): min_lr: float = 0.0 +@dataclass +class NoamAnnealingParams(WarmupSchedulerParams): + """ + Cosine Annealing parameter config + It is not derived from Config as it is not a NeMo object (and in particular it doesn't need a name). + """ + + min_lr: float = 0.0 + + @dataclass class WarmupAnnealingParams(WarmupSchedulerParams): """ @@ -233,6 +243,7 @@ def get_scheduler_config(name: str, **kwargs: Optional[Dict[str, Any]]) -> Sched 'SquareRootAnnealingParams': SquareRootAnnealingParams, 'InverseSquareRootAnnealingParams': InverseSquareRootAnnealingParams, 'CosineAnnealingParams': CosineAnnealingParams, + 'NoamAnnealingParams': NoamAnnealingParams, 'WarmupAnnealingParams': WarmupAnnealingParams, 'PolynomialDecayAnnealingParams': PolynomialDecayAnnealingParams, 'PolynomialHoldDecayAnnealingParams': PolynomialHoldDecayAnnealingParams, diff --git a/nemo/core/optim/__init__.py b/nemo/core/optim/__init__.py index a194068db7a8..e847d121df36 100644 --- a/nemo/core/optim/__init__.py +++ b/nemo/core/optim/__init__.py @@ -14,6 +14,7 @@ from nemo.core.optim.lr_scheduler import ( CosineAnnealing, + NoamAnnealing, InverseSquareRootAnnealing, PolynomialDecayAnnealing, PolynomialHoldDecayAnnealing, diff --git a/nemo/core/optim/lr_scheduler.py b/nemo/core/optim/lr_scheduler.py index ea4ca7f51bc8..ea61f67682ca 100644 --- a/nemo/core/optim/lr_scheduler.py +++ b/nemo/core/optim/lr_scheduler.py @@ -62,7 +62,7 @@ def __init__(self, optimizer, *, warmup_steps=None, warmup_ratio=None, max_steps def get_lr(self): if not self._get_lr_called_within_step: warnings.warn( - "To get the last learning rate computed by the scheduler, " "please use `get_last_lr()`.", UserWarning + "To get the last learning rate computed by the scheduler, please use `get_last_lr()`.", UserWarning ) step = self.last_epoch @@ -192,10 +192,26 @@ def _poly_decay(initial_lr, step, decay_steps, power, min_lr, cycle): class SquareAnnealing(WarmupPolicy): - def __init__(self, optimizer, *, max_steps, min_lr=1e-5, last_epoch=-1, **kwargs): - super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) + def __init__(self, optimizer, *, warmup_steps=None, warmup_ratio=None, max_steps=None, min_lr=0.0, last_epoch=-1): + assert not ( + warmup_steps is not None and warmup_ratio is not None + ), "Either use particular number of step or ratio" + assert warmup_ratio is None or max_steps is not None, "If there is a ratio, there should be a total steps" - def _get_lr(self, step): + # It is necessary to assign all attributes *before* __init__, + # as class is wrapped by an inner class. + self.max_steps = max_steps + if warmup_steps is not None: + self.warmup_steps = warmup_steps + elif warmup_ratio is not None: + self.warmup_steps = int(warmup_ratio * max_steps) + else: + self.warmup_steps = 0 + + self.min_lr = min_lr + super().__init__(optimizer, last_epoch) + + def get_lr(self, step): new_lrs = [ _square_annealing( initial_lr=initial_lr, @@ -214,7 +230,7 @@ def __init__(self, optimizer, *, max_steps, min_lr=0, last_epoch=-1, **kwargs): def _get_lr(self, step): new_lrs = [ - _squareroot_annealing(initial_lr=initial_lr, step=step, max_steps=self.max_steps, min_lr=self.min_lr,) + _squareroot_annealing(initial_lr=initial_lr, step=step, max_steps=self.max_steps, min_lr=self.min_lr) for initial_lr in self.base_lrs ] return new_lrs @@ -228,7 +244,7 @@ def _get_lr(self, step): for initial_lr in self.base_lrs: if initial_lr < self.min_lr: raise ValueError( - f"{self} received an initial learning rate that " f"was lower than the minimum learning rate." + f"{self} received an initial learning rate that was lower than the minimum learning rate." ) new_lrs = [ @@ -243,6 +259,58 @@ def _get_lr(self, step): return new_lrs +class NoamAnnealing(_LRScheduler): + def __init__( + self, optimizer, *, d_model, warmup_steps=None, warmup_ratio=None, max_steps=None, min_lr=0.0, last_epoch=-1 + ): + self._normalize = d_model ** (-0.5) + assert not ( + warmup_steps is not None and warmup_ratio is not None + ), "Either use particular number of step or ratio" + assert warmup_ratio is None or max_steps is not None, "If there is a ratio, there should be a total steps" + + # It is necessary to assign all attributes *before* __init__, + # as class is wrapped by an inner class. + self.max_steps = max_steps + if warmup_steps is not None: + self.warmup_steps = warmup_steps + elif warmup_ratio is not None: + self.warmup_steps = int(warmup_ratio * max_steps) + else: + self.warmup_steps = 0 + + self.min_lr = min_lr + super().__init__(optimizer, last_epoch) + + def get_lr(self): + if not self._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, please use `get_last_lr()`.", UserWarning + ) + + step = max(1, self.last_epoch) + + if step > self.max_steps: + return [self.min_lr for _ in self.base_lrs] + + for initial_lr in self.base_lrs: + if initial_lr < self.min_lr: + raise ValueError( + f"{self} received an initial learning rate that was lower than the minimum learning rate." + ) + + new_lrs = [self._noam_annealing(initial_lr=initial_lr, step=step) for initial_lr in self.base_lrs] + return new_lrs + + def _noam_annealing(self, initial_lr, step): + mult = self._normalize * min(step ** (-0.5), step * (self.warmup_steps ** (-1.5))) + out_lr = initial_lr * mult + if step > self.warmup_steps: + out_lr = max(out_lr, self.min_lr) + print(out_lr) + return out_lr + + class WarmupAnnealing(WarmupPolicy): def __init__(self, optimizer, *, max_steps, last_epoch=-1, min_lr=0.0, **kwargs): super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) @@ -576,6 +644,7 @@ def prepare_lr_scheduler( 'WarmupHoldPolicy': WarmupHoldPolicy, 'SquareAnnealing': SquareAnnealing, 'CosineAnnealing': CosineAnnealing, + 'NoamAnnealing': NoamAnnealing, 'WarmupAnnealing': WarmupAnnealing, 'InverseSquareRootAnnealing': InverseSquareRootAnnealing, 'SquareRootAnnealing': SquareRootAnnealing, From 428a72d3463ecaa1d11dee86742ade414cdf67b5 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 16:02:00 -0700 Subject: [PATCH 03/67] Added log_every_n_steps. Signed-off-by: Vahid --- examples/asr/experimental/{configs => }/bpe_vocabs/1k/vocab.txt | 0 examples/asr/experimental/{configs => }/bpe_vocabs/2k/vocab.txt | 0 examples/asr/experimental/{configs => }/bpe_vocabs/4k/vocab.txt | 0 examples/asr/experimental/{configs => }/bpe_vocabs/8k/vocab.txt | 0 examples/asr/experimental/configs/conformer/conformer_bpe.yaml | 2 +- examples/asr/experimental/configs/conformer/conformer_char.yaml | 2 +- 6 files changed, 2 insertions(+), 2 deletions(-) rename examples/asr/experimental/{configs => }/bpe_vocabs/1k/vocab.txt (100%) rename examples/asr/experimental/{configs => }/bpe_vocabs/2k/vocab.txt (100%) rename examples/asr/experimental/{configs => }/bpe_vocabs/4k/vocab.txt (100%) rename examples/asr/experimental/{configs => }/bpe_vocabs/8k/vocab.txt (100%) diff --git a/examples/asr/experimental/configs/bpe_vocabs/1k/vocab.txt b/examples/asr/experimental/bpe_vocabs/1k/vocab.txt similarity index 100% rename from examples/asr/experimental/configs/bpe_vocabs/1k/vocab.txt rename to examples/asr/experimental/bpe_vocabs/1k/vocab.txt diff --git a/examples/asr/experimental/configs/bpe_vocabs/2k/vocab.txt b/examples/asr/experimental/bpe_vocabs/2k/vocab.txt similarity index 100% rename from examples/asr/experimental/configs/bpe_vocabs/2k/vocab.txt rename to examples/asr/experimental/bpe_vocabs/2k/vocab.txt diff --git a/examples/asr/experimental/configs/bpe_vocabs/4k/vocab.txt b/examples/asr/experimental/bpe_vocabs/4k/vocab.txt similarity index 100% rename from examples/asr/experimental/configs/bpe_vocabs/4k/vocab.txt rename to examples/asr/experimental/bpe_vocabs/4k/vocab.txt diff --git a/examples/asr/experimental/configs/bpe_vocabs/8k/vocab.txt b/examples/asr/experimental/bpe_vocabs/8k/vocab.txt similarity index 100% rename from examples/asr/experimental/configs/bpe_vocabs/8k/vocab.txt rename to examples/asr/experimental/bpe_vocabs/8k/vocab.txt diff --git a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml index 107168fb6cbe..495af71a3660 100644 --- a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml @@ -136,7 +136,7 @@ trainer: gradient_clip_val: 0.0 amp_level: O0 # O1/O2 for mixed precision precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. - row_log_interval: 1 # Interval of logging. + log_every_n_steps: 10 # Interval of logging. resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs diff --git a/examples/asr/experimental/configs/conformer/conformer_char.yaml b/examples/asr/experimental/configs/conformer/conformer_char.yaml index 24959752e20b..01fbd405fca8 100644 --- a/examples/asr/experimental/configs/conformer/conformer_char.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_char.yaml @@ -133,7 +133,7 @@ trainer: gradient_clip_val: 0.0 amp_level: O0 # O1/O2 for mixed precision precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. - row_log_interval: 1 # Interval of logging. + log_every_n_steps: 10 # Interval of logging. resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs From 56770ad8736d8b528d4bf26608fbb86532975610 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 16:18:11 -0700 Subject: [PATCH 04/67] dropped older multi_head_att modules. Signed-off-by: Vahid --- .../configs/conformer/conformer_bpe.yaml | 2 +- .../configs/conformer/conformer_char.yaml | 2 +- nemo/collections/asr/modules/__init__.py | 2 - .../asr/modules/conformer_encoder.py | 40 +--- .../asr/modules/multi_head_attention.py | 209 +----------------- 5 files changed, 11 insertions(+), 244 deletions(-) diff --git a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml index 495af71a3660..4a049a5e5253 100644 --- a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml @@ -85,7 +85,7 @@ model: ff_expansion_factor: 4 # Multi-headed Attention Module's params - self_attention_model: rel_pos # rel_pos, abs_pos, rel_pos_old, rel_pos_old_xl + self_attention_model: rel_pos # rel_pos, abs_pos n_heads: 4 xscaling: true diff --git a/examples/asr/experimental/configs/conformer/conformer_char.yaml b/examples/asr/experimental/configs/conformer/conformer_char.yaml index 01fbd405fca8..263c98cb88e3 100644 --- a/examples/asr/experimental/configs/conformer/conformer_char.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_char.yaml @@ -82,7 +82,7 @@ model: ff_expansion_factor: 4 # Multi-headed Attention Module's params - self_attention_model: rel_pos # rel_pos, abs_pos, rel_pos_old, rel_pos_old_xl + self_attention_model: rel_pos # rel_pos, abs_pos n_heads: 4 xscaling: true diff --git a/nemo/collections/asr/modules/__init__.py b/nemo/collections/asr/modules/__init__.py index 68b972cdb243..502d036ea771 100644 --- a/nemo/collections/asr/modules/__init__.py +++ b/nemo/collections/asr/modules/__init__.py @@ -35,8 +35,6 @@ from nemo.collections.asr.modules.multi_head_attention import ( MultiHeadAttention, RelPositionMultiHeadAttention, - RelPositionMultiHeadAttention_old, - RelPositionalEncoding_old, RelPositionalEncoding, PositionalEncoding, ) \ No newline at end of file diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 60866dbbc4be..34df21c6bb2f 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -24,9 +24,7 @@ MultiHeadAttention, PositionalEncoding, RelPositionalEncoding, - RelPositionalEncoding_old, RelPositionMultiHeadAttention, - RelPositionMultiHeadAttention_old, ) from nemo.collections.asr.modules.subsampling import ConvSubsampling from nemo.core.classes.common import typecheck @@ -142,24 +140,16 @@ def __init__( self.u_bias = None self.v_bias = None - if 'old' in self_attention_model: - self.pos_enc = RelPositionalEncoding_old( - d_model=d_model, dropout=dropout, dropout_emb_rate=dropout_emb, xscale=self.xscale + if self_attention_model == "rel_pos": + self.pos_enc = RelPositionalEncoding( + d_model=d_model, dropout_rate=dropout_in, dropout_emb_rate=dropout_emb, xscale=self.xscale + ) + elif self_attention_model == "abs_pos": + self.pos_enc = PositionalEncoding( + d_model=d_model, dropout_rate=dropout_in, max_len=6000, reverse=False, xscale=self.xscale ) - if "xl" in self_attention_model: - self.u_bias = nn.Parameter(torch.Tensor(n_heads, d_model // n_heads)) - self.v_bias = nn.Parameter(torch.Tensor(n_heads, d_model // n_heads)) else: - if self_attention_model == "rel_pos": - self.pos_enc = RelPositionalEncoding( - d_model=d_model, dropout_rate=dropout_in, dropout_emb_rate=dropout_emb, xscale=self.xscale - ) - elif self_attention_model == "abs_pos": - self.pos_enc = PositionalEncoding( - d_model=d_model, dropout_rate=dropout_in, max_len=6000, reverse=False, xscale=self.xscale - ) - else: - raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") + raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") self.layers = nn.ModuleList() for i in range(n_layers): @@ -271,18 +261,6 @@ def __init__( self.self_attn = RelPositionMultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) elif self_attention_model == 'abs_pos': self.self_attn = MultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) - elif 'rel_pos_old' in self_attention_model: - self.self_attn = RelPositionMultiHeadAttention_old( - kdim=d_model, - qdim=d_model, - adim=d_model, - odim=d_model, - n_heads=n_heads, - dropout=dropout_att, - bias=False, - param_init='xavier_uniform', - xl_like=True if 'xl' in self_attention_model else False, - ) else: raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") @@ -317,8 +295,6 @@ def forward(self, x, att_mask=None, pos_emb=None, u_bias=None, v_bias=None, pad_ x = self.self_attn(query=x, key=x, value=x, pos_emb=pos_emb, mask=att_mask) elif self.self_attention_model == 'abs_pos': x = self.self_attn(query=x, key=x, value=x, mask=att_mask) - elif 'old' in self.self_attention_model: - x = self.self_attn(query=x, key=x, pos_emb=pos_emb, mask=att_mask, u_bias=u_bias, v_bias=v_bias) else: x = None x = self.dropout(x) + residual diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/modules/multi_head_attention.py index 44211d4b3680..f2c26a1e39ad 100644 --- a/nemo/collections/asr/modules/multi_head_attention.py +++ b/nemo/collections/asr/modules/multi_head_attention.py @@ -16,7 +16,7 @@ # """ -Part of this code is adopted from https://github.com/espnet/espnet and https://github.com/hirofumi0810/neural_sp/ +Part of this code is adopted from https://github.com/espnet/espnet """ import math @@ -28,9 +28,7 @@ from nemo.utils import logging __all__ = [ - 'RelPositionMultiHeadAttention_old', 'RelPositionMultiHeadAttention', - 'RelPositionalEncoding_old', 'RelPositionalEncoding', 'PositionalEncoding', ] @@ -276,208 +274,3 @@ def forward(self, x): if self.dropout_emb: pos_emb = self.dropout_emb(pos_emb) return self.dropout(x), pos_emb - - -class RelPositionMultiHeadAttention_old(nn.Module): - """Relative multi-head attention layer for TransformerXL. - Args: - kdim (int): dimension of key - qdim (int): dimension of query - adim: (int) dimension of attention space - odim: (int) dimension of output - n_heads (int): number of heads - dropout (float): dropout probability for attenion weights - dropout_head (float): HeadDrop probability - bias (bool): use bias term in linear layers - param_init (str): parameter initialization method - xl_like (bool): use TransformerXL like relative positional encoding. - Otherwise, use relative positional encoding like Shaw et al. 2018 - """ - - def __init__( - self, kdim, qdim, adim, odim, n_heads, dropout, dropout_head=0.0, bias=False, param_init='', xl_like=False - ): - - super().__init__() - - assert adim % n_heads == 0 - self.d_k = adim // n_heads - self.n_heads = n_heads - self.scale = math.sqrt(self.d_k) - self.xl_like = xl_like - - self.dropout_attn = nn.Dropout(p=dropout) - self.dropout_head = dropout_head - - assert kdim == qdim - # NOTE: relative attention is supprted for self-attention only - self.w_key = nn.Linear(kdim, adim, bias=bias) - self.w_value = nn.Linear(kdim, adim, bias=bias) - self.w_query = nn.Linear(qdim, adim, bias=bias) - self.w_out = nn.Linear(adim, odim, bias=bias) - - if xl_like: - self.w_pos = nn.Linear(qdim, adim, bias=bias) - - if param_init == 'xavier_uniform': - self.reset_parameters(bias) - else: - logging.info('Parameter initialization for RelativeMultiheadAttentionMechanism skipped.') - - def reset_parameters(self, bias): - """Initialize parameters with Xavier uniform distribution.""" - # NOTE: see https://github.com/pytorch/fairseq/blob/master/fairseq/modules/multihead_attention.py - nn.init.xavier_uniform_(self.w_key.weight, gain=1 / math.sqrt(2)) - nn.init.xavier_uniform_(self.w_value.weight, gain=1 / math.sqrt(2)) - nn.init.xavier_uniform_(self.w_query.weight, gain=1 / math.sqrt(2)) - if bias: - nn.init.constant_(self.w_key.bias, 0.0) - nn.init.constant_(self.w_value.bias, 0.0) - nn.init.constant_(self.w_query.bias, 0.0) - - nn.init.xavier_uniform_(self.w_out.weight) - if bias: - nn.init.constant_(self.w_out.bias, 0.0) - - if self.xl_like: - nn.init.xavier_uniform_(self.w_pos.weight) - if bias: - nn.init.constant_(self.w_pos.bias, 0.0) - - def _rel_shift(self, xs): - """Calculate relative positional attention efficiently. - Args: - xs (FloatTensor): `[B, qlen, klen, H]` - Returns: - xs_shifted (FloatTensor): `[B, qlen, klen, H]` - """ - bs, qlen, klen, n_heads = xs.size() - # `[qlen, klen, B, H]` -> `[B, qlen, klen, H]` - xs = xs.permute(1, 2, 0, 3).contiguous().view(qlen, klen, bs * n_heads) - - zero_pad = xs.new_zeros((qlen, 1, bs * n_heads)) - xs_shifted = torch.cat([zero_pad, xs], dim=1).view(klen + 1, qlen, bs * n_heads)[1:].view_as(xs) - return xs_shifted.view(qlen, klen, bs, n_heads).permute(2, 0, 1, 3) - - def forward(self, query, key, pos_emb, mask, u_bias=None, v_bias=None): - """Forward pass. - Args: - cat (FloatTensor): `[B, mlen+qlen, kdim]` - mask (ByteTensor): `[B, qlen, mlen+qlen]` - pos_emb (LongTensor): `[qlen, 1, d_model]` - u_bias (nn.Parameter): `[H, d_k]` - v_bias (nn.Parameter): `[H, d_k]` - Returns: - cv (FloatTensor): `[B, qlen, vdim]` - aw (FloatTensor): `[B, H, qlen, mlen+qlen]` - """ - bs, qlen = query.size()[:2] - mlen = key.size(1) - qlen - # NOTE: cat already includes memory, i.e., klen=mlen+qlen - - if mask is not None: - mask = mask.unsqueeze(3).repeat([1, 1, 1, self.n_heads]) - assert mask.size() == (bs, qlen, mlen + qlen, self.n_heads), ( - mask.size(), - (bs, qlen, mlen + qlen, self.n_heads), - ) - - k = self.w_key(key).view(bs, -1, self.n_heads, self.d_k) # `[B, mlen+qlen, H, d_k]` - v = self.w_value(key).view(bs, -1, self.n_heads, self.d_k) # `[B, mlen+qlen, H, d_k]` - q = self.w_query(key[:, -qlen:]).view(bs, -1, self.n_heads, self.d_k) # `[B, qlen, H, d_k]` - - if self.xl_like: - _pos_embs = self.w_pos(pos_emb) - else: - _pos_embs = self.w_value(pos_emb) - _pos_embs = _pos_embs.view(-1, self.n_heads, self.d_k) # `[mlen+qlen, H, d_k]` - - # content-based attention term: (a) + (c) - if u_bias is not None: - assert self.xl_like - AC = torch.einsum("bihd,bjhd->bijh", ((q + u_bias[None, None]), k)) # `[B, qlen, mlen+qlen, H]` - else: - AC = torch.einsum("bihd,bjhd->bijh", (q, k)) # `[B, qlen, mlen+qlen, H]` - - # position-based attention term: (b) + (d) - if v_bias is not None: - assert self.xl_like - BD = torch.einsum("bihd,jhd->bijh", ((q + v_bias[None, None]), _pos_embs)) # `[B, qlen, mlen+qlen, H]` - else: - BD = torch.einsum("bihd,jhd->bijh", (q, _pos_embs)) # `[B, qlen, mlen+qlen, H]` - - # Compute positional attention efficiently - BD = self._rel_shift(BD) - - # the attention is the sum of content-based and position-based attention - e = (AC + BD) / self.scale # `[B, qlen, mlen+qlen, H]` - - # Compute attention weights - if mask is not None: - NEG_INF = float(np.finfo(torch.tensor(0, dtype=e.dtype).numpy().dtype).min) - e = e.masked_fill_(mask == 0, NEG_INF) # `[B, qlen, mlen+qlen, H]` - aw = torch.softmax(e, dim=2) - aw = self.dropout_attn(aw) # `[B, qlen, mlen+qlen, H]` - - # mask out each head independently (HeadDrop) - if self.dropout_head > 0 and self.training: - aw_masked = aw.clone() - aw_masked = aw_masked.permute(0, 3, 1, 2) - aw_masked = self.dropout_head(aw_masked, self.n_heads, self.dropout_head) # `[B, H, qlen, klen]` - aw_masked = aw_masked.permute(0, 2, 3, 1) - - cv = torch.einsum("bijh,bjhd->bihd", (aw, v)) # `[B, qlen, H, d_k]` - cv = cv.contiguous().view(bs, -1, self.n_heads * self.d_k) # `[B, qlen, H * d_k]` - cv = self.w_out(cv) - # aw = aw.permute(0, 3, 1, 2) # `[B, H, qlen, mlen+qlen]` - - return cv # , aw - - -class RelPositionalEncoding_old(nn.Module): - def __init__(self, d_model, dropout, dropout_emb_rate=0.0, xscale=None): - """Positional embedding for TransformerXL.""" - super().__init__() - self.d_model = d_model - self.xscale = xscale - inv_freq = 1 / (10000 ** (torch.arange(0.0, d_model, 2.0) / d_model)) - self.register_buffer("inv_freq", inv_freq) - - self.dropout = nn.Dropout(p=dropout) - - if dropout_emb_rate > 0: - self.dropout_emb = nn.Dropout(dropout_emb_rate) - else: - self.dropout_emb = None - - self.xscale = 1 # math.sqrt(self.d_model) - - def forward(self, xs, mlen=0, clamp_len=-1, zero_center_offset=False): - """Forward pass. - Args: - xs (FloatTensor): `[B, L, d_model]` - mlen (int); length of memory - clamp_len (int): - zero_center_offset (bool): - Returns: - pos_emb (LongTensor): `[L, 1, d_model]` - """ - if zero_center_offset: - pos_idxs = torch.arange(mlen - 1, -xs.size(1) - 1, -1.0, dtype=torch.float, device=xs.device) - else: - pos_idxs = torch.arange(mlen + xs.size(1) - 1, -1, -1.0, dtype=torch.float, device=xs.device) - - # truncate by maximum length - if clamp_len > 0: - pos_idxs.clamp_(max=clamp_len) - - # outer product - sinusoid_inp = torch.einsum("i,j->ij", pos_idxs, self.inv_freq) - pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1) - - if self.dropout_emb: - pos_emb = self.dropout_emb(pos_emb) - if self.xscale: - xs = self.xscale * xs - - return self.dropout(xs), pos_emb.unsqueeze(1) From 7213125547c8f6b2a7b913c67c62cea321ac0c7e Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 16:25:01 -0700 Subject: [PATCH 05/67] Dropped dropout_in and params. Signed-off-by: Vahid --- .../experimental/configs/conformer/conformer_bpe.yaml | 7 +++---- .../experimental/configs/conformer/conformer_char.yaml | 7 +++---- examples/asr/speech_to_text.py | 4 ++-- nemo/collections/asr/models/ctc_bpe_models.py | 10 +++++----- nemo/collections/asr/models/ctc_models.py | 4 ++-- nemo/collections/asr/modules/conformer_encoder.py | 5 ++--- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml index 4a049a5e5253..8905850df972 100644 --- a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_bpe.yaml @@ -93,10 +93,9 @@ model: conv_kernel_size: 31 ### regularization - dropout: 0.1 - dropout_emb: 0.1 - dropout_in: 0.1 - dropout_att: 0.0 + dropout: 0.1 # The dropout used inside the Conformer Modules + dropout_emb: 0.1 # The dropout used embeddings + dropout_att: 0.0 # The dropout for multi-headed attention modules decoder: _target_: nemo.collections.asr.modules.LSTMDecoder diff --git a/examples/asr/experimental/configs/conformer/conformer_char.yaml b/examples/asr/experimental/configs/conformer/conformer_char.yaml index 263c98cb88e3..c6f997d527f8 100644 --- a/examples/asr/experimental/configs/conformer/conformer_char.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_char.yaml @@ -90,10 +90,9 @@ model: conv_kernel_size: 31 ### regularization - dropout: 0.1 - dropout_emb: 0.1 - dropout_in: 0.1 - dropout_att: 0.0 + dropout: 0.1 # The dropout used inside the Conformer Modules + dropout_emb: 0.1 # The dropout used embeddings + dropout_att: 0.0 # The dropout for multi-headed attention modules decoder: _target_: nemo.collections.asr.modules.LSTMDecoder diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index c77d5cf015f7..47e205d25cab 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -45,8 +45,8 @@ hydra.run.dir="." \ trainer.gpus=2 \ trainer.max_epochs=2 \ - model.optim.args.params.betas=[0.8,0.5] \ - model.optim.args.params.weight_decay=0.0001 + model.optim.args.betas=[0.8,0.5] \ + model.optim.args.weight_decay=0.0001 Overide optimizer entirely python speech_to_text.py \ diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index 1ee033766961..f55357d4d1ff 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -67,16 +67,16 @@ def __init__(self, cfg: DictConfig, trainer=None): vocabulary = self.tokenizer.tokenizer.get_vocab() # Set the new vocabulary - cfg.decoder.params.vocabulary = ListConfig(list(vocabulary.values())) + cfg.decoder.vocabulary = ListConfig(list(vocabulary.values())) # Override number of classes if placeholder provided - if cfg.decoder.params['num_classes'] < 1: + if cfg.decoder['num_classes'] < 1: logging.info( "\nReplacing placeholder number of classes ({}) with actual number of classes - {}".format( - cfg.decoder.params['num_classes'], len(vocabulary) + cfg.decoder['num_classes'], len(vocabulary) ) ) - cfg.decoder.params['num_classes'] = len(vocabulary) + cfg.decoder['num_classes'] = len(vocabulary) super().__init__(cfg=cfg, trainer=trainer) @@ -288,7 +288,7 @@ def change_vocabulary(self, new_tokenizer_dir: str, new_tokenizer_type: str): # Set the new vocabulary decoder_config = copy.deepcopy(self.decoder.to_config_dict()) - decoder_config.params.vocabulary = ListConfig(list(vocabulary.values())) + decoder_config.vocabulary = ListConfig(list(vocabulary.values())) # Override number of classes if placeholder provided logging.info( diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 906bf6d592ad..5110db9c38ff 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -204,8 +204,8 @@ def change_vocabulary(self, new_vocabulary: List[str]): raise ValueError(f'New vocabulary must be non-empty list of chars. But I got: {new_vocabulary}') decoder_config = self.decoder.to_config_dict() new_decoder_config = copy.deepcopy(decoder_config) - new_decoder_config['params']['vocabulary'] = new_vocabulary - new_decoder_config['params']['num_classes'] = len(new_vocabulary) + new_decoder_config['vocabulary'] = new_vocabulary + new_decoder_config['num_classes'] = len(new_vocabulary) del self.decoder self.decoder = EncDecCTCModel.from_config_dict(new_decoder_config) del self.loss diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 34df21c6bb2f..2656005eb410 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -109,7 +109,6 @@ def __init__( conv_kernel_size=31, dropout=0.1, dropout_emb=0.1, - dropout_in=0.1, dropout_att=0.0, ): super().__init__() @@ -142,11 +141,11 @@ def __init__( if self_attention_model == "rel_pos": self.pos_enc = RelPositionalEncoding( - d_model=d_model, dropout_rate=dropout_in, dropout_emb_rate=dropout_emb, xscale=self.xscale + d_model=d_model, dropout_rate=dropout, dropout_emb_rate=dropout_emb, xscale=self.xscale ) elif self_attention_model == "abs_pos": self.pos_enc = PositionalEncoding( - d_model=d_model, dropout_rate=dropout_in, max_len=6000, reverse=False, xscale=self.xscale + d_model=d_model, dropout_rate=dropout, max_len=6000, reverse=False, xscale=self.xscale ) else: raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") From a4c4c79b4c7cb33c2da18a37810395c263567a98 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 16:26:20 -0700 Subject: [PATCH 06/67] Fixed code style. Signed-off-by: Vahid --- nemo/collections/asr/losses/__init__.py | 2 +- nemo/collections/asr/models/ctc_bpe_models.py | 15 +++++++++++++-- nemo/collections/asr/modules/__init__.py | 17 +++++++---------- .../asr/modules/conformer_encoder.py | 4 +--- nemo/core/config/__init__.py | 2 +- nemo/core/optim/__init__.py | 2 +- 6 files changed, 24 insertions(+), 18 deletions(-) diff --git a/nemo/collections/asr/losses/__init__.py b/nemo/collections/asr/losses/__init__.py index aa0a5ac45449..d5d8200ead58 100644 --- a/nemo/collections/asr/losses/__init__.py +++ b/nemo/collections/asr/losses/__init__.py @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nemo.collections.asr.losses.ctc import CTCLoss from nemo.collections.asr.losses.angularloss import AngularSoftmaxLoss +from nemo.collections.asr.losses.ctc import CTCLoss diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index f55357d4d1ff..adc94ccb3b4b 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -82,7 +82,12 @@ def __init__(self, cfg: DictConfig, trainer=None): # Setup metric objects self._wer = WERBPE( - tokenizer=self.tokenizer, batch_dim_index=0, use_cer=False, ctc_decode=True, dist_sync_on_step=True, log_prediction=self._cfg.get("log_prediction", False) + tokenizer=self.tokenizer, + batch_dim_index=0, + use_cer=False, + ctc_decode=True, + dist_sync_on_step=True, + log_prediction=self._cfg.get("log_prediction", False), ) def _setup_tokenizer(self): @@ -302,7 +307,13 @@ def change_vocabulary(self, new_tokenizer_dir: str, new_tokenizer_type: str): self.decoder = EncDecCTCModelBPE.from_config_dict(decoder_config) del self.loss self.loss = CTCLoss(num_classes=self.decoder.num_classes_with_blank - 1, zero_infinity=True) - self._wer = WERBPE(tokenizer=self.tokenizer, batch_dim_index=0, use_cer=False, ctc_decode=True, log_prediction=self._cfg.get("log_prediction", False)) + self._wer = WERBPE( + tokenizer=self.tokenizer, + batch_dim_index=0, + use_cer=False, + ctc_decode=True, + log_prediction=self._cfg.get("log_prediction", False), + ) # Update config OmegaConf.set_struct(self._cfg.decoder, False) diff --git a/nemo/collections/asr/modules/__init__.py b/nemo/collections/asr/modules/__init__.py index 502d036ea771..49e6cf502e6a 100644 --- a/nemo/collections/asr/modules/__init__.py +++ b/nemo/collections/asr/modules/__init__.py @@ -12,29 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +from nemo.collections.asr.modules.activations import Swish from nemo.collections.asr.modules.audio_preprocessing import ( AudioToMelSpectrogramPreprocessor, AudioToMFCCPreprocessor, CropOrPadSpectrogramAugmentation, SpectrogramAugmentation, ) +from nemo.collections.asr.modules.conformer_encoder import ConformerEncoder +from nemo.collections.asr.modules.conformer_modules import ConformerConvolution, ConformerFeedForward from nemo.collections.asr.modules.conv_asr import ( ConvASRDecoder, ConvASRDecoderClassification, ConvASREncoder, SpeakerDecoder, ) - -from nemo.collections.asr.modules.conformer_encoder import ConformerEncoder from nemo.collections.asr.modules.lstm_decoder import LSTMDecoder - -from nemo.collections.asr.modules.activations import Swish -from nemo.collections.asr.modules.conformer_modules import ConformerConvolution, ConformerFeedForward - -from nemo.collections.asr.modules.subsampling import ConvSubsampling from nemo.collections.asr.modules.multi_head_attention import ( MultiHeadAttention, - RelPositionMultiHeadAttention, - RelPositionalEncoding, PositionalEncoding, -) \ No newline at end of file + RelPositionalEncoding, + RelPositionMultiHeadAttention, +) +from nemo.collections.asr.modules.subsampling import ConvSubsampling diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 2656005eb410..779394a5ba05 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -237,9 +237,7 @@ class ConformerEncoderBlock(torch.nn.Module): dropout_att (float): dropout probabilities for attention distributions """ - def __init__( - self, d_model, d_ff, conv_kernel_size, self_attention_model, n_heads, dropout, dropout_att - ): + def __init__(self, d_model, d_ff, conv_kernel_size, self_attention_model, n_heads, dropout, dropout_att): super(ConformerEncoderBlock, self).__init__() self.self_attention_model = self_attention_model diff --git a/nemo/core/config/__init__.py b/nemo/core/config/__init__.py index 0e192771be3c..9eb915cc4ac5 100644 --- a/nemo/core/config/__init__.py +++ b/nemo/core/config/__init__.py @@ -31,8 +31,8 @@ from nemo.core.config.pytorch_lightning import TrainerConfig from nemo.core.config.schedulers import ( CosineAnnealingParams, - NoamAnnealingParams, InverseSquareRootAnnealingParams, + NoamAnnealingParams, PolynomialDecayAnnealingParams, PolynomialHoldDecayAnnealingParams, SchedulerParams, diff --git a/nemo/core/optim/__init__.py b/nemo/core/optim/__init__.py index e847d121df36..b421582c0034 100644 --- a/nemo/core/optim/__init__.py +++ b/nemo/core/optim/__init__.py @@ -14,8 +14,8 @@ from nemo.core.optim.lr_scheduler import ( CosineAnnealing, - NoamAnnealing, InverseSquareRootAnnealing, + NoamAnnealing, PolynomialDecayAnnealing, PolynomialHoldDecayAnnealing, SquareAnnealing, From 2c900aa2210681fd54480024469dcdc0af1ea802 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 17:21:34 -0700 Subject: [PATCH 07/67] Updated docs. Signed-off-by: Vahid --- .../asr/modules/conformer_encoder.py | 4 +- nemo/collections/asr/modules/lstm_decoder.py | 9 +- .../asr/modules/multi_head_attention.py | 107 +++++++++--------- nemo/collections/asr/modules/subsampling.py | 42 ++++--- 4 files changed, 85 insertions(+), 77 deletions(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 779394a5ba05..41416b06ee1b 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -126,8 +126,8 @@ def __init__( self.pre_encode = ConvSubsampling( subsampling=subsampling, subsampling_factor=subsampling_factor, - idim=feat_in, - odim=d_model, + feat_in=feat_in, + feat_out=d_model, conv_channels=subsampling_conv_channels, activation=nn.ReLU(), ) diff --git a/nemo/collections/asr/modules/lstm_decoder.py b/nemo/collections/asr/modules/lstm_decoder.py index 149d9b1fb9fa..1a00c9077be0 100644 --- a/nemo/collections/asr/modules/lstm_decoder.py +++ b/nemo/collections/asr/modules/lstm_decoder.py @@ -27,7 +27,14 @@ class LSTMDecoder(NeuralModule, Exportable): """ - Simple LSTM Decoder for use with CTC-based ASR models + Simple LSTM Decoder for ASR models + Args: + feat_in (int): size of the input features + num_classes (int): the size of the vocabulary + lstm_hidden_size (int): hidden size of the LSTM layers + vocabulary (vocab): The vocabulary + bidirectional (bool): default is False. Whether LSTMs are bidirectional or not + num_layers (int): default is 1. Number of LSTM layers stacked """ def save_to(self, save_path: str): diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/modules/multi_head_attention.py index f2c26a1e39ad..5a9501d963a5 100644 --- a/nemo/collections/asr/modules/multi_head_attention.py +++ b/nemo/collections/asr/modules/multi_head_attention.py @@ -36,9 +36,10 @@ class MultiHeadAttention(nn.Module): """Multi-Head Attention layer. - :param int n_head: the number of head s - :param int n_feat: the number of features - :param float dropout_rate: dropout rate + Args: + n_head (int): number of heads + n_feat (int): size of the features + dropout_rate (float): dropout rate """ def __init__(self, n_head, n_feat, dropout_rate): @@ -56,29 +57,34 @@ def __init__(self, n_head, n_feat, dropout_rate): self.dropout = nn.Dropout(p=dropout_rate) def forward_qkv(self, query, key, value): - """Transform query, key and value. - :param torch.Tensor query: (batch, time1, size) - :param torch.Tensor key: (batch, time2, size) - :param torch.Tensor value: (batch, time2, size) - :return torch.Tensor transformed query, key and value + """Transforms query, key and value. + Args: + query (torch.Tensor): (batch, time1, size) + key (torch.Tensor): (batch, time2, size) + value (torch.Tensor): (batch, time2, size) + returns: + q (torch.Tensor): (batch, head, time1, size) + k (torch.Tensor): (batch, head, time2, size) + v (torch.Tensor): (batch, head, time2, size) """ n_batch = query.size(0) q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) - q = q.transpose(1, 2) # (batch, head, time1, d_k) - k = k.transpose(1, 2) # (batch, head, time2, d_k) - v = v.transpose(1, 2) # (batch, head, time2, d_k) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) return q, k, v def forward_attention(self, value, scores, mask): """Compute attention context vector. - :param torch.Tensor value: (batch, time2, size) - :param torch.Tensor scores: (batch, time1, time2) - :param torch.Tensor mask: (batch, time1, time2) - :return torch.Tensor transformed `value` (batch, time2, d_model) - weighted by the attention score (batch, time1, time2) + Args: + value (torch.Tensor): (batch, time2, size) + scores(torch.Tensor): (batch, time1, time2) + mask(torch.Tensor): (batch, time1, time2) + returns: + value (torch.Tensor): transformed `value` (batch, time2, d_model) weighted by the attention scores """ n_batch = value.size(0) if mask is not None: @@ -97,13 +103,13 @@ def forward_attention(self, value, scores, mask): def forward(self, query, key, value, mask): """Compute 'Scaled Dot Product Attention'. - :param torch.Tensor query: (batch, time1, size) - :param torch.Tensor key: (batch, time2, size) - :param torch.Tensor value: (batch, time2, size) - :param torch.Tensor mask: (batch, time1, time2) - :param torch.nn.Dropout dropout: - :return torch.Tensor: attentined and transformed `value` (batch, time1, d_model) - weighted by the query dot key attention (batch, head, time1, time2) + Args: + query (torch.Tensor): (batch, time1, size) + key (torch.Tensor): (batch, time2, size) + value(torch.Tensor): (batch, time2, size) + mask (torch.Tensor): (batch, time1, time2) + returns: + output (torch.Tensor): transformed `value` (batch, time1, d_model) weighted by the query dot key attention """ q, k, v = self.forward_qkv(query, key, value) scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) @@ -113,9 +119,10 @@ def forward(self, query, key, value, mask): class RelPositionMultiHeadAttention(MultiHeadAttention): """Multi-Head Attention layer with relative position encoding. Paper: https://arxiv.org/abs/1901.02860 - :param int n_head: the number of head s - :param int n_feat: the number of features - :param float dropout_rate: dropout rate + Args: + n_head (int): number of heads + n_feat (int): size of the features + dropout_rate (float): dropout rate """ def __init__(self, n_head, n_feat, dropout_rate): @@ -132,8 +139,9 @@ def __init__(self, n_head, n_feat, dropout_rate): def rel_shift(self, x, zero_triu=False): """Compute relative positinal encoding. - :param torch.Tensor x: (batch, time, size) - :param bool zero_triu: return the lower triangular part of the matrix + Args: + x (torch.Tensor): (batch, time, size) + zero_triu (bool): return the lower triangular part of the matrix """ zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) x_padded = torch.cat([zero_pad, x], dim=-1) @@ -149,14 +157,14 @@ def rel_shift(self, x, zero_triu=False): def forward(self, query, key, value, mask, pos_emb): """Compute 'Scaled Dot Product Attention' with rel. positional encoding. - :param torch.Tensor query: (batch, time1, size) - :param torch.Tensor key: (batch, time2, size) - :param torch.Tensor value: (batch, time2, size) - :param torch.Tensor pos_emb: (batch, time1, size) - :param torch.Tensor mask: (batch, time1, time2) - :param torch.nn.Dropout dropout: - :return torch.Tensor: attentined and transformed `value` (batch, time1, d_model) - weighted by the query dot key attention (batch, head, time1, time2) + Args: + query (torch.Tensor): (batch, time1, size) + key (torch.Tensor): (batch, time2, size) + value(torch.Tensor): (batch, time2, size) + mask (torch.Tensor): (batch, time1, time2) + pos_emb (torch.Tensor) : (batch, time1, size) + Returns: + output (torch.Tensor): transformed `value` (batch, time1, d_model) weighted by the query dot key attention """ q, k, v = self.forward_qkv(query, key, value) q = q.transpose(1, 2) # (batch, time1, head, d_k) @@ -188,10 +196,11 @@ def forward(self, query, key, value, mask, pos_emb): class PositionalEncoding(torch.nn.Module): """Positional encoding. - :param int d_model: embedding dim - :param float dropout_rate: dropout rate - :param int max_len: maximum input length - :param reverse: whether to reverse the input position + Args: + d_model (int): embedding dim + dropout_rate (float): dropout rate + max_len (int): maximum input length + reverse (int): whether to reverse the input position """ def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False, xscale=None): @@ -229,7 +238,7 @@ def forward(self, x: torch.Tensor): Args: x (torch.Tensor): Input. Its shape is (batch, time, ...) Returns: - torch.Tensor: Encoded tensor. Its shape is (batch, time, ...) + Encoded Output (torch.Tensor): Its shape is (batch, time, ...) """ self.extend_pe(x) if self.xscale: @@ -241,17 +250,13 @@ def forward(self, x: torch.Tensor): class RelPositionalEncoding(PositionalEncoding): """Relitive positional encoding module. See : Appendix B in https://arxiv.org/abs/1901.02860 - :param int d_model: embedding dim - :param float dropout_rate: dropout rate - :param int max_len: maximum input length + Args: + d_model (int): embedding dim + dropout_rate (float): dropout rate + max_len (int): maximum input length """ def __init__(self, d_model, dropout_rate, max_len=5000, dropout_emb_rate=0.0, xscale=None): - """Initialize class. - :param int d_model: embedding dim - :param float dropout_rate: dropout rate - :param int max_len: maximum input length - """ super().__init__(d_model, dropout_rate, max_len, reverse=True, xscale=xscale) if dropout_emb_rate > 0: @@ -264,8 +269,8 @@ def forward(self, x): Args: x (torch.Tensor): Input. Its shape is (batch, time, ...) Returns: - torch.Tensor: x. Its shape is (batch, time, ...) - torch.Tensor: pos_emb. Its shape is (1, time, ...) + x (torch.Tensor): Its shape is (batch, time, ...) + pos_emb (torch.Tensor): Its shape is (1, time, ...) """ self.extend_pe(x) if self.xscale: diff --git a/nemo/collections/asr/modules/subsampling.py b/nemo/collections/asr/modules/subsampling.py index 2d1d5e07c320..85f31611b8b4 100644 --- a/nemo/collections/asr/modules/subsampling.py +++ b/nemo/collections/asr/modules/subsampling.py @@ -19,19 +19,21 @@ class ConvSubsampling(torch.nn.Module): - """Convolutional 2D subsampling (to 1/4 length). - - :param int idim: input dim - :param int odim: output dim - :param str activation: activation functions - :param flaot dropout_rate: dropout rate - + """Convolutional subsampling which supports VGGNet and striding approach introduced in: + VGGNet Subsampling: https://arxiv.org/pdf/1910.12977.pdf + Striding Subsampling: + "Speech-Transformer: A No-Recurrence Sequence-to-Sequence Model for Speech Recognition" by Linhao Dong et al. + Args: + subsampling (str): The subsampling technique from {"vggnet", "striding"} + subsampling_factor (int): The subsampling factor which should be a power of 2 + feat_in (int): size of the input features + feat_out (int): size of the output features + conv_channels (int): Number of channels for the convolution layers. + activation (Module): activation function, default is nn.ReLU() """ - def __init__(self, subsampling, subsampling_factor, idim, odim, conv_channels=-1, activation=nn.ReLU()): + def __init__(self, subsampling, subsampling_factor, feat_in, feat_out, conv_channels, activation=nn.ReLU()): super(ConvSubsampling, self).__init__() - if conv_channels <= 0: - conv_channels = odim self._subsampling = subsampling if subsampling_factor % 2 != 0: @@ -44,7 +46,7 @@ def __init__(self, subsampling, subsampling_factor, idim, odim, conv_channels=-1 self._padding = 0 self._stride = 2 self._kernel_size = 2 - self._ceil_mode = True # TODO: is False better? + self._ceil_mode = True for i in range(self._sampling_num): layers.append( @@ -69,7 +71,7 @@ def __init__(self, subsampling, subsampling_factor, idim, odim, conv_channels=-1 ) in_channels = conv_channels elif subsampling == 'striding': - self._padding = 1 # TODO: is 0 better? + self._padding = 1 self._stride = 2 self._kernel_size = 3 self._ceil_mode = False @@ -89,7 +91,7 @@ def __init__(self, subsampling, subsampling_factor, idim, odim, conv_channels=-1 else: raise ValueError(f"Not valid sub-sampling: {subsampling}!") - in_length = idim + in_length = feat_in for i in range(self._sampling_num): out_length = calc_length( length=int(in_length), @@ -100,23 +102,16 @@ def __init__(self, subsampling, subsampling_factor, idim, odim, conv_channels=-1 ) in_length = out_length - self.out = torch.nn.Linear(conv_channels * out_length, odim) + self.out = torch.nn.Linear(conv_channels * out_length, feat_out) self.conv = torch.nn.Sequential(*layers) def forward(self, x, lengths): - """Subsample x. - - :param torch.Tensor x: input tensor - :param torch.Tensor x_mask: input mask - :return: subsampled x and mask - :rtype Tuple[torch.Tensor or Tuple[torch.Tensor, torch.Tensor], torch.Tensor] - """ - x = x.unsqueeze(1) # (b, c, t, f) + x = x.unsqueeze(1) x = self.conv(x) b, c, t, f = x.size() x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) - # TODO: improve the performance of here + # TODO: improve the performance of length calculation new_lengths = lengths for i in range(self._sampling_num): new_lengths = [ @@ -135,6 +130,7 @@ def forward(self, x, lengths): def calc_length(length, padding, kernel_size, stride, ceil_mode): + """ Calculates the output length of a Tensor passed through a convolution or max pooling layer""" if ceil_mode: length = math.ceil((length + (2 * padding) - (kernel_size - 1) - 1) / float(stride) + 1) else: From ac86cb3b4c4de575e84503a732df7492248d3fd4 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 18:20:56 -0700 Subject: [PATCH 08/67] Removed unused import. Signed-off-by: Vahid --- nemo/collections/asr/modules/multi_head_attention.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/modules/multi_head_attention.py index 5a9501d963a5..7e12a724bcee 100644 --- a/nemo/collections/asr/modules/multi_head_attention.py +++ b/nemo/collections/asr/modules/multi_head_attention.py @@ -1,6 +1,5 @@ # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # Copyright 2017 Johns Hopkins University (Shinji Watanabe) -# Copyright 2020 Hirofumi Inaguma(Kyoto University) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,8 +24,6 @@ import torch import torch.nn as nn -from nemo.utils import logging - __all__ = [ 'RelPositionMultiHeadAttention', 'RelPositionalEncoding', From 75aa1e4754ad21e689693ca56a1c4d12194f595d Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 18:39:13 -0700 Subject: [PATCH 09/67] fixed docs. Signed-off-by: Vahid --- .../asr/modules/conformer_encoder.py | 28 +++++++------------ .../asr/modules/conformer_modules.py | 1 - 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 41416b06ee1b..458cb413f010 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -39,7 +39,8 @@ class ConformerEncoder(NeuralModule, Exportable): """ The encoder for ASR model of Conformer. Based on this paper: - https://arxiv.org/abs/2005.08100 + 'Conformer: Convolution-augmented Transformer for Speech Recognition' by Anmol Gulati et al. + https://arxiv.org/abs/2005.08100 """ def _prepare_for_export(self): @@ -98,7 +99,7 @@ def __init__( feat_in, n_layers, d_model, - feat_out=0, + feat_out=-1, subsampling='vggnet', subsampling_factor=4, subsampling_conv_channels=64, @@ -206,13 +207,7 @@ def forward(self, audio_signal, length): @staticmethod def make_pad_mask(seq_lens, max_time, device=None): - """Make masking for padding. - Args: - seq_lens (IntTensor): `[B]` - device_id (int): - Returns: - mask (IntTensor): `[B, T]` - """ + """Make masking for padding.""" bs = seq_lens.size(0) seq_range = torch.arange(0, max_time, dtype=torch.int32) seq_range_expand = seq_range.unsqueeze(0).expand(bs, max_time) @@ -268,18 +263,15 @@ def __init__(self, d_model, d_ff, conv_kernel_size, self_attention_model, n_head self.dropout = nn.Dropout(dropout) self.norm_out = LayerNorm(d_model) - def forward(self, x, att_mask=None, pos_emb=None, u_bias=None, v_bias=None, pad_mask=None): + def forward(self, x, att_mask=None, pos_emb=None, pad_mask=None): """ Args: - x (FloatTensor): `[B, T, d_model]` - att_mask (ByteTensor): `[B, T, T]` - pos_emb (LongTensor): `[L, 1, d_model]` - u (FloatTensor): global parameter for relative positional embedding - v (FloatTensor): global parameter for relative positional embedding + x (torch.Tensor): input signals (B, T, d_model) + att_mask (torch.Tensor): attention masks(B, T, T) + pos_emb (torch.Tensor): (L, 1, d_model) + pad_mask (torch.tensor): padding mask Returns: - xs (FloatTensor): `[B, T, d_model]` - xx_aws (FloatTensor): `[B, H, T, T]` - + x (torch.Tensor): (B, T, d_model) """ residual = x x = self.norm_feed_forward1(x) diff --git a/nemo/collections/asr/modules/conformer_modules.py b/nemo/collections/asr/modules/conformer_modules.py index 5662ddbfe439..8e6e0a0fefbc 100644 --- a/nemo/collections/asr/modules/conformer_modules.py +++ b/nemo/collections/asr/modules/conformer_modules.py @@ -70,7 +70,6 @@ class ConformerFeedForward(nn.Module): """ feed-forward module of Conformer model. """ - def __init__(self, d_model, d_ff, dropout, activation=Swish()): super(ConformerFeedForward, self).__init__() self.linear1 = nn.Linear(d_model, d_ff) From 25fff7f7a5b080b21cba7d6f07353435c803e047 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 21:52:27 -0700 Subject: [PATCH 10/67] Fixed license header. Signed-off-by: Vahid --- .../asr/modules/multi_head_attention.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/modules/multi_head_attention.py index 7e12a724bcee..47549a0f7ca7 100644 --- a/nemo/collections/asr/modules/multi_head_attention.py +++ b/nemo/collections/asr/modules/multi_head_attention.py @@ -14,6 +14,21 @@ # limitations under the License. # +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """ Part of this code is adopted from https://github.com/espnet/espnet """ From 09d217c72f92ae9a2df77a454834f69fff413646 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 21 Oct 2020 23:59:39 -0700 Subject: [PATCH 11/67] Fixed license header. Signed-off-by: Vahid --- nemo/collections/asr/modules/multi_head_attention.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/modules/multi_head_attention.py index 47549a0f7ca7..9d1d5bbbdbd1 100644 --- a/nemo/collections/asr/modules/multi_head_attention.py +++ b/nemo/collections/asr/modules/multi_head_attention.py @@ -1,5 +1,4 @@ # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# Copyright 2017 Johns Hopkins University (Shinji Watanabe) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From cda38ae4df75c0be717278d93ff0cd60013afd28 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 00:47:23 -0700 Subject: [PATCH 12/67] fixed style. Signed-off-by: Vahid --- nemo/collections/asr/modules/conformer_modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nemo/collections/asr/modules/conformer_modules.py b/nemo/collections/asr/modules/conformer_modules.py index 8e6e0a0fefbc..5662ddbfe439 100644 --- a/nemo/collections/asr/modules/conformer_modules.py +++ b/nemo/collections/asr/modules/conformer_modules.py @@ -70,6 +70,7 @@ class ConformerFeedForward(nn.Module): """ feed-forward module of Conformer model. """ + def __init__(self, d_model, d_ff, dropout, activation=Swish()): super(ConformerFeedForward, self).__init__() self.linear1 = nn.Linear(d_model, d_ff) From c24b0f697722bf781b0c6680d6f096d12acc7ff7 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 11:59:36 -0700 Subject: [PATCH 13/67] Updated tests with _target_. Signed-off-by: Vahid --- .../asr/test_asr_ctc_encoder_model_bpe.py | 44 ++++---- .../asr/test_asr_ctcencdec_model.py | 106 +++++++++--------- 2 files changed, 72 insertions(+), 78 deletions(-) diff --git a/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py b/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py index 3b9cca9ad302..0da826d3e3ed 100644 --- a/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py +++ b/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py @@ -25,33 +25,31 @@ @pytest.fixture() def asr_model(test_data_dir): - preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} + preprocessor = {'_target_': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor'} encoder = { - 'cls': 'nemo.collections.asr.modules.ConvASREncoder', - 'params': { - 'feat_in': 64, - 'activation': 'relu', - 'conv_mask': True, - 'jasper': [ - { - 'filters': 1024, - 'repeat': 1, - 'kernel': [1], - 'stride': [1], - 'dilation': [1], - 'dropout': 0.0, - 'residual': False, - 'separable': True, - 'se': True, - 'se_context_size': -1, - } - ], - }, + '_target_': 'nemo.collections.asr.modules.ConvASREncoder', + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 1024, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], } decoder = { - 'cls': 'nemo.collections.asr.modules.ConvASRDecoder', - 'params': {'feat_in': 1024, 'num_classes': -1, 'vocabulary': None}, + '_target_': 'nemo.collections.asr.modules.ConvASRDecoder', + 'feat_in': 1024, 'num_classes': -1, 'vocabulary': None, } tokenizer = {'dir': os.path.join(test_data_dir, "asr", "tokenizers", "an4_wpe_128"), 'type': 'wpe'} diff --git a/tests/collections/asr/test_asr_ctcencdec_model.py b/tests/collections/asr/test_asr_ctcencdec_model.py index 151b7e96fb69..7df1c8393e4d 100644 --- a/tests/collections/asr/test_asr_ctcencdec_model.py +++ b/tests/collections/asr/test_asr_ctcencdec_model.py @@ -23,64 +23,60 @@ def asr_model(): preprocessor = {'cls': 'nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor', 'params': dict({})} encoder = { - 'cls': 'nemo.collections.asr.modules.ConvASREncoder', - 'params': { - 'feat_in': 64, - 'activation': 'relu', - 'conv_mask': True, - 'jasper': [ - { - 'filters': 1024, - 'repeat': 1, - 'kernel': [1], - 'stride': [1], - 'dilation': [1], - 'dropout': 0.0, - 'residual': False, - 'separable': True, - 'se': True, - 'se_context_size': -1, - } - ], - }, + '_target_': 'nemo.collections.asr.modules.ConvASREncoder', + 'feat_in': 64, + 'activation': 'relu', + 'conv_mask': True, + 'jasper': [ + { + 'filters': 1024, + 'repeat': 1, + 'kernel': [1], + 'stride': [1], + 'dilation': [1], + 'dropout': 0.0, + 'residual': False, + 'separable': True, + 'se': True, + 'se_context_size': -1, + } + ], } decoder = { - 'cls': 'nemo.collections.asr.modules.ConvASRDecoder', - 'params': { - 'feat_in': 1024, - 'num_classes': 28, - 'vocabulary': [ - ' ', - 'a', - 'b', - 'c', - 'd', - 'e', - 'f', - 'g', - 'h', - 'i', - 'j', - 'k', - 'l', - 'm', - 'n', - 'o', - 'p', - 'q', - 'r', - 's', - 't', - 'u', - 'v', - 'w', - 'x', - 'y', - 'z', - "'", - ], - }, + '_target_': 'nemo.collections.asr.modules.ConvASRDecoder', + 'feat_in': 1024, + 'num_classes': 28, + 'vocabulary': [ + ' ', + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + 'l', + 'm', + 'n', + 'o', + 'p', + 'q', + 'r', + 's', + 't', + 'u', + 'v', + 'w', + 'x', + 'y', + 'z', + "'", + ], } modelConfig = DictConfig( {'preprocessor': DictConfig(preprocessor), 'encoder': DictConfig(encoder), 'decoder': DictConfig(decoder)} From 7a619052ec03643f2ae77fdb74749bcbe76458b8 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 14:59:38 -0700 Subject: [PATCH 14/67] Updated tests with _target_. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_bpe_models.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index adc94ccb3b4b..c5af8ae6ff6b 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -70,13 +70,21 @@ def __init__(self, cfg: DictConfig, trainer=None): cfg.decoder.vocabulary = ListConfig(list(vocabulary.values())) # Override number of classes if placeholder provided - if cfg.decoder['num_classes'] < 1: + if "params" in cfg.decoder: + num_c = cfg.decoder["params"]["num_classes"] + else: + num_c = cfg.decoder["num_classes"] + + if num_c < 1: logging.info( "\nReplacing placeholder number of classes ({}) with actual number of classes - {}".format( - cfg.decoder['num_classes'], len(vocabulary) + num_c, len(vocabulary) ) ) - cfg.decoder['num_classes'] = len(vocabulary) + if "params" in cfg.decoder: + cfg.decoder["params"]["num_classes"] = len(vocabulary) + else: + cfg.decoder['num_classes'] = len(vocabulary) super().__init__(cfg=cfg, trainer=trainer) From 9420578d205c2a66a1df110eb3e442b0bf5d6dc9 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 15:14:55 -0700 Subject: [PATCH 15/67] Fixed style. Signed-off-by: Vahid --- tests/collections/asr/test_asr_ctc_encoder_model_bpe.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py b/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py index 0da826d3e3ed..fa8bba3afd03 100644 --- a/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py +++ b/tests/collections/asr/test_asr_ctc_encoder_model_bpe.py @@ -49,7 +49,9 @@ def asr_model(test_data_dir): decoder = { '_target_': 'nemo.collections.asr.modules.ConvASRDecoder', - 'feat_in': 1024, 'num_classes': -1, 'vocabulary': None, + 'feat_in': 1024, + 'num_classes': -1, + 'vocabulary': None, } tokenizer = {'dir': os.path.join(test_data_dir, "asr", "tokenizers", "an4_wpe_128"), 'type': 'wpe'} From 3402668a89b2e8caa20e740ba734dddbb350dc48 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 16:28:24 -0700 Subject: [PATCH 16/67] Fixed style. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 5110db9c38ff..4f005cfa8571 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -105,7 +105,9 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) self.loss = CTCLoss( - num_classes=self.decoder.num_classes_with_blank - 1, zero_infinity=True, reduction=self._cfg.ctc_reduction + num_classes=self.decoder.num_classes_with_blank - 1, + zero_infinity=True, + reduction=self._cfg.get("ctc_reduction", "mean_batch"), ) if hasattr(self._cfg, 'spec_augment') and self._cfg.spec_augment is not None: From fb324d033bec62286263665f4e89387259d7a3c1 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 16:52:29 -0700 Subject: [PATCH 17/67] Fixed missing params. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 4486f91b0f0f..b8e01eca047c 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -97,10 +97,16 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self.preprocessor = EncDecCTCModel.from_config_dict(self._cfg.preprocessor) self.encoder = EncDecCTCModel.from_config_dict(self._cfg.encoder) - if not self._cfg.decoder.feat_in and hasattr(self.encoder, 'feat_out'): - self._cfg.decoder.feat_in = self.encoder.feat_out + if "params" in self.encoder: + if not self._cfg.decoder.params.feat_in and hasattr(self.encoder.params, 'feat_out'): + self._cfg.decoder.params.feat_in = self.encoder.params.feat_out + else: + logging.error("param feat_in of the decoder's config is not set!") else: - logging.error("param feat_in of the decoder' config is not set!") + if not self._cfg.decoder.feat_in and hasattr(self.encoder, 'feat_out'): + self._cfg.decoder.feat_in = self.encoder.feat_out + else: + logging.error("param feat_in of the decoder's config is not set!") self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) From 93adc1aaa4dbac25f3444d098db02c71e2952752 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 17:08:10 -0700 Subject: [PATCH 18/67] Fixed missing params. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index b8e01eca047c..2c4bebf9efd8 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -97,9 +97,9 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self.preprocessor = EncDecCTCModel.from_config_dict(self._cfg.preprocessor) self.encoder = EncDecCTCModel.from_config_dict(self._cfg.encoder) - if "params" in self.encoder: - if not self._cfg.decoder.params.feat_in and hasattr(self.encoder.params, 'feat_out'): - self._cfg.decoder.params.feat_in = self.encoder.params.feat_out + if "params" in self._cfg.decoder: + if not self._cfg.decoder.params.feat_in and hasattr(self.encoder, 'feat_out'): + self._cfg.decoder.params.feat_in = self.encoder.feat_out else: logging.error("param feat_in of the decoder's config is not set!") else: From 2473812f8c73d3d5b4136430e9807214796de87c Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 17:12:43 -0700 Subject: [PATCH 19/67] Dropped u and v biases. Signed-off-by: Vahid --- nemo/collections/asr/modules/conformer_encoder.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 458cb413f010..0cc408b34d41 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -137,9 +137,6 @@ def __init__( self.feat_out = d_model self.pre_encode = nn.Linear(feat_in, d_model) - self.u_bias = None - self.v_bias = None - if self_attention_model == "rel_pos": self.pos_enc = RelPositionalEncoding( d_model=d_model, dropout_rate=dropout, dropout_emb_rate=dropout_emb, xscale=self.xscale @@ -194,8 +191,6 @@ def forward(self, audio_signal, length): x=audio_signal, att_mask=xx_mask, pos_emb=pos_emb, - u_bias=self.u_bias, - v_bias=self.v_bias, pad_mask=pad_mask, ) From 165359935df936c5aee631e5ff0044ddddb8e56b Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 22 Oct 2020 17:23:02 -0700 Subject: [PATCH 20/67] Fixed params. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_bpe_models.py | 13 +++++++++++-- nemo/collections/asr/modules/conformer_encoder.py | 7 +------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index a4071cc3b5f1..89f18b0e1337 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -303,13 +303,22 @@ def change_vocabulary(self, new_tokenizer_dir: str, new_tokenizer_type: str): decoder_config = copy.deepcopy(self.decoder.to_config_dict()) decoder_config.vocabulary = ListConfig(list(vocabulary.values())) + if "params" in decoder_config: + decoder_num_classes = decoder_config['params']['num_classes'] + else: + decoder_num_classes = decoder_config['num_classes'] + # Override number of classes if placeholder provided logging.info( "\nReplacing old number of classes ({}) with new number of classes - {}".format( - decoder_config['params']['num_classes'], len(vocabulary) + decoder_num_classes, len(vocabulary) ) ) - decoder_config['params']['num_classes'] = len(vocabulary) + + if "params" in decoder_config: + decoder_config['params']['num_classes'] = len(vocabulary) + else: + decoder_config['num_classes'] = len(vocabulary) del self.decoder self.decoder = EncDecCTCModelBPE.from_config_dict(decoder_config) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 0cc408b34d41..3e9460389b5d 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -187,12 +187,7 @@ def forward(self, audio_signal, length): pad_mask = (~pad_mask).unsqueeze(2) for lth, layer in enumerate(self.layers): - audio_signal = layer( - x=audio_signal, - att_mask=xx_mask, - pos_emb=pos_emb, - pad_mask=pad_mask, - ) + audio_signal = layer(x=audio_signal, att_mask=xx_mask, pos_emb=pos_emb, pad_mask=pad_mask,) if self.out_proj is not None: audio_signal = self.out_proj(audio_signal) From 81dff89ee556df1a2367e3b19b792b4e46200c4f Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 23 Oct 2020 14:56:02 -0700 Subject: [PATCH 21/67] moved back padding in features.py. Signed-off-by: Vahid --- nemo/collections/asr/parts/features.py | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/nemo/collections/asr/parts/features.py b/nemo/collections/asr/parts/features.py index e9491df38201..424ea65daaa9 100644 --- a/nemo/collections/asr/parts/features.py +++ b/nemo/collections/asr/parts/features.py @@ -357,21 +357,6 @@ def forward(self, x, seq_len): # dot with filterbank energies x = torch.matmul(self.fb.to(x.dtype), x) - # mask to zero any values beyond seq_len in batch, pad to multiple of - # `pad_to` (for efficiency) - max_len = x.size(-1) - mask = torch.arange(max_len).to(x.device) - mask = mask.expand(x.size(0), max_len) >= seq_len.unsqueeze(1) - x = x.masked_fill(mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value) - del mask - pad_to = self.pad_to - if pad_to == "max": - x = nn.functional.pad(x, (0, self.max_length - x.size(-1)), value=self.pad_value) - elif pad_to > 0: - pad_amt = x.size(-1) % pad_to - if pad_amt != 0: - x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value) - # log features if required if self.log: if self.log_zero_guard_type == "add": @@ -389,4 +374,19 @@ def forward(self, x, seq_len): if self.normalize: x = normalize_batch(x, seq_len, normalize_type=self.normalize) + # mask to zero any values beyond seq_len in batch, pad to multiple of + # `pad_to` (for efficiency) + max_len = x.size(-1) + mask = torch.arange(max_len).to(x.device) + mask = mask.expand(x.size(0), max_len) >= seq_len.unsqueeze(1) + x = x.masked_fill(mask.unsqueeze(1).type(torch.bool).to(device=x.device), self.pad_value) + del mask + pad_to = self.pad_to + if pad_to == "max": + x = nn.functional.pad(x, (0, self.max_length - x.size(-1)), value=self.pad_value) + elif pad_to > 0: + pad_amt = x.size(-1) % pad_to + if pad_amt != 0: + x = nn.functional.pad(x, (0, pad_to - pad_amt), value=self.pad_value) + return x, seq_len From ea93557ad4e22a503a99c6f6cb66ffd2049041a5 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 26 Oct 2020 11:00:40 -0700 Subject: [PATCH 22/67] fixed optimzier. Signed-off-by: Vahid --- nemo/core/optim/lr_scheduler.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/nemo/core/optim/lr_scheduler.py b/nemo/core/optim/lr_scheduler.py index ea61f67682ca..3c011c512594 100644 --- a/nemo/core/optim/lr_scheduler.py +++ b/nemo/core/optim/lr_scheduler.py @@ -192,26 +192,10 @@ def _poly_decay(initial_lr, step, decay_steps, power, min_lr, cycle): class SquareAnnealing(WarmupPolicy): - def __init__(self, optimizer, *, warmup_steps=None, warmup_ratio=None, max_steps=None, min_lr=0.0, last_epoch=-1): - assert not ( - warmup_steps is not None and warmup_ratio is not None - ), "Either use particular number of step or ratio" - assert warmup_ratio is None or max_steps is not None, "If there is a ratio, there should be a total steps" - - # It is necessary to assign all attributes *before* __init__, - # as class is wrapped by an inner class. - self.max_steps = max_steps - if warmup_steps is not None: - self.warmup_steps = warmup_steps - elif warmup_ratio is not None: - self.warmup_steps = int(warmup_ratio * max_steps) - else: - self.warmup_steps = 0 - - self.min_lr = min_lr - super().__init__(optimizer, last_epoch) + def __init__(self, optimizer, *, max_steps, min_lr=1e-5, last_epoch=-1, **kwargs): + super().__init__(optimizer=optimizer, max_steps=max_steps, last_epoch=last_epoch, min_lr=min_lr, **kwargs) - def get_lr(self, step): + def _get_lr(self, step): new_lrs = [ _square_annealing( initial_lr=initial_lr, From a7c72073b342dfe35a18cadde0b7ce373d1c541b Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 26 Oct 2020 16:51:03 -0700 Subject: [PATCH 23/67] fixed vocab bug. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_bpe_models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index 587faf939254..b655bcad9e7b 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -67,7 +67,10 @@ def __init__(self, cfg: DictConfig, trainer=None): vocabulary = self.tokenizer.tokenizer.get_vocab() # Set the new vocabulary - cfg.decoder.vocabulary = ListConfig(list(vocabulary.values())) + if "params" in cfg.decoder: + cfg.decoder.params.vocabulary = ListConfig(list(vocabulary.values())) + else: + cfg.decoder.vocabulary = ListConfig(list(vocabulary.values())) # Override number of classes if placeholder provided if "params" in cfg.decoder: From 3ceeebd09a37082f4121651f84d5430f6c697afe Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 26 Oct 2020 17:42:39 -0700 Subject: [PATCH 24/67] added load_weights_from_checkpoint. Signed-off-by: Vahid --- examples/asr/speech_to_text.py | 4 ++-- examples/asr/speech_to_text_bpe.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index 47e205d25cab..6a715bb33ed6 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -70,8 +70,8 @@ def main(cfg): exp_manager(trainer, cfg.get("exp_manager", None)) asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) - if cfg.model.get("load_from_checkpoint", None): - logging.info(f"Loading checkpoint '{cfg.model.load_from_checkpoint}' ...") + if cfg.model.get("load_weights_from_checkpoint", None): + logging.info(f"Loading checkpoint '{cfg.model.load_weights_from_checkpoint}' ...") asr_model.update_weights(checkpoint_path=cfg.model.load_weights_from_checkpoint) trainer.fit(asr_model) diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index 98bbc03fcf39..973426c942cd 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -62,8 +62,8 @@ def main(cfg): asr_model = EncDecCTCModelBPE(cfg=cfg.model, trainer=trainer) - if cfg.model.get("load_from_checkpoint", None): - logging.info(f"Loading checkpoint '{cfg.model.load_from_checkpoint}' ...") + if cfg.model.get("load_weights_from_checkpoint", None): + logging.info(f"Loading checkpoint '{cfg.model.load_weights_from_checkpoint}' ...") asr_model.update_weights(checkpoint_path=cfg.model.load_weights_from_checkpoint) trainer.fit(asr_model) From acfb8e6721378d4310de1a89e9fc03acb935829a Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 26 Oct 2020 17:51:36 -0700 Subject: [PATCH 25/67] Added jenkins test for Conformer. Updated names from bpe to subword. Signed-off-by: Vahid --- Jenkinsfile | 15 +++++++++++++++ ...{conformer_bpe.yaml => conformer_subword.yaml} | 0 .../{bpe_vocabs => wpe_vocabs}/1k/vocab.txt | 0 .../{bpe_vocabs => wpe_vocabs}/2k/vocab.txt | 0 .../{bpe_vocabs => wpe_vocabs}/4k/vocab.txt | 0 .../{bpe_vocabs => wpe_vocabs}/8k/vocab.txt | 0 6 files changed, 15 insertions(+) rename examples/asr/experimental/configs/conformer/{conformer_bpe.yaml => conformer_subword.yaml} (100%) rename examples/asr/experimental/{bpe_vocabs => wpe_vocabs}/1k/vocab.txt (100%) rename examples/asr/experimental/{bpe_vocabs => wpe_vocabs}/2k/vocab.txt (100%) rename examples/asr/experimental/{bpe_vocabs => wpe_vocabs}/4k/vocab.txt (100%) rename examples/asr/experimental/{bpe_vocabs => wpe_vocabs}/8k/vocab.txt (100%) diff --git a/Jenkinsfile b/Jenkinsfile index d574a2c262cd..a99b45a3797b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -223,6 +223,21 @@ pipeline { sh 'rm -rf examples/asr/speech_to_text_wpe_results' } } + + stage('L2: Speech to Text WPE - Conformer') { + steps { + sh 'python examples/asr/speech_to_text_bpe.py \ + --config-path="experimental/configs/conformer" --config-name="conformer_subword" \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + model.tokenizer.dir="experimental/wpe_vocabs/1k/" \ + model.tokenizer.type="wpe" \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_wpe_conformer_results' + sh 'rm -rf examples/asr/speech_to_text_wpe_conformer_results' + } + } } } diff --git a/examples/asr/experimental/configs/conformer/conformer_bpe.yaml b/examples/asr/experimental/configs/conformer/conformer_subword.yaml similarity index 100% rename from examples/asr/experimental/configs/conformer/conformer_bpe.yaml rename to examples/asr/experimental/configs/conformer/conformer_subword.yaml diff --git a/examples/asr/experimental/bpe_vocabs/1k/vocab.txt b/examples/asr/experimental/wpe_vocabs/1k/vocab.txt similarity index 100% rename from examples/asr/experimental/bpe_vocabs/1k/vocab.txt rename to examples/asr/experimental/wpe_vocabs/1k/vocab.txt diff --git a/examples/asr/experimental/bpe_vocabs/2k/vocab.txt b/examples/asr/experimental/wpe_vocabs/2k/vocab.txt similarity index 100% rename from examples/asr/experimental/bpe_vocabs/2k/vocab.txt rename to examples/asr/experimental/wpe_vocabs/2k/vocab.txt diff --git a/examples/asr/experimental/bpe_vocabs/4k/vocab.txt b/examples/asr/experimental/wpe_vocabs/4k/vocab.txt similarity index 100% rename from examples/asr/experimental/bpe_vocabs/4k/vocab.txt rename to examples/asr/experimental/wpe_vocabs/4k/vocab.txt diff --git a/examples/asr/experimental/bpe_vocabs/8k/vocab.txt b/examples/asr/experimental/wpe_vocabs/8k/vocab.txt similarity index 100% rename from examples/asr/experimental/bpe_vocabs/8k/vocab.txt rename to examples/asr/experimental/wpe_vocabs/8k/vocab.txt From fd8764a5d51723e1ca7da895437b8e9efbbf0507 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 26 Oct 2020 19:54:50 -0700 Subject: [PATCH 26/67] Added jenkins test for Conformer. Updated names from bpe to subword. Signed-off-by: Vahid --- Jenkinsfile | 2 +- .../asr/experimental/configs/conformer/conformer_char.yaml | 2 -- .../asr/experimental/configs/conformer/conformer_subword.yaml | 4 +--- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a99b45a3797b..3af042101238 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -230,7 +230,7 @@ pipeline { --config-path="experimental/configs/conformer" --config-name="conformer_subword" \ model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ - model.tokenizer.dir="experimental/wpe_vocabs/1k/" \ + model.tokenizer.dir="examples/asr/experimental/wpe_vocabs/1k/" \ model.tokenizer.type="wpe" \ trainer.gpus=[1] \ +trainer.fast_dev_run=True \ diff --git a/examples/asr/experimental/configs/conformer/conformer_char.yaml b/examples/asr/experimental/configs/conformer/conformer_char.yaml index c6f997d527f8..31772e8d5a3a 100644 --- a/examples/asr/experimental/configs/conformer/conformer_char.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_char.yaml @@ -106,7 +106,6 @@ model: optim: name: novograd lr: 0.01 - # optimizer arguments betas: [0.8, 0.5] weight_decay: 0.001 @@ -114,7 +113,6 @@ model: # scheduler setup sched: name: CosineAnnealing - # scheduler config override warmup_steps: 1000 warmup_ratio: null diff --git a/examples/asr/experimental/configs/conformer/conformer_subword.yaml b/examples/asr/experimental/configs/conformer/conformer_subword.yaml index 8905850df972..4eea7f4a0e34 100644 --- a/examples/asr/experimental/configs/conformer/conformer_subword.yaml +++ b/examples/asr/experimental/configs/conformer/conformer_subword.yaml @@ -1,4 +1,4 @@ -name: &name "Conformer-BPE" +name: &name "Conformer-Subword" model: sample_rate: &sample_rate 16000 @@ -109,7 +109,6 @@ model: optim: name: novograd lr: 0.01 - # optimizer arguments betas: [0.8, 0.5] weight_decay: 0.001 @@ -117,7 +116,6 @@ model: # scheduler setup sched: name: CosineAnnealing - # scheduler config override warmup_steps: 4000 warmup_ratio: null From 5f241d467acc436b36b538d6fd72f178d6ed402b Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 27 Oct 2020 14:19:48 -0700 Subject: [PATCH 27/67] enabled ddp. Signed-off-by: Vahid --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 3af042101238..bc9387f859b9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -543,6 +543,7 @@ pipeline { model.language_model.pretrained_model_name=megatron-bert-uncased \ model.language_model.config_file=/home/TestData/nlp/mp_2_bert_toy/config.json \ model.language_model.lm_checkpoint=/home/TestData/nlp/mp_2_bert_toy/iter_2000000 \ + model.nemo_path=null \ ' sh 'rm -rf examples/nlp/text_classification/exp_mp_2_megatron_bert' } From f0cae87e9702662c2fd4b9c3f091f113be482544 Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 27 Oct 2020 23:47:53 -0700 Subject: [PATCH 28/67] removed extra prints. Signed-off-by: Vahid --- nemo/core/optim/lr_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo/core/optim/lr_scheduler.py b/nemo/core/optim/lr_scheduler.py index 42a6e9f66fc4..0bcb7a574712 100644 --- a/nemo/core/optim/lr_scheduler.py +++ b/nemo/core/optim/lr_scheduler.py @@ -291,7 +291,6 @@ def _noam_annealing(self, initial_lr, step): out_lr = initial_lr * mult if step > self.warmup_steps: out_lr = max(out_lr, self.min_lr) - print(out_lr) return out_lr From b625c95939c286e7dd8afab4bed8257caf07f083 Mon Sep 17 00:00:00 2001 From: Vahid Date: Wed, 28 Oct 2020 17:25:37 -0700 Subject: [PATCH 29/67] reorg the folders. Signed-off-by: Vahid --- examples/asr/experimental/{configs => citrinet}/config_bpe.yaml | 0 .../contextnet_bpe => citrinet}/contextnet_192_2x_stride.yaml | 0 .../contextnet_bpe => citrinet}/contextnet_192_4x_stride.yaml | 0 .../contextnet_bpe => citrinet}/contextnet_192_8x_stride.yaml | 0 .../{configs/conformer => conformer/confs}/conformer_char.yaml | 0 .../{configs/conformer => conformer/confs}/conformer_subword.yaml | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename examples/asr/experimental/{configs => citrinet}/config_bpe.yaml (100%) rename examples/asr/experimental/{configs/contextnet_bpe => citrinet}/contextnet_192_2x_stride.yaml (100%) rename examples/asr/experimental/{configs/contextnet_bpe => citrinet}/contextnet_192_4x_stride.yaml (100%) rename examples/asr/experimental/{configs/contextnet_bpe => citrinet}/contextnet_192_8x_stride.yaml (100%) rename examples/asr/experimental/{configs/conformer => conformer/confs}/conformer_char.yaml (100%) rename examples/asr/experimental/{configs/conformer => conformer/confs}/conformer_subword.yaml (100%) diff --git a/examples/asr/experimental/configs/config_bpe.yaml b/examples/asr/experimental/citrinet/config_bpe.yaml similarity index 100% rename from examples/asr/experimental/configs/config_bpe.yaml rename to examples/asr/experimental/citrinet/config_bpe.yaml diff --git a/examples/asr/experimental/configs/contextnet_bpe/contextnet_192_2x_stride.yaml b/examples/asr/experimental/citrinet/contextnet_192_2x_stride.yaml similarity index 100% rename from examples/asr/experimental/configs/contextnet_bpe/contextnet_192_2x_stride.yaml rename to examples/asr/experimental/citrinet/contextnet_192_2x_stride.yaml diff --git a/examples/asr/experimental/configs/contextnet_bpe/contextnet_192_4x_stride.yaml b/examples/asr/experimental/citrinet/contextnet_192_4x_stride.yaml similarity index 100% rename from examples/asr/experimental/configs/contextnet_bpe/contextnet_192_4x_stride.yaml rename to examples/asr/experimental/citrinet/contextnet_192_4x_stride.yaml diff --git a/examples/asr/experimental/configs/contextnet_bpe/contextnet_192_8x_stride.yaml b/examples/asr/experimental/citrinet/contextnet_192_8x_stride.yaml similarity index 100% rename from examples/asr/experimental/configs/contextnet_bpe/contextnet_192_8x_stride.yaml rename to examples/asr/experimental/citrinet/contextnet_192_8x_stride.yaml diff --git a/examples/asr/experimental/configs/conformer/conformer_char.yaml b/examples/asr/experimental/conformer/confs/conformer_char.yaml similarity index 100% rename from examples/asr/experimental/configs/conformer/conformer_char.yaml rename to examples/asr/experimental/conformer/confs/conformer_char.yaml diff --git a/examples/asr/experimental/configs/conformer/conformer_subword.yaml b/examples/asr/experimental/conformer/confs/conformer_subword.yaml similarity index 100% rename from examples/asr/experimental/configs/conformer/conformer_subword.yaml rename to examples/asr/experimental/conformer/confs/conformer_subword.yaml From 09ee00fd73d7e9de20d9c8a0eec2b1868a35f342 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 29 Oct 2020 17:29:49 -0700 Subject: [PATCH 30/67] reorg the folders. Signed-off-by: Vahid --- examples/asr/experimental/conformer/confs/conformer_char.yaml | 2 +- .../asr/experimental/conformer/confs/conformer_subword.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/asr/experimental/conformer/confs/conformer_char.yaml b/examples/asr/experimental/conformer/confs/conformer_char.yaml index 31772e8d5a3a..46baa9b57318 100644 --- a/examples/asr/experimental/conformer/confs/conformer_char.yaml +++ b/examples/asr/experimental/conformer/confs/conformer_char.yaml @@ -75,7 +75,7 @@ model: # Sub-sampling params subsampling: vggnet # vggnet or striding - subsampling_factor: 4 # must be a multiply of 2 + subsampling_factor: 4 # must be power of 2 subsampling_conv_channels: 64 # set to -1 to make it equal to the d_model # Feed forward module's params diff --git a/examples/asr/experimental/conformer/confs/conformer_subword.yaml b/examples/asr/experimental/conformer/confs/conformer_subword.yaml index 4eea7f4a0e34..e08fd1835936 100644 --- a/examples/asr/experimental/conformer/confs/conformer_subword.yaml +++ b/examples/asr/experimental/conformer/confs/conformer_subword.yaml @@ -78,7 +78,7 @@ model: # Sub-sampling params subsampling: vggnet # vggnet or striding - subsampling_factor: 4 # must be a multiply of 2 + subsampling_factor: 4 # must be power of 2 subsampling_conv_channels: 64 # set to -1 to make it equal to the d_model # Feed forward module's params From cdfd2ecbb03ea636674444f6dcfc675cf5159023 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 29 Oct 2020 17:31:02 -0700 Subject: [PATCH 31/67] reorg the folders. Signed-off-by: Vahid --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bc9387f859b9..80478963fb67 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -212,7 +212,7 @@ pipeline { stage('L2: Speech to Text WPE') { steps { sh 'python examples/asr/speech_to_text_bpe.py \ - --config-path="experimental/configs/" --config-name="config_bpe" \ + --config-path="experimental/conformer/confs" --config-name="config_bpe" \ model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ @@ -227,7 +227,7 @@ pipeline { stage('L2: Speech to Text WPE - Conformer') { steps { sh 'python examples/asr/speech_to_text_bpe.py \ - --config-path="experimental/configs/conformer" --config-name="conformer_subword" \ + --config-path="experimental/conformer/confs" --config-name="conformer_subword" \ model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ model.tokenizer.dir="examples/asr/experimental/wpe_vocabs/1k/" \ From b0f22cf27294761d5ce268d09cf089d544151052 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 29 Oct 2020 17:32:44 -0700 Subject: [PATCH 32/67] reorg the folders. Signed-off-by: Vahid --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 80478963fb67..44cb7adc7502 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -212,7 +212,7 @@ pipeline { stage('L2: Speech to Text WPE') { steps { sh 'python examples/asr/speech_to_text_bpe.py \ - --config-path="experimental/conformer/confs" --config-name="config_bpe" \ + --config-path="experimental/citrinet/configs/" --config-name="config_bpe" \ model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ From 175cd4877af0c5800f8c65f985a0a731d40cef50 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 10:34:55 -0700 Subject: [PATCH 33/67] reverted back subword. Signed-off-by: Vahid --- .../confs/{conformer_subword.yaml => conformer_bpe.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/asr/experimental/conformer/confs/{conformer_subword.yaml => conformer_bpe.yaml} (99%) diff --git a/examples/asr/experimental/conformer/confs/conformer_subword.yaml b/examples/asr/experimental/conformer/confs/conformer_bpe.yaml similarity index 99% rename from examples/asr/experimental/conformer/confs/conformer_subword.yaml rename to examples/asr/experimental/conformer/confs/conformer_bpe.yaml index e08fd1835936..2105b3697ac4 100644 --- a/examples/asr/experimental/conformer/confs/conformer_subword.yaml +++ b/examples/asr/experimental/conformer/confs/conformer_bpe.yaml @@ -1,4 +1,4 @@ -name: &name "Conformer-Subword" +name: &name "Conformer-BPE" model: sample_rate: &sample_rate 16000 From 0efb275b9e3344b04e7d654fcd1921c4fc8236c1 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 10:44:58 -0700 Subject: [PATCH 34/67] reverted back subword. Signed-off-by: Vahid --- .../asr/experimental/conformer/{confs => }/conformer_bpe.yaml | 0 .../asr/experimental/conformer/{confs => }/conformer_char.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename examples/asr/experimental/conformer/{confs => }/conformer_bpe.yaml (100%) rename examples/asr/experimental/conformer/{confs => }/conformer_char.yaml (100%) diff --git a/examples/asr/experimental/conformer/confs/conformer_bpe.yaml b/examples/asr/experimental/conformer/conformer_bpe.yaml similarity index 100% rename from examples/asr/experimental/conformer/confs/conformer_bpe.yaml rename to examples/asr/experimental/conformer/conformer_bpe.yaml diff --git a/examples/asr/experimental/conformer/confs/conformer_char.yaml b/examples/asr/experimental/conformer/conformer_char.yaml similarity index 100% rename from examples/asr/experimental/conformer/confs/conformer_char.yaml rename to examples/asr/experimental/conformer/conformer_char.yaml From c6b5b3c567d613f6be8b93fc9f9989387a4b993c Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 11:02:01 -0700 Subject: [PATCH 35/67] updated test evaluation. Signed-off-by: Vahid --- examples/asr/experimental/conformer/conformer_bpe.yaml | 5 ++--- examples/asr/experimental/conformer/conformer_char.yaml | 5 ++--- examples/asr/speech_to_text.py | 2 +- examples/asr/speech_to_text_bpe.py | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/asr/experimental/conformer/conformer_bpe.yaml b/examples/asr/experimental/conformer/conformer_bpe.yaml index 2105b3697ac4..a79c4fcc8b4a 100644 --- a/examples/asr/experimental/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/conformer/conformer_bpe.yaml @@ -127,7 +127,7 @@ trainer: num_nodes: 1 max_epochs: 100 max_steps: null # computed at runtime if not set - val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + val_check_interval: 1 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations distributed_backend: ddp accumulate_grad_batches: 2 gradient_clip_val: 0.0 @@ -136,9 +136,8 @@ trainer: log_every_n_steps: 10 # Interval of logging. resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it - check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + check_val_every_n_epoch: 10 # number of evaluations on validation every n epochs sync_batchnorm: true - checkpoint_callback: false # Provided by exp_manager logger: false # Provided by exp_manager diff --git a/examples/asr/experimental/conformer/conformer_char.yaml b/examples/asr/experimental/conformer/conformer_char.yaml index 46baa9b57318..56af8604f209 100644 --- a/examples/asr/experimental/conformer/conformer_char.yaml +++ b/examples/asr/experimental/conformer/conformer_char.yaml @@ -124,7 +124,7 @@ trainer: num_nodes: 1 max_epochs: 100 max_steps: null # computed at runtime if not set - val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations + val_check_interval: 1 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations distributed_backend: ddp accumulate_grad_batches: 2 gradient_clip_val: 0.0 @@ -133,9 +133,8 @@ trainer: log_every_n_steps: 10 # Interval of logging. resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it - check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs + check_val_every_n_epoch: 10 # number of evaluations on validation every n epochs sync_batchnorm: true - checkpoint_callback: false # Provided by exp_manager logger: false # Provided by exp_manager diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index 6a715bb33ed6..b248a3022623 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -78,7 +78,7 @@ def main(cfg): if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: gpu = 1 if cfg.trainer.gpus != 0 else 0 - trainer = pl.Trainer(gpus=gpu) + trainer = pl.Trainer(gpus=gpu, precision=cfg.trainer.precision, amp_level=cfg.trainer.amp_level, amp_backend=cfg.trainer.amp_backend) if asr_model.prepare_test(trainer): trainer.test(asr_model) diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index 973426c942cd..c04e116bd3e0 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -70,7 +70,7 @@ def main(cfg): if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: gpu = 1 if cfg.trainer.gpus != 0 else 0 - trainer = pl.Trainer(gpus=gpu) + trainer = pl.Trainer(gpus=gpu, precision=cfg.trainer.precision, amp_level=cfg.trainer.amp_level, amp_backend=cfg.trainer.amp_backend) if asr_model.prepare_test(trainer): trainer.test(asr_model) From e60bc1554aedcebdcb47e3797a8abb260ea64f97 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 11:19:45 -0700 Subject: [PATCH 36/67] Updated the code. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_bpe_models.py | 12 ++++++------ nemo/collections/asr/models/ctc_models.py | 8 ++++---- .../collections/asr/modules/conformer_encoder.py | 16 ---------------- nemo/collections/asr/modules/lstm_decoder.py | 7 ------- 4 files changed, 10 insertions(+), 33 deletions(-) diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index b655bcad9e7b..90c71f7cdacf 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -74,14 +74,14 @@ def __init__(self, cfg: DictConfig, trainer=None): # Override number of classes if placeholder provided if "params" in cfg.decoder: - num_c = cfg.decoder["params"]["num_classes"] + num_classes = cfg.decoder["params"]["num_classes"] else: - num_c = cfg.decoder["num_classes"] + num_classes = cfg.decoder["num_classes"] - if num_c < 1: + if num_classes < 1: logging.info( "\nReplacing placeholder number of classes ({}) with actual number of classes - {}".format( - num_c, len(vocabulary) + num_classes, len(vocabulary) ) ) if "params" in cfg.decoder: @@ -95,7 +95,7 @@ def __init__(self, cfg: DictConfig, trainer=None): self._wer = WERBPE( tokenizer=self.tokenizer, batch_dim_index=0, - use_cer=False, + use_cer=self._cfg.get('use_cer', False), ctc_decode=True, dist_sync_on_step=True, log_prediction=self._cfg.get("log_prediction", False), @@ -315,7 +315,7 @@ def change_vocabulary(self, new_tokenizer_dir: str, new_tokenizer_type: str): self._wer = WERBPE( tokenizer=self.tokenizer, batch_dim_index=0, - use_cer=False, + use_cer=self._cfg.get('use_cer', False), ctc_decode=True, log_prediction=self._cfg.get("log_prediction", False), ) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index d091b5d93765..e6d7b3d3f7ed 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -104,12 +104,12 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): if not self._cfg.decoder.params.feat_in and hasattr(self.encoder, 'feat_out'): self._cfg.decoder.params.feat_in = self.encoder.feat_out else: - logging.error("param feat_in of the decoder's config is not set!") + raise ValueError("Parameter feat_in of the decoder's config is not set!") else: if not self._cfg.decoder.feat_in and hasattr(self.encoder, 'feat_out'): self._cfg.decoder.feat_in = self.encoder.feat_out else: - logging.error("param feat_in of the decoder's config is not set!") + raise ValueError("param feat_in of the decoder's config is not set!") self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) @@ -128,7 +128,7 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self._wer = WER( vocabulary=self.decoder.vocabulary, batch_dim_index=0, - use_cer=False, + use_cer=self._cfg.get('use_cer', False), ctc_decode=True, dist_sync_on_step=True, log_prediction=self._cfg.get("log_prediction", False), @@ -228,7 +228,7 @@ def change_vocabulary(self, new_vocabulary: List[str]): self._wer = WER( vocabulary=self.decoder.vocabulary, batch_dim_index=0, - use_cer=False, + use_cer=self._cfg.get('use_cer', False), ctc_decode=True, dist_sync_on_step=True, log_prediction=self._cfg.get("log_prediction", False), diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 3e9460389b5d..b483c0c38e44 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -55,22 +55,6 @@ def input_example(self): input_example = torch.randn(16, self.__feat_in, 256).to(next(self.parameters()).device) return tuple([input_example]) - @property - def disabled_deployment_input_names(self): - """Implement this method to return a set of input names disabled for export""" - return set(["length"]) - - @property - def disabled_deployment_output_names(self): - """Implement this method to return a set of output names disabled for export""" - return set(["encoded_lengths"]) - - def save_to(self, save_path: str): - pass - - @classmethod - def restore_from(cls, restore_path: str): - pass @property def input_types(self): diff --git a/nemo/collections/asr/modules/lstm_decoder.py b/nemo/collections/asr/modules/lstm_decoder.py index 1a00c9077be0..5552638c8b8b 100644 --- a/nemo/collections/asr/modules/lstm_decoder.py +++ b/nemo/collections/asr/modules/lstm_decoder.py @@ -37,13 +37,6 @@ class LSTMDecoder(NeuralModule, Exportable): num_layers (int): default is 1. Number of LSTM layers stacked """ - def save_to(self, save_path: str): - pass - - @classmethod - def restore_from(cls, restore_path: str): - pass - @property def input_types(self): return OrderedDict({"encoder_output": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation())}) From 8ea849b0bd2558c3da12de27e1f2c347564c4e75 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 11:23:11 -0700 Subject: [PATCH 37/67] fixed the style. Signed-off-by: Vahid --- examples/asr/speech_to_text.py | 7 ++++++- examples/asr/speech_to_text_bpe.py | 7 ++++++- nemo/collections/asr/modules/conformer_encoder.py | 1 - 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index b248a3022623..d456b61c21b8 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -78,7 +78,12 @@ def main(cfg): if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: gpu = 1 if cfg.trainer.gpus != 0 else 0 - trainer = pl.Trainer(gpus=gpu, precision=cfg.trainer.precision, amp_level=cfg.trainer.amp_level, amp_backend=cfg.trainer.amp_backend) + trainer = pl.Trainer( + gpus=gpu, + precision=cfg.trainer.precision, + amp_level=cfg.trainer.amp_level, + amp_backend=cfg.trainer.amp_backend, + ) if asr_model.prepare_test(trainer): trainer.test(asr_model) diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index c04e116bd3e0..2b9b8a873143 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -70,7 +70,12 @@ def main(cfg): if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: gpu = 1 if cfg.trainer.gpus != 0 else 0 - trainer = pl.Trainer(gpus=gpu, precision=cfg.trainer.precision, amp_level=cfg.trainer.amp_level, amp_backend=cfg.trainer.amp_backend) + trainer = pl.Trainer( + gpus=gpu, + precision=cfg.trainer.precision, + amp_level=cfg.trainer.amp_level, + amp_backend=cfg.trainer.amp_backend, + ) if asr_model.prepare_test(trainer): trainer.test(asr_model) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index b483c0c38e44..1584a71a0e94 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -55,7 +55,6 @@ def input_example(self): input_example = torch.randn(16, self.__feat_in, 256).to(next(self.parameters()).device) return tuple([input_example]) - @property def input_types(self): """Returns definitions of module input ports. From be0887825351eaff026ab0d4820cb1f7e263a5c2 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 11:43:52 -0700 Subject: [PATCH 38/67] Added docstring. Signed-off-by: Vahid --- .../asr/modules/conformer_encoder.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 1584a71a0e94..2d48c8769e88 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -41,6 +41,34 @@ class ConformerEncoder(NeuralModule, Exportable): Based on this paper: 'Conformer: Convolution-augmented Transformer for Speech Recognition' by Anmol Gulati et al. https://arxiv.org/abs/2005.08100 + + Args: + feat_in (int): the size of feature channels + n_layers (int): number of layers of ConformerBlock + d_model (int): the hidden size of the model + feat_out (int): the size of the output features + Defaults to -1 (means feat_out is d_model) + subsampling (str): the method of subsampling, choices=['vggnet', 'striding'] + subsampling_factor (int): the subsampling factor which should be power of 2 + Defaults to 4. + subsampling_conv_channels (int): the size of the convolutions in the subsampling module + Defaults to 64. + ff_expansion_factor (int): the expansion factor in feed forward layers + Defaults to 4. + self_attention_model (str): type of the attention layer and positional encoding + choices=['rel_pos', 'abs_pos']. + n_heads (int): number of heads in multi-headed attention layers + Defaults to 4. + xscaling (bool): enables scaling the inputs to the multi-headed attention layers by sqrt(d_model) + Defaults to True. + conv_kernel_size (int): the size of the convolutions in the convolutional modules + Defaults to 31. + dropout (float): the dropout rate used in all layers except the attention layers + Defaults to 0.1. + dropout_emb (float): the dropout rate used for the positional embeddings + Defaults to 0.1. + dropout_att (float): the dropout rate used for the attention layer + Defaults to 0.0. """ def _prepare_for_export(self): From d1ed6402c74b1061b253ceb4e4678a9e3be5ddf9 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 12:38:00 -0700 Subject: [PATCH 39/67] dropped load_weights. Signed-off-by: Vahid --- examples/asr/speech_to_text.py | 4 ---- examples/asr/speech_to_text_bpe.py | 4 ---- nemo/core/classes/modelPT.py | 16 ---------------- 3 files changed, 24 deletions(-) diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index d456b61c21b8..4620d0bde13d 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -70,10 +70,6 @@ def main(cfg): exp_manager(trainer, cfg.get("exp_manager", None)) asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) - if cfg.model.get("load_weights_from_checkpoint", None): - logging.info(f"Loading checkpoint '{cfg.model.load_weights_from_checkpoint}' ...") - asr_model.update_weights(checkpoint_path=cfg.model.load_weights_from_checkpoint) - trainer.fit(asr_model) if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index 2b9b8a873143..98306d557210 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -62,10 +62,6 @@ def main(cfg): asr_model = EncDecCTCModelBPE(cfg=cfg.model, trainer=trainer) - if cfg.model.get("load_weights_from_checkpoint", None): - logging.info(f"Loading checkpoint '{cfg.model.load_weights_from_checkpoint}' ...") - asr_model.update_weights(checkpoint_path=cfg.model.load_weights_from_checkpoint) - trainer.fit(asr_model) if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: diff --git a/nemo/core/classes/modelPT.py b/nemo/core/classes/modelPT.py index 607816b3de41..a0d60b5af10b 100644 --- a/nemo/core/classes/modelPT.py +++ b/nemo/core/classes/modelPT.py @@ -1019,19 +1019,3 @@ def __is_model_being_restored() -> bool: def __set_model_restore_state(is_being_restored: bool): global _MODEL_IS_RESTORED _MODEL_IS_RESTORED = is_being_restored - - def update_weights(self, checkpoint_path): - if os.path.exists(checkpoint_path): - raise FileExistsError(f"Could not find checkpoint file '{checkpoint_path}'!") - checkpoint = torch.load(checkpoint_path) - model_dict = self.state_dict() - for weights_name, weights in model_dict.items(): - if weights_name not in checkpoint['state_dict']: - logging.warning(f"Weights {weights_name} missing from the checkpoint!") - for weights_name, weights in checkpoint['state_dict'].items(): - if weights_name not in model_dict: - logging.warning(f"Weights {weights_name} missing from the model while exists in the checkpoint!") - pretrained_dict = {k: v for k, v in checkpoint['state_dict'].items() if k in model_dict} - model_dict.update(pretrained_dict) - self.load_state_dict(model_dict) - return self From c5922ac6eabf3eb7d58236777c0d14bc81ce4cef Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 12:55:36 -0700 Subject: [PATCH 40/67] fixed feat_out. Signed-off-by: Vahid --- examples/asr/experimental/conformer/conformer_bpe.yaml | 2 +- examples/asr/experimental/conformer/conformer_char.yaml | 2 +- nemo/collections/asr/models/ctc_models.py | 4 ++-- nemo/collections/asr/modules/conformer_encoder.py | 8 ++++---- nemo/collections/asr/modules/conv_asr.py | 3 +++ 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/asr/experimental/conformer/conformer_bpe.yaml b/examples/asr/experimental/conformer/conformer_bpe.yaml index a79c4fcc8b4a..7da7ab8e7e29 100644 --- a/examples/asr/experimental/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/conformer/conformer_bpe.yaml @@ -99,7 +99,7 @@ model: decoder: _target_: nemo.collections.asr.modules.LSTMDecoder - feat_in: null + feat_in: null # If not provided, the feat_out of the encoder would be used num_classes: -1 # filled with vocabulary size from tokenizer at runtime vocabulary: [] # filled with vocabulary from tokenizer at runtime lstm_hidden_size: 640 diff --git a/examples/asr/experimental/conformer/conformer_char.yaml b/examples/asr/experimental/conformer/conformer_char.yaml index 56af8604f209..ef7de1ecc7d4 100644 --- a/examples/asr/experimental/conformer/conformer_char.yaml +++ b/examples/asr/experimental/conformer/conformer_char.yaml @@ -96,7 +96,7 @@ model: decoder: _target_: nemo.collections.asr.modules.LSTMDecoder - feat_in: null + feat_in: null # If not provided, the feat_out of the encoder would be used num_classes: 28 vocabulary: *labels lstm_hidden_size: 640 diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index e6d7b3d3f7ed..9ac398e94871 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -102,12 +102,12 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): if "params" in self._cfg.decoder: if not self._cfg.decoder.params.feat_in and hasattr(self.encoder, 'feat_out'): - self._cfg.decoder.params.feat_in = self.encoder.feat_out + self._cfg.decoder.params.feat_in = self.encoder.__feat_out else: raise ValueError("Parameter feat_in of the decoder's config is not set!") else: if not self._cfg.decoder.feat_in and hasattr(self.encoder, 'feat_out'): - self._cfg.decoder.feat_in = self.encoder.feat_out + self._cfg.decoder.feat_in = self.encoder.__feat_out else: raise ValueError("param feat_in of the decoder's config is not set!") diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 2d48c8769e88..df65f72d2c68 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -143,9 +143,9 @@ def __init__( conv_channels=subsampling_conv_channels, activation=nn.ReLU(), ) - self.feat_out = d_model + self.__feat_out = d_model else: - self.feat_out = d_model + self.__feat_out = d_model self.pre_encode = nn.Linear(feat_in, d_model) if self_attention_model == "rel_pos": @@ -174,10 +174,10 @@ def __init__( if feat_out > 0 and feat_out != self.output_dim: self.out_proj = nn.Linear(self.feat_out, feat_out) - self.feat_out = feat_out + self.__feat_out = feat_out else: self.out_proj = None - self.feat_out = d_model + self.__feat_out = d_model @typecheck() def forward(self, audio_signal, length): diff --git a/nemo/collections/asr/modules/conv_asr.py b/nemo/collections/asr/modules/conv_asr.py index 83b50f41e375..22d91d29e1a9 100644 --- a/nemo/collections/asr/modules/conv_asr.py +++ b/nemo/collections/asr/modules/conv_asr.py @@ -174,9 +174,12 @@ def __init__( ) feat_in = lcfg['filters'] + self.__feat_out = feat_in + self.encoder = torch.nn.Sequential(*encoder_layers) self.apply(lambda x: init_weights(x, mode=init_mode)) + @typecheck() def forward(self, audio_signal, length=None): s_input, length = self.encoder(([audio_signal], length)) From 58b85f096ead1808ef75ac021b458ebe693bedd9 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 13:00:07 -0700 Subject: [PATCH 41/67] fixed feat_out. Signed-off-by: Vahid --- nemo/collections/asr/modules/conv_asr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo/collections/asr/modules/conv_asr.py b/nemo/collections/asr/modules/conv_asr.py index 22d91d29e1a9..17b1db1e8c30 100644 --- a/nemo/collections/asr/modules/conv_asr.py +++ b/nemo/collections/asr/modules/conv_asr.py @@ -179,7 +179,6 @@ def __init__( self.encoder = torch.nn.Sequential(*encoder_layers) self.apply(lambda x: init_weights(x, mode=init_mode)) - @typecheck() def forward(self, audio_signal, length=None): s_input, length = self.encoder(([audio_signal], length)) From 42fea1165e6e30b156481b6c15e8c77793eac73f Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 13:56:57 -0700 Subject: [PATCH 42/67] dropped the vocab files. Signed-off-by: Vahid --- .../asr/experimental/wpe_vocabs/1k/vocab.txt | 1024 --- .../asr/experimental/wpe_vocabs/2k/vocab.txt | 2048 ----- .../asr/experimental/wpe_vocabs/4k/vocab.txt | 4096 --------- .../asr/experimental/wpe_vocabs/8k/vocab.txt | 8192 ----------------- 4 files changed, 15360 deletions(-) delete mode 100644 examples/asr/experimental/wpe_vocabs/1k/vocab.txt delete mode 100644 examples/asr/experimental/wpe_vocabs/2k/vocab.txt delete mode 100644 examples/asr/experimental/wpe_vocabs/4k/vocab.txt delete mode 100644 examples/asr/experimental/wpe_vocabs/8k/vocab.txt diff --git a/examples/asr/experimental/wpe_vocabs/1k/vocab.txt b/examples/asr/experimental/wpe_vocabs/1k/vocab.txt deleted file mode 100644 index e0e2d5c408f1..000000000000 --- a/examples/asr/experimental/wpe_vocabs/1k/vocab.txt +++ /dev/null @@ -1,1024 +0,0 @@ -[PAD] -[UNK] -[CLS] -[SEP] -[MASK] -' -a -b -c -d -e -f -g -h -i -j -k -l -m -n -o -p -q -r -s -t -u -v -w -x -y -z -##o -##l -##i -##a -##n -##t -##r -##e -##u -##s -##y -##c -##b -##g -##m -##d -##w -##v -##k -##h -##p -##f -##x -##q -##j -##z -th -the -##er -##nd -##in -##ed -##ou -##at -##en -and -##or -##es -to -of -##on -##is -##ing -##ar -##it -##as -##an -##ll -in -##re -wh -he -##om -be -ha -##le -##ot -##ow -##ic -##ut -it -##ld -that -sh -##ly -was -##gh -##id -##se -on -##ve -##et -##ent -you -##im -##st -##ce -##ith -for -##ir -##ion -as -##ay -his -##al -with -##ur -st -##ter -her -##ver -##ch -re -##ad -##ght -an -not -is -had -sa -at -##oo -##her -but -##ould -she -##ct -##ess -fr -se -##am -pr -we -him -so -##ill -##ain -su -##ight -me -##ere -##all -##ke -##il -de -my -they -all -##est -ch -##ich -ne -by -have -##ome -##pp -##ck -li -this -do -##ore -con -##us -which -##ri -##ul -up -said -from -##th -##ard -##ge -##ra -or -one -no -ex -were -there -##os -##pe -man -##ble -who -al -##ant -##ie -##ous -##ust -##ation -fe -##and -them -##ers -##lf -when -##ind -##ast -go -##ood -kn -##un -##our -##ist -##em -are -##out -would -##art -what -le -their -##ong -en -if -un -##if -com -##ound -ab -out -wor -##ol -will -int -br -##ak -mis -##ate -##op -##um -##ough -##la -##itt -ar -##ook -##ish -##one -##ap -been -##own -##ame -then -##ven -##self -##ive -tr -ro -##ig -fa -##reat -af -sp -qu -##ery -##ity -##ort -##ost -tw -ag -could -am -##el -##ake -any -##ure -some -your -can -more -tim -##ep -##nder -ever -##ence -##ther -##fe -into -##ry -##ose -now -##au -##ag -cl -very -##qu -##ved -bo -other -did -##ance -than -##ittle -little -##ine -##ies -##oun -look -##way -##urn -like -##ite -gr -##ear -##ach -lo -##ass -##other -##able -##ought -know -time -im -us -co -##fore -##ell -how -##hing -##cc -ad -day -##ack -about -##ade -##ice -see -over -##ci -##ide -##pt -dis -##ink -too -##wn -great -after -per -pro -again -##ment -upon -hand -has -##ree -##ked -##ather -only -car -##tain -##ally -str -mar -po -fl -well -should -##ab -##pl -say -good -##ings -two -our -thr -##ud -##ving -##oss -down -##ty -##ire -pe -before -##ves -app -##ile -every -its -pl -cr -old -off -made -##ick -##ied -long -bl -##age -##ber -where -##ress -fir -##ang -mu -never -must -sm -##ful -such -##ull -##ions -##ft -pre -came -##ro -miss -##ward -##een -##ild -##orn -##ue -even -under -these -##ord -come -part -fo -sc -##ated -rem -ind -##ness -bec -##per -may -much -think -##ious -way -mister -##ount -##led -let -ey -##ath -thought -##ect -back -spe -gl -##ady -bet -##ff -##ans -res -just -first -##ise -here -ho -own -des -##ried -ob -##ces -mo -pla -##ced -cont -##ary -##ered -went -himself -men -##me -##co -##int -might -##ily -seem -##ip -beg -##oth -##ouse -##ave -wat -comp -sw -through -##igh -away -em -get -head -don -imp -##llow -bel -##ened -without -##iz -life -most -pass -ear -make -cons -som -turn -##ac -##iv -shall -##ng -pres -acc -those -eyes -house -att -somet -jo -still -night -call -##red -op -##ew -##ens -wom -last -##ign -tell -##ause -nothing -##ark -sl -new -##ction -la -##iff -take -being -comm -many -##les -ser -word -inter -##atter -wr -##ons -ret -found -##are -##ase -while -##ory -##ix -saw -put -year -##iend -##ople -##dy -young -far -##aking -once -friend -people -##air -##ether -work -##ors -##ible -room -##ently -three -##ted -yet -##pect -##ever -count -same -father -right -child -cour -##ne -place -another -##ition -want -though -##less -##ince -king -##ian -happ -heart -end -gra -nat -face -##thing -love -took -dist -arm -##ace -inst -rep -##alk -##man -##ult -char -nor -te -ple -set -##cl -light -looked -##gg -missus -asked -mind -yes -supp -wa -ll -left -##cess -##ting -ac -##ub -##augh -##ents -##te -##ished -pers -##ways -things -sir -mom -##irl -near -##sw -##ations -sat -find -##der -##sel -##ia -world -##ower -##rew -##ject -##fect -give -cap -##av -why -mother -gu -gen -##ict -always -##so -##ling -answ -##pped -soon -##iver -##den -form -heard -war -thing -something -seemed -sub -god -door -girl -##ange -##ph -##und -wind -because -diff -mon -going -##ertain -##de -told -home -ke -##orm -got -##ither -el -##ret -##ock -sur -high -##ched -eng -gi -ma -ph -##ank -hands -moment -woman -follow -##land -dr -ve -vo -feel -##aut -##ning -poss -ev -##ical -##lt -##ple -boy -return -rest -reg -knew -each -inf -conf -oh -sil -##ner -kind -cle -##ush -hel -rel -##orr -tra -##ual -ass -wo -##ars -##cept -belie -bre -certain -against -water -##uck -exp -hard -prop -years -del -unt -quite -beh -side -##iness -##ined -appear -four -few -##ool -done -better -##ited -half -min -##ail -tre -ref -also -##cy -##oub -##vent -began -##most -##ows -ter -##ully -##act -enough -pur -##ised -dra -act -dear -having -seen -gener -lady -hum -##aster -##ared -##aken -##ash -##oke -hour -inde -##ps -##ying -cur -called -##ier -##ason -does -whole -##uth -morn -turned -pleas -##ial -gave -betw -##ense -occ -##aps -sou -course -##ib -ins -##gether -##ream -##ired -both -add -poor -col -read -bla -##ten -##selves -ste -among -between -keep -##ular -beaut -matter -care -morning -white -##ained -sure -name -##ged -toward -##ute -##owed -small -whom -par -exc -sk -pa -lar -felt -##ks -##ating -##osed -##bo -myself -open -six -es -##ru -herself -however -##xt -##ond -hund -stood -hundred -##plied -sent -inc -round -##tered -sim -##ife -##ates -##ters -days -almost -point -##eng -gre -eight -words -hor -fam -stand -land -##dden -##ung -mean -sun -##ird -##haps -##iet -perhaps -##ures -art -sle -ed -##iss -best -dark -sudden -hear -bu -replied -bar -voice -met -##ton -che -anything -till -underst -##ren -##its -until -black -##oud -looking -cried -##ned -suff -aw -##ins -prin -red -##amp -less -lay -next -law -##end -power -##vel -fact -##not -brought -##up -##ately -##enty -country -help -##als -sn -quest -wonder -use -vis -laugh -##med -##fully -together -air -unc -need -adv -wish -##ove -person -indeed -since -contin -##oney -gent -present -har -forg -full -run -rather -ra -##ok -cond -ide -sy -##eter -##rib -feet -fin -fall -##cei -five -mil -talk -##ater -bed -##ured -doct -##sh -reach -interest -lord -gone -##tle -death -consid -##fort -serv -wife -doub -adm -##ief -##pr -##ene -taken diff --git a/examples/asr/experimental/wpe_vocabs/2k/vocab.txt b/examples/asr/experimental/wpe_vocabs/2k/vocab.txt deleted file mode 100644 index 08203cc74baa..000000000000 --- a/examples/asr/experimental/wpe_vocabs/2k/vocab.txt +++ /dev/null @@ -1,2048 +0,0 @@ -[PAD] -[UNK] -[CLS] -[SEP] -[MASK] -' -a -b -c -d -e -f -g -h -i -j -k -l -m -n -o -p -q -r -s -t -u -v -w -x -y -z -##l -##a -##u -##d -##i -##n -##s -##e -##h -##y -##o -##c -##k -##r -##t -##m -##v -##b -##g -##p -##f -##w -##z -##j -##q -##x -th -the -##er -##nd -##in -##ed -##ou -##at -##en -and -##or -##es -to -of -##on -##is -##ing -##ar -##it -##as -##an -##ll -in -##re -wh -he -##om -be -ha -##le -##ot -##ow -##ic -##ut -it -##ld -that -sh -##ly -was -##gh -##id -##se -on -##ve -##et -##ent -you -##im -##st -##ce -##ith -for -##ir -##ion -as -##ay -his -##al -with -##ur -st -##ter -her -##ver -##ch -re -##ad -##ght -an -not -is -had -sa -at -##oo -##her -but -##ould -she -##ct -##ess -fr -se -##am -pr -we -him -so -##ill -##ain -su -##ight -me -##ere -##all -##ke -##il -de -my -they -all -##est -ch -##ich -ne -by -have -##ome -##pp -##ck -li -this -do -##ore -con -##us -which -##ri -##ul -up -said -from -##th -##ard -##ge -##ra -or -one -no -ex -were -there -##os -##pe -man -##ble -who -al -##ant -##ie -##ous -##ust -##ation -fe -##and -them -##ers -##lf -when -##ind -##ast -go -##ood -kn -##un -##our -##ist -##em -are -##out -would -##art -what -le -their -##ong -en -if -un -##if -com -##ound -ab -out -wor -##ol -will -int -br -##ak -mis -##ate -##op -##um -##ough -##la -##itt -ar -##ook -##ish -##one -##ap -been -##own -##ame -then -##ven -##self -##ive -tr -ro -##ig -fa -##reat -af -sp -qu -##ery -##ity -##ort -##ost -tw -ag -could -am -##el -##ake -any -##ure -some -your -can -more -tim -##ep -##nder -ever -##ence -##ther -##fe -into -##ry -##ose -now -##au -##ag -cl -very -##qu -##ved -bo -other -did -##ance -than -##ittle -little -##ine -##ies -##oun -look -##way -##urn -like -##ite -gr -##ear -##ach -lo -##ass -##other -##able -##ought -know -time -im -us -co -##fore -##ell -how -##hing -##cc -ad -day -##ack -about -##ade -##ice -see -over -##ci -##ide -##pt -dis -##ink -too -##wn -great -after -per -pro -again -##ment -upon -hand -has -##ree -##ked -##ather -only -car -##tain -##ally -str -mar -po -fl -well -should -##ab -##pl -say -good -##ings -two -our -thr -##ud -##ving -##oss -down -##ty -##ire -pe -before -##ves -app -##ile -every -its -pl -cr -old -off -made -##ick -##ied -long -bl -##age -##ber -where -##ress -fir -##ang -mu -never -must -sm -##ful -such -##ull -##ions -##ft -pre -came -##ro -miss -##ward -##een -##ild -##orn -##ue -even -under -these -##ord -come -part -fo -sc -##ated -rem -ind -##ness -bec -##per -may -much -think -##ious -way -mister -##ount -##led -let -ey -##ath -thought -##ect -back -spe -gl -##ady -bet -##ff -##ans -res -just -first -##ise -here -ho -own -des -##ried -ob -##ces -mo -pla -##ced -cont -##ary -##ered -went -himself -men -##me -##co -##int -might -##ily -seem -##ip -beg -##oth -##ouse -##ave -wat -comp -sw -through -##igh -away -em -get -head -don -imp -##llow -bel -##ened -without -##iz -life -most -pass -ear -make -cons -som -turn -##ac -##iv -shall -##ng -pres -acc -those -eyes -house -att -somet -jo -still -night -call -##red -op -##ew -##ens -wom -last -##ign -tell -##ause -nothing -##ark -sl -new -##ction -la -##iff -take -being -comm -many -##les -ser -word -inter -##atter -wr -##ons -ret -found -##are -##ase -while -##ory -##ix -saw -put -year -##iend -##ople -##dy -young -far -##aking -once -friend -people -##air -##ether -work -##ors -##ible -room -##ently -three -##ted -yet -##pect -##ever -count -same -father -right -child -cour -##ne -place -another -##ition -want -though -##less -##ince -king -##ian -happ -heart -end -gra -nat -face -##thing -love -took -dist -arm -##ace -inst -rep -##alk -##man -##ult -char -nor -te -ple -set -##cl -light -looked -##gg -missus -asked -mind -yes -supp -wa -ll -left -##cess -##ting -ac -##ub -##augh -##ents -##te -##ished -pers -##ways -things -sir -mom -##irl -near -##sw -##ations -sat -find -##der -##sel -##ia -world -##ower -##rew -##ject -##fect -give -cap -##av -why -mother -gu -gen -##ict -always -##so -##ling -answ -##pped -soon -##iver -##den -form -heard -war -thing -something -seemed -sub -god -door -girl -##ange -##ph -##und -wind -because -diff -mon -going -##ertain -##de -told -home -ke -##orm -got -##ither -el -##ret -##ock -sur -high -##ched -eng -gi -ma -ph -##ank -hands -moment -woman -follow -##land -dr -ve -vo -feel -##aut -##ning -poss -ev -##ical -##lt -##ple -boy -return -rest -reg -knew -each -inf -conf -oh -sil -##ner -kind -cle -##ush -hel -rel -##orr -tra -##ual -ass -wo -##ars -##cept -belie -bre -certain -against -water -##uck -exp -hard -prop -years -del -unt -quite -beh -side -##iness -##ined -appear -four -few -##ool -done -better -##ited -half -min -##ail -tre -ref -also -##cy -##oub -##vent -began -##most -##ows -ter -##ully -##act -enough -pur -##ised -dra -act -dear -having -seen -gener -lady -hum -##aster -##ared -##aken -##ash -##oke -hour -inde -##ps -##ying -cur -called -##ier -##ason -does -whole -##uth -morn -turned -pleas -##ial -gave -betw -##ense -occ -##aps -sou -course -##ib -ins -##gether -##ream -##ired -both -add -poor -col -read -bla -##ten -##selves -ste -among -between -keep -##ular -beaut -matter -care -morning -white -##ained -sure -name -##ged -toward -##ute -##owed -small -whom -par -exc -sk -pa -lar -felt -##ks -##ating -##osed -##bo -myself -open -six -es -##ru -herself -however -##xt -##ond -hund -stood -hundred -##plied -sent -inc -round -##tered -sim -##ife -##ates -##ters -days -almost -point -##eng -gre -eight -words -hor -fam -stand -land -##dden -##ung -mean -sun -##ird -##haps -##iet -perhaps -##ures -art -sle -ed -##iss -best -dark -sudden -hear -bu -replied -bar -voice -met -##ton -che -anything -till -underst -##ren -##its -until -black -##oud -looking -cried -##ned -suff -aw -##ins -prin -red -##amp -less -lay -next -law -##end -power -##vel -fact -##not -brought -##up -##ately -##enty -country -help -##als -sn -quest -wonder -use -vis -laugh -##med -##fully -together -air -unc -need -adv -wish -##ove -person -indeed -since -contin -##oney -gent -present -har -forg -full -run -rather -ra -##ok -cond -ide -sy -##eter -##rib -feet -fin -fall -##cei -five -mil -talk -##ater -bed -##ured -doct -##sh -reach -interest -lord -gone -##tle -death -consid -##fort -serv -wife -doub -adm -##ief -##pr -##ene -taken -others -hope -cannot -speak -themselves -along -ext -dire -bes -ent -master -##od -##ued -gentle -##lic -##uring -fire -large -pol -##ention -##oy -thus -##iled -sec -##ets -play -real -prom -##ments -##ield -bus -differ -##ison -thou -##ember -##ead -##ches -reason -thous -##iting -brother -##og -##akes -thousand -money -dep -answered -true -##unt -children -behind -sound -##ants -##ably -wood -##used -whose -ele -twenty -mer -rec -believe -inv -hon -book -dec -opp -hold -##ude -fair -passed -sch -##ided -##ities -##ength -##cious -sing -##cover -##ys -nature -##ering -alone -dead -##app -sever -cat -close -gold -start -hur -fur -##ont -doctor -ask -son -ground -pat -sea -strong -leave -compan -##illed -##ept -##cond -##ides -obs -fell -##body -doubt -sign -earth -remember -fear -given -##ances -therefore -ir -##oc -##ually -deep -##iful -##ressed -body -often -##du -pap -##ane -short -second -tem -##owing -##ceed -stre -fort -##mb -##ale -horse -mad -chr -order -const -ten -clear -##irc -fif -eas -##vers -bur -really -spir -##ceived -##read -##laim -ill -##osition -bra -understand -list -abs -spr -##az -pain -question -##anger -everything -aff -circ -wall -coming -##cted -##ching -##ready -above -already -least -expl -##esh -step -used -prince -itself -##ister -necess -case -around -tri -##hn -soul -##ized -suddenly -lad -evening -general -mag -num -##imes -known -quick -captain -##ific -whether -lear -within -live -times -expect -state -friends -swe -##gn -bring -##alked -sort -##men -women -table -meet -john -sum -returned -held -dri -big -walk -##eth -says -lea -##cer -obser -else -mid -during -##ident -mus -##ape -beautiful -ord -either -town -lost -sight -sleep -##vern -means -foot -cut -cal -kept -##ety -##gar -##ience -prof -fellow -dream -mist -past -became -pret -bad -##ages -stra -making -##ught -##ey -##ution -##lish -object -est -##ts -towards -dev -human -manner -strange -low -##old -##ott -##ient -##gs -##ines -ann -##here -ang -daugh -##oon -city -rich -sometimes -##aught -##airs -rose -dem -##ility -ready -ru -##cing -##itch -pal -road -ran -corn -thy -cold -##ians -##iew -##aring -acr -bit -won -didn -christ -squ -truth -##mer -##orrow -##nded -thir -soci -perfect -subject -##br -ship -mur -answer -pie -disc -arr -free -business -river -##arr -cast -family -##ountain -hus -jud -letter -##ony -hair -arms -##oken -##ote -become -sit -ut -ah -##ingly -##ouble -val -##aint -natur -possible -several -fine -##ffect -lead -express -sus -glad -arri -blood -##itting -##rie -quiet -able -##ert -idea -##iously -##itted -##bed -charac -##ah -begin -chur -story -eye -##band -##ative -consider -across -##li -except -equ -tou -##dle -cent -##isf -sto -partic -diffic -window -surpr -prov -grand -##use -flo -pen -ca -govern -wild -soft -cross -different -effect -##ased -def -longer -discover -followed -sold -thee -husband -fre -##antly -##by -exper -##ards -pub -cor -enter -win -suppose -##teen -respect -hast -##ices -proper -sal -comple -show -heav -##cul -happy -creat -taking -##essed -stay -requ -appro -spoke -daughter -wee -##une -##lly -##alth -anim -din -green -bird -mere -##ny -cra -gard -##empt -##ms -draw -reached -##ov -appeared -cover -##ley -exist -##ug -##ination -pretty -hea -opened -##anged -slow -##ize -imag -saying -##utes -common -tom -mor -coll -ap -occas -##ames -##ench -mountain -satisf -sense -succ -fav -##oura -##ising -rock -accord -scar -break -month -continued -stri -contr -church -tree -carried -spirit -##outh -cry -##ump -wanted -certainly -secret -command -##be -dest -##lled -move -lu -street -account -ought -prep -sens -##ats -##ement -purp -##ots -##riage -tear -esc -##owers -breat -dro -fan -character -##res -##ime -decl -##medi -stud -school -##itude -sla -quar -heaven -feeling -##son -sad -regard -bright -worth -##ney -dut -hot -hung -##laimed -##ric -disp -trans -delight -reco -##iling -rid -##eric -blue -hill -cloth -div -mot -disapp -##idence -produ -sweet -success -##ury -fast -english -ye -dress -port -neither -court -var -seven -##amed -##ains -princess -lived -view -##tern -##ushed -immedi -self -chief -hours -mill -exam -##wh -ce -tried -##day -entered -##ask -eat -carry -import -bear -danger -##ledge -offic -du -cause -none -forward -##umb -uncle -number -det -further -##gr -pay -##gin -added -eff -particular -deal -queen -prot -led -front -acqu -pray -##ored -happened -alth -fight -walked -later -##bit -joy -intell -##ster -comfort -prob -maj -##ening -public -bene -##uss -##ship -although -fig -direct -##iar -##ger -bat -mouth -##ality -mem -seems -sho -##side -##iam -sol -yourself -fore -##to -doing -living -desp -##oof -##pend -field -received -shad -bey -beyond -phil -line -visit -##inct -party -garden -hall -vi -##uff -americ -##ism -mat -gentleman -burn -##ested -laid -##irt -##raid -occasion -entire -age -sister -repe -##ously -prison -accom -whis -nearly -##ndon -trees -##cri -ge -eighteen -send -##wards -early -breath -lab -forth -##ior -try -##ones -med -terri -da -##ilt -princi -enem -thinking -##where -chance -minutes -cre -anx -mary -##anced -pict -wait -vill -afraid -crow -smile -##els -england -late -pleasure -bank -##bert -##ming -faith -super -news -fle -seeing -rob -neigh -##oint -##iment -hat -auth -bill -plain -trouble -silence -aunt -exclaimed -shar -coun -er -##gy -miles -##ply -glass -drew -neighb -mine -pract -heavy -standing -sevent -change -necessary -chap -##pping -purpose -concer -length -inqu -bell -##iest -natural -deter -##icked -hardly -top -caught -##ues -strength -##ives -##ounded -##aw -##ession -circum -fing -stopped -av -opin -##ament -priv -instead -##ences -##pper -grew -loved -island -knight -ago -inn -peace -watch -##inary -##iced -third -beauty -##uct -emb -conv -approach -determ -ver -##ler -##ining -##hood -##king -convers -straight -besides -position -thirty -##iter -den -attention -south -##ager -##ically -hist -##eless -comes -##aged -clock -force -plan -sitting -please -dog -whatever -##ugg -inform -hop -chair -lie -##form -build -bab -##ma -##ustom -je -girls -rom -french -struck -pull -wrong -knowledge -grace -##oom -scarce -resol -thoughts -attempt -fifty -rap -box -getting -company -crowd -##tained -##asp -##ils -class -die -drink -dinner -exact -enj -thick -saint -slight -save -maid -immediately -born -main -##ius -##crib -rev -remark -ast -##oose -north -##ists -tears -treat -##usion -meant -boys -fat -##ls -slowly -author -incl -lim -died -infl -fool -fish -sugg -post -accept -##ern -##elve -##ceive -##cient -stone -boat -paper -fright -conse -jour -ty -enc -quickly -youth -##ging -van -vict -##ified -belong -warm -##par -fix -imposs -forgot -beside -margar -leg -tone -camp -bound -makes -margaret -##mon -brown -##bs -##val -moon -##rupt -gather -lips -placed -circumst -food -filled -##iers -##ift -desire -##ee -london -distance -forest -floor -obl -##itions -spring -jack -viol -##ics -pock -following -remained -ger -grow -snow -government -ball -hors -##ented -##aded -broken -laughed -##ann -##oured -refle -safe -##ries -hen -##ively -##itten -depart -neck -easy -thank -##dom -##ham -##ope -knows -waiting -remain -hid -fresh -und -anne -dread -ash -##arch -result -proceed -gaz -##uc -shout -##dd -started -gall diff --git a/examples/asr/experimental/wpe_vocabs/4k/vocab.txt b/examples/asr/experimental/wpe_vocabs/4k/vocab.txt deleted file mode 100644 index d65cec73c945..000000000000 --- a/examples/asr/experimental/wpe_vocabs/4k/vocab.txt +++ /dev/null @@ -1,4096 +0,0 @@ -[PAD] -[UNK] -[CLS] -[SEP] -[MASK] -' -a -b -c -d -e -f -g -h -i -j -k -l -m -n -o -p -q -r -s -t -u -v -w -x -y -z -##e -##s -##c -##r -##i -##p -##t -##v -##a -##n -##d -##b -##u -##l -##y -##o -##g -##w -##h -##m -##x -##k -##f -##z -##q -##j -th -the -##er -##nd -##in -##ed -##ou -##at -##en -and -##or -##es -to -of -##on -##is -##ing -##ar -##it -##as -##an -##ll -in -##re -wh -he -##om -be -ha -##le -##ot -##ow -##ic -##ut -it -##ld -that -sh -##ly -was -##gh -##id -##se -on -##ve -##et -##ent -you -##im -##st -##ce -##ith -for -##ir -##ion -as -##ay -his -##al -with -##ur -st -##ter -her -##ver -##ch -re -##ad -##ght -an -not -is -had -sa -at -##oo -##her -but -##ould -she -##ct -##ess -fr -se -##am -pr -we -him -so -##ill -##ain -su -##ight -me -##ere -##all -##ke -##il -de -my -they -all -##est -ch -##ich -ne -by -have -##ome -##pp -##ck -li -this -do -##ore -con -##us -which -##ri -##ul -up -said -from -##th -##ard -##ge -##ra -or -one -no -ex -were -there -##os -##pe -man -##ble -who -al -##ant -##ie -##ous -##ust -##ation -fe -##and -them -##ers -##lf -when -##ind -##ast -go -##ood -kn -##un -##our -##ist -##em -are -##out -would -##art -what -le -their -##ong -en -if -un -##if -com -##ound -ab -out -wor -##ol -will -int -br -##ak -mis -##ate -##op -##um -##ough -##la -##itt -ar -##ook -##ish -##one -##ap -been -##own -##ame -then -##ven -##self -##ive -tr -ro -##ig -fa -##reat -af -sp -qu -##ery -##ity -##ort -##ost -tw -ag -could -am -##el -##ake -any -##ure -some -your -can -more -tim -##ep -##nder -ever -##ence -##ther -##fe -into -##ry -##ose -now -##au -##ag -cl -very -##qu -##ved -bo -other -did -##ance -than -##ittle -little -##ine -##ies -##oun -look -##way -##urn -like -##ite -gr -##ear -##ach -lo -##ass -##other -##able -##ought -know -time -im -us -co -##fore -##ell -how -##hing -##cc -ad -day -##ack -about -##ade -##ice -see -over -##ci -##ide -##pt -dis -##ink -too -##wn -great -after -per -pro -again -##ment -upon -hand -has -##ree -##ked -##ather -only -car -##tain -##ally -str -mar -po -fl -well -should -##ab -##pl -say -good -##ings -two -our -thr -##ud -##ving -##oss -down -##ty -##ire -pe -before -##ves -app -##ile -every -its -pl -cr -old -off -made -##ick -##ied -long -bl -##age -##ber -where -##ress -fir -##ang -mu -never -must -sm -##ful -such -##ull -##ions -##ft -pre -came -##ro -miss -##ward -##een -##ild -##orn -##ue -even -under -these -##ord -come -part -fo -sc -##ated -rem -ind -##ness -bec -##per -may -much -think -##ious -way -mister -##ount -##led -let -ey -##ath -thought -##ect -back -spe -gl -##ady -bet -##ff -##ans -res -just -first -##ise -here -ho -own -des -##ried -ob -##ces -mo -pla -##ced -cont -##ary -##ered -went -himself -men -##me -##co -##int -might -##ily -seem -##ip -beg -##oth -##ouse -##ave -wat -comp -sw -through -##igh -away -em -get -head -don -imp -##llow -bel -##ened -without -##iz -life -most -pass -ear -make -cons -som -turn -##ac -##iv -shall -##ng -pres -acc -those -eyes -house -att -somet -jo -still -night -call -##red -op -##ew -##ens -wom -last -##ign -tell -##ause -nothing -##ark -sl -new -##ction -la -##iff -take -being -comm -many -##les -ser -word -inter -##atter -wr -##ons -ret -found -##are -##ase -while -##ory -##ix -saw -put -year -##iend -##ople -##dy -young -far -##aking -once -friend -people -##air -##ether -work -##ors -##ible -room -##ently -three -##ted -yet -##pect -##ever -count -same -father -right -child -cour -##ne -place -another -##ition -want -though -##less -##ince -king -##ian -happ -heart -end -gra -nat -face -##thing -love -took -dist -arm -##ace -inst -rep -##alk -##man -##ult -char -nor -te -ple -set -##cl -light -looked -##gg -missus -asked -mind -yes -supp -wa -ll -left -##cess -##ting -ac -##ub -##augh -##ents -##te -##ished -pers -##ways -things -sir -mom -##irl -near -##sw -##ations -sat -find -##der -##sel -##ia -world -##ower -##rew -##ject -##fect -give -cap -##av -why -mother -gu -gen -##ict -always -##so -##ling -answ -##pped -soon -##iver -##den -form -heard -war -thing -something -seemed -sub -god -door -girl -##ange -##ph -##und -wind -because -diff -mon -going -##ertain -##de -told -home -ke -##orm -got -##ither -el -##ret -##ock -sur -high -##ched -eng -gi -ma -ph -##ank -hands -moment -woman -follow -##land -dr -ve -vo -feel -##aut -##ning -poss -ev -##ical -##lt -##ple -boy -return -rest -reg -knew -each -inf -conf -oh -sil -##ner -kind -cle -##ush -hel -rel -##orr -tra -##ual -ass -wo -##ars -##cept -belie -bre -certain -against -water -##uck -exp -hard -prop -years -del -unt -quite -beh -side -##iness -##ined -appear -four -few -##ool -done -better -##ited -half -min -##ail -tre -ref -also -##cy -##oub -##vent -began -##most -##ows -ter -##ully -##act -enough -pur -##ised -dra -act -dear -having -seen -gener -lady -hum -##aster -##ared -##aken -##ash -##oke -hour -inde -##ps -##ying -cur -called -##ier -##ason -does -whole -##uth -morn -turned -pleas -##ial -gave -betw -##ense -occ -##aps -sou -course -##ib -ins -##gether -##ream -##ired -both -add -poor -col -read -bla -##ten -##selves -ste -among -between -keep -##ular -beaut -matter -care -morning -white -##ained -sure -name -##ged -toward -##ute -##owed -small -whom -par -exc -sk -pa -lar -felt -##ks -##ating -##osed -##bo -myself -open -six -es -##ru -herself -however -##xt -##ond -hund -stood -hundred -##plied -sent -inc -round -##tered -sim -##ife -##ates -##ters -days -almost -point -##eng -gre -eight -words -hor -fam -stand -land -##dden -##ung -mean -sun -##ird -##haps -##iet -perhaps -##ures -art -sle -ed -##iss -best -dark -sudden -hear -bu -replied -bar -voice -met -##ton -che -anything -till -underst -##ren -##its -until -black -##oud -looking -cried -##ned -suff -aw -##ins -prin -red -##amp -less -lay -next -law -##end -power -##vel -fact -##not -brought -##up -##ately -##enty -country -help -##als -sn -quest -wonder -use -vis -laugh -##med -##fully -together -air -unc -need -adv -wish -##ove -person -indeed -since -contin -##oney -gent -present -har -forg -full -run -rather -ra -##ok -cond -ide -sy -##eter -##rib -feet -fin -fall -##cei -five -mil -talk -##ater -bed -##ured -doct -##sh -reach -interest -lord -gone -##tle -death -consid -##fort -serv -wife -doub -adm -##ief -##pr -##ene -taken -others -hope -cannot -speak -themselves -along -ext -dire -bes -ent -master -##od -##ued -gentle -##lic -##uring -fire -large -pol -##ention -##oy -thus -##iled -sec -##ets -play -real -prom -##ments -##ield -bus -differ -##ison -thou -##ember -##ead -##ches -reason -thous -##iting -brother -##og -##akes -thousand -money -dep -answered -true -##unt -children -behind -sound -##ants -##ably -wood -##used -whose -ele -twenty -mer -rec -believe -inv -hon -book -dec -opp -hold -##ude -fair -passed -sch -##ided -##ities -##ength -##cious -sing -##cover -##ys -nature -##ering -alone -dead -##app -sever -cat -close -gold -start -hur -fur -##ont -doctor -ask -son -ground -pat -sea -strong -leave -compan -##illed -##ept -##cond -##ides -obs -fell -##body -doubt -sign -earth -remember -fear -given -##ances -therefore -ir -##oc -##ually -deep -##iful -##ressed -body -often -##du -pap -##ane -short -second -tem -##owing -##ceed -stre -fort -##mb -##ale -horse -mad -chr -order -const -ten -clear -##irc -fif -eas -##vers -bur -really -spir -##ceived -##read -##laim -ill -##osition -bra -understand -list -abs -spr -##az -pain -question -##anger -everything -aff -circ -wall -coming -##cted -##ching -##ready -above -already -least -expl -##esh -step -used -prince -itself -##ister -necess -case -around -tri -##hn -soul -##ized -suddenly -lad -evening -general -mag -num -##imes -known -quick -captain -##ific -whether -lear -within -live -times -expect -state -friends -swe -##gn -bring -##alked -sort -##men -women -table -meet -john -sum -returned -held -dri -big -walk -##eth -says -lea -##cer -obser -else -mid -during -##ident -mus -##ape -beautiful -ord -either -town -lost -sight -sleep -##vern -means -foot -cut -cal -kept -##ety -##gar -##ience -prof -fellow -dream -mist -past -became -pret -bad -##ages -stra -making -##ught -##ey -##ution -##lish -object -est -##ts -towards -dev -human -manner -strange -low -##old -##ott -##ient -##gs -##ines -ann -##here -ang -daugh -##oon -city -rich -sometimes -##aught -##airs -rose -dem -##ility -ready -ru -##cing -##itch -pal -road -ran -corn -thy -cold -##ians -##iew -##aring -acr -bit -won -didn -christ -squ -truth -##mer -##orrow -##nded -thir -soci -perfect -subject -##br -ship -mur -answer -pie -disc -arr -free -business -river -##arr -cast -family -##ountain -hus -jud -letter -##ony -hair -arms -##oken -##ote -become -sit -ut -ah -##ingly -##ouble -val -##aint -natur -possible -several -fine -##ffect -lead -express -sus -glad -arri -blood -##itting -##rie -quiet -able -##ert -idea -##iously -##itted -##bed -charac -##ah -begin -chur -story -eye -##band -##ative -consider -across -##li -except -equ -tou -##dle -cent -##isf -sto -partic -diffic -window -surpr -prov -grand -##use -flo -pen -ca -govern -wild -soft -cross -different -effect -##ased -def -longer -discover -followed -sold -thee -husband -fre -##antly -##by -exper -##ards -pub -cor -enter -win -suppose -##teen -respect -hast -##ices -proper -sal -comple -show -heav -##cul -happy -creat -taking -##essed -stay -requ -appro -spoke -daughter -wee -##une -##lly -##alth -anim -din -green -bird -mere -##ny -cra -gard -##empt -##ms -draw -reached -##ov -appeared -cover -##ley -exist -##ug -##ination -pretty -hea -opened -##anged -slow -##ize -imag -saying -##utes -common -tom -mor -coll -ap -occas -##ames -##ench -mountain -satisf -sense -succ -fav -##oura -##ising -rock -accord -scar -break -month -continued -stri -contr -church -tree -carried -spirit -##outh -cry -##ump -wanted -certainly -secret -command -##be -dest -##lled -move -lu -street -account -ought -prep -sens -##ats -##ement -purp -##ots -##riage -tear -esc -##owers -breat -dro -fan -character -##res -##ime -decl -##medi -stud -school -##itude -sla -quar -heaven -feeling -##son -sad -regard -bright -worth -##ney -dut -hot -hung -##laimed -##ric -disp -trans -delight -reco -##iling -rid -##eric -blue -hill -cloth -div -mot -disapp -##idence -produ -sweet -success -##ury -fast -english -ye -dress -port -neither -court -var -seven -##amed -##ains -princess -lived -view -##tern -##ushed -immedi -self -chief -hours -mill -exam -##wh -ce -tried -##day -entered -##ask -eat -carry -import -bear -danger -##ledge -offic -du -cause -none -forward -##umb -uncle -number -det -further -##gr -pay -##gin -added -eff -particular -deal -queen -prot -led -front -acqu -pray -##ored -happened -alth -fight -walked -later -##bit -joy -intell -##ster -comfort -prob -maj -##ening -public -bene -##uss -##ship -although -fig -direct -##iar -##ger -bat -mouth -##ality -mem -seems -sho -##side -##iam -sol -yourself -fore -##to -doing -living -desp -##oof -##pend -field -received -shad -bey -beyond -phil -line -visit -##inct -party -garden -hall -vi -##uff -americ -##ism -mat -gentleman -burn -##ested -laid -##irt -##raid -occasion -entire -age -sister -repe -##ously -prison -accom -whis -nearly -##ndon -trees -##cri -ge -eighteen -send -##wards -early -breath -lab -forth -##ior -try -##ones -med -terri -da -##ilt -princi -enem -thinking -##where -chance -minutes -cre -anx -mary -##anced -pict -wait -vill -afraid -crow -smile -##els -england -late -pleasure -bank -##bert -##ming -faith -super -news -fle -seeing -rob -neigh -##oint -##iment -hat -auth -bill -plain -trouble -silence -aunt -exclaimed -shar -coun -er -##gy -miles -##ply -glass -drew -neighb -mine -pract -heavy -standing -sevent -change -necessary -chap -##pping -purpose -concer -length -inqu -bell -##iest -natural -deter -##icked -hardly -top -caught -##ues -strength -##ives -##ounded -##aw -##ession -circum -fing -stopped -av -opin -##ament -priv -instead -##ences -##pper -grew -loved -island -knight -ago -inn -peace -watch -##inary -##iced -third -beauty -##uct -emb -conv -approach -determ -ver -##ler -##ining -##hood -##king -convers -straight -besides -position -thirty -##iter -den -attention -south -##ager -##ically -hist -##eless -comes -##aged -clock -force -plan -sitting -please -dog -whatever -##ugg -inform -hop -chair -lie -##form -build -bab -##ma -##ustom -je -girls -rom -french -struck -pull -wrong -knowledge -grace -##oom -scarce -resol -thoughts -attempt -fifty -rap -box -getting -company -crowd -##tained -##asp -##ils -class -die -drink -dinner -exact -enj -thick -saint -slight -save -maid -immediately -born -main -##crib -##ius -rev -remark -ast -##oose -north -##ists -tears -treat -##usion -meant -boys -fat -##ls -slowly -author -incl -lim -died -infl -fool -fish -sugg -post -accept -##ern -##elve -##ceive -##cient -stone -boat -paper -fright -conse -jour -ty -enc -quickly -youth -##ging -van -vict -##ified -belong -warm -##par -fix -imposs -forgot -beside -margar -leg -tone -camp -bound -makes -margaret -##mon -brown -##bs -##val -moon -##rupt -gather -lips -placed -circumst -food -filled -##iers -##ift -desire -##ee -london -distance -forest -floor -obl -##itions -spring -jack -viol -##ics -pock -following -remained -ger -grow -snow -government -ball -hors -##ented -##aded -broken -laughed -##ann -##oured -refle -safe -##ries -hen -##ively -##itten -depart -neck -easy -thank -##dom -##ope -##ham -knows -waiting -remain -hid -fresh -und -anne -dread -ash -##arch -result -proceed -gaz -##uc -shout -##dd -started -gall -complete -whisper -wide -horses -##isted -##cially -##ges -service -piece -##eness -forget -palace -guard -doll -##erv -talking -yell -##cial -lift -fun -grave -week -impossible -effort -##nding -imm -army -difficult -##let -##reme -stop -mess -narr -deg -incre -corner -society -path -weak -shut -hy -teach -cloud -##ency -##ivid -demand -##ended -##unk -nine -recogn -##cell -##fer -turning -sky -ori -##asure -##olog -opinion -sin -beat -single -dw -fly -lang -mass -outside -kiss -trust -occup -##uit -below -appearance -silver -bro -west -aftern -gun -couldn -ant -conn -showed -conversation -running -mention -nin -greater -music -##ases -##ises -##fection -mark -espe -##ror -##ories -une -spot -##ourage -peter -##ork -##anch -presence -winter -battle -hun -morrow -probably -##ection -clothes -fash -compl -wished -appar -married -arrived -suffic -eager -former -giving -pop -##pic -sand -usual -relig -simple -sym -##itation -neg -sor -moved -months -speaking -silent -cab -pun -expression -covered -afternoon -pet -##aped -occur -states -##rief -admir -##light -easily -##unity -phys -tender -beginning -duty -chapter -influ -##zz -instant -polit -enjoy -sick -tur -stream -figure -##ald -##ract -jew -vol -minute -pleasant -scarcely -conscious -terrible -kill -concl -pale -raised -##oop -fashion -thor -##yal -##ze -twelve -leaving -##ature -mel -fut -threw -flowers -attack -trying -##cked -sword -tall -marry -expected -according -##uel -forty -stick -temper -iron -oblig -train -rapid -ladies -learn -especially -##ipped -##orted -isn -##amb -evil -bread -learned -##orth -office -surprise -write -purs -written -killed -star -consequ -exh -places -condition -direction -##jo -entirely -action -##ding -##wise -##ibly -##mit -happiness -id -decided -golden -langu -##uous -note -bott -unless -books -fal -##aled -wonderful -shook -##ounds -##ech -##onel -perfectly -geor -sail -##ume -broad -closed -##atic -latter -steps -utter -merely -history -##gan -##osing -wise -wouldn -understood -middle -prev -enemy -sole -gar -simply -tast -shot -board -ring -sav -bow -##io -reply -##gress -changed -excell -grass -finally -##ital -suc -summer -sharp -fortune -experience -succeed -##uted -##retched -journey -observed -##osp -afterwards -##fast -##ec -earn -persons -lean -broke -deser -fit -mir -marriage -##acy -repres -bitter -lying -cou -rain -trave -listen -situ -##anc -##cience -##ected -origin -wit -##olute -##oses -calm -lives -escape -beneath -##ouses -clim -hes -ven -repeated -pocket -##lling -liber -oper -tail -whe -passion -dick -mut -becom -dar -allowed -famil -ow -##ivers -##mn -gate -spite -health -##str -##ission -ign -cand -habit -empl -strugg -firm -sorry -##under -madame -furn -sigh -future -##umed -nice -separ -presently -circumstances -##reh -##ples -sett -loud -worse -corr -wand -spread -letters -magn -disg -passing -pleased -darkness -##come -civ -##cies -tea -apart -disappear -important -##erable -legs -nation -delic -dressed -game -extra -walls -dry -virt -dim -rub -##idently -absolute -blind -discovered -exactly -dam -##oved -unh -cost -arch -employ -##velop -cous -beast -opport -hu -dropped -pack -subst -smiled -chee -resp -promise -##urs -host -creature -yellow -cla -notice -priest -knowing -heads -##esty -cook -shore -##ishment -individ -existence -determined -##lend -storm -seat -ears -##ours -mount -ur -##ina -conce -##anging -fixed -press -wear -major -ves -pros -##iod -motion -empt -leaves -bottom -arg -##iety -##sy -##enth -nobody -pick -##amber -##que -bag -intellig -stir -assist -dom -advant -tired -vent -##ument -##ashed -dan -##action -wrote -considered -houses -suit -cheer -clean -fla -lou -##path -##rel -cho -castle -frequ -drawing -luck -##cent -##pan -##idge -##ulty -ones -noble -splend -honor -paid -speech -estab -faint -##ires -individual -vall -##inite -##rodu -birds -allow -impress -jane -##my -song -touch -various -moder -believed -##istr -mod -somewh -period -shadow -affect -mal -extreme -judge -village -royal -somewhat -lower -##go -##zen -agree -remembered -aston -declared -##ays -ham -parts -colonel -##cil -breakfast -shr -tow -drop -surely -##ayed -##uded -scene -describ -comb -greatest -influence -custom -animal -##itary -sake -perform -soldiers -##iny -ancient -##ests -evidently -drawn -looks -ways -harm -destro -revol -##nes -##ator -##anted -picture -##ption -likely -laws -holding -difficulty -courage -inj -prevent -mort -troub -burst -angry -proud -spoken -##ron -##head -##ision -attend -watched -east -horr -pot -##na -agre -private -##bing -shop -jul -finished -anxious -##cise -fifteen -social -dism -wine -ideas -murder -mater -george -space -scr -inside -gentlemen -civil -##iently -formed -fol -fail -constit -lot -goes -thin -surf -servant -ourselves -fallen -fancy -henry -##ium -kne -advent -carriage -baby -tong -everybody -commun -##itive -ine -waited -grou -##gged -vain -##arked -##otion -favor -bal -speed -guess -narrow -windows -carefully -ice -jim -noise -meth -hero -william -pecul -promised -walking -forgotten -obliged -earnest -lose -imper -vessel -grad -glance -shoulder -animals -noticed -knock -peculiar -##cks -quietly -cup -ta -serious -tremb -generally -american -france -exer -##iction -property -stranger -shoulders -##osity -sympath -bod -flor -cow -dust -affection -conduct -##rop -##raph -higher -handsome -##iant -##nday -recover -wel -poet -pure -chamber -distinct -gh -mam -pier -grown -##uly -rough -##enced -desce -murm -##stem -personal -##osite -offered -##onsie -edge -built -whispered -skin -pieces -##arily -##itated -suffer -butter -faces -spent -##ables -interrupt -##how -##isters -develop -unk -fond -coat -touched -hol -jump -pi -##ingu -wasn -surprised -race -##af -##uce -##oted -dish -##ob -smo -educ -brave -fer -##over -weather -eld -wealth -system -german -false -##yn -##aked -cousin -##key -ben -prepared -##wood -fingers -surr -intellect -##ipp -##ites -opportunity -introdu -lovely -rag -crown -matters -companion -alar -innoc -ris -sorrow -##hern -quarter -band -lake -mix -meaning -mountains -memory -dor -##ring -##leep -##ctions -frank -finding -mach -charge -pan -gray -avo -inh -grat -greatly -dare -asleep -offer -dav -alto -##ounced -opposite -##ett -##unch -breast -bare -reading -altogether -writing -##pected -degree -exec -##fortun -hath -##eal -feelings -rail -##night -honour -fate -##attered -meeting -##ilar -alive -hide -cruel -blow -hurt -loss -defe -thrown -caused -##oned -seek -serve -profess -duke -bent -##kes -united -kingdom -situation -empty -wis -liked -swift -opening -servants -bush -suspic -freed -##ointed -surface -questions -##aff -curious -ess -accompan -christian -fill -wid -paris -satisfied -ere -pity -sides -##nds -equal -height -##osop -ordered -depend -grey -listened -##pet -##eer -rejo -capt -##ibility -despair -mart -happen -hurried -dollars -lin -loc -language -unf -yours -supposed -laughing -settled -talked -rode -perm -distingu -hurry -lifted -destroy -value -taste -##ores -##ners -square -vast -render -labor -rul -roof -stock -telling -papa -study -##ara -pin -rising -tal -sufficient -due -rise -##eds -forced -philosop -nose -sixty -egg -stat -fault -##asc -station -##ille -distur -regular -kitch -special -honest -building -shape -season -pride -smiling -orders -accompl -sacri -usually -watching -impro -##ushing -passage -valley -material -stairs -remarked -frightened -woods -wed -catch -mistress -directly -##he -prete -gloom -lines -justice -doesn -suggest -diam -growing -bless -##like -eur -##erence -ordinary -gathered -tit -threat -weeks -method -ale -ange -seized -search -discuss -ble -##ify -ignor -deb -##eping -supper -##uses -##ral -##bur -##chief -march -bold -putting -explain -##ik -nearer -york -affairs -contrary -##yr -##ishing -moving -streets -bir -##icated -wash -kitchen -bas -cit -##eder -imagine -prefer -possess -##rance -deliver -##imate -receive -monsie -por -twice -monsieur -##min -hole -sacrif -##hip -rever -reck -process -favour -dign -##aly -europe -unw -working -heat -##antic -cler -proved -##ades -pra -##lin -arth -lod -prove -##ception -pulled -pit -##ette -authority -jer -##ole -haven -movement -##tering -##usted -engaged -brothers -advantage -ber -##lished -arthur -stones -farm -difference -aside -hence -possession -observ -##etic -exte -soldier -hills -fortun -##uls -instance -ren -##iding -holy -color -flew -rate -wal -doors -trib -busy -lat -address -familiar -weight -played -##set -flower -aware -solemn -di -liter -test -indian -emper -distant -interesting -bull -thorough -ol -wore -worked -amb -explained -excellent -splendid -tongue -named -pard -shame -sam -franc -moments -trium -wil -##eav -myster -seated -spect -instantly -unkn -endeav -similar -naturally -triumph -##cr -measure -dogs -amount -elect -officer -##itable -uns -necessity -rank -##theless -brit -confidence -vir -##illa -##ulous -nevertheless -agreed -hearing -##ancho -takes -##ental -univers -fier -earl -keeping -drive -produced -aud -aug -cott -names -##con -disappeared -##agn -throw -president -gods -magic -prisoner -represent -unknown -##por -##ague -terror -contain -bri -smoke -support -detail -wicked -flat -yester -artic -dull -falling -##vey -worthy -liberty -##ulation -design -wants -evidence -companions -spirits -bid -##bers -coast -thro -mighty -particularly -witness -##ky -clever -sought -span -rare -officers -##lv -##ford -##zy -yesterday -powers -yield -organ -amaz -pointed -claim -content -possibly -cool -hom -shock -preser -terms -dwell -persu -drove -ceased -acquaint -occurred -lies -gri -gree -tor -otherwise -finger -emperor -bask -stars -knees -pair -paused -##ruction -failed -required -unhapp -diamond -aut -rat -ali -double -forms -gives -##race -bob -eliz -travel -carrying -gle -##iles -teeth -freedom -shown -forb -fruit -##umn -waters -hearts -ener -pill -##soci -example -upper -milk -affair -softly -moral -foreign -charles -excit -huge -##oes -possessed -cases -fet -##arther -counten -report -principle -inhab -proof -intended -satisfaction -shouted -farther -group -der -##asm -unfortun -unto -singing -arrange -religion -rocks -seventeen -james -works -buy -america -succeeded -rooms -events -leading -majesty -gently -dance -tele -##using -##ached -ga -##father -key -mentioned -seventy -ros -suffering -record -cabin -invent -diss -##ival -demanded -excitement -associ -progress -##angers -##road -curi -##erous -rule -vig -clearly -sheep -regret -neighbour -##iance -instinct -awful -advice -ban -fully -sen -##ena -##mend -papers -hidden -smooth -birth -hither -##bor -stuff -impat -calling -fourth -dreadful -ain -grief -pos -##pap -##ese -brill -powerful -presented -fairy -shoot -##ker -proc -joined -afford -##iments -shel -considerable -obey -interv -voices -interested -card -conqu -cred -needed -cord -pounds -wag -advanced -undert -resolved -##istic -excited -paul -bath -asse -conditions -pictures -##gen -shining -sunday -served -steam -police -sprang -jes -salt -fields -bore -##sie -indians -fierce -ride -nodd -aband -desired -edward -importance -information -hosp -memb -perceived -task -yard -##mond -crit -control -##ternal -fold -##ensive -sooner -merch -dur -suffered -absolutely -citiz -##isc -iss -##illy -completely -cart -rad -##ois -##hold -share -willing -devil -ships -imagination -superior -ep -##ancy -feather -env -anybody -stead -dawn -##asped -occupied -curiosity -fox -yo -##inal -##asant -##asons -##ams -myst -smith -##room -type -apparently -couple -extraord -welcome -bay -daily -modern -dying -##pir -feat -accident -##ension -countenance -##iot -##llen -##ortion -crime -mult -##get -reb -alas -refused -inquir -hate -whenever -thanks -regarded -suggested -##ulf -actually -pardon -size -##gment -cult -flesh -hotel -bodies -shell -##ishop -truly -betr -##uced -##with -rapidly -odd -##alous -club -horror -mile -san -sour -flight -crossed -slave -professor -oce -rushed -excl -worst -##ization -event -corre -science -##sed -##ainted -stepped -midst -foolish -##ha -##thus -brain -##acle -cottage -buck -brief -gain -tight -expressed -drag -equally -education -august -dun -nay -##com -##abel -tempt -picked -taught -elsie -lands -driven -political -resist -resem -instr -cell -lack -barb -dangerous -ken -keen -##hy -store -dozen -trade -points -ninet -sty -nob -##ability -crim -interp -relations -delighted -members -sisters -##ially -belief -asking -anger -park -wet -plat -relief -##ification -hunting -alex -cath -##aries -meat -obst -behold -mistake -inquired -remarkable -cop -nerv -##now -careful -cere -wounded -##ading -enemies -gradually -pu -##for -thorn -worn -interrupted -stup -silk -sixteen -##racted -flour -sylv -lone -nap -##kins -##asing -depth -dreams -##esp -kindly -##reg -##acity -eleven -invol -severe -##ocr -oak -wings -hal -##lace -begun -distr -##while -benef -moreover -exped -independ -artagn -buried -approached -artagnan -saved -sex -species -harry -physical -thunder -glory -creatures -newspap -sang -##kin -plenty -useful -shoes -hoped -frequently -##vert -saf -princip -##oe -chest -bran -##ication -roll -lover -famous -recoll -queer -grim -charming -aim -loose -nur -ath -indiff -consciousness -mamma -enthus -##ips -slept -fighting -enthusi -dig -##acc -##mas -##ids -stage -anyone -thrust -desper -drunk -lamp -##rant -bew -##stone -stern -evident -meanwhile -forgive -accepted -ocean -bot -tot -wondered -playing -hyp -hale -knife -closely -meas -proceeded -message -mour -##lied -sell -union -##ustomed -exceed -precious -century -unex -kat -##oops -##aming -everywhere -minds -extraordinary -arose -capital -entrance -recall -burning -magnific -##stand -plant -neighbor -assemb -lest -##orious -##uments -colle -virtue -retreat -##iana -capable -assured -##ograph -constant -governor -increased -horn -rush -removed -facts -absence -explan -ack -mud -fever -somebody -awa -admit -jealous -kissed -popular -hut -##pelled -grant -indign -friendship -sympathy -ug -##iable -thom -alice -level -objects -pressed -qual -begged -hind -highest -clouds -ghost -acknow -##oused -emp -wishes -##oking -strike -becomes -trembling -##gu -kindness -accordingly -throat -stretched -frag -confess -grandfather -wheel -bon -russ -deck -choose -eighty -##pher -miserable -contempt -##igned -military -basket -##nal -blank -listening -ahead -fro -##elt -larger -divine -##cription -tar -##iber -stories -reve -##izing -excuse -innocent -aid -et -hadn -remind -slaves -ara -##stairs -reward -manage -smell -throughout -waves -judgment -arrival -choice -unhappy -##astic -curr -advance -informed -impression -mysterious -acquaintance -ly -notes -comr -indust -ended -lights -nurse -bought -wra -##itor -spo -marked -scream -uneas -delicate -weary -##estic -join -prompt -experi -hungry -flying -pow -bridge -##aker -##umber -visible -understanding -crying -avoid -tis -stiff -sounds -bowed -caut -goods -david -stupid -##ancing -unable -bos -spark -amid -somewhere -infinite -ashamed -dignity -gay -vic -##ka -hollow -emotion -parents -admitted -hint -##pse -temple -comfortable -intelligence -virgin -bearing -##ldom -hers -##abeth -remains -contem -settle -immense -cher -weap -##ulated -lighted -baron -##gypt -elder -thoroughly -egypt -##ilst -##anges -friendly -rome -meal -intention -##achel -returning -billy -convin -ox -price -sup -rope -project -escaped -hopes -##ology -elizabeth -safety -wound -unus -##onscious -minister -horri -helped -secure -plainly -seldom -thinks -fellows -mood -pushed -exhib -##inging -fairly -pear -##irm -maiden -##mother -##ora -##orous -appears -cheeks -ease -skill -extent -practice -religious -becoming -features -pink -tied -##ential -whence -somehow -displ -greet -faithful -concerned -bishop -##isher -gro -eagerly -eating -waste -fem -bride -##ceiving -concerning -original -accustomed diff --git a/examples/asr/experimental/wpe_vocabs/8k/vocab.txt b/examples/asr/experimental/wpe_vocabs/8k/vocab.txt deleted file mode 100644 index 378e9cebe711..000000000000 --- a/examples/asr/experimental/wpe_vocabs/8k/vocab.txt +++ /dev/null @@ -1,8192 +0,0 @@ -[PAD] -[UNK] -[CLS] -[SEP] -[MASK] -' -a -b -c -d -e -f -g -h -i -j -k -l -m -n -o -p -q -r -s -t -u -v -w -x -y -z -##n -##g -##e -##r -##l -##t -##i -##a -##m -##p -##v -##o -##w -##d -##u -##s -##b -##k -##c -##h -##f -##y -##x -##z -##q -##j -th -the -##er -##nd -##in -##ed -##ou -##at -##en -and -##or -##es -to -of -##on -##is -##ing -##ar -##it -##as -##an -##ll -in -##re -wh -he -##om -be -ha -##le -##ot -##ow -##ic -##ut -it -##ld -that -sh -##ly -was -##gh -##id -##se -on -##ve -##et -##ent -you -##im -##st -##ce -##ith -for -##ir -##ion -as -##ay -his -##al -with -##ur -st -##ter -her -##ver -##ch -re -##ad -##ght -an -not -is -had -sa -at -##oo -##her -but -##ould -she -##ct -##ess -fr -se -##am -pr -we -him -so -##ill -##ain -su -##ight -me -##ere -##all -##ke -##il -de -my -they -all -##est -ch -##ich -ne -by -have -##ome -##pp -##ck -li -this -do -##ore -con -##us -which -##ri -##ul -up -said -from -##th -##ard -##ge -##ra -or -one -no -ex -were -there -##os -##pe -man -##ble -who -al -##ant -##ie -##ous -##ust -##ation -fe -##and -them -##ers -##lf -when -##ind -##ast -go -##ood -kn -##un -##our -##ist -##em -are -##out -would -##art -what -le -their -##ong -en -if -un -##if -com -##ound -ab -out -wor -##ol -will -int -br -##ak -mis -##ate -##op -##um -##ough -##la -##itt -ar -##ook -##ish -##one -##ap -been -##own -##ame -then -##ven -##self -##ive -tr -ro -##ig -fa -##reat -af -sp -qu -##ery -##ity -##ort -##ost -tw -ag -could -am -##el -##ake -any -##ure -some -your -can -more -tim -##ep -##nder -ever -##ence -##ther -##fe -into -##ry -##ose -now -##au -##ag -cl -very -##qu -##ved -bo -other -did -##ance -than -##ittle -little -##ine -##ies -##oun -look -##way -##urn -like -##ite -gr -##ear -##ach -lo -##ass -##other -##able -##ought -know -time -im -us -co -##fore -##ell -how -##hing -##cc -ad -day -##ack -about -##ade -##ice -see -over -##ci -##ide -##pt -dis -##ink -too -##wn -great -after -per -pro -again -##ment -upon -hand -has -##ree -##ked -##ather -only -car -##tain -##ally -str -mar -po -fl -well -should -##ab -##pl -say -good -##ings -two -our -thr -##ud -##ving -##oss -down -##ty -##ire -pe -before -##ves -app -##ile -every -its -pl -cr -old -off -made -##ick -##ied -long -bl -##age -##ber -where -##ress -fir -##ang -mu -never -must -sm -##ful -such -##ull -##ions -##ft -pre -came -##ro -miss -##ward -##een -##ild -##orn -##ue -even -under -these -##ord -come -part -fo -sc -##ated -rem -ind -##ness -bec -##per -may -much -think -##ious -way -mister -##ount -##led -let -ey -##ath -thought -##ect -back -spe -gl -##ady -bet -##ff -##ans -res -just -first -##ise -here -ho -own -des -##ried -ob -##ces -mo -pla -##ced -cont -##ary -##ered -went -himself -men -##me -##co -##int -might -##ily -seem -##ip -beg -##oth -##ouse -##ave -wat -comp -sw -through -##igh -away -em -get -head -don -imp -##llow -bel -##ened -without -##iz -life -most -pass -ear -make -cons -som -turn -##ac -##iv -shall -##ng -pres -acc -those -eyes -house -att -somet -jo -still -night -call -##red -op -##ew -##ens -wom -last -##ign -tell -##ause -nothing -##ark -sl -new -##ction -la -##iff -take -being -comm -many -##les -ser -word -inter -##atter -wr -##ons -ret -found -##are -##ase -while -##ory -##ix -saw -put -year -##iend -##ople -##dy -young -far -##aking -once -friend -people -##air -##ether -work -##ors -##ible -room -##ently -three -##ted -yet -##pect -##ever -count -same -father -right -child -cour -##ne -place -another -##ition -want -though -##less -##ince -king -##ian -happ -heart -end -gra -nat -face -##thing -love -took -dist -arm -##ace -inst -rep -##alk -##man -##ult -char -nor -te -ple -set -##cl -light -looked -##gg -missus -asked -mind -yes -supp -wa -ll -left -##cess -##ting -ac -##ub -##augh -##ents -##te -##ished -pers -##ways -things -sir -mom -##irl -near -##sw -##ations -sat -find -##der -##sel -##ia -world -##ower -##rew -##ject -##fect -give -cap -##av -why -mother -gu -gen -##ict -always -##so -##ling -answ -##pped -soon -##iver -##den -form -heard -war -thing -something -seemed -sub -god -door -girl -##ange -##ph -##und -wind -because -diff -mon -going -##ertain -##de -told -home -ke -##orm -got -##ither -el -##ret -##ock -sur -high -##ched -eng -gi -ma -ph -##ank -hands -moment -woman -follow -##land -dr -ve -vo -feel -##aut -##ning -poss -ev -##ical -##lt -##ple -boy -return -rest -reg -knew -each -inf -conf -oh -sil -##ner -kind -cle -##ush -hel -rel -##orr -tra -##ual -ass -wo -##ars -##cept -belie -bre -certain -against -water -##uck -exp -hard -prop -years -del -unt -quite -beh -side -##iness -##ined -appear -four -few -##ool -done -better -##ited -half -min -##ail -tre -ref -also -##cy -##oub -##vent -began -##most -##ows -ter -##ully -##act -enough -pur -##ised -dra -act -dear -having -seen -gener -lady -hum -##aster -##ared -##aken -##ash -##oke -hour -inde -##ps -##ying -cur -called -##ier -##ason -does -whole -##uth -morn -turned -pleas -##ial -gave -betw -##ense -occ -##aps -sou -course -##ib -ins -##gether -##ream -##ired -both -add -poor -col -read -bla -##ten -##selves -ste -among -between -keep -##ular -beaut -matter -care -morning -white -##ained -sure -name -##ged -toward -##ute -##owed -small -whom -par -exc -sk -pa -lar -felt -##ks -##ating -##osed -##bo -myself -open -six -es -##ru -herself -however -##xt -##ond -hund -stood -hundred -##plied -sent -inc -round -##tered -sim -##ife -##ates -##ters -days -almost -point -##eng -gre -eight -words -hor -fam -stand -land -##dden -##ung -mean -sun -##ird -##haps -##iet -perhaps -##ures -art -sle -ed -##iss -best -dark -sudden -hear -bu -replied -bar -voice -met -##ton -che -anything -till -underst -##ren -##its -until -black -##oud -looking -cried -##ned -suff -aw -##ins -prin -red -##amp -less -lay -next -law -##end -power -##vel -fact -##not -brought -##up -##ately -##enty -country -help -##als -sn -quest -wonder -use -vis -laugh -##med -##fully -together -air -unc -need -adv -wish -##ove -person -indeed -since -contin -##oney -gent -present -har -forg -full -run -rather -ra -##ok -cond -ide -sy -##eter -##rib -feet -fin -fall -##cei -five -mil -talk -##ater -bed -##ured -doct -##sh -reach -interest -lord -gone -##tle -death -consid -##fort -serv -wife -doub -adm -##ief -##pr -##ene -taken -others -hope -cannot -speak -themselves -along -ext -dire -bes -ent -master -##od -##ued -gentle -##lic -##uring -fire -large -pol -##ention -##oy -thus -##iled -sec -##ets -play -real -prom -##ments -##ield -bus -differ -##ison -thou -##ember -##ead -##ches -reason -thous -##iting -brother -##og -##akes -thousand -money -dep -answered -true -##unt -children -behind -sound -##ants -##ably -wood -##used -whose -ele -twenty -mer -rec -believe -inv -hon -book -dec -opp -hold -##ude -fair -passed -sch -##ided -##ities -##ength -##cious -sing -##cover -##ys -nature -##ering -alone -dead -##app -sever -cat -close -gold -start -hur -fur -##ont -doctor -ask -son -ground -pat -sea -strong -leave -compan -##illed -##ept -##cond -##ides -obs -fell -##body -doubt -sign -earth -remember -fear -given -##ances -therefore -ir -##oc -##ually -deep -##iful -##ressed -body -often -##du -pap -##ane -short -second -tem -##owing -##ceed -stre -fort -##mb -##ale -horse -mad -chr -order -const -ten -clear -##irc -fif -eas -##vers -bur -really -spir -##ceived -##read -##laim -ill -##osition -bra -understand -list -abs -spr -##az -pain -question -##anger -everything -aff -circ -wall -coming -##cted -##ching -##ready -above -already -least -expl -##esh -step -used -prince -itself -##ister -necess -case -around -tri -##hn -soul -##ized -suddenly -lad -evening -general -mag -num -##imes -known -quick -captain -##ific -whether -lear -within -live -times -expect -state -friends -swe -##gn -bring -##alked -sort -##men -women -table -meet -john -sum -returned -held -dri -big -walk -##eth -says -lea -##cer -obser -else -mid -during -##ident -mus -##ape -beautiful -ord -either -town -lost -sight -sleep -##vern -means -foot -cut -cal -kept -##ety -##gar -##ience -prof -fellow -dream -mist -past -became -pret -bad -##ages -stra -making -##ught -##ey -##ution -##lish -object -est -##ts -towards -dev -human -manner -strange -low -##old -##ott -##ient -##gs -##ines -ann -##here -ang -daugh -##oon -city -rich -sometimes -##aught -##airs -rose -dem -##ility -ready -ru -##cing -##itch -pal -road -ran -corn -thy -cold -##ians -##iew -##aring -acr -bit -won -didn -christ -squ -truth -##mer -##orrow -##nded -thir -soci -perfect -subject -##br -ship -mur -answer -pie -disc -arr -free -business -river -##arr -cast -family -##ountain -hus -jud -letter -##ony -hair -arms -##oken -##ote -become -sit -ut -ah -##ingly -##ouble -val -##aint -natur -possible -several -fine -##ffect -lead -express -sus -glad -arri -blood -##itting -##rie -quiet -able -##ert -idea -##iously -##itted -##bed -charac -##ah -begin -chur -story -eye -##band -##ative -consider -across -##li -except -equ -tou -##dle -cent -##isf -sto -partic -diffic -window -surpr -prov -grand -##use -flo -pen -ca -govern -wild -soft -cross -different -effect -##ased -def -longer -discover -followed -sold -thee -husband -fre -##antly -##by -exper -##ards -pub -cor -enter -win -suppose -##teen -respect -hast -##ices -proper -sal -comple -show -heav -##cul -happy -creat -taking -##essed -stay -requ -appro -spoke -daughter -wee -##une -##lly -##alth -anim -din -green -bird -mere -##ny -cra -gard -##empt -##ms -draw -reached -##ov -appeared -cover -##ley -exist -##ug -##ination -pretty -hea -opened -##anged -slow -##ize -imag -saying -##utes -common -tom -mor -coll -ap -occas -##ames -##ench -mountain -satisf -sense -succ -fav -##oura -##ising -rock -accord -scar -break -month -continued -stri -contr -church -tree -carried -spirit -##outh -cry -##ump -wanted -certainly -secret -command -##be -dest -##lled -move -lu -street -account -ought -prep -sens -##ats -##ement -purp -##ots -##riage -tear -esc -##owers -breat -dro -fan -character -##res -##ime -decl -##medi -stud -school -##itude -sla -quar -heaven -feeling -##son -sad -regard -bright -worth -##ney -dut -hot -hung -##laimed -##ric -disp -trans -delight -reco -##iling -rid -##eric -blue -hill -cloth -div -mot -disapp -##idence -produ -sweet -success -##ury -fast -english -ye -dress -port -neither -court -var -seven -##amed -##ains -princess -lived -view -##tern -##ushed -immedi -self -chief -hours -mill -exam -##wh -ce -tried -##day -entered -##ask -eat -carry -import -bear -danger -##ledge -offic -du -cause -none -forward -##umb -uncle -number -det -further -##gr -pay -##gin -added -eff -particular -deal -queen -prot -led -front -acqu -pray -##ored -happened -alth -fight -walked -later -##bit -joy -intell -##ster -comfort -prob -maj -##ening -public -bene -##uss -##ship -although -fig -direct -##ger -##iar -bat -mouth -##ality -mem -seems -sho -##side -##iam -sol -yourself -fore -##to -doing -living -desp -##oof -##pend -field -received -shad -bey -beyond -phil -line -visit -##inct -party -garden -hall -vi -##uff -americ -##ism -mat -gentleman -burn -##ested -laid -##irt -##raid -occasion -entire -age -sister -repe -##ously -prison -accom -whis -nearly -##ndon -trees -##cri -ge -eighteen -send -##wards -early -breath -lab -forth -##ior -try -##ones -med -terri -da -##ilt -princi -enem -thinking -##where -chance -minutes -cre -anx -mary -##anced -pict -wait -vill -afraid -crow -smile -##els -england -late -pleasure -bank -##bert -##ming -faith -super -news -fle -seeing -rob -neigh -##oint -##iment -hat -auth -bill -plain -trouble -silence -aunt -exclaimed -shar -coun -er -##gy -miles -##ply -glass -drew -neighb -mine -pract -heavy -standing -sevent -change -necessary -chap -##pping -purpose -concer -length -inqu -bell -##iest -natural -deter -##icked -hardly -top -caught -##ues -strength -##ives -##ounded -##aw -##ession -circum -fing -stopped -av -opin -##ament -priv -instead -##ences -##pper -grew -loved -island -knight -ago -inn -peace -watch -##inary -##iced -third -beauty -##uct -emb -conv -approach -determ -ver -##ler -##ining -##hood -##king -convers -straight -besides -position -thirty -##iter -den -attention -south -##ager -##ically -hist -##eless -comes -##aged -clock -force -plan -sitting -please -dog -whatever -##ugg -inform -hop -chair -lie -##form -build -bab -##ma -##ustom -je -girls -rom -french -struck -pull -wrong -knowledge -grace -##oom -scarce -resol -thoughts -attempt -fifty -rap -box -getting -company -crowd -##tained -##asp -##ils -class -die -drink -dinner -exact -enj -thick -saint -slight -save -maid -immediately -born -main -##ius -##crib -rev -remark -ast -##oose -north -##ists -tears -treat -##usion -meant -boys -fat -##ls -slowly -author -incl -lim -died -infl -fool -fish -sugg -post -accept -##ern -##elve -##ceive -##cient -stone -boat -paper -fright -conse -jour -ty -enc -quickly -youth -##ging -van -vict -##ified -belong -warm -##par -fix -imposs -forgot -beside -margar -leg -tone -camp -bound -makes -margaret -##mon -brown -##bs -##val -moon -##rupt -gather -lips -placed -circumst -food -filled -##iers -##ift -desire -##ee -london -distance -forest -floor -obl -##itions -spring -jack -viol -##ics -pock -following -remained -ger -grow -snow -government -ball -hors -##ented -##aded -broken -laughed -##ann -##oured -refle -safe -##ries -hen -##ively -##itten -depart -neck -easy -thank -##ope -##dom -##ham -knows -waiting -remain -hid -fresh -und -anne -dread -ash -##arch -result -proceed -gaz -##uc -shout -##dd -started -gall -complete -whisper -wide -horses -##isted -##cially -##ges -service -piece -##eness -forget -palace -guard -doll -##erv -talking -yell -##cial -lift -fun -grave -week -impossible -effort -##nding -imm -army -difficult -##let -##reme -stop -mess -narr -deg -incre -corner -society -path -weak -shut -hy -teach -cloud -##ency -##ivid -demand -##ended -##unk -nine -recogn -##cell -##fer -turning -sky -ori -##asure -##olog -opinion -sin -beat -single -dw -fly -lang -mass -outside -kiss -trust -occup -##uit -below -appearance -silver -bro -west -aftern -gun -couldn -ant -conn -showed -conversation -running -mention -nin -greater -music -##ases -##ises -##fection -mark -espe -##ror -##ories -une -spot -##ourage -peter -##ork -##anch -presence -winter -battle -hun -morrow -probably -##ection -clothes -fash -compl -wished -appar -married -arrived -suffic -eager -former -giving -pop -##pic -sand -usual -relig -simple -sym -##itation -neg -sor -moved -months -speaking -silent -cab -pun -expression -covered -afternoon -pet -##aped -occur -states -##rief -admir -##light -easily -##unity -phys -tender -beginning -duty -chapter -influ -##zz -instant -polit -enjoy -sick -tur -stream -figure -##ald -##ract -jew -vol -minute -pleasant -scarcely -conscious -terrible -kill -concl -pale -raised -##oop -fashion -thor -##yal -##ze -twelve -leaving -##ature -mel -fut -threw -flowers -attack -trying -##cked -sword -tall -marry -expected -according -##uel -forty -stick -temper -iron -oblig -train -rapid -ladies -learn -especially -##ipped -##orted -isn -##amb -evil -bread -learned -##orth -office -surprise -write -purs -written -killed -star -consequ -exh -places -condition -direction -##jo -entirely -action -##ding -##wise -##ibly -##mit -happiness -id -decided -golden -langu -##uous -note -bott -unless -books -fal -##aled -wonderful -shook -##ounds -##ech -##onel -perfectly -geor -sail -##ume -broad -closed -##atic -latter -steps -utter -merely -history -##gan -##osing -wise -wouldn -understood -middle -prev -enemy -sole -gar -simply -tast -shot -board -ring -sav -bow -##io -reply -##gress -changed -excell -grass -finally -##ital -suc -summer -sharp -fortune -experience -succeed -##uted -##retched -journey -observed -##osp -afterwards -##fast -##ec -earn -persons -lean -broke -deser -fit -mir -marriage -##acy -repres -bitter -lying -cou -rain -trave -listen -situ -##anc -##cience -##ected -origin -wit -##olute -##oses -calm -lives -escape -beneath -##ouses -clim -hes -ven -repeated -pocket -##lling -liber -oper -tail -whe -passion -dick -mut -becom -dar -allowed -famil -ow -##ivers -##mn -gate -spite -health -##str -##ission -ign -cand -habit -empl -strugg -firm -sorry -##under -madame -furn -sigh -future -##umed -nice -separ -presently -circumstances -##reh -##ples -sett -loud -worse -corr -wand -spread -letters -magn -disg -passing -pleased -darkness -##come -civ -##cies -tea -apart -disappear -important -##erable -legs -nation -delic -dressed -game -extra -walls -dry -virt -dim -rub -##idently -absolute -blind -discovered -exactly -dam -##oved -unh -cost -arch -employ -##velop -cous -beast -opport -hu -dropped -pack -subst -smiled -chee -resp -promise -##urs -host -creature -yellow -cla -notice -priest -knowing -heads -##esty -cook -shore -##ishment -individ -existence -determined -##lend -storm -seat -ears -##ours -mount -ur -##ina -conce -##anging -fixed -press -wear -major -ves -pros -##iod -motion -empt -leaves -bottom -arg -##iety -##sy -##enth -nobody -pick -##amber -##que -bag -intellig -stir -assist -dom -advant -tired -vent -##ument -##ashed -dan -##action -wrote -considered -houses -suit -cheer -clean -fla -lou -##path -##rel -cho -castle -frequ -drawing -luck -##pan -##cent -##idge -##ulty -ones -noble -splend -honor -paid -speech -estab -faint -##ires -individual -vall -##inite -##rodu -birds -allow -impress -jane -##my -song -touch -various -moder -believed -##istr -mod -somewh -period -shadow -affect -mal -extreme -judge -village -royal -somewhat -lower -##go -##zen -agree -remembered -aston -declared -##ays -ham -parts -colonel -##cil -breakfast -shr -tow -drop -surely -##ayed -##uded -scene -describ -comb -greatest -influence -custom -animal -##itary -sake -perform -soldiers -##iny -ancient -##ests -evidently -drawn -looks -ways -harm -destro -revol -##nes -##ator -##anted -picture -##ption -likely -laws -holding -difficulty -courage -inj -prevent -mort -troub -burst -angry -proud -spoken -##ron -##head -##ision -attend -watched -east -horr -pot -##na -agre -private -##bing -shop -jul -finished -anxious -##cise -fifteen -social -dism -wine -ideas -murder -mater -george -space -scr -inside -gentlemen -civil -##iently -formed -fol -fail -constit -lot -goes -thin -surf -servant -ourselves -fallen -fancy -henry -##ium -kne -advent -carriage -baby -tong -everybody -commun -##itive -ine -waited -grou -##gged -vain -##arked -##otion -favor -bal -speed -guess -narrow -windows -carefully -ice -jim -noise -meth -hero -william -pecul -promised -walking -forgotten -obliged -earnest -lose -imper -vessel -grad -glance -shoulder -animals -noticed -knock -peculiar -##cks -quietly -cup -ta -serious -tremb -generally -american -france -exer -##iction -property -stranger -shoulders -##osity -sympath -bod -flor -cow -dust -affection -conduct -##rop -##raph -higher -handsome -##iant -##nday -recover -wel -poet -pure -chamber -distinct -gh -mam -pier -grown -##uly -rough -##enced -desce -murm -##stem -personal -##osite -offered -##onsie -edge -built -whispered -skin -pieces -##arily -##itated -suffer -butter -faces -spent -##ables -interrupt -##how -##isters -develop -unk -fond -coat -touched -hol -jump -pi -##ingu -wasn -surprised -race -##af -##uce -##oted -dish -##ob -smo -educ -brave -fer -##over -weather -eld -wealth -system -german -false -##yn -##aked -cousin -##key -ben -prepared -##wood -fingers -surr -intellect -##ipp -##ites -opportunity -introdu -lovely -rag -crown -matters -companion -alar -innoc -ris -sorrow -##hern -quarter -band -lake -mix -meaning -mountains -memory -dor -##ring -##leep -##ctions -frank -finding -mach -charge -pan -gray -avo -inh -grat -greatly -dare -asleep -offer -dav -alto -##ounced -opposite -##ett -##unch -breast -bare -reading -altogether -writing -##pected -degree -exec -##fortun -hath -##eal -feelings -rail -##night -honour -fate -##attered -meeting -##ilar -alive -hide -cruel -blow -hurt -loss -defe -thrown -caused -##oned -seek -serve -profess -duke -bent -##kes -united -kingdom -situation -empty -wis -liked -swift -opening -servants -bush -suspic -freed -##ointed -surface -questions -##aff -curious -ess -accompan -christian -fill -wid -paris -satisfied -ere -pity -sides -##nds -equal -height -##osop -ordered -depend -grey -listened -##pet -##eer -rejo -capt -##ibility -despair -mart -happen -hurried -dollars -lin -loc -language -unf -yours -supposed -laughing -settled -talked -rode -perm -distingu -hurry -lifted -destroy -value -taste -##ores -##ners -square -vast -render -labor -rul -roof -stock -telling -papa -study -##ara -pin -rising -tal -sufficient -due -rise -##eds -forced -philosop -nose -sixty -egg -stat -fault -##asc -station -##ille -distur -regular -kitch -special -honest -building -shape -season -pride -smiling -orders -accompl -sacri -usually -watching -impro -##ushing -passage -valley -material -stairs -remarked -frightened -woods -wed -catch -mistress -directly -##he -prete -gloom -lines -justice -doesn -suggest -diam -growing -bless -##like -eur -##erence -ordinary -gathered -tit -threat -weeks -method -ale -ange -seized -search -discuss -ble -##ify -ignor -deb -##eping -supper -##uses -##ral -##bur -##chief -march -bold -putting -explain -##ik -nearer -york -affairs -contrary -##yr -##ishing -moving -streets -bir -##icated -wash -kitchen -bas -cit -##eder -imagine -prefer -possess -##rance -deliver -##imate -receive -monsie -por -twice -monsieur -##min -hole -sacrif -##hip -rever -reck -process -favour -dign -##aly -europe -unw -working -heat -##antic -cler -proved -##ades -pra -##lin -arth -lod -prove -##ception -pulled -pit -##ette -authority -jer -##ole -haven -movement -##tering -##usted -engaged -brothers -advantage -ber -##lished -arthur -stones -farm -difference -aside -hence -possession -observ -##etic -exte -soldier -hills -fortun -##uls -instance -ren -##iding -holy -color -flew -rate -wal -doors -trib -busy -lat -address -familiar -weight -played -##set -flower -aware -solemn -di -liter -test -indian -emper -distant -interesting -bull -thorough -ol -wore -worked -amb -explained -excellent -splendid -tongue -named -pard -shame -sam -franc -moments -trium -wil -##eav -myster -seated -spect -instantly -unkn -endeav -similar -naturally -triumph -##cr -measure -dogs -amount -elect -officer -##itable -uns -necessity -rank -##theless -brit -confidence -vir -##illa -##ulous -nevertheless -agreed -hearing -##ancho -takes -##ental -univers -fier -earl -keeping -drive -produced -aud -aug -cott -names -##con -disappeared -##agn -throw -president -gods -magic -prisoner -represent -unknown -##por -##ague -terror -contain -bri -smoke -support -detail -wicked -flat -yester -artic -dull -falling -##vey -worthy -liberty -##ulation -design -wants -evidence -companions -spirits -bid -##bers -coast -thro -mighty -particularly -witness -##ky -clever -sought -span -rare -officers -##lv -##ford -##zy -yesterday -powers -yield -organ -amaz -pointed -claim -content -possibly -cool -hom -shock -preser -terms -dwell -persu -drove -ceased -acquaint -occurred -lies -gri -gree -tor -otherwise -finger -emperor -bask -stars -knees -pair -paused -##ruction -failed -required -unhapp -diamond -aut -rat -ali -double -forms -gives -##race -bob -eliz -travel -carrying -gle -##iles -teeth -freedom -shown -forb -fruit -##umn -waters -hearts -ener -pill -##soci -example -upper -milk -affair -softly -moral -foreign -charles -excit -huge -##oes -possessed -cases -fet -##arther -counten -report -principle -inhab -proof -intended -satisfaction -shouted -farther -group -der -##asm -unfortun -unto -singing -arrange -religion -rocks -seventeen -james -works -buy -america -succeeded -rooms -events -leading -majesty -gently -dance -tele -##using -##ached -ga -##father -key -mentioned -seventy -ros -suffering -record -cabin -invent -diss -##ival -demanded -excitement -associ -progress -##angers -##road -curi -##erous -rule -vig -clearly -sheep -regret -neighbour -##iance -instinct -awful -advice -ban -fully -sen -##ena -##mend -papers -hidden -smooth -birth -hither -##bor -stuff -impat -calling -fourth -dreadful -ain -grief -pos -##pap -##ese -brill -powerful -presented -fairy -shoot -##ker -proc -joined -afford -##iments -shel -considerable -obey -interv -voices -interested -card -conqu -cred -needed -cord -pounds -wag -advanced -undert -resolved -##istic -excited -paul -bath -asse -conditions -pictures -##gen -shining -sunday -served -steam -police -sprang -jes -salt -fields -bore -##sie -indians -fierce -ride -nodd -aband -desired -edward -importance -information -hosp -memb -perceived -task -yard -##mond -crit -control -##ternal -fold -##ensive -sooner -merch -dur -suffered -absolutely -citiz -##isc -iss -##illy -completely -cart -rad -##ois -##hold -share -willing -devil -ships -imagination -superior -ep -##ancy -feather -env -anybody -stead -dawn -##asped -occupied -curiosity -fox -yo -##inal -##asant -##asons -##ams -myst -smith -##room -type -apparently -couple -extraord -welcome -bay -daily -modern -dying -##pir -feat -accident -##ension -countenance -##iot -##llen -##ortion -crime -mult -##get -reb -alas -refused -inquir -hate -whenever -thanks -regarded -suggested -##ulf -actually -pardon -size -##gment -cult -flesh -hotel -bodies -shell -##ishop -truly -betr -##with -##uced -rapidly -odd -##alous -club -horror -mile -san -sour -flight -crossed -slave -professor -oce -rushed -excl -worst -##ization -event -corre -science -##sed -##ainted -stepped -midst -foolish -##ha -##thus -brain -##acle -cottage -buck -brief -gain -tight -expressed -drag -equally -education -august -dun -nay -##com -##abel -tempt -picked -taught -elsie -lands -driven -political -resist -resem -instr -cell -lack -barb -dangerous -ken -keen -##hy -store -dozen -trade -points -ninet -sty -nob -##ability -crim -interp -relations -delighted -members -sisters -##ially -belief -asking -anger -park -wet -plat -relief -##ification -hunting -alex -cath -##aries -meat -obst -behold -mistake -inquired -remarkable -cop -nerv -##now -careful -cere -wounded -##ading -enemies -gradually -pu -##for -thorn -worn -interrupted -stup -silk -sixteen -##racted -flour -sylv -lone -nap -##kins -##asing -depth -dreams -##esp -kindly -##reg -##acity -eleven -invol -severe -##ocr -oak -wings -hal -##lace -begun -distr -##while -benef -moreover -exped -independ -artagn -buried -approached -artagnan -saved -sex -species -harry -physical -thunder -glory -creatures -newspap -sang -##kin -plenty -useful -shoes -hoped -frequently -##vert -saf -princip -##oe -chest -bran -##ication -roll -lover -famous -recoll -queer -grim -charming -aim -loose -nur -ath -indiff -consciousness -mamma -enthus -##ips -slept -fighting -enthusi -dig -##acc -##mas -##ids -stage -anyone -thrust -desper -drunk -lamp -##rant -bew -##stone -stern -evident -meanwhile -forgive -accepted -ocean -bot -tot -wondered -playing -hyp -hale -knife -closely -meas -proceeded -message -mour -##lied -sell -union -##ustomed -exceed -precious -century -unex -kat -##oops -##aming -everywhere -minds -extraordinary -arose -capital -entrance -recall -burning -magnific -##stand -plant -neighbor -assemb -lest -##orious -##uments -colle -virtue -retreat -##iana -capable -assured -##ograph -constant -governor -increased -horn -rush -removed -facts -absence -explan -ack -mud -fever -somebody -awa -admit -jealous -kissed -popular -hut -##pelled -grant -indign -friendship -sympathy -ug -##iable -thom -alice -level -objects -pressed -qual -begged -hind -highest -clouds -ghost -acknow -##oused -emp -wishes -##oking -strike -becomes -trembling -##gu -kindness -accordingly -throat -stretched -frag -confess -grandfather -wheel -bon -russ -deck -choose -eighty -##pher -miserable -contempt -##igned -military -basket -##nal -blank -listening -ahead -fro -##elt -larger -divine -##cription -tar -##iber -stories -reve -##izing -excuse -innocent -aid -et -hadn -remind -slaves -ara -##stairs -reward -manage -smell -throughout -waves -judgment -arrival -choice -unhappy -##astic -curr -advance -informed -impression -mysterious -acquaintance -ly -notes -comr -indust -ended -lights -nurse -bought -wra -##itor -spo -marked -scream -uneas -delicate -weary -##estic -join -prompt -experi -hungry -flying -pow -bridge -##aker -##umber -visible -understanding -crying -avoid -tis -stiff -sounds -bowed -caut -goods -david -stupid -##ancing -unable -bos -spark -amid -somewhere -infinite -ashamed -dignity -gay -vic -##ka -hollow -emotion -parents -admitted -hint -##pse -temple -comfortable -intelligence -virgin -bearing -##ldom -hers -##abeth -remains -contem -settle -immense -cher -weap -##ulated -lighted -baron -##gypt -elder -thoroughly -egypt -##ilst -##anges -friendly -rome -meal -intention -##achel -returning -billy -convin -ox -price -sup -rope -project -escaped -hopes -##ology -elizabeth -safety -wound -unus -##onscious -minister -horri -helped -secure -plainly -seldom -thinks -fellows -mood -pushed -exhib -##inging -fairly -pear -##irm -maiden -##mother -##ora -##orous -appears -cheeks -ease -skill -extent -practice -religious -becoming -features -pink -tied -##ential -whence -somehow -displ -greet -faithful -concerned -bishop -##isher -gro -eagerly -eating -waste -fem -bride -##ceiving -concerning -original -accustomed -recom -patient -arranged -##ointment -##ville -##iture -ate -wholly -older -mont -colour -provided -eggs -##ration -partly -prospect -##rees -fee -fled -laughter -branches -##right -whilst -slipped -violent -inhabit -sons -##bb -##yer -engage -uncom -deeply -substance -tale -tiny -yards -##icks -hastily -vague -hell -amuse -wolf -hans -##illing -supply -constantly -christmas -million -mental -wes -washing -log -corresp -nodded -correspond -rust -##oms -obtain -unconscious -prepar -roman -struggle -established -##ws -bee -hunt -##very -signs -uttered -constitution -cave -crew -##well -##reci -quant -##aud -image -merry -treated -efforts -lonely -nut -spare -glanced -portion -##ax -withd -prim -mead -feast -cliff -emer -proportion -consideration -haste -gaze -savage -resemb -##ffe -coach -conscience -mercy -exha -consent -##bits -##ators -##urd -adop -amongst -tower -hanging -rab -##oul -compar -##lessly -circle -entertain -brilliant -gained -row -troops -repro -dared -lion -ming -scot -joe -helen -bringing -anxiety -consequence -establish -fought -sout -##eem -sha -##chan -pred -scra -glim -victory -##field -obv -winds -restr -bol -##ode -##lect -##ourse -handed -indul -patience -jose -rolled -##udd -setting -profession -sylvia -##utions -feared -clo -boots -forehead -principles -rig -sink -##ture -purch -gazed -employed -murmured -##fl -##ashing -##ural -exch -trad -active -bottle -rage -##aval -treasure -invest -lux -protect -encoun -hang -sank -beard -##vere -loving -obtained -native -instru -waist -relation -discovery -nervous -tort -trail -distinguished -ach -pig -stared -print -misery -##cip -guil -warri -jumped -sac -swim -approaching -suspicion -iv -rum -managed -match -guilty -sar -wretched -bree -kings -theory -sher -shone -showing -##atherine -throne -robin -##utation -addressed -protest -admiration -troubled -ugly -flung -subs -relie -thousands -lan -od -wooden -member -sacrifice -apr -##aling -##urity -##acles -farmer -needs -condem -bade -reasons -swall -shadows -dancing -nineteen -style -manif -grate -appreh -desert -engine -terr -wisdom -jesus -pir -##ols -preced -interests -trial -nest -unl -phr -voy -illust -##iven -sear -worship -ay -adam -principal -##more -spend -stands -respons -hit -ner -whist -flash -kinds -require -described -angel -precise -proposed -produce -utterly -##ulse -novel -blame -credit -ja -pause -##ses -##osen -household -armed -follows -##upon -ninety -unb -block -##ivity -refuse -musc -sensible -national -cam -grie -##othing -reven -##plic -delightful -melancho -extremely -melancholy -lunch -enorm -spec -slightly -recognized -mystery -sacred -compreh -distress -useless -trif -philip -mounted -energy -explanation -pour -##xed -##atory -haw -chosen -everyone -##umbled -solid -correct -appreci -resumed -alexand -aven -ech -wing -intense -highly -schem -lucy -departure -agreeable -exercise -ward -dece -max -request -unexpected -joseph -cas -nort -##col -convent -leap -attract -refer -victim -lit -##ndy -forces -sport -drift -sighed -wedding -british -attitude -vision -pipe -southern -manners -flood -commission -guide -observe -concern -affected -##fall -##onstr -stret -poison -directed -medic -gest -younger -rested -confusion -parli -continue -absor -centre -##conom -horrible -bark -gown -bye -aloud -feed -careless -profound -cheerful -torn -orn -unp -sentence -mistaken -figures -accompanied -catherine -econom -##rated -##lot -##uing -shaking -atm -##ella -council -ends -knights -atmosp -shade -brow -refr -sentiment -subjects -duties -recovered -##oll -locked -composed -swept -reality -singular -assistance -louis -##vereign -wake -sovereign -loves -absurd -souls -riding -immediate -labour -connection -cheek -magnificent -prud -sugar -adj -dise -plans -leaning -surrounded -ling -proble -strict -##uding -##ailed -awake -praise -convinced -##zing -rele -frame -trot -breaking -curtain -stayed -craw -divided -inclined -previous -##edy -stair -sees -pron -glo -##board -complex -prayer -pierre -unfortunate -genius -banks -increase -sufficiently -revolution -aust -countess -sleeping -devoted -utmost -market -bosom -##ev -##alt -handker -admin -midnight -collect -senses -connected -ident -permitted -fil -gift -theat -brid -faced -desir -horiz -college -handkerchief -nurs -##isions -reader -polic -announced -nons -##ales -##ounce -fleet -ragged -##ped -##icate -coffe -parties -reser -nights -delay -sounded -merchant -cities -##zed -##owned -appointed -instit -striking -guns -astonishment -parliament -pil -page -##atched -##itate -##iform -plate -straw -thirst -##eld -asce -ruin -negro -inhabitants -rib -##inate -addition -argument -tad -##ox -##arf -##igg -scen -guests -wondering -acquainted -ax -nich -intent -##pless -treas -destroyed -coffee -##inent -tos -##lebr -sob -gets -assure -raise -valu -sharply -privile -prosper -alarm -machine -##gue -##can -##fied -appeal -deliber -reflect -mich -lock -afterward -guest -conclusion -risk -ignorant -pian -##bury -##ghter -##uries -soil -tend -commanded -practical -easter -fisher -##sey -arist -strain -beheld -quarters -degrees -nonsense -mc -offe -mechan -##keep -doubtless -violence -folk -bul -ral -##rate -##liness -##lected -##char -##ilities -contained -impr -leaf -celebr -reflection -concluded -district -scient -intend -scorn -steep -crack -muttered -darling -adventure -##win -andrew -chim -quality -polly -travell -universal -cig -dir -##atives -##uten -##cher -subm -##aces -##omp -##ulted -leis -##quis -emph -job -surround -lev -overt -retired -irr -extended -puzz -##bour -##kind -libr -spl -realized -causes -punishment -physic -leisure -##isp -shake -charm -belonged -bones -range -##iation -##mber -##pent -##nders -prec -proph -invited -detect -troubles -katy -june -tables -##non -hay -##icip -ages -anywhere -maybe -##ffin -properly -local -improve -atmosphere -rang -shed -compass -lieuten -leaned -firmly -aver -nations -##arse -##add -wept -chin -conven -uniform -julia -##amm -##unted -commer -track -obsc -sorts -difficulties -intellectual -introduced -bushes -rout -##mith -##iday -##elia -throwing -plun -humble -polite -numerous -movements -successful -protection -candle -separate -rendered -thomas -enormous -repub -sunsh -descended -unusual -fid -##ived -shows -##inder -robert -cattle -crept -deserted -astonished -arab -lap -##udes -nearest -ideal -entering -##cript -##standing -##uf -clos -ours -wherever -visited -calcul -base -gates -##sing -stamp -official -northern -##agger -##enger -painful -burned -republic -lately -ital -actions -final -startled -sensation -fatal -sore -##olic -appet -bloom -stronger -numbers -gratitude -female -western -sed -##lest -stoop -fourteen -madam -learning -forbid -citizens -rude -vers -compelled -abroad -##teenth -##arrass -reckon -francis -##house -##aine -changes -mankind -contempl -thread -stout -alt -perpet -cared -plants -weakness -apartment -accomplished -title -dart -habits -##rier -scattered -parl -lawyer -trip -hunger -prisoners -details -interview -bord -rays -##oqu -##rey -fears -##ems -rod -obed -jean -gloomy -empire -investig -grasp -##elle -richard -##atively -fanci -embarrass -slightest -climbed -coo -signal -painted -vanished -ralph -lieutenant -witch -##onies -isabel -##itty -likewise -contri -hearted -giant -helpless -actual -badly -effects -beaten -knocked -diana -mild -ray -perse -carpet -term -steady -conceal -supreme -kate -oil -rif -ruth -##inated -aspect -veget -esteem -grandmother -inex -hitherto -cultiv -source -gla -##ico -marilla -description -swear -attended -committed -countries -infer -trem -irish -squire -hastened -eaten -resolution -current -pist -pitch -##eenth -##bly -neat -##ulate -ens -ard -##icken -resc -guards -warning -spiritual -disposition -examined -results -consequently -widow -##port -abrupt -fifth -exception -occasionally -instrument -sunshine -poured -psy -##urance -anna -penet -spanish -voyage -staring -steel -check -created -ridic -pursued -shelter -qualities -psych -cries -honey -rachel -tent -flame -beloved -smaller -occasions -departed -hesitated -goodness -driving -complain -furniture -folks -execution -quantity -fulf -lamb -sof -intr -##umes -purple -##engers -leader -destruction -crowded -reflected -caval -##geon -counter -observation -alike -##acious -##ads -##unning -outl -string -desk -attached -chris -disposed -borne -safely -##uits -unm -disci -tells -continually -wandered -electric -bern -##lie -##bre -##cts -excess -unple -seriously -barn -fancied -gil -mos -wave -##enom -chain -liqu -##rily -parted -mode -committ -domestic -##vation -##otes -forever -abst -happens -venture -flag -eternal -mas -silly -##cere -##oring -##icing -intim -##quet -drank -incident -craft -operation -florence -valuable -dang -##born -##verty -##pended -trick -poverty -dearest -oliver -steadily -thornton -dil -ned -pant -beings -##ama -##olas -tristr -crist -##ewell -imagined -heavens -tristram -cy -cot -vac -##gl -##bow -##ken -##att -##cture -leather -mission -calls -separated -beasts -piano -library -swell -phenom -readily -annoy -enjoyed -##burg -walter -expedition -ju -mouse -nav -someone -seeking -eyed -glit -signific -meantime -lincol -disturbed -powder -lincoln -##bon -acts -##liest -cigar -hated -##inity -staff -marsh -##fulness -entreat -suspect -ventured -failure -philosopher -audience -pine -##ault -##uch -##ki -tones -attent -related -assumed -hundreds -washington -beach -carol -masters -dressing -repeat -tax -greek -lam -pace -##rous -##ipping -##iece -##aks -quiver -consist -farewell -insisted -humanity -heavily -sne -##fty -generous -chiefly -development -manifest -sult -granted -preach -strongly -islands -classes -dus -##pes -tomb -region -burden -summon -sadly -saddle -counsel -drinking -disturb -desperate -##hen -heap -##ola -brush -gracious -remaining -sailor -reproach -gas -pony -##worth -ton -abund -applied -families -echo -##eral -attrib -infant -dragged -attempted -anticip -describe -delivered -groan -italian -fund -##cast -##rem -heed -shrie -weeping -chuck -##eps -permit -holl -trace -propos -daughters -disappointed -##ground -probable -sincere -concealed -virginia -##isl -apple -headed -satur -reaching -services -legisl -circumstance -dwarf -noon -laun -colored -coloured -simon -suspected -fastened -abandoned -beth -anch -prize -shouldn -rights -dining -##ourable -disappointment -directions -histor -growth -acknowled -shy -weigh -chose -chann -strangers -lightly -doctr -estate -definite -wandering -dirty -##uge -gover -grav -lightning -decor -towns -freely -motive -exhausted -jen -pars -##inations -##itating -stuck -wearing -brook -perceive -flow -propri -opposition -quarrel -essential -platform -newspaper -enthusiasm -cunning -oath -pool -whit -##verse -leaped -glorious -emily -##umbling -mortal -exerc -zeal -##wick -sailed -convey -cru -series -monte -##arsh -cutting -views -washed -brig -achie -pound -outer -##umble -clay -turns -finds -trap -assert -fatig -bitterly -rules -gal -gif -##ea -##ipe -ench -cock -overcome -disag -marble -gazing -anti -mixed -critic -hed -hook -##dered -toby -inher -branch -desires -acted -falls -article -##net -##rim -##work -##inction -whirl -##ica -notion -##ails -slee -shortly -variety -dragon -indiffer -grateful -##iveness -nicholas -gor -non -##smith -knee -misfort -##ression -smoking -consult -endure -maintain -belongs -fetch -experienced -misfortune -thither -beating -timid -owner -slip -silently -deny -hiding -disgu -ambition -mingled -##keeper -apt -tick -boast -boats -flut -slain -objection -duch -pockets -volume -respond -frequent -disease -wre -##osure -contrib -writ -reputation -fearful -roses -##yard -##iel -trim -afric -metal -##ribly -recon -selfish -apparent -trembled -doroth -swiftly -wire -##uine -column -incon -invitation -formerly -fed -jun -ped -thine -##ares -italy -frog -pointing -poetry -suggestion -depths -bench -gw -jar -tin -shakes -sheet -mischief -lesson -treatment -fasc -##ums -perple -flock -fired -mock -represented -descend -threatened -horizon -unpleasant -duck -##jud -##atch -##asper -shield -consol -monarch -philosophy -bewild -closer -bernard -bert -toler -rear -rolling -tremend -reform -wishing -touching -fountain -##gest -inner -grin -coal -community -##asha -cats -millions -vessels -earlier -hospital -barbar -jan -oct -writer -uncertain -##etera -fury -enterpr -commander -encoura -majority -dismiss -pretended -ignorance -wagon -mac -nan -os -##ean -herbert -albert -intimate -marian -blessed -confused -contrast -##grat -hatred -articles -precisely -committee -ot -button -confident -thirteen -purposes -effic -conviction -furnished -benefit -harsh -pup -##mony -##otions -orch -##olate -boil -cro -smart -missed -warn -monster -male -reduced -beds -starting -luc -acquired -mirror -autumn -gesture -batter -daz -sway -vivid -heels -sust -deem -worry -thence -slender -eleg -johnson -visitor -motionless -neighbourhood -ceremony -maxim -sultan -cream -lament -fores -stable -anth -marve -apply -##coat -regiment -incred -cheap -absent -venge -discussion -renew -scientific -jap -pac -noted -deed -decision -ruff -roads -characters -tyr -teacher -stirred -yonder -feeble -scheme -problem -bun -date -eve -##ih -reign -##ifying -schol -disgrace -priests -##withstanding -fest -##isk -heir -notwithstanding -china -mans -marks -swung -consum -veh -opinions -francs -cristo -fant -lust -rh -stom -##olen -pole -fires -prey -insult -naked -torr -##ds -##case -lily -depri -portra -embra -victor -urged -pressure -absorbed -cetera -##anqu -##lla -impl -specul -compet -##acon -contrad -declare -owing -shooting -duchess -##lies -inch -##gging -phot -magistr -inev -multitude -recollect -idle -attain -shepher -unre -compared -interf -grounds -independent -stomach -ela -fitted -seas -prose -##akened -trunk -persist -possibility -##chester -element -necessarily -modest -distinctly -clerk -wrapped -medicine -tun -shiver -marvel -peasant -applic -surre -veil -deeper -earnestly -puff -porth -tide -##lict -##ino -tob -##itory -##iring -flatter -spell -##actory -cheese -considering -susan -experiment -gardens -secretary -batt -visitors -fashioned -statement -##ocratic -jaw -wives -asc -##inkled -norm -activity -tempor -central -tenderness -jewels -performed -clergy -rabbit -bud -rud -##lers -##leness -##we -##uish -fac -begins -insist -landl -landed -finish -arrive -provisions -sending -coward -tub -##nel -##gie -##eys -ingen -fred -##rand -agony -dischar -decide -respectable -wheels -japan -dale -pall -##upt -inches -sober -pepper -score -harr -collar -##ropri -withdra -##sters -##agg -repent -##iverse -stole -gravely -uneasy -##uary -##isy -##erson -flashed -prejud -happily -guessed -feels -irres -limbs -teaching -climb -fortunate -obeyed -puzzled -porthos -wol -##ael -##mes -##ata -##isible -inse -##rage -alter -remote -wrath -distinction -awk -solitary -buildings -moonlight -persuaded -scotland -revenge -eastern -ire -losing -champ -upstairs -aged -##ache -insp -raising -lively -prayers -cloak -fow -doom -altern -##apt -coin -childhood -heartily -merit -intelligent -drops -##rence -##raction -##peare -anyhow -##agon -advert -impulse -acting -merc -constru -belonging -pursuit -murderer -linen -saturday -vengeance -awkward -bund -chase -##ifully -marched -precip -passions -breed -cease -shouting -permission -containing -preserved -scenes -##rels -##emon -peril -peas -contents -capacity -surv -regarding -blew -napole -aly -damp -eag -moss -rot -##back -##inth -##anz -reasonable -elements -dutch -literature -abandon -abruptly -disagree -##off -passengers -supported -##ieth -buff -bedroom -county -gathering -russian -##lars -chall -##opes -using -wanting -norman -career -awoke -depos -strangely -stopping -preparing -slavery -pulling -ashes -germany -##guer -theatre -bor -##ike -##ule -anyway -folly -reference -prepare -wander -louise -developed -distinguish -inquiry -warriors -indifferent -cake -danced -fence -faded -jest -rav -tour -conc -agent -clut -whereas -##rov -contented -presents -princes -pains -crossing -examination -encourage -domin -favorite -revealed -engagement -breeze -bind -kitty -tip -ult -##cest -sale -##ational -brings -gran -attracted -confeder -breathing -messenger -dimin -ba -chairs -twil -lofty -daylight -marguer -rivers -attacked -plunged -appetite -marguerite -lane -rent -tap -##nat -##mo -shirt -external -india -repair -artist -musk -gladly -wilder -cryst -calmly -blessing -measures -##place -##ura -congress -concent -serge -restless -refuge -footst -gallant -gallop -gallery -funny -hunter -owe -folded -display -condemned -surrender -twilight -today -whither -receiving -emotions -expense -dreamed -songs -weapons -ornament -gifts -ireland -##oying -shir -##osha -##ooked -quoth -handle -plot -waved -neighbors -##ological -exceedingly -relieved -administr -dot -doth -pile -wilt -##inter -##irmed -weep -encl -trump -thanked -##ensible -reluct -keeps -arrow -floating -jeff -sailors -virtues -arrangement -industry -alyosha -cyr -tie -tang -vul -brut -stroke -contract -doorway -satisfy -scarlet -##whel -tales -amazement -catholic -napoleon -roused -tig -spain -##elly -genuine -resting -laying -garments -murmur -indignation -ecc -whip -stated -mex -manuf -lovers -longed -resour -natasha -henri -feathers -mourn -adopted -##ada -contact -cleared -parad -security -summ -centuries -error -increasing -masses -struggling -alarmed -physician -tremendous -cub -ded -sul -##pel -unlike -tranqu -overwhel -perfection -flushed -longing -exposed -inspect -patch -##oonful -examine -burnt -remarks -diamonds -exchange -tobacc -fain -##oot -##uously -haz -exert -roast -##cca -##inkle -commenced -lords -expecting -demonstr -penny -##ovitch -communication -ske -sancho -##eus -##ucted -##oration -solic -##ante -clothing -disord -joke -annie -center -existed -civilization -lodge -cardinal -emphas -rifle -bone -tore -velve -sauce -deer -##aph -perf -remem -remove -retorted -secured -matth -frightful -crimson -promptly -shakespeare -shepherd -bible -daring -reject -frost -conception -comment -accur -##ensity -snake -recent -driver -anxiously -jackson -consequences -intervals -comrades -pronounced -cotton -rival -thief -##itz -shaw -fog -distrib -asser -brand -ruined -romans -recognize -population -enjoyment -homes -nancy -wret -##lo -##iated -##just -##inous -##inger -##uting -##amation -restored -sunk -transl -bells -porter -meadow -convention -bess -les -mould -exquis -unse -ability -##ensions -income -preparations -jaco -danglars -dried -goss -##jean -tops -##usions -enl -spin -coffin -scre -indicated -moist -attach -avail -lucky -defend -issue -glimpse -##plicity -dorothy -bars -stu -##ria -##izes -purse -barg -finest -invisible -opposed -palm -funeral -comrade -obvious -bust -cree -cape -fame -loo -wick -##eman -outward -perman -charity -##oubted -realize -sensitive -mixture -porch -torment -positive -issued -weapon -refresh -bald -##iate -##vering -atl -tramp -glow -charged -behav -keeper -streams -detective -sofa -goose -mul -##of -heath -sew -##ulating -goat -pluck -lawn -minor -awakened -paint -corners -valjean -glasses -hideous -martin -reminded -manufact -garr -##tte -##aign -sept -allah -##outs -roots -quix -##inking -peep -advised -murdered -completed -undoubted -advantages -eldest -rejoined -favourite -exquisite -gig -##lon -##orable -##itudes -##idents -##usive -##olent -##ints -arts -temp -defence -succession -loudly -methods -conquer -betrayed -july -push -rushing -##low -##ider -sooth -soph -roar -quit -deadly -patri -solitude -despite -##oiselle -newspapers -indifference -privilege -eth -##ari -shaped -##sell -deceived -##pled -minded -steal -landing -conducted -hopeless -vanity -trusted -suitable -holes -wilson -amusement -tobacco -jos -tum -##ram -##lance -##esy -##aments -##rious -root -loaded -images -contro -emma -passes -retain -brandon -boxes -disgust -individuals -essence -copy -chimney -cure -kid -##lets -freder -bred -trained -indic -lav -provin -apolog -travelling -preserve -jealousy -uncomfort -mate -naught -congrat -##tha -##abled -apples -whereupon -youngest -confined -sunlight -signed -tomorrow -flee -determination -planet -fishing -excellency -sarah -cards -divers -pages -##iency -##inery -changing -appe -bloss -wherefore -natured -monk -eloqu -edith -accounts -inclination -forests -groups -depends -shells -betray -resistance -responded -rand -pope -lessons -stephen -musical -arrested -miller -paying -##cribed -including -politics -idiot -innocence -accomplish -debt -hospit -ivan -obedience -au -dre -om -umb -wip -##eous -##ients -shag -##lyle -ancest -thereby -copper -carlyle -throng -letting -resent -belt -fortnight -devotion -published -department -hesitation -rubbed -pillow -lantern -celebrated -dumb -kit -##nie -##alled -fathers -admired -characteristic -punish -angels -##vest -##any -inward -stim -deeds -dost -##emoiselle -arise -##igs -load -thoughtfully -subd -mademoiselle -prayed -resolute -literary -autom -citizen -inferior -matthew -jacob -jones -nic -patter -##oked -##ston -stolen -chances -clar -loyal -disple -appropri -conflict -##bridge -englishman -limit -packed -adventures -dwelling -rigid -pistol -disagreeable -gn -lil -sire -##eur -##rid -bees -##leton -##utor -##laced -##aping -mars -plains -##rog -parting -limited -legal -ashore -jews -prevented -ratt -beef -michael -holland -quixote -dad -sie -##omin -bears -alm -facing -boiled -grows -pond -flames -scare -relative -treach -easier -ladder -practically -hoping -romantic -campaign -sticks -mutual -corrid -tossed -ascertain -##ana -##ills -whoever -universe -blows -lasted -drowned -parish -standard -sunset -elsewhere -angrily -province -authorities -risen -teleph -involunt -nerves -rescue -gang -pip -til -##aves -##itively -bending -reception -orange -##olence -clasped -bond -grove -ade -interior -dram -##gram -immort -dishes -preferred -procession -yielded -##piration -landsc -criminal -ridiculous -net -##ai -##inence -##ulations -unjust -##law -adorn -wherein -scale -seconds -collected -graceful -warmth -##after -assembled -searching -average -dusk -bost -##rom -##omon -beck -bron -aren -offering -impressed -slid -wax -generation -holds -studied -denied -avoided -pretend -instruments -staircase -administration -etern -hull -ink -rein -cannon -boot -resign -counted -sixth -##okes -illness -transfer -err -captured -rosy -instructions -glaci -map -##gent -##enn -shru -##ctive -deaf -likes -miser -##eling -curiously -artific -furious -illum -apost -fanny -previously -operations -impatient -total -assembly -encounter -april -imprison -channel -vulgar -faster -hood -##eon -##oen -##icted -frown -seize -##arts -abb -brass -amused -overl -swar -jolly -mail -followers -teleg -##oman -shalt -stores -##racy -leon -brisk -##aging -namely -sweep -floated -exists -commonly -satisfactory -training -candid -faintly -erect -tribe -unfortunately -prudence -doctrine -boston -kar -revel -anton -prow -spur -##ago -##agues -crou -blown -supposing -regions -assume -extrem -balls -occupation -archite -combat -jimmy -machinery -railway -fortunes -acknowledged -velvet -oy -tid -##eg -##orter -neph -reported -richmond -corpor -americans -embro -convict -dwelt -latin -phrase -raggedy -summoned -dangers -pave -##aved -##och -##anim -##sequ -##entine -seats -conceived -brood -aroused -##udder -glen -helm -woe -proposition -continent -speaker -doubtful -##cules -pressing -balance -fortunately -neighborhood -surrounding -footsteps -cush -rings -ted -vest -##iour -##mans -##encies -rela -doc -appla -prem -##izz -regards -confession -questioned -profit -solomon -verse -wheat -horrid -railroad -recalled -recommend -fatigue -frederick -dine -shudder -felic -struct -thresh -spear -glowing -##anks -##rup -helping -runs -johnny -summit -seventh -deserve -spani -rarely -gilbert -wilderness -september -nephew -dug -##idable -upward -##ander -tram -downstairs -residence -##ubb -warned -propose -sentiments -motor -maintained -compliment -furnish -assistant -rebell -pirates -travellers -discipl -eagle -##natural -tumult -shaggy -telegraph -bou -dashed -##forth -shores -upright -##berry -##fts -moan -repose -monstr -phoen -assurance -advancing -breathe -inquire -demands -healthy -solemnly -swallowed -undoubtedly -rasc -##olet -sails -lik -monkey -terribly -pleasures -eighth -sensations -medical -lifting -delicious -imperfect -accompany -##force -intentions -##rane -toil -##iser -##anor -boiling -##enses -instances -vex -believing -humour -secretly -breathed -creation -killing -swords -injured -addressing -acknowledge -specim -portrait -bargain -creek -threshold -fi -gos -jury -link -##urer -analy -cha -nove -##ops -canv -hasn -missing -thoughtful -slope -submit -springs -schools -violently -performance -affectionate -spectacle -envy -levin -convenient -vine -woo -##iving -##uity -##cle -##fic -ing -wasted -##celot -butt -franz -brains -blast -impart -founded -survey -bloody -cheerfully -vigor -positively -curtains -launcelot -cose -mast -naut -##ya -##itute -liver -altered -brick -overhead -##athan -owned -supplied -incap -simplicity -raw -oppress -dropping -jewel -travelled -stirring -hurrying -persuade -displayed -territory -avenue -altar -africa -cu -jail -##hand -shif -rect -theirs -ponder -backward -harder -merr -catching -observing -penn -coron -requires -forgetting -injury -magician -strictly -complexion -prosperity -dense -hush -rice -##ek -##ray -##onic -##arn -##ita -saving -friday -prime -##keys -trous -fawn -agitation -monday -consented -guilt -subl -dreaming -anguish -ruins -heavenly -turkey -potat -searched -offended -compassion -persec -gossip -landscape -jason -kin -##lock -haired -section -dew -##ements -enqu -willingly -spots -prod -poets -planted -offence -lace -serpent -confessed -##ceptible -humor -recently -##ocracy -expectation -churches -transform -ridge -creep -embrace -sympt -neglected -politeness -tribes -establishment -purchase -bapt -##enched -lip -lif -disd -flies -partner -conspic -traged -decent -irre -painting -announce -leads -journal -bags -speedily -impatience -impatiently -swimming -engineer -precaut -wreck -dazz -normal -uncomfortable -cosette -sins -##fold -bea -recei -butler -conceive -##apes -grain -humil -sket -covering -motives -connect -punished -struggled -dishon -cruelty -testim -ranks -steamer -kenned -independence -frozen -alexander -anchor -gorge -bother -jam -mity -pad -stall -hercules -##umps -ambit -natives -checked -speaks -leaders -##ovna -division -protested -shoe -limits -venus -endeavour -##regular -comprehend -challen -wretch -kennedy -cents -kick -tu -##oard -##adel -alight -locks -slay -forming -confirmed -adding -irregular -staying -function -admiral -cellar -cautiously -obviously -hedge -peasants -bret -eust -wy -##rical -##atically -##orse -stove -prost -chill -##rig -spar -twisted -coarse -poem -blowing -scott -waving -keys -##grim -superst -impressions -interpret -sheriff -mitya -feder -lur -##aur -##course -##aration -bend -sailing -prol -##ula -knot -robe -amiable -flowing -beggar -judged -temperature -moderate -##etta -derived -association -greeted -cens -gasped -##burn -##fish -##esis -tools -shrew -rebe -sem -prair -##alla -##ilus -##rian -yourselves -nowhere -carr -consisted -nighting -foundation -elbow -highness -walks -riches -cavern -memories -bills -romance -boldly -withdrew -torture -magistrate -cough -cash -hare -ost -rows -wounds -##nic -##elf -##ober -##chio -seal -marius -##angle -justified -mob -certainty -planned -liberal -vigorous -reserve -pilgrim -obscure -achieve -mansion -prodig -mun -na -vap -vote -##greg -##don -##bar -##hin -chau -aboard -twas -muse -retire -##lands -arrest -slau -attempts -tenderly -couch -passionate -employment -flora -hurriedly -unwilling -bidding -mourning -encountered -perplex -inspired -rhy -rational -##gers -seed -exagger -feature -carved -induced -graham -armies -woke -steward -lads -providence -proceeding -decidedly -##burgh -arrangements -ceremon -halted -glittering -proposal -bewildered -enterprise -elegant -embraced -bundle -umbre -embroid -fare -##nut -##ta -##mie -##verted -sheer -brist -pled -mule -glancing -consists -intercourse -breth -##pson -##ibilities -smallest -needle -lordship -madness -judges -collection -rocky -gleam -tightly -temptation -caution -verses -application -rebecca -sorr -viz -##ban -sullen -lend -clam -##roll -seeming -accused -##ucky -raoul -corpse -morgan -destiny -neglect -bowl -welcom -remembr -brethren -lid -vice -##ique -##pers -##wart -##ded -##asy -##anny -##che -attained -allan -chatter -crushed -descent -endless -assum -##ashes -pleasing -colors -symb -doubts -clearing -peaceful -balloon -combined -fiercely -meadows -improvement -perpetual -photograph -prosec -bows -pav -##sis -thumb -conj -offices -smiles -backs -mostly -drum -curse -admire -remembering -respects -harmony -dismay -observations -energ -recollection -hypnot -referred -commerce -propriet -inconven -november -pul -##itious -shan -shaken -stagg -ree -brute -agric -discer -scold -swam -supplies -mothers -sna -entit -reasoning -estr -ceiling -hugh -sufferings -dissol -severely -cliffs -sinking -stretch -bart -foul -ram -##cow -haught -fors -##alf -##ieu -abode -clad -bonds -flu -mustn -remedy -interval -soup -##licted -eleph -penc -caes -crazy -secrets -terrified -prevail -picking -educated -filling -basin -russia -marvell -landlord -irresist -inspector -disorder -nautilus -dict -lent -nom -wages -##uz -##nde -reass -fruits -##allel -dod -stricken -foe -indef -incom -awe -syn -introduce -claims -inquiries -measured -preparation -gloves -policy -tendency -prophet -moscow -disguise -ec -lun -##awn -##asion -enable -cares -##ilet -##ounted -godd -hearth -advise -harmon -maggie -commands -melted -davy -britain -duration -restrain -tradition -luxury -sergeant -bite -##za -##inating -##ulars -upset -dislike -prel -laura -tread -bestowed -awaken -arrows -dresses -violet -medium -jacket -deserved -extrav -wealthy -awaiting -trousers -kent -lions -math -wont -##uated -thie -##chet -##raham -brows -clung -##assed -peer -fogg -impuls -formal -monot -existing -neighbours -enjoying -guarded -eagerness -materials -procure -awaited -december -treasures -parlour -renewed -tiger -extravag -diver -eh -lemon -##ingham -##imer -stub -abraham -closing -##cian -placing -plague -slipp -answering -largest -editor -polished -patron -entertained -superint -robbed -julius -representative -unconsciously -commercial -paradise -oven -tenth -##oons -haul -##cel -##esses -enabled -amusing -clover -slumber -serving -text -subtle -drake -prett -equality -hasty -recognition -savages -louder -comparison -axe -route -bough -dy -ell -jasper -mol -og -pag -##oh -##hov -##inge -##ington -whim -shine -sack -##usal -##lac -facul -##ques -##assad -perish -##udge -blush -answers -beautifully -plays -squir -sandy -admirable -released -encouraged -faculty -bits -dame -fern -lic -pud -pushing -wits -##lor -##dles -knelt -amend -contest -##mended -wrought -ministers -promises -election -inval -bigger -dispute -occasional -occupy -climbing -release -tremble -conspicuous -##aid -##hire -##ania -chat -conject -##arded -lee -discl -firing -scand -mayor -composition -##neys -errand -recognised -sickness -experiences -navig -inevitable -corridor -telephone -fiery -nightingale -cows -gab -games -ket -##loc -##mouth -##vous -##hib -##ican -stained -select -##antial -##ourt -amelia -childish -hearty -formidable -personally -doubted -provide -student -ringing -situated -faults -ambassad -reports -pearl -pearls -reserved -resources -gulf -nigh -vale -##tur -##iary -##yl -##lessness -behalf -visits -largely -transport -narrative -prevailed -contains -preceding -cavalry -spargo -cel -rash -sid -vib -wer -##master -sund -##rial -##acks -flowed -remorse -swinging -restore -dearly -discourse -peters -turtle -hostile -purely -unworth -cordial -adopt -printed -responsible -responsibility -muscular -economic -traveller -phenomena -gravity -october -pavement -likeness -disdain -federal -ellen -nails -psmith -rus -##ody -halt -stool -chamb -orph -unne -cove -crumb -scent -indisp -workmen -wars -drown -veins -paces -raft -milton -##inese -valentine -included -thankful -dolls -apartments -cooking -arguments -invention -horns -professional -stooped -echoed -dismissed -deemed -interfere -japanese -superintend -bunch -dix -oars -##row -##rating -##line -##ien -##atus -##adem -martha -poly -##oths -attorn -##arks -colon -skil -surprising -sweetness -rubbing -heroes -reverence -lodging -undertake -fragments -gigantic -remembrance -cakes -##xter -selected -chem -chop -bru -greatness -access -housekeeper -winding -appearances -edison -eleanor -fairies -estim -destined -rider -executed -tempted -screamed -sparkling -globe -forbidden -deprived -elabor -permanent -snapped -caesar -unnecess -dum -fitting -nun -waking -##foot -##zil -##arian -heel -shelf -defect -##iforn -ark -clust -grasped -perch -blade -droop -colours -sunny -slew -harness -californ -squee -wildly -convuls -learnt -condemn -leslie -structure -monstrous -prolong -mask -##itors -hem -thereupon -pencil -benev -reflections -guardian -candles -bodily -conquered -sanct -uncommon -apron -pouring -weighed -jaws -withdraw -discipline -eug -lots -##ji -##atisf -##omb -stif -suited -degr -nois -leagues -plough -scri -induce -betty -charlot -nora -tracks -patiently -earthly -effective -ville -villain -dreaded -paths -roughly -durham -spoil -amidst -stamped -contrived -patriot -breton -dub -gy -moses -pang -sant -wig -##ainder -##irs -##ilton -chicken -##riages -org -nod -##anda -trop -appointment -compare -passages -pleaded -persisted -##retion -raven -lowest -lowered -democratic -product -tailor -holiday -threatening -##ikov -briefly -drunken -resemblance -muscles -deliberately From c28b31209c43e847e93afb10ab852b7f5d05bb87 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 14:47:24 -0700 Subject: [PATCH 43/67] fixed the bug. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 9ac398e94871..16c75dec9f0d 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -101,12 +101,12 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self.encoder = EncDecCTCModel.from_config_dict(self._cfg.encoder) if "params" in self._cfg.decoder: - if not self._cfg.decoder.params.feat_in and hasattr(self.encoder, 'feat_out'): + if not self._cfg.decoder.params.feat_in and hasattr(self.encoder, '__feat_out'): self._cfg.decoder.params.feat_in = self.encoder.__feat_out else: raise ValueError("Parameter feat_in of the decoder's config is not set!") else: - if not self._cfg.decoder.feat_in and hasattr(self.encoder, 'feat_out'): + if not self._cfg.decoder.feat_in and hasattr(self.encoder, '__feat_out'): self._cfg.decoder.feat_in = self.encoder.__feat_out else: raise ValueError("param feat_in of the decoder's config is not set!") From 4283e977a8ca2cf288cc4fa9fe2a0a94dba8f70f Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 14:53:59 -0700 Subject: [PATCH 44/67] added logging of config. Signed-off-by: Vahid --- examples/asr/speech_to_text.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index 4620d0bde13d..e6ddfe630982 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -66,6 +66,7 @@ @hydra_runner(config_path="conf", config_name="config") def main(cfg): + logging.info(f'Hydra config: {cfg.pretty()}') trainer = pl.Trainer(**cfg.trainer) exp_manager(trainer, cfg.get("exp_manager", None)) asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) From 83ef7e1b0cf2914e2f3403cf7b0a511e4e74d2fd Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 14:57:39 -0700 Subject: [PATCH 45/67] moved swish to activations. Signed-off-by: Vahid --- nemo/collections/asr/parts/jasper.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/nemo/collections/asr/parts/jasper.py b/nemo/collections/asr/parts/jasper.py index 633f135ca4fd..a3923eb7edd8 100644 --- a/nemo/collections/asr/parts/jasper.py +++ b/nemo/collections/asr/parts/jasper.py @@ -19,11 +19,9 @@ import torch.nn as nn from torch import Tensor -jasper_activations = { - "hardtanh": nn.Hardtanh, - "relu": nn.ReLU, - "selu": nn.SELU, -} +from nemo.collections.asr.modules.activations import Swish + +jasper_activations = {"hardtanh": nn.Hardtanh, "relu": nn.ReLU, "selu": nn.SELU, "swish": Swish} def init_weights(m, mode='xavier_uniform'): @@ -257,11 +255,6 @@ def forward(self, x): return x * y -class Swish(nn.Module): - def forward(self, x): - return x * torch.sigmoid(x) - - class JasperBlock(nn.Module): __constants__ = ["conv_mask", "separable", "residual_mode", "res", "mconv"] @@ -561,7 +554,3 @@ def forward(self, input_: Tuple[List[Tensor], Optional[Tensor]]): return xs + [out], lens return [out], lens - - -# Register swish activation function -jasper_activations['swish'] = Swish From 2ccbb6aa552381bb485d8737b99a2ef5ae1d228b Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 15:01:40 -0700 Subject: [PATCH 46/67] fixed to_yaml. Signed-off-by: Vahid --- examples/asr/speech_to_text.py | 3 ++- examples/asr/speech_to_text_bpe.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/asr/speech_to_text.py b/examples/asr/speech_to_text.py index e6ddfe630982..fb164727f8f9 100644 --- a/examples/asr/speech_to_text.py +++ b/examples/asr/speech_to_text.py @@ -13,6 +13,7 @@ # limitations under the License. import pytorch_lightning as pl +from omegaconf import OmegaConf from nemo.collections.asr.models import EncDecCTCModel from nemo.core.config import hydra_runner @@ -66,7 +67,7 @@ @hydra_runner(config_path="conf", config_name="config") def main(cfg): - logging.info(f'Hydra config: {cfg.pretty()}') + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') trainer = pl.Trainer(**cfg.trainer) exp_manager(trainer, cfg.get("exp_manager", None)) asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer) diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index 98306d557210..c0552991a888 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -47,6 +47,7 @@ ``` """ import pytorch_lightning as pl +from omegaconf import OmegaConf from nemo.collections.asr.models.ctc_bpe_models import EncDecCTCModelBPE from nemo.core.config import hydra_runner @@ -56,7 +57,8 @@ @hydra_runner(config_path="experimental/configs/", config_name="config_bpe") def main(cfg): - logging.info(f'Hydra config: {cfg.pretty()}') + logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}') + print(OmegaConf.to_yaml(cfg)) trainer = pl.Trainer(**cfg.trainer) exp_manager(trainer, cfg.get("exp_manager", None)) From 4d04686874d2dfb86799c68bce992a9ffcf3ccd3 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 16:08:29 -0700 Subject: [PATCH 47/67] fixed the feat_in bug. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 16c75dec9f0d..6a754a449589 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -101,15 +101,18 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self.encoder = EncDecCTCModel.from_config_dict(self._cfg.encoder) if "params" in self._cfg.decoder: - if not self._cfg.decoder.params.feat_in and hasattr(self.encoder, '__feat_out'): + if "feat_in" not in self._cfg.decoder.params or ( + not self._cfg.decoder.params.feat_in and not hasattr(self.encoder, '__feat_out') + ): self._cfg.decoder.params.feat_in = self.encoder.__feat_out - else: - raise ValueError("Parameter feat_in of the decoder's config is not set!") else: - if not self._cfg.decoder.feat_in and hasattr(self.encoder, '__feat_out'): + if "feat_in" not in self._cfg.decoder or ( + not self._cfg.decoder.params.feat_in and not hasattr(self.encoder, '__feat_out') + ): self._cfg.decoder.feat_in = self.encoder.__feat_out - else: - raise ValueError("param feat_in of the decoder's config is not set!") + + if "feat_in" not in self._cfg.deocder or not self._cfg.decoder.feat_in: + raise ValueError("param feat_in of the decoder's config is not set!") self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) From 7d859fe5418e82a181ac31802685b405072ee45d Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 19:10:34 -0700 Subject: [PATCH 48/67] fixed the feat_in bug. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 6a754a449589..06b4449e37fe 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -107,7 +107,7 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self._cfg.decoder.params.feat_in = self.encoder.__feat_out else: if "feat_in" not in self._cfg.decoder or ( - not self._cfg.decoder.params.feat_in and not hasattr(self.encoder, '__feat_out') + not self._cfg.decoder.feat_in and not hasattr(self.encoder, '__feat_out') ): self._cfg.decoder.feat_in = self.encoder.__feat_out From a1647480f4c56bddd3e8f18fdadd1a736e0c5491 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 19:29:22 -0700 Subject: [PATCH 49/67] fixed the feat_in bug. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 8 ++++---- nemo/collections/asr/modules/conformer_encoder.py | 10 +++++----- nemo/collections/asr/modules/conv_asr.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 06b4449e37fe..6fc92546dc66 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -102,14 +102,14 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): if "params" in self._cfg.decoder: if "feat_in" not in self._cfg.decoder.params or ( - not self._cfg.decoder.params.feat_in and not hasattr(self.encoder, '__feat_out') + not self._cfg.decoder.params.feat_in and not hasattr(self.encoder, '_feat_out') ): - self._cfg.decoder.params.feat_in = self.encoder.__feat_out + self._cfg.decoder.params.feat_in = self.encoder._feat_out else: if "feat_in" not in self._cfg.decoder or ( - not self._cfg.decoder.feat_in and not hasattr(self.encoder, '__feat_out') + not self._cfg.decoder.feat_in and not hasattr(self.encoder, '_feat_out') ): - self._cfg.decoder.feat_in = self.encoder.__feat_out + self._cfg.decoder.feat_in = self.encoder._feat_out if "feat_in" not in self._cfg.deocder or not self._cfg.decoder.feat_in: raise ValueError("param feat_in of the decoder's config is not set!") diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index df65f72d2c68..acff0d0e5ba8 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -80,7 +80,7 @@ def input_example(self): Returns: A tuple of input examples. """ - input_example = torch.randn(16, self.__feat_in, 256).to(next(self.parameters()).device) + input_example = torch.randn(16, self._feat_in, 256).to(next(self.parameters()).device) return tuple([input_example]) @property @@ -143,9 +143,9 @@ def __init__( conv_channels=subsampling_conv_channels, activation=nn.ReLU(), ) - self.__feat_out = d_model + self._feat_out = d_model else: - self.__feat_out = d_model + self._feat_out = d_model self.pre_encode = nn.Linear(feat_in, d_model) if self_attention_model == "rel_pos": @@ -174,10 +174,10 @@ def __init__( if feat_out > 0 and feat_out != self.output_dim: self.out_proj = nn.Linear(self.feat_out, feat_out) - self.__feat_out = feat_out + self._feat_out = feat_out else: self.out_proj = None - self.__feat_out = d_model + self._feat_out = d_model @typecheck() def forward(self, audio_signal, length): diff --git a/nemo/collections/asr/modules/conv_asr.py b/nemo/collections/asr/modules/conv_asr.py index 17b1db1e8c30..45073fbbc4de 100644 --- a/nemo/collections/asr/modules/conv_asr.py +++ b/nemo/collections/asr/modules/conv_asr.py @@ -63,7 +63,7 @@ def input_example(self): Returns: A tuple of input examples. """ - input_example = torch.randn(16, self.__feat_in, 256).to(next(self.parameters()).device) + input_example = torch.randn(16, self._feat_in, 256).to(next(self.parameters()).device) return tuple([input_example]) @property @@ -124,7 +124,7 @@ def __init__( activation = jasper_activations[activation]() feat_in = feat_in * frame_splicing - self.__feat_in = feat_in + self._feat_in = feat_in residual_panes = [] encoder_layers = [] @@ -174,7 +174,7 @@ def __init__( ) feat_in = lcfg['filters'] - self.__feat_out = feat_in + self._feat_out = feat_in self.encoder = torch.nn.Sequential(*encoder_layers) self.apply(lambda x: init_weights(x, mode=init_mode)) From 8c92c5425b82a38292512c50a8fc6bf62bbc7c6e Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 19:33:02 -0700 Subject: [PATCH 50/67] fixed the feat_in bug. Signed-off-by: Vahid --- examples/asr/speech_to_text_bpe.py | 10 +++++----- nemo/collections/asr/models/ctc_models.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index c0552991a888..c2735212a328 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -68,13 +68,13 @@ def main(cfg): if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: gpu = 1 if cfg.trainer.gpus != 0 else 0 - trainer = pl.Trainer( + test_trainer = pl.Trainer( gpus=gpu, - precision=cfg.trainer.precision, - amp_level=cfg.trainer.amp_level, - amp_backend=cfg.trainer.amp_backend, + precision=trainer.precision, + amp_level=trainer.amp_level, + amp_backend=trainer.amp_backend, ) - if asr_model.prepare_test(trainer): + if asr_model.prepare_test(test_trainer): trainer.test(asr_model) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 6fc92546dc66..a04ce4714403 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -111,7 +111,7 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): ): self._cfg.decoder.feat_in = self.encoder._feat_out - if "feat_in" not in self._cfg.deocder or not self._cfg.decoder.feat_in: + if "feat_in" not in self._cfg.decoder or not self._cfg.decoder.feat_in: raise ValueError("param feat_in of the decoder's config is not set!") self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) From d929c1df6fd0cfd2137e23e1627154d6e2084ce2 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 19:56:17 -0700 Subject: [PATCH 51/67] fixed the feat_in bug. Signed-off-by: Vahid --- examples/asr/speech_to_text_bpe.py | 5 +---- nemo/collections/asr/models/ctc_models.py | 11 ++++++----- nemo/collections/asr/modules/multi_head_attention.py | 6 +++++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/examples/asr/speech_to_text_bpe.py b/examples/asr/speech_to_text_bpe.py index c2735212a328..400ffe3ef479 100644 --- a/examples/asr/speech_to_text_bpe.py +++ b/examples/asr/speech_to_text_bpe.py @@ -69,10 +69,7 @@ def main(cfg): if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.manifest_filepath is not None: gpu = 1 if cfg.trainer.gpus != 0 else 0 test_trainer = pl.Trainer( - gpus=gpu, - precision=trainer.precision, - amp_level=trainer.amp_level, - amp_backend=trainer.amp_backend, + gpus=gpu, precision=trainer.precision, amp_level=trainer.amp_level, amp_backend=trainer.amp_backend, ) if asr_model.prepare_test(test_trainer): trainer.test(asr_model) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index a04ce4714403..3e611fa6440a 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -102,17 +102,18 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): if "params" in self._cfg.decoder: if "feat_in" not in self._cfg.decoder.params or ( - not self._cfg.decoder.params.feat_in and not hasattr(self.encoder, '_feat_out') + not self._cfg.decoder.params.feat_in and hasattr(self.encoder, '_feat_out') ): self._cfg.decoder.params.feat_in = self.encoder._feat_out + if "feat_in" not in self._cfg.decoder.params or not self._cfg.decoder.prams.feat_in: + raise ValueError("param feat_in of the decoder's config is not set!") else: if "feat_in" not in self._cfg.decoder or ( - not self._cfg.decoder.feat_in and not hasattr(self.encoder, '_feat_out') + not self._cfg.decoder.feat_in and hasattr(self.encoder, '_feat_out') ): self._cfg.decoder.feat_in = self.encoder._feat_out - - if "feat_in" not in self._cfg.decoder or not self._cfg.decoder.feat_in: - raise ValueError("param feat_in of the decoder's config is not set!") + if "feat_in" not in self._cfg.decoder or not self._cfg.decoder.feat_in: + raise ValueError("param feat_in of the decoder's config is not set!") self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/modules/multi_head_attention.py index 9d1d5bbbdbd1..415652a36763 100644 --- a/nemo/collections/asr/modules/multi_head_attention.py +++ b/nemo/collections/asr/modules/multi_head_attention.py @@ -100,7 +100,11 @@ def forward_attention(self, value, scores, mask): n_batch = value.size(0) if mask is not None: mask = mask.unsqueeze(1).eq(0) # (batch, 1, time1, time2) - min_value = float(np.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min) + if scores.dtype == torch.float16: + dtype = np.float16 + else: + dtype = np.float32 + min_value = np.finfo(dtype).min scores = scores.masked_fill(mask, min_value) self.attn = torch.softmax(scores, dim=-1).masked_fill(mask, 0.0) # (batch, head, time1, time2) else: From f336ed9b8d16e8b206995b9a1f97c76c2d4dd976 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 20:10:05 -0700 Subject: [PATCH 52/67] fixed the feat_in bug. Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 3e611fa6440a..19cc9962c61a 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -105,7 +105,7 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): not self._cfg.decoder.params.feat_in and hasattr(self.encoder, '_feat_out') ): self._cfg.decoder.params.feat_in = self.encoder._feat_out - if "feat_in" not in self._cfg.decoder.params or not self._cfg.decoder.prams.feat_in: + if "feat_in" not in self._cfg.decoder.params or not self._cfg.decoder.params.feat_in: raise ValueError("param feat_in of the decoder's config is not set!") else: if "feat_in" not in self._cfg.decoder or ( From dbfe71e72fd43734d980505a8c125cbaa012b81e Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 22:34:06 -0700 Subject: [PATCH 53/67] fixed the feat_in bug. Signed-off-by: Vahid --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a096a2f1e5a2..643c3ffa987c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -227,7 +227,7 @@ pipeline { stage('L2: Speech to Text WPE - Conformer') { steps { sh 'python examples/asr/speech_to_text_bpe.py \ - --config-path="experimental/conformer/confs" --config-name="conformer_subword" \ + --config-path="experimental/conformer" --config-name="conformer_bpe" \ model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ model.tokenizer.dir="examples/asr/experimental/wpe_vocabs/1k/" \ From 5a0cf54e098ddc0e510aabe0019a854c68c9cc7b Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 30 Oct 2020 23:22:14 -0700 Subject: [PATCH 54/67] fixed the feat_in bug. Signed-off-by: Vahid --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 643c3ffa987c..92b0a09dffcc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -209,7 +209,7 @@ pipeline { } } - stage('L2: Speech to Text WPE') { + stage('L2: Speech to Text WPE - CitriNet') { steps { sh 'python examples/asr/speech_to_text_bpe.py \ --config-path="experimental/citrinet/" --config-name="config_bpe" \ @@ -230,7 +230,7 @@ pipeline { --config-path="experimental/conformer" --config-name="conformer_bpe" \ model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ - model.tokenizer.dir="examples/asr/experimental/wpe_vocabs/1k/" \ + model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ model.tokenizer.type="wpe" \ trainer.gpus=[1] \ +trainer.fast_dev_run=True \ From 6791136e671db999c8856a585e2be4f05f2133d8 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 31 Oct 2020 01:11:38 -0700 Subject: [PATCH 55/67] fixed the feat_in bug. Signed-off-by: Vahid --- Jenkinsfile | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 92b0a09dffcc..bb518c6923c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -224,20 +224,20 @@ pipeline { } } - stage('L2: Speech to Text WPE - Conformer') { - steps { - sh 'python examples/asr/speech_to_text_bpe.py \ - --config-path="experimental/conformer" --config-name="conformer_bpe" \ - model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ - model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ - model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ - model.tokenizer.type="wpe" \ - trainer.gpus=[1] \ - +trainer.fast_dev_run=True \ - exp_manager.exp_dir=examples/asr/speech_to_text_wpe_conformer_results' - sh 'rm -rf examples/asr/speech_to_text_wpe_conformer_results' - } - } +// stage('L2: Speech to Text WPE - Conformer') { +// steps { +// sh 'python examples/asr/speech_to_text_bpe.py \ +// --config-path="experimental/conformer" --config-name="conformer_bpe" \ +// model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ +// model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ +// model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ +// model.tokenizer.type="wpe" \ +// trainer.gpus=[1] \ +// +trainer.fast_dev_run=True \ +// exp_manager.exp_dir=examples/asr/speech_to_text_wpe_conformer_results' +// sh 'rm -rf examples/asr/speech_to_text_wpe_conformer_results' +// } +// } } } From dfeafd1d204306640429bfef4aa11ed300c92ccc Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 1 Nov 2020 19:55:07 -0800 Subject: [PATCH 56/67] fixed the feat_in bug. Signed-off-by: Vahid --- Jenkinsfile | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bb518c6923c2..92b0a09dffcc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -224,20 +224,20 @@ pipeline { } } -// stage('L2: Speech to Text WPE - Conformer') { -// steps { -// sh 'python examples/asr/speech_to_text_bpe.py \ -// --config-path="experimental/conformer" --config-name="conformer_bpe" \ -// model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ -// model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ -// model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ -// model.tokenizer.type="wpe" \ -// trainer.gpus=[1] \ -// +trainer.fast_dev_run=True \ -// exp_manager.exp_dir=examples/asr/speech_to_text_wpe_conformer_results' -// sh 'rm -rf examples/asr/speech_to_text_wpe_conformer_results' -// } -// } + stage('L2: Speech to Text WPE - Conformer') { + steps { + sh 'python examples/asr/speech_to_text_bpe.py \ + --config-path="experimental/conformer" --config-name="conformer_bpe" \ + model.train_ds.manifest_filepath=/home/TestData/an4_dataset/an4_train.json \ + model.validation_ds.manifest_filepath=/home/TestData/an4_dataset/an4_val.json \ + model.tokenizer.dir="/home/TestData/asr_tokenizers/an4_wpe_128/" \ + model.tokenizer.type="wpe" \ + trainer.gpus=[1] \ + +trainer.fast_dev_run=True \ + exp_manager.exp_dir=examples/asr/speech_to_text_wpe_conformer_results' + sh 'rm -rf examples/asr/speech_to_text_wpe_conformer_results' + } + } } } From a74a327fae54aeee1db7b12d4b00fcd0e3722ae1 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 00:30:51 -0800 Subject: [PATCH 57/67] fixed log_predciction. Signed-off-by: Vahid --- examples/asr/experimental/conformer/conformer_bpe.yaml | 2 +- examples/asr/experimental/conformer/conformer_char.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/asr/experimental/conformer/conformer_bpe.yaml b/examples/asr/experimental/conformer/conformer_bpe.yaml index 7da7ab8e7e29..abb4a7d8616c 100644 --- a/examples/asr/experimental/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/conformer/conformer_bpe.yaml @@ -2,7 +2,7 @@ name: &name "Conformer-BPE" model: sample_rate: &sample_rate 16000 - log_predictions: true + log_prediction: true load_weights_from_checkpoint: null ctc_reduction: 'mean_batch' diff --git a/examples/asr/experimental/conformer/conformer_char.yaml b/examples/asr/experimental/conformer/conformer_char.yaml index ef7de1ecc7d4..32f94857e016 100644 --- a/examples/asr/experimental/conformer/conformer_char.yaml +++ b/examples/asr/experimental/conformer/conformer_char.yaml @@ -4,7 +4,7 @@ model: sample_rate: &sample_rate 16000 labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"] - log_predictions: true + log_prediction: true load_weights_from_checkpoint: null ctc_reduction: 'mean_batch' From 6600de9341ad5d2bff64e8c8d463a1c757bc37b5 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 11:48:39 -0800 Subject: [PATCH 58/67] fixed log bug. Signed-off-by: Vahid --- .../experimental/conformer/conformer_bpe.yaml | 11 ++--------- .../experimental/conformer/conformer_char.yaml | 17 +++++------------ nemo/collections/asr/modules/__init__.py | 8 -------- .../asr/modules/conformer_encoder.py | 4 ++-- .../asr/{modules => parts}/activations.py | 0 .../asr/{modules => parts}/conformer_modules.py | 2 +- nemo/collections/asr/parts/jasper.py | 2 +- .../{modules => parts}/multi_head_attention.py | 0 8 files changed, 11 insertions(+), 33 deletions(-) rename nemo/collections/asr/{modules => parts}/activations.py (100%) rename nemo/collections/asr/{modules => parts}/conformer_modules.py (97%) rename nemo/collections/asr/{modules => parts}/multi_head_attention.py (100%) diff --git a/examples/asr/experimental/conformer/conformer_bpe.yaml b/examples/asr/experimental/conformer/conformer_bpe.yaml index abb4a7d8616c..635ace3ba45c 100644 --- a/examples/asr/experimental/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/conformer/conformer_bpe.yaml @@ -129,14 +129,14 @@ trainer: max_steps: null # computed at runtime if not set val_check_interval: 1 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations distributed_backend: ddp - accumulate_grad_batches: 2 + accumulate_grad_batches: 1 gradient_clip_val: 0.0 amp_level: O0 # O1/O2 for mixed precision precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. log_every_n_steps: 10 # Interval of logging. resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it - check_val_every_n_epoch: 10 # number of evaluations on validation every n epochs + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs sync_batchnorm: true checkpoint_callback: false # Provided by exp_manager logger: false # Provided by exp_manager @@ -153,10 +153,3 @@ exp_manager: project: null resume_if_exists: false resume_ignore_no_checkpoint: false - -hydra: - run: - dir: . - job_logging: - root: - handlers: null diff --git a/examples/asr/experimental/conformer/conformer_char.yaml b/examples/asr/experimental/conformer/conformer_char.yaml index 32f94857e016..a8f2d368e6a6 100644 --- a/examples/asr/experimental/conformer/conformer_char.yaml +++ b/examples/asr/experimental/conformer/conformer_char.yaml @@ -12,7 +12,7 @@ model: manifest_filepath: ??? labels: *labels sample_rate: 16000 - batch_size: 16 + batch_size: 32 trim_silence: false max_duration: 16.7 min_duration: 0.1 @@ -26,7 +26,7 @@ model: manifest_filepath: ??? labels: *labels sample_rate: 16000 - batch_size: 16 + batch_size: 32 shuffle: false num_workers: 4 pin_memory: true @@ -35,7 +35,7 @@ model: manifest_filepath: null labels: *labels sample_rate: 16000 - batch_size: 16 + batch_size: 32 shuffle: false num_workers: 4 pin_memory: true @@ -126,14 +126,14 @@ trainer: max_steps: null # computed at runtime if not set val_check_interval: 1 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations distributed_backend: ddp - accumulate_grad_batches: 2 + accumulate_grad_batches: 1 gradient_clip_val: 0.0 amp_level: O0 # O1/O2 for mixed precision precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP. log_every_n_steps: 10 # Interval of logging. resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it - check_val_every_n_epoch: 10 # number of evaluations on validation every n epochs + check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs sync_batchnorm: true checkpoint_callback: false # Provided by exp_manager logger: false # Provided by exp_manager @@ -150,10 +150,3 @@ exp_manager: project: null resume_if_exists: false resume_ignore_no_checkpoint: false - -hydra: - run: - dir: . - job_logging: - root: - handlers: null diff --git a/nemo/collections/asr/modules/__init__.py b/nemo/collections/asr/modules/__init__.py index 9abb025a6021..a054bbb3bdab 100644 --- a/nemo/collections/asr/modules/__init__.py +++ b/nemo/collections/asr/modules/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nemo.collections.asr.modules.activations import Swish from nemo.collections.asr.modules.audio_preprocessing import ( AudioToMelSpectrogramPreprocessor, AudioToMFCCPreprocessor, @@ -21,7 +20,6 @@ ) from nemo.collections.asr.modules.beam_search_decoder import BeamSearchDecoderWithLM from nemo.collections.asr.modules.conformer_encoder import ConformerEncoder -from nemo.collections.asr.modules.conformer_modules import ConformerConvolution, ConformerFeedForward from nemo.collections.asr.modules.conv_asr import ( ConvASRDecoder, ConvASRDecoderClassification, @@ -29,10 +27,4 @@ SpeakerDecoder, ) from nemo.collections.asr.modules.lstm_decoder import LSTMDecoder -from nemo.collections.asr.modules.multi_head_attention import ( - MultiHeadAttention, - PositionalEncoding, - RelPositionalEncoding, - RelPositionMultiHeadAttention, -) from nemo.collections.asr.modules.subsampling import ConvSubsampling diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index acff0d0e5ba8..48a7e802891b 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -19,8 +19,8 @@ import torch.nn as nn from torch.nn.modules import LayerNorm -from nemo.collections.asr.modules.conformer_modules import ConformerConvolution, ConformerFeedForward -from nemo.collections.asr.modules.multi_head_attention import ( +from nemo.collections.asr.parts.conformer_modules import ConformerConvolution, ConformerFeedForward +from nemo.collections.asr.parts.multi_head_attention import ( MultiHeadAttention, PositionalEncoding, RelPositionalEncoding, diff --git a/nemo/collections/asr/modules/activations.py b/nemo/collections/asr/parts/activations.py similarity index 100% rename from nemo/collections/asr/modules/activations.py rename to nemo/collections/asr/parts/activations.py diff --git a/nemo/collections/asr/modules/conformer_modules.py b/nemo/collections/asr/parts/conformer_modules.py similarity index 97% rename from nemo/collections/asr/modules/conformer_modules.py rename to nemo/collections/asr/parts/conformer_modules.py index 5662ddbfe439..a459dcf58d2a 100644 --- a/nemo/collections/asr/modules/conformer_modules.py +++ b/nemo/collections/asr/parts/conformer_modules.py @@ -15,7 +15,7 @@ import torch.nn as nn -from nemo.collections.asr.modules.activations import Swish +from nemo.collections.asr.parts.activations import Swish __all__ = ['ConformerConvolution', 'ConformerFeedForward'] diff --git a/nemo/collections/asr/parts/jasper.py b/nemo/collections/asr/parts/jasper.py index a3923eb7edd8..c2f4cdd7ccbc 100644 --- a/nemo/collections/asr/parts/jasper.py +++ b/nemo/collections/asr/parts/jasper.py @@ -19,7 +19,7 @@ import torch.nn as nn from torch import Tensor -from nemo.collections.asr.modules.activations import Swish +from nemo.collections.asr.parts.activations import Swish jasper_activations = {"hardtanh": nn.Hardtanh, "relu": nn.ReLU, "selu": nn.SELU, "swish": Swish} diff --git a/nemo/collections/asr/modules/multi_head_attention.py b/nemo/collections/asr/parts/multi_head_attention.py similarity index 100% rename from nemo/collections/asr/modules/multi_head_attention.py rename to nemo/collections/asr/parts/multi_head_attention.py From 4b314db38d6949594556f846f751fee942957398 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 11:54:04 -0800 Subject: [PATCH 59/67] fixed style. Signed-off-by: Vahid --- nemo/collections/asr/modules/conformer_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 48a7e802891b..4ad7960002d1 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -19,6 +19,7 @@ import torch.nn as nn from torch.nn.modules import LayerNorm +from nemo.collections.asr.modules.subsampling import ConvSubsampling from nemo.collections.asr.parts.conformer_modules import ConformerConvolution, ConformerFeedForward from nemo.collections.asr.parts.multi_head_attention import ( MultiHeadAttention, @@ -26,7 +27,6 @@ RelPositionalEncoding, RelPositionMultiHeadAttention, ) -from nemo.collections.asr.modules.subsampling import ConvSubsampling from nemo.core.classes.common import typecheck from nemo.core.classes.exportable import Exportable from nemo.core.classes.module import NeuralModule From 58f9588b0ca2872247e75c85f40ef8c0f33f9b3b Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 12:02:42 -0800 Subject: [PATCH 60/67] added pos_emb_max_len. Signed-off-by: Vahid --- examples/asr/experimental/conformer/conformer_bpe.yaml | 1 + examples/asr/experimental/conformer/conformer_char.yaml | 1 + nemo/collections/asr/modules/conformer_encoder.py | 7 +++++-- nemo/collections/asr/parts/multi_head_attention.py | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/asr/experimental/conformer/conformer_bpe.yaml b/examples/asr/experimental/conformer/conformer_bpe.yaml index 635ace3ba45c..0e5e19427288 100644 --- a/examples/asr/experimental/conformer/conformer_bpe.yaml +++ b/examples/asr/experimental/conformer/conformer_bpe.yaml @@ -86,6 +86,7 @@ model: # Multi-headed Attention Module's params self_attention_model: rel_pos # rel_pos, abs_pos + pos_emb_max_len: 5000 n_heads: 4 xscaling: true diff --git a/examples/asr/experimental/conformer/conformer_char.yaml b/examples/asr/experimental/conformer/conformer_char.yaml index a8f2d368e6a6..6129c0ce5bc8 100644 --- a/examples/asr/experimental/conformer/conformer_char.yaml +++ b/examples/asr/experimental/conformer/conformer_char.yaml @@ -83,6 +83,7 @@ model: # Multi-headed Attention Module's params self_attention_model: rel_pos # rel_pos, abs_pos + pos_emb_max_len: 5000 n_heads: 4 xscaling: true diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 4ad7960002d1..bbf3f693e54f 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -57,6 +57,8 @@ class ConformerEncoder(NeuralModule, Exportable): Defaults to 4. self_attention_model (str): type of the attention layer and positional encoding choices=['rel_pos', 'abs_pos']. + pos_emb_max_len (int): the maximum length of positional embeddings + Defaulst to 5000 n_heads (int): number of heads in multi-headed attention layers Defaults to 4. xscaling (bool): enables scaling the inputs to the multi-headed attention layers by sqrt(d_model) @@ -116,6 +118,7 @@ def __init__( subsampling_conv_channels=64, ff_expansion_factor=4, self_attention_model='rel_pos', + pos_emb_max_len=5000, n_heads=4, xscaling=True, conv_kernel_size=31, @@ -150,11 +153,11 @@ def __init__( if self_attention_model == "rel_pos": self.pos_enc = RelPositionalEncoding( - d_model=d_model, dropout_rate=dropout, dropout_emb_rate=dropout_emb, xscale=self.xscale + d_model=d_model, dropout_rate=dropout, max_len=pos_emb_max_len, xscale=self.xscale, dropout_emb_rate=dropout_emb ) elif self_attention_model == "abs_pos": self.pos_enc = PositionalEncoding( - d_model=d_model, dropout_rate=dropout, max_len=6000, reverse=False, xscale=self.xscale + d_model=d_model, dropout_rate=dropout, max_len=pos_emb_max_len, reverse=False, xscale=self.xscale ) else: raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") diff --git a/nemo/collections/asr/parts/multi_head_attention.py b/nemo/collections/asr/parts/multi_head_attention.py index 415652a36763..b36631d9eb46 100644 --- a/nemo/collections/asr/parts/multi_head_attention.py +++ b/nemo/collections/asr/parts/multi_head_attention.py @@ -263,7 +263,7 @@ def forward(self, x: torch.Tensor): class RelPositionalEncoding(PositionalEncoding): - """Relitive positional encoding module. + """Relative positional encoding module. See : Appendix B in https://arxiv.org/abs/1901.02860 Args: d_model (int): embedding dim @@ -271,7 +271,7 @@ class RelPositionalEncoding(PositionalEncoding): max_len (int): maximum input length """ - def __init__(self, d_model, dropout_rate, max_len=5000, dropout_emb_rate=0.0, xscale=None): + def __init__(self, d_model, dropout_rate, max_len=5000, xscale=None, dropout_emb_rate=0.0): super().__init__(d_model, dropout_rate, max_len, reverse=True, xscale=xscale) if dropout_emb_rate > 0: From 306ddf6734464b9d0b110475c8fe38ea664912fb Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 12:03:03 -0800 Subject: [PATCH 61/67] added pos_emb_max_len. Signed-off-by: Vahid --- nemo/collections/asr/modules/conformer_encoder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index bbf3f693e54f..8516b1e46222 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -153,7 +153,11 @@ def __init__( if self_attention_model == "rel_pos": self.pos_enc = RelPositionalEncoding( - d_model=d_model, dropout_rate=dropout, max_len=pos_emb_max_len, xscale=self.xscale, dropout_emb_rate=dropout_emb + d_model=d_model, + dropout_rate=dropout, + max_len=pos_emb_max_len, + xscale=self.xscale, + dropout_emb_rate=dropout_emb, ) elif self_attention_model == "abs_pos": self.pos_enc = PositionalEncoding( From 7dabb7844dbce1128c9bceb308718c69c4ac91f1 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 12:52:05 -0800 Subject: [PATCH 62/67] moved subsampling. Signed-off-by: Vahid --- nemo/collections/asr/modules/__init__.py | 1 - nemo/collections/asr/modules/conformer_encoder.py | 2 +- nemo/collections/asr/{modules => parts}/subsampling.py | 0 3 files changed, 1 insertion(+), 2 deletions(-) rename nemo/collections/asr/{modules => parts}/subsampling.py (100%) diff --git a/nemo/collections/asr/modules/__init__.py b/nemo/collections/asr/modules/__init__.py index a054bbb3bdab..36d553d1b34f 100644 --- a/nemo/collections/asr/modules/__init__.py +++ b/nemo/collections/asr/modules/__init__.py @@ -27,4 +27,3 @@ SpeakerDecoder, ) from nemo.collections.asr.modules.lstm_decoder import LSTMDecoder -from nemo.collections.asr.modules.subsampling import ConvSubsampling diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 8516b1e46222..12428ca73582 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -19,7 +19,6 @@ import torch.nn as nn from torch.nn.modules import LayerNorm -from nemo.collections.asr.modules.subsampling import ConvSubsampling from nemo.collections.asr.parts.conformer_modules import ConformerConvolution, ConformerFeedForward from nemo.collections.asr.parts.multi_head_attention import ( MultiHeadAttention, @@ -27,6 +26,7 @@ RelPositionalEncoding, RelPositionMultiHeadAttention, ) +from nemo.collections.asr.parts.subsampling import ConvSubsampling from nemo.core.classes.common import typecheck from nemo.core.classes.exportable import Exportable from nemo.core.classes.module import NeuralModule diff --git a/nemo/collections/asr/modules/subsampling.py b/nemo/collections/asr/parts/subsampling.py similarity index 100% rename from nemo/collections/asr/modules/subsampling.py rename to nemo/collections/asr/parts/subsampling.py From 90cb95ccadc07566c966c0e80280fca8d8b70838 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 13:06:17 -0800 Subject: [PATCH 63/67] added open_dict() Signed-off-by: Vahid --- nemo/collections/asr/models/ctc_bpe_models.py | 14 ++++----- nemo/collections/asr/models/ctc_models.py | 31 ++++++++++--------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/nemo/collections/asr/models/ctc_bpe_models.py b/nemo/collections/asr/models/ctc_bpe_models.py index 90c71f7cdacf..6fdce8a96106 100644 --- a/nemo/collections/asr/models/ctc_bpe_models.py +++ b/nemo/collections/asr/models/ctc_bpe_models.py @@ -17,7 +17,7 @@ from typing import Dict, Optional import torch -from omegaconf import DictConfig, ListConfig, OmegaConf +from omegaconf import DictConfig, ListConfig, OmegaConf, open_dict from nemo.collections.asr.data.audio_to_text import AudioToBPEDataset, TarredAudioToBPEDataset from nemo.collections.asr.losses.ctc import CTCLoss @@ -26,7 +26,6 @@ from nemo.collections.asr.parts.perturb import process_augmentations from nemo.collections.common import tokenizers from nemo.core.classes.common import PretrainedModelInfo -from nemo.core.neural_types import * from nemo.utils import logging __all__ = ['EncDecCTCModelBPE', 'JasperNetBPE', 'QuartzNetBPE'] @@ -67,10 +66,11 @@ def __init__(self, cfg: DictConfig, trainer=None): vocabulary = self.tokenizer.tokenizer.get_vocab() # Set the new vocabulary - if "params" in cfg.decoder: - cfg.decoder.params.vocabulary = ListConfig(list(vocabulary.values())) - else: - cfg.decoder.vocabulary = ListConfig(list(vocabulary.values())) + with open_dict(cfg): + if "params" in cfg.decoder: + cfg.decoder.params.vocabulary = ListConfig(list(vocabulary.values())) + else: + cfg.decoder.vocabulary = ListConfig(list(vocabulary.values())) # Override number of classes if placeholder provided if "params" in cfg.decoder: @@ -87,7 +87,7 @@ def __init__(self, cfg: DictConfig, trainer=None): if "params" in cfg.decoder: cfg.decoder["params"]["num_classes"] = len(vocabulary) else: - cfg.decoder['num_classes'] = len(vocabulary) + cfg.decoder["num_classes"] = len(vocabulary) super().__init__(cfg=cfg, trainer=trainer) diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py index 19cc9962c61a..8257e4170f1e 100644 --- a/nemo/collections/asr/models/ctc_models.py +++ b/nemo/collections/asr/models/ctc_models.py @@ -20,7 +20,7 @@ import onnx import torch -from omegaconf import DictConfig, OmegaConf +from omegaconf import DictConfig, OmegaConf, open_dict from pytorch_lightning import Trainer from nemo.collections.asr.data.audio_to_text import AudioToCharDataset, TarredAudioToCharDataset @@ -100,20 +100,21 @@ def __init__(self, cfg: DictConfig, trainer: Trainer = None): self.preprocessor = EncDecCTCModel.from_config_dict(self._cfg.preprocessor) self.encoder = EncDecCTCModel.from_config_dict(self._cfg.encoder) - if "params" in self._cfg.decoder: - if "feat_in" not in self._cfg.decoder.params or ( - not self._cfg.decoder.params.feat_in and hasattr(self.encoder, '_feat_out') - ): - self._cfg.decoder.params.feat_in = self.encoder._feat_out - if "feat_in" not in self._cfg.decoder.params or not self._cfg.decoder.params.feat_in: - raise ValueError("param feat_in of the decoder's config is not set!") - else: - if "feat_in" not in self._cfg.decoder or ( - not self._cfg.decoder.feat_in and hasattr(self.encoder, '_feat_out') - ): - self._cfg.decoder.feat_in = self.encoder._feat_out - if "feat_in" not in self._cfg.decoder or not self._cfg.decoder.feat_in: - raise ValueError("param feat_in of the decoder's config is not set!") + with open_dict(self._cfg): + if "params" in self._cfg.decoder: + if "feat_in" not in self._cfg.decoder.params or ( + not self._cfg.decoder.params.feat_in and hasattr(self.encoder, '_feat_out') + ): + self._cfg.decoder.params.feat_in = self.encoder._feat_out + if "feat_in" not in self._cfg.decoder.params or not self._cfg.decoder.params.feat_in: + raise ValueError("param feat_in of the decoder's config is not set!") + else: + if "feat_in" not in self._cfg.decoder or ( + not self._cfg.decoder.feat_in and hasattr(self.encoder, '_feat_out') + ): + self._cfg.decoder.feat_in = self.encoder._feat_out + if "feat_in" not in self._cfg.decoder or not self._cfg.decoder.feat_in: + raise ValueError("param feat_in of the decoder's config is not set!") self.decoder = EncDecCTCModel.from_config_dict(self._cfg.decoder) From 353e6529e55b66ea013349058cc626f3bcb33fbd Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 17:35:39 -0800 Subject: [PATCH 64/67] moved conformerblock. Signed-off-by: Vahid --- .../asr/modules/conformer_encoder.py | 92 +------------------ .../asr/parts/conformer_modules.py | 87 +++++++++++++++++- 2 files changed, 88 insertions(+), 91 deletions(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 12428ca73582..4202d4234944 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -14,18 +14,12 @@ import math from collections import OrderedDict +from collections.asr.parts.conformer_modules import ConformerEncoderBlock import torch import torch.nn as nn -from torch.nn.modules import LayerNorm -from nemo.collections.asr.parts.conformer_modules import ConformerConvolution, ConformerFeedForward -from nemo.collections.asr.parts.multi_head_attention import ( - MultiHeadAttention, - PositionalEncoding, - RelPositionalEncoding, - RelPositionMultiHeadAttention, -) +from nemo.collections.asr.parts.multi_head_attention import PositionalEncoding, RelPositionalEncoding from nemo.collections.asr.parts.subsampling import ConvSubsampling from nemo.core.classes.common import typecheck from nemo.core.classes.exportable import Exportable @@ -226,85 +220,3 @@ def make_pad_mask(seq_lens, max_time, device=None): if device: mask = mask.to(device) return mask - - -class ConformerEncoderBlock(torch.nn.Module): - """A single block of the Conformer encoder. - - Args: - d_model (int): input dimension of MultiheadAttentionMechanism and PositionwiseFeedForward - d_ff (int): hidden dimension of PositionwiseFeedForward - n_heads (int): number of heads for multi-head attention - conv_kernel_size (int): kernel size for depthwise convolution in convolution module - dropout (float): dropout probabilities for linear layers - dropout_att (float): dropout probabilities for attention distributions - """ - - def __init__(self, d_model, d_ff, conv_kernel_size, self_attention_model, n_heads, dropout, dropout_att): - super(ConformerEncoderBlock, self).__init__() - - self.self_attention_model = self_attention_model - self.n_heads = n_heads - self.fc_factor = 0.5 - - # first feed forward module - self.norm_feed_forward1 = LayerNorm(d_model) - self.feed_forward1 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) - - # convolution module - self.norm_conv = LayerNorm(d_model) - self.conv = ConformerConvolution(d_model=d_model, kernel_size=conv_kernel_size) - - # multi-headed self-attention module - self.norm_self_att = LayerNorm(d_model) - if self_attention_model == 'rel_pos': - self.self_attn = RelPositionMultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) - elif self_attention_model == 'abs_pos': - self.self_attn = MultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) - else: - raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") - - # second feed forward module - self.norm_feed_forward2 = LayerNorm(d_model) - self.feed_forward2 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) - - self.dropout = nn.Dropout(dropout) - self.norm_out = LayerNorm(d_model) - - def forward(self, x, att_mask=None, pos_emb=None, pad_mask=None): - """ - Args: - x (torch.Tensor): input signals (B, T, d_model) - att_mask (torch.Tensor): attention masks(B, T, T) - pos_emb (torch.Tensor): (L, 1, d_model) - pad_mask (torch.tensor): padding mask - Returns: - x (torch.Tensor): (B, T, d_model) - """ - residual = x - x = self.norm_feed_forward1(x) - x = self.feed_forward1(x) - x = self.fc_factor * self.dropout(x) + residual - - residual = x - x = self.norm_self_att(x) - if self.self_attention_model == 'rel_pos': - x = self.self_attn(query=x, key=x, value=x, pos_emb=pos_emb, mask=att_mask) - elif self.self_attention_model == 'abs_pos': - x = self.self_attn(query=x, key=x, value=x, mask=att_mask) - else: - x = None - x = self.dropout(x) + residual - - residual = x - x = self.norm_conv(x) - x = self.conv(x) - x = self.dropout(x) + residual - - residual = x - x = self.norm_feed_forward2(x) - x = self.feed_forward2(x) - x = self.fc_factor * self.dropout(x) + residual - - x = self.norm_out(x) - return x diff --git a/nemo/collections/asr/parts/conformer_modules.py b/nemo/collections/asr/parts/conformer_modules.py index a459dcf58d2a..0d9d20572478 100644 --- a/nemo/collections/asr/parts/conformer_modules.py +++ b/nemo/collections/asr/parts/conformer_modules.py @@ -12,14 +12,99 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from collections.asr.parts.multi_head_attention import MultiHeadAttention, RelPositionMultiHeadAttention -import torch.nn as nn +import torch +from torch import nn as nn +from torch.nn import LayerNorm from nemo.collections.asr.parts.activations import Swish __all__ = ['ConformerConvolution', 'ConformerFeedForward'] +class ConformerEncoderBlock(torch.nn.Module): + """A single block of the Conformer encoder. + + Args: + d_model (int): input dimension of MultiheadAttentionMechanism and PositionwiseFeedForward + d_ff (int): hidden dimension of PositionwiseFeedForward + n_heads (int): number of heads for multi-head attention + conv_kernel_size (int): kernel size for depthwise convolution in convolution module + dropout (float): dropout probabilities for linear layers + dropout_att (float): dropout probabilities for attention distributions + """ + + def __init__(self, d_model, d_ff, conv_kernel_size, self_attention_model, n_heads, dropout, dropout_att): + super(ConformerEncoderBlock, self).__init__() + + self.self_attention_model = self_attention_model + self.n_heads = n_heads + self.fc_factor = 0.5 + + # first feed forward module + self.norm_feed_forward1 = LayerNorm(d_model) + self.feed_forward1 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + + # convolution module + self.norm_conv = LayerNorm(d_model) + self.conv = ConformerConvolution(d_model=d_model, kernel_size=conv_kernel_size) + + # multi-headed self-attention module + self.norm_self_att = LayerNorm(d_model) + if self_attention_model == 'rel_pos': + self.self_attn = RelPositionMultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) + elif self_attention_model == 'abs_pos': + self.self_attn = MultiHeadAttention(n_head=n_heads, n_feat=d_model, dropout_rate=dropout_att) + else: + raise ValueError(f"Not valid self_attention_model: '{self_attention_model}'!") + + # second feed forward module + self.norm_feed_forward2 = LayerNorm(d_model) + self.feed_forward2 = ConformerFeedForward(d_model=d_model, d_ff=d_ff, dropout=dropout) + + self.dropout = nn.Dropout(dropout) + self.norm_out = LayerNorm(d_model) + + def forward(self, x, att_mask=None, pos_emb=None, pad_mask=None): + """ + Args: + x (torch.Tensor): input signals (B, T, d_model) + att_mask (torch.Tensor): attention masks(B, T, T) + pos_emb (torch.Tensor): (L, 1, d_model) + pad_mask (torch.tensor): padding mask + Returns: + x (torch.Tensor): (B, T, d_model) + """ + residual = x + x = self.norm_feed_forward1(x) + x = self.feed_forward1(x) + x = self.fc_factor * self.dropout(x) + residual + + residual = x + x = self.norm_self_att(x) + if self.self_attention_model == 'rel_pos': + x = self.self_attn(query=x, key=x, value=x, pos_emb=pos_emb, mask=att_mask) + elif self.self_attention_model == 'abs_pos': + x = self.self_attn(query=x, key=x, value=x, mask=att_mask) + else: + x = None + x = self.dropout(x) + residual + + residual = x + x = self.norm_conv(x) + x = self.conv(x) + x = self.dropout(x) + residual + + residual = x + x = self.norm_feed_forward2(x) + x = self.feed_forward2(x) + x = self.fc_factor * self.dropout(x) + residual + + x = self.norm_out(x) + return x + + class ConformerConvolution(nn.Module): """The convolution module for the Conformer model. Args: From 5e974b8fbd5605b0f794ca3bfac4631e7e46f323 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 19:11:07 -0800 Subject: [PATCH 65/67] fixed import bug. Signed-off-by: Vahid --- nemo/collections/asr/modules/conformer_encoder.py | 2 +- nemo/collections/asr/parts/conformer_modules.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 4202d4234944..3185dde9326b 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -14,7 +14,7 @@ import math from collections import OrderedDict -from collections.asr.parts.conformer_modules import ConformerEncoderBlock +from nemo.collections.asr.parts.conformer_modules import ConformerEncoderBlock import torch import torch.nn as nn diff --git a/nemo/collections/asr/parts/conformer_modules.py b/nemo/collections/asr/parts/conformer_modules.py index 0d9d20572478..5f2d29449c4d 100644 --- a/nemo/collections/asr/parts/conformer_modules.py +++ b/nemo/collections/asr/parts/conformer_modules.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from collections.asr.parts.multi_head_attention import MultiHeadAttention, RelPositionMultiHeadAttention +from nemo.collections.asr.parts.multi_head_attention import MultiHeadAttention, RelPositionMultiHeadAttention import torch from torch import nn as nn From 3eef0628d341293b626511252752f9e39c12c68c Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 2 Nov 2020 19:55:14 -0800 Subject: [PATCH 66/67] fixed code style. Signed-off-by: Vahid --- nemo/collections/asr/modules/conformer_encoder.py | 2 +- nemo/collections/asr/parts/conformer_modules.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nemo/collections/asr/modules/conformer_encoder.py b/nemo/collections/asr/modules/conformer_encoder.py index 3185dde9326b..e80d1aa64b84 100644 --- a/nemo/collections/asr/modules/conformer_encoder.py +++ b/nemo/collections/asr/modules/conformer_encoder.py @@ -14,11 +14,11 @@ import math from collections import OrderedDict -from nemo.collections.asr.parts.conformer_modules import ConformerEncoderBlock import torch import torch.nn as nn +from nemo.collections.asr.parts.conformer_modules import ConformerEncoderBlock from nemo.collections.asr.parts.multi_head_attention import PositionalEncoding, RelPositionalEncoding from nemo.collections.asr.parts.subsampling import ConvSubsampling from nemo.core.classes.common import typecheck diff --git a/nemo/collections/asr/parts/conformer_modules.py b/nemo/collections/asr/parts/conformer_modules.py index 5f2d29449c4d..212b6ce4248c 100644 --- a/nemo/collections/asr/parts/conformer_modules.py +++ b/nemo/collections/asr/parts/conformer_modules.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from nemo.collections.asr.parts.multi_head_attention import MultiHeadAttention, RelPositionMultiHeadAttention - import torch from torch import nn as nn from torch.nn import LayerNorm from nemo.collections.asr.parts.activations import Swish +from nemo.collections.asr.parts.multi_head_attention import MultiHeadAttention, RelPositionMultiHeadAttention __all__ = ['ConformerConvolution', 'ConformerFeedForward'] From 302b9f330cc60cef4d81e476bceb9bdffe700f5c Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 3 Nov 2020 11:29:10 -0800 Subject: [PATCH 67/67] added conformerencoder to all. Signed-off-by: Vahid --- nemo/collections/asr/parts/conformer_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo/collections/asr/parts/conformer_modules.py b/nemo/collections/asr/parts/conformer_modules.py index 212b6ce4248c..ef8943d4d510 100644 --- a/nemo/collections/asr/parts/conformer_modules.py +++ b/nemo/collections/asr/parts/conformer_modules.py @@ -19,7 +19,7 @@ from nemo.collections.asr.parts.activations import Swish from nemo.collections.asr.parts.multi_head_attention import MultiHeadAttention, RelPositionMultiHeadAttention -__all__ = ['ConformerConvolution', 'ConformerFeedForward'] +__all__ = ['ConformerConvolution', 'ConformerFeedForward', 'ConformerEncoderBlock'] class ConformerEncoderBlock(torch.nn.Module):