Updated bioguide ID assignment code to specifically append unmatched …

…names to the file for manual matching, and updated the file to include a spelling of Bill Young

process_new_release/2_add_bioguide_id/2_assign_bioguide_ids.rb

@@ -25,6 +25,8 @@
 end
 
 @@misses = 0
+@@missed_names = []
+
 @@duplicates = 0
 Sunlight::Base.api_key = 'sunlight9'
 
@@ -53,8 +55,7 @@ def legislator_for_name(name)
   puts "Couldn't find #{name} cached, checking with the Sunlight Labs Congress API..."
 
   # get rid of "HON." prefix and split on spaces
-  name = name.gsub /^HON\.\s?/i, ''
-  pieces = name.split /\s+/
+  pieces = name.gsub(/^HON\.\s?/i, '').split /\s+/
 
   # might be a state in parentheses at the end
   options[:state] = pieces.pop.gsub(/[\(\)]/, '') if pieces.last =~ /^\([a-zA-Z]+\)$/
@@ -118,6 +119,8 @@ def legislator_for_name(name)
             else
               @@misses += 1
               puts "I GIVE UP. Couldn't match on options: #{options.merge(:pieces => pieces).inspect}"
+              @@missed_names << name
+              puts "Added name to bottom of bioguide_ids.csv WITHOUT a bioguide_id, match by hand"
             end
           end
 
@@ -206,11 +209,15 @@ def name_for(legislator)
   names.each do |name, values|
     csv << [values[:bioguide_id], name, values[:name_confirm_from_sunlight], values[:in_office]]
   end
+
+  @@missed_names.each do |name|
+    csv << [nil, name, nil]
+  end
 end
 
 puts ""
 puts "Out of #{names.keys.size} names:"
-puts "#{@@misses} attempts failed to match a legislator entirely."
+puts "#{@@misses} attempts failed to match a legislator entirely, requires manual matching."
 puts "#{@@duplicates} attempts matched too many legislators."
 puts ""
-puts "Appended any new names and bioguide IDs on to #{bioguide_file}."
+puts "Appended any new names and bioguide IDs to #{bioguide_file}."

process_new_release/2_add_bioguide_id/3_update_expenditures.rb

@@ -31,7 +31,9 @@
 legislators = {}
 FasterCSV.foreach(bioguide_file) do |row|
   # key is name, value is bioguide_id
-  legislators[row[1]] = row[0]
+  if row[0] and row[0] != ""
+    legislators[row[1]] = row[0] 
+  end
 end
 
 # open up a file for writing, and in it:

process_new_release/2_add_bioguide_id/bioguide_ids.csv

@@ -470,93 +470,94 @@
 "C001079","HON. ANH ""JOSEPH"" CAO","REP. ANH 'JOSEPH' CAO","false"
 "C001078","HON. GERALD E ""GERRY"" CONNOLLY","REP. GERALD 'GERRY' CONNOLLY","true"
 "M000508","HON. HOWARD ""BUCK"" MCKEON","REP. HOWARD 'BUCK' MCKEON","true"
-B001260,HON. VERN G. BUCHANAN,REP. VERN BUCHANAN,true
-H000067,HON. RALPH M. HALL,REP. RALPH HALL,true
-A000022,HON. GARY L. ACKERMAN,REP. GARY ACKERMAN,true
-P000258,HON. COLLIN C. PETERSON,REP. COLLIN PETERSON,true
-M000133,HON. EDWARD J. MARKEY,REP. EDWARD 'ED' MARKEY,true
-H000874,HON. STENY H. HOYER,REP. STENY HOYER,true
-A000210,HON. ROBERT E. ANDREWS,REP. ROBERT 'ROB' ANDREWS,true
-B001255,"HON. CHARLES W. BOUSTANY, JR.",REP. CHARLES BOUSTANY JR.,true
-K000371,"HON. FRANK KRATOVIL, JR.",REP. FRANK KRATOVIL JR.,false
-R000004,HON. GEORGE P. RADANOVICH,REP. GEORGE RADANOVICH,false
-E000092,HON. VERNON J. EHLERS,REP. VERNON 'VERN' EHLERS,false
-S000185,HON. ROBERT C. SCOTT,REP. ROBERT 'BOBBY' SCOTT,true
-K000210,HON. PETER T. KING,REP. PETER 'PETE' KING,true
-M000689,HON. JOHN L. MICA,REP. JOHN MICA,true
-O000007,HON. DAVID R. OBEY,REP. DAVID ROSS 'DAVE' OBEY,false
-P000034,"HON. FRANK PALLONE, JR.",REP. FRANK PALLONE JR.,true
-K000008,HON. PAUL E. KANJORSKI,REP. PAUL KANJORSKI,false
-M000933,HON. JAMES P. MORAN,REP. JAMES 'JIM' MORAN JR.,true
-E000179,HON. ELIOT L. ENGEL,REP. ELIOT ENGEL,true
-C001078,"HON. GERALD E. ""GERRY"" CONNOLLY",REP. GERALD 'GERRY' CONNOLLY,true
-F000455,HON. MARCIA FUDGE,REP. MARCIA FUDGE,true
-N000015,HON. RICHARD E. NEAL,REP. RICHARD NEAL,true
-G000535,HON. LUIS V. GUTIERREZ,REP. LUIS GUTIERREZ,true
-E000290,HON. DONNA EDWARDS,REP. DONNA EDWARDS,true
-B000208,HON. ROSCOE G. BARTLETT,REP. ROSCOE BARTLETT,true
-R000053,HON. CHARLES B. RANGEL,REP. CHARLES 'CHARLIE' RANGEL,true
-R000572,HON. MIKE ROGERS,REP. MIKE ROGERS,true
-M000087,HON. CAROLYN B. MALONEY,REP. CAROLYN MALONEY,true
-C001074,HON. TRAVIS W CHILDERS,REP. TRAVIS CHILDERS,false
-D000216,HON. ROSA L. DELAURO,REP. ROSA DELAURO,true
-F000010,HON. ENI F. H. FALEOMAVAEGA,DEL. ENI FALEOMAVAEGA JR,true
-F000372,HON. RODNEY P. FRELINGHUYSEN,REP. RODNEY FRELINGHUYSEN,true
-L000566,HON. ROBERT E LATTA,REP. ROBERT 'BOB' LATTA,true
-H000324,HON. ALCEE L. HASTINGS,REP. ALCEE HASTINGS SR.,true
-C000537,HON. JAMES E. CLYBURN,REP. JAMES 'JIM' CLYBURN,true
-B000410,HON. HOWARD BERMAN,REP. HOWARD BERMAN,true
-P000265,HON. THOMAS E. PETRI,REP. THOMAS 'TOM' PETRI,true
-D000191,HON. PETER A. DEFAZIO,REP. PETER DEFAZIO,true
-W000215,HON. HENRY A. WAXMAN,REP. HENRY WAXMAN,true
-O000006,HON. JAMES L. OBERSTAR,REP. JAMES 'JIM' OBERSTAR,false
-O000107,HON. SOLOMON P. ORTIZ,REP. SOLOMON ORTIZ SR.,false
-P000149,HON. DONALD M. PAYNE,REP. DONALD PAYNE,true
-P000523,HON. DAVID E. PRICE,REP. DAVID PRICE,true
-M000508,"HON. HOWARD P. ""BUCK"" MCKEON",REP. HOWARD 'BUCK' MCKEON,true
-C000714,"HON. JOHN CONYERS, JR.",REP. JOHN CONYERS JR.,true
-B000490,HON. SANFORD BISHOP,REP. SANFORD BISHOP JR.,true
-S000749,"HON. JOHN M. SPRATT, JR.",REP. JOHN SPRATT JR.,false
-W000672,HON. FRANK R. WOLF,REP. FRANK WOLF,true
-M000844,HON. ALAN B. MOLLOHAN,REP. ALAN MOLLOHAN,false
-S000522,HON. CHRISTOPHER H. SMITH,REP. CHRISTOPHER 'CHRIS' SMITH,true
-B000461,HON. BRIAN P BILBRAY,REP. BRIAN BILBRAY,true
-W000207,HON. MELVIN L. WATT,REP. MELVIN 'MEL' WATT,true
-L000263,HON. SANDER M. LEVIN,REP. SANDER 'SANDY' LEVIN,true
-C000794,HON. JERRY F. COSTELLO,REP. JERRY COSTELLO,true
-K000172,HON. DALE E. KILDEE,REP. DALE KILDEE,true
-A000363,HON. MICHAEL A. ARCURI,REP. MICHAEL ARCURI,false
-H000627,HON. MAURICE D. HINCHEY,REP. MAURICE HINCHEY,true
-S000248,HON. JOSE E. SERRANO,REP. JOSE SERRANO,true
-L000553,HON. STEVEN C. LATOURETTE,REP. STEVEN LATOURETTE,true
-E000215,HON. ANNA G. ESHOO,REP. ANNA ESHOO,true
-K000113,HON. PATRICK J. KENNEDY,REP. PATRICK KENNEDY,false
-T000038,HON. JOHN S. TANNER,REP. JOHN TANNER,false
-D000327,HON. NORMAN D. DICKS,REP. NORMAN 'NORM' DICKS,true
-J000288,"HON. HENRY C. ""HANK"" JOHNSON, JR.",REP. HENRY 'HANK' JOHNSON JR.,true
-S000244,"HON. F. JAMES SENSENBRENNER, JR.",REP. JAMES 'JIM' SENSENBRENNER JR.,true
-R000578,HON. DAVID G REICHERT,REP. DAVID 'DAVE' REICHERT,true
-V000081,HON. NYDIA M. VELAZQUEZ,REP. NYDIA VELAZQUEZ,true
-D000482,HON. MICHAEL F. DOYLE,REP. MICHAEL 'MIKE' DOYLE JR.,true
-C000243,HON. MICHAEL N. CASTLE,REP. MICHAEL CASTLE,false
-L000491,HON. FRANK D. LUCAS,REP. FRANK LUCAS,true
-L000480,HON. NITA M. LOWEY,REP. NITA LOWEY,true
-R000487,HON. EDWARD R. ROYCE,REP. EDWARD 'ED' ROYCE,true
-M001167,HON. HARRY E MITCHELL,REP. HARRY MITCHELL,false
-O000085,HON. JOHN W. OLVER,REP. JOHN OLVER,true
-L000517,HON. DANIEL E LUNGREN,REP. DANIEL 'DAN' LUNGREN,true
-M001138,HON. DONALD A. MANZULLO,REP. DONALD 'DON' MANZULLO,true
-B000589,HON. JOHN A. BOEHNER,REP. JOHN BOEHNER,true
-S001188,HON. MARLIN A. STUTZMAN,REP. MARLIN STUTZMAN,true
-R000011,HON. NICK J. RAHALL II,REP. NICK JOE 'NICK' RAHALL II,true
-J000255,HON. WALTER B. JONES,REP. WALTER JONES JR.,true
-V000108,HON. PETER J. VISCLOSKY,REP. PETER 'PETE' VISCLOSKY,true
-L000554,HON. FRANK A. LOBIONDO,REP. FRANK LOBIONDO,true
-R000515,HON. BOBBY L. RUSH,REP. BOBBY RUSH,true
-C001062,HON. K.MICHAEL CONAWAY,REP. K. MICHAEL 'MIKE' CONAWAY,true
-D000355,HON. JOHN D. DINGELL,REP. JOHN DINGELL,true
-R000585,HON. TOM REED,REP. THOMAS 'TOM' REED II,true
-J000283,"HON. JESSE L. JACKSON, JR.",REP. JESSE JACKSON JR.,true
-S000275,HON. JOHN B. SHADEGG,REP. JOHN SHADEGG,false
-F000443,HON. MIKE FERGUSON,REP. MICHAEL 'MIKE' FERGUSON,false
-C000266,HON. STEVE CHABOT,REP. STEVEN 'STEVE' CHABOT,true
+"B001260","HON. VERN G. BUCHANAN","REP. VERN BUCHANAN","true"
+"H000067","HON. RALPH M. HALL","REP. RALPH HALL","true"
+"A000022","HON. GARY L. ACKERMAN","REP. GARY ACKERMAN","true"
+"P000258","HON. COLLIN C. PETERSON","REP. COLLIN PETERSON","true"
+"M000133","HON. EDWARD J. MARKEY","REP. EDWARD 'ED' MARKEY","true"
+"H000874","HON. STENY H. HOYER","REP. STENY HOYER","true"
+"A000210","HON. ROBERT E. ANDREWS","REP. ROBERT 'ROB' ANDREWS","true"
+"B001255","HON. CHARLES W. BOUSTANY, JR.","REP. CHARLES BOUSTANY JR.","true"
+"K000371","HON. FRANK KRATOVIL, JR.","REP. FRANK KRATOVIL JR.","false"
+"R000004","HON. GEORGE P. RADANOVICH","REP. GEORGE RADANOVICH","false"
+"E000092","HON. VERNON J. EHLERS","REP. VERNON 'VERN' EHLERS","false"
+"S000185","HON. ROBERT C. SCOTT","REP. ROBERT 'BOBBY' SCOTT","true"
+"K000210","HON. PETER T. KING","REP. PETER 'PETE' KING","true"
+"M000689","HON. JOHN L. MICA","REP. JOHN MICA","true"
+"O000007","HON. DAVID R. OBEY","REP. DAVID ROSS 'DAVE' OBEY","false"
+"P000034","HON. FRANK PALLONE, JR.","REP. FRANK PALLONE JR.","true"
+"K000008","HON. PAUL E. KANJORSKI","REP. PAUL KANJORSKI","false"
+"M000933","HON. JAMES P. MORAN","REP. JAMES 'JIM' MORAN JR.","true"
+"E000179","HON. ELIOT L. ENGEL","REP. ELIOT ENGEL","true"
+"C001078","HON. GERALD E. ""GERRY"" CONNOLLY","REP. GERALD 'GERRY' CONNOLLY","true"
+"F000455","HON. MARCIA FUDGE","REP. MARCIA FUDGE","true"
+"N000015","HON. RICHARD E. NEAL","REP. RICHARD NEAL","true"
+"G000535","HON. LUIS V. GUTIERREZ","REP. LUIS GUTIERREZ","true"
+"E000290","HON. DONNA EDWARDS","REP. DONNA EDWARDS","true"
+"B000208","HON. ROSCOE G. BARTLETT","REP. ROSCOE BARTLETT","true"
+"R000053","HON. CHARLES B. RANGEL","REP. CHARLES 'CHARLIE' RANGEL","true"
+"R000572","HON. MIKE ROGERS","REP. MIKE ROGERS","true"
+"M000087","HON. CAROLYN B. MALONEY","REP. CAROLYN MALONEY","true"
+"C001074","HON. TRAVIS W CHILDERS","REP. TRAVIS CHILDERS","false"
+"D000216","HON. ROSA L. DELAURO","REP. ROSA DELAURO","true"
+"F000010","HON. ENI F. H. FALEOMAVAEGA","DEL. ENI FALEOMAVAEGA JR","true"
+"F000372","HON. RODNEY P. FRELINGHUYSEN","REP. RODNEY FRELINGHUYSEN","true"
+"L000566","HON. ROBERT E LATTA","REP. ROBERT 'BOB' LATTA","true"
+"H000324","HON. ALCEE L. HASTINGS","REP. ALCEE HASTINGS SR.","true"
+"C000537","HON. JAMES E. CLYBURN","REP. JAMES 'JIM' CLYBURN","true"
+"B000410","HON. HOWARD BERMAN","REP. HOWARD BERMAN","true"
+"P000265","HON. THOMAS E. PETRI","REP. THOMAS 'TOM' PETRI","true"
+"D000191","HON. PETER A. DEFAZIO","REP. PETER DEFAZIO","true"
+"W000215","HON. HENRY A. WAXMAN","REP. HENRY WAXMAN","true"
+"O000006","HON. JAMES L. OBERSTAR","REP. JAMES 'JIM' OBERSTAR","false"
+"O000107","HON. SOLOMON P. ORTIZ","REP. SOLOMON ORTIZ SR.","false"
+"P000149","HON. DONALD M. PAYNE","REP. DONALD PAYNE","true"
+"P000523","HON. DAVID E. PRICE","REP. DAVID PRICE","true"
+"M000508","HON. HOWARD P. ""BUCK"" MCKEON","REP. HOWARD 'BUCK' MCKEON","true"
+"C000714","HON. JOHN CONYERS, JR.","REP. JOHN CONYERS JR.","true"
+"B000490","HON. SANFORD BISHOP","REP. SANFORD BISHOP JR.","true"
+"S000749","HON. JOHN M. SPRATT, JR.","REP. JOHN SPRATT JR.","false"
+"W000672","HON. FRANK R. WOLF","REP. FRANK WOLF","true"
+"M000844","HON. ALAN B. MOLLOHAN","REP. ALAN MOLLOHAN","false"
+"S000522","HON. CHRISTOPHER H. SMITH","REP. CHRISTOPHER 'CHRIS' SMITH","true"
+"B000461","HON. BRIAN P BILBRAY","REP. BRIAN BILBRAY","true"
+"W000207","HON. MELVIN L. WATT","REP. MELVIN 'MEL' WATT","true"
+"L000263","HON. SANDER M. LEVIN","REP. SANDER 'SANDY' LEVIN","true"
+"C000794","HON. JERRY F. COSTELLO","REP. JERRY COSTELLO","true"
+"K000172","HON. DALE E. KILDEE","REP. DALE KILDEE","true"
+"A000363","HON. MICHAEL A. ARCURI","REP. MICHAEL ARCURI","false"
+"H000627","HON. MAURICE D. HINCHEY","REP. MAURICE HINCHEY","true"
+"S000248","HON. JOSE E. SERRANO","REP. JOSE SERRANO","true"
+"L000553","HON. STEVEN C. LATOURETTE","REP. STEVEN LATOURETTE","true"
+"E000215","HON. ANNA G. ESHOO","REP. ANNA ESHOO","true"
+"K000113","HON. PATRICK J. KENNEDY","REP. PATRICK KENNEDY","false"
+"T000038","HON. JOHN S. TANNER","REP. JOHN TANNER","false"
+"D000327","HON. NORMAN D. DICKS","REP. NORMAN 'NORM' DICKS","true"
+"J000288","HON. HENRY C. ""HANK"" JOHNSON, JR.","REP. HENRY 'HANK' JOHNSON JR.","true"
+"S000244","HON. F. JAMES SENSENBRENNER, JR.","REP. JAMES 'JIM' SENSENBRENNER JR.","true"
+"R000578","HON. DAVID G REICHERT","REP. DAVID 'DAVE' REICHERT","true"
+"V000081","HON. NYDIA M. VELAZQUEZ","REP. NYDIA VELAZQUEZ","true"
+"D000482","HON. MICHAEL F. DOYLE","REP. MICHAEL 'MIKE' DOYLE JR.","true"
+"C000243","HON. MICHAEL N. CASTLE","REP. MICHAEL CASTLE","false"
+"L000491","HON. FRANK D. LUCAS","REP. FRANK LUCAS","true"
+"L000480","HON. NITA M. LOWEY","REP. NITA LOWEY","true"
+"R000487","HON. EDWARD R. ROYCE","REP. EDWARD 'ED' ROYCE","true"
+"M001167","HON. HARRY E MITCHELL","REP. HARRY MITCHELL","false"
+"O000085","HON. JOHN W. OLVER","REP. JOHN OLVER","true"
+"L000517","HON. DANIEL E LUNGREN","REP. DANIEL 'DAN' LUNGREN","true"
+"M001138","HON. DONALD A. MANZULLO","REP. DONALD 'DON' MANZULLO","true"
+"B000589","HON. JOHN A. BOEHNER","REP. JOHN BOEHNER","true"
+"S001188","HON. MARLIN A. STUTZMAN","REP. MARLIN STUTZMAN","true"
+"R000011","HON. NICK J. RAHALL II","REP. NICK JOE 'NICK' RAHALL II","true"
+"J000255","HON. WALTER B. JONES","REP. WALTER JONES JR.","true"
+"V000108","HON. PETER J. VISCLOSKY","REP. PETER 'PETE' VISCLOSKY","true"
+"L000554","HON. FRANK A. LOBIONDO","REP. FRANK LOBIONDO","true"
+"R000515","HON. BOBBY L. RUSH","REP. BOBBY RUSH","true"
+"C001062","HON. K.MICHAEL CONAWAY","REP. K. MICHAEL 'MIKE' CONAWAY","true"
+"D000355","HON. JOHN D. DINGELL","REP. JOHN DINGELL","true"
+"R000585","HON. TOM REED","REP. THOMAS 'TOM' REED II","true"
+"J000283","HON. JESSE L. JACKSON, JR.","REP. JESSE JACKSON JR.","true"
+"S000275","HON. JOHN B. SHADEGG","REP. JOHN SHADEGG","false"
+"F000443","HON. MIKE FERGUSON","REP. MICHAEL 'MIKE' FERGUSON","false"
+"C000266","HON. STEVE CHABOT","REP. STEVEN 'STEVE' CHABOT","true"
+"Y000031","HON. C.W. BILL YOUNG",,