Skip to content
This repository has been archived by the owner on Mar 6, 2019. It is now read-only.

Commit

Permalink
added support for F3L schedule A transactions
Browse files Browse the repository at this point in the history
  • Loading branch information
Derek Willis committed Mar 28, 2015
1 parent 182ff96 commit 5e5b058
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 29 deletions.
9 changes: 5 additions & 4 deletions lib/fech/fech_utils.rb
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Contains helper functions and static variables used by various
# Fech classes.
module FechUtils

# All supported row types pointed to regular expressions that will correctly
# match that row type in the wild. If multiple matches exist, Fech will match
# the longest regex pattern found.
Expand Down Expand Up @@ -43,6 +43,7 @@ module FechUtils
:h5 => /^h5/i,
:h6 => /^h6/i,
:sa => /^sa/i,
:sa3l => /^sa3l/i,
:sb => /^sb/i,
:sc => /^sc[^1-2]/i,
:sc1 => /^sc1/i,
Expand All @@ -53,7 +54,7 @@ module FechUtils
:sl => /^sl/i,
:text => /^text/i,
}

# Converts symbols and strings to Regexp objects for use in regex-keyed maps.
# Assumes that symbols should be matched literally, strings unanchored.
# @param [String,Symbol,Regexp] label the object to convert to a Regexp
Expand All @@ -70,7 +71,7 @@ def regexify(label)
Regexp.new(Regexp.escape(label.to_s), Regexp::IGNORECASE)
end
end



end
52 changes: 28 additions & 24 deletions lib/fech/map_generator.rb
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
module Fech

# Helper class to generate mapping hashes from source csv data.
# Needed to rebuild rendered_maps.rb with new source data, not used
# in main gem.
# rake fech:maps
class MapGenerator

attr_accessor :map
PAPER_FILING_VERSIONS = ["3.1", "3.0", "2.6", "2.4", "2.3", "2.2", "1.0"]
PAPER_BASE_ROW_TYPES = ["HDR", "F3", "F3X", "SchA", "SchB"]

FILING_VERSIONS = ["8.0", "7.0", "6.4", "6.3", "6.2", "6.1",
"5.3", "5.2", "5.1", "5.0", "3"]
BASE_ROW_TYPES = ["HDR", "F1", "F13", "F132", "F133", "F1M", "F1S", "F2", "F24", "F3", "F3L", "F3P", "F3P31", "F3PS",
"F3S", "F3X", "F4", "F5", "F56", "F57", "F6", "F65", "F7", "F76", "F9", "F91", "F92", "F93",
BASE_ROW_TYPES = ["HDR", "F1", "F13", "F132", "F133", "F1M", "F1S", "F2", "F24", "F3", "F3L", "F3P", "F3P31", "F3PS",
"F3S", "F3X", "F4", "F5", "F56", "F57", "F6", "F65", "F7", "F76", "F9", "F91", "F92", "F93",
"F94", "F99", "H1", "H2", "H3", "H4", "H5", "H6",
"SchA", "SchB", "SchC", "SchC1", "SchC2", "SchD", "SchE", "SchF", "SchL", "TEXT"]
"SchA", "SchA3L", "SchB", "SchC", "SchC1", "SchC2", "SchD", "SchE", "SchF", "SchL", "TEXT"]
ROW_TYPE_MATCHERS = {
"HDR" => FechUtils::ROW_TYPES[:hdr],
"F1" => FechUtils::ROW_TYPES[:f1],
Expand Down Expand Up @@ -51,6 +54,7 @@ class MapGenerator
"H5" => FechUtils::ROW_TYPES[:h5],
"H6" => FechUtils::ROW_TYPES[:h6],
"SchA" => FechUtils::ROW_TYPES[:sa],
"SchA3L" => FechUtils::ROW_TYPES[:sa3l],
"SchB" => FechUtils::ROW_TYPES[:sb],
"SchC" => FechUtils::ROW_TYPES[:sc],
"SchC1" => FechUtils::ROW_TYPES[:sc1],
Expand All @@ -61,15 +65,15 @@ class MapGenerator
"SchL" => FechUtils::ROW_TYPES[:sl],
"TEXT" => FechUtils::ROW_TYPES[:text],
}

# Goes through all version header summary files and generates
# row map files for each type of row inside them.
def self.convert_header_file_to_row_files(source_dir)
data = {}
hybrid_data = {}

ignored_fields = File.open(ignored_fields_file(source_dir)).readlines.map { |l| l.strip }

# Create a hash of data with an entry for each row type found in the source
# version summary files. Each row has an entry for each version map that
# exists for it. If maps for two different versions are identical, they
Expand Down Expand Up @@ -101,12 +105,12 @@ def self.convert_header_file_to_row_files(source_dir)
data[row.first][version] = row_version_data
data[row.first].each do |k, v|
# skip the row we just added

next if k == version
if v == row_version_data
# Create the new hybrid entry
hybrid_data[row.first]["#{k}|#{version}"] = row_version_data

# Delete the old entry, and the one for this version only
data[row.first].delete(k)
data[row.first].delete(version)
Expand All @@ -115,7 +119,7 @@ def self.convert_header_file_to_row_files(source_dir)
data[row.first].update(hybrid_data[row.first])
end
end

# Go through each row type and create a base map management file that
# will serve as a template for organizing which fields are the same
# between versions. This file will need to then be arranged by hand to
Expand All @@ -126,18 +130,18 @@ def self.convert_header_file_to_row_files(source_dir)
next unless File.exists?(file_path)
File.open(file_path, 'w') do |f|
f.write('canonical')

to_transpose = []
row_data.sort.reverse.each do |version, version_data|
to_transpose << ["^#{version}", version_data.each_with_index.collect {|x, idx| idx+1}].flatten
to_transpose << [nil, version_data].flatten
end

# standardize row size
max_size = to_transpose.max { |r1, r2| r1.size <=> r2.size }.size
to_transpose.each { |r| r[max_size - 1] ||= nil }
transposed = to_transpose.transpose

transposed.each do |transposed_data|
transposed_data.collect! {|x| x.to_s.gsub(/\r/, ' ')}
canonical = transposed_data[1] # first description
Expand All @@ -152,7 +156,7 @@ def self.convert_header_file_to_row_files(source_dir)
end

end

# Generates the mapping for each row type in BASE_ROW_TYPES, writes them out
# to file for inclusion in the gem.
def self.dump_row_maps_to_ruby(source_dir, file_path)
Expand Down Expand Up @@ -194,44 +198,44 @@ def self.generate_row_map_from_file(source_dir, row_type)
row.each_with_index {|x, ind| version_indexes << ind unless (x.nil? || x.empty?)}.slice!(1)
version_indexes.slice!(0, 1)
versions.each {|x| data[x] = [] }

elsif row.first.size > 0
canonical = row.first

versions.zip(version_indexes).each do |version, row_index|
index = row[row_index]
data[version][index.to_i - 1] = canonical.to_sym if index.to_i > 0
end
end
end

row_map = {}
data.each {|key, value| row_map[key] = value}
row_map
end

# Remove both the row type from the beginning of the row,
# and any fields marked as "ignore" in sources/headers/ignore.csv
def self.remove_ignored_fields(row, ignore)
data = row[1..-1].compact # strip off the row type
data.reject { |f| ignore.include?(f) }
end

def self.row_map_file(source_dir, row_type)
File.join(source_dir, row_type + '.csv')
end

def self.ignored_fields_file(source_dir)
File.join(source_dir, 'headers', 'ignore.csv')
end

def self.version_summary_file(source_dir, version)
File.join(source_dir, 'headers', version + '.csv')
end

def self.write_row_map_file(source_dir, row_type)
File.join(source_dir, 'rows', row_type + '.csv')
end

end
end
4 changes: 4 additions & 0 deletions lib/fech/rendered_maps.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,10 @@ module Fech
'^5.0' => [:form_type, :filer_committee_id_number, :entity_type, :contributor_name, :contributor_street_1, :contributor_street_2, :contributor_city, :contributor_state, :contributor_zip_code, :election_code, :election_other_description, :contributor_employer, :contributor_occupation, :contribution_aggregate, :contribution_date, :contribution_amount, :contribution_purpose_code, :contribution_purpose_descrip, :donor_committee_fec_id, :donor_candidate_fec_id, :donor_candidate_name, :donor_candidate_office, :donor_candidate_state, :donor_candidate_district, :conduit_name, :conduit_street1, :conduit_street2, :conduit_city, :conduit_state, :conduit_zip_code, :memo_code, :memo_text_description, nil, :transaction_id, :back_reference_tran_id_number, :back_reference_sched_name, :reference_code, :increased_limit_code],
'^3' => [:form_type, :filer_committee_id_number, :entity_type, :contributor_name, :contributor_street_1, :contributor_street_2, :contributor_city, :contributor_state, :contributor_zip_code, :election_code, :election_other_description, :contributor_employer, :contributor_occupation, :contribution_aggregate, :contribution_date, :contribution_amount, :contribution_purpose_code, :contribution_purpose_descrip, :donor_committee_fec_id, :donor_candidate_fec_id, :donor_candidate_name, :donor_candidate_office, :donor_candidate_state, :donor_candidate_district, :conduit_name, :conduit_street1, :conduit_street2, :conduit_city, :conduit_state, :conduit_zip_code, :memo_code, :memo_text_description, nil, :transaction_id, :back_reference_tran_id_number, :back_reference_sched_name, :reference_code],
},
"^sa3l" => {
'^8.1|8.0' => [:form_type, :filer_committee_id_number, :transaction_id, :back_reference_tran_id_number, :back_reference_sched_name, :entity_type, :lobbyist_registrant_organization_name, :lobbyist_registrant_last_name, :lobbyist_registrant_first_name, :lobbyist_registrant_middle_name, :lobbyist_registrant_prefix, :lobbyist_registrant_suffix, :lobbyist_registrant_street_1, :lobbyist_registrant_street_2, :lobbyist_registrant_city, :lobbyist_registrant_state, :lobbyist_registrant_zip_code, :election_code, :election_other_description, :contribution_date, :bundled_amount_period, :bundled_amount_semi_annual, :contribution_purpose_descrip, :lobbyist_registrant_employer, :lobbyist_registrant_occupation, :donor_committee_fec_id, :donor_committee_name, :donor_candidate_fec_id, :donor_candidate_last_name, :donor_candidate_first_name, :donor_candidate_middle_name, :donor_candidate_prefix, :donor_candidate_suffix, :donor_candidate_office, :donor_candidate_state, :donor_candidate_district, :conduit_name, :conduit_street1, :conduit_street2, :conduit_city, :conduit_state, :conduit_zip_code, :associated_text_record, :memo_text, :reference_code],
'^7.0|6.4' => [:form_type, :filer_committee_id_number, :transaction_id, :back_reference_tran_id_number, :back_reference_sched_name, :entity_type, :lobbyist_registrant_organization_name, :lobbyist_registrant_last_name, :lobbyist_registrant_first_name, :lobbyist_registrant_middle_name, :lobbyist_registrant_prefix, :lobbyist_registrant_suffix, :lobbyist_registrant_street_1, :lobbyist_registrant_street_2, :lobbyist_registrant_city, :lobbyist_registrant_state, :lobbyist_registrant_zip_code, :election_code, :election_other_description, :contribution_date, :bundled_amount_period, :bundled_amount_semi_annual, :contribution_purpose_code, :contribution_purpose_descrip, :lobbyist_registrant_employer, :lobbyist_registrant_occupation, :donor_committee_fec_id, :donor_committee_name, :donor_candidate_fec_id, :donor_candidate_last_name, :donor_candidate_first_name, :donor_candidate_middle_name, :donor_candidate_prefix, :donor_candidate_suffix, :donor_candidate_office, :donor_candidate_state, :donor_candidate_district, :conduit_name, :conduit_street1, :conduit_street2, :conduit_city, :conduit_state, :conduit_zip_code, :associated_text_record, :memo_text, :reference_code],
},
"^sb" => {
'^8.1|8.0' => [:form_type, :filer_committee_id_number, :transaction_id_number, :back_reference_tran_id_number, :back_reference_sched_name, :entity_type, :payee_organization_name, :payee_last_name, :payee_first_name, :payee_middle_name, :payee_prefix, :payee_suffix, :payee_street_1, :payee_street_2, :payee_city, :payee_state, :payee_zip_code, :election_code, :election_other_description, :expenditure_date, :expenditure_amount, :semi_annual_refunded_bundled_amt, :expenditure_purpose_descrip, :category_code, :beneficiary_committee_fec_id, :beneficiary_committee_name, :beneficiary_candidate_fec_id, :beneficiary_candidate_last_name, :beneficiary_candidate_first_name, :beneficiary_candidate_middle_name, :beneficiary_candidate_prefix, :beneficiary_candidate_suffix, :beneficiary_candidate_office, :beneficiary_candidate_state, :beneficiary_candidate_district, :conduit_name, :conduit_street_1, :conduit_street_2, :conduit_city, :conduit_state, :conduit_zip_code, :memo_code, :memo_text_description, :reference_to_si_or_sl_system_code_that_identifies_the_account],
'^7.0|6.4' => [:form_type, :filer_committee_id_number, :transaction_id_number, :back_reference_tran_id_number, :back_reference_sched_name, :entity_type, :payee_organization_name, :payee_last_name, :payee_first_name, :payee_middle_name, :payee_prefix, :payee_suffix, :payee_street_1, :payee_street_2, :payee_city, :payee_state, :payee_zip_code, :election_code, :election_other_description, :expenditure_date, :expenditure_amount, :semi_annual_refunded_bundled_amt, :expenditure_purpose_code, :expenditure_purpose_descrip, :category_code, :beneficiary_committee_fec_id, :beneficiary_committee_name, :beneficiary_candidate_fec_id, :beneficiary_candidate_last_name, :beneficiary_candidate_first_name, :beneficiary_candidate_middle_name, :beneficiary_candidate_prefix, :beneficiary_candidate_suffix, :beneficiary_candidate_office, :beneficiary_candidate_state, :beneficiary_candidate_district, :conduit_name, :conduit_street_1, :conduit_street_2, :conduit_city, :conduit_state, :conduit_zip_code, :memo_code, :memo_text_description, :reference_to_si_or_sl_system_code_that_identifies_the_account],
Expand Down
2 changes: 1 addition & 1 deletion sources/SchA3L.csv
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
canonical,^8.1|8.0|7.0|6.4,form_type,1,FORM TYPEfiler_committee_id_number,2,FILER COMMITTEE ID NUMBERtransaction_id,3,TRANSACTION ID back_reference_tran_id_number,4,BACK REFERENCE TRAN ID NUMBERback_reference_sched_name,5,BACK REFERENCE SCHED NAMEentity_type,6,ENTITY TYPElobbyist_registrant_organization_name,7,LOBBYIST/REGISTRANT ORGANIZATION NAMElobbyist_registrant_last_name,8,LOBBYIST/REGISTRANT LAST NAMElobbyist_registrant_first_name,9,LOBBYIST/REGISTRANT FIRST NAMElobbyist_registrant_middle_name,10,LOBBYIST/REGISTRANT MIDDLE NAMElobbyist_registrant_prefix,11,LOBBYIST/REGISTRANT PREFIXlobbyist_registrant_suffix,12,LOBBYIST/REGISTRANT SUFFIXlobbyist_registrant_street_1,13,LOBBYIST/REGISTRANT STREET 1lobbyist_registrant_street_2,14,LOBBYIST/REGISTRANT STREET 2lobbyist_registrant_city,15,LOBBYIST/REGISTRANT CITYlobbyist_registrant_state,16,LOBBYIST/REGISTRANT STATElobbyist_registrant_zip_code,17,LOBBYIST/REGISTRANT ZIPbundled_amount_period,18,BUNDLED AMOUNT PERIODbundled_amount_semi_annual,19,BUNDLED AMOUNT SEMI-ANNUALlobbyist_registrant_employer,20,LOBBYIST/REGISTRANT EMPLOYERlobbyist_registrant_occupation,21,LOBBYIST/REGISTRANT OCCUPATIONassociated_text_record,22,ASSOCIATED TEXT RECORDmemo_text,23,MEMO TEXT
canonical,^8.1|8.0,,^7.0|6.4,form_type,1,FORM TYPE,1,FORM TYPEfiler_committee_id_number,2,FILER COMMITTEE ID NUMBER,2,FILER COMMITTEE ID NUMBERtransaction_id,3,TRANSACTION ID ,3,TRANSACTION ID back_reference_tran_id_number,4,BACK REFERENCE TRAN ID NUMBER,4,BACK REFERENCE TRAN ID NUMBERback_reference_sched_name,5,BACK REFERENCE SCHED NAME,5,BACK REFERENCE SCHED NAMEentity_type,6,ENTITY TYPE,6,ENTITY TYPElobbyist_registrant_organization_name,7,LOBBYIST/REGISTRANT ORGANIZATION NAME,7,LOBBYIST/REGISTRANT ORGANIZATION NAMElobbyist_registrant_last_name,8,LOBBYIST/REGISTRANT LAST NAME,8,LOBBYIST/REGISTRANT LAST NAMElobbyist_registrant_first_name,9,LOBBYIST/REGISTRANT FIRST NAME,9,LOBBYIST/REGISTRANT FIRST NAMElobbyist_registrant_middle_name,10,LOBBYIST/REGISTRANT MIDDLE NAME,10,LOBBYIST/REGISTRANT MIDDLE NAMElobbyist_registrant_prefix,11,LOBBYIST/REGISTRANT PREFIX,11,LOBBYIST/REGISTRANT PREFIXlobbyist_registrant_suffix,12,LOBBYIST/REGISTRANT SUFFIX,12,LOBBYIST/REGISTRANT SUFFIXlobbyist_registrant_street_1,13,LOBBYIST/REGISTRANT STREET 1,13,LOBBYIST/REGISTRANT STREET 1lobbyist_registrant_street_2,14,LOBBYIST/REGISTRANT STREET 2,14,LOBBYIST/REGISTRANT STREET 2lobbyist_registrant_city,15,LOBBYIST/REGISTRANT CITY,15,LOBBYIST/REGISTRANT CITYlobbyist_registrant_state,16,LOBBYIST/REGISTRANT STATE,16,LOBBYIST/REGISTRANT STATElobbyist_registrant_zip_code,17,LOBBYIST/REGISTRANT ZIP,17,LOBBYIST/REGISTRANT ZIPelection_code,18,ELECTION CODE,18,ELECTION CODEelection_other_description,19,ELECTION OTHER DESCRIPTION ,19,ELECTION OTHER DESCRIPTION contribution_date,20,CONTRIBUTION DATE,20,CONTRIBUTION DATEbundled_amount_period,21,BUNDLED AMOUNT PERIOD,21,BUNDLED AMOUNT PERIODbundled_amount_semi_annual,22,BUNDLED AMOUNT SEMI-ANNUAL,22,BUNDLED AMOUNT SEMI-ANNUALcontribution_purpose_code,,,23,CONTRIBUTION PURPOSE CODEcontribution_purpose_descrip,23,CONTRIBUTION PURPOSE DESCRIP,24,CONTRIBUTION PURPOSE DESCRIPlobbyist_registrant_employer,24,LOBBYIST/REGISTRANT EMPLOYER,25,LOBBYIST/REGISTRANT EMPLOYERlobbyist_registrant_occupation,25,LOBBYIST/REGISTRANT OCCUPATION,26,LOBBYIST/REGISTRANT OCCUPATIONdonor_committee_fec_id,26,,27,DONOR COMMITTEE FEC IDdonor_committee_name,27,,28,DONOR COMMITTEE NAMEdonor_candidate_fec_id,28,,29,DONOR CANDIDATE FEC IDdonor_candidate_last_name,29,,30,DONOR CANDIDATE LAST NAMEdonor_candidate_first_name,30,,31,DONOR CANDIDATE FIRST NAMEdonor_candidate_middle_name,31,,32,DONOR CANDIDATE MIDDLE NAMEdonor_candidate_prefix,32,,33,DONOR CANDIDATE PREFIXdonor_candidate_suffix,33,,34,DONOR CANDIDATE SUFFIXdonor_candidate_office,34,,35,DONOR CANDIDATE OFFICEdonor_candidate_state,35,,36,DONOR CANDIDATE STATE donor_candidate_district,36,,37,DONOR CANDIDATE DISTRICTconduit_name,37,,38,CONDUIT NAMEconduit_street1,38,,39,CONDUIT STREET1conduit_street2,39,,40,CONDUIT STREET2conduit_city,40,,41,CONDUIT CITYconduit_state,41,,42,CONDUIT STATEconduit_zip_code,42,,43,CONDUIT ZIPassociated_text_record,43,ASSOCIATED TEXT RECORD,44,ASSOCIATED TEXT RECORDmemo_text,44,MEMO TEXT,45,MEMO TEXTreference_code,45,Reference to SI or SL system code that identifies the Account,46,Reference to SI or SL system code that identifies the Account
Expand Down

0 comments on commit 5e5b058

Please sign in to comment.