add netfile v2 data to database during import

Makefile

@@ -105,7 +105,7 @@ do-import-spreadsheets:
 
 import-data: 496 497 A-Contributions B1-Loans B2-Loans C-Contributions \
 	D-Expenditure E-Expenditure F-Expenses F461P5-Expenditure F465P3-Expenditure \
-	F496P3-Contributions G-Expenditure H-Loans I-Contributions Summary
+	F496P3-Contributions G-Expenditure H-Loans I-Contributions Summary elections_v2 committees_v2 a_contributions_v2
 	echo 'CREATE TABLE IF NOT EXISTS "calculations" (id SERIAL PRIMARY KEY, subject_id integer, subject_type varchar(30), name varchar(40), value jsonb);' | psql $(DATABASE_NAME)
 	./bin/remove_duplicate_transactions
 	./bin/make_view
@@ -120,6 +120,9 @@ reindex:
 496 497 A-Contributions B1-Loans B2-Loans C-Contributions D-Expenditure E-Expenditure F-Expenses F461P5-Expenditure F465P3-Expenditure F496P3-Contributions G-Expenditure H-Loans I-Contributions Summary:
 	DATABASE_NAME=$(DATABASE_NAME) ./bin/import-file $(CSV_PATH) $@
 
+elections_v2 committees_v2 a_contributions_v2:
+	DATABASE_NAME=$(DATABASE_NAME) ./bin/import-file $(CSV_PATH) $@ 0
+
 downloads/csv/candidates.csv:
 	mkdir -p downloads/csv downloads/raw
 	$(WGET) -O- \

bin/clean

@@ -18,4 +18,6 @@ cat <<-QUERY | psql ${database_name}
   DELETE FROM "$table_name"
   WHERE "Tran_Date" is NULL;
 QUERY
+else
+  echo
 fi

bin/import-file

@@ -2,10 +2,12 @@
 # Contains logic to import files regardless of how many there are.
 # If there's no file, don't do anything with the database.
 # If the table already exists in the database, don't try to re-create it.
+# The fix_pending parameter defaults to 1 if not set.  Set it to 0
+# to skip the section that fixes the pending Filer_ID
 #
 # Usage:
-#   bin/import-file [csv_path] [table]
-#   bin/import-file downloads/csv A-Contributions
+#   bin/import-file [csv_path] [table] [fix_pending]
+#   bin/import-file downloads/csv A-Contributions 1
 set -euo pipefail
 
 if [ -z "${DATABASE_NAME:-""}" ]; then
@@ -14,12 +16,13 @@ if [ -z "${DATABASE_NAME:-""}" ]; then
 fi
 
 if [ $# -eq 0 ]; then
-  echo "Usage: bin/import-file [csv_path] [table]"
+  echo "Usage: bin/import-file [csv_path] [table] [fix_pending]"
   exit 1
 fi
 
 csv_path=$1
 table_name=$2
+fix_pending=${3:-1}
 filename_glob=$csv_path'/*'${table_name}'.csv'
 table_exists=
 if psql disclosure-backend -c '\d "'${table_name}'"' >/dev/null 2>&1; then
@@ -45,9 +48,10 @@ if ls $filename_glob 2>/dev/null >/dev/null; then
     csvsql --db postgresql:///$DATABASE_NAME --tables $table_name --insert --no-inference ${table_exists:+--no-create}
   echo -n '  Removing empty Tran_Date... '
   ./bin/clean "$DATABASE_NAME" "$table_name"
-  echo
-  echo -n '  Fixing pending Filer_IDs... '
-  ./bin/fix-pending "$DATABASE_NAME" "$table_name"
+  if [ "$fix_pending" = "1" ]; then
+    echo -n '  Fixing pending Filer_IDs... '
+    ./bin/fix-pending "$DATABASE_NAME" "$table_name"
+  fi
 else
   echo 'Found no files to import'
 fi

dbschema/committees_v2.sql

@@ -0,0 +1,15 @@
+CREATE TABLE committees_v2 (
+	filer_nid DECIMAL NOT NULL, 
+	"Ballot_Measure_Election" VARCHAR, 
+	"Filer_ID" VARCHAR, 
+	"Filer_NamL" VARCHAR NOT NULL, 
+	"_Status" VARCHAR NOT NULL, 
+	"_Committee_Type" VARCHAR NOT NULL, 
+	"Ballot_Measure" VARCHAR, 
+	"Support_Or_Oppose" VARCHAR, 
+	candidate_controlled_id BOOLEAN, 
+	"Start_Date" DATE, 
+	"End_Date" BOOLEAN, 
+	data_warning BOOLEAN, 
+	"Make_Active" BOOLEAN
+);

dbschema/elections_v2.sql

@@ -0,0 +1,6 @@
+CREATE TABLE elections_v2 (
+	location VARCHAR NOT NULL, 
+	date DATE NOT NULL, 
+	name VARCHAR NOT NULL, 
+	title VARCHAR NOT NULL
+);

download/main.py

@@ -82,9 +82,9 @@ def main():
             'XRef_Match',
         ]).sample(n=20))
 
-    elections.df.to_csv('.local/elections.csv', index=False)
-    committees.df.to_csv('.local/committees.csv', index=False)
-    a_contributions.df.to_csv('.local/a_contributions.csv', index=False)
+    elections.df.to_csv('downloads/csv/elections_v2.csv', index=False)
+    committees.df.to_csv('downloads/csv/committees_v2.csv', index=False)
+    a_contributions.df.to_csv('downloads/csv/a_contributions_v2.csv', index=False)
 
     '''
     with engine.connect() as conn: