larger dataset imported with tab_cfg.txt

Need:
- testing spec files on variables imported
- inspect specific parts of database (e.g. ordered varlist)

data/2015/ind10/tab_cfg.txt

@@ -1,12 +1,12 @@
 - 3.5. Индекс потребительских цен
 - 4. Социальная сфера 
 - spec_cpi.txt
-#---
-#- 2.4.1. Кредиторская задолженность
-#- 2.4.2. Дебиторская задолженность
-#- spec_overdue.txt
-#---
-#- 2.2. Сальдированный финансовый результат
-#- Убыточные организации
-#- spec_profit.txt
+---
+- 2.4.1. Кредиторская задолженность
+- 2.4.2. Дебиторская задолженность
+- spec_overdue.txt
+---
+- 2.2. Сальдированный финансовый результат
+- Убыточные организации
+- spec_profit.txt
 

kep/importer/parser/stream.py

@@ -58,6 +58,8 @@ def yield_flat_tuples(row_tuple):
 #  Read rows by annual, qtr, month section 
 #------------------------------------------------------------------------------
 
+# split* functions return (year, annual value, quarterly values list, monthly values list) 
+
 def split_row_by_periods(row):           
     """Year A Q Q Q Q M*12"""
     return int(row[0]), row[1], row[2:2+4], row[2+4:2+4+12]
@@ -70,9 +72,15 @@ def split_row_by_months_and_annual(row):
     """Year A M*12"""
     return int(row[0]), row[1], None, row[2:12+2]
 
+def split_row_by_accum_qtrs(row):         
+    """Year AccumQ1 AccumH1 Accum9mo"""
+    # WARNING: may interfere with other qtr readers  
+    return int(row[0]), row[1], row[2:2+3] + [row[1]], None    
+
 ROW_LENGTH_TO_FUNC = { 1+1+4+12: split_row_by_periods, 
                            1+12: split_row_by_months,
-                         1+1+12: split_row_by_months_and_annual }
+                         1+1+12: split_row_by_months_and_annual,
+                            1+4: split_row_by_accum_qtrs  }
 
 def get_reader_func_by_row_length(row):
     return ROW_LENGTH_TO_FUNC[len(row)]       

kep/query/plots.py

@@ -29,24 +29,21 @@
 
 def write_plots():
     """Write monthly graphs as *.png plots with markdown showcase file and as a PDF file."""
-    print("Reading stored data...")
+    #print("Reading stored data...")
     df = get_dfm()
 
     # png images    
-    print("Writing .png images...")
+    print("Plotting: writing .png images...")
     write_png_pictures(df)
 
     # md file
-    print("Writing markdown (.md) showcase file with images...")
+    print("Plotting: writing markdown (.md) showcase file with images...")
     generate_md(df)
 
     # PDF output
-    print("Writing PDF file...")
+    print("Plotting: writing PDF file...")
     write_monthly_pdf()
 
-    print("Done.")
-
-
 def write_monthly_pdf():
     df = get_dfm()
     save_plots_as_pdf(df, PDF_FILE)

kep/query/save.py

@@ -121,8 +121,9 @@ def get_dfs():
 
 def db_dump():
     dfa, dfq, dfm = get_reshaped_dfs()
+    print("Writing database to CSV and XLS(X) files...")
     write_to_xl(dfa, dfq, dfm)
-    write_to_csv(dfa, dfq, dfm)
+    write_to_csv(dfa, dfq, dfm)       
 
 if __name__ == "__main__":
     # repeat db_dump() here

kep/test/pytest/_package/test_new_specfiles.py

@@ -0,0 +1,17 @@
+from kep.importer.csv2db import to_database
+from kep.inspection.var_check import inspect_db
+from kep.database.db import wipe_db_tables
+
+#to_database(raw_data_file, spec_file, cfg_file = None):
+#    lab_rows = get_labelled_rows(raw_data_file, spec_file, cfg_file)
+#    db_rows = stream_flat_data(lab_rows)
+#    stream_to_database(db_rows)
+
+folder = "C:\\Users\\Евгений\\Documents\\GitHub\\rosstat-kep-data\\data\\2015\\ind10\\"
+raw_data_file = folder + "tab.csv"
+spec_file = folder + "spec_profit.txt" # "spec_overdue.txt" #"spec_profit.txt" # "spec_cpi.txt"
+
+wipe_db_tables()
+to_database(raw_data_file, spec_file)
+# Inspection procedure
+inspect_db(folder) 

output/data_annual.txt

@@ -1,17 +1,17 @@
-year,CONSTR_bln_rub_fix,CONSTR_yoy,CPI_ALCOHOL_rog,CPI_FOOD_rog,CPI_NONFOOD_rog,CPI_SERVICES_rog,CPI_rog,IND_PROD_yoy,I_bln_rub,I_yoy,PROD_AUTO_BUS_units,PROD_AUTO_PSGR_th,PROD_AUTO_TRUCKS_AND_CHASSIS_th,PROD_AUTO_TRUCKS_th,PROD_BYCYCLES_th,PROD_E_TWh,PROD_RAILWAY_CARGO_WAGONS_units,PROD_RAILWAY_PSGR_WAGONS_units,RETAIL_SALES_bln_rub,RETAIL_SALES_yoy,RUR_EUR_eop,RUR_USD_eop,SOC_EMPLOYED_mln,SOC_EMPLOYED_yoy,SOC_PENSION_rub,SOC_UNEMPLOYED_bln,SOC_UNEMPLOYMENT_percent,SOC_WAGE_rub,SOC_WAGE_yoy,TRANS_COM_bln_t_km,TRANS_COM_yoy,TRANS_RAILLOAD_mln_t,TRANS_RAILLOAD_yoy,TRANS_bln_t_km,TRANS_yoy,USLUGI_bln_rub,USLUGI_yoy
-1999,307.8,103.9,143.2,135.0,139.2,134.0,136.5,,670.4,105.3,50021.0,954.0,,176.1,425.0,846.0,4140.0,716.0,1797.4,94.2,27.23,27.0,62.9,106.3,449.0,9.4,13.0,1523.0,144.8,3372.0,105.8,947.0,113.5,3461.0,,443.7,106.6
-2000,503.8,113.5,125.0,117.1,118.5,133.7,120.2,,1165.2,117.4,54001.0,969.0,,184.5,639.0,878.0,4029.0,802.0,2352.3,109.0,26.14,28.16,65.0,103.4,694.3,7.7,10.5,2223.0,146.0,3542.0,105.0,1047.0,110.5,3638.0,105.1,602.8,104.7
-2001,703.8,110.4,112.6,117.8,112.7,136.9,118.6,,1504.7,111.7,56506.0,1022.0,,172.6,514.0,891.0,6518.0,859.0,3070.0,111.0,26.49,30.14,65.2,100.8,1023.5,6.4,9.0,3240.0,145.7,3651.0,103.1,1057.0,101.0,3754.0,103.2,811.7,101.6
-2002,831.0,102.9,108.9,111.3,110.9,136.2,115.1,103.1,1762.4,102.9,66692.0,981.0,,172.6,509.0,891.0,10740.0,867.0,3765.4,109.3,33.11,31.78,66.7,102.2,1378.5,5.8,8.0,4360.0,134.6,3868.0,106.0,1084.0,102.5,3976.0,105.9,1088.0,103.7
-2003,1042.7,112.8,109.9,110.2,109.2,122.3,112.0,108.9,2186.4,112.7,76228.0,1012.0,,192.9,415.0,916.0,26973.0,1025.0,4529.7,108.8,36.82,29.45,66.3,99.6,1637.0,5.9,8.2,5499.0,126.1,4171.0,107.8,1161.0,107.1,4282.0,107.7,1430.7,106.6
-2004,1313.6,110.1,108.7,113.0,107.4,117.7,111.7,108.0,2865.0,116.8,75724.0,1110.0,,200.2,344.0,932.0,35340.0,1211.0,5642.5,113.3,37.81,27.75,67.3,101.5,1914.5,5.6,7.7,6740.0,122.6,4441.0,106.5,1221.0,105.2,4558.0,106.4,1789.7,108.4
-2005,1754.4,113.2,107.6,109.9,106.4,121.0,110.9,105.1,3611.1,110.2,78235.0,1069.0,,205.1,236.0,953.0,35160.0,1221.0,7041.5,112.8,34.19,28.78,68.3,101.4,2364.0,5.3,7.1,8555.0,126.9,4550.0,102.4,1273.0,104.3,4675.0,102.5,2264.7,107.5
-2006,2350.8,118.1,110.1,108.4,106.0,113.9,109.0,106.3,4730.0,117.8,88655.0,1178.0,,245.3,1048.0,996.0,33656.0,1557.0,8711.9,114.1,34.7,26.33,69.2,101.3,2726.1,5.2,7.0,10634.0,124.3,4675.0,102.7,1311.0,102.9,4800.0,102.7,2798.4,107.9
-2007,3293.3,118.2,107.7,117.1,106.5,113.3,111.9,106.8,6716.2,123.8,88862.0,1294.0,,285.0,1513.0,1015.0,38575.0,1823.0,10869.0,116.1,35.93,24.55,70.7,102.2,3115.5,4.5,6.0,13593.0,127.8,4788.0,102.4,1344.0,102.5,4915.0,102.4,3407.0,107.1
-2008,4528.1,112.8,110.9,117.6,108.0,115.9,113.3,100.6,8781.6,109.5,66477.0,1470.0,,256.0,1435.0,1040.0,42681.0,2143.0,13944.2,113.7,41.44,29.38,71.0,100.3,4198.6,4.7,6.2,17290.0,127.2,4820.0,100.7,1304.0,97.0,4948.0,100.7,4079.8,104.8
-2009,3998.3,86.8,108.9,105.5,109.7,111.6,108.8,89.3,7976.0,86.5,35483.0,600.0,92.7,91.4,388.0,992.0,23584.0,1384.0,14599.2,94.9,43.39,30.24,69.5,97.9,5191.1,6.2,8.2,18638.0,107.8,4344.0,90.1,1108.0,85.0,4446.0,89.9,4420.0,95.8
-2010,4454.1,105.0,108.3,113.7,105.0,108.1,108.8,107.3,9152.1,106.3,40944.0,1210.0,155.5,,1169.0,1038.0,50472.0,1234.0,16512.0,106.5,40.33,30.48,69.9,100.7,7476.3,5.5,7.3,20952.0,112.4,4645.0,106.9,1206.0,108.8,4752.0,106.9,4851.0,101.5
-2011,5140.3,105.1,108.4,103.2,106.7,108.7,106.1,105.0,11035.7,110.8,44244.0,1740.0,207.0,,1133.0,1052.0,62856.0,1205.0,19104.3,107.1,41.67,32.2,70.9,101.3,8202.9,4.9,6.5,23369.0,111.5,4799.0,103.3,1242.0,103.0,4914.0,103.4,5424.8,103.0
-2012,5714.1,102.5,112.1,106.7,105.2,107.3,106.6,103.4,12586.1,106.8,57961.0,1964.0,212.0,,1674.0,1064.0,71697.0,880.0,21394.5,106.3,40.23,30.37,71.5,101.0,9040.5,4.1,5.5,26629.0,113.9,4934.0,102.8,1272.0,102.4,5055.0,102.9,5926.2,103.7
-2013,6019.5,100.1,114.6,106.1,104.5,108.0,106.5,100.4,13450.3,100.8,53237.0,1924.0,207.4,,2014.0,1051.0,60121.0,760.0,23685.9,103.9,44.97,32.73,71.4,99.8,9917.5,4.1,5.5,29792.0,111.9,4958.0,100.5,1237.0,97.2,5084.0,100.6,6512.3,102.1
-2014,6125.2,97.7,113.7,115.7,108.1,110.5,111.4,101.7,13527.7,97.3,43208.0,1740.0,154.0,,1763.0,1059.0,54519.0,853.0,26356.2,102.7,68.34,56.26,71.5,100.2,10786.1,3.9,5.2,32495.0,109.1,4955.0,99.9,1231.0,99.2,5080.0,99.9,7302.3,101.3
+year,CONSTR_bln_rub_fix,CONSTR_yoy,CPI_ALCOHOL_rog,CPI_FOOD_rog,CPI_NONFOOD_rog,CPI_SERVICES_rog,CPI_rog,IND_PROD_yoy,I_bln_rub,I_yoy,NONFINANCIALS_PROFIT_CONSTRUCTION_bln_rub,NONFINANCIALS_PROFIT_MANUF_bln_rub,NONFINANCIALS_PROFIT_MINING_bln_rub,NONFINANCIALS_PROFIT_POWER_GAS_WATER_bln_rub,NONFINANCIALS_PROFIT_TRANS_COMM_bln_rub,PROD_AUTO_BUS_units,PROD_AUTO_PSGR_th,PROD_AUTO_TRUCKS_AND_CHASSIS_th,PROD_AUTO_TRUCKS_th,PROD_BYCYCLES_th,PROD_E_TWh,PROD_RAILWAY_CARGO_WAGONS_units,PROD_RAILWAY_PSGR_WAGONS_units,RETAIL_SALES_bln_rub,RETAIL_SALES_yoy,RUR_EUR_eop,RUR_USD_eop,SOC_EMPLOYED_mln,SOC_EMPLOYED_yoy,SOC_PENSION_rub,SOC_UNEMPLOYED_bln,SOC_UNEMPLOYMENT_percent,SOC_WAGE_rub,SOC_WAGE_yoy,TRANS_COM_bln_t_km,TRANS_COM_yoy,TRANS_RAILLOAD_mln_t,TRANS_RAILLOAD_yoy,TRANS_bln_t_km,TRANS_yoy,USLUGI_bln_rub,USLUGI_yoy
+1999,307.8,103.9,143.2,135.0,139.2,134.0,136.5,,670.4,105.3,14822.0,137998.0,109148.0,8241.0,97060.0,50021.0,954.0,,176.1,425.0,846.0,4140.0,716.0,1797.4,94.2,27.23,27.0,62.9,106.3,449.0,9.4,13.0,1523.0,144.8,3372.0,105.8,947.0,113.5,3461.0,,443.7,106.6
+2000,503.8,113.5,125.0,117.1,118.5,133.7,120.2,,1165.2,117.4,30359.0,266128.0,188810.0,21280.0,171970.0,54001.0,969.0,,184.5,639.0,878.0,4029.0,802.0,2352.3,109.0,26.14,28.16,65.0,103.4,694.3,7.7,10.5,2223.0,146.0,3542.0,105.0,1047.0,110.5,3638.0,105.1,602.8,104.7
+2001,703.8,110.4,112.6,117.8,112.7,136.9,118.6,,1504.7,111.7,32331.0,286115.0,206748.0,21945.0,187521.0,56506.0,1022.0,,172.6,514.0,891.0,6518.0,859.0,3070.0,111.0,26.49,30.14,65.2,100.8,1023.5,6.4,9.0,3240.0,145.7,3651.0,103.1,1057.0,101.0,3754.0,103.2,811.7,101.6
+2002,831.0,102.9,108.9,111.3,110.9,136.2,115.1,103.1,1762.4,102.9,22279.0,212476.0,174694.0,11039.0,154100.0,66692.0,981.0,,172.6,509.0,891.0,10740.0,867.0,3765.4,109.3,33.11,31.78,66.7,102.2,1378.5,5.8,8.0,4360.0,134.6,3868.0,106.0,1084.0,102.5,3976.0,105.9,1088.0,103.7
+2003,1042.7,112.8,109.9,110.2,109.2,122.3,112.0,108.9,2186.4,112.7,31922.0,339308.0,241097.0,22194.0,213945.0,76228.0,1012.0,,192.9,415.0,916.0,26973.0,1025.0,4529.7,108.8,36.82,29.45,66.3,99.6,1637.0,5.9,8.2,5499.0,126.1,4171.0,107.8,1161.0,107.1,4282.0,107.7,1430.7,106.6
+2004,1313.6,110.1,108.7,113.0,107.4,117.7,111.7,108.0,2865.0,116.8,33047.0,601635.0,425039.0,82676.0,263439.0,75724.0,1110.0,,200.2,344.0,932.0,35340.0,1211.0,5642.5,113.3,37.81,27.75,67.3,101.5,1914.5,5.6,7.7,6740.0,122.6,4441.0,106.5,1221.0,105.2,4558.0,106.4,1789.7,108.4
+2005,1754.4,113.2,107.6,109.9,106.4,121.0,110.9,105.1,3611.1,110.2,39804.0,943674.0,804198.0,117805.0,326903.0,78235.0,1069.0,,205.1,236.0,953.0,35160.0,1221.0,7041.5,112.8,34.19,28.78,68.3,101.4,2364.0,5.3,7.1,8555.0,126.9,4550.0,102.4,1273.0,104.3,4675.0,102.5,2264.7,107.5
+2006,2350.8,118.1,110.1,108.4,106.0,113.9,109.0,106.3,4730.0,117.8,66431.0,1483567.0,712883.0,90461.0,410994.0,88655.0,1178.0,,245.3,1048.0,996.0,33656.0,1557.0,8711.9,114.1,34.7,26.33,69.2,101.3,2726.1,5.2,7.0,10634.0,124.3,4675.0,102.7,1311.0,102.9,4800.0,102.7,2798.4,107.9
+2007,3293.3,118.2,107.7,117.1,106.5,113.3,111.9,106.8,6716.2,123.8,108823.0,1739360.0,906679.0,119690.0,659277.0,88862.0,1294.0,,285.0,1513.0,1015.0,38575.0,1823.0,10869.0,116.1,35.93,24.55,70.7,102.2,3115.5,4.5,6.0,13593.0,127.8,4788.0,102.4,1344.0,102.5,4915.0,102.4,3407.0,107.1
+2008,4528.1,112.8,110.9,117.6,108.0,115.9,113.3,100.6,8781.6,109.5,130421.0,1694862.0,872481.0,97862.0,573243.0,66477.0,1470.0,,256.0,1435.0,1040.0,42681.0,2143.0,13944.2,113.7,41.44,29.38,71.0,100.3,4198.6,4.7,6.2,17290.0,127.2,4820.0,100.7,1304.0,97.0,4948.0,100.7,4079.8,104.8
+2009,3998.3,86.8,108.9,105.5,109.7,111.6,108.8,89.3,7976.0,86.5,104861.0,969591.0,892471.0,220009.0,564638.0,35483.0,600.0,92.7,91.4,388.0,992.0,23584.0,1384.0,14599.2,94.9,43.39,30.24,69.5,97.9,5191.1,6.2,8.2,18638.0,107.8,4344.0,90.1,1108.0,85.0,4446.0,89.9,4420.0,95.8
+2010,4454.1,105.0,108.3,113.7,105.0,108.1,108.8,107.3,9152.1,106.3,98327.0,1595787.0,1377510.0,373766.0,719034.0,40944.0,1210.0,155.5,,1169.0,1038.0,50472.0,1234.0,16512.0,106.5,40.33,30.48,69.9,100.7,7476.3,5.5,7.3,20952.0,112.4,4645.0,106.9,1206.0,108.8,4752.0,106.9,4851.0,101.5
+2011,5140.3,105.1,108.4,103.2,106.7,108.7,106.1,105.0,11035.7,110.8,137870.0,1914115.0,1981014.0,120558.0,768112.0,44244.0,1740.0,207.0,,1133.0,1052.0,62856.0,1205.0,19104.3,107.1,41.67,32.2,70.9,101.3,8202.9,4.9,6.5,23369.0,111.5,4799.0,103.3,1242.0,103.0,4914.0,103.4,5424.8,103.0
+2012,5714.1,102.5,112.1,106.7,105.2,107.3,106.6,103.4,12586.1,106.8,157476.0,2082317.0,1833460.0,201356.0,924919.0,57961.0,1964.0,212.0,,1674.0,1064.0,71697.0,880.0,21394.5,106.3,40.23,30.37,71.5,101.0,9040.5,4.1,5.5,26629.0,113.9,4934.0,102.8,1272.0,102.4,5055.0,102.9,5926.2,103.7
+2013,6019.5,100.1,114.6,106.1,104.5,108.0,106.5,100.4,13450.3,100.8,120979.0,1544702.0,1707119.0,129309.0,781415.0,53237.0,1924.0,207.4,,2014.0,1051.0,60121.0,760.0,23685.9,103.9,44.97,32.73,71.4,99.8,9917.5,4.1,5.5,29792.0,111.9,4958.0,100.5,1237.0,97.2,5084.0,100.6,6512.3,102.1
+2014,6125.2,97.7,113.7,115.7,108.1,110.5,111.4,101.7,13527.7,97.3,91446.0,877139.0,2648591.0,164130.0,307569.0,43208.0,1740.0,154.0,,1763.0,1059.0,54519.0,853.0,26356.2,102.7,68.34,56.26,71.5,100.2,10786.1,3.9,5.2,32495.0,109.1,4955.0,99.9,1231.0,99.2,5080.0,99.9,7302.3,101.3