From b8581db803a7b3dd100ca255a5da7c987e9e349b Mon Sep 17 00:00:00 2001 From: Charlie Garrett Date: Sat, 13 Jun 2020 17:31:30 -0700 Subject: [PATCH] Add test for internal database errors --- commcare_export/cli.py | 5 ++++ tests/013_ConflictingTypes.xlsx | Bin 0 -> 7867 bytes tests/test_cli.py | 51 +++++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/013_ConflictingTypes.xlsx diff --git a/commcare_export/cli.py b/commcare_export/cli.py index 2c44a5c2..c65c8cf0 100644 --- a/commcare_export/cli.py +++ b/commcare_export/cli.py @@ -10,6 +10,7 @@ import dateutil.parser import requests +import sqlalchemy from six.moves import input from commcare_export import excel_query @@ -268,6 +269,10 @@ def evaluate_query(env, query): print(e.message) print('Try increasing --batch-size to overcome the error') return EXIT_STATUS_ERROR + except (sqlalchemy.exc.DataError, sqlalchemy.exc.InternalError, + sqlalchemy.exc.ProgrammingError) as e: + print('Stopping because of database error:\n', e) + return EXIT_STATUS_ERROR except KeyboardInterrupt: print('\nExport aborted', file=sys.stderr) return EXIT_STATUS_ERROR diff --git a/tests/013_ConflictingTypes.xlsx b/tests/013_ConflictingTypes.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..a54a4b48e5f5499476f4ce5d90b51077d52e367b GIT binary patch literal 7867 zcmd^E2UL?^vZqSuND&c4nt*_`5PFfSbQGjW3B3~_AYG6mAV`fO9T5c7D4|M?^db%Q(G2s9Y_nZMsNQ)>H**XK7=K8eDjnfROj8+4BqBOJT zA*?A$Z)ghMhrZR4bO|P8_vlf=t@S~CtGCr`8A7<1%IQgim?|-r+%#*R6%Y8OBSx(w zJ2fnl+abG^ET9+t4aHn5!B)lPCvV+ry`g2SAUi8g^<}7%`XfhCs_av8l%}6C>iLrT z*GquR;I^7P-VaO7#F@nN8wa|hQPm)dE#(}x9cj*}Q?@&#TaL5vr(`d$Dj4h$c7#O$ zbiz}GXn0KV(cV5tKTYAs~Xf5Q^sqBggI#VuZF*J zCO#Y9(^VtAJwH_@&@EZTFX&!oHT|Nug6kF7K${FeX6q73N=QIJMod6p@PCQY-xKQQ zm&00@k5kTQxMipmJ_prUlahcNo}}1F&@%{ghw*`XC!okb+jwhO}R)@%ctr^ z)b~w`7%pQ~i!7oGnMB^qk{_pI4~-Zh+z>XjsYZ z%Z`O?UGB^2sp&mL&R56D_GR2i$i|E%SbwNNNuj~=qLq9vrqz(u-_FXt|M0^;cEk?T znfzi+M6uUfUUB)rj{k5*#|FyQGHn#Nv*0x6q1&J5SXb?mOCPu$mz{$y+Pitv4+^{` zx9d@j+y68=f*J3|3}L2d4|Dq!;d%QZUR$7BjTXrIU`$W@;@rn|mw}CmJpb^6I$7!0 zmY!*&VOf1MR*>p2niw+X3aO#V=Y2g3BVnJIvwBv_LhcU@VZwx!uA%ro3LaCv%=UrJ{F6Kg}CVio;J+w zDAe~{N{Shd;lvuc!8~({6p%Ukkvmr@>z!AX8ZG%ti?SBSyIpZ})>W+ayUFucC;eWX z8``w@R}Y&qe&Js=S+KnFcFPF{o8R&uQZ(B8=Agda@L}O06J{gMWQ)mrbgjCvn4KGY z+P`3^0ykupw8exoQ=eYOWHFVp3vLGM4BG{L!EBd3*(F{WC-1S+-eSUUpQ2w?hRrSI~X{XTiJeb$Q*y&0yV` zI`8bud10W8BUX{KxH{}Yrp{aYI9@}EjK|%KKUhyQbw=!;@fw0Mo^>1P7P*RFhka*d z@(xL46m?7J7P*M)!dx zO4rd@TouNW**if)Qr^O_T0r@_A9s4OtM4B(8U1m-jMeg+NEL#(c z44?)eO zOQ#{4RJN7}z{$dK9EfuyC!(4n=+fzkSXHdm0B}G!3nzk`Bs!`ol`fs0$Xvx*2LLAz zm*GV4k({{G^q4N4ktj>W+6Vxr2sh(I2$DqKX)32nKSeaDVr>C{Q-%j}B1A||L^rk2 zrJp8ZRkgMUz^TGBI1xZKzvwO$-8c)EoT|N3ESf4MhqL_m%*Qifvr-%(oo4~rDKn9? zQk>hCfKb+mnV4BAu73pY?;rVJ`Q_al&N3j6UvwKPd5nc#PSw`w3YzL4WTq-%W|QQ% zfwVK20b-|*NebIOvlBZspBXzv%@i%K465tmc_ zr!&h-%Vy(+wgo7Wy)FLmiY6AE7A0~xqO@u@{vzUEI`ZGS8iO$w3OQ99Cw?^5ot%6G zkkK!?1@-S_#+5F8>5<*3OGF|HO3~YAwdJ51$eM!E)h|7A<^R6v82LrNK^cs*2*{~A zIF+EO9^~Yg1OGFa&(^r-ST(ox=#dYi*3QpmX@+z{=(AJS_~ux(5dZ8sCgkLo0VVvR zJ5U2-ENA3Y?VOy^RR8JB%r%QebCaFv2gtxDYgUVsCgs4&wXvpD;Ma>nDs*}?fG*wPc{(l|wgj}n&(`htSWR492NaYvZj1nDV`LAPg>L979JC}RL z-8Wq8GIDd?!#~CMDM2JR7Hk{$NBX|4px1^*CULg6)`M!t6W=GwOrW+MYv1=)Z|JXD zS)huT^+jGfdP@2$P3O4Z3`^T(&RXc(pN$C&%kqGD(xg4dxtC>3=d6EQcdVU&7j5)e zcH_P-6_2>@KyG;<8!XX53N^XHkm9Wvup+y>Twr66zvyde1$cc#rxaG=0R7Pxv@Z&7 zP%v$*5j5Y2dV?`GwL!0fmz@rYkCMP-8Rv}G)C2@ipZ_iiwkPzU$psi5%D zdqTf9MT;oiiCKn#7|~$ZyF0NGD;iOdC+yZ&ifG;0c`$D$vl1T9wk-7ahqQu0w7?41 zJJK?O`5#F|%ik<^)6k=)yc>y6G2BR~sNk{52k16F)041!BjuePV$V``wIw}uhVtad zl@r|lx$b&F%v444(aHfv%xqWO<7@?6iV3j~yKh|NtgIt>cQN;%%yLFQjz-Utdi>k` zodLqNT2o#zs{1@%2NR90sZOWVEbo@aVHu8O@Sb@LQ#3mFMmli^9?gh(H1or8B(|%KhUP8r&(#D zG4)n2VOITFCf4_?MN0~VIJXC^Rusd?FW03M$_G-?EauuiZ;+p^I^4|O zU@@Y(QtfwB80B}q5Dwo^54O%)uX_l9-7bwpW=DycJ#2H2YO~!Gr%$|f;Z!pq{>cU- zWlvX}tOUUWrnW5sjZ0CnA4+-+7s61h?LdX|j{f7TU0;dTY%^AqMXSvpMwL+;GR`4! zEwwZe7^C_EiBmnVx~f18to+NHO_8!bK=2*%t7Xw;b;CwLK_mHQdH|j7u zbhd74K+a zOJn?DX+F+2PzQTMU+7T@@2GO;k=(4;DoUsJ`gRy%Q;$vBh*V~kaG0BUme?3IXc{-# zT(7*a=AFslktpY>0leWaUHo>>ASJyvGNy=Bl?{3hH{ATW6BA0S!C2cV2E4hhO+*&< zS#5ZjUz;p)>6`nDsBn5M7~cmiBvi8mE5s*g$cc8~7?haktezT=N@?-|ZGjcX%klAeNpF;D=1Z?u?$M~<= z_g=9}nKG6U?y(PvFgI+s`#=kEd*i<*hcW2S(lMK0XQ zwcXE{&AtqxxO*sN#%C;$_pdI2^!#2h?S$HC&0S`NlrRfryE`;UQ558*`LRwX%^}i| zGTM+GdMLB273?h6zqmhv^NOB&wMEaeoY@X}$g+ZGH74OLGTnTja?UwCam2 z(LqFTQ-0bssY&}2U+LtoOVBSo-&B_6IPd%1GuR%gHKQx%jE|I*wF)XFA3B9bdnk&Y zf^}5!79Rx0J2t#`cpf-Sn}q9_Sw+{Q7s3jK7R)xus_2tE#lYHKccV5$hmxpHUMQ}) z6=Veo3O(K7Cd*A)F`|5q3eiSUxR7`dn;P6$X2R&6wrw%gwl)%XGBSyd^g;C_#S(!5 zB%{Wr?)F&u9_i5@w2xvPFXQ)s=eNc3Ge7p=M{sm}C%%VAFH)6V@%~F4za6pTqvfz` zT6eQ``Ryl*$mj;g#LLce>Q3xz=19)U(y1_8-92pII~;JX*gq453BTaqELajpPZGr7 z{@I*#E#~1aA6-V}t2$bCrhDl&<})5vWutAt@tb0lG#ImR?zsMIj`LmvwSbcfGZLKC znWja?8Xk#t>a_wrmoq-G7=Z2L8m}jBRug%j=04z)lhAeYV(xXhq;Dd?gAsy0(z}=% z0S=UW9?^Qq%E|O%cd`o_N6i;ofxQ0Os&aZUYBPaeBtTE>m1ncO5cvknpqQN64sizj zoRMNtdsqB9ld$iRWg1Jf;SydRwM4(uT1xyEKg9LF5u`FrgeV>906v0r+`Q&xb1xqu z#_LYa{-|1&^HQUM>iW32q>@K zxX4$|Z{1FCv+9C~bay&LMX9EV-Zt{KqK5xR`4V;~7ICQaP!YFMPr8O@4b1ia37l~M z*8A722Dj4TZ;3(*D6LnP*3V_tK@0-8Zsxu1IL8FO%cJcnN@2=-z@s!B+aZ+S^gK08 z-n0=PFp>cZJ@Yxr@eASqtw%o!Ak9@hA-)a}U$9Z2mxGVxk&#vz_8gsRdqfAEUztD< z8IGvT&8B(4mHV`&XkEJzk-(|ZnF`GtJkL17hpif_JLQ1ORmB@Q740V&GG+zEE_6P= zR2m@}YIZlNedCKBpyUYCT?9zbsZpl!1>szkfvMt zlf#WS09R7&`%?3m7e>ua-vaNO(}-m$+ETk^#m9uce)5u}%{d{W7s&&=`@yBt+Oxv4 zN9m(SgQ6Hv_VfjYiB&2P?;!RmLBX&r!i=8IA>JFC77YWDmvYAskSgeZ*>=-RGD4 zgX3H8_;<-X0ESVrr7qnLZE$`rlSXszaHa27G(vW%XVs}6`CaNBv6TiOyu?{ZjuecL zh>_suW#I9&=10Es# zHE#K}>v2zdbY}geZ>Rn$j=!8>zji+ERF0hOPYS?q5PsZ0alOCd9Jkmb7xa^!vHZay k{o3)k(*05P5AgH(8|H1GO?u)8odW;M#JfUDwxhTI0XJE#C;$Ke literal 0 HcmV?d00001 diff --git a/tests/test_cli.py b/tests/test_cli.py index 1e085154..c31adfc6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import csv342 as csv import os +import re import unittest from argparse import Namespace from copy import copy @@ -10,7 +11,7 @@ from mock import mock from commcare_export.checkpoint import CheckpointManager -from commcare_export.cli import CLI_ARGS, main_with_args +from commcare_export.cli import CLI_ARGS, EXIT_STATUS_ERROR, main_with_args from commcare_export.commcare_hq_client import MockCommCareHqClient from commcare_export.specs import TableSpec from commcare_export.writers import JValueTableWriter, SqlTableWriter @@ -395,3 +396,51 @@ def _check_checkpoints(self, caplog, expected): else: message += '✓ {}: {} in {}\n'.format(i, items[0], items[1]) assert not fail, 'Checkpoint comparison failed:\n' + message + + +# Conflicting types for 'count' will cause errors when inserting into database. +CONFLICTING_TYPES_CLIENT = MockCommCareHqClient({ + 'form': [ + ( + {'limit': DEFAULT_BATCH_SIZE, 'order_by': ['server_modified_on', 'received_on']}, + [ + {'id': 1, 'form': {'name': 'n1', 'count': 10}}, + {'id': 2, 'form': {'name': 'f2', 'count': 'abc'}} + ] + ), + ], +}) + +@pytest.fixture(scope='class') +def strict_writer(db_params): + return SqlTableWriter(db_params['url'], poolclass=sqlalchemy.pool.NullPool, strict_types=True) + +@pytest.fixture(scope='class') +def all_db_checkpoint_manager(db_params): + cm = CheckpointManager(db_params['url'], 'query', '123', 'test', 'hq', poolclass=sqlalchemy.pool.NullPool) + cm.create_checkpoint_table() + return cm + +@pytest.mark.dbtest +class TestCLIWithDatabaseErrors(object): + def test_cli_database_error(self, strict_writer, all_db_checkpoint_manager, capfd): + args = make_args( + query='tests/013_ConflictingTypes.xlsx', + output_format='sql' + ) + # set this so that it get's written to the checkpoints + checkpoint_manager.query = args.query + + api_client_patch = mock.patch('commcare_export.cli._get_api_client', + return_value=CONFLICTING_TYPES_CLIENT) + # have to mock these to override the pool class otherwise they hold the db connection open + strict_writer_patch = mock.patch('commcare_export.cli._get_writer', + return_value=strict_writer) + checkpoint_patch = mock.patch('commcare_export.cli._get_checkpoint_manager', + return_value=all_db_checkpoint_manager) + with api_client_patch, strict_writer_patch, checkpoint_patch: + assert main_with_args(args) == EXIT_STATUS_ERROR + out, err = capfd.readouterr() + + expected_re = re.compile('Stopping because of database error') + assert re.search(expected_re, out)