From 5c1f76799608786ada9fca366f9dca7842d98862 Mon Sep 17 00:00:00 2001 From: Andrew Paroz Date: Thu, 28 Mar 2024 12:51:06 +1000 Subject: [PATCH 1/3] Add column check when reading excel file --- Models/PostSimulationTools/ExcelInput.cs | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Models/PostSimulationTools/ExcelInput.cs b/Models/PostSimulationTools/ExcelInput.cs index e9a872dac6..4ff2dc60e0 100644 --- a/Models/PostSimulationTools/ExcelInput.cs +++ b/Models/PostSimulationTools/ExcelInput.cs @@ -130,6 +130,40 @@ public void Run() { if (SheetNames.Any(str => string.Equals(str.Trim(), table.TableName, StringComparison.InvariantCultureIgnoreCase))) { + //Check if any columns that only contain dates are being read in as strings (and won't graph properly because of it) + List replaceColumns = new List(); + foreach (DataColumn column in table.Columns) + { + if (column.DataType == typeof(string)) { + bool isDate = true; + int count = 0; + while(isDate && count < table.Rows.Count) { + if (DateUtilities.ValidateDateString(table.Rows[count][column.ColumnName].ToString()) == null) { + isDate = false; + } + count += 1; + } + if (isDate) { + replaceColumns.Add(column.ColumnName); + } + } + } + foreach (string name in replaceColumns) + { + DataColumn column = table.Columns[name]; + int ordinal = column.Ordinal; + + DataColumn newColumn = new DataColumn("NewColumn"+name, typeof(DateTime)); + table.Columns.Add(newColumn); + newColumn.SetOrdinal(ordinal); + + foreach (DataRow row in table.Rows) + row[newColumn.ColumnName] = DateUtilities.GetDate(row[name].ToString()); + + table.Columns.Remove(name); + newColumn.ColumnName = name; + } + TruncateDates(table); // Don't delete previous data existing in this table. Doing so would From b66c4a3fbcb85d93f22865364a6c9142ccb66f47 Mon Sep 17 00:00:00 2001 From: Andrew Paroz Date: Thu, 28 Mar 2024 14:20:32 +1000 Subject: [PATCH 2/3] Only convert full dates, not partials --- APSIM.Shared/Utilities/DateUtilities.cs | 73 ++++++++++++++++-------- Models/PostSimulationTools/ExcelInput.cs | 2 +- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/APSIM.Shared/Utilities/DateUtilities.cs b/APSIM.Shared/Utilities/DateUtilities.cs index 97869d0692..6339d7ff4c 100644 --- a/APSIM.Shared/Utilities/DateUtilities.cs +++ b/APSIM.Shared/Utilities/DateUtilities.cs @@ -345,30 +345,18 @@ public static bool IsEndOfYear(DateTime date) /// Return null if not valid, otherwise it returns a string with the valid dd-MMM string or a valid date as a string (yyyy-mm-dd) public static string ValidateDateString(string dateStr) { - DateAsParts parts; - parts = ParseDateString(dateStr); - if (parts.parseError) - return null; - - DateTime date; - try - { - date = GetDate(parts); - } - catch - { - return null; - } + return Validate(dateStr, true); + } - if (parts.yearWasMissing) - { - //for consistency, return it as 'Title' case (ie, 01-Jan, not 1-jan) - return date.ToString(DEFAULT_FORMAT_DAY_MONTH, CultureInfo.InvariantCulture); - } - else - { - return date.ToString(DEFAULT_FORMAT_DAY_MONTH_YEAR, CultureInfo.InvariantCulture); - } + /// + /// Takes in a string and checks to see if it is in the correct format for either a full date with year, month and date (in any recognised date format). + /// Will return null if not valid or if only a day and month was provided + /// + /// + /// Return null if not valid, otherwise it returns a string with the valid string (yyyy-mm-dd) + public static string ValidateDateStringWithYear(string dateStr) + { + return Validate(dateStr, false); } /// @@ -423,6 +411,45 @@ private static DateTime GetDate(DateAsParts parts) return GetDate(parts.day, parts.month, parts.year); } + /// + /// Checks if a string is formatted to be a date, returns null if it can't be a date, or a formatted date string if it can. + /// + /// The given string to be checked + /// If a day-month is allowed or if it must be a full date + /// A formatted date as a string + private static string Validate(string input, bool allowPartialDate) + { + DateAsParts parts; + parts = ParseDateString(input); + if (parts.parseError) + return null; + + DateTime date; + try + { + date = GetDate(parts); + } + catch + { + return null; + } + + if (parts.yearWasMissing) { + if (allowPartialDate) { + //for consistency, return it as 'Title' case (ie, 01-Jan, not 1-jan) + return date.ToString(DEFAULT_FORMAT_DAY_MONTH, CultureInfo.InvariantCulture); + } + else + { + return null; + } + } + else + { + return date.ToString(DEFAULT_FORMAT_DAY_MONTH_YEAR, CultureInfo.InvariantCulture); + } + } + /// /// Convert any valid date string into a DateTime objects. /// Valid seprators are: / - , . _ diff --git a/Models/PostSimulationTools/ExcelInput.cs b/Models/PostSimulationTools/ExcelInput.cs index 4ff2dc60e0..a238eff70e 100644 --- a/Models/PostSimulationTools/ExcelInput.cs +++ b/Models/PostSimulationTools/ExcelInput.cs @@ -138,7 +138,7 @@ public void Run() bool isDate = true; int count = 0; while(isDate && count < table.Rows.Count) { - if (DateUtilities.ValidateDateString(table.Rows[count][column.ColumnName].ToString()) == null) { + if (DateUtilities.ValidateDateStringWithYear(table.Rows[count][column.ColumnName].ToString()) == null) { isDate = false; } count += 1; From a1b0f687be07ae46540a6a7996c99c0249208aea Mon Sep 17 00:00:00 2001 From: Andrew Paroz Date: Thu, 4 Apr 2024 09:46:16 +1000 Subject: [PATCH 3/3] Unit tests for this functionality --- .../PostSimulationTools/ExcelInputTests.cs | 61 ++++++++++++++++++ .../UnitTests/PostSimulationTools/Input.xlsx | Bin 0 -> 10024 bytes .../UtilityTests/DataUtilitiesTests.cs | 4 ++ 3 files changed, 65 insertions(+) create mode 100644 Tests/UnitTests/PostSimulationTools/ExcelInputTests.cs create mode 100644 Tests/UnitTests/PostSimulationTools/Input.xlsx diff --git a/Tests/UnitTests/PostSimulationTools/ExcelInputTests.cs b/Tests/UnitTests/PostSimulationTools/ExcelInputTests.cs new file mode 100644 index 0000000000..dcfdb6f09a --- /dev/null +++ b/Tests/UnitTests/PostSimulationTools/ExcelInputTests.cs @@ -0,0 +1,61 @@ +using APSIM.Shared.Utilities; +using Models.PostSimulationTools; +using Models.Storage; +using NUnit.Framework; +using System; +using System.Data; +using System.IO; +using System.Reflection; + +namespace UnitTests +{ + public class ExcelInputTests + { + private IDatabaseConnection database; + + /// Find and return the file name of SQLite runtime .dll + public static string FindSqlite3DLL() + { + string directory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string[] files = Directory.GetFiles(directory, "sqlite3.dll"); + if (files.Length == 1) + return files[0]; + + throw new Exception("Cannot find sqlite3 dll directory"); + } + + [Test] + public void LoadExcelInput() + { + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + + if (ProcessUtilities.CurrentOS.IsWindows) + { + string sqliteSourceFileName = FindSqlite3DLL(); + Directory.SetCurrentDirectory(Path.GetDirectoryName(sqliteSourceFileName)); + } + + database = new SQLite(); + database.OpenDatabase(":memory:", readOnly: false); + + var dataStore = new DataStore(database); + dataStore.Writer.TablesModified.Add("Observed"); + + ExcelInput excelInput = new ExcelInput(); + excelInput.FileNames = new string[] {"%root%/Tests/UnitTests/PostSimulationTools/Input.xlsx"}; + excelInput.SheetNames = new string[] {"Sheet1"}; + + Utilities.InjectLink(excelInput, "storage", dataStore); + + excelInput.Run(); + dataStore.Writer.Stop(); + dataStore.Reader.Refresh(); + + DataTable dt = dataStore.Reader.GetData("Sheet1"); + + Assert.AreEqual(dt.Columns[4].DataType, typeof(DateTime)); + Assert.AreEqual(dt.Columns[5].DataType, typeof(string)); + Assert.AreEqual(dt.Columns[6].DataType, typeof(string)); + } + } +} diff --git a/Tests/UnitTests/PostSimulationTools/Input.xlsx b/Tests/UnitTests/PostSimulationTools/Input.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..990cbb381ca2d31e1367a5bbc9bad5cf19049144 GIT binary patch literal 10024 zcmeHtg;!L2_x?x^-6cpP9RkuNpoD;QwRy`_y2eN7e}B_wb!Pd1G`CMOH{I1J>=8d z{CpI$dNLi_P2`WR(VDc?Ya|ReUXR}>sg(Czy5f@Fd?k2GF|{IYi1n9YLvj+%~;rxKNGqwPrH$FqW9 zQ3<*xruBTd%c*VQ-gYfGIYuX9V?I}ZWmZzzd5y+Kr6*kKv?WOnwyRUqFk;&75co!q0 zsA7RrbOb;29SM80sk3mh%Yqrqxs(iXip97?%!g1|p55KS177_tO{>&6Xii|*dkI@b zgQcl~J;=&|o$crOf9d&OoRfcf^ulO)#SRYipnd6^w>{?*bMLXA$+?J2H&Cg1`^wB> zRYhddKAdZLOopvW>JR_Ir^)-KZ)Q#?VylPhY?-?-3>ROJrrM<_DE`XM37LuBKJK|) z;Zi4#^Tg@IS;7lhHwNdXNT!0?+)r}73pCHi4<*a6huJhpfq2wdJcLaVHCOZ zu#WZ5B!LF&Uqr$K01L3zh6Rg^s};Ko*xu3@47U8)#|qV~!I^y6Q17u@N zA^~^Pi%8^%&qQk!>b)BTSG1$bOE8V8XRWi6C$Xz08nAia?Rs`}jnix>A^|Gxt%_r` zyQ^9I39N&KRxGC>pf)K9e3q`UI7@LEI*pk8oNDp_xCW|19^?ScWJntkn)(=!y&n|3Am1o^F`E4=iP#MMkIW0sa!%-b$gUlxg2a)Pcv?d7l3y%}NG}tgAtKn2 z`1T1YlKH822$fr!sm9o*>_mlKs?d=h}>ga}FF=0zq zZLiAZB05t{D|cQjO^{^kZ@uWT|3F>V$TCob!r#0WFqMK}wEijT4jzIdu4_M*H=%$? z>Tc!!>WF5X=ldQjaTj^Pyjf$$Tw3-gA*|^Ew6u zVG{PY7~zI>KL*bWl?%##Wb_0Mk)A=*t?J;0_)uu(Y}!X>zpPslxApSLu{^eW5l3X$ zjrxMWyqxkVnU=Af+sR$wOi=lD4t4eFyV3o$^`xwF ze7cYD={ua~3R%LN>UuI0!<52EC#7$raYBXA){4PyMhGWDlVwl9%d1d^v{iEhs$1*e zyU2&-`@XJsqoXVulDP?#Yj~ddIgU}dz-Ur9xaR%d9Pf|RL=5hTY%Ey*rDNO&EI!J$ zP!z@TS@z0gq;i3quE+bINA?zSB%MYY>aDMx3cHPKJf?KfA2tYhZ8p0KjJho}-GblM zV0+0cI0OyfA>+SEHBpyqhn~$ zw;V1>&ICB$9oeaO)V8rudMg=c=&888PZl0B)$3_~lVgVm2f6JJb~~T*U@pVsowhOO zhTe zJgFNC;bWSh`BLv@^pa@~gaQOUg zg@kbU@%Ie)$>+Ij-JMvR%j7Dto7Mc#%ghrIqt`6EOw;-AHZBG*y!E|rO8Ji4r;2rR3OPfQV zp*p^?f=2Q`owe3ik5}%GF zkIN$@K9E$dHNX%EPkmmJQ7->3vqup(I>4N}BX~DL?dZJrnf^+*bnXB)aK%nHZ2^JQ z!}%8V0)w|2e`qp^FL%(_|B&E?O@yu+SQAo-j1>NX4tHqfanEbG+DvlHB~chO?(Tk_ zVxo0WJ1DUMCAdOjgB{Hf9191WvS%%0xDsRY%@$!s8t>PKKN=CoubtTpd@YaxiipG+ z??_9bUaR^$eU6J-R^!Z`nQ7ovPcqt)uCokYoX#Zbu79}QY-+deNSM5}Ao@eNjWC21 zcQS^7YV@-t0dULR#IcF09<#21K}gCSen2>VHpc9g&Cmn;B#b)m=?bxm5Ay(rNCsXy z;=VzCv0@$n77a+CE;q>!436^vdllnIdxuWm_~MzHUxeq^LvTlkqqd8oOO-rX1*qfT zWREw66}v-vvbe-k0=TJN-F1wd@4Q^^&O;NPR}MG<6%&QqyW}dg-Cc0cZ%EIa`}7Jdn^js_QG4HT*W4AAo{cug*tu zR7Xp&Jo{(iJ~ONouZ6APQ2!JVer2Jf8OR#M{_FfJ0e7?qBOlffv>^A~i4Ev|8`$(P z;vf0Ud$#Ak6sxx?EfW)@KV&PW+`x?g0CnT5oFAO|sGQ%tJ3}avnJ-e^Y9yy2jr?7( zkp1(-rO?nOs93bI3p>}>ePQT&Q+GFSq{I1#f{&ljsA<<^CeijT9#Qv(51W_oxGU1H z$j(S8FTH+8q9{G7gZIPBet#u$ZyBB->LcZQ#nlIU?orA=#H*N0Il|Y-Q=<{Qb!>4= z5NnLJQqVks?k4IOUM*S?obp03i^N|=v{^R8(?isM%*`-RaMsGle1+$tAF8x4mLcA` z5gI|Mi-V^hL42c!wVrU?53EyuWUR?_@oD>VeYjx0L8lA34qL_JwVb`laeV(8-0|_q z0At+vZWffmYSO>R8{W`@h}UBKu1EyFwSq**{7ZHrml^l1^G6f8^qsL!4dbX5Pn)w^ zsyp~Mxv!1p#=v3;(5pSJcY>Kl9qkWa1%Y=KJ-ny(zp^ww09~wOHj8XkwMPZ=rU#F% zdJ@f9Rm}{NH3hQp2|Bh54(2S7t{=c5j+RES33ovrTNo5}U62r&0L&;mLg}T8RLWbq zf|vF4Yb7O^RDvcS$=bEkpoEwG1 zf#U$IQ(ZbVMfKh>;X2J&yGYJv+d<*hkhR58roa~myPzj!kZQ%Q0Pto0#^uAiZfo-ZON?>L>S z$g;S!h~NnMA!y{NA_PN0bY#tqinUFTCawHJnEOv8tw58h=|h{FC2{Iz6gce%bg6id zRL?ELk&QgiG33M_rM!>I4)GhjRd;@~VsO9{nMs}o#-m_a=GVhL(fdmdjm>ec-}pkx zraoUHG};92skRzBzH;q+vjJKJ1|DH8cbUh$z;%p)cvLEGn+Fs%$t}paG!_T0weV`0 zl(MDGIqE^s>&nn+&v>f`5x6WN)Q0pQGiWsG5pnu&1B2%^BBuHnYf6c%myj633*8^* z^lpcy;jtjJ+_+h!VX2?i612!QRj`+k=@;0aEqMo44Puu>tH90l+APw$^_IIyY7)(qp2798oizt94Jja~qiK#K> zFKw=U%@N|S+zjE+YVfCeDBC8$xuHYU$<^p~{c zx*)}^cR2IbbR=V$*1r9sCYS@T&?#y4;Dywj7Ek7yhNsgY^IYj(Gdi4pX_)5UpBTRm z@9o0@6m{t-x^Pc6MSf0WL#yIFsFZ5wqeEj(3OI~qJa~-&?r?LAyWSHeu;o2<`fQsg zrI=(~L5fORboTb3N`ZRP9$v050YbNl`^^<4Y)FDD*|`bvBx31_0+nl)_j$V&@GUzR zT{*$s(>yu?WsI`$Ct*p2A1)u~grans&7PhdgFF1_=tEww3Y0qfGlfQF57F_Mu2Oif`D_ndD*x$nD>e-Z{ru6x$n`c-{@8VP#6W&021{nGu$ z-DdBSZ*k0L-3n#q+U8Q=q&=&vCFe`r0%M{oO;Fvn`_QgcG&w@PLz3%xQ;THL$AOF- zlKpj~l*(hJq-l>r6xjwvhse3=q=0c;aKowp>}l?VU}sJ#w;vdDsNZzP^9nt>8(RTK zgGF%n4RAJL?1?bJml_ojgN^u?Z$jvg>)2!~RG}YPuu}}2nnu+4*WY%#@wDQKB_$H6 zPpOY+?S6;2SM*C&AT|&T^-Wox=Cd}&PLWmE1j<{EO6llXD|~e-vU&3jvp;n@%gBVW zcDjnXvKKLEj9+kI(4KuYfjL%V+BQ%f;@q@w6`}8i{S=9yWsX>FtE%3>oY%cdwq!IH zG4uH%*7eecn4*;3NKdmTafdm!&c=f1$HNpbU9QvUlAPrDJ4(;=Oay zSpAl#x!g0E-h7zPPd{NTdS2H-YKT)D9CTWJnDDeAKDlY1{1)eLW$Qm&>H0|i1q|b*OZJ$p~wM(BPWKI%QYvlM#ADDZA*UnAPFsBo* zN! zwH$~=-KCwS4-a3<9lRbN-zTLoQ>wxMdN^zq z|4+c18QFtgt2o-5+nD~sz2ium0!(y8S`@_H$G#kjF2;SvFo^k5kg^SQ08qF5CLJ+8 z!Db9rq~}X7C%?Uwz3KNpjzv0_`OrClp~{RLR6D4*DgnO_ZfHQ}$mk)*Ovv-Y16sB3 zMi42f%$IYI6ugW4COf0@OlvKwmX^XzHW!qPss1IT^h2ujxZ&i`+)O}JTcz6@Ru4-K zuI`nWiY<$jt|4ufQ_DZ-D$eFD4$iDZn_Aiwt zAhIpAodaL$93Cp+>ha-QAi9JKFLkZj2E6grG}27``*ZwTj|wtS>)M*_@Y>U)YoWY{ zZ!n`+zShG9u9^^;l^%FZv$lwr@omlG16hUM8;!FyC&DWz$cA*k*CaqI3sj=LCGPO< z{9g7BnD<^}{ku$v8%OVD*u#;+d<*L%1LR6}g{pm)RWA*H0byk==*~MY3L%@!mLch* z;Q$%__l6=F()>MAyGOpp;@d7~sg_=t2szdR{l{m(e~SQq3io41*we4Vir?cuB47&x zfg{LX1?1@X6AkS5?m5|v&24sK%M?1q(1S!50-vfOYc4*R0?H8+qvd8ui%I#%!1gR? zxNxe5k_Y(cYdo%i=P(II993alR|1Ll6;jYe%q+!>>Gh7F>A=T_^t!fPJ);rq)cTM zJsS91quIA-AQGSAHO#T@a+*mXesB@KI4dLYlPIfA%~RTJ-4Wd`WhD0rsFhYC$RbXK z##ox8oZPBTDw^j^3V!HPhUyHz&K;$|52^@#@-^5|)(>?*_~53=J)=;{&|y{602a%C z?kPiq4Ptd*(T;_+8yuJz_!?~T${uX%z-|Jz2mLvc{+|X23tf1$fm{a%e%AtGi$t$K zL!tbnK-9GJOOPTak~;o^v3(6ek|^z{jko)_b3yDSh(%;Qr0+YNu%OOP7Jlg z3Z>*D6ecakZzS$a>rkGfa5_koENagPCVJxf=f{j{z*y2aZKjcGgcbg14xk#zv}OTr zIK~{{F}*G7QX+F6f7xE{q$bXl!Z+Vcj>dK}7NbtwS%B9B-sq!Vvq)8ndW#3)#<{-! zrAU$foX}D2k=Yym5oaV>{pl3@gfr*2R?i#Wl*}iV6Mb_c|7vI1aY?Zf-_>FAaDrW` zf*F-TkwiWh$vAr`Ub4R=iWl#t#BR#)5Q+xxag>I%7HksLBNd8#^rnh{(|JkXNDZ!G z?l#O;XicaxN|2^ih=}?JHHE(r-&Z<3n_XP=rVP(TKnmSf@01zii({~SAZyV!#EZ)!a^!kv>_$l z5vJtqYPpo@ZgnNum7weTr=&Jb-j;1RqQl3`m^yG8;!cE^K~|2sjwgEKecP#62HzUq z`(|I@KzOv35|u@W$%U^H%hPl))DO&6XzOuLbhs-&)Gdyy#%OD4(wk^8hf!XbiqHCQ z#aAoSXW)~w3XH6p1g#4f?IweK2%bf%lqyJNlF=3FKc$zjf>)05@$~FM@_ltcOB^P_ zH=$tSz_{U0iP+sfZ-^ihVbWPeg?6D?qMF4XyrHt}XS7Og*Ov&ts zdHoE_jK92H+pC;e^u5XRhon%-xq_dVR_ za=$&$k=>8MzcRb~4)0T=-wq#>|8jVb82u|zy6^Pw4B)p*7$FTHF8t literal 0 HcmV?d00001 diff --git a/Tests/UnitTests/UtilityTests/DataUtilitiesTests.cs b/Tests/UnitTests/UtilityTests/DataUtilitiesTests.cs index d2bc34a3f7..e8655b2e45 100644 --- a/Tests/UnitTests/UtilityTests/DataUtilitiesTests.cs +++ b/Tests/UnitTests/UtilityTests/DataUtilitiesTests.cs @@ -204,6 +204,10 @@ public void TestDateFunctions() Assert.Null(DateUtilities.ValidateDateString("FakeMonth 10")); + //ValidateDateStringWithYear + Assert.AreEqual(yyyymmdd, DateUtilities.ValidateDateStringWithYear("10/January/2000")); + Assert.Null(DateUtilities.ValidateDateStringWithYear("10 January")); + //GetNextDate Assert.AreEqual(DateUtilities.GetDate("2-Jan-2001"), DateUtilities.GetNextDate("2-Jan", date)); //2-Jan is before date Assert.AreEqual(DateUtilities.GetDate("20-Jan-2000"), DateUtilities.GetNextDate("20-Jan", date)); //20-Jan is after date