Merge pull request #8791 from par456/8784DataDateChecks

8784 Check if string column should be date when reading from excel

APSIM.Shared/Utilities/DateUtilities.cs

@@ -345,30 +345,18 @@ public static bool IsEndOfYear(DateTime date)
         /// <returns>Return null if not valid, otherwise it returns a string with the valid dd-MMM string or a valid date as a string (yyyy-mm-dd)</returns>
         public static string ValidateDateString(string dateStr)
         {
-            DateAsParts parts;
-            parts = ParseDateString(dateStr);
-            if (parts.parseError)
-                return null;
-
-            DateTime date;
-            try
-            {
-                date = GetDate(parts);
-            }
-            catch
-            {
-                return null;
-            }
+            return Validate(dateStr, true);
+        }
 
-            if (parts.yearWasMissing)
-            {
-                //for consistency, return it as 'Title' case (ie, 01-Jan, not 1-jan)
-                return date.ToString(DEFAULT_FORMAT_DAY_MONTH, CultureInfo.InvariantCulture);
-            }
-            else
-            {
-                return date.ToString(DEFAULT_FORMAT_DAY_MONTH_YEAR, CultureInfo.InvariantCulture);
-            }
+        /// <summary>
+        /// Takes in a string and checks to see if it is in the correct format for either a full date with year, month and date (in any recognised date format).
+        /// Will return null if not valid or if only a day and month was provided
+        /// </summary>
+        /// <param name="dateStr"></param>
+        /// <returns>Return null if not valid, otherwise it returns a string with the valid string (yyyy-mm-dd)</returns>
+        public static string ValidateDateStringWithYear(string dateStr)
+        {
+            return Validate(dateStr, false);
         }
 
         /// <summary>
@@ -423,6 +411,45 @@ private static DateTime GetDate(DateAsParts parts)
             return GetDate(parts.day, parts.month, parts.year);
         }
 
+        /// <summary>
+        /// Checks if a string is formatted to be a date, returns null if it can't be a date, or a formatted date string if it can.
+        /// </summary>
+        /// <param name="input">The given string to be checked</param>
+        /// <param name="allowPartialDate">If a day-month is allowed or if it must be a full date</param>
+        /// <returns>A formatted date as a string</returns>
+        private static string Validate(string input, bool allowPartialDate)
+        {
+            DateAsParts parts;
+            parts = ParseDateString(input);
+            if (parts.parseError)
+                return null;
+
+            DateTime date;
+            try
+            {
+                date = GetDate(parts);
+            }
+            catch
+            {
+                return null;
+            }
+
+            if (parts.yearWasMissing) {
+                if (allowPartialDate) {
+                    //for consistency, return it as 'Title' case (ie, 01-Jan, not 1-jan)
+                    return date.ToString(DEFAULT_FORMAT_DAY_MONTH, CultureInfo.InvariantCulture);
+                } 
+                else
+                {
+                    return null;
+                }
+            }
+            else 
+            {
+                return date.ToString(DEFAULT_FORMAT_DAY_MONTH_YEAR, CultureInfo.InvariantCulture);
+            }
+        }
+
         /// <summary>
         /// Convert any valid date string into a DateTime objects.
         /// Valid seprators are: / - , . _

Models/PostSimulationTools/ExcelInput.cs

@@ -130,6 +130,40 @@ public void Run()
                         {
                             if (SheetNames.Any(str => string.Equals(str.Trim(), table.TableName, StringComparison.InvariantCultureIgnoreCase)))
                             {
+                                //Check if any columns that only contain dates are being read in as strings (and won't graph properly because of it)
+                                List<string> replaceColumns = new List<string>();
+                                foreach (DataColumn column in table.Columns) 
+                                {
+                                    if (column.DataType == typeof(string)) {
+                                        bool isDate = true;
+                                        int count = 0;
+                                        while(isDate && count < table.Rows.Count) {
+                                            if (DateUtilities.ValidateDateStringWithYear(table.Rows[count][column.ColumnName].ToString()) == null) {
+                                                isDate = false;
+                                            }
+                                            count += 1;
+                                        }
+                                        if (isDate) {
+                                            replaceColumns.Add(column.ColumnName);
+                                        }
+                                    }
+                                }
+                                foreach (string name in replaceColumns) 
+                                {
+                                    DataColumn column = table.Columns[name];
+                                    int ordinal = column.Ordinal;
+
+                                    DataColumn newColumn = new DataColumn("NewColumn"+name, typeof(DateTime));
+                                    table.Columns.Add(newColumn);
+                                    newColumn.SetOrdinal(ordinal);
+
+                                    foreach (DataRow row in table.Rows)
+                                        row[newColumn.ColumnName] = DateUtilities.GetDate(row[name].ToString());
+
+                                    table.Columns.Remove(name);
+                                    newColumn.ColumnName = name;
+                                }
+
                                 TruncateDates(table);
 
                                 // Don't delete previous data existing in this table. Doing so would

Tests/UnitTests/PostSimulationTools/ExcelInputTests.cs

@@ -0,0 +1,61 @@
+using APSIM.Shared.Utilities;
+using Models.PostSimulationTools;
+using Models.Storage;
+using NUnit.Framework;
+using System;
+using System.Data;
+using System.IO;
+using System.Reflection;
+
+namespace UnitTests
+{
+    public class ExcelInputTests
+    {
+        private IDatabaseConnection database;
+
+        /// <summary>Find and return the file name of SQLite runtime .dll</summary>
+        public static string FindSqlite3DLL()
+        {
+            string directory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+            string[] files = Directory.GetFiles(directory, "sqlite3.dll");
+            if (files.Length == 1)
+                return files[0];
+
+            throw new Exception("Cannot find sqlite3 dll directory");
+        }
+
+        [Test]
+        public void LoadExcelInput()
+        {
+            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
+
+            if (ProcessUtilities.CurrentOS.IsWindows)
+            {
+                string sqliteSourceFileName = FindSqlite3DLL();
+                Directory.SetCurrentDirectory(Path.GetDirectoryName(sqliteSourceFileName));
+            }
+
+            database = new SQLite();
+            database.OpenDatabase(":memory:", readOnly: false);
+
+            var dataStore = new DataStore(database);
+            dataStore.Writer.TablesModified.Add("Observed");
+
+            ExcelInput excelInput = new ExcelInput();
+            excelInput.FileNames = new string[] {"%root%/Tests/UnitTests/PostSimulationTools/Input.xlsx"};
+            excelInput.SheetNames = new string[] {"Sheet1"};
+
+            Utilities.InjectLink(excelInput, "storage", dataStore);
+
+            excelInput.Run();
+            dataStore.Writer.Stop();
+            dataStore.Reader.Refresh();
+
+            DataTable dt = dataStore.Reader.GetData("Sheet1");
+
+            Assert.AreEqual(dt.Columns[4].DataType, typeof(DateTime));
+            Assert.AreEqual(dt.Columns[5].DataType, typeof(string));
+            Assert.AreEqual(dt.Columns[6].DataType, typeof(string));
+        }
+    }
+}

Tests/UnitTests/UtilityTests/DataUtilitiesTests.cs

@@ -204,6 +204,10 @@ public void TestDateFunctions()
 
             Assert.Null(DateUtilities.ValidateDateString("FakeMonth 10"));
 
+            //ValidateDateStringWithYear
+            Assert.AreEqual(yyyymmdd, DateUtilities.ValidateDateStringWithYear("10/January/2000"));
+            Assert.Null(DateUtilities.ValidateDateStringWithYear("10 January"));
+
             //GetNextDate
             Assert.AreEqual(DateUtilities.GetDate("2-Jan-2001"), DateUtilities.GetNextDate("2-Jan", date)); //2-Jan is before date
             Assert.AreEqual(DateUtilities.GetDate("20-Jan-2000"), DateUtilities.GetNextDate("20-Jan", date)); //20-Jan is after date