[Issue #421] update docs and clean Date related code.

README.md

@@ -6,8 +6,8 @@ It is optimized for analytical tables stored in on-premises and cloud-native sto
 including S3, GCS, HDFS, Redis, and local file systems.
 Pixels outperforms Parquet, which is the most widely used columnar format in today's lakehouses, by up to two orders of magnitude.
 
-We have integrated Pixels with popular query engines including Trino (405), Presto (0.279), and Hive (2.3+). Please find these
-integrations in separate repositories:
+We have integrated Pixels with popular query engines including Trino (405), Presto (0.279), and Hive (2.3+). These
+integrations are opensourced in separate repositories:
 * [Pixels Connector for Trino](https://github.com/pixelsdb/pixels-trino)
 * [Pixels Connector for Presto](https://github.com/pixelsdb/pixels-presto)
 * [Pixels SerDe for Hive](https://github.com/pixelsdb/pixels-hive)
@@ -20,7 +20,7 @@ for workload spikes.
 
 ## Publications
 
-Pixels is an academic system aims at providing product-level quality. It supports all the functionalities required by TPC-H and
+Pixels is an academic system aims at providing production-grade quality. It supports all the functionalities required by TPC-H and
 is compatible with the mainstream data analytic ecosystems.
 The key ideas and insights in Pixels are elaborated in the following publications.
 
@@ -36,32 +36,13 @@ The key ideas and insights in Pixels are elaborated in the following publication
 > `CIDR'20` Pixels: Multiversion Wide Table Store for Data Lakes (abstract) [[Cite]](https://dblp.org/rec/conf/cidr/Bian20.html?view=bibtex)\
 > Haoqiong Bian
 
-> `Journal of Software 2020,31(1):137-161` Survey on Storage and Optimization Techniques of HDFS (Chinese: HDFS存储和优化技术研究综述) [[Cite]](https://www.jos.org.cn/josen/article/export/5872)\
-> Guodong Jin, Haoqiong Bian, Yueguo Chen, Xiaoyong Du
-
 > `ICDE'18` Rainbow: Adaptive Layout Optimization for Wide Tables (demo) [[Cite]](https://dblp.org/rec/conf/icde/BianTJCQD18.html?view=bibtex)\
 > Haoqiong Bian, Youxian Tao, Guodong Jin, Yueguo Chen, Xiongpai Qin, Xiaoyong Du
 
 > `SIGMOD'17` Wide Table Layout Optimization by Column Ordering and Duplication [[Cite]](https://dblp.org/rec/conf/sigmod/BianYTCCDM17.html?view=bibtex)\
 > Haoqiong Bian, Ying Yan, Wenbo Tao, Liang Jeff Chen, Yueguo Chen, Xiaoyong Du, Thomas Moscibroda
 
 
-## Development Environment*
-
-This is optional if you want to develop Pixels in an IDE.
-
-In the directory that you prefer, clone the Pixels codebase:
-```bash
-git clone https://github.com/pixelsdb/pixels.git
-```
-Install JDK 17.0.3 or above, open Pixels as a maven project in IntelliJ.
-When the project is fully indexed and the dependencies are successfully downloaded,
-add a `PIXELS_HOME` environment variable in *Run -> Edit Configurations -> Edit configuration templates*,
-point it to a directory that you want to put the pixels configuration file and the runtime logs.
-
-Now, in Intellij, you can build Pixels using the maven plugin, run and debug unit tests, and debug Pixels by 
-setting up a *Remote JVM Debug*.
-
 ## Build Pixels
 
 Install JDK 17.0.3 or above, and clone the Pixels codebase into any `SRC_BASE` directory:
@@ -93,6 +74,18 @@ They will be used in the installation.
 > the `pixels.properties` into `PIXELS_HOME` and create the `logs` directory where the log files will be
 > written.
 
+
+## Development Environment*
+
+If you want to develop Pixels in Intellij, open `SRC_BASE/pixels` as a maven project.
+When the project is fully indexed and the dependencies are successfully downloaded,
+add a `PIXELS_HOME` environment variable in *Run -> Edit Configurations -> Edit configuration templates*,
+point it to your Pixels home that contains the configuration file (i.e., `pixels.properties`) and stores the runtime logs.
+
+Now, in Intellij, you can build Pixels using the maven plugin, run and debug unit tests, and debug Pixels by
+setting up a *Remote JVM Debug*.
+
+
 ## Installation in AWS
 
 Create an EC2 Ubuntu-20.04 or 22.04 instance with x86 arch and at least 20GB root volume. 8GB or larger memory is recommended. Login the instance as `ubuntu` user, 

pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/CompactExecutor.java

@@ -164,7 +164,6 @@ public void execute(Namespace ns, String command) throws Exception
                             .setBlockPadding(false);
 
             long threadStart = System.currentTimeMillis();
-            String finalFilePath = targetFilePath;
             compactExecutor.execute(() -> {
                 // Issue #192: run compaction in threads.
                 try
@@ -177,7 +176,7 @@ public void execute(Namespace ns, String command) throws Exception
                 {
                     e.printStackTrace();
                 }
-                System.out.println("Compact file '" + finalFilePath + "' is built in " +
+                System.out.println("Compact file '" + targetFilePath + "' is built in " +
                         ((System.currentTimeMillis() - threadStart) / 1000.0) + "s");
             });
         }

pixels-core/src/main/java/io/pixelsdb/pixels/core/stats/DateStatsRecorder.java

@@ -24,7 +24,7 @@
 import java.sql.Date;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static io.pixelsdb.pixels.core.utils.DatetimeUtils.millisToDay;
+import static io.pixelsdb.pixels.core.utils.DatetimeUtils.localMillisToUtcDays;
 
 /**
  * pixels
@@ -101,7 +101,7 @@ public void updateDate(int value)
     @Override
     public void updateDate(Date value)
     {
-        this.updateDate(millisToDay(value.getTime()));
+        this.updateDate(localMillisToUtcDays(value.getTime()));
     }
 
     @Override

pixels-core/src/main/java/io/pixelsdb/pixels/core/utils/DatetimeUtils.java

@@ -37,33 +37,33 @@ public static void resetTimezoneOffset()
     }
 
     /**
-     * Convert the days to milliseconds, both since the Unix epoch ('1970-01-01 00:00:00 UTC').
+     * Convert the epoch days in UTC of a date (e.g., '1999-12-31') to the first millisecond in local time
+     * of the date since the Unix epoch ('1970-01-01 00:00:00 UTC').
      * Leap seconds are not considered.
      */
-    public static long dayToMillis (int day)
+    public static long utcDaysToLocalMillis(int day)
     {
         /**
          * Issue #419:
          * No need to add the timezone offset, because both days and milliseconds
          * are since the Unix epoch.
-         * TODO: add leap seconds.
          */
-        return day * 86400000L;
+        return Date.valueOf(LocalDate.ofEpochDay(day)).getTime();
     }
 
     /**
-     * Convert the milliseconds to days, both since the Unix epoch ('1970-01-01 00:00:00 UTC').
+     * Convert the first millisecond in local time of a date (e.g., '1999-12-31') since the
+     * Unix epoch ('1970-01-01 00:00:00 UTC') to the epoch days in UTC of the date.
      * Leap seconds are not considered.
      * <b>If the default timezone is changed, must call {@link #resetTimezoneOffset()} before this method.</b>
      */
-    public static int millisToDay (long millis)
+    public static int localMillisToUtcDays(long millis)
     {
-        // TODO: add leap seconds.
         return Math.round((millis + TIMEZONE_OFFSET) / 86400000f);
     }
 
     /**
-     * Convert the {@link Date} of local time to the days since the Unix epoch ('1970-01-01 00:00:00 UTC').
+     * Convert the {@link Date} in local time to the days in epoch day in UTC of the date (e.g., '1999-12-31').
      * This method produces more temporary objects than:<br/>
      * {@code millisToDay(date.getTime())}.
      */
@@ -73,7 +73,7 @@ public static int sqlDateToDay (Date date)
     }
 
     /**
-     * Convert the {@link Date} of local time to the days since the Unix epoch ('1970-01-01 00:00:00 UTC').
+     * Convert the {@link Date} to the days since the Unix epoch ('1970-01-01').
      * @param date
      * @return
      */
@@ -82,7 +82,33 @@ public static int stringToDay (String date)
         return (int) LocalDate.parse(date).toEpochDay();
     }
 
-    public static int roundSqlTime (long millis)
+    public static long daysToMillis (int days)
+    {
+        return (long) days * 86400000L;
+    }
+
+    /**
+     * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of
+     * seconds. 500 would round to 0, -500 would round to -1.
+     */
+    public static long millisToSeconds(long millis)
+    {
+        if (millis >= 0)
+        {
+            return millis / 1000;
+        }
+        else
+        {
+            return (millis - 999) / 1000;
+        }
+    }
+
+    /**
+     * Get the milliseconds in a day.
+     * @param millis the milliseconds since the Unix epoch ('1970-01-01 00:00:00 UTC');
+     * @return the milliseconds in a day.
+     */
+    public static int millisInDay(long millis)
     {
         return (int)(millis % 86400000);
     }

pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/DateColumnVector.java

@@ -144,71 +144,6 @@ public int getDate(int elementNum)
         return dates[elementNum];
     }
 
-    /**
-     * Set a Date object from a row of the column.
-     * We assume the entry has already been NULL checked and isRepeated adjusted.
-     *
-     * @param date
-     * @param elementNum
-     */
-    public void dateUpdate(Date date, int elementNum)
-    {
-        date.setTime(dayToMillis(this.dates[elementNum]));
-    }
-
-    /**
-     * Return the scratch Date object set from a row.
-     * We assume the entry has already been NULL checked and isRepeated adjusted.
-     *
-     * @param elementNum
-     * @return
-     */
-    public Date asScratchDate(int elementNum)
-    {
-        scratchDate.setTime(dayToMillis(dates[elementNum]));
-        return scratchDate;
-    }
-
-    /**
-     * Return a long representation, i.e., the seconds since the Unix epoch ('1970-01-01 00:00:00' UTC/GMT), of the element.
-     *
-     * @param elementNum the index of the element in this vector
-     * @return
-     */
-    public long getDateAsLong(int elementNum)
-    {
-        return millisToSeconds(dayToMillis(dates[elementNum]));
-    }
-
-    /**
-     * Return a long representation, i.e., the seconds since the Unix epoch ('1970-01-01 00:00:00' UTC/GMT), of a Date.
-     *
-     * @param date the date
-     * @return
-     */
-    public static long getDateAsLong(Date date)
-    {
-        return millisToSeconds(date.getTime());
-    }
-
-    // Copy of TimestampWritable.millisToSeconds
-
-    /**
-     * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of
-     * seconds. 500 would round to 0, -500 would round to -1.
-     */
-    private static long millisToSeconds(long millis)
-    {
-        if (millis >= 0)
-        {
-            return millis / 1000;
-        }
-        else
-        {
-            return (millis - 999) / 1000;
-        }
-    }
-
     /**
      * Compare row to Date.
      * We assume the entry has already been NULL checked and isRepeated adjusted.
@@ -219,7 +154,7 @@ private static long millisToSeconds(long millis)
      */
     public int compareTo(int elementNum, Date date)
     {
-        return Long.compare(getDateAsLong(elementNum), getDateAsLong(date));
+        return Integer.compare(dates[elementNum], localMillisToUtcDays(date.getTime()));
     }
 
     /**
@@ -232,7 +167,7 @@ public int compareTo(int elementNum, Date date)
      */
     public int compareTo(Date date, int elementNum)
     {
-        return Long.compare(getDateAsLong(date), getDateAsLong(elementNum));
+        return Integer.compare(localMillisToUtcDays(date.getTime()), dates[elementNum]);
     }
 
     /**
@@ -430,7 +365,7 @@ public void set(int elementNum, Date date)
         }
         else
         {
-            this.dates[elementNum] = millisToDay(date.getTime());
+            this.dates[elementNum] = localMillisToUtcDays(date.getTime());
             this.isNull[elementNum] = false;
         }
     }
@@ -452,21 +387,6 @@ public void set(int elementNum, int days)
         this.isNull[elementNum] = false;
     }
 
-    /**
-     * Set a row from the current value in the scratch date.
-     *
-     * @param elementNum
-     */
-    public void setFromScratchDate(int elementNum)
-    {
-        if (elementNum >= writeIndex)
-        {
-            writeIndex = elementNum + 1;
-        }
-        this.dates[elementNum] = millisToDay(scratchDate.getTime());
-        this.isNull[elementNum] = false;
-    }
-
     /**
      * Set row to standard null value(s).
      * We assume the entry has already been isRepeated adjusted.
@@ -492,7 +412,7 @@ public void fill(Date date)
     {
         noNulls = true;
         isRepeating = true;
-        dates[0] = millisToDay(date.getTime());
+        dates[0] = localMillisToUtcDays(date.getTime());
     }
 
     @Override
@@ -504,7 +424,7 @@ public void stringifyValue(StringBuilder buffer, int row)
         }
         if (noNulls || !isNull[row])
         {
-            scratchDate.setTime(dayToMillis(dates[row]));
+            scratchDate.setTime(utcDaysToLocalMillis(dates[row]));
             buffer.append(scratchDate.toString());
         }
         else

pixels-core/src/main/java/io/pixelsdb/pixels/core/vector/TimeColumnVector.java

@@ -25,7 +25,7 @@
 import java.util.Arrays;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static io.pixelsdb.pixels.core.utils.DatetimeUtils.roundSqlTime;
+import static io.pixelsdb.pixels.core.utils.DatetimeUtils.millisInDay;
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -382,7 +382,7 @@ public void add(int value)
         {
             ensureSize(writeIndex * 2, true);
         }
-        this.times[writeIndex] = roundSqlTime(value);
+        this.times[writeIndex] = millisInDay(value);
         this.isNull[writeIndex++] = false;
     }
 
@@ -427,7 +427,7 @@ public void set(int elementNum, Time t)
         else
         {
             this.isNull[elementNum] = false;
-            this.times[elementNum] = roundSqlTime(t.getTime());
+            this.times[elementNum] = millisInDay(t.getTime());
         }
     }
 
@@ -460,7 +460,7 @@ public void setFromScratchTime(int elementNum)
             writeIndex = elementNum + 1;
         }
         // scratchTime may be changed outside this class, so we also mod it by millis in a day.
-        this.times[elementNum] = roundSqlTime(scratchTime.getTime());
+        this.times[elementNum] = millisInDay(scratchTime.getTime());
         this.isNull[elementNum] = false;
     }
 
@@ -490,7 +490,7 @@ public void fill(Time t)
     {
         noNulls = true;
         isRepeating = true;
-        times[0] = roundSqlTime(t.getTime());
+        times[0] = millisInDay(t.getTime());
     }
 
     @Override