added TimestampFormatConfig, TimestampFormatter and TimestampParser

lib/quickload.rb

@@ -1,2 +1,3 @@
 require 'quickload/error'
 require 'quickload/plugin'
+require 'quickload/java/time_helper'

lib/quickload/java/imports.rb

@@ -2,9 +2,9 @@
 
 module QuickLoad::Java
   java_import 'org.quickload.time.Timestamp'
-  java_import 'org.quickload.time.TimestampFormat'
   java_import 'org.quickload.time.TimestampFormatter'
   java_import 'org.quickload.time.TimestampParser'
+  java_import 'org.quickload.time.TimestampParseException'
 
   java_import 'org.quickload.buffer.Buffer'
   java_import 'org.quickload.buffer.BufferAllocator'
@@ -73,7 +73,7 @@ module QuickLoad::Java
   java_import 'org.quickload.spi.PartialTransferException'
   java_import 'org.quickload.spi.PluginThread'
   java_import 'org.quickload.spi.ExecControl'
-  java_import 'org.quickload.spi.ProcTask'
+  java_import 'org.quickload.spi.ExecTask'
   java_import 'org.quickload.spi.ExecConfig'
   java_import 'org.quickload.spi.TextGuessPlugin'
 end

lib/quickload/java/time_helper.rb

@@ -0,0 +1,75 @@
+require 'java'
+
+module QuickLoad::Java
+  require 'quickload/java/imports'
+
+  class TimeParserHelper
+    java_implements 'org.quickload.time.JRubyTimeParserHelper'
+
+    class Factory
+      java_implements 'org.quickload.time.JRubyTimeParserHelperFactory'
+
+      # Override
+      def newInstance(format_string, year, mon, day, hour, min, sec, usec)
+        default_time = Time.utc(year, mon, day, hour, min, sec, usec)
+        TimeParserHelper.new(format_string, default_time)
+      end
+    end
+
+    def initialize(format_string, default_time)
+      @format_string = format_string
+      @default_time = default_time
+    end
+
+    # Override
+    def strptime(text)
+      hash = Date._strptime(text, @format_string)
+      unless hash
+        raise Java::TimestampParseException.new
+      end
+
+      if seconds = d[:seconds]
+        return seconds * 1_000_000
+
+      else
+        year = d[:year]
+        mon = d[:mon]
+        day = d[:mday]
+        hour = d[:hour]
+        min = d[:min]
+        sec = d[:sec]
+        sec_fraction = d[:sec_fraction]
+        zone = d[:zone]
+        usec = d[:sec_fraction] ? d[:sec_fraction] * 1000000 : nil
+
+        now = @default_time
+        begin
+          break if year; year = now.year
+          break if mon; mon = now.mon
+          break if day; day = now.day
+          break if hour; hour = now.hour
+          break if min; min = now.min
+          break if sec; sec = now.sec
+          break if sec_fraction; usec = now.tv_usec
+        end until true
+
+        year ||= 1970
+        mon ||= 1
+        day ||= 1
+        hour ||= 0
+        min ||= 0
+        sec ||= 0
+        usec ||= 0
+
+        @zone = zone
+        time = Time.utc(year, mon, day, hour, min, sec, usec).to_i
+        return time.tv_sec * 1_000_000 + time.tv_usec
+      end
+    end
+
+    # Override
+    def getZone
+      @zone
+    end
+  end
+end

pom.xml

@@ -31,6 +31,7 @@
         <project.build.jvmsize>1024m</project.build.jvmsize>
         <project.test.fork-mode>once</project.test.fork-mode>
 
+        <dep.joda-time.version>2.3</dep.joda-time.version>
         <dep.jruby.version>1.7.16.1</dep.jruby.version>
 
         <project.check.skip-all>false</project.check.skip-all>
@@ -136,6 +137,12 @@
                 <version>${dep.jruby.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>joda-time</groupId>
+                <artifactId>joda-time</artifactId>
+                <version>${dep.joda-time.version}</version>
+            </dependency>
+
             <!-- ==================== -->
             <!-- Testing dependencies -->
             <!-- ==================== -->

quickload-core/src/main/java/org/quickload/config/TaskSerDe.java

@@ -26,7 +26,7 @@
 import com.fasterxml.jackson.databind.DeserializationConfig;
 import com.fasterxml.jackson.databind.JsonMappingException;
 
-public class TaskSerDe
+class TaskSerDe
 {
     public static class TaskSerializer
             extends JsonSerializer<Task>

quickload-core/src/main/java/org/quickload/exec/ExecModule.java

@@ -5,6 +5,7 @@
 import com.google.common.base.Preconditions;
 import com.google.inject.Binder;
 import com.google.inject.Scopes;
+import org.quickload.time.TimestampFormatConfigSerDe;
 import org.quickload.config.ModelManager;
 import org.quickload.record.TypeManager;
 import org.quickload.config.DataSourceSerDe;
@@ -20,6 +21,7 @@ public void configure(Binder binder)
         binder.bind(ModelManager.class).in(Scopes.SINGLETON);
         binder.bind(TypeManager.class).asEagerSingleton();
         binder.bind(DataSourceSerDe.class).asEagerSingleton();
+        binder.bind(TimestampFormatConfigSerDe.class).asEagerSingleton();
         binder.bind(BufferManager.class).in(Scopes.SINGLETON);
         binder.bind(ParserPlugin.class).annotatedWith(Names.named("system_guess")).to(GuessExecutor.GuessParserPlugin.class);
     }

quickload-core/src/main/java/org/quickload/time/JRubyTimeParserHelper.java

@@ -0,0 +1,8 @@
+package org.quickload.time;
+
+public interface JRubyTimeParserHelper
+{
+    public long strptime(String text) throws TimestampParseException;
+
+    public String getZone();
+}

quickload-core/src/main/java/org/quickload/time/JRubyTimeParserHelperFactory.java

@@ -0,0 +1,6 @@
+package org.quickload.time;
+
+public interface JRubyTimeParserHelperFactory
+{
+    public JRubyTimeParserHelper newInstance(String formatString, int year, int mon, int day, int hour, int min, int sec, int usec);
+}

quickload-core/src/main/java/org/quickload/time/TimestampFormatConfig.java

@@ -0,0 +1,99 @@
+package org.quickload.time;
+
+import java.util.Set;
+import java.util.Date;
+import java.text.SimpleDateFormat;
+import java.text.ParsePosition;
+import com.google.common.collect.ImmutableSet;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.jruby.embed.ScriptingContainer;
+
+public class TimestampFormatConfig
+{
+    private final ScriptingContainer jruby;
+    private final String format;
+
+    public TimestampFormatConfig(ScriptingContainer jruby, String format)
+    {
+        this.jruby = jruby;
+        this.format = format;
+    }
+
+    @JsonValue
+    public String getFormat()
+    {
+        return format;
+    }
+
+    public TimestampFormatter newFormatter(TimestampFormatterTask task)
+    {
+        return new TimestampFormatter(jruby, format, task);
+    }
+
+    public TimestampParser newParser(TimestampParserTask task)
+    {
+        return new TimestampParser(jruby, format, task);
+    }
+
+    private static Set<String> availableTimeZoneNames = ImmutableSet.copyOf(DateTimeZone.getAvailableIDs());
+
+    static DateTimeZone parseDateTimeZone(String s)
+    {
+        if(s.startsWith("+") || s.startsWith("-")) {
+            return DateTimeZone.forID("GMT"+s);
+
+        } else {
+            try {
+                int rawOffset = (int) DateTimeFormat.forPattern("z").parseMillis(s);
+                if(rawOffset == 0) {
+                    return DateTimeZone.UTC;
+                }
+                int offset = rawOffset / -1000;
+                int h = offset / 3600;
+                int m = offset % 3600;
+                return DateTimeZone.forOffsetHoursMinutes(h, m);
+            } catch (IllegalArgumentException ex) {
+                // parseMillis failed
+            }
+
+            // TimeZone.getTimeZone returns GMT zone if given timezone id is not found
+            // we want to only return timezone if exact match, otherwise exception
+            if (availableTimeZoneNames.contains(s)) {
+                //return TimeZone.getTimeZone(s);
+                return DateTimeZone.forID(s);
+            }
+            return null;
+        }
+    }
+
+    //// Java standard TimeZone
+    //static TimeZone parseDateTimeZone(String s)
+    //{
+    //    if(s.startsWith("+") || s.startsWith("-")) {
+    //        return TimeZone.getTimeZone("GMT"+s);
+    //
+    //    } else {
+    //        ParsePosition pp = new ParsePosition(0);
+    //        Date off = new SimpleDateFormat("z").parse(s, pp);
+    //        if(off != null && pp.getErrorIndex() == -1) {
+    //            int rawOffset = (int) off.getTime();
+    //            if(rawOffset == 0) {
+    //                return TimeZone.UTC;
+    //            }
+    //            int offset = rawOffset / -1000;
+    //            int h = offset / 3600;
+    //            int m = offset % 3600;
+    //            return DateTimeZone.getTimeZone(String.format("GMT%+02d%02d", h, m));
+    //        }
+    //
+    //        // TimeZone.getTimeZone returns GMT zone if given timezone id is not found
+    //        // we want to only return timezone if exact match, otherwise exception
+    //        if (availableTimeZoneNames.contains(s)) {
+    //            return TimeZone.getTimeZone(s);
+    //        }
+    //        return null;
+    //    }
+    //}
+}

quickload-core/src/main/java/org/quickload/time/TimestampFormatConfigSerDe.java

@@ -0,0 +1,39 @@
+package org.quickload.time;
+
+import com.google.inject.Inject;
+import org.jruby.embed.ScriptingContainer;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import org.quickload.config.ModelManager;
+
+public class TimestampFormatConfigSerDe
+{
+    @Inject
+    public TimestampFormatConfigSerDe(ModelManager modelManager, ScriptingContainer jruby)
+    {
+        SimpleModule module = new SimpleModule();
+
+        module.addDeserializer(TimestampFormatConfig.class, new TimestampFormatConfigDeserializer(jruby));
+
+        modelManager.addObjectMapperModule(module);
+    }
+
+    public static class TimestampFormatConfigDeserializer
+            extends FromStringDeserializer<TimestampFormatConfig>
+    {
+        private final ScriptingContainer jruby;
+
+        public TimestampFormatConfigDeserializer(ScriptingContainer jruby)
+        {
+            super(TimestampFormatConfig.class);
+            this.jruby = jruby;
+        }
+
+        @Override
+        protected TimestampFormatConfig _deserialize(String value, DeserializationContext context)
+        {
+            return new TimestampFormatConfig(jruby, value);
+        }
+    }
+}

quickload-core/src/main/java/org/quickload/time/TimestampFormatter.java

@@ -1,14 +1,39 @@
 package org.quickload.time;
 
+import java.util.Locale;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.util.RubyDateFormat;
+import static org.quickload.time.TimestampFormatConfig.parseDateTimeZone;
+import org.quickload.spi.LineEncoder;
+import org.quickload.config.ConfigException;
+
 public class TimestampFormatter
 {
-    public TimestampFormatter(TimestampFormat format)
+    private final RubyDateFormat dateFormat;
+    private final DateTimeZone timeZone;
+
+    public TimestampFormatter(ScriptingContainer jruby, String format, TimestampFormatterTask task)
+    {
+        this.timeZone = parseDateTimeZone(task.getTimeZone());
+        if (timeZone == null) {
+            throw new ConfigException("Unsupported timezone '"+task.getTimeZone()+"'");
+        }
+        this.dateFormat = new RubyDateFormat(format, Locale.ENGLISH, true);
+    }
+
+    public void format(Timestamp value, LineEncoder encoder)
     {
+        // TODO optimize by directly appending to internal buffer
+        encoder.addText(format(value));
     }
 
-    public String format()
+    public String format(Timestamp value)
     {
-        // TODO
-        return "";
+        // TODO optimize by using reused StringBuilder
+        dateFormat.setDateTime(new DateTime(value.toEpochMilli(), timeZone));
+        dateFormat.setNSec(value.getNano());
+        return dateFormat.format(null);
     }
 }

quickload-core/src/main/java/org/quickload/time/TimestampFormatterTask.java

@@ -0,0 +1,12 @@
+package org.quickload.time;
+
+import org.quickload.config.Config;
+import org.quickload.config.ConfigDefault;
+
+public interface TimestampFormatterTask
+{
+    @Config("timezone")
+    @ConfigDefault("\"UTC\"")
+    // TODO TimeZone SerDe
+    public String getTimeZone();
+}

quickload-core/src/main/java/org/quickload/time/TimestampParseException.java

@@ -0,0 +1,6 @@
+package org.quickload.time;
+
+public class TimestampParseException
+        extends Exception
+{
+}