chore: rewritten double-to-string algo that prints to highest possibl…

…e precision.
questdb · Jan 9, 2020 · 5e9d11e · 5e9d11e
1 parent a859868
commit 5e9d11e
Show file tree

Hide file tree

Showing 102 changed files with 8,711 additions and 5,580 deletions.
diff --git a/benchmarks/src/main/java/org/questdb/PrintDoubleBenchmark.java b/benchmarks/src/main/java/org/questdb/PrintDoubleBenchmark.java
@@ -0,0 +1,79 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2020 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package org.questdb;
+
+import io.questdb.std.Numbers;
+import io.questdb.std.str.StringSink;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.concurrent.TimeUnit;
+
+@State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+public class PrintDoubleBenchmark {
+
+    private static StringSink sink = new StringSink();
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder()
+                .include(PrintDoubleBenchmark.class.getSimpleName())
+                .warmupIterations(5)
+                .measurementIterations(5)
+                .addProfiler("gc")
+                .forks(1)
+                .build();
+
+        new Runner(opt).run();
+    }
+
+    @Benchmark
+    public void testCurrent() {
+        sink.clear();
+        Numbers.append(sink, 0.8998893432, 16);
+    }
+
+    @Benchmark
+    public void testCurrent2() {
+        sink.clear();
+        Numbers.append(sink, 0.1253852754303285, 16);
+    }
+
+    @Benchmark
+    public void testNew1() {
+        sink.clear();
+        Numbers.append(sink, 0.8998893432);
+    }
+
+    @Benchmark
+    public void testNew2() {
+        sink.clear();
+        Numbers.append(sink, 0.1253852754303285);
+    }
+}
diff --git a/core/src/main/java/io/questdb/PropServerConfiguration.java b/core/src/main/java/io/questdb/PropServerConfiguration.java
@@ -169,7 +169,6 @@ public int getWorkerCount() {
     private boolean lineUdpUnicast;
     private boolean lineUdpOwnThread;
     private int jsonQueryFloatScale;
-    private int jsonQueryDoubleScale;
     private int sqlCopyBufferSize;
     private int jsonQueryConnectionCheckFrequency;
     private boolean httpFrozenClock;
@@ -249,7 +248,6 @@ public PropServerConfiguration(String root, Properties properties) throws Server
             this.utf8SinkSize = getIntSize(properties, "http.text.utf8.sink.size", 4096);
 
             this.jsonQueryConnectionCheckFrequency = getInt(properties, "http.json.query.connection.check.frequency", 1_000_000);
-            this.jsonQueryDoubleScale = getInt(properties, "http.json.query.double.scale", 10);
             this.jsonQueryFloatScale = getInt(properties, "http.json.query.float.scale", 10);
 
             parseBindTo(properties, "http.bind.to", "0.0.0.0:9000", (a, p) -> {
@@ -1141,11 +1139,6 @@ public int getConnectionCheckFrequency() {
             return jsonQueryConnectionCheckFrequency;
         }
 
-        @Override
-        public int getDoubleScale() {
-            return jsonQueryDoubleScale;
-        }
-
         @Override
         public FilesFacade getFilesFacade() {
             return FilesFacadeImpl.INSTANCE;

diff --git a/core/src/main/java/io/questdb/cutlass/http/DefaultHttpServerConfiguration.java b/core/src/main/java/io/questdb/cutlass/http/DefaultHttpServerConfiguration.java
@@ -76,11 +76,6 @@ public int getConnectionCheckFrequency() {
             return 1_000_000;
         }
 
-        @Override
-        public int getDoubleScale() {
-            return 10;
-        }
-
         @Override
         public FilesFacade getFilesFacade() {
             return FilesFacadeImpl.INSTANCE;

diff --git a/core/src/main/java/io/questdb/cutlass/http/HttpHeaderParser.java b/core/src/main/java/io/questdb/cutlass/http/HttpHeaderParser.java
@@ -500,7 +500,7 @@ public DirectByteCharSequence of(CharSequence value) {
 
         private void of0(CharSequence value) {
             int len = value.length();
-            Chars.strcpy(value, len, _wptr);
+            Chars.asciiStrCpy(value, len, _wptr);
             _wptr += len;
         }
 

diff --git a/core/src/main/java/io/questdb/cutlass/http/HttpResponseSink.java b/core/src/main/java/io/questdb/cutlass/http/HttpResponseSink.java
@@ -391,7 +391,7 @@ public CharSink put(CharSequence cs) {
             int len = cs.length();
             long p = _wptr;
             if (p + len < limit) {
-                Chars.strcpy(cs, len, p);
+                Chars.asciiStrCpy(cs, len, p);
                 _wptr += len;
             } else {
                 throw NoSpaceLeftInResponseBufferException.INSTANCE;
@@ -408,6 +408,15 @@ public CharSink put(char c) {
             throw NoSpaceLeftInResponseBufferException.INSTANCE;
         }
 
+        @Override
+        public CharSink put(char[] chars, int start, int len) {
+            if (_wPtr + len < limit) {
+                Chars.asciiCopyTo(chars, start, len, _wPtr);
+                _wPtr += len;
+            }
+            throw NoSpaceLeftInResponseBufferException.INSTANCE;
+        }
+
         @Override
         public void send() throws PeerDisconnectedException, PeerIsSlowToReadException {
             headerImpl.prepareToSend();
@@ -477,7 +486,7 @@ public CharSink put(CharSequence seq) {
             int len = seq.length();
             long p = _wPtr;
             if (p + len < limit) {
-                Chars.strcpy(seq, len, p);
+                Chars.asciiStrCpy(seq, len, p);
                 _wPtr = p + len;
             } else {
                 throw NoSpaceLeftInResponseBufferException.INSTANCE;
@@ -494,6 +503,16 @@ public CharSink put(char c) {
             throw NoSpaceLeftInResponseBufferException.INSTANCE;
         }
 
+        @Override
+        public CharSink put(char[] chars, int start, int len) {
+            if (_wPtr + len < limit) {
+                Chars.asciiCopyTo(chars, start, len, _wPtr);
+                _wPtr += len;
+                return this;
+            }
+            throw NoSpaceLeftInResponseBufferException.INSTANCE;
+        }
+
         @Override
         public CharSink put(float value, int scale) {
             if (Float.isNaN(value)) {
@@ -504,12 +523,12 @@ public CharSink put(float value, int scale) {
         }
 
         @Override
-        public CharSink put(double value, int scale) {
+        public CharSink put(double value) {
             if (Double.isNaN(value)) {
                 put("null");
                 return this;
             }
-            return super.put(value, scale);
+            return super.put(value);
         }
 
         @Override

diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessor.java b/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessor.java
@@ -63,7 +63,6 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable {
     private final SqlCompiler compiler;
     private final JsonQueryProcessorConfiguration configuration;
     private final int floatScale;
-    private final int doubleScale;
     private final SqlExecutionContextImpl sqlExecutionContext = new SqlExecutionContextImpl();
     private final ObjList<ValueWriter> valueWriters = new ObjList<>();
     private final Path path = new Path();
@@ -77,7 +76,6 @@ public JsonQueryProcessor(
         this.configuration = configuration;
         this.compiler = new SqlCompiler(engine);
         this.floatScale = configuration.getFloatScale();
-        this.doubleScale = configuration.getDoubleScale();
         this.valueWriters.extendAndSet(ColumnType.BOOLEAN, JsonQueryProcessor::putBooleanValue);
         this.valueWriters.extendAndSet(ColumnType.BYTE, JsonQueryProcessor::putByteValue);
         this.valueWriters.extendAndSet(ColumnType.DOUBLE, this::putDoubleValue);
@@ -119,7 +117,7 @@ private static void doResumeSend(
             HttpConnectionContext context,
             ObjList<ValueWriter> valueWriters
     ) throws PeerDisconnectedException, PeerIsSlowToReadException {
-        JsonQueryProcessorState state = LV.get(context);
+        final JsonQueryProcessorState state = LV.get(context);
         if (state == null || state.cursor == null) {
             return;
         }
@@ -531,7 +529,7 @@ private boolean parseUrl(
     }
 
     private void putDoubleValue(HttpChunkedResponseSocket socket, Record rec, int col) {
-        socket.put(rec.getDouble(col), doubleScale);
+        socket.put(rec.getDouble(col));
     }
 
     private void putFloatValue(HttpChunkedResponseSocket socket, Record rec, int col) {

diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessorConfiguration.java b/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessorConfiguration.java
@@ -33,8 +33,6 @@ public interface JsonQueryProcessorConfiguration {
 
     int getConnectionCheckFrequency();
 
-    int getDoubleScale();
-
     FilesFacade getFilesFacade();
 
     int getFloatScale();

diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/TextQueryProcessor.java b/core/src/main/java/io/questdb/cutlass/http/processors/TextQueryProcessor.java
@@ -66,7 +66,6 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable {
     private final SqlCompiler compiler;
     private final JsonQueryProcessorConfiguration configuration;
     private final int floatScale;
-    private final int doubleScale;
     private final SqlExecutionContextImpl sqlExecutionContext = new SqlExecutionContextImpl();
     private final MillisecondClock clock;
 
@@ -75,7 +74,6 @@ public TextQueryProcessor(JsonQueryProcessorConfiguration configuration, CairoEn
         this.configuration = configuration;
         this.compiler = new SqlCompiler(engine);
         this.floatScale = configuration.getFloatScale();
-        this.doubleScale = configuration.getDoubleScale();
         this.clock = configuration.getClock();
     }
 
@@ -390,7 +388,7 @@ private void putValue(HttpChunkedResponseSocket socket, int type, Record rec, in
             case ColumnType.DOUBLE:
                 double d = rec.getDouble(col);
                 if (d == d) {
-                    socket.put(d, doubleScale);
+                    socket.put(d);
                 }
                 break;
             case ColumnType.FLOAT:

diff --git a/core/src/main/java/io/questdb/cutlass/line/LineProtoLexer.java b/core/src/main/java/io/questdb/cutlass/line/LineProtoLexer.java
@@ -343,7 +343,6 @@ public CharSink put(char c) {
             if (dstPos == bufferHi) {
                 extend();
             }
-
             Unsafe.getUnsafe().putChar(dstPos, c);
             return this;
         }
@@ -364,6 +363,11 @@ private void extend() {
             dstPos = buf + offset + (dstPos - dstTop);
             dstTop = buf + offset;
         }
+
+        @Override
+        public CharSink put(char[] chars, int start, int len) {
+            throw new UnsupportedOperationException();
+        }
     }
 
     private class ArrayBackedCharSequence extends AbstractCharSequence implements CachedCharSequence {

diff --git a/core/src/main/java/io/questdb/cutlass/line/udp/LineProtoSender.java b/core/src/main/java/io/questdb/cutlass/line/udp/LineProtoSender.java
@@ -134,8 +134,8 @@ public LineProtoSender field(CharSequence name, CharSequence value) {
         return this;
     }
 
-    public LineProtoSender field(CharSequence name, double value, int scale) {
-        field(name).put(value, scale);
+    public LineProtoSender field(CharSequence name, double value) {
+        field(name).put(value);
         return this;
     }
 
@@ -154,11 +154,11 @@ public void flush() {
     public LineProtoSender put(CharSequence cs) {
         int l = cs.length();
         if (ptr + l < hi) {
-            Chars.strcpy(cs, l, ptr);
+            Chars.asciiStrCpy(cs, l, ptr);
         } else {
             send00();
             if (ptr + l < hi) {
-                Chars.strcpy(cs, l, ptr);
+                Chars.asciiStrCpy(cs, l, ptr);
             } else {
                 throw CairoException.instance(0).put("value too long");
             }
@@ -167,6 +167,22 @@ public LineProtoSender put(CharSequence cs) {
         return this;
     }
 
+    @Override
+    public CharSink put(char[] chars, int start, int len) {
+        if (ptr + len < hi) {
+            Chars.asciiCopyTo(chars, start, len, ptr);
+        } else {
+            send00();
+            if (ptr + len < hi) {
+                Chars.asciiCopyTo(chars, start, len, ptr);
+            } else {
+                throw CairoException.instance(0).put("value too long");
+            }
+        }
+        ptr += len;
+        return this;
+    }
+
     public LineProtoSender metric(CharSequence metric) {
         if (hasMetric) {
             throw CairoException.instance(0).put("duplicate metric");

diff --git a/core/src/main/java/io/questdb/cutlass/pgwire/PGConnectionContext.java b/core/src/main/java/io/questdb/cutlass/pgwire/PGConnectionContext.java
@@ -559,7 +559,7 @@ private void appendDoubleColumn(Record record, int columnIndex) {
             responseAsciiSink.setNullValue();
         } else {
             final long a = responseAsciiSink.skip();
-            responseAsciiSink.put(doubleValue, 3);
+            responseAsciiSink.put(doubleValue);
             responseAsciiSink.putLenEx(a);
         }
     }
@@ -1516,6 +1516,14 @@ public void put(BinarySequence sequence) {
             }
         }
 
+        @Override
+        public CharSink put(char[] chars, int start, int len) {
+            ensureCapacity(len);
+            Chars.asciiCopyTo(chars, start, len, sendBufferPtr);
+            sendBufferPtr += len;
+            return this;
+        }
+
         public void putLen(long start) {
             putInt(start, (int) (sendBufferPtr - start));
         }

diff --git a/core/src/main/java/io/questdb/cutlass/text/TextException.java b/core/src/main/java/io/questdb/cutlass/text/TextException.java
@@ -61,7 +61,7 @@ public TextException put(char c) {
     }
 
     public TextException put(double c) {
-        message.put(c, 6);
+        message.put(c);
         return this;
     }
 

diff --git a/core/src/main/java/io/questdb/griffin/CharacterStore.java b/core/src/main/java/io/questdb/griffin/CharacterStore.java
@@ -92,6 +92,14 @@ public CharSink put(char c) {
         return this;
     }
 
+    @Override
+    public CharSink put(char[] chars, int start, int len) {
+        for (int i = 0; i < len; i++) {
+            put(chars[start + i]);
+        }
+        return this;
+    }
+
     private void resizeAndPut(char c) {
         char[] next = new char[capacity * 2];
         System.arraycopy(chars, 0, next, 0, capacity);