Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit 969b429

Browse files
JacobStocklassJacobStocklass
authored
feat: Add CivilTimeEncoder to encode and decode DateTime/Time as numerics (#937)
* Adding Time Encoding Integration Test Placeholder * Removing Stress test from this branch and moving it to the appropriate branch * Add integration test to make sure that encoding and decoding across a table insertion holds up * Added Integration test, renamed functions and got rid of redundant functions * Fix License Header * Java Lang set to 8 in order to use Java Local Time * Removing nano functions, cleaning up comments * Added round trip test to unit tests * Moving to threeten time instead of java time * Removing Java Time Dependency * Lint * Adding Time Encoding Integration Test Placeholder * Removing Stress test from this branch and moving it to the appropriate branch * Add integration test to make sure that encoding and decoding across a table insertion holds up * Added Integration test, renamed functions and got rid of redundant functions * Fix License Header * Java Lang set to 8 in order to use Java Local Time * Removing nano functions, cleaning up comments * Added round trip test to unit tests * Moving to threeten time instead of java time * Removing Java Time Dependency * Lint * Remove E2E test for another PR. Split Unit tests into better named tests * Lint * Combining Encode and Decode Test for easier reading * Removing unused methods
1 parent d781dc5 commit 969b429

File tree

2 files changed

+648
-0
lines changed

2 files changed

+648
-0
lines changed

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/CivilTimeEncoder.java‎

Lines changed: 314 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery.storage.v1beta2;
17+
18+
import static com.google.common.base.Preconditions.checkArgument;
19+
20+
import org.threeten.bp.DateTimeException;
21+
import org.threeten.bp.LocalDateTime;
22+
import org.threeten.bp.LocalTime;
23+
import org.threeten.bp.temporal.ChronoUnit;
24+
25+
/**
26+
* Ported from ZetaSQL CivilTimeEncoder Original code can be found at:
27+
* https://github.com/google/zetasql/blob/master/java/com/google/zetasql/CivilTimeEncoder.java
28+
* Encoder for TIME and DATETIME values, according to civil_time encoding.
29+
*
30+
* <p>The valid range and number of bits required by each date/time field is as the following:
31+
*
32+
* <table>
33+
* <tr> <th> Field </th> <th> Range </th> <th> #Bits </th> </tr>
34+
* <tr> <td> Year </td> <td> [1, 9999] </td> <td> 14 </td> </tr>
35+
* <tr> <td> Month </td> <td> [1, 12] </td> <td> 4 </td> </tr>
36+
* <tr> <td> Day </td> <td> [1, 31] </td> <td> 5 </td> </tr>
37+
* <tr> <td> Hour </td> <td> [0, 23] </td> <td> 5 </td> </tr>
38+
* <tr> <td> Minute </td> <td> [0, 59] </td> <td> 6 </td> </tr>
39+
* <tr> <td> Second </td> <td> [0, 59]* </td> <td> 6 </td> </tr>
40+
* <tr> <td> Micros </td> <td> [0, 999999] </td> <td> 20 </td> </tr>
41+
* <tr> <td> Nanos </td> <td> [0, 999999999] </td> <td> 30 </td> </tr>
42+
* </table>
43+
*
44+
* <p>* Leap second is not supported.
45+
*
46+
* <p>When encoding the TIME or DATETIME into a bit field, larger date/time field is on the more
47+
* significant side.
48+
*/
49+
public final class CivilTimeEncoder {
50+
private static final int NANO_LENGTH = 30;
51+
private static final int MICRO_LENGTH = 20;
52+
53+
private static final int NANO_SHIFT = 0;
54+
private static final int MICRO_SHIFT = 0;
55+
private static final int SECOND_SHIFT = 0;
56+
private static final int MINUTE_SHIFT = 6;
57+
private static final int HOUR_SHIFT = 12;
58+
private static final int DAY_SHIFT = 17;
59+
private static final int MONTH_SHIFT = 22;
60+
private static final int YEAR_SHIFT = 26;
61+
62+
private static final long NANO_MASK = 0x3FFFFFFFL;
63+
private static final long MICRO_MASK = 0xFFFFFL;
64+
private static final long SECOND_MASK = 0x3FL;
65+
private static final long MINUTE_MASK = 0xFC0L;
66+
private static final long HOUR_MASK = 0x1F000L;
67+
private static final long DAY_MASK = 0x3E0000L;
68+
private static final long MONTH_MASK = 0x3C00000L;
69+
private static final long YEAR_MASK = 0xFFFC000000L;
70+
71+
private static final long TIME_SECONDS_MASK = 0x1FFFFL;
72+
private static final long TIME_MICROS_MASK = 0x1FFFFFFFFFL;
73+
private static final long TIME_NANOS_MASK = 0x7FFFFFFFFFFFL;
74+
private static final long DATETIME_SECONDS_MASK = 0xFFFFFFFFFFL;
75+
private static final long DATETIME_MICROS_MASK = 0xFFFFFFFFFFFFFFFL;
76+
77+
/**
78+
* Encodes {@code time} as a 4-byte integer with seconds precision.
79+
*
80+
* <p>Encoding is as the following:
81+
*
82+
* <pre>
83+
* 3 2 1
84+
* MSB 10987654321098765432109876543210 LSB
85+
* | H || M || S |
86+
* </pre>
87+
*
88+
* @see #decodePacked32TimeSeconds(int)
89+
*/
90+
@SuppressWarnings("GoodTime-ApiWithNumericTimeUnit")
91+
private static int encodePacked32TimeSeconds(LocalTime time) {
92+
checkValidTimeSeconds(time);
93+
int bitFieldTimeSeconds = 0x0;
94+
bitFieldTimeSeconds |= time.getHour() << HOUR_SHIFT;
95+
bitFieldTimeSeconds |= time.getMinute() << MINUTE_SHIFT;
96+
bitFieldTimeSeconds |= time.getSecond() << SECOND_SHIFT;
97+
return bitFieldTimeSeconds;
98+
}
99+
100+
/**
101+
* Decodes {@code bitFieldTimeSeconds} as a {@link LocalTime} with seconds precision.
102+
*
103+
* <p>Encoding is as the following:
104+
*
105+
* <pre>
106+
* 3 2 1
107+
* MSB 10987654321098765432109876543210 LSB
108+
* | H || M || S |
109+
* </pre>
110+
*
111+
* @see #encodePacked32TimeSeconds(LocalTime)
112+
*/
113+
@SuppressWarnings("GoodTime-ApiWithNumericTimeUnit")
114+
private static LocalTime decodePacked32TimeSeconds(int bitFieldTimeSeconds) {
115+
checkValidBitField(bitFieldTimeSeconds, TIME_SECONDS_MASK);
116+
int hourOfDay = getFieldFromBitField(bitFieldTimeSeconds, HOUR_MASK, HOUR_SHIFT);
117+
int minuteOfHour = getFieldFromBitField(bitFieldTimeSeconds, MINUTE_MASK, MINUTE_SHIFT);
118+
int secondOfMinute = getFieldFromBitField(bitFieldTimeSeconds, SECOND_MASK, SECOND_SHIFT);
119+
// LocalTime validates the input parameters.
120+
try {
121+
return LocalTime.of(hourOfDay, minuteOfHour, secondOfMinute);
122+
} catch (DateTimeException e) {
123+
throw new IllegalArgumentException(e.getMessage(), e);
124+
}
125+
}
126+
127+
/**
128+
* Encodes {@code time} as a 8-byte integer with microseconds precision.
129+
*
130+
* <p>Encoding is as the following:
131+
*
132+
* <pre>
133+
* 6 5 4 3 2 1
134+
* MSB 3210987654321098765432109876543210987654321098765432109876543210 LSB
135+
* | H || M || S ||-------micros-----|
136+
* </pre>
137+
*
138+
* @see #decodePacked64TimeMicros(long)
139+
* @see #encodePacked64TimeMicros(LocalTime)
140+
*/
141+
@SuppressWarnings("GoodTime")
142+
public static long encodePacked64TimeMicros(LocalTime time) {
143+
checkValidTimeMicros(time);
144+
return (((long) encodePacked32TimeSeconds(time)) << MICRO_LENGTH) | (time.getNano() / 1_000L);
145+
}
146+
147+
/**
148+
* Decodes {@code bitFieldTimeMicros} as a {@link LocalTime} with microseconds precision.
149+
*
150+
* <p>Encoding is as the following:
151+
*
152+
* <pre>
153+
* 6 5 4 3 2 1
154+
* MSB 3210987654321098765432109876543210987654321098765432109876543210 LSB
155+
* | H || M || S ||-------micros-----|
156+
* </pre>
157+
*
158+
* @see #encodePacked64TimeMicros(LocalTime)
159+
*/
160+
@SuppressWarnings("GoodTime-ApiWithNumericTimeUnit")
161+
public static LocalTime decodePacked64TimeMicros(long bitFieldTimeMicros) {
162+
checkValidBitField(bitFieldTimeMicros, TIME_MICROS_MASK);
163+
int bitFieldTimeSeconds = (int) (bitFieldTimeMicros >> MICRO_LENGTH);
164+
LocalTime timeSeconds = decodePacked32TimeSeconds(bitFieldTimeSeconds);
165+
int microOfSecond = getFieldFromBitField(bitFieldTimeMicros, MICRO_MASK, MICRO_SHIFT);
166+
checkValidMicroOfSecond(microOfSecond);
167+
LocalTime time = timeSeconds.withNano(microOfSecond * 1000);
168+
checkValidTimeMicros(time);
169+
return time;
170+
}
171+
172+
/**
173+
* Encodes {@code dateTime} as a 8-byte integer with seconds precision.
174+
*
175+
* <p>Encoding is as the following:
176+
*
177+
* <pre>
178+
* 6 5 4 3 2 1
179+
* MSB 3210987654321098765432109876543210987654321098765432109876543210 LSB
180+
* |--- year ---||m || D || H || M || S |
181+
* </pre>
182+
*
183+
* @see #decodePacked64DatetimeSeconds(long)
184+
*/
185+
@SuppressWarnings("GoodTime-ApiWithNumericTimeUnit")
186+
private static long encodePacked64DatetimeSeconds(LocalDateTime dateTime) {
187+
checkValidDateTimeSeconds(dateTime);
188+
long bitFieldDatetimeSeconds = 0x0L;
189+
bitFieldDatetimeSeconds |= (long) dateTime.getYear() << YEAR_SHIFT;
190+
bitFieldDatetimeSeconds |= (long) dateTime.getMonthValue() << MONTH_SHIFT;
191+
bitFieldDatetimeSeconds |= (long) dateTime.getDayOfMonth() << DAY_SHIFT;
192+
bitFieldDatetimeSeconds |= (long) encodePacked32TimeSeconds(dateTime.toLocalTime());
193+
return bitFieldDatetimeSeconds;
194+
}
195+
196+
/**
197+
* Decodes {@code bitFieldDatetimeSeconds} as a {@link LocalDateTime} with seconds precision.
198+
*
199+
* <p>Encoding is as the following:
200+
*
201+
* <pre>
202+
* 6 5 4 3 2 1
203+
* MSB 3210987654321098765432109876543210987654321098765432109876543210 LSBa
204+
* |--- year ---||m || D || H || M || S |
205+
* </pre>
206+
*
207+
* @see #encodePacked64DatetimeSeconds(LocalDateTime)
208+
*/
209+
@SuppressWarnings("GoodTime-ApiWithNumericTimeUnit")
210+
private static LocalDateTime decodePacked64DatetimeSeconds(long bitFieldDatetimeSeconds) {
211+
checkValidBitField(bitFieldDatetimeSeconds, DATETIME_SECONDS_MASK);
212+
int bitFieldTimeSeconds = (int) (bitFieldDatetimeSeconds & TIME_SECONDS_MASK);
213+
LocalTime timeSeconds = decodePacked32TimeSeconds(bitFieldTimeSeconds);
214+
int year = getFieldFromBitField(bitFieldDatetimeSeconds, YEAR_MASK, YEAR_SHIFT);
215+
int monthOfYear = getFieldFromBitField(bitFieldDatetimeSeconds, MONTH_MASK, MONTH_SHIFT);
216+
int dayOfMonth = getFieldFromBitField(bitFieldDatetimeSeconds, DAY_MASK, DAY_SHIFT);
217+
try {
218+
LocalDateTime dateTime =
219+
LocalDateTime.of(
220+
year,
221+
monthOfYear,
222+
dayOfMonth,
223+
timeSeconds.getHour(),
224+
timeSeconds.getMinute(),
225+
timeSeconds.getSecond());
226+
checkValidDateTimeSeconds(dateTime);
227+
return dateTime;
228+
} catch (DateTimeException e) {
229+
throw new IllegalArgumentException(e.getMessage(), e);
230+
}
231+
}
232+
233+
/**
234+
* Encodes {@code dateTime} as a 8-byte integer with microseconds precision.
235+
*
236+
* <p>Encoding is as the following:
237+
*
238+
* <pre>
239+
* 6 5 4 3 2 1
240+
* MSB 3210987654321098765432109876543210987654321098765432109876543210 LSB
241+
* |--- year ---||m || D || H || M || S ||-------micros-----|
242+
* </pre>
243+
*
244+
* @see #decodePacked64DatetimeMicros(long)
245+
*/
246+
@SuppressWarnings({"GoodTime-ApiWithNumericTimeUnit", "JavaLocalDateTimeGetNano"})
247+
public static long encodePacked64DatetimeMicros(LocalDateTime dateTime) {
248+
checkValidDateTimeMicros(dateTime);
249+
return (encodePacked64DatetimeSeconds(dateTime) << MICRO_LENGTH)
250+
| (dateTime.getNano() / 1_000L);
251+
}
252+
253+
/**
254+
* Decodes {@code bitFieldDatetimeMicros} as a {@link LocalDateTime} with microseconds precision.
255+
*
256+
* <p>Encoding is as the following:
257+
*
258+
* <pre>
259+
* 6 5 4 3 2 1
260+
* MSB 3210987654321098765432109876543210987654321098765432109876543210 LSB
261+
* |--- year ---||m || D || H || M || S ||-------micros-----|
262+
* </pre>
263+
*
264+
* @see #encodePacked64DatetimeMicros(LocalDateTime)
265+
*/
266+
@SuppressWarnings("GoodTime-ApiWithNumericTimeUnit")
267+
public static LocalDateTime decodePacked64DatetimeMicros(long bitFieldDatetimeMicros) {
268+
checkValidBitField(bitFieldDatetimeMicros, DATETIME_MICROS_MASK);
269+
long bitFieldDatetimeSeconds = bitFieldDatetimeMicros >> MICRO_LENGTH;
270+
LocalDateTime dateTimeSeconds = decodePacked64DatetimeSeconds(bitFieldDatetimeSeconds);
271+
int microOfSecond = getFieldFromBitField(bitFieldDatetimeMicros, MICRO_MASK, MICRO_SHIFT);
272+
checkValidMicroOfSecond(microOfSecond);
273+
LocalDateTime dateTime = dateTimeSeconds.withNano(microOfSecond * 1_000);
274+
checkValidDateTimeMicros(dateTime);
275+
return dateTime;
276+
}
277+
278+
private static int getFieldFromBitField(long bitField, long mask, int shift) {
279+
return (int) ((bitField & mask) >> shift);
280+
}
281+
282+
private static void checkValidTimeSeconds(LocalTime time) {
283+
checkArgument(time.getHour() >= 0 && time.getHour() <= 23);
284+
checkArgument(time.getMinute() >= 0 && time.getMinute() <= 59);
285+
checkArgument(time.getSecond() >= 0 && time.getSecond() <= 59);
286+
}
287+
288+
private static void checkValidDateTimeSeconds(LocalDateTime dateTime) {
289+
checkArgument(dateTime.getYear() >= 1 && dateTime.getYear() <= 9999);
290+
checkArgument(dateTime.getMonthValue() >= 1 && dateTime.getMonthValue() <= 12);
291+
checkArgument(dateTime.getDayOfMonth() >= 1 && dateTime.getDayOfMonth() <= 31);
292+
checkValidTimeSeconds(dateTime.toLocalTime());
293+
}
294+
295+
private static void checkValidTimeMicros(LocalTime time) {
296+
checkValidTimeSeconds(time);
297+
checkArgument(time.equals(time.truncatedTo(ChronoUnit.MICROS)));
298+
}
299+
300+
private static void checkValidDateTimeMicros(LocalDateTime dateTime) {
301+
checkValidDateTimeSeconds(dateTime);
302+
checkArgument(dateTime.equals(dateTime.truncatedTo(ChronoUnit.MICROS)));
303+
}
304+
305+
private static void checkValidMicroOfSecond(int microOfSecond) {
306+
checkArgument(microOfSecond >= 0 && microOfSecond <= 999999);
307+
}
308+
309+
private static void checkValidBitField(long bitField, long mask) {
310+
checkArgument((bitField & ~mask) == 0x0L);
311+
}
312+
313+
private CivilTimeEncoder() {}
314+
}

0 commit comments

Comments
 (0)