OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DateTimeParser.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "DateTimeParser.h"
18 #include "StringTransform.h"
19 
20 #include <boost/algorithm/string/predicate.hpp>
21 
22 #include <algorithm>
23 #include <array>
24 #include <cctype>
25 #include <charconv>
26 #include <limits>
27 #include <sstream>
28 #include <vector>
29 
30 namespace {
31 
32 constexpr std::array<int, 12> month_prefixes{{int('j') << 16 | int('a') << 8 | int('n'),
33  int('f') << 16 | int('e') << 8 | int('b'),
34  int('m') << 16 | int('a') << 8 | int('r'),
35  int('a') << 16 | int('p') << 8 | int('r'),
36  int('m') << 16 | int('a') << 8 | int('y'),
37  int('j') << 16 | int('u') << 8 | int('n'),
38  int('j') << 16 | int('u') << 8 | int('l'),
39  int('a') << 16 | int('u') << 8 | int('g'),
40  int('s') << 16 | int('e') << 8 | int('p'),
41  int('o') << 16 | int('c') << 8 | int('t'),
42  int('n') << 16 | int('o') << 8 | int('v'),
43  int('d') << 16 | int('e') << 8 | int('c')}};
44 
45 constexpr std::array<std::string_view, 13> month_suffixes{
46  {""
47  "uary",
48  "ruary",
49  "ch",
50  "il",
51  "",
52  "e",
53  "y",
54  "ust",
55  "tember",
56  "ober",
57  "ember",
58  "ember"}};
59 
60 constexpr unsigned
61  pow_10[10]{1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000};
62 
63 // Return y-m-d minus 1970-01-01 in days according to Gregorian calendar.
64 // Credit: http://howardhinnant.github.io/date_algorithms.html#days_from_civil
65 int64_t daysFromCivil(int64_t y, unsigned const m, unsigned const d) {
66  y -= m <= 2;
67  int64_t const era = (y < 0 ? y - 399 : y) / 400;
68  unsigned const yoe = static_cast<unsigned>(y - era * 400); // [0, 399]
69  unsigned const doy = (153 * (m + (m <= 2 ? 9 : -3)) + 2) / 5 + d - 1; // [0, 365]
70  unsigned const doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096]
71  return era * 146097 + static_cast<int64_t>(doe) - 719468;
72 }
73 
74 // Order of entries correspond to enum class FormatType { Date, Time, Timezone }.
75 std::vector<std::vector<std::string_view>> formatViews() {
76  return {{{"%Y-%m-%d", "%m/%d/%y", "%m/%d/%Y", "%Y/%m/%d", "%d-%b-%y", "%d/%b/%Y"},
77  {"%I:%M:%S %p",
78  "%H:%M:%S",
79  "%I:%M %p",
80  "%H:%M",
81  "%H%M%S",
82  "%I . %M . %S %p",
83  "%I %p"},
84  {"%z"}}};
85 }
86 
87 // Optionally eat month name after first 3 letters. Assume first 3 letters are correct.
88 void eatMonth(unsigned const month, std::string_view& str) {
89  str.remove_prefix(3);
90  std::string_view const suffix = month_suffixes[month];
91  if (boost::algorithm::istarts_with(str, suffix)) {
92  str.remove_prefix(suffix.size());
93  }
94 }
95 
96 void eatSpace(std::string_view& str) {
97  while (!str.empty() && isspace(str.front())) {
98  str.remove_prefix(1);
99  }
100 }
101 
102 // Parse str as a number of maxlen and type T.
103 // Return value and consume from str on success,
104 // otherwise return std::nullopt and do not change str.
105 template <typename T>
106 std::optional<T> fromChars(std::string_view& str,
107  size_t maxlen = std::numeric_limits<size_t>::max()) {
108  T retval;
109  maxlen = std::min(maxlen, str.size());
110  auto const result = std::from_chars(str.data(), str.data() + maxlen, retval);
111  if (result.ec == std::errc()) {
112  str.remove_prefix(result.ptr - str.data());
113  return retval;
114  } else {
115  return std::nullopt;
116  }
117 }
118 
119 std::optional<int64_t> unixTime(std::string_view const str) {
120  int64_t time{0};
121  auto const result = std::from_chars(str.data(), str.data() + str.size(), time);
122  // is_valid = str =~ /^-?\d+(\.\d*)$/
123  bool const is_valid = result.ec == std::errc() &&
124  (result.ptr == str.data() + str.size() ||
125  (*result.ptr == '.' &&
126  std::all_of(result.ptr + 1, str.data() + str.size(), isdigit)));
127  return is_valid ? std::make_optional(time) : std::nullopt;
128 }
129 
130 } // namespace
131 
132 // Interpret str according to DateTimeParser::FormatType::Time.
133 // Return number of (s,ms,us,ns) since midnight based on dim in (0,3,6,9) resp.
134 template <>
135 std::optional<int64_t> dateTimeParseOptional<kTIME>(std::string_view str,
136  unsigned const dim) {
137  if (!str.empty() && str.front() == 'T') {
138  str.remove_prefix(1);
139  }
142  std::optional<int64_t> time = parser.parse(str, dim);
143  if (!time) {
144  return std::nullopt;
145  }
146  // Parse optional timezone
147  std::string_view timezone = parser.unparsed();
149  std::optional<int64_t> tz = parser.parse(timezone, dim);
150  if (!parser.unparsed().empty()) {
151  return std::nullopt;
152  }
153  return *time + tz.value_or(0);
154 }
155 
156 // Interpret str according to DateTimeParser::FormatType::Date and Time.
157 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
158 template <>
159 std::optional<int64_t> dateTimeParseOptional<kTIMESTAMP>(std::string_view str,
160  unsigned const dim) {
161  if (!str.empty() && str.front() == 'T') {
162  str.remove_prefix(1);
163  }
165  // Parse date
167  std::optional<int64_t> date = parser.parse(str, dim);
168  if (!date) {
169  return unixTime(str);
170  }
171  // Parse time-of-day
172  std::string_view time_of_day = parser.unparsed();
173  if (time_of_day.empty()) {
174  return std::nullopt;
175  } else if (time_of_day.front() == 'T' || time_of_day.front() == ':') {
176  time_of_day.remove_prefix(1);
177  }
179  std::optional<int64_t> time = parser.parse(time_of_day, dim);
180  // Parse optional timezone
181  std::string_view timezone = parser.unparsed();
183  std::optional<int64_t> tz = parser.parse(timezone, dim);
184  return *date + time.value_or(0) + tz.value_or(0);
185 }
186 
187 // Interpret str according to DateTimeParser::FormatType::Date.
188 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
189 template <>
190 std::optional<int64_t> dateTimeParseOptional<kDATE>(std::string_view str,
191  unsigned const dim) {
193  // Parse date
195  std::optional<int64_t> date = parser.parse(str, dim);
196  if (!date) {
197  return unixTime(str);
198  }
199  // Parse optional timezone
200  std::string_view timezone = parser.unparsed();
202  std::optional<int64_t> tz = parser.parse(timezone, dim);
203  return *date + tz.value_or(0);
204 }
205 
206 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
207 int64_t DateTimeParser::DateTime::getTime(unsigned const dim) const {
208  int64_t const days = daysFromCivil(Y, m, d);
209  int const seconds = static_cast<int>(3600 * H + 60 * M + S) - z +
210  (p ? *p && H != 12 ? 12 * 3600
211  : !*p && H == 12 ? -12 * 3600
212  : 0
213  : 0);
214  return (24 * 3600 * days + seconds) * pow_10[dim] + n / pow_10[9 - dim];
215 }
216 
217 // Return true if successful parse, false otherwise. Update dt_ and str.
218 // OK to be destructive to str on failed match.
219 bool DateTimeParser::parseWithFormat(std::string_view format, std::string_view& str) {
220  while (!format.empty()) {
221  if (format.front() == '%') {
222  eatSpace(str);
223  if (!updateDateTimeAndStr(format[1], str)) {
224  return false;
225  }
226  format.remove_prefix(2);
227  } else if (isspace(format.front())) {
228  eatSpace(format);
229  eatSpace(str);
230  } else if (!str.empty() && format.front() == str.front()) {
231  format.remove_prefix(1);
232  str.remove_prefix(1);
233  } else {
234  return false;
235  }
236  }
237  return true;
238 }
239 
240 // Update dt_ based on given str and current value of format_type_.
241 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
242 // or std::nullopt if no format matches str.
243 // In either case, update unparsed_ to the remaining part of str that was not matched.
244 std::optional<int64_t> DateTimeParser::parse(std::string_view const str, unsigned dim) {
245  static std::vector<std::vector<std::string_view>> const& format_views = formatViews();
246  auto const& formats = format_views.at(static_cast<int>(format_type_));
247  for (std::string_view const format : formats) {
248  std::string_view str_unparsed = str;
249  if (parseWithFormat(format, str_unparsed)) {
250  unparsed_ = str_unparsed;
251  return dt_.getTime(dim);
252  }
253  }
254  unparsed_ = str;
255  return std::nullopt;
256 }
257 
259  dt_ = DateTime();
260 }
261 
263  resetDateTime();
264  format_type_ = format_type;
265 }
266 
267 std::string_view DateTimeParser::unparsed() const {
268  return unparsed_;
269 }
270 
271 // Return true if successful parse, false otherwise. Update dt_ and str on success.
272 // OK to be destructive to str on failed parse.
273 bool DateTimeParser::updateDateTimeAndStr(char const field, std::string_view& str) {
274  switch (field) {
275  case 'Y':
276  if (auto const year = fromChars<int64_t>(str)) {
277  dt_.Y = *year;
278  return true;
279  }
280  return false;
281  case 'y':
282  // %y matches 1 or 2 digits. If 3 or more digits are provided,
283  // then it is considered an unsuccessful parse.
284  if (auto const year = fromChars<unsigned>(str)) {
285  if (*year < 69) {
286  dt_.Y = 2000 + *year;
287  return true;
288  } else if (*year < 100) {
289  dt_.Y = 1900 + *year;
290  return true;
291  }
292  }
293  return false;
294  case 'm':
295  if (auto const month = fromChars<unsigned>(str, 2)) {
296  if (1 <= *month && *month <= 12) {
297  dt_.m = *month;
298  return true;
299  }
300  }
301  return false;
302  case 'b':
303  if (3 <= str.size()) {
304  int const key =
305  std::tolower(str[0]) << 16 | std::tolower(str[1]) << 8 | std::tolower(str[2]);
306  constexpr auto end = month_prefixes.data() + month_prefixes.size();
307  // This is faster than a lookup into a std::unordered_map.
308  auto const ptr = std::find(month_prefixes.data(), end, key);
309  if (ptr != end) {
310  dt_.m = ptr - month_prefixes.data() + 1;
311  eatMonth(dt_.m, str);
312  return true;
313  }
314  }
315  return false;
316  case 'd':
317  if (auto const day = fromChars<unsigned>(str, 2)) {
318  if (1 <= *day && *day <= 31) {
319  dt_.d = *day;
320  return true;
321  }
322  }
323  return false;
324  case 'H':
325  if (auto const hour = fromChars<unsigned>(str, 2)) {
326  if (*hour <= 23) {
327  dt_.H = *hour;
328  return true;
329  }
330  }
331  return false;
332  case 'I':
333  if (auto const hour = fromChars<unsigned>(str, 2)) {
334  if (1 <= *hour && *hour <= 12) {
335  dt_.H = *hour;
336  return true;
337  }
338  }
339  return false;
340  case 'M':
341  if (auto const minute = fromChars<unsigned>(str, 2)) {
342  if (*minute <= 59) {
343  dt_.M = *minute;
344  return true;
345  }
346  }
347  return false;
348  case 'S':
349  if (auto const second = fromChars<unsigned>(str, 2)) {
350  if (*second <= 61) {
351  dt_.S = *second;
352  if (!str.empty() && str.front() == '.') {
353  str.remove_prefix(1);
354  size_t len = str.size();
355  if (auto const ns = fromChars<unsigned>(str, 9)) {
356  len -= str.size();
357  dt_.n = *ns * pow_10[9 - len];
358  } else {
359  return false; // Reject period not followed by a digit
360  }
361  }
362  return true;
363  }
364  }
365  return false;
366  case 'z':
367  // [-+]\d\d:?\d\d
368  if (5 <= str.size() && (str.front() == '-' || str.front() == '+') &&
369  isdigit(str[1]) && isdigit(str[2]) && isdigit(str[4]) &&
370  (str[3] == ':' ? 6 <= str.size() && isdigit(str[5]) : isdigit(str[3]))) {
371  char const* sep = &str[3];
372  int hours{0}, minutes{0};
373  std::from_chars(str.data() + 1, sep, hours);
374  sep += *sep == ':';
375  std::from_chars(sep, sep + 2, minutes);
376  dt_.z = (str.front() == '-' ? -60 : 60) * (60 * hours + minutes);
377  str.remove_prefix(sep - str.data() + 2);
378  return true;
379  }
380  return false;
381  case 'p':
382  // %p implies optional, so never return false
383  if (boost::algorithm::istarts_with(str, "am") ||
384  boost::algorithm::istarts_with(str, "pm") ||
385  boost::algorithm::istarts_with(str, "a.m.") ||
386  boost::algorithm::istarts_with(str, "p.m.")) {
387  dt_.p = std::tolower(str.front()) == 'p';
388  str.remove_prefix(std::tolower(str[1]) == 'm' ? 2 : 4);
389  } else {
390  dt_.p.reset();
391  }
392  return true;
393  default:
394  throw std::runtime_error(cat("Unrecognized format: %", field));
395  }
396 }
397 
398 std::ostream& operator<<(std::ostream& out, DateTimeParser::DateTime const& dt) {
399  return out << dt.Y << '-' << dt.m << '-' << dt.d << ' ' << dt.H << ':' << dt.M << ':'
400  << dt.S << '.' << dt.n << " p("
401  << (dt.p ? *dt.p ? "true" : "false" : "unset") << ") z(" << dt.z << ')';
402 }
std::optional< bool > p
std::optional< int64_t > parse(std::string_view const, unsigned dim)
std::string cat(Ts &&...args)
bool updateDateTimeAndStr(char const field, std::string_view &)
std::optional< int64_t > unixTime(std::string_view const str)
std::ostream & operator<<(std::ostream &os, const SessionInfo &session_info)
Definition: SessionInfo.cpp:57
std::string_view unparsed_
std::vector< std::vector< std::string_view > > formatViews()
void eatMonth(unsigned const month, std::string_view &str)
bool parseWithFormat(std::string_view format, std::string_view &str)
void eatSpace(std::string_view &str)
std::string suffix(SQLTypes type)
Definition: Codegen.cpp:69
std::optional< int64_t > dateTimeParseOptional< kTIME >(std::string_view str, unsigned const dim)
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:33
std::optional< T > fromChars(std::string_view &str, size_t maxlen=std::numeric_limits< size_t >::max())
std::optional< int64_t > dateTimeParseOptional< kDATE >(std::string_view str, unsigned const dim)
std::optional< int64_t > dateTimeParseOptional< kTIMESTAMP >(std::string_view str, unsigned const dim)
int64_t getTime(unsigned const dim) const
constexpr std::array< int, 12 > month_prefixes
constexpr std::array< std::string_view, 13 > month_suffixes
FormatType format_type_
int64_t daysFromCivil(int64_t y, unsigned const m, unsigned const d)
void setFormatType(FormatType)
std::string_view unparsed() const