OmniSciDB  2e3a973ef4
DateTimeParser.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "DateTimeParser.h"
18 #include "StringTransform.h"
19 
20 #include <boost/algorithm/string/predicate.hpp>
21 
22 #include <algorithm>
23 #include <array>
24 #include <cctype>
25 #include <charconv>
26 #include <limits>
27 #include <sstream>
28 #include <vector>
29 
30 namespace {
31 
32 constexpr std::array<int, 12> month_prefixes{{int('j') << 16 | int('a') << 8 | int('n'),
33  int('f') << 16 | int('e') << 8 | int('b'),
34  int('m') << 16 | int('a') << 8 | int('r'),
35  int('a') << 16 | int('p') << 8 | int('r'),
36  int('m') << 16 | int('a') << 8 | int('y'),
37  int('j') << 16 | int('u') << 8 | int('n'),
38  int('j') << 16 | int('u') << 8 | int('l'),
39  int('a') << 16 | int('u') << 8 | int('g'),
40  int('s') << 16 | int('e') << 8 | int('p'),
41  int('o') << 16 | int('c') << 8 | int('t'),
42  int('n') << 16 | int('o') << 8 | int('v'),
43  int('d') << 16 | int('e') << 8 | int('c')}};
44 
45 constexpr std::array<std::string_view, 13> month_suffixes{
46  {""
47  "uary",
48  "ruary",
49  "ch",
50  "il",
51  "",
52  "e",
53  "y",
54  "ust",
55  "tember",
56  "ober",
57  "ember",
58  "ember"}};
59 
60 constexpr unsigned
61  pow_10[10]{1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000};
62 
63 // Return y-m-d minus 1970-01-01 in days according to Gregorian calendar.
64 // Credit: http://howardhinnant.github.io/date_algorithms.html#days_from_civil
65 int64_t daysFromCivil(int64_t y, unsigned const m, unsigned const d) {
66  y -= m <= 2;
67  int64_t const era = (y < 0 ? y - 399 : y) / 400;
68  unsigned const yoe = static_cast<unsigned>(y - era * 400); // [0, 399]
69  unsigned const doy = (153 * (m + (m <= 2 ? 9 : -3)) + 2) / 5 + d - 1; // [0, 365]
70  unsigned const doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096]
71  return era * 146097 + static_cast<int64_t>(doe) - 719468;
72 }
73 
74 // Order of entries correspond to enum class FormatType { Date, Time, Timezone }.
75 std::vector<std::vector<std::string_view>> formatViews() {
76  return {{{"%Y-%m-%d", "%m/%d/%Y", "%Y/%m/%d", "%d-%b-%y", "%d/%b/%Y"},
77  {"%I:%M:%S %p",
78  "%H:%M:%S",
79  "%I:%M %p",
80  "%H:%M",
81  "%H%M%S",
82  "%I %p",
83  "%I . %M . %S %p"},
84  {"%z"}}};
85 }
86 
87 // Optionally eat month name after first 3 letters. Assume first 3 letters are correct.
88 void eatMonth(unsigned const month, std::string_view& str) {
89  str.remove_prefix(3);
90  std::string_view const& suffix = month_suffixes[month];
91  if (boost::algorithm::istarts_with(str, suffix)) {
92  str.remove_prefix(suffix.size());
93  }
94 }
95 
96 void eatSpace(std::string_view& str) {
97  while (!str.empty() && isspace(str.front())) {
98  str.remove_prefix(1);
99  }
100 }
101 
102 // Parse str as a number of maxlen and type T.
103 // Return value and consume from str on success,
104 // otherwise return std::nullopt and do not change str.
105 template <typename T>
106 std::optional<T> fromChars(std::string_view& str,
107  size_t maxlen = std::numeric_limits<size_t>::max()) {
108  T retval;
109  maxlen = std::min(maxlen, str.size());
110  auto const result = std::from_chars(str.data(), str.data() + maxlen, retval);
111  if (result.ec == std::errc()) {
112  str.remove_prefix(result.ptr - str.data());
113  return retval;
114  } else {
115  return std::nullopt;
116  }
117 }
118 
119 // Parse str into int64_t or throw exception.
120 int64_t unixTime(std::string_view const& str) {
121  int64_t time{0};
122  auto const result = std::from_chars(str.data(), str.data() + str.size(), time);
123  if (result.ec == std::errc()) {
124  return time;
125  } else {
126  throw std::runtime_error(cat("Invalid DATE/TIMESTAMP string (", str, ')'));
127  }
128 }
129 } // namespace
130 
131 // Interpret str according to DateTimeParser::FormatType::Time.
132 // Return number of (s,ms,us,ns) since midnight based on dim in (0,3,6,9) resp.
133 template <>
134 int64_t dateTimeParse<kTIME>(std::string_view str, unsigned const dim) {
135  if (!str.empty() && str.front() == 'T') {
136  str.remove_prefix(1);
137  }
140  std::optional<int64_t> time = parser.parse(str, dim);
141  if (!time) {
142  throw std::runtime_error(cat("Invalid TIME string (", str, ')'));
143  }
144  // Parse optional timezone
145  std::string_view timezone = parser.unparsed();
147  std::optional<int64_t> tz = parser.parse(timezone, dim);
148  if (!parser.unparsed().empty()) {
149  throw std::runtime_error(cat("Invalid TIME string (", str, ')'));
150  }
151  return *time + tz.value_or(0);
152 }
153 
154 // Interpret str according to DateTimeParser::FormatType::Date and Time.
155 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
156 template <>
157 int64_t dateTimeParse<kTIMESTAMP>(std::string_view str, unsigned const dim) {
158  if (!str.empty() && str.front() == 'T') {
159  str.remove_prefix(1);
160  }
162  // Parse date
164  std::optional<int64_t> date = parser.parse(str, dim);
165  if (!date) {
166  return unixTime(str);
167  }
168  // Parse optional time-of-day
169  std::string_view time_of_day = parser.unparsed();
170  if (time_of_day.empty()) {
171  throw std::runtime_error(cat("TIMESTAMP requires a time-of-day (", str, ')'));
172  } else if (time_of_day.front() == 'T' || time_of_day.front() == ':') {
173  time_of_day.remove_prefix(1);
174  }
176  std::optional<int64_t> time = parser.parse(time_of_day, dim);
177  // Parse optional timezone
178  std::string_view timezone = parser.unparsed();
180  std::optional<int64_t> tz = parser.parse(timezone, dim);
181  return *date + time.value_or(0) + tz.value_or(0);
182 }
183 
184 // Interpret str according to DateTimeParser::FormatType::Date.
185 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
186 template <>
187 int64_t dateTimeParse<kDATE>(std::string_view str, unsigned const dim) {
189  // Parse date
191  std::optional<int64_t> date = parser.parse(str, dim);
192  if (!date) {
193  return unixTime(str);
194  }
195  // Parse optional timezone
196  std::string_view timezone = parser.unparsed();
198  std::optional<int64_t> tz = parser.parse(timezone, dim);
199  return *date + tz.value_or(0);
200 }
201 
202 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
203 int64_t DateTimeParser::DateTime::getTime(unsigned const dim) const {
204  int64_t const days = daysFromCivil(Y, m, d);
205  int const seconds =
206  static_cast<int>(3600 * H + 60 * M + S) - z +
207  (p ? *p && H != 12 ? 12 * 3600 : !*p && H == 12 ? -12 * 3600 : 0 : 0);
208  return (24 * 3600 * days + seconds) * pow_10[dim] + n / pow_10[9 - dim];
209 }
210 
211 // Return true if successful parse, false otherwise. Update dt_ and str.
212 // OK to be destructive to str on failed match.
213 bool DateTimeParser::parseWithFormat(std::string_view format, std::string_view& str) {
214  while (!format.empty()) {
215  if (format.front() == '%') {
216  eatSpace(str);
217  if (!updateDateTimeAndStr(format[1], str)) {
218  return false;
219  }
220  format.remove_prefix(2);
221  } else if (isspace(format.front())) {
222  eatSpace(format);
223  eatSpace(str);
224  } else if (!str.empty() && format.front() == str.front()) {
225  format.remove_prefix(1);
226  str.remove_prefix(1);
227  } else {
228  return false;
229  }
230  }
231  return true;
232 }
233 
234 // Update dt_ based on given str and current value of format_type_.
235 // Return number of (s,ms,us,ns) since epoch based on dim in (0,3,6,9) resp.
236 // or std::nullopt if no format matches str.
237 // In either case, update unparsed_ to the remaining part of str that was not matched.
238 std::optional<int64_t> DateTimeParser::parse(std::string_view const& str, unsigned dim) {
239  static std::vector<std::vector<std::string_view>> const& format_views = formatViews();
240  auto const& formats = format_views.at(static_cast<int>(format_type_));
241  for (std::string_view const& format : formats) {
242  std::string_view str_unparsed = str;
243  if (parseWithFormat(format, str_unparsed)) {
244  unparsed_ = str_unparsed;
245  return dt_.getTime(dim);
246  }
247  }
248  unparsed_ = str;
249  return std::nullopt;
250 }
251 
253  dt_ = DateTime();
254 }
255 
257  resetDateTime();
258  format_type_ = format_type;
259 }
260 
261 std::string_view DateTimeParser::unparsed() const {
262  return unparsed_;
263 }
264 
265 // Return true if successful parse, false otherwise. Update dt_ and str on success.
266 // OK to be destructive to str on failed parse.
267 bool DateTimeParser::updateDateTimeAndStr(char const field, std::string_view& str) {
268  switch (field) {
269  case 'Y':
270  if (auto const year = fromChars<int64_t>(str)) {
271  dt_.Y = *year;
272  return true;
273  }
274  return false;
275  case 'y':
276  if (2 <= str.size() && isdigit(str[0]) && isdigit(str[1])) {
277  if (auto const year = fromChars<unsigned>(str, 2)) {
278  dt_.Y = (*year < 69 ? 2000 : 1900) + *year;
279  return true;
280  }
281  }
282  return false;
283  case 'm':
284  if (auto const month = fromChars<unsigned>(str, 2)) {
285  if (1 <= *month && *month <= 12) {
286  dt_.m = *month;
287  return true;
288  }
289  }
290  return false;
291  case 'b':
292  if (3 <= str.size()) {
293  int const key =
294  std::tolower(str[0]) << 16 | std::tolower(str[1]) << 8 | std::tolower(str[2]);
295  constexpr auto end = month_prefixes.data() + month_prefixes.size();
296  // This is faster than a lookup into a std::unordered_map.
297  auto const ptr = std::find(month_prefixes.data(), end, key);
298  if (ptr != end) {
299  dt_.m = ptr - month_prefixes.data() + 1;
300  eatMonth(dt_.m, str);
301  return true;
302  }
303  }
304  return false;
305  case 'd':
306  if (auto const day = fromChars<unsigned>(str, 2)) {
307  if (1 <= *day && *day <= 31) {
308  dt_.d = *day;
309  return true;
310  }
311  }
312  return false;
313  case 'H':
314  if (auto const hour = fromChars<unsigned>(str, 2)) {
315  if (*hour <= 23) {
316  dt_.H = *hour;
317  return true;
318  }
319  }
320  return false;
321  case 'I':
322  if (auto const hour = fromChars<unsigned>(str, 2)) {
323  if (1 <= *hour && *hour <= 12) {
324  dt_.H = *hour;
325  return true;
326  }
327  }
328  return false;
329  case 'M':
330  if (auto const minute = fromChars<unsigned>(str, 2)) {
331  if (*minute <= 59) {
332  dt_.M = *minute;
333  return true;
334  }
335  }
336  return false;
337  case 'S':
338  if (auto const second = fromChars<unsigned>(str, 2)) {
339  if (*second <= 61) {
340  dt_.S = *second;
341  if (!str.empty() && str.front() == '.') {
342  str.remove_prefix(1);
343  size_t len = str.size();
344  if (auto const ns = fromChars<unsigned>(str, 9)) {
345  len -= str.size();
346  dt_.n = *ns * pow_10[9 - len];
347  } else {
348  return false; // Reject period not followed by a digit
349  }
350  }
351  return true;
352  }
353  }
354  return false;
355  case 'z':
356  // [-+]\d\d:?\d\d
357  if (5 <= str.size() && (str.front() == '-' || str.front() == '+') &&
358  isdigit(str[1]) && isdigit(str[2]) && isdigit(str[4]) &&
359  (str[3] == ':' ? 6 <= str.size() && isdigit(str[5]) : isdigit(str[3]))) {
360  char const* sep = &str[3];
361  int hours{0}, minutes{0};
362  std::from_chars(str.data() + 1, sep, hours);
363  sep += *sep == ':';
364  std::from_chars(sep, sep + 2, minutes);
365  dt_.z = (str.front() == '-' ? -60 : 60) * (60 * hours + minutes);
366  str.remove_prefix(sep - str.data() + 2);
367  return true;
368  }
369  return false;
370  case 'p':
371  // %p implies optional, so never return false
372  if (boost::algorithm::istarts_with(str, "am") ||
373  boost::algorithm::istarts_with(str, "pm") ||
374  boost::algorithm::istarts_with(str, "a.m.") ||
375  boost::algorithm::istarts_with(str, "p.m.")) {
376  dt_.p = std::tolower(str.front()) == 'p';
377  str.remove_prefix(std::tolower(str[1]) == 'm' ? 2 : 4);
378  } else {
379  dt_.p.reset();
380  }
381  return true;
382  default:
383  throw std::runtime_error(cat("Unrecognized format: %", field));
384  }
385 }
386 
387 std::ostream& operator<<(std::ostream& out, DateTimeParser::DateTime const& dt) {
388  return out << dt.Y << '-' << dt.m << '-' << dt.d << ' ' << dt.H << ':' << dt.M << ':'
389  << dt.S << '.' << dt.n << " p("
390  << (dt.p ? *dt.p ? "true" : "false" : "unset") << ") z(" << dt.z << ')';
391 }
std::optional< bool > p
bool updateDateTimeAndStr(char const field, std::string_view &)
int64_t dateTimeParse< kTIMESTAMP >(std::string_view str, unsigned const dim)
std::string_view unparsed_
std::vector< std::vector< std::string_view > > formatViews()
friend std::ostream & operator<<(std::ostream &, DateTime const &)
int64_t dateTimeParse< kDATE >(std::string_view str, unsigned const dim)
void eatMonth(unsigned const month, std::string_view &str)
bool parseWithFormat(std::string_view format, std::string_view &str)
void eatSpace(std::string_view &str)
int64_t getTime(unsigned const dim) const
int64_t unixTime(std::string_view const &str)
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
std::optional< int64_t > parse(std::string_view const &, unsigned dim)
std::optional< T > fromChars(std::string_view &str, size_t maxlen=std::numeric_limits< size_t >::max())
std::string cat(Ts &&... args)
constexpr std::array< int, 12 > month_prefixes
int64_t dateTimeParse< kTIME >(std::string_view str, unsigned const dim)
constexpr std::array< std::string_view, 13 > month_suffixes
std::string_view unparsed() const
FormatType format_type_
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t ** out
int64_t daysFromCivil(int64_t y, unsigned const m, unsigned const d)
void setFormatType(FormatType)