OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
StringOps.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "StringOps.h"
18 #include "Shared/base64.h"
19 
20 #include <rapidjson/document.h>
21 #include <boost/algorithm/string/predicate.hpp>
22 
23 namespace StringOps_Namespace {
24 
25 boost::regex StringOp::generateRegex(const std::string& op_name,
26  const std::string& regex_pattern,
27  const std::string& regex_params,
28  const bool supports_sub_matches) {
29  bool is_case_sensitive = false;
30  bool is_case_insensitive = false;
31 
32  for (const auto& c : regex_params) {
33  switch (c) {
34  case 'c':
35  is_case_sensitive = true;
36  break;
37  case 'i':
38  is_case_insensitive = true;
39  break;
40  case 'e': {
41  if (!supports_sub_matches) {
42  throw std::runtime_error(op_name +
43  " does not support 'e' (sub-matches) option.");
44  }
45  // We use e to set sub-expression group in a separate initializer
46  // but need to have this entry to not error on the default path
47  break;
48  }
49  default: {
50  if (supports_sub_matches) {
51  throw std::runtime_error("Unrecognized regex parameter for " + op_name +
52  ", expected either 'c' 'i', or 'e'.");
53  }
54  throw std::runtime_error("Unrecognized regex parameter for " + op_name +
55  ", expected either 'c' or 'i'.");
56  }
57  }
58  }
59  if (!is_case_sensitive && !is_case_insensitive) {
60  throw std::runtime_error(op_name +
61  " params must either specify case-sensitivity ('c') or "
62  "case-insensitivity ('i').");
63  }
64  if (is_case_sensitive && is_case_insensitive) {
65  throw std::runtime_error(op_name +
66  " params cannot specify both case-sensitivity ('c') and "
67  "case-insensitivity ('i').");
68  }
69  if (is_case_insensitive) {
70  return boost::regex(regex_pattern,
71  boost::regex_constants::extended |
72  boost::regex_constants::optimize |
73  boost::regex_constants::icase);
74  } else {
75  return boost::regex(
76  regex_pattern,
77  boost::regex_constants::extended | boost::regex_constants::optimize);
78  }
79 }
80 
81 NullableStrType TryStringCast::operator()(const std::string& str) const {
82  UNREACHABLE() << "Invalid string output for TryStringCast";
83  return NullableStrType();
84 }
85 
86 Datum TryStringCast::numericEval(const std::string_view str) const {
87  if (str.empty()) {
88  return NullDatum(return_ti_);
89  }
90  // Need to make copy for now b/c StringToDatum can mod SQLTypeInfo arg
91  SQLTypeInfo return_ti(return_ti_);
92  try {
93  return StringToDatum(str, return_ti);
94  } catch (std::runtime_error& e) {
95  return NullDatum(return_ti);
96  }
97 }
98 
99 NullableStrType Position::operator()(const std::string& str) const {
100  UNREACHABLE() << "Invalid string output for Position";
101  return {};
102 }
103 
104 Datum Position::numericEval(const std::string_view str) const {
105  if (str.empty()) {
106  return NullDatum(return_ti_);
107  } else {
108  const int64_t str_len = str.size();
109  const int64_t wrapped_start = start_ >= 0 ? start_ : str_len + start_;
110  Datum return_datum;
111  const auto search_index = str.find(search_str_, wrapped_start);
112  if (search_index == std::string::npos) {
113  return_datum.bigintval = 0;
114  } else {
115  return_datum.bigintval = static_cast<int64_t>(search_index) + 1;
116  }
117  return return_datum;
118  }
119 }
120 
121 // Prefix length to consider for the Jaro-Winkler score.
122 constexpr int winkler_k_prefix_length = 4;
123 
124 // Scaling factor for the adjustment of the score.
125 constexpr double winkler_k_scaling_factor = 0.1;
126 
127 double compute_jaro_score(std::string_view s1, std::string_view s2) {
128  int s1_len = s1.size();
129  int s2_len = s2.size();
130 
131  if (s1_len == 0 || s2_len == 0) {
132  return 0.0;
133  }
134 
135  int match_distance = std::max(s1_len, s2_len) / 2 - 1;
136  std::vector<bool> s1_match(s1_len, false);
137  std::vector<bool> s2_match(s2_len, false);
138 
139  int matches = 0;
140  int transpositions = 0;
141 
142  for (int i = 0; i < s1_len; ++i) {
143  int start = std::max(0, i - match_distance);
144  int end = std::min(i + match_distance + 1, s2_len);
145 
146  for (int j = start; j < end; ++j) {
147  if (s2_match[j]) {
148  continue;
149  }
150  if (s1[i] != s2[j]) {
151  continue;
152  }
153  s1_match[i] = true;
154  s2_match[j] = true;
155  ++matches;
156  break;
157  }
158  }
159 
160  if (matches == 0) {
161  return 0.0;
162  }
163 
164  int k = 0;
165  for (int i = 0; i < s1_len; ++i) {
166  if (!s1_match[i]) {
167  continue;
168  }
169  while (!s2_match[k]) {
170  ++k;
171  }
172  if (s1[i] != s2[k]) {
173  ++transpositions;
174  }
175  ++k;
176  }
177 
178  double score = ((matches / (double)s1_len) + (matches / (double)s2_len) +
179  ((matches - transpositions / 2.0) / matches)) /
180  3.0;
181 
182  return score;
183 }
184 
185 double compute_jaro_winkler_score(std::string_view s1, std::string_view s2) {
186  double jaro_score = compute_jaro_score(s1, s2);
187 
188  int l = 0;
189  int n = std::min({static_cast<int>(s1.size()),
190  static_cast<int>(s2.size()),
192 
193  for (; l < n; ++l) {
194  if (s1[l] != s2[l]) {
195  break;
196  }
197  }
198 
199  double winkler_adjustment = l * winkler_k_scaling_factor * (1 - jaro_score);
200  double jaro_winkler_score = jaro_score + winkler_adjustment;
201 
202  return jaro_winkler_score * 100;
203 }
204 
205 NullableStrType JarowinklerSimilarity::operator()(const std::string& str) const {
206  UNREACHABLE() << "Invalid string output for Jarowinkler Similarity";
207  return {};
208 }
209 
210 Datum JarowinklerSimilarity::numericEval(const std::string_view str) const {
211  if (str.empty()) {
212  return NullDatum(return_ti_);
213  }
214  const double jaro_winkler_score = compute_jaro_winkler_score(str, str_literal_);
215  Datum return_datum;
216  return_datum.bigintval = static_cast<int64_t>(std::round(jaro_winkler_score));
217  return return_datum;
218 }
219 
220 Datum JarowinklerSimilarity::numericEval(const std::string_view str1,
221  const std::string_view str2) const {
222  if (str1.empty() || str2.empty()) {
223  return NullDatum(return_ti_);
224  }
225  const double jaro_winkler_score = compute_jaro_winkler_score(str1, str2);
226  Datum return_datum;
227  return_datum.bigintval = static_cast<int64_t>(std::round(jaro_winkler_score));
228  return return_datum;
229 }
230 
231 // int64_t compute_levenshtein_distance(std::string_view s1, std::string_view s2) {
232 // const size_t len1 = s1.size(), len2 = s2.size();
233 // std::vector<std::vector<size_t>> d(len1 + 1, std::vector<size_t>(len2 + 1));
234 //
235 // d[0][0] = 0;
236 // for (size_t i = 1; i <= len1; ++i) {
237 // d[i][0] = i;
238 // }
239 // for (size_t i = 1; i <= len2; ++i) {
240 // d[0][i] = i;
241 // }
242 //
243 // for (size_t i = 1; i <= len1; ++i) {
244 // for (size_t j = 1; j <= len2; ++j) {
245 // d[i][j] = std::min({d[i - 1][j] + 1,
246 // d[i][j - 1] + 1,
247 // d[i - 1][j - 1] + (s1[i - 1] == s2[j - 1] ? 0 : 1)});
248 // }
249 // }
250 //
251 // return d[len1][len2];
252 // }
253 
254 template <typename T>
255 T compute_levenshtein_distance_template(std::string_view s1, std::string_view s2) {
256  const size_t len1 = s1.size(), len2 = s2.size();
257  std::vector<std::vector<T>> d(len1 + 1, std::vector<T>(len2 + 1));
258 
259  d[0][0] = 0;
260  for (size_t i = 1; i <= len1; ++i) {
261  d[i][0] = i;
262  }
263  for (size_t i = 1; i <= len2; ++i) {
264  d[0][i] = i;
265  }
266 
267  for (size_t i = 1; i <= len1; ++i) {
268  for (size_t j = 1; j <= len2; ++j) {
269  d[i][j] = std::min({d[i - 1][j] + 1,
270  d[i][j - 1] + 1,
271  d[i - 1][j - 1] + (s1[i - 1] == s2[j - 1] ? 0 : 1)});
272  }
273  }
274 
275  return d[len1][len2];
276 }
277 
278 int64_t compute_levenshtein_distance(std::string_view s1, std::string_view s2) {
279  const size_t max_len = std::max(s1.size(), s2.size());
280 
281  if (max_len < 256) {
282  return compute_levenshtein_distance_template<uint8_t>(s1, s2);
283  } else if (max_len < 65536) {
284  return compute_levenshtein_distance_template<uint16_t>(s1, s2);
285  } else if (max_len < std::numeric_limits<uint32_t>::max()) {
286  return compute_levenshtein_distance_template<uint32_t>(s1, s2);
287  } else {
288  return compute_levenshtein_distance_template<uint64_t>(s1, s2);
289  }
290 }
291 
292 NullableStrType LevenshteinDistance::operator()(const std::string& str) const {
293  UNREACHABLE() << "Invalid string output for Levenshtein Distance";
294  return {};
295 }
296 
297 Datum LevenshteinDistance::numericEval(const std::string_view str) const {
298  if (str.empty()) {
299  return NullDatum(return_ti_);
300  }
301  const double levenshtein_distance = compute_levenshtein_distance(str, str_literal_);
302  Datum return_datum;
303  return_datum.bigintval = static_cast<int64_t>(std::round(levenshtein_distance));
304  return return_datum;
305 }
306 
307 Datum LevenshteinDistance::numericEval(const std::string_view str1,
308  const std::string_view str2) const {
309  if (str1.empty() || str2.empty()) {
310  return NullDatum(return_ti_);
311  }
312  const double levenshtein_distance = compute_levenshtein_distance(str1, str2);
313  Datum return_datum;
314  return_datum.bigintval = static_cast<int64_t>(std::round(levenshtein_distance));
315  return return_datum;
316 }
317 
318 NullableStrType Lower::operator()(const std::string& str) const {
319  std::string output_str(str);
321  output_str.begin(), output_str.end(), output_str.begin(), [](unsigned char c) {
322  return std::tolower(c);
323  });
324  return output_str;
325 }
326 
327 NullableStrType Upper::operator()(const std::string& str) const {
328  std::string output_str(str);
330  output_str.begin(), output_str.end(), output_str.begin(), [](unsigned char c) {
331  return std::toupper(c);
332  });
333  return output_str;
334 }
335 
336 NullableStrType InitCap::operator()(const std::string& str) const {
337  std::string output_str(str);
338  bool last_char_whitespace = true; // Beginning of string counts as whitespace
339  for (auto& c : output_str) {
340  if (isspace(c) || delimiter_bitmap_[reinterpret_cast<const uint8_t&>(c)]) {
341  last_char_whitespace = true;
342  continue;
343  }
344  if (last_char_whitespace) {
345  c = toupper(c);
346  last_char_whitespace = false;
347  } else {
348  c = tolower(c);
349  }
350  }
351  return output_str;
352 }
353 
354 NullableStrType Reverse::operator()(const std::string& str) const {
355  const std::string reversed_str = std::string(str.rbegin(), str.rend());
356  return reversed_str;
357 }
358 
359 NullableStrType Repeat::operator()(const std::string& str) const {
360  std::string repeated_str;
361  repeated_str.reserve(str.size() * n_);
362  for (size_t r = 0; r < n_; ++r) {
363  repeated_str += str;
364  }
365  return repeated_str;
366 }
367 
368 NullableStrType Concat::operator()(const std::string& str) const {
369  return reverse_order_ ? str_literal_ + str : str + str_literal_;
370 }
371 
372 NullableStrType Concat::operator()(const std::string& str1,
373  const std::string& str2) const {
374  return str1 + str2;
375 }
376 
377 NullableStrType Pad::operator()(const std::string& str) const {
378  return pad_mode_ == Pad::PadMode::LEFT ? lpad(str) : rpad(str);
379 }
380 
381 std::string Pad::lpad(const std::string& str) const {
382  const auto str_len = str.size();
383  const size_t chars_to_fill = str_len < padded_length_ ? padded_length_ - str_len : 0UL;
384  if (chars_to_fill == 0UL) {
385  return str.substr(0, padded_length_);
386  }
387  // If here we need to add characters from the padding_string_
388  // to fill the difference between str_len and padded_length_
389  if (padding_string_length_ == 1UL) {
390  return std::string(chars_to_fill, padding_char_) + str;
391  }
392 
393  std::string fitted_padding_str;
394  fitted_padding_str.reserve(chars_to_fill);
395  for (size_t i = 0; i < chars_to_fill; ++i) {
396  fitted_padding_str.push_back(padding_string_[i % padding_string_length_]);
397  }
398  return fitted_padding_str + str;
399 }
400 
401 std::string Pad::rpad(const std::string& str) const {
402  const auto str_len = str.size();
403  const size_t chars_to_fill = str_len < padded_length_ ? padded_length_ - str_len : 0UL;
404  if (chars_to_fill == 0UL) {
405  return str.substr(str_len - padded_length_, std::string::npos);
406  }
407  // If here we need to add characters from the padding_string_
408  // to fill the difference between str_len and padded_length_
409  if (padding_string_length_ == 1UL) {
410  return str + std::string(chars_to_fill, padding_char_);
411  }
412 
413  std::string fitted_padding_str;
414  fitted_padding_str.reserve(chars_to_fill);
415  for (size_t i = 0; i < chars_to_fill; ++i) {
416  fitted_padding_str.push_back(padding_string_[i % padding_string_length_]);
417  }
418  return str + fitted_padding_str;
419 }
420 
421 Pad::PadMode Pad::op_kind_to_pad_mode(const SqlStringOpKind op_kind) {
422  switch (op_kind) {
424  return PadMode::LEFT;
426  return PadMode::RIGHT;
427  default:
428  UNREACHABLE();
429  // Not reachable, but make compiler happy
430  return PadMode::LEFT;
431  };
432 }
433 
434 NullableStrType Trim::operator()(const std::string& str) const {
435  const auto str_len = str.size();
436  size_t trim_begin = 0;
437  if (trim_mode_ == TrimMode::LEFT || trim_mode_ == TrimMode::BOTH) {
438  while (trim_begin < str_len &&
439  trim_char_bitmap_[reinterpret_cast<const uint8_t&>(str[trim_begin])]) {
440  ++trim_begin;
441  }
442  }
443  size_t trim_end = str_len - 1;
444  if (trim_mode_ == TrimMode::RIGHT || trim_mode_ == TrimMode::BOTH) {
445  while (trim_end > trim_begin &&
446  trim_char_bitmap_[reinterpret_cast<const uint8_t&>(str[trim_end])]) {
447  --trim_end;
448  }
449  }
450  if (trim_begin == 0 && trim_end == str_len - 1) {
451  return str;
452  }
453  return str.substr(trim_begin, trim_end + 1 - trim_begin);
454 }
455 
456 Trim::TrimMode Trim::op_kind_to_trim_mode(const SqlStringOpKind op_kind) {
457  switch (op_kind) {
459  return Trim::TrimMode::BOTH;
461  return Trim::TrimMode::LEFT;
463  return Trim::TrimMode::RIGHT;
464  default:
465  UNREACHABLE();
466  // Not reachable, but make compiler happy
467  return Trim::TrimMode::BOTH;
468  };
469 }
470 
471 NullableStrType Substring::operator()(const std::string& str) const {
472  // If start_ is negative then we start abs(start_) characters from the end
473  // of the string
474  const int64_t str_len = str.size();
475  const int64_t wrapped_start = start_ >= 0 ? start_ : str_len + start_;
476  const size_t capped_start =
477  wrapped_start > str_len ? str_len : (wrapped_start < 0 ? 0 : wrapped_start);
478  return str.substr(capped_start, length_);
479 }
480 
481 NullableStrType Overlay::operator()(const std::string& base_str) const {
482  // If start_ is negative then we start abs(start_) characters from the end
483  // of the string
484  const int64_t str_len = base_str.size();
485  const int64_t wrapped_start = start_ >= 0 ? start_ : str_len + start_;
486  const size_t capped_start =
487  wrapped_start > str_len ? str_len : (wrapped_start < 0 ? 0 : wrapped_start);
488  std::string replaced_str = base_str.substr(0, capped_start);
489  replaced_str += insert_str_;
490  const size_t remainder_start =
491  std::min(wrapped_start + replacement_length_, size_t(str_len));
492  const size_t remainder_length = static_cast<size_t>(str_len) - remainder_start;
493  replaced_str += base_str.substr(remainder_start, remainder_length);
494  return replaced_str;
495 }
496 
497 NullableStrType Replace::operator()(const std::string& str) const {
498  std::string replaced_str(str);
499 
500  size_t search_start_index = 0;
501  while (true) {
502  search_start_index = replaced_str.find(pattern_str_, search_start_index);
503  if (search_start_index == std::string::npos) {
504  break;
505  }
506  replaced_str.replace(search_start_index, pattern_str_len_, replacement_str_);
507  search_start_index += replacement_str_len_;
508  }
509  return replaced_str;
510 }
511 
512 NullableStrType SplitPart::operator()(const std::string& str) const {
513  // If split_part_ is negative then it is taken as the number
514  // of split parts from the end of the string
515 
516  if (delimiter_ == "") {
517  return str;
518  }
519 
520  const size_t str_len = str.size();
521  size_t delimiter_pos = reverse_ ? str_len : 0UL;
522  size_t last_delimiter_pos;
523  size_t delimiter_idx = 0UL;
524 
525  do {
526  last_delimiter_pos = delimiter_pos;
527  delimiter_pos = reverse_ ? str.rfind(delimiter_, delimiter_pos - 1UL)
528  : str.find(delimiter_, delimiter_pos + delimiter_length_);
529  } while (delimiter_pos != std::string::npos && ++delimiter_idx < split_part_);
530 
531  if (delimiter_idx == 0UL && split_part_ == 1UL) {
532  // No delimiter was found, but the first match is requested, which here is
533  // the whole string
534  return str;
535  }
536 
537  if (delimiter_pos == std::string::npos &&
538  (delimiter_idx < split_part_ - 1UL || delimiter_idx < 1UL)) {
539  // split_part_ was out of range
540  return NullableStrType(); // null string
541  }
542 
543  if (reverse_) {
544  const size_t substr_start =
545  delimiter_pos == std::string::npos ? 0UL : delimiter_pos + delimiter_length_;
546  return str.substr(substr_start, last_delimiter_pos - substr_start);
547  } else {
548  const size_t substr_start =
549  split_part_ == 1UL ? 0UL : last_delimiter_pos + delimiter_length_;
550  return str.substr(substr_start, delimiter_pos - substr_start);
551  }
552 }
553 
554 NullableStrType RegexpReplace::operator()(const std::string& str) const {
555  const int64_t str_len = str.size();
556  const int64_t pos = start_pos_ < 0 ? str_len + start_pos_ : start_pos_;
557  const size_t wrapped_start = std::clamp(pos, int64_t(0), str_len);
558  if (occurrence_ == 0L) {
559  std::string result;
560  std::string::const_iterator replace_start(str.cbegin() + wrapped_start);
561  boost::regex_replace(std::back_inserter(result),
562  replace_start,
563  str.cend(),
564  regex_pattern_,
565  replacement_);
566  return str.substr(0UL, wrapped_start) + result;
567  } else {
568  const auto occurrence_match_pos = RegexpReplace::get_nth_regex_match(
569  str,
570  wrapped_start,
571  regex_pattern_,
572  occurrence_ > 0 ? occurrence_ - 1 : occurrence_);
573  if (occurrence_match_pos.first == std::string::npos) {
574  // No match found, return original string
575  return str;
576  }
577  std::string result;
578  std::string::const_iterator replace_start(str.cbegin() + occurrence_match_pos.first);
579  std::string::const_iterator replace_end(str.cbegin() + occurrence_match_pos.second);
580  std::string replaced_match;
581  boost::regex_replace(std::back_inserter(replaced_match),
582  replace_start,
583  replace_end,
584  regex_pattern_,
585  replacement_);
586  return str.substr(0UL, occurrence_match_pos.first) + replaced_match +
587  str.substr(occurrence_match_pos.second, std::string::npos);
588  }
589 }
590 
591 std::pair<size_t, size_t> RegexpReplace::get_nth_regex_match(
592  const std::string& str,
593  const size_t start_pos,
594  const boost::regex& regex_pattern,
595  const int64_t occurrence) {
596  std::vector<std::pair<size_t, size_t>> regex_match_positions;
597  std::string::const_iterator search_start(str.cbegin() + start_pos);
598  boost::smatch match;
599  int64_t match_idx = 0;
600  size_t string_pos = start_pos;
601  while (boost::regex_search(search_start, str.cend(), match, regex_pattern)) {
602  string_pos += match.position(size_t(0)) + match.length(0);
603  regex_match_positions.emplace_back(
604  std::make_pair(string_pos - match.length(0), string_pos));
605  if (match_idx++ == occurrence) {
606  return regex_match_positions.back();
607  }
608  search_start =
609  match.suffix().first; // Move to position after last char of matched string
610  // Position is relative to last match/initial iterator, so need to increment our
611  // string_pos accordingly
612  }
613  // occurrence only could have a valid match if negative here,
614  // but don't want to check in inner loop for performance reasons
615  const int64_t wrapped_match = occurrence >= 0 ? occurrence : match_idx + occurrence;
616  if (wrapped_match < 0 || wrapped_match >= match_idx) {
617  // Represents a non-match
618  return std::make_pair(std::string::npos, std::string::npos);
619  }
620  return regex_match_positions[wrapped_match];
621 }
622 
623 NullableStrType RegexpSubstr::operator()(const std::string& str) const {
624  const int64_t str_len = str.size();
625  const int64_t pos = start_pos_ < 0 ? str_len + start_pos_ : start_pos_;
626  const size_t wrapped_start = std::clamp(pos, int64_t(0), str_len);
627  int64_t match_idx = 0;
628  // Apears std::regex_search does not support string_view?
629  std::vector<std::string> regex_matches;
630  std::string::const_iterator search_start(str.cbegin() + wrapped_start);
631  boost::smatch match;
632  while (boost::regex_search(search_start, str.cend(), match, regex_pattern_)) {
633  if (match_idx++ == occurrence_) {
634  if (sub_match_info_.first) {
635  return RegexpSubstr::get_sub_match(match, sub_match_info_);
636  }
637  return NullableStrType(match[0]);
638  }
639  regex_matches.emplace_back(match[0]);
640  search_start =
641  match.suffix().first; // Move to position after last char of matched string
642  }
643  const int64_t wrapped_match = occurrence_ >= 0 ? occurrence_ : match_idx + occurrence_;
644  if (wrapped_match < 0 || wrapped_match >= match_idx) {
645  return NullableStrType();
646  }
647  if (sub_match_info_.first) {
648  return RegexpSubstr::get_sub_match(match, sub_match_info_);
649  }
650  return regex_matches[wrapped_match];
651 }
652 
653 std::string RegexpSubstr::get_sub_match(const boost::smatch& match,
654  const std::pair<bool, int64_t> sub_match_info) {
655  const int64_t num_sub_matches = match.size() - 1;
656  const int64_t wrapped_sub_match = sub_match_info.second >= 0
657  ? sub_match_info.second
658  : num_sub_matches + sub_match_info.second;
659  if (wrapped_sub_match < 0 || wrapped_sub_match >= num_sub_matches) {
660  return "";
661  }
662  return match[wrapped_sub_match + 1];
663 }
664 
665 std::pair<bool, int64_t> RegexpSubstr::set_sub_match_info(
666  const std::string& regex_pattern,
667  const int64_t sub_match_group_idx) {
668  if (regex_pattern.find("e", 0UL) == std::string::npos) {
669  return std::make_pair(false, 0UL);
670  }
671  return std::make_pair(
672  true, sub_match_group_idx > 0L ? sub_match_group_idx - 1 : sub_match_group_idx);
673 }
674 
675 // json_path must start with "lax $", "strict $" or "$" (case-insensitive).
676 JsonValue::JsonParseMode JsonValue::parse_json_parse_mode(std::string_view json_path) {
677  size_t const string_pos = json_path.find('$');
678  if (string_pos == 0) {
679  // Parsing mode was not explicitly specified, default to PARSE_MODE_LAX
680  return JsonValue::JsonParseMode::PARSE_MODE_LAX;
681  } else if (string_pos == std::string::npos) {
682  throw std::runtime_error("JSON search path must include a '$' literal.");
683  }
684  std::string_view const prefix = json_path.substr(0, string_pos);
685  if (boost::iequals(prefix, std::string_view("lax "))) {
686  return JsonValue::JsonParseMode::PARSE_MODE_LAX;
687  } else if (boost::iequals(prefix, std::string_view("strict "))) {
688  if constexpr (JsonValue::allow_strict_json_parsing) {
689  return JsonValue::JsonParseMode::PARSE_MODE_STRICT;
690  } else {
691  throw std::runtime_error("Strict parsing not currently supported for JSON_VALUE.");
692  }
693  } else {
694  throw std::runtime_error("Issue parsing JSON_VALUE Parse Mode.");
695  }
696 }
697 
698 std::vector<JsonValue::JsonKey> JsonValue::parse_json_path(const std::string& json_path) {
699  // Assume that parse_key_error_mode validated strict/lax mode
700  size_t string_pos = json_path.find("$");
701  if (string_pos == std::string::npos) {
702  throw std::runtime_error("JSON search path must begin with '$' literal.");
703  }
704  string_pos += 1; // Go to next character after $
705 
706  // Use tildas to enclose escaped regex string due to embedded ')"'
707  static const auto& key_regex = *new boost::regex(
708  R"~(^(\.(([[:alpha:]][[:alnum:]_-]*)|"([[:alpha:]][ [:alnum:]_-]*)"))|\[([[:digit:]]+)\])~",
709  boost::regex_constants::extended | boost::regex_constants::optimize);
710  static_assert(std::is_trivially_destructible_v<decltype(key_regex)>);
711 
712  std::string::const_iterator search_start(json_path.cbegin() + string_pos);
713  boost::smatch match;
714  std::vector<JsonKey> json_keys;
715  while (boost::regex_search(search_start, json_path.cend(), match, key_regex)) {
716  CHECK_EQ(match.size(), 6UL);
717  if (match.position(size_t(0)) != 0L) {
718  // Match wasn't found at beginning of string
719  throw std::runtime_error("JSON search path parsing error: '" + json_path + "'");
720  }
721  size_t matching_expr = 0;
722  if (match[3].matched) {
723  // simple object key
724  matching_expr = 3;
725  } else if (match[4].matched) {
726  // complex object key
727  matching_expr = 4;
728  } else if (match[5].matched) {
729  // array key
730  matching_expr = 5;
731  }
732  CHECK_GT(matching_expr, 0UL);
733  string_pos += match.length(0);
734 
735  const std::string key_match(match[matching_expr].first, match[matching_expr].second);
736  CHECK_GE(key_match.length(), 1UL);
737  if (isalpha(key_match[0])) {
738  // Object key
739  json_keys.emplace_back(JsonKey(key_match));
740  } else {
741  // Array key
742  json_keys.emplace_back(JsonKey(std::stoi(key_match)));
743  }
744  search_start =
745  match.suffix().first; // Move to position after last char of matched string
746  }
747  if (json_keys.empty()) {
748  throw std::runtime_error("No keys found in JSON search path.");
749  }
750  if (string_pos < json_path.size()) {
751  throw std::runtime_error("JSON path parsing error.");
752  }
753  return json_keys;
754 }
755 
756 NullableStrType JsonValue::operator()(const std::string& str) const {
757  rapidjson::Document document;
758  if (document.Parse(str.c_str()).HasParseError()) {
759  if constexpr (JsonValue::allow_strict_json_parsing) {
760  return handle_parse_error(str);
761  } else {
762  return NullableStrType();
763  }
764  }
765  rapidjson::Value& json_val = document;
766  for (const auto& json_key : json_keys_) {
767  switch (json_key.key_kind) {
768  case JsonKeyKind::JSON_OBJECT: {
769  if (!json_val.IsObject() || !json_val.HasMember(json_key.object_key)) {
770  if constexpr (JsonValue::allow_strict_json_parsing) {
771  return handle_key_error(str);
772  } else {
773  return NullableStrType();
774  }
775  }
776  json_val = json_val[json_key.object_key];
777  break;
778  }
779  case JsonKeyKind::JSON_ARRAY: {
780  if (!json_val.IsArray() || json_val.Size() <= json_key.array_key) {
781  if constexpr (JsonValue::allow_strict_json_parsing) {
782  return handle_key_error(str);
783  } else {
784  return NullableStrType();
785  }
786  }
787  json_val = json_val[json_key.array_key];
788  break;
789  }
790  }
791  }
792  // Now get value as string
793  if (json_val.IsString()) {
794  return NullableStrType(std::string(json_val.GetString()));
795  } else if (json_val.IsNumber()) {
796  if (json_val.IsDouble()) {
797  return NullableStrType(std::to_string(json_val.GetDouble()));
798  } else if (json_val.IsInt64()) {
799  return NullableStrType(std::to_string(json_val.GetInt64()));
800  } else if (json_val.IsUint64()) {
801  // Need to cover range of uint64 that can't fit int in64
802  return NullableStrType(std::to_string(json_val.GetUint64()));
803  } else {
804  // A bit defensive, as I'm fairly sure json does not
805  // support numeric types with widths > 64 bits, so may drop
806  if constexpr (JsonValue::allow_strict_json_parsing) {
807  return handle_key_error(str);
808  } else {
809  return NullableStrType();
810  }
811  }
812  } else if (json_val.IsBool()) {
813  return NullableStrType(std::string(json_val.IsTrue() ? "true" : "false"));
814  } else if (json_val.IsNull()) {
815  return NullableStrType();
816  } else {
817  // For any unhandled type - we may move this to a CHECK after gaining
818  // more confidence in prod
819  if constexpr (JsonValue::allow_strict_json_parsing) {
820  return handle_key_error(str);
821  } else {
822  return NullableStrType();
823  }
824  }
825 }
826 
827 NullableStrType Base64Encode::operator()(const std::string& str) const {
828  return shared::encode_base64(str);
829 }
830 
831 NullableStrType Base64Decode::operator()(const std::string& str) const {
832  return shared::decode_base64(str);
833 }
834 
835 std::string StringOps::operator()(const std::string& str) const {
836  NullableStrType modified_str(str);
837  if (modified_str.is_null) {
838  return ""; // How we currently represent dictionary-encoded nulls
839  }
840  for (const auto& string_op : string_ops_) {
841  modified_str = string_op->operator()(modified_str.str);
842  if (modified_str.is_null) {
843  return ""; // How we currently represent dictionary-encoded nulls
844  }
845  }
846  return modified_str.str;
847 }
848 
849 std::string StringOps::multi_input_eval(const std::string_view str1,
850  const std::string_view str2) const {
851  NullableStrType modified_str1(str1);
852  NullableStrType modified_str2(str2);
853  if (modified_str1.is_null || modified_str2.is_null) {
854  return ""; // How we currently represent dictionary-encoded nulls
855  }
856  for (const auto& string_op : string_ops_) {
857  modified_str1 = string_op->operator()(modified_str1.str, modified_str2.str);
858  if (modified_str1.is_null) {
859  return ""; // How we currently represent dictionary-encoded nulls
860  }
861  }
862  return modified_str1.str;
863 }
864 
865 std::string_view StringOps::operator()(const std::string_view sv,
866  std::string& sv_storage) const {
867  sv_storage = sv;
868  NullableStrType nullable_str(sv);
869  for (const auto& string_op : string_ops_) {
870  nullable_str = string_op->operator()(nullable_str.str);
871  if (nullable_str.is_null) {
872  return "";
873  }
874  }
875  sv_storage = nullable_str.str;
876  return sv_storage;
877 }
878 
879 Datum StringOps::numericEval(const std::string_view str) const {
880  const auto num_string_producing_ops = string_ops_.size() - 1;
881  if (num_string_producing_ops == 0UL) {
882  // Short circuit and avoid transformation to string if
883  // only have one string->numeric op
884  return string_ops_.back()->numericEval(str);
885  }
886  NullableStrType modified_str(str);
887  for (size_t string_op_idx = 0; string_op_idx < num_string_producing_ops;
888  ++string_op_idx) {
889  const auto& string_op = string_ops_[string_op_idx];
890  modified_str = string_op->operator()(modified_str.str);
891  if (modified_str.is_null) {
892  break;
893  }
894  }
895  return string_ops_.back()->numericEval(modified_str.str);
896 }
897 
898 Datum StringOps::numericEval(const std::string_view str1,
899  const std::string_view str2) const {
900  const auto num_string_producing_ops = string_ops_.size() - 1;
901  // All string ops should be evaluated before invoking
902  // numericEval with two non-literal string inputs, so
903  // num string producing ops should be 0 here
904  CHECK_EQ(num_string_producing_ops, 0UL);
905  return string_ops_.back()->numericEval(str1, str2);
906 }
907 
908 std::vector<std::unique_ptr<const StringOp>> StringOps::genStringOpsFromOpInfos(
909  const std::vector<StringOpInfo>& string_op_infos) const {
910  // Should we handle pure literal expressions here as well
911  // even though they are currently rewritten to string literals?
912  std::vector<std::unique_ptr<const StringOp>> string_ops;
913  string_ops.reserve(string_op_infos.size());
914  for (const auto& string_op_info : string_op_infos) {
915  string_ops.emplace_back(gen_string_op(string_op_info));
916  }
917  return string_ops;
918 }
919 
920 // Free functions follow
921 
922 std::unique_ptr<const StringOp> gen_string_op(const StringOpInfo& string_op_info) {
923  std::optional<std::string> var_string_optional_literal;
924  const auto op_kind = string_op_info.getOpKind();
925  const auto& return_ti = string_op_info.getReturnType();
926 
927  if (string_op_info.hasNullLiteralArg()) {
928  return std::make_unique<const NullOp>(var_string_optional_literal, op_kind);
929  }
930 
931  const auto num_non_variable_literals = string_op_info.numNonVariableLiterals();
932  if (string_op_info.hasVarStringLiteral()) {
933  CHECK_EQ(num_non_variable_literals + 1UL, string_op_info.numLiterals());
934  var_string_optional_literal = string_op_info.getStringLiteral(0);
935  }
936 
937  switch (op_kind) {
938  case SqlStringOpKind::LOWER: {
939  CHECK_EQ(num_non_variable_literals, 0UL);
940  return std::make_unique<const Lower>(var_string_optional_literal);
941  }
942  case SqlStringOpKind::UPPER: {
943  CHECK_EQ(num_non_variable_literals, 0UL);
944  return std::make_unique<const Upper>(var_string_optional_literal);
945  }
947  CHECK_EQ(num_non_variable_literals, 0UL);
948  return std::make_unique<const InitCap>(var_string_optional_literal);
949  }
951  CHECK_EQ(num_non_variable_literals, 0UL);
952  return std::make_unique<const Reverse>(var_string_optional_literal);
953  }
955  CHECK_EQ(num_non_variable_literals, 1UL);
956  const auto num_repeats_literal = string_op_info.getIntLiteral(1);
957  return std::make_unique<const Repeat>(var_string_optional_literal,
958  num_repeats_literal);
959  }
962  CHECK_GE(num_non_variable_literals, 0UL);
963  CHECK_LE(num_non_variable_literals, 1UL);
964  if (num_non_variable_literals == 1UL) {
965  const auto str_literal = string_op_info.getStringLiteral(1);
966  // Handle lhs literals by having RCONCAT operator set a flag
967  return std::make_unique<const Concat>(var_string_optional_literal,
968  str_literal,
969  op_kind == SqlStringOpKind::RCONCAT);
970  } else {
971  return std::make_unique<const Concat>(var_string_optional_literal);
972  }
973  }
975  case SqlStringOpKind::RPAD: {
976  CHECK_EQ(num_non_variable_literals, 2UL);
977  const auto padded_length_literal = string_op_info.getIntLiteral(1);
978  const auto padding_string_literal = string_op_info.getStringLiteral(2);
979  return std::make_unique<Pad>(var_string_optional_literal,
980  op_kind,
981  padded_length_literal,
982  padding_string_literal);
983  }
986  case SqlStringOpKind::RTRIM: {
987  CHECK_EQ(num_non_variable_literals, 1UL);
988  const auto trim_chars_literal = string_op_info.getStringLiteral(1);
989  return std::make_unique<Trim>(
990  var_string_optional_literal, op_kind, trim_chars_literal);
991  }
993  CHECK_GE(num_non_variable_literals, 1UL);
994  CHECK_LE(num_non_variable_literals, 2UL);
995  const auto start_pos_literal = string_op_info.getIntLiteral(1);
996  const bool has_length_literal = string_op_info.intLiteralArgAtIdxExists(2);
997  if (has_length_literal) {
998  const auto length_literal = string_op_info.getIntLiteral(2);
999  return std::make_unique<const Substring>(
1000  var_string_optional_literal, start_pos_literal, length_literal);
1001  } else {
1002  return std::make_unique<const Substring>(var_string_optional_literal,
1003  start_pos_literal);
1004  }
1005  }
1006  case SqlStringOpKind::OVERLAY: {
1007  CHECK_GE(num_non_variable_literals, 2UL);
1008  CHECK_LE(num_non_variable_literals, 3UL);
1009  const auto replace_string_literal = string_op_info.getStringLiteral(1);
1010  const auto start_pos_literal = string_op_info.getIntLiteral(2);
1011  const bool has_length_literal = string_op_info.intLiteralArgAtIdxExists(3);
1012  if (has_length_literal) {
1013  const auto length_literal = string_op_info.getIntLiteral(3);
1014  return std::make_unique<const Overlay>(var_string_optional_literal,
1015  replace_string_literal,
1016  start_pos_literal,
1017  length_literal);
1018  } else {
1019  return std::make_unique<const Overlay>(
1020  var_string_optional_literal, replace_string_literal, start_pos_literal);
1021  }
1022  }
1023  case SqlStringOpKind::REPLACE: {
1024  CHECK_GE(num_non_variable_literals, 2UL);
1025  CHECK_LE(num_non_variable_literals, 2UL);
1026  const auto pattern_string_literal = string_op_info.getStringLiteral(1);
1027  const auto replacement_string_literal = string_op_info.getStringLiteral(2);
1028  return std::make_unique<const Replace>(var_string_optional_literal,
1029  pattern_string_literal,
1030  replacement_string_literal);
1031  }
1033  CHECK_GE(num_non_variable_literals, 2UL);
1034  CHECK_LE(num_non_variable_literals, 2UL);
1035  const auto delimiter_literal = string_op_info.getStringLiteral(1);
1036  const auto split_part_literal = string_op_info.getIntLiteral(2);
1037  return std::make_unique<const SplitPart>(
1038  var_string_optional_literal, delimiter_literal, split_part_literal);
1039  }
1041  CHECK_GE(num_non_variable_literals, 5UL);
1042  CHECK_LE(num_non_variable_literals, 5UL);
1043  const auto pattern_literal = string_op_info.getStringLiteral(1);
1044  const auto replacement_literal = string_op_info.getStringLiteral(2);
1045  const auto start_pos_literal = string_op_info.getIntLiteral(3);
1046  const auto occurrence_literal = string_op_info.getIntLiteral(4);
1047  const auto regex_params_literal = string_op_info.getStringLiteral(5);
1048  return std::make_unique<const RegexpReplace>(var_string_optional_literal,
1049  pattern_literal,
1050  replacement_literal,
1051  start_pos_literal,
1052  occurrence_literal,
1053  regex_params_literal);
1054  }
1056  CHECK_GE(num_non_variable_literals, 5UL);
1057  CHECK_LE(num_non_variable_literals, 5UL);
1058  const auto pattern_literal = string_op_info.getStringLiteral(1);
1059  const auto start_pos_literal = string_op_info.getIntLiteral(2);
1060  const auto occurrence_literal = string_op_info.getIntLiteral(3);
1061  const auto regex_params_literal = string_op_info.getStringLiteral(4);
1062  const auto sub_match_idx_literal = string_op_info.getIntLiteral(5);
1063  return std::make_unique<const RegexpSubstr>(var_string_optional_literal,
1064  pattern_literal,
1065  start_pos_literal,
1066  occurrence_literal,
1067  regex_params_literal,
1068  sub_match_idx_literal);
1069  }
1071  CHECK_EQ(num_non_variable_literals, 1UL);
1072  const auto json_path_literal = string_op_info.getStringLiteral(1);
1073  return std::make_unique<const JsonValue>(var_string_optional_literal,
1074  json_path_literal);
1075  }
1077  CHECK_EQ(num_non_variable_literals, 0UL);
1078  return std::make_unique<const Base64Encode>(var_string_optional_literal);
1079  }
1081  CHECK_EQ(num_non_variable_literals, 0UL);
1082  return std::make_unique<const Base64Decode>(var_string_optional_literal);
1083  }
1085  CHECK_EQ(num_non_variable_literals, 0UL);
1086  return std::make_unique<const TryStringCast>(return_ti,
1087  var_string_optional_literal);
1088  }
1090  CHECK_GE(num_non_variable_literals, 1UL);
1091  CHECK_LE(num_non_variable_literals, 2UL);
1092  const auto search_literal = string_op_info.getStringLiteral(1);
1093  const bool has_start_pos_literal = string_op_info.intLiteralArgAtIdxExists(2);
1094  if (has_start_pos_literal) {
1095  const auto start_pos_literal = string_op_info.getIntLiteral(2);
1096  return std::make_unique<const Position>(
1097  var_string_optional_literal, search_literal, start_pos_literal);
1098  } else {
1099  return std::make_unique<const Position>(var_string_optional_literal,
1100  search_literal);
1101  }
1102  }
1104  CHECK_GE(num_non_variable_literals, 0UL);
1105  CHECK_LE(num_non_variable_literals, 1UL);
1106  if (num_non_variable_literals == 1UL) {
1107  const auto str_literal = string_op_info.getStringLiteral(1);
1108  return std::make_unique<const JarowinklerSimilarity>(var_string_optional_literal,
1109  str_literal);
1110  } else {
1111  return std::make_unique<const JarowinklerSimilarity>(var_string_optional_literal);
1112  }
1113  }
1115  CHECK_GE(num_non_variable_literals, 0UL);
1116  CHECK_LE(num_non_variable_literals, 1UL);
1117  if (num_non_variable_literals == 1UL) {
1118  const auto str_literal = string_op_info.getStringLiteral(1);
1119  return std::make_unique<const LevenshteinDistance>(var_string_optional_literal,
1120  str_literal);
1121  } else {
1122  return std::make_unique<const LevenshteinDistance>(var_string_optional_literal);
1123  }
1124  }
1125  default: {
1126  UNREACHABLE();
1127  return std::make_unique<NullOp>(var_string_optional_literal, op_kind);
1128  }
1129  }
1130  // Make compiler happy
1131  return std::make_unique<NullOp>(var_string_optional_literal, op_kind);
1132 }
1133 
1134 std::pair<std::string, bool /* is null */> apply_string_op_to_literals(
1135  const StringOpInfo& string_op_info) {
1136  CHECK(string_op_info.hasVarStringLiteral());
1137  if (string_op_info.hasNullLiteralArg()) {
1138  const std::string null_str{""};
1139  return std::make_pair(null_str, true);
1140  }
1141  const auto string_op = gen_string_op(string_op_info);
1142  return string_op->operator()().toPair();
1143 }
1144 
1146  CHECK(string_op_info.hasVarStringLiteral());
1147  const auto string_op = gen_string_op(string_op_info);
1148  return string_op->numericEval();
1149 }
1150 
1151 } // namespace StringOps_Namespace
#define CHECK_EQ(x, y)
Definition: Logger.h:301
const SQLTypeInfo & getReturnType() const
Definition: StringOpInfo.h:58
Datum apply_numeric_op_to_literals(const StringOpInfo &string_op_info)
Definition: StringOps.cpp:1145
T compute_levenshtein_distance_template(std::string_view s1, std::string_view s2)
Definition: StringOps.cpp:255
#define UNREACHABLE()
Definition: Logger.h:338
#define CHECK_GE(x, y)
Definition: Logger.h:306
SqlStringOpKind
Definition: sqldefs.h:89
bool intLiteralArgAtIdxExists(const size_t index) const
double compute_jaro_winkler_score(std::string_view s1, std::string_view s2)
Definition: StringOps.cpp:185
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
int64_t getIntLiteral(const size_t index) const
size_t numNonVariableLiterals() const
Definition: StringOpInfo.h:54
int64_t bigintval
Definition: Datum.h:74
std::pair< std::string, bool > apply_string_op_to_literals(const StringOpInfo &string_op_info)
Definition: StringOps.cpp:1134
Datum StringToDatum(const std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:339
OUTPUT transform(INPUT const &input, FUNC const &func)
Definition: misc.h:320
Datum NullDatum(const SQLTypeInfo &ti)
Definition: Datum.cpp:288
constexpr int winkler_k_prefix_length
Definition: StringOps.cpp:122
#define CHECK_LE(x, y)
Definition: Logger.h:304
double compute_jaro_score(std::string_view s1, std::string_view s2)
Definition: StringOps.cpp:127
int64_t compute_levenshtein_distance(std::string_view s1, std::string_view s2)
Definition: StringOps.cpp:278
std::string getStringLiteral(const size_t index) const
#define CHECK(condition)
Definition: Logger.h:291
std::string decode_base64(const std::string &val, bool trim_nulls)
Definition: base64.h:27
static std::string encode_base64(const std::string &val)
Definition: base64.h:45
const SqlStringOpKind & getOpKind() const
Definition: StringOpInfo.h:42
constexpr double n
Definition: Utm.h:38
constexpr double winkler_k_scaling_factor
Definition: StringOps.cpp:125
Definition: Datum.h:69
std::unique_ptr< const StringOp > gen_string_op(const StringOpInfo &string_op_info)
Definition: StringOps.cpp:922