OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
StringTransform.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "StringTransform.h"
18 #include "Logger/Logger.h"
19 
20 #include <boost/algorithm/string/classification.hpp> // Include boost::for is_any_of
21 #include <boost/algorithm/string/split.hpp> // Include for boost::split
22 
23 #include <numeric>
24 #include <random>
25 #include <regex>
26 #include <string>
27 
28 #include <cmath> // format_bytes round call
29 #include <vector> // format_bytes
30 
31 #ifndef __CUDACC__
32 #include <boost/filesystem.hpp>
33 #include <iomanip>
34 #endif
35 
36 void apply_shim(std::string& result,
37  const boost::regex& reg_expr,
38  const std::function<void(std::string&, const boost::smatch&)>& shim_fn) {
39  boost::smatch what;
40  std::vector<std::pair<size_t, size_t>> lit_pos = find_string_literals(result);
41  auto start_it = result.cbegin();
42  auto end_it = result.cend();
43  while (true) {
44  if (!boost::regex_search(start_it, end_it, what, reg_expr)) {
45  break;
46  }
47  const auto next_start =
48  inside_string_literal(what.position(), what.length(), lit_pos);
49  if (next_start) {
50  start_it = result.cbegin() + *next_start;
51  } else {
52  shim_fn(result, what);
53  lit_pos = find_string_literals(result);
54  start_it = result.cbegin();
55  end_it = result.cend();
56  }
57  }
58 }
59 
60 std::vector<std::pair<size_t, size_t>> find_string_literals(const std::string& query) {
61  boost::regex literal_string_regex{R"(([^']+)('(?:[^']+|'')+'))", boost::regex::perl};
62  boost::smatch what;
63  auto it = query.begin();
64  auto prev_it = it;
65  std::vector<std::pair<size_t, size_t>> positions;
66  while (true) {
67  try {
68  if (!boost::regex_search(it, query.end(), what, literal_string_regex)) {
69  break;
70  }
71  } catch (const std::exception& e) {
72  LOG(WARNING) << "Error processing literals: " << e.what()
73  << "\nContinuing query parse...";
74  // boost::regex throws an exception about the complexity of matching when
75  // the wrong type of quotes are used or they're mismatched. Let the query
76  // through unmodified, the parser will throw a much more informative error.
77  // This can also throw on very long queries
78  break;
79  }
80  CHECK_GT(what[1].length(), 0);
81  prev_it = it;
82  it += what.length();
83  positions.emplace_back(prev_it + what[1].length() - query.begin(),
84  it - query.begin());
85  }
86  return positions;
87 }
88 
89 std::string hide_sensitive_data_from_query(std::string const& query_str) {
90  constexpr std::regex::flag_type flags =
91  std::regex::ECMAScript | std::regex::icase | std::regex::optimize;
92  static const std::initializer_list<std::pair<std::regex, std::string>> rules{
93  {std::regex(
94  R"(\b((?:password|s3_access_key|s3_secret_key|s3_session_token|username|credential_string)\s*=\s*)'.+?')",
95  flags),
96  "$1'XXXXXXXX'"},
97  {std::regex(R"((\\set_license\s+)\S+)", flags), "$1XXXXXXXX"}};
98  return std::accumulate(
99  rules.begin(), rules.end(), query_str, [](auto& str, auto& rule) {
100  return std::regex_replace(str, rule.first, rule.second);
101  });
102 }
103 
104 std::string format_num_bytes(const size_t bytes) {
105  const size_t units_per_k_unit{1024};
106  const std::vector<std::string> byte_units = {" bytes", "KB", "MB", "GB", "TB", "PB"};
107  const std::vector<size_t> bytes_per_scale_unit = {size_t(1),
108  size_t(1) << 10,
109  size_t(1) << 20,
110  size_t(1) << 30,
111  size_t(1) << 40,
112  size_t(1) << 50,
113  size_t(1) << 60};
114  if (bytes < units_per_k_unit) {
115  return std::to_string(bytes) + " bytes";
116  }
117  CHECK_GE(bytes, units_per_k_unit);
118  const size_t byte_scale = log(bytes) / log(units_per_k_unit);
119  CHECK_GE(byte_scale, size_t(1));
120  CHECK_LE(byte_scale, size_t(5));
121  const size_t scaled_bytes_left_of_decimal = bytes / bytes_per_scale_unit[byte_scale];
122  const size_t scaled_bytes_right_of_decimal = bytes % bytes_per_scale_unit[byte_scale];
123  const size_t fractional_digits = static_cast<double>(scaled_bytes_right_of_decimal) /
124  bytes_per_scale_unit[byte_scale] * 100.;
125  return std::to_string(scaled_bytes_left_of_decimal) + "." +
126  std::to_string(fractional_digits) + " " + byte_units[byte_scale];
127 }
128 
129 template <>
130 std::string to_string(char const*&& v) {
131  return std::string(v);
132 }
133 
134 template <>
135 std::string to_string(std::string&& v) {
136  return std::move(v);
137 }
138 
139 std::pair<std::string_view, const char*> substring(const std::string& str,
140  size_t substr_length) {
141  // return substring with a post_fix
142  // assume input str is valid and we perform substring starting from str's initial pos
143  // (=0)
144  const auto str_size = str.size();
145  if (substr_length >= str_size) {
146  return {str, ""};
147  }
148  std::string_view substr(str.c_str(), substr_length);
149  return {substr, "..."};
150 }
151 
152 std::string generate_random_string(const size_t len) {
153  static char charset[] =
154  "0123456789"
155  "abcdefghijklmnopqrstuvwxyz"
156  "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
157 
158  static std::mt19937 prng{std::random_device{}()};
159  static std::uniform_int_distribution<size_t> dist(0, strlen(charset) - 1);
160 
161  std::string str;
162  str.reserve(len);
163  for (size_t i = 0; i < len; i++) {
164  str += charset[dist(prng)];
165  }
166  return str;
167 }
168 
169 #ifndef __CUDACC__
170 // This version of split works almost exactly like Python's split,
171 // which is very convienently-designed.
172 // See also: https://docs.python.org/3.8/library/stdtypes.html#str.split
173 std::vector<std::string> split(std::string_view str,
174  std::string_view delim,
175  std::optional<size_t> maxsplit) {
176  std::vector<std::string> result;
177 
178  // Use an explicit delimiter.
179  if (!delim.empty()) {
180  std::string::size_type i = 0, j = 0;
181  while ((i = str.find(delim, i)) != std::string::npos &&
182  (!maxsplit || result.size() < maxsplit.value())) {
183  result.emplace_back(str, j, i - j);
184  i += delim.size();
185  j = i;
186  }
187  result.emplace_back(str, j, std::string::npos);
188  return result;
189 
190  // Treat any number of consecutive whitespace characters as a delimiter.
191  } else {
192  bool prev_ws = true;
193  std::string::size_type i = 0, j = 0;
194  for (; i < str.size(); ++i) {
195  if (prev_ws) {
196  if (!isspace(str[i])) {
197  // start of word
198  prev_ws = false;
199  j = i;
200  }
201  } else {
202  if (isspace(str[i])) {
203  // start of space
204  result.emplace_back(str, j, i - j);
205  prev_ws = true;
206  j = i;
207  if ((maxsplit && result.size() == maxsplit.value())) {
208  // stop early if maxsplit was reached
209  result.emplace_back(str, j, std::string::npos);
210  return result;
211  }
212  }
213  }
214  }
215  if (!prev_ws) {
216  result.emplace_back(str, j, std::string::npos);
217  }
218  return result;
219  }
220 }
221 
222 std::string_view sv_strip(std::string_view str) {
223  std::string::size_type i, j;
224  for (i = 0; i < str.size() && std::isspace(str[i]); ++i) {
225  }
226  for (j = str.size(); j > i && std::isspace(str[j - 1]); --j) {
227  }
228  return str.substr(i, j - i);
229 }
230 
231 std::string strip(std::string_view str) {
232  return std::string(sv_strip(str));
233 }
234 
235 std::optional<size_t> inside_string_literal(
236  const size_t start,
237  const size_t length,
238  const std::vector<std::pair<size_t, size_t>>& literal_positions) {
239  const auto end = start + length;
240  for (const auto& literal_position : literal_positions) {
241  if (literal_position.first <= start && end <= literal_position.second) {
242  return literal_position.second;
243  }
244  }
245  return std::nullopt;
246 }
247 
248 #endif // __CUDACC__
249 
251  std::string& str) noexcept {
252  char inside_quote = 0;
253  bool previous_c_was_backslash = false;
254  for (auto& c : str) {
255  // if this character is a quote of either type
256  if (c == '\'' || c == '\"') {
257  // ignore if previous character was a backslash
258  if (!previous_c_was_backslash) {
259  // start or end of a quoted region
260  if (inside_quote == c) {
261  // end region
262  inside_quote = 0;
263  } else if (inside_quote == 0) {
264  // start region
265  inside_quote = c;
266  }
267  }
268  } else if (inside_quote == 0) {
269  // outside quoted region
270  if (c == '\n' || c == '\t' || c == '\r') {
271  // replace these with space
272  c = ' ';
273  }
274  // otherwise leave alone, including quotes of a different type
275  }
276  // handle backslashes, except for double backslashes
277  if (c == '\\') {
278  previous_c_was_backslash = !previous_c_was_backslash;
279  } else {
280  previous_c_was_backslash = false;
281  }
282  }
283  // if we didn't end a region, there were unclosed or mixed-nested quotes
284  // accounting for backslashes should mean that this should only be the
285  // case with truly malformed strings which Calcite will barf on anyway
286  return (inside_quote == 0);
287 }
288 
289 #ifndef __CUDACC__
290 std::string get_quoted_string(const std::string& filename, char quote, char escape) {
291  std::stringstream ss;
292  ss << std::quoted(filename, quote, escape); // TODO: prevents string_view Jun 2020
293  return ss.str();
294 }
295 #endif // __CUDACC__
296 
297 #ifndef __CUDACC__
298 std::string simple_sanitize(const std::string& str) {
299  auto sanitized_str{str};
300  for (auto& c : sanitized_str) {
301  c = (c < 32) ? ' ' : c;
302  }
303  return sanitized_str;
304 }
305 #endif // __CUDACC__
std::string hide_sensitive_data_from_query(std::string const &query_str)
std::vector< std::pair< size_t, size_t > > find_string_literals(const std::string &query)
std::optional< size_t > inside_string_literal(const size_t start, const size_t length, const std::vector< std::pair< size_t, size_t >> &literal_positions)
std::string_view sv_strip(std::string_view str)
return trimmed string_view
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
#define LOG(tag)
Definition: Logger.h:285
#define CHECK_GE(x, y)
Definition: Logger.h:306
std::string simple_sanitize(const std::string &str)
simple sanitize string (replace control characters with space)
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
std::string get_quoted_string(const std::string &filename, char quote, char escape)
Quote a string while escaping any existing quotes in the string.
std::string generate_random_string(const size_t len)
DEVICE auto accumulate(ARGS &&...args)
Definition: gpu_enabled.h:42
void apply_shim(std::string &result, const boost::regex &reg_expr, const std::function< void(std::string &, const boost::smatch &)> &shim_fn)
std::string format_num_bytes(const size_t bytes)
#define CHECK_LE(x, y)
Definition: Logger.h:304
bool remove_unquoted_newlines_linefeeds_and_tabs_from_sql_string(std::string &str) noexcept
sanitize an SQL string
std::pair< std::string_view, const char * > substring(const std::string &str, size_t substr_length)
return substring of str with postfix if str.size() &gt; substr_length