OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
TableFunctionsFactory.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #include <boost/algorithm/string.hpp>
20 #include <mutex>
21 
22 extern bool g_enable_table_functions;
23 
24 namespace table_functions {
25 
26 namespace {
27 
29  switch (ext_arg_type) {
31  return SQLTypeInfo(kTINYINT, false);
33  return SQLTypeInfo(kSMALLINT, false);
35  return SQLTypeInfo(kINT, false);
37  return SQLTypeInfo(kBIGINT, false);
39  return SQLTypeInfo(kFLOAT, false);
41  return SQLTypeInfo(kDOUBLE, false);
43  return SQLTypeInfo(kBOOLEAN, false);
49  return generate_column_type(kINT);
72  default:
73  LOG(WARNING) << "ext_arg_pointer_type_to_type_info: ExtArgumentType `"
75  << "` conversion to SQLTypeInfo not implemented.";
76  UNREACHABLE();
77  }
78  UNREACHABLE();
79  return SQLTypeInfo(kNULLT, false);
80 }
81 
83  switch (ext_arg_type) {
88  return SQLTypeInfo(kTINYINT, false);
93  return SQLTypeInfo(kSMALLINT, false);
98  return SQLTypeInfo(kINT, false);
103  return SQLTypeInfo(kBIGINT, false);
108  return SQLTypeInfo(kFLOAT, false);
113  return SQLTypeInfo(kDOUBLE, false);
118  return SQLTypeInfo(kBOOLEAN, false);
122  return SQLTypeInfo(kTEXT, false, kENCODING_DICT);
123  default:
124  LOG(WARNING) << "ext_arg_pointer_type_to_type_info: ExtArgumentType `"
126  << "` conversion to SQLTypeInfo not implemented.";
127  UNREACHABLE();
128  }
129  UNREACHABLE();
130  return SQLTypeInfo(kNULLT, false);
131 }
132 
133 } // namespace
134 
136  CHECK_LT(idx, input_args_.size());
138 }
139 
141  CHECK_LT(idx, output_args_.size());
142  // TODO(adb): conditionally handle nulls
144 }
145 
147  int32_t scalar_args = 0;
148  for (const auto& ext_arg : input_args_) {
149  if (is_ext_arg_type_scalar(ext_arg)) {
150  scalar_args += 1;
151  }
152  }
153  return scalar_args;
154 }
155 
157  // workaround for default args
158  for (size_t idx = 0; idx < std::min(input_args_.size(), annotations_.size()); idx++) {
159  const auto& ann = getInputAnnotation(idx);
160  if (ann.find("require") != ann.end()) {
161  return true;
162  }
163  }
164  return false;
165 }
166 
167 const std::map<std::string, std::string>& TableFunction::getAnnotation(
168  const size_t idx) const {
169  if (annotations_.size() == 0) {
170  static const std::map<std::string, std::string> empty = {};
171  return empty;
172  }
173  CHECK_LT(idx, annotations_.size());
174  return annotations_[idx];
175 }
176 
177 const std::map<std::string, std::string>& TableFunction::getInputAnnotation(
178  const size_t input_arg_idx) const {
179  CHECK_LT(input_arg_idx, input_args_.size());
180  return getAnnotation(input_arg_idx);
181 }
182 
183 const std::map<std::string, std::string>& TableFunction::getOutputAnnotation(
184  const size_t output_arg_idx) const {
185  CHECK_LT(output_arg_idx, output_args_.size());
186  return getAnnotation(output_arg_idx + sql_args_.size());
187 }
188 
189 const std::map<std::string, std::string>& TableFunction::getFunctionAnnotation() const {
190  return getAnnotation(annotations_.size() - 1);
191 }
192 
193 std::pair<int32_t, int32_t> TableFunction::getInputID(const size_t idx) const {
194  // if the annotation is of the form args<INT,INT>, it is refering to a column list
195 #define PREFIX_LENGTH 5
196  const auto& annotation = getOutputAnnotation(idx);
197  auto annot = annotation.find("input_id");
198  if (annot == annotation.end()) {
199  size_t lo = 0;
200  for (const auto& ext_arg : input_args_) {
201  switch (ext_arg) {
205  return std::make_pair(lo, 0);
206  default:
207  lo++;
208  }
209  }
210  UNREACHABLE();
211  }
212 
213  const std::string& input_id = annot->second;
214 
215  size_t comma = input_id.find(",");
216  int32_t gt = input_id.size() - 1;
217  int32_t lo = std::stoi(input_id.substr(PREFIX_LENGTH, comma - 1));
218 
219  if (comma == std::string::npos) {
220  return std::make_pair(lo, 0);
221  }
222  int32_t hi = std::stoi(input_id.substr(comma + 1, gt - comma - 1));
223  return std::make_pair(lo, hi);
224 }
225 
227  /*
228  This function differs from getOutputRowSizeParameter() since it returns the correct
229  index for the sizer in the sql_args list. For instance, consider the example below:
230 
231  RowMultiplier=4
232  input_args=[{i32*, i64}, {i32*, i64}, {i32*, i64}, i32, {i32*, i64}, {i32*, i64},
233  i32] sql_args=[cursor, i32, cursor, i32]
234 
235  Non-scalar args are aggregated in a cursor inside the sql_args list and the new
236  sizer index is 2 rather than 4 originally specified.
237  */
238 
240  size_t sizer = getOutputRowSizeParameter(); // lookup until reach the sizer arg
241  int32_t ext_arg_index = 0, sql_arg_index = 0;
242 
243  auto same_kind = [&](const ExtArgumentType& ext_arg, const ExtArgumentType& sql_arg) {
244  return ((is_ext_arg_type_scalar(ext_arg) && is_ext_arg_type_scalar(sql_arg)) ||
246  };
247 
248  while ((size_t)ext_arg_index < sizer) {
249  if ((size_t)ext_arg_index == sizer - 1)
250  return sql_arg_index;
251 
252  const auto& ext_arg = input_args_[ext_arg_index];
253  const auto& sql_arg = sql_args_[sql_arg_index];
254 
255  if (same_kind(ext_arg, sql_arg)) {
256  ++ext_arg_index;
257  ++sql_arg_index;
258  } else {
259  CHECK(same_kind(ext_arg, sql_args_[sql_arg_index - 1]));
260  ext_arg_index += 1;
261  }
262  }
263 
264  CHECK(false);
265  }
266 
267  return getOutputRowSizeParameter();
268 }
269 
271  const std::string& name,
272  const TableFunctionOutputRowSizer sizer,
273  const std::vector<ExtArgumentType>& input_args,
274  const std::vector<ExtArgumentType>& output_args,
275  const std::vector<ExtArgumentType>& sql_args,
276  const std::vector<std::map<std::string, std::string>>& annotations,
277  bool is_runtime,
278  bool uses_manager) {
279  auto tf = TableFunction(name,
280  sizer,
281  input_args,
282  output_args,
283  sql_args,
284  annotations,
285  is_runtime,
286  uses_manager);
287  auto sig = tf.getSignature();
288  for (auto it = functions_.begin(); it != functions_.end();) {
289  if (it->second.getName() == name) {
290  if (it->second.isRuntime()) {
291  LOG(WARNING)
292  << "Overriding existing run-time table function (reset not called?): "
293  << name;
294  it = functions_.erase(it);
295  } else {
296  throw std::runtime_error("Will not override existing load-time table function: " +
297  name);
298  }
299  } else {
300  if (sig == it->second.getSignature() &&
301  ((tf.isCPU() && it->second.isCPU()) || (tf.isGPU() && it->second.isGPU()))) {
302  LOG(WARNING)
303  << "The existing (1) and added (2) table functions have the same signature `"
304  << sig << "`:\n"
305  << " 1: " << it->second.toString() << "\n 2: " << tf.toString() << "\n";
306  }
307  ++it;
308  }
309  }
310 
311  functions_.emplace(name, tf);
313  auto input_args2 = input_args;
314  input_args2.erase(input_args2.begin() + sizer.val - 1);
315 
316  auto sql_args2 = sql_args;
317  auto sql_sizer_pos = tf.getSqlOutputRowSizeParameter();
318  sql_args2.erase(sql_args2.begin() + sql_sizer_pos);
319 
321  sizer,
322  input_args2,
323  output_args,
324  sql_args2,
325  annotations,
326  is_runtime,
327  uses_manager);
328  auto sig = tf2.getSignature();
329  for (auto it = functions_.begin(); it != functions_.end();) {
330  if (sig == it->second.getSignature() &&
331  ((tf2.isCPU() && it->second.isCPU()) || (tf2.isGPU() && it->second.isGPU()))) {
332  LOG(WARNING)
333  << "The existing (1) and added (2) table functions have the same signature `"
334  << sig << "`:\n"
335  << " 1: " << it->second.toString() << "\n 2: " << tf2.toString() << "\n";
336  }
337  ++it;
338  }
339  functions_.emplace(name + DEFAULT_ROW_MULTIPLIER_SUFFIX, tf2);
340  }
341 }
342 
343 /*
344  The implementation for `void TableFunctionsFactory::init()` is
345  generated by QueryEngine/scripts/generate_TableFunctionsFactory_init.py
346 */
347 
348 // removes existing runtime table functions
351  return;
352  }
353  for (auto it = functions_.begin(); it != functions_.end();) {
354  if (it->second.isRuntime()) {
355  it = functions_.erase(it);
356  } else {
357  ++it;
358  }
359  }
360 }
361 
362 namespace {
363 
364 std::string drop_suffix_impl(const std::string& str) {
365  const auto idx = str.find("__");
366  if (idx == std::string::npos) {
367  return str;
368  }
369  CHECK_GT(idx, std::string::size_type(0));
370  return str.substr(0, idx);
371 }
372 
373 } // namespace
374 
375 std::string TableFunction::getName(const bool drop_suffix, const bool lower) const {
376  std::string result = name_;
377  if (drop_suffix) {
378  result = drop_suffix_impl(result);
379  }
380  if (lower) {
382  }
383  return result;
384 }
385 
387  // gets the name of the require check function associated with this table function
388  return getName() + REQUIRE_CHECK_SUFFIX;
389 }
390 
391 std::vector<TableFunction> TableFunctionsFactory::get_table_funcs(const std::string& name,
392  const bool is_gpu) {
393  std::vector<TableFunction> table_funcs;
394  auto table_func_name = name;
395  boost::algorithm::to_lower(table_func_name);
396  for (const auto& pair : functions_) {
397  auto fname = drop_suffix_impl(pair.first);
398  if (fname == table_func_name &&
399  (is_gpu ? pair.second.isGPU() : pair.second.isCPU())) {
400  table_funcs.push_back(pair.second);
401  }
402  }
403  return table_funcs;
404 }
405 
406 std::vector<TableFunction> TableFunctionsFactory::get_table_funcs(const bool is_runtime) {
407  std::vector<TableFunction> table_funcs;
408  for (const auto& pair : functions_) {
409  if (pair.second.isRuntime() == is_runtime) {
410  table_funcs.push_back(pair.second);
411  }
412  }
413  return table_funcs;
414 }
415 
416 std::unordered_map<std::string, TableFunction> TableFunctionsFactory::functions_;
417 
418 } // namespace table_functions
SQLTypeInfo getOutputSQLType(const size_t idx) const
std::string to_lower(const std::string &str)
bool is_ext_arg_type_scalar(const ExtArgumentType ext_arg_type)
#define PREFIX_LENGTH
#define LOG(tag)
Definition: Logger.h:203
auto generate_column_type(const SQLTypes subtype)
Definition: sqltypes.h:1119
string name
Definition: setup.in.py:72
SQLTypeInfo ext_arg_pointer_type_to_type_info(const ExtArgumentType ext_arg_type)
#define UNREACHABLE()
Definition: Logger.h:253
static void add(const std::string &name, const TableFunctionOutputRowSizer sizer, const std::vector< ExtArgumentType > &input_args, const std::vector< ExtArgumentType > &output_args, const std::vector< ExtArgumentType > &sql_args, const std::vector< std::map< std::string, std::string >> &annotations, bool is_runtime=false, bool uses_manager=false)
SQLTypeInfo ext_arg_type_to_type_info_output(const ExtArgumentType ext_arg_type)
const std::map< std::string, std::string > & getInputAnnotation(const size_t input_arg_idx) const
const std::vector< std::map< std::string, std::string > > annotations_
const std::vector< ExtArgumentType > output_args_
#define DEFAULT_ROW_MULTIPLIER_SUFFIX
std::pair< int32_t, int32_t > getInputID(const size_t idx) const
#define CHECK_GT(x, y)
Definition: Logger.h:221
const std::map< std::string, std::string > & getFunctionAnnotation() const
bool is_ext_arg_type_nonscalar(const ExtArgumentType ext_arg_type)
const std::vector< ExtArgumentType > sql_args_
SQLTypeInfo getInputSQLType(const size_t idx) const
auto generate_column_list_type(const SQLTypes subtype)
Definition: sqltypes.h:1133
const std::map< std::string, std::string > & getOutputAnnotation(const size_t output_arg_idx) const
std::string getName(const bool drop_suffix=false, const bool lower=false) const
#define CHECK_LT(x, y)
Definition: Logger.h:219
Definition: sqltypes.h:52
const std::map< std::string, std::string > & getAnnotation(const size_t idx) const
static std::string toString(const std::vector< ExtensionFunction > &ext_funcs, std::string tab="")
#define REQUIRE_CHECK_SUFFIX
#define CHECK(condition)
Definition: Logger.h:209
static std::vector< TableFunction > get_table_funcs(const std::string &name, const bool is_gpu)
static std::unordered_map< std::string, TableFunction > functions_
Definition: sqltypes.h:45
const std::vector< ExtArgumentType > input_args_
bool g_enable_table_functions
Definition: Execute.cpp:108