#include <ParserNode.h>

Inheritance diagram for Parser::CreateModelStmt:

Collaboration diagram for Parser::CreateModelStmt:

Public Member Functions
	CreateModelStmt (const rapidjson::Value &payload)

const std::string &	get_model_name () const

const std::string &	get_select_query () const

void	execute (const Catalog_Namespace::SessionInfo &session, bool read_only_mode) override

void	train_model (const Catalog_Namespace::SessionInfo &session)

Public Member Functions inherited from Parser::DDLStmt
void	setColumnDescriptor (ColumnDescriptor &cd, const ColumnDef *coldef)

Public Member Functions inherited from Parser::Node
virtual	~Node ()

Private Member Functions
bool	check_model_exists ()

void	parse_model_options ()

std::string	build_model_query (const std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr)

Private Attributes
MLModelType	model_type_

std::string	model_name_

std::string	select_query_

bool	replace_

bool	if_not_exists_

std::list< std::unique_ptr < NameValueAssign > >	model_options_

std::ostringstream	options_oss_

size_t	num_options_ {0}

double	data_split_train_fraction_ {1.0}

double	data_split_eval_fraction_ {0.0}

std::string	model_predicted_var_

std::vector< std::string >	model_feature_vars_

std::vector< int64_t >	feature_permutations_

Detailed Description

Definition at line 1959 of file ParserNode.h.

Constructor & Destructor Documentation

Parser::CreateModelStmt::CreateModelStmt ( const rapidjson::Value & payload )

Definition at line 3439 of file ParserNode.cpp.

References CHECK, g_enable_ml_functions, get_ml_model_type_from_str(), if_not_exists_, json_bool(), json_str(), model_name_, model_options_, model_type_, Parser::anonymous_namespace{ParserNode.cpp}::parse_options(), replace_, and select_query_.

                                                               {
   if (!g_enable_ml_functions) {
     throw std::runtime_error("Cannot create model. ML functions are disabled.");
   }
   CHECK(payload.HasMember("name"));
   const std::string model_type_str = json_str(payload["type"]);
   model_type_ = get_ml_model_type_from_str(model_type_str);
   model_name_ = json_str(payload["name"]);
   replace_ = false;
   if (payload.HasMember("replace")) {
     replace_ = json_bool(payload["replace"]);
   }
 
   if_not_exists_ = false;
   if (payload.HasMember("ifNotExists")) {
     if_not_exists_ = json_bool(payload["ifNotExists"]);
   }
 
   CHECK(payload.HasMember("query"));
   select_query_ = json_str(payload["query"]);
   std::regex newline_re("\\n");
   std::regex backtick_re("`");
   select_query_ = std::regex_replace(select_query_, newline_re, " ");
   select_query_ = std::regex_replace(select_query_, backtick_re, "");
 
   // No need to ensure trailing semicolon as we will wrap this select statement
   // in a CURSOR as input to the train model table function
   parse_options(payload, model_options_);
 }

Here is the call graph for this function:

Member Function Documentation

std::string Parser::CreateModelStmt::build_model_query ( const std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr )

private

Definition at line 3643 of file ParserNode.cpp.

References query_state::QueryState::create(), data_split_train_fraction_, feature_permutations_, Parser::LocalQueryConnector::getColumnDescriptors(), is_regression_model(), model_feature_vars_, model_predicted_var_, model_type_, Parser::LocalQueryConnector::query(), and select_query_.

Referenced by train_model().

                                                                    {
   auto validate_query_state = query_state::QueryState::create(session_ptr, select_query_);
 
   LocalQueryConnector local_connector;
 
   auto validate_result = local_connector.query(
       validate_query_state->createQueryStateProxy(), select_query_, {}, true, false);
 
   auto column_descriptors_for_model_create =
       local_connector.getColumnDescriptors(validate_result, true);
 
   std::vector<size_t> categorical_feature_idxs;
   std::vector<size_t> numeric_feature_idxs;
   bool numeric_feature_seen = false;
   bool all_categorical_features_placed_first = true;
   bool model_has_predicted_var = is_regression_model(model_type_);
   model_feature_vars_.reserve(column_descriptors_for_model_create.size() -
                               (model_has_predicted_var ? 1 : 0));
   bool is_predicted = model_has_predicted_var ? true : false;
   size_t feature_idx = 0;
   for (auto& cd : column_descriptors_for_model_create) {
     // Check to see if the projected column is an expression without a user-provided
     // alias, as we don't allow this.
     if (cd.columnName.rfind("EXPR$", 0) == 0) {
       throw std::runtime_error(
           "All projected expressions (i.e. col * 2) that are not column references (i.e. "
           "col) must be aliased.");
     }
     if (is_predicted) {
       model_predicted_var_ = cd.columnName;
       if (!cd.columnType.is_number()) {
         throw std::runtime_error(
             "Numeric predicted column expression should be first argument to CREATE "
             "MODEL.");
       }
       is_predicted = false;
     } else {
       if (cd.columnType.is_number()) {
         numeric_feature_idxs.emplace_back(feature_idx);
         numeric_feature_seen = true;
       } else if (cd.columnType.is_string()) {
         categorical_feature_idxs.emplace_back(feature_idx);
         if (numeric_feature_seen) {
           all_categorical_features_placed_first = false;
         }
       } else {
         throw std::runtime_error("Feature column expression should be numeric or TEXT.");
       }
       model_feature_vars_.emplace_back(cd.columnName);
       feature_idx++;
     }
   }
   auto modified_select_query = select_query_;
   if (!all_categorical_features_placed_first) {
     std::ostringstream modified_query_oss;
     modified_query_oss << "SELECT ";
     if (model_has_predicted_var) {
       modified_query_oss << model_predicted_var_ << ", ";
     }
     for (auto categorical_feature_idx : categorical_feature_idxs) {
       modified_query_oss << model_feature_vars_[categorical_feature_idx] << ", ";
       feature_permutations_.emplace_back(static_cast<int64_t>(categorical_feature_idx));
     }
     for (auto numeric_feature_idx : numeric_feature_idxs) {
       modified_query_oss << model_feature_vars_[numeric_feature_idx];
       feature_permutations_.emplace_back(static_cast<int64_t>(numeric_feature_idx));
       if (numeric_feature_idx != numeric_feature_idxs.back()) {
         modified_query_oss << ", ";
       }
     }
     modified_query_oss << " FROM (" << modified_select_query << ")";
     modified_select_query = modified_query_oss.str();
   }
 
   if (data_split_train_fraction_ < 1.0) {
     std::ostringstream modified_query_oss;
     if (all_categorical_features_placed_first) {
       modified_query_oss << "SELECT * FROM (" << modified_select_query << ")";
     } else {
       modified_query_oss << modified_select_query;
     }
     modified_query_oss << " WHERE SAMPLE_RATIO(" << data_split_train_fraction_ << ")";
     modified_select_query = modified_query_oss.str();
   }
   return modified_select_query;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool Parser::CreateModelStmt::check_model_exists ( )

private

Definition at line 3529 of file ParserNode.cpp.

References g_ml_models, get_model_name(), if_not_exists_, MLModelMap::modelExists(), and replace_.

Referenced by train_model().

                                          {
   if (g_ml_models.modelExists(get_model_name())) {
     if (if_not_exists_) {
       // Returning true tells the caller we should just return early and silently (without
       // error)
       return true;
     }
     if (!replace_) {
       std::ostringstream error_oss;
       error_oss << "Model " << get_model_name() << " already exists.";
       throw std::runtime_error(error_oss.str());
     }
   }
   // Returning false tells the caller all is clear to proceed with the create model,
   // whether that means creating a new one or overwriting an existing model
   return false;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Parser::CreateModelStmt::execute	(	const Catalog_Namespace::SessionInfo &	session,
		bool	read_only_mode
	)

overridevirtual

Implements Parser::DDLStmt.

Definition at line 3790 of file ParserNode.cpp.

References model_name_, and train_model().

Referenced by heavydb.cursor.Cursor::executemany().

                                                    {
   if (read_only_mode) {
     throw std::runtime_error("CREATE MODEL invalid in read only mode.");
   }
 
   try {
     train_model(session);
   } catch (std::exception& e) {
     std::ostringstream error_oss;
     // Error messages from table functions come back like this:
     // Error executing table function: MLTableFunctions.hpp:269 linear_reg_fit_impl: No
     // rows exist in training input. Training input must at least contain 1 row.
 
     // We want to take everything after the function name, so we will search for the
     // third colon.
     // Todo(todd): Look at making this less hacky by setting a mode for the table
     // function that will return only the core error string and not the preprending
     // metadata
 
     auto get_error_substring = [](const std::string& message) -> std::string {
       size_t colon_position = std::string::npos;
       for (int i = 0; i < 3; ++i) {
         colon_position = message.find(':', colon_position + 1);
         if (colon_position == std::string::npos) {
           return message;
         }
       }
 
       if (colon_position + 2 >= message.length()) {
         return message;
       }
       return message.substr(colon_position + 2);
     };
 
     const auto error_substr = get_error_substring(e.what());
 
     error_oss << "Could not create model " << model_name_ << ". " << error_substr;
     throw std::runtime_error(error_oss.str());
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const std::string& Parser::CreateModelStmt::get_model_name ( ) const

inline

Definition at line 1963 of file ParserNode.h.

References model_name_.

Referenced by check_model_exists(), and train_model().

1963 { return model_name_; }

Parser::CreateModelStmt::model_name_

std::string model_name_

Definition: ParserNode.h:1971

Here is the caller graph for this function:

const std::string& Parser::CreateModelStmt::get_select_query ( ) const

inline

Definition at line 1964 of file ParserNode.h.

References select_query_.

1964 { return select_query_; }

Parser::CreateModelStmt::select_query_

std::string select_query_

Definition: ParserNode.h:1972

void Parser::CreateModelStmt::parse_model_options ( )

private

Definition at line 3547 of file ParserNode.cpp.

References data_split_eval_fraction_, data_split_train_fraction_, Parser::DoubleLiteral::get_doubleval(), Parser::IntLiteral::get_intval(), Parser::StringLiteral::get_stringval(), model_options_, num_options_, and options_oss_.

Referenced by train_model().

                                           {
   bool train_fraction_specified = false;
   bool eval_fraction_specified = false;
   for (auto& p : model_options_) {
     const auto key = boost::to_lower_copy<std::string>(*p->get_name());
     if (key == "train_fraction" || key == "data_split_train_fraction") {
       if (train_fraction_specified) {
         throw std::runtime_error(
             "Error parsing DATA_SPLIT_TRAIN_FRACTION value. "
             "Expected only one value.");
       }
       const DoubleLiteral* fp_literal =
           dynamic_cast<const DoubleLiteral*>(p->get_value());
       if (fp_literal != nullptr) {
         data_split_train_fraction_ = fp_literal->get_doubleval();
         if (data_split_train_fraction_ <= 0.0 || data_split_train_fraction_ > 1.0) {
           throw std::runtime_error(
               "Error parsing DATA_SPLIT_TRAIN_FRACTION value. "
               "Expected value between 0.0 and 1.0.");
         }
       } else {
         throw std::runtime_error(
             "Error parsing DATA_SPLIT_TRAIN_FRACTION value. "
             "Expected floating point value betwen 0.0 and 1.0.");
       }
       train_fraction_specified = true;
       continue;
     }
     if (key == "eval_fraction" || key == "data_split_eval_fraction") {
       if (eval_fraction_specified) {
         throw std::runtime_error(
             "Error parsing DATA_SPLIT_EVAL_FRACTION value. "
             "Expected only one value.");
       }
       const DoubleLiteral* fp_literal =
           dynamic_cast<const DoubleLiteral*>(p->get_value());
       if (fp_literal != nullptr) {
         data_split_eval_fraction_ = fp_literal->get_doubleval();
         if (data_split_eval_fraction_ < 0.0 || data_split_eval_fraction_ >= 1.0) {
           throw std::runtime_error(
               "Error parsing DATA_SPLIT_EVAL_FRACTION value. "
               "Expected value between 0.0 and 1.0.");
         }
       } else {
         throw std::runtime_error(
             "Error parsing DATA_SPLIT_EVAL_FRACTION value. "
             "Expected floating point value betwen 0.0 and 1.0.");
       }
       eval_fraction_specified = true;
       continue;
     }
     if (num_options_) {
       options_oss_ << ", ";
     }
     num_options_++;
     options_oss_ << key << " => ";
     const StringLiteral* str_literal = dynamic_cast<const StringLiteral*>(p->get_value());
     if (str_literal != nullptr) {
       options_oss_ << "'"
                    << boost::to_lower_copy<std::string>(*str_literal->get_stringval())
                    << "'";
       continue;
     }
     const IntLiteral* int_literal = dynamic_cast<const IntLiteral*>(p->get_value());
     if (int_literal != nullptr) {
       options_oss_ << int_literal->get_intval();
       continue;
     }
     const DoubleLiteral* fp_literal = dynamic_cast<const DoubleLiteral*>(p->get_value());
     if (fp_literal != nullptr) {
       options_oss_ << fp_literal->get_doubleval();
       continue;
     }
     throw std::runtime_error("Error parsing value.");
   }
 
   // First handle case where data_split_train_fraction was left to default value
   // and data_split_eval_fraction was specified. We shouldn't error here,
   // but rather set data_split_train_fraction to 1.0 - data_split_eval_fraction
   // Likewise if data_split_eval_fraction was left to default value and we have
   // a specified data_split_train_fraction, we should set data_split_eval_fraction
   // to 1.0 - data_split_train_fraction
   if (data_split_eval_fraction_ > 0.0 && data_split_train_fraction_ == 1.0) {
     data_split_train_fraction_ = 1.0 - data_split_eval_fraction_;
   } else if (data_split_eval_fraction_ == 0.0 && data_split_train_fraction_ < 1.0) {
     data_split_eval_fraction_ = 1.0 - data_split_train_fraction_;
   }
 
   // If data_split_train_fraction was specified, and data_split_train_fraction +
   // data_split_eval_fraction > 1.0, then we should error
   if (data_split_eval_fraction_ + data_split_train_fraction_ > 1.0) {
     throw std::runtime_error(
         "Error parsing DATA_SPLIT_TRAIN_FRACTION and DATA_SPLIT_EVAL_FRACTION values. "
         "Expected sum of values to be less than or equal to 1.0.");
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Parser::CreateModelStmt::train_model ( const Catalog_Namespace::SessionInfo & session )

Definition at line 3731 of file ParserNode.cpp.

References build_model_query(), check_model_exists(), query_state::QueryState::create(), data_split_eval_fraction_, data_split_train_fraction_, shared::encode_base64(), feature_permutations_, get_ml_model_type_str(), get_model_name(), model_feature_vars_, model_predicted_var_, model_type_, num_options_, options_oss_, parse_model_options(), Parser::LocalQueryConnector::query(), select_query_, and Parser::write_model_params_to_json().

Referenced by execute().

                                                                              {
   if (check_model_exists()) {
     // Will return true if model exists and if_not_exists_ is true, in this
     // case we should return only
     return;
   }
 
   parse_model_options();
 
   auto session_copy = session;
   auto session_ptr = std::shared_ptr<Catalog_Namespace::SessionInfo>(
       &session_copy, boost::null_deleter());
 
   // We need to do various manipulations on the raw select query, such
   // as adding in any sampling or feature permutation logic. All of this
   // work is encapsulated in build_model_query
 
   const auto modified_select_query = build_model_query(session_ptr);
 
   // We have to base64 encode the model metadata because depending on the query,
   // the training data can have single quotes that trips up the parsing of the combined
   // select query with this metadata embedded.
 
   // This is just a temporary workaround until we store this info in the Catalog
   // rather than in the stored model pointer itself (and have to pass the metadata
   // down through the table function call)
   const auto model_metadata =
       shared::encode_base64(write_model_params_to_json(model_predicted_var_,
                                                        model_feature_vars_,
                                                        select_query_,
                                                        data_split_train_fraction_,
                                                        data_split_eval_fraction_,
                                                        feature_permutations_));
   if (num_options_) {
     // The options string does not have a trailing comma,
     // so add it
     options_oss_ << ", ";
   }
   options_oss_ << "model_metadata => '" << model_metadata << "'";
 
   const std::string options_str = options_oss_.str();
 
   const std::string model_train_func = get_ml_model_type_str(model_type_) + "_FIT";
 
   std::ostringstream model_query_oss;
   model_query_oss << "SELECT * FROM TABLE(" << model_train_func << "(model_name=>'"
                   << get_model_name() << "', data=>CURSOR(" << modified_select_query
                   << ")";
   model_query_oss << ", " << options_str;
   model_query_oss << "))";
 
   std::string wrapped_model_query = model_query_oss.str();
   auto query_state = query_state::QueryState::create(session_ptr, wrapped_model_query);
   // Don't need result back from query, as the query will create the model
   LocalQueryConnector local_connector;
   local_connector.query(
       query_state->createQueryStateProxy(), wrapped_model_query, {}, false);
 }