EvaluateModelCommand Class Reference

#include <DdlCommandExecutor.h>

Inheritance diagram for EvaluateModelCommand:

Collaboration diagram for EvaluateModelCommand:

Public Member Functions
	EvaluateModelCommand (const DdlCommandData &ddl_data, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
ExecutionResult	execute (bool read_only_mode) override
Public Member Functions inherited from DdlCommand
	DdlCommand (const DdlCommandData &ddl_data, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)

Additional Inherited Members
Protected Attributes inherited from DdlCommand
const DdlCommandData &	ddl_data_
std::shared_ptr < Catalog_Namespace::SessionInfo const >	session_ptr_

Detailed Description

Definition at line 317 of file DdlCommandExecutor.h.

Constructor & Destructor Documentation

EvaluateModelCommand::EvaluateModelCommand	(	const DdlCommandData &	ddl_data,
		std::shared_ptr< Catalog_Namespace::SessionInfo const >	session_ptr
	)

Definition at line 2383 of file DdlCommandExecutor.cpp.

References g_enable_ml_functions.

    : DdlCommand(ddl_data, session_ptr) {
  if (!g_enable_ml_functions) {
    throw std::runtime_error("Cannot evaluate model. ML functions are disabled.");
  }
}

Member Function Documentation

ExecutionResult EvaluateModelCommand::execute ( bool  read_only_mode )

overridevirtual

Executes the DDL command corresponding to provided JSON payload.

Parameters

_return

result of DDL command execution (if applicable)

Implements DdlCommand.

Definition at line 2392 of file DdlCommandExecutor.cpp.

References CHECK_EQ, CHECK_LE, ResultSetLogicalValuesBuilder::create(), query_state::QueryState::create(), DdlCommand::ddl_data_, anonymous_namespace{DdlCommandExecutor.cpp}::extractPayload(), g_ml_models, anonymous_namespace{DdlCommandExecutor.cpp}::genLiteralDouble(), Parser::LocalQueryConnector::getColumnDescriptors(), legacylockmgr::getExecuteReadLock(), MLModelMap::getModelMetadata(), is_regression_model(), kDOUBLE, run_benchmark_import::label, Parser::LocalQueryConnector::query(), run_benchmark_import::result, and DdlCommand::session_ptr_.

Referenced by heavydb.cursor.Cursor::executemany().

                                                                 {
  auto execute_read_lock = legacylockmgr::getExecuteReadLock();
  auto& ddl_payload = extractPayload(ddl_data_);
  std::string model_name;
  std::string select_query;
  if (ddl_payload.HasMember("modelName")) {
    model_name = ddl_payload["modelName"].GetString();
  }
  if (ddl_payload.HasMember("query")) {
    select_query = ddl_payload["query"].GetString();
  }
  std::regex newline_re("\\n");
  std::regex backtick_re("`");
  select_query = std::regex_replace(select_query, newline_re, " ");
  select_query = std::regex_replace(select_query, backtick_re, "");
  const auto model_metadata = g_ml_models.getModelMetadata(model_name);
  const auto model_type = model_metadata.getModelType();
  if (!is_regression_model(model_type)) {
    const auto& model_type_str = model_metadata.getModelTypeStr();
    std::ostringstream error_oss;
    error_oss << "EVALUATE MODEL not supported for " << model_type_str << " models.";
    throw std::runtime_error(error_oss.str());
  }

  if (select_query.empty()) {
    const double data_split_eval_fraction = model_metadata.getDataSplitEvalFraction();
    CHECK_LE(data_split_eval_fraction, 1.0);
    if (data_split_eval_fraction <= 0.0) {
      throw std::runtime_error(
          "Unable to evaluate model: " + model_name +
          ". Model was not trained with a data split evaluation fraction.");
    }
    const auto& training_query = model_metadata.getTrainingQuery();
    const auto& feature_permutations = model_metadata.getFeaturePermutations();
    std::ostringstream select_query_oss;
    // To get a non-overlapping eval dataset (that does not overlap with the training
    // dataset), we need to use NOT SAMPLE_RATIO(training_fraction) and not
    // SAMPLE_RATIO(eval_fraction), as the latter will be a subset of the training
    // dataset
    const double data_split_train_fraction = 1.0 - data_split_eval_fraction;
    select_query_oss << "SELECT ";
    if (!feature_permutations.empty()) {
      select_query_oss << model_metadata.getPredicted() << ", ";
      const auto& features = model_metadata.getFeatures();
      for (const auto feature_permutation : feature_permutations) {
        select_query_oss << features[feature_permutation];
        if (feature_permutation != feature_permutations.back()) {
          select_query_oss << ", ";
        }
      }
    } else {
      select_query_oss << " * ";
    }

    select_query_oss << " FROM (" << training_query << ") WHERE NOT SAMPLE_RATIO("
                     << data_split_train_fraction << ")";
    select_query = select_query_oss.str();
  } else {
    const auto& feature_permutations = model_metadata.getFeaturePermutations();
    if (!feature_permutations.empty()) {
      Parser::LocalQueryConnector local_connector;
      auto validate_query_state =
          query_state::QueryState::create(session_ptr_, select_query);
      auto validate_result = local_connector.query(
          validate_query_state->createQueryStateProxy(), select_query, {}, true, false);
      auto column_descriptors_list =
          local_connector.getColumnDescriptors(validate_result, true);
      std::vector<ColumnDescriptor> column_descriptors;
      for (auto& cd : column_descriptors_list) {
        column_descriptors.emplace_back(cd);
      }
      std::ostringstream select_query_oss;
      select_query_oss << "SELECT " << model_metadata.getPredicted() << ", ";
      for (const auto feature_permutation : feature_permutations) {
        select_query_oss << column_descriptors[feature_permutation + 1].columnName;
        if (feature_permutation != feature_permutations.back()) {
          select_query_oss << ", ";
        }
      }
      select_query_oss << " FROM (" << select_query << ")";
      select_query = select_query_oss.str();
    }
  }
  std::ostringstream r2_query_oss;
  r2_query_oss << "SELECT * FROM TABLE(r2_score(model_name => '" << model_name << "', "
               << "data => CURSOR(" << select_query << ")))";
  std::string r2_query = r2_query_oss.str();

  try {
    Parser::LocalQueryConnector local_connector;
    auto query_state = query_state::QueryState::create(session_ptr_, r2_query);
    auto result =
        local_connector.query(query_state->createQueryStateProxy(), r2_query, {}, false);
    std::vector<std::string> labels{"r2"};
    std::vector<TargetMetaInfo> label_infos;
    for (const auto& label : labels) {
      label_infos.emplace_back(label, SQLTypeInfo(kDOUBLE, true));
    }
    std::vector<RelLogicalValues::RowValues> logical_values;
    logical_values.emplace_back(RelLogicalValues::RowValues{});

    CHECK_EQ(result.size(), size_t(1));
    CHECK_EQ(result[0].rs->rowCount(), size_t(1));
    CHECK_EQ(result[0].rs->colCount(), size_t(1));

    auto result_row = result[0].rs->getNextRow(true, true);

    auto scalar_r = boost::get<ScalarTargetValue>(&result_row[0]);
    auto p = boost::get<double>(scalar_r);

    logical_values.back().emplace_back(genLiteralDouble(*p));

    std::shared_ptr<ResultSet> rSet = std::shared_ptr<ResultSet>(
        ResultSetLogicalValuesBuilder::create(label_infos, logical_values));

    return ExecutionResult(rSet, label_infos);
  } catch (const std::exception& e) {
    std::ostringstream error_oss;
    // Error messages from table functions come back like this:
    // Error executing table function: MLTableFunctions.hpp:1416 r2_score_impl: No
    // rows exist in evaluation data. Evaluation data must at least contain 1 row.

    // We want to take everything after the function name, so we will search for the
    // third colon. Todo(todd): Look at making this less hacky by setting a mode for the
    // table function that will return only the core error string and not the
    // preprending metadata

    auto get_error_substring = [](const std::string& message) -> std::string {
      size_t colon_position = std::string::npos;
      for (int i = 0; i < 3; ++i) {
        colon_position = message.find(':', colon_position + 1);
        if (colon_position == std::string::npos) {
          return message;
        }
      }

      if (colon_position + 2 >= message.length()) {
        return message;
      }
      return message.substr(colon_position + 2);
    };

    const auto error_substr = get_error_substring(e.what());
    error_oss << "Could not evaluate model " << model_name << ". " << error_substr;
    throw std::runtime_error(error_oss.str());
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/Catalog/DdlCommandExecutor.h
• /home/jenkins-slave/workspace/core-os-doxygen/Catalog/DdlCommandExecutor.cpp