anonymous_namespace{DBHandler.cpp} Namespace Reference

Classes
struct	ForceDisconnect
struct	ProjectionTokensForCompletion

Functions
SessionMap::iterator	get_session_from_map (const TSessionId &session, SessionMap &session_map)
ProjectionTokensForCompletion	extract_projection_tokens_for_completion (const std::string &sql)
std::string	dump_table_col_names (const std::map< std::string, std::vector< std::string >> &table_col_names)
void	check_table_not_sharded (const TableDescriptor *td)
void	check_valid_column_names (const std::list< const ColumnDescriptor * > &descs, const std::vector< std::string > &column_names)
std::vector< int >	column_ids_by_names (const std::list< const ColumnDescriptor * > &descs, const std::vector< std::string > &column_names)
size_t	get_column_size (const TColumn &column)
RecordBatchVector	loadArrowStream (const std::string &stream)
void	add_vsi_network_prefix (std::string &path)
void	add_vsi_geo_prefix (std::string &path)
void	add_vsi_archive_prefix (std::string &path)
std::string	remove_vsi_prefixes (const std::string &path_in)
bool	path_is_relative (const std::string &path)
bool	path_has_valid_filename (const std::string &path)
bool	is_a_supported_geo_file (const std::string &path, bool include_gz)
bool	is_a_supported_archive_file (const std::string &path)
std::string	find_first_geo_file_in_archive (const std::string &archive_path, const import_export::CopyParams &copy_params)
bool	is_local_file (const std::string &file_path)
void	validate_import_file_path_if_local (const std::string &file_path)
bool	TTypeInfo_IsGeo (const TDatumType::type &t)

Function Documentation

void anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix

(

std::string &

path

)

Definition at line 3206 of file DBHandler.cpp.

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::import_geo_table().

                                             {
  // check for compressed file or file bundle
  if (boost::iends_with(path, ".zip")) {
    // zip archive
    path = "/vsizip/" + path;
  } else if (boost::iends_with(path, ".tar") || boost::iends_with(path, ".tgz") ||
             boost::iends_with(path, ".tar.gz")) {
    // tar archive (compressed or uncompressed)
    path = "/vsitar/" + path;
  }
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::add_vsi_geo_prefix

(

std::string &

path

)

Definition at line 3199 of file DBHandler.cpp.

Referenced by DBHandler::detect_column_types(), DBHandler::get_layers_in_geo_file(), and DBHandler::import_geo_table().

                                         {
  // single gzip'd file (not an archive)?
  if (boost::iends_with(path, ".gz") && !boost::iends_with(path, ".tar.gz")) {
    path = "/vsigzip/" + path;
  }
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix

(

std::string &

path

)

Definition at line 3177 of file DBHandler.cpp.

References Geospatial::GDAL::supportsNetworkFileAccess(), and THROW_MAPD_EXCEPTION.

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::import_geo_table().

                                             {
  // do we support network file access?
  bool gdal_network = Geospatial::GDAL::supportsNetworkFileAccess();

  // modify head of filename based on source location
  if (boost::istarts_with(path, "http://") || boost::istarts_with(path, "https://")) {
    if (!gdal_network) {
      THROW_MAPD_EXCEPTION(
          "HTTP geo file import not supported! Update to GDAL 2.2 or later!");
    }
    // invoke GDAL CURL virtual file reader
    path = "/vsicurl/" + path;
  } else if (boost::istarts_with(path, "s3://")) {
    if (!gdal_network) {
      THROW_MAPD_EXCEPTION(
          "S3 geo file import not supported! Update to GDAL 2.2 or later!");
    }
    // invoke GDAL S3 virtual file reader
    boost::replace_first(path, "s3://", "/vsis3/");
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::check_table_not_sharded

(

const TableDescriptor *

td

)

Definition at line 2404 of file DBHandler.cpp.

References TableDescriptor::nShards.

Referenced by DBHandler::load_table(), DBHandler::load_table_binary(), and DBHandler::prepare_columnar_loader().

                                                        {
  if (td && td->nShards) {
    throw std::runtime_error("Cannot import a sharded table directly to a leaf");
  }
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::check_valid_column_names	(	const std::list< const ColumnDescriptor * > &	descs,
		const std::vector< std::string > &	column_names
	)

Definition at line 2410 of file DBHandler.cpp.

References setup::name, THROW_MAPD_EXCEPTION, and to_lower().

Referenced by DBHandler::load_table(), DBHandler::load_table_binary(), and DBHandler::prepare_columnar_loader().

                                                                        {
  std::unordered_set<std::string> unique_names;
  for (const auto& name : column_names) {
    auto lower_name = to_lower(name);
    if (unique_names.find(lower_name) != unique_names.end()) {
      THROW_MAPD_EXCEPTION("Column " + name + " is mentioned multiple times");
    } else {
      unique_names.insert(lower_name);
    }
  }
  for (const auto& cd : descs) {
    auto iter = unique_names.find(to_lower(cd->columnName));
    if (iter != unique_names.end()) {
      unique_names.erase(iter);
    }
  }
  if (!unique_names.empty()) {
    THROW_MAPD_EXCEPTION("Column " + *unique_names.begin() + " does not exist");
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<int> anonymous_namespace{DBHandler.cpp}::column_ids_by_names	(	const std::list< const ColumnDescriptor * > &	descs,
		const std::vector< std::string > &	column_names
	)

Definition at line 2436 of file DBHandler.cpp.

References THROW_MAPD_EXCEPTION, and to_lower().

Referenced by DBHandler::load_table(), DBHandler::load_table_binary(), DBHandler::load_table_binary_arrow(), and DBHandler::load_table_binary_columnar().

                                                                               {
  std::vector<int> desc_to_column_ids;
  if (column_names.empty()) {
    int col_idx = 0;
    for (const auto& cd : descs) {
      if (!cd->isGeoPhyCol) {
        desc_to_column_ids.push_back(col_idx);
        ++col_idx;
      }
    }
  } else {
    for (const auto& cd : descs) {
      if (!cd->isGeoPhyCol) {
        bool found = false;
        for (size_t j = 0; j < column_names.size(); ++j) {
          if (to_lower(cd->columnName) == to_lower(column_names[j])) {
            found = true;
            desc_to_column_ids.push_back(j);
            break;
          }
        }
        if (!found) {
          if (!cd->columnType.get_notnull()) {
            desc_to_column_ids.push_back(-1);
          } else {
            THROW_MAPD_EXCEPTION("Column '" + cd->columnName +
                                 "' cannot be omitted due to NOT NULL constraint");
          }
        }
      }
    }
  }
  return desc_to_column_ids;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::dump_table_col_names

(

const std::map< std::string, std::vector< std::string >> &

table_col_names

)

Definition at line 1875 of file DBHandler.cpp.

Referenced by DBHandler::get_result_row_for_pixel().

                                                                    {
  std::ostringstream oss;
  for (const auto& [table_name, col_names] : table_col_names) {
    oss << ":" << table_name;
    for (const auto& col_name : col_names) {
      oss << "," << col_name;
    }
  }
  return oss.str();
}

Here is the caller graph for this function:

ProjectionTokensForCompletion anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion

(

const std::string &

sql

)

Definition at line 1281 of file DBHandler.cpp.

References split(), and to_upper().

Referenced by DBHandler::get_completion_hints().

                          {
  boost::regex id_regex{R"(([[:alnum:]]|_|\.)+)",
                        boost::regex::extended | boost::regex::icase};
  boost::sregex_token_iterator tok_it(sql.begin(), sql.end(), id_regex, 0);
  boost::sregex_token_iterator end;
  std::unordered_set<std::string> uc_column_names;
  std::unordered_set<std::string> uc_column_table_qualifiers;
  for (; tok_it != end; ++tok_it) {
    std::string column_name = *tok_it;
    std::vector<std::string> column_tokens;
    boost::split(column_tokens, column_name, boost::is_any_of("."));
    if (column_tokens.size() == 2) {
      // If the column name is qualified, take user's word.
      uc_column_table_qualifiers.insert(to_upper(column_tokens.front()));
    } else {
      uc_column_names.insert(to_upper(column_name));
    }
  }
  return {uc_column_names, uc_column_table_qualifiers};
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive	(	const std::string &	archive_path,
		const import_export::CopyParams &	copy_params
	)

Definition at line 3287 of file DBHandler.cpp.

References import_export::Importer::gdalGetAllFilesInArchive(), logger::INFO, is_a_supported_geo_file(), LOG, and remove_vsi_prefixes().

Referenced by DBHandler::detect_column_types(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::import_geo_table().

                                                                                       {
  // get the recursive list of all files in the archive
  std::vector<std::string> files =
      import_export::Importer::gdalGetAllFilesInArchive(archive_path, copy_params);

  // report the list
  LOG(INFO) << "Found " << files.size() << " files in Archive "
            << remove_vsi_prefixes(archive_path);
  for (const auto& file : files) {
    LOG(INFO) << "  " << file;
  }

  // scan the list for the first candidate file
  bool found_suitable_file = false;
  std::string file_name;
  for (const auto& file : files) {
    if (is_a_supported_geo_file(file, false)) {
      file_name = file;
      found_suitable_file = true;
      break;
    }
  }

  // if we didn't find anything
  if (!found_suitable_file) {
    LOG(INFO) << "Failed to find any supported geo files in Archive: " +
                     remove_vsi_prefixes(archive_path);
    file_name.clear();
  }

  // done
  return file_name;
}

Here is the call graph for this function:

Here is the caller graph for this function:

size_t anonymous_namespace{DBHandler.cpp}::get_column_size

(

const TColumn &

column

)

Definition at line 2614 of file DBHandler.cpp.

Referenced by DBHandler::load_table_binary_columnar().

                                              {
  if (!column.nulls.empty()) {
    return column.nulls.size();
  } else {
    // it is a very bold estimate but later we check it against REAL data
    // and if this function returns a wrong result (e.g. both int and string
    // vectors are filled with values), we get an error
    return column.data.int_col.size() + column.data.arr_col.size() +
           column.data.real_col.size() + column.data.str_col.size();
  }
}

Here is the caller graph for this function:

SessionMap::iterator anonymous_namespace{DBHandler.cpp}::get_session_from_map	(	const TSessionId &	session,
		SessionMap &	session_map
	)

Definition at line 127 of file DBHandler.cpp.

References THROW_MAPD_EXCEPTION.

Referenced by DBHandler::get_session_it_unsafe().

                                                                   {
  auto session_it = session_map.find(session);
  if (session_it == session_map.end()) {
    THROW_MAPD_EXCEPTION("Session not valid.");
  }
  return session_it;
}

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file

(

const std::string &

path

)

Definition at line 3274 of file DBHandler.cpp.

References path_has_valid_filename().

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::import_geo_table().

                                                        {
  if (!path_has_valid_filename(path)) {
    return false;
  }
  if (boost::iends_with(path, ".zip") && !boost::iends_with(path, ".gdb.zip")) {
    return true;
  } else if (boost::iends_with(path, ".tar") || boost::iends_with(path, ".tgz") ||
             boost::iends_with(path, ".tar.gz")) {
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::is_a_supported_geo_file	(	const std::string &	path,
		bool	include_gz
	)

Definition at line 3256 of file DBHandler.cpp.

References path_has_valid_filename().

Referenced by DBHandler::detect_column_types(), find_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::import_geo_table().

                                                                     {
  if (!path_has_valid_filename(path)) {
    return false;
  }
  if (include_gz) {
    if (boost::iends_with(path, ".geojson.gz") || boost::iends_with(path, ".json.gz")) {
      return true;
    }
  }
  if (boost::iends_with(path, ".shp") || boost::iends_with(path, ".geojson") ||
      boost::iends_with(path, ".json") || boost::iends_with(path, ".kml") ||
      boost::iends_with(path, ".kmz") || boost::iends_with(path, ".gdb") ||
      boost::iends_with(path, ".gdb.zip")) {
    return true;
  }
  return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::is_local_file

(

const std::string &

file_path

)

Definition at line 3322 of file DBHandler.cpp.

Referenced by validate_import_file_path_if_local().

                                               {
  return (!boost::istarts_with(file_path, "s3://") &&
          !boost::istarts_with(file_path, "http://") &&
          !boost::istarts_with(file_path, "https://"));
}

Here is the caller graph for this function:

RecordBatchVector anonymous_namespace{DBHandler.cpp}::loadArrowStream

(

const std::string &

stream

)

Definition at line 2739 of file DBHandler.cpp.

References ARROW_ASSIGN_OR_THROW, ARROW_THRIFT_THROW_NOT_OK, logger::ERROR, and LOG.

Referenced by DBHandler::load_table_binary_arrow().

                                                           {
  RecordBatchVector batches;
  try {
    // TODO(wesm): Make this simpler in general, see ARROW-1600
    auto stream_buffer =
        std::make_shared<arrow::Buffer>(reinterpret_cast<const uint8_t*>(stream.c_str()),
                                        static_cast<int64_t>(stream.size()));

    arrow::io::BufferReader buf_reader(stream_buffer);
    std::shared_ptr<arrow::RecordBatchReader> batch_reader;
    ARROW_ASSIGN_OR_THROW(batch_reader,
                          arrow::ipc::RecordBatchStreamReader::Open(&buf_reader));

    while (true) {
      std::shared_ptr<arrow::RecordBatch> batch;
      // Read batch (zero-copy) from the stream
      ARROW_THRIFT_THROW_NOT_OK(batch_reader->ReadNext(&batch));
      if (batch == nullptr) {
        break;
      }
      batches.emplace_back(std::move(batch));
    }
  } catch (const std::exception& e) {
    LOG(ERROR) << "Error parsing Arrow stream: " << e.what() << ". Import aborted";
  }
  return batches;
}

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::path_has_valid_filename

(

const std::string &

path

)

Definition at line 3248 of file DBHandler.cpp.

References logger::filename().

Referenced by is_a_supported_archive_file(), and is_a_supported_geo_file().

                                                    {
  auto filename = boost::filesystem::path(path).filename().string();
  if (filename.size() == 0 || filename[0] == '.' || filename[0] == '/') {
    return false;
  }
  return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::path_is_relative

(

const std::string &

path

)

Definition at line 3240 of file DBHandler.cpp.

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), and DBHandler::import_geo_table().

                                             {
  if (boost::istarts_with(path, "s3://") || boost::istarts_with(path, "http://") ||
      boost::istarts_with(path, "https://")) {
    return false;
  }
  return !boost::filesystem::path(path).is_absolute();
}

Here is the caller graph for this function:

std::string anonymous_namespace{DBHandler.cpp}::remove_vsi_prefixes

(

const std::string &

path_in

)

Definition at line 3218 of file DBHandler.cpp.

Referenced by find_first_geo_file_in_archive().

                                                        {
  std::string path(path_in);

  // these will be first
  if (boost::istarts_with(path, "/vsizip/")) {
    boost::replace_first(path, "/vsizip/", "");
  } else if (boost::istarts_with(path, "/vsitar/")) {
    boost::replace_first(path, "/vsitar/", "");
  } else if (boost::istarts_with(path, "/vsigzip/")) {
    boost::replace_first(path, "/vsigzip/", "");
  }

  // then these
  if (boost::istarts_with(path, "/vsicurl/")) {
    boost::replace_first(path, "/vsicurl/", "");
  } else if (boost::istarts_with(path, "/vsis3/")) {
    boost::replace_first(path, "/vsis3/", "s3://");
  }

  return path;
}

Here is the caller graph for this function:

bool anonymous_namespace{DBHandler.cpp}::TTypeInfo_IsGeo

(

const TDatumType::type &

t

)

Definition at line 4123 of file DBHandler.cpp.

Referenced by DBHandler::import_geo_table().

                                              {
  return (t == TDatumType::POLYGON || t == TDatumType::MULTIPOLYGON ||
          t == TDatumType::LINESTRING || t == TDatumType::POINT);
}

Here is the caller graph for this function:

void anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local

(

const std::string &

file_path

)

Definition at line 3328 of file DBHandler.cpp.

References ddl_utils::IMPORT, is_local_file(), and ddl_utils::validate_allowed_file_path().

Referenced by DBHandler::detect_column_types(), DBHandler::get_all_files_in_archive(), DBHandler::get_first_geo_file_in_archive(), DBHandler::get_layers_in_geo_file(), DBHandler::import_geo_table(), and DBHandler::import_table().

                                                                    {
  if (is_local_file(file_path)) {
    ddl_utils::validate_allowed_file_path(file_path, ddl_utils::DataTransferType::IMPORT);
  }
}

Here is the call graph for this function:

Here is the caller graph for this function: