OmniSciDB
a667adc9c8
|
Namespaces | |
anonymous_namespace{Importer.cpp} | |
anonymous_namespace{QueryExporterGDAL.cpp} | |
delimited_parser | |
Classes | |
struct | CopyParams |
struct | GeoImportException |
struct | BadRowsTracker |
class | TypedImportBuffer |
class | Loader |
struct | ImportStatus |
class | DataStreamSink |
class | Detector |
class | ImporterUtils |
class | RenderGroupAnalyzer |
class | Importer |
class | QueryExporter |
class | QueryExporterCSV |
class | QueryExporterGDAL |
Typedefs | |
using | FieldNameToIndexMapType = std::map< std::string, size_t > |
using | ColumnNameToSourceNameMapType = std::map< std::string, std::string > |
using | ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer >> |
using | FeaturePtrVector = std::vector< OGRFeatureUqPtr > |
using | ArraySliceRange = std::pair< size_t, size_t > |
Enumerations | |
enum | FileType { FileType::DELIMITED, FileType::POLYGON } |
enum | ImportHeaderRow { ImportHeaderRow::AUTODETECT, ImportHeaderRow::NO_HEADER, ImportHeaderRow::HAS_HEADER } |
Functions | |
static const std::string | trim_space (const char *field, const size_t len) |
Datum | NullDatum (SQLTypeInfo &ti) |
Datum | NullArrayDatum (SQLTypeInfo &ti) |
ArrayDatum | StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams ©_params) |
ArrayDatum | NullArray (const SQLTypeInfo &ti) |
void | addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec) |
Datum | TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti) |
ArrayDatum | TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti) |
bool | importGeoFromLonLat (double lon, double lat, std::vector< double > &coords, SQLTypeInfo &ti) |
static ImportStatus | import_thread_delimited (int thread_id, Importer *importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer, const Catalog_Namespace::SessionInfo *session_info, Executor *executor) |
static ImportStatus | import_thread_shapefile (int thread_id, Importer *importer, OGRSpatialReference *poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, const Catalog_Namespace::SessionInfo *session_info, Executor *executor) |
template<class T > | |
bool | try_cast (const std::string &str) |
std::pair< SQLTypes, bool > | ogr_to_type (const OGRFieldType &ogr_type) |
SQLTypes | ogr_to_type (const OGRwkbGeometryType &ogr_type) |
void | gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files) |
std::vector< std::unique_ptr < TypedImportBuffer > > | setup_column_loaders (const TableDescriptor *td, Loader *loader) |
std::vector< std::unique_ptr < TypedImportBuffer > > | fill_missing_columns (const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data) |
Variables | |
static constexpr size_t | kImportFileBufferSize = (1 << 23) |
static constexpr bool | PROMOTE_POLYGON_TO_MULTIPOLYGON = true |
static mapd_shared_mutex | status_mutex |
static std::map< std::string, ImportStatus > | import_status_map |
using import_export::ArraySliceRange = typedef std::pair<size_t, size_t> |
Definition at line 72 of file Importer.h.
using import_export::ColumnIdToRenderGroupAnalyzerMapType = typedef std::map<int, std::shared_ptr<RenderGroupAnalyzer>> |
Definition at line 157 of file Importer.cpp.
using import_export::ColumnNameToSourceNameMapType = typedef std::map<std::string, std::string> |
Definition at line 155 of file Importer.cpp.
using import_export::FeaturePtrVector = typedef std::vector<OGRFeatureUqPtr> |
Definition at line 158 of file Importer.cpp.
using import_export::FieldNameToIndexMapType = typedef std::map<std::string, size_t> |
Definition at line 154 of file Importer.cpp.
|
strong |
|
strong |
Enumerator | |
---|---|
AUTODETECT | |
NO_HEADER | |
HAS_HEADER |
Definition at line 43 of file CopyParams.h.
void import_export::addBinaryStringArray | ( | const TDatum & | datum, |
std::vector< std::string > & | string_vec | ||
) |
Definition at line 421 of file Importer.cpp.
Referenced by import_export::TypedImportBuffer::add_value().
std::vector< std::unique_ptr< TypedImportBuffer > > import_export::fill_missing_columns | ( | const Catalog_Namespace::Catalog * | cat, |
Fragmenter_Namespace::InsertData & | insert_data | ||
) |
Definition at line 5365 of file Importer.cpp.
References CHECK, ColumnDescriptor::columnId, Fragmenter_Namespace::InsertData::columnIds, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Fragmenter_Namespace::InsertData::data, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), import_export::TypedImportBuffer::get_data_block_pointers(), SQLTypeInfo::get_physical_cols(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Geospatial::GeoTypesFactory::getGeoColumns(), Catalog_Namespace::Catalog::getMetadataForDict(), i, Fragmenter_Namespace::InsertData::is_default, SQLTypeInfo::is_geometry(), IS_STRING, SQLTypeInfo::is_string(), kARRAY, kENCODING_DICT, import_export::Importer::set_geo_physical_import_buffer(), gpu_enabled::sort(), and Fragmenter_Namespace::InsertData::tableId.
Referenced by RelAlgExecutor::executeSimpleInsert(), DBHandler::insert_data(), and Parser::InsertIntoTableAsSelectStmt::populateData().
void import_export::gdalGatherFilesInArchiveRecursive | ( | const std::string & | archive_path, |
std::vector< std::string > & | files | ||
) |
Definition at line 4745 of file Importer.cpp.
References LOG, run_benchmark_import::result, and logger::WARNING.
Referenced by import_export::Importer::gdalGetAllFilesInArchive().
|
static |
Definition at line 1826 of file Importer.cpp.
References CHECK, CHECK_LT, anonymous_namespace{Importer.cpp}::checkInterrupt(), Geospatial::GeoTypesFactory::createOGRGeometry(), DEBUG_TIMING, DELIMITED, logger::ERROR, measure< TimeT >::execution(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::file_type, import_export::delimited_parser::find_beginning(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), import_export::Importer::get_is_array(), import_export::delimited_parser::get_row(), Catalog_Namespace::SessionInfo::get_session_id(), import_export::Importer::getCatalog(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), i, importGeoFromLonLat(), logger::INFO, IS_GEO, is_null(), kMULTIPOLYGON, kPOINT, kPOLYGON, import_export::Importer::load(), import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, LOG, import_export::CopyParams::lonlat, import_export::CopyParams::max_reject, import_export::CopyParams::null_str, shared::printContainer(), PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::Importer::set_geo_physical_import_buffer(), import_export::CopyParams::source_srid, gpu_enabled::swap(), import_export::ImportStatus::thread_id, logger::thread_id(), to_string(), and UNLIKELY.
Referenced by import_export::Importer::importDelimited().
|
static |
Definition at line 2192 of file Importer.cpp.
References CHECK, anonymous_namespace{Importer.cpp}::checkInterrupt(), Geospatial::compress_coords(), DEBUG_TIMING, Executor::ERR_INTERRUPTED, logger::ERROR, import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), Catalog_Namespace::SessionInfo::get_session_id(), QueryExecutionError::getErrorCode(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), logger::INFO, kLINESTRING, kMULTIPOLYGON, kPOLYGON, import_export::Importer::load(), import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, LOG, import_export::CopyParams::null_str, PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::ImportStatus::thread_id, logger::thread_id(), timer_start(), timer_stop(), to_string(), and UNLIKELY.
Referenced by import_export::Importer::importGDAL().
bool import_export::importGeoFromLonLat | ( | double | lon, |
double | lat, | ||
std::vector< double > & | coords, | ||
SQLTypeInfo & | ti | ||
) |
Definition at line 1439 of file Importer.cpp.
References Geospatial::GeoPoint::getColumns(), and SQLTypeInfo::transforms().
Referenced by import_thread_delimited().
ArrayDatum import_export::NullArray | ( | const SQLTypeInfo & | ti | ) |
Definition at line 389 of file Importer.cpp.
References appendDatum(), CHECK, checked_malloc(), test_fsi::d, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArrayDatum(), and NullDatum().
Referenced by import_export::TypedImportBuffer::add_value(), import_export::TypedImportBuffer::add_values(), import_export::ImporterUtils::composeNullArray(), and TDatumToArrayDatum().
Datum import_export::NullArrayDatum | ( | SQLTypeInfo & | ti | ) |
Definition at line 295 of file Importer.cpp.
References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_array_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.
Referenced by NullArray().
Datum import_export::NullDatum | ( | SQLTypeInfo & | ti | ) |
Definition at line 254 of file Importer.cpp.
References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.
Referenced by NullArray(), and StringToArray().
Definition at line 4557 of file Importer.cpp.
References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().
Referenced by import_export::Importer::gdalToColumnDescriptors().
Definition at line 4595 of file Importer.cpp.
References kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().
std::vector< std::unique_ptr< TypedImportBuffer > > import_export::setup_column_loaders | ( | const TableDescriptor * | td, |
Loader * | loader | ||
) |
Definition at line 5350 of file Importer.cpp.
References CHECK, import_export::Loader::get_column_descs(), and import_export::Loader::getStringDict().
Referenced by DBHandler::prepare_loader_generic().
ArrayDatum import_export::StringToArray | ( | const std::string & | s, |
const SQLTypeInfo & | ti, | ||
const CopyParams & | copy_params | ||
) |
Definition at line 336 of file Importer.cpp.
References appendDatum(), import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, checked_malloc(), test_fsi::d, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), i, is_null(), SQLTypeInfo::is_number(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), LOG, import_export::CopyParams::null_str, NullDatum(), StringToDatum(), trim_space(), and logger::WARNING.
Referenced by import_export::TypedImportBuffer::add_value().
ArrayDatum import_export::TDatumToArrayDatum | ( | const TDatum & | datum, |
const SQLTypeInfo & | ti | ||
) |
Definition at line 473 of file Importer.cpp.
References appendDatum(), CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArray(), and TDatumToDatum().
Referenced by import_export::TypedImportBuffer::add_value().
Datum import_export::TDatumToDatum | ( | const TDatum & | datum, |
SQLTypeInfo & | ti | ||
) |
Definition at line 428 of file Importer.cpp.
References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.
Referenced by TDatumToArrayDatum().
Definition at line 242 of file Importer.cpp.
References i, and generate_TableFunctionsFactory_init::j.
Referenced by import_export::delimited_parser::get_row(), and StringToArray().
bool import_export::try_cast | ( | const std::string & | str | ) |
|
static |
Definition at line 169 of file Importer.cpp.
Referenced by import_export::Importer::get_import_status(), and import_export::Importer::set_import_status().
|
static |
Definition at line 32 of file CopyParams.h.
|
static |
Definition at line 166 of file Importer.cpp.
Referenced by import_export::Detector::detect_sqltype(), import_export::Importer::gdalToColumnDescriptors(), import_thread_delimited(), and import_thread_shapefile().
|
static |
Definition at line 168 of file Importer.cpp.
Referenced by import_export::Importer::get_import_status(), and import_export::Importer::set_import_status().