OmniSciDB
bf83d84833
|
Namespaces | |
anonymous_namespace{Importer.cpp} | |
anonymous_namespace{QueryExporterGDAL.cpp} | |
delimited_parser | |
Classes | |
struct | CopyParams |
struct | GeoImportException |
struct | BadRowsTracker |
class | TypedImportBuffer |
class | Loader |
struct | ImportStatus |
class | DataStreamSink |
class | Detector |
class | ImporterUtils |
class | RenderGroupAnalyzer |
class | Importer |
class | QueryExporter |
class | QueryExporterCSV |
class | QueryExporterGDAL |
Typedefs | |
using | FieldNameToIndexMapType = std::map< std::string, size_t > |
using | ColumnNameToSourceNameMapType = std::map< std::string, std::string > |
using | ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer >> |
using | FeaturePtrVector = std::vector< OGRFeatureUqPtr > |
using | ArraySliceRange = std::pair< size_t, size_t > |
Enumerations | |
enum | FileType { FileType::DELIMITED, FileType::POLYGON } |
enum | ImportHeaderRow { ImportHeaderRow::AUTODETECT, ImportHeaderRow::NO_HEADER, ImportHeaderRow::HAS_HEADER } |
Functions | |
static const std::string | trim_space (const char *field, const size_t len) |
Datum | NullDatum (SQLTypeInfo &ti) |
Datum | NullArrayDatum (SQLTypeInfo &ti) |
ArrayDatum | StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams ©_params) |
ArrayDatum | NullArray (const SQLTypeInfo &ti) |
void | addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec) |
Datum | TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti) |
ArrayDatum | TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti) |
bool | importGeoFromLonLat (double lon, double lat, std::vector< double > &coords, SQLTypeInfo &ti) |
static ImportStatus | import_thread_delimited (int thread_id, Importer *importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer) |
static ImportStatus | import_thread_shapefile (int thread_id, Importer *importer, OGRSpatialReference *poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap) |
template<class T > | |
bool | try_cast (const std::string &str) |
char * | try_strptimes (const char *str, const std::vector< std::string > &formats) |
std::pair< SQLTypes, bool > | ogr_to_type (const OGRFieldType &ogr_type) |
SQLTypes | ogr_to_type (const OGRwkbGeometryType &ogr_type) |
void | gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files) |
std::vector< std::unique_ptr < TypedImportBuffer > > | setup_column_loaders (const TableDescriptor *td, Loader *loader) |
Variables | |
static constexpr size_t | kImportFileBufferSize = (1 << 23) |
static constexpr bool | PROMOTE_POLYGON_TO_MULTIPOLYGON = true |
static mapd_shared_mutex | status_mutex |
static std::map< std::string, ImportStatus > | import_status_map |
using import_export::ArraySliceRange = typedef std::pair<size_t, size_t> |
Definition at line 72 of file Importer.h.
using import_export::ColumnIdToRenderGroupAnalyzerMapType = typedef std::map<int, std::shared_ptr<RenderGroupAnalyzer>> |
Definition at line 139 of file Importer.cpp.
using import_export::ColumnNameToSourceNameMapType = typedef std::map<std::string, std::string> |
Definition at line 137 of file Importer.cpp.
using import_export::FeaturePtrVector = typedef std::vector<OGRFeatureUqPtr> |
Definition at line 140 of file Importer.cpp.
using import_export::FieldNameToIndexMapType = typedef std::map<std::string, size_t> |
Definition at line 136 of file Importer.cpp.
|
strong |
|
strong |
Enumerator | |
---|---|
AUTODETECT | |
NO_HEADER | |
HAS_HEADER |
Definition at line 43 of file CopyParams.h.
void import_export::addBinaryStringArray | ( | const TDatum & | datum, |
std::vector< std::string > & | string_vec | ||
) |
Definition at line 403 of file Importer.cpp.
Referenced by import_export::TypedImportBuffer::add_value().
void import_export::gdalGatherFilesInArchiveRecursive | ( | const std::string & | archive_path, |
std::vector< std::string > & | files | ||
) |
Definition at line 4609 of file Importer.cpp.
References LOG, run_benchmark_import::result, and logger::WARNING.
Referenced by import_export::Importer::gdalGetAllFilesInArchive().
|
static |
Definition at line 1808 of file Importer.cpp.
References CHECK, CHECK_LT, Geospatial::GeoTypesFactory::createOGRGeometry(), DEBUG_TIMING, DELIMITED, logger::ERROR, measure< TimeT >::execution(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::file_type, import_export::delimited_parser::find_beginning(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), import_export::Importer::get_is_array(), import_export::delimited_parser::get_row(), import_export::Importer::getCatalog(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), importGeoFromLonLat(), logger::INFO, IS_GEO, is_null(), kMULTIPOLYGON, kPOINT, kPOLYGON, import_export::Importer::load(), LOG, import_export::CopyParams::lonlat, import_export::CopyParams::max_reject, import_export::CopyParams::null_str, shared::printContainer(), PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::Importer::set_geo_physical_import_buffer(), import_export::CopyParams::source_srid, gpu_enabled::swap(), import_export::ImportStatus::thread_id, logger::thread_id(), and to_string().
Referenced by import_export::Importer::importDelimited().
|
static |
Definition at line 2149 of file Importer.cpp.
References CHECK, Geospatial::compress_coords(), DEBUG_TIMING, logger::ERROR, import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), logger::INFO, kLINESTRING, kMULTIPOLYGON, kPOLYGON, import_export::Importer::load(), LOG, import_export::CopyParams::null_str, PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::ImportStatus::thread_id, logger::thread_id(), timer_start(), timer_stop(), and to_string().
Referenced by import_export::Importer::importGDAL().
bool import_export::importGeoFromLonLat | ( | double | lon, |
double | lat, | ||
std::vector< double > & | coords, | ||
SQLTypeInfo & | ti | ||
) |
Definition at line 1413 of file Importer.cpp.
References Geospatial::GeoPoint::getColumns(), and SQLTypeInfo::transforms().
Referenced by import_thread_delimited().
ArrayDatum import_export::NullArray | ( | const SQLTypeInfo & | ti | ) |
Definition at line 371 of file Importer.cpp.
References appendDatum(), CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArrayDatum(), and NullDatum().
Referenced by import_export::TypedImportBuffer::add_value(), import_export::TypedImportBuffer::add_values(), import_export::ImporterUtils::composeNullArray(), and TDatumToArrayDatum().
Datum import_export::NullArrayDatum | ( | SQLTypeInfo & | ti | ) |
Definition at line 277 of file Importer.cpp.
References Datum::bigintval, Datum::boolval, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_array_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.
Referenced by NullArray().
Datum import_export::NullDatum | ( | SQLTypeInfo & | ti | ) |
Definition at line 236 of file Importer.cpp.
References Datum::bigintval, Datum::boolval, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.
Referenced by NullArray(), and StringToArray().
std::pair<SQLTypes, bool> import_export::ogr_to_type | ( | const OGRFieldType & | ogr_type | ) |
Definition at line 4421 of file Importer.cpp.
References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().
Referenced by import_export::Importer::gdalToColumnDescriptors().
SQLTypes import_export::ogr_to_type | ( | const OGRwkbGeometryType & | ogr_type | ) |
Definition at line 4459 of file Importer.cpp.
References kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().
std::vector< std::unique_ptr< TypedImportBuffer > > import_export::setup_column_loaders | ( | const TableDescriptor * | td, |
Loader * | loader | ||
) |
Definition at line 5189 of file Importer.cpp.
References CHECK, import_export::Loader::get_column_descs(), and import_export::Loader::getStringDict().
Referenced by Parser::AddColumnStmt::execute(), and DBHandler::prepare_columnar_loader().
ArrayDatum import_export::StringToArray | ( | const std::string & | s, |
const SQLTypeInfo & | ti, | ||
const CopyParams & | copy_params | ||
) |
Definition at line 318 of file Importer.cpp.
References appendDatum(), import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), is_null(), SQLTypeInfo::is_number(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), LOG, import_export::CopyParams::null_str, NullDatum(), StringToDatum(), trim_space(), and logger::WARNING.
Referenced by import_export::TypedImportBuffer::add_value().
ArrayDatum import_export::TDatumToArrayDatum | ( | const TDatum & | datum, |
const SQLTypeInfo & | ti | ||
) |
Definition at line 455 of file Importer.cpp.
References appendDatum(), CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArray(), and TDatumToDatum().
Referenced by import_export::TypedImportBuffer::add_value().
Datum import_export::TDatumToDatum | ( | const TDatum & | datum, |
SQLTypeInfo & | ti | ||
) |
Definition at line 410 of file Importer.cpp.
References Datum::bigintval, Datum::boolval, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.
Referenced by TDatumToArrayDatum().
|
static |
Definition at line 224 of file Importer.cpp.
Referenced by import_export::delimited_parser::get_row(), and StringToArray().
bool import_export::try_cast | ( | const std::string & | str | ) |
|
inline |
Definition at line 3007 of file Importer.cpp.
Referenced by import_export::Detector::detect_sqltype().
|
static |
Definition at line 151 of file Importer.cpp.
Referenced by import_export::Importer::get_import_status(), and import_export::Importer::set_import_status().
|
static |
Definition at line 32 of file CopyParams.h.
|
static |
Definition at line 148 of file Importer.cpp.
Referenced by import_export::Detector::detect_sqltype(), import_export::Importer::gdalToColumnDescriptors(), import_thread_delimited(), and import_thread_shapefile().
|
static |
Definition at line 150 of file Importer.cpp.
Referenced by import_export::Importer::get_import_status(), import_export::DataStreamSink::import_compressed(), import_export::Importer::importDelimited(), and import_export::Importer::set_import_status().