OmniSciDB  340b00dbf6
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer.cpp File Reference
#include "ImportExport/Importer.h"
#include <arrow/api.h>
#include <arrow/io/api.h>
#include <gdal.h>
#include <ogrsf_frmts.h>
#include <boost/algorithm/string.hpp>
#include <boost/dynamic_bitset.hpp>
#include <boost/filesystem.hpp>
#include <boost/variant.hpp>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <future>
#include <iomanip>
#include <list>
#include <memory>
#include <mutex>
#include <numeric>
#include <stack>
#include <stdexcept>
#include <thread>
#include <typeinfo>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#include "Archive/PosixFileArchive.h"
#include "Archive/S3Archive.h"
#include "ArrowImporter.h"
#include "Geospatial/Compression.h"
#include "Geospatial/GDAL.h"
#include "Geospatial/Transforms.h"
#include "Geospatial/Types.h"
#include "ImportExport/DelimitedParserUtils.h"
#include "Logger/Logger.h"
#include "OSDependent/omnisci_glob.h"
#include "QueryEngine/TypePunning.h"
#include "Shared/SqlTypesLayout.h"
#include "Shared/import_helpers.h"
#include "Shared/measure.h"
#include "Shared/misc.h"
#include "Shared/scope.h"
#include "Shared/shard_key.h"
#include "Shared/thread_count.h"
#include "Utils/ChunkAccessorTable.h"
#include "gen-cpp/OmniSci.h"
+ Include dependency graph for Importer.cpp:

Go to the source code of this file.

Classes

struct  anonymous_namespace{Importer.cpp}::OGRDataSourceDeleter
 
struct  anonymous_namespace{Importer.cpp}::OGRFeatureDeleter
 
struct  anonymous_namespace{Importer.cpp}::OGRSpatialReferenceDeleter
 
struct  import_export::GeoImportException
 

Namespaces

 anonymous_namespace{Importer.cpp}
 
 boost
 
 boost::log
 
 import_export
 
 import_export::anonymous_namespace{Importer.cpp}
 

Macros

#define DEBUG_TIMING   false
 
#define DEBUG_RENDER_GROUP_ANALYZER   0
 
#define DEBUG_AWS_AUTHENTICATION   0
 
#define DISABLE_MULTI_THREADED_SHAPEFILE_IMPORT   0
 

Typedefs

using anonymous_namespace{Importer.cpp}::OGRDataSourceUqPtr = std::unique_ptr< OGRDataSource, OGRDataSourceDeleter >
 
using anonymous_namespace{Importer.cpp}::OGRFeatureUqPtr = std::unique_ptr< OGRFeature, OGRFeatureDeleter >
 
using anonymous_namespace{Importer.cpp}::OGRSpatialReferenceUqPtr = std::unique_ptr< OGRSpatialReference, OGRSpatialReferenceDeleter >
 
using import_export::FieldNameToIndexMapType = std::map< std::string, size_t >
 
using import_export::ColumnNameToSourceNameMapType = std::map< std::string, std::string >
 
using import_export::ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer >>
 
using import_export::FeaturePtrVector = std::vector< OGRFeatureUqPtr >
 

Functions

auto get_filesize (const std::string &file_path)
 
formatting_ostream & boost::log::operator<< (formatting_ostream &out, std::vector< std::string > &row)
 
static const std::string import_export::trim_space (const char *field, const size_t len)
 
Datum import_export::NullDatum (SQLTypeInfo &ti)
 
Datum import_export::NullArrayDatum (SQLTypeInfo &ti)
 
ArrayDatum import_export::StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
 
ArrayDatum import_export::NullArray (const SQLTypeInfo &ti)
 
void import_export::addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec)
 
Datum import_export::TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti)
 
ArrayDatum import_export::TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti)
 
bool import_export::importGeoFromLonLat (double lon, double lat, std::vector< double > &coords)
 
std::tuple< int, SQLTypes,
std::string > 
import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1 (const std::list< const ColumnDescriptor * > &col_descs)
 
int64_t import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
 
static ImportStatus import_export::import_thread_delimited (int thread_id, Importer *importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer)
 
static ImportStatus import_export::import_thread_shapefile (int thread_id, Importer *importer, OGRSpatialReference *poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap)
 
int64_t import_export::anonymous_namespace{Importer.cpp}::int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float import_export::anonymous_namespace{Importer.cpp}::float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double import_export::anonymous_namespace{Importer.cpp}::double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
template<class T >
bool import_export::try_cast (const std::string &str)
 
char * import_export::try_strptimes (const char *str, const std::vector< std::string > &formats)
 
OGRLayer & import_export::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName (const std::string &geo_layer_name, const OGRDataSourceUqPtr &poDS, const std::string &file_name)
 
std::pair< SQLTypes, bool > import_export::ogr_to_type (const OGRFieldType &ogr_type)
 
SQLTypes import_export::ogr_to_type (const OGRwkbGeometryType &ogr_type)
 
void import_export::gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files)
 
std::vector< std::unique_ptr
< TypedImportBuffer > > 
import_export::setup_column_loaders (const TableDescriptor *td, Loader *loader)
 

Variables

size_t g_archive_read_buf_size = 1 << 20
 
static constexpr bool import_export::PROMOTE_POLYGON_TO_MULTIPOLYGON = true
 
static mapd_shared_mutex import_export::status_mutex
 
static std::map< std::string,
ImportStatus > 
import_export::import_status_map
 

Macro Definition Documentation

#define DEBUG_AWS_AUTHENTICATION   0

Definition at line 139 of file Importer.cpp.

#define DEBUG_RENDER_GROUP_ANALYZER   0
#define DISABLE_MULTI_THREADED_SHAPEFILE_IMPORT   0

Definition at line 141 of file Importer.cpp.

Function Documentation

auto get_filesize ( const std::string &  file_path)
inline

Definition at line 76 of file Importer.cpp.

References omnisci::file_size().

Referenced by import_export::DataStreamSink::archivePlumber().

76  {
77  boost::filesystem::path boost_file_path{file_path};
78  boost::system::error_code ec;
79  const auto filesize = boost::filesystem::file_size(boost_file_path, ec);
80  return ec ? 0 : filesize;
81 }
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

size_t g_archive_read_buf_size = 1 << 20