OmniSciDB  5ade3759e0
Importer.cpp File Reference
#include <gdal.h>
#include <ogrsf_frmts.h>
#include <unistd.h>
#include <boost/algorithm/string.hpp>
#include <boost/dynamic_bitset.hpp>
#include <boost/filesystem.hpp>
#include <boost/variant.hpp>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <future>
#include <iomanip>
#include <list>
#include <memory>
#include <mutex>
#include <numeric>
#include <stack>
#include <stdexcept>
#include <thread>
#include <typeinfo>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#include "../QueryEngine/TypePunning.h"
#include "../Shared/geo_compression.h"
#include "../Shared/geo_types.h"
#include "../Shared/geosupport.h"
#include "../Shared/import_helpers.h"
#include "../Shared/mapd_glob.h"
#include "../Shared/mapdpath.h"
#include "../Shared/measure.h"
#include "../Shared/scope.h"
#include "../Shared/shard_key.h"
#include "../Shared/thread_count.h"
#include "Shared/Logger.h"
#include "Shared/SqlTypesLayout.h"
#include "Importer.h"
#include "QueryRunner/QueryRunner.h"
#include "Utils/ChunkAccessorTable.h"
#include "gen-cpp/MapD.h"
#include <arrow/api.h>
#include <arrow/io/api.h>
#include "../Archive/PosixFileArchive.h"
#include "../Archive/S3Archive.h"
#include "ArrowImporter.h"
+ Include dependency graph for Importer.cpp:

Go to the source code of this file.

Classes

struct  anonymous_namespace{Importer.cpp}::OGRDataSourceDeleter
 
struct  anonymous_namespace{Importer.cpp}::OGRFeatureDeleter
 
struct  anonymous_namespace{Importer.cpp}::OGRSpatialReferenceDeleter
 
struct  Importer_NS::GeoImportException
 

Namespaces

 anonymous_namespace{Importer.cpp}
 
 boost
 
 boost::log
 
 Importer_NS
 
 Importer_NS::anonymous_namespace{Importer.cpp}
 

Macros

#define DEBUG_TIMING   false
 
#define DEBUG_RENDER_GROUP_ANALYZER   0
 
#define DEBUG_AWS_AUTHENTICATION   0
 
#define DISABLE_MULTI_THREADED_SHAPEFILE_IMPORT   0
 

Typedefs

using anonymous_namespace{Importer.cpp}::OGRDataSourceUqPtr = std::unique_ptr< OGRDataSource, OGRDataSourceDeleter >
 
using anonymous_namespace{Importer.cpp}::OGRFeatureUqPtr = std::unique_ptr< OGRFeature, OGRFeatureDeleter >
 
using anonymous_namespace{Importer.cpp}::OGRSpatialReferenceUqPtr = std::unique_ptr< OGRSpatialReference, OGRSpatialReferenceDeleter >
 
using Importer_NS::FieldNameToIndexMapType = std::map< std::string, size_t >
 
using Importer_NS::ColumnNameToSourceNameMapType = std::map< std::string, std::string >
 
using Importer_NS::ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer > >
 
using Importer_NS::FeaturePtrVector = std::vector< OGRFeatureUqPtr >
 

Functions

auto get_filesize (const std::string &file_path)
 
formatting_ostream & boost::log::operator<< (formatting_ostream &out, std::vector< std::string > &row)
 
static const std::string Importer_NS::trim_space (const char *field, const size_t len)
 
static const bool Importer_NS::is_eol (const char &p, const std::string &line_delims)
 
static const char * Importer_NS::get_row (const char *buf, const char *buf_end, const char *entire_buf_end, const CopyParams &copy_params, bool is_begin, const bool *is_array, std::vector< std::string > &row, bool &try_single_thread)
 
int8_t * Importer_NS::appendDatum (int8_t *buf, Datum d, const SQLTypeInfo &ti)
 
Datum Importer_NS::NullDatum (SQLTypeInfo &ti)
 
Datum Importer_NS::NullArrayDatum (SQLTypeInfo &ti)
 
ArrayDatum Importer_NS::StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
 
ArrayDatum Importer_NS::NullArray (const SQLTypeInfo &ti)
 
void Importer_NS::addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec)
 
Datum Importer_NS::TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti)
 
ArrayDatum Importer_NS::TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti)
 
static size_t Importer_NS::find_beginning (const char *buffer, size_t begin, size_t end, const CopyParams &copy_params)
 
bool Importer_NS::importGeoFromLonLat (double lon, double lat, std::vector< double > &coords)
 
uint64_t Importer_NS::compress_coord (double coord, const SQLTypeInfo &ti, bool x)
 
std::vector< uint8_t > Importer_NS::compress_coords (std::vector< double > &coords, const SQLTypeInfo &ti)
 
static ImportStatus Importer_NS::import_thread_delimited (int thread_id, Importer *importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer)
 
static ImportStatus Importer_NS::import_thread_shapefile (int thread_id, Importer *importer, OGRSpatialReference *poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap)
 
static size_t Importer_NS::find_end (const char *buffer, size_t size, const CopyParams &copy_params)
 
int64_t Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
template<class T >
bool Importer_NS::try_cast (const std::string &str)
 
char * Importer_NS::try_strptimes (const char *str, const std::vector< std::string > &formats)
 
void Importer_NS::GDALErrorHandler (CPLErr eErrClass, int err_no, const char *msg)
 
OGRLayer & Importer_NS::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName (const std::string &geo_layer_name, const OGRDataSourceUqPtr &poDS, const std::string &file_name)
 
std::pair< SQLTypes, bool > Importer_NS::ogr_to_type (const OGRFieldType &ogr_type)
 
SQLTypes Importer_NS::ogr_to_type (const OGRwkbGeometryType &ogr_type)
 
void Importer_NS::gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files)
 

Variables

static constexpr bool Importer_NS::PROMOTE_POLYGON_TO_MULTIPOLYGON = true
 
static mapd_shared_mutex Importer_NS::status_mutex
 
static std::map< std::string, ImportStatus > Importer_NS::import_status_map
 

Macro Definition Documentation

◆ DEBUG_AWS_AUTHENTICATION

#define DEBUG_AWS_AUTHENTICATION   0

Definition at line 137 of file Importer.cpp.

◆ DEBUG_RENDER_GROUP_ANALYZER

#define DEBUG_RENDER_GROUP_ANALYZER   0

◆ DEBUG_TIMING

#define DEBUG_TIMING   false

◆ DISABLE_MULTI_THREADED_SHAPEFILE_IMPORT

#define DISABLE_MULTI_THREADED_SHAPEFILE_IMPORT   0

Definition at line 139 of file Importer.cpp.

Function Documentation

◆ get_filesize()

auto get_filesize ( const std::string &  file_path)
inline

Definition at line 74 of file Importer.cpp.

References error_code, and anonymous_namespace{StringDictionary.cpp}::file_size().

Referenced by Importer_NS::DataStreamSink::archivePlumber().

74  {
75  boost::filesystem::path boost_file_path{file_path};
77  const auto filesize = boost::filesystem::file_size(boost_file_path, ec);
78  return ec ? 0 : filesize;
79 }
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t uint32_t const int64_t int32_t * error_code
+ Here is the call graph for this function:
+ Here is the caller graph for this function: