OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer.cpp File Reference
#include "Import/Importer.h"
#include <arrow/api.h>
#include <arrow/io/api.h>
#include <gdal.h>
#include <ogrsf_frmts.h>
#include <unistd.h>
#include <boost/algorithm/string.hpp>
#include <boost/dynamic_bitset.hpp>
#include <boost/filesystem.hpp>
#include <boost/variant.hpp>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <future>
#include <iomanip>
#include <list>
#include <memory>
#include <mutex>
#include <numeric>
#include <stack>
#include <stdexcept>
#include <thread>
#include <typeinfo>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#include "../Archive/PosixFileArchive.h"
#include "../Archive/S3Archive.h"
#include "../QueryEngine/TypePunning.h"
#include "../Shared/geo_compression.h"
#include "../Shared/geo_types.h"
#include "../Shared/geosupport.h"
#include "../Shared/import_helpers.h"
#include "../Shared/mapd_glob.h"
#include "../Shared/mapdpath.h"
#include "../Shared/measure.h"
#include "../Shared/scope.h"
#include "../Shared/shard_key.h"
#include "../Shared/thread_count.h"
#include "ArrowImporter.h"
#include "Import/DelimitedParserUtils.h"
#include "QueryRunner/QueryRunner.h"
#include "Shared/Logger.h"
#include "Shared/SqlTypesLayout.h"
#include "Utils/ChunkAccessorTable.h"
#include "gen-cpp/MapD.h"
+ Include dependency graph for Importer.cpp:

Go to the source code of this file.

Classes

struct  anonymous_namespace{Importer.cpp}::OGRDataSourceDeleter
 
struct  anonymous_namespace{Importer.cpp}::OGRFeatureDeleter
 
struct  anonymous_namespace{Importer.cpp}::OGRSpatialReferenceDeleter
 
struct  Importer_NS::GeoImportException
 

Namespaces

 anonymous_namespace{Importer.cpp}
 
 boost
 
 boost::log
 
 Importer_NS
 
 Importer_NS::anonymous_namespace{Importer.cpp}
 

Macros

#define DEBUG_TIMING   false
 
#define DEBUG_RENDER_GROUP_ANALYZER   0
 
#define DEBUG_AWS_AUTHENTICATION   0
 
#define DISABLE_MULTI_THREADED_SHAPEFILE_IMPORT   0
 

Typedefs

using anonymous_namespace{Importer.cpp}::OGRDataSourceUqPtr = std::unique_ptr< OGRDataSource, OGRDataSourceDeleter >
 
using anonymous_namespace{Importer.cpp}::OGRFeatureUqPtr = std::unique_ptr< OGRFeature, OGRFeatureDeleter >
 
using anonymous_namespace{Importer.cpp}::OGRSpatialReferenceUqPtr = std::unique_ptr< OGRSpatialReference, OGRSpatialReferenceDeleter >
 
using Importer_NS::FieldNameToIndexMapType = std::map< std::string, size_t >
 
using Importer_NS::ColumnNameToSourceNameMapType = std::map< std::string, std::string >
 
using Importer_NS::ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer >>
 
using Importer_NS::FeaturePtrVector = std::vector< OGRFeatureUqPtr >
 

Functions

auto get_filesize (const std::string &file_path)
 
formatting_ostream & boost::log::operator<< (formatting_ostream &out, std::vector< std::string > &row)
 
static const std::string Importer_NS::trim_space (const char *field, const size_t len)
 
int8_t * Importer_NS::appendDatum (int8_t *buf, Datum d, const SQLTypeInfo &ti)
 
Datum Importer_NS::NullDatum (SQLTypeInfo &ti)
 
Datum Importer_NS::NullArrayDatum (SQLTypeInfo &ti)
 
ArrayDatum Importer_NS::StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
 
ArrayDatum Importer_NS::NullArray (const SQLTypeInfo &ti)
 
void Importer_NS::addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec)
 
Datum Importer_NS::TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti)
 
ArrayDatum Importer_NS::TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti)
 
bool Importer_NS::importGeoFromLonLat (double lon, double lat, std::vector< double > &coords)
 
uint64_t Importer_NS::compress_coord (double coord, const SQLTypeInfo &ti, bool x)
 
std::vector< uint8_t > Importer_NS::compress_coords (std::vector< double > &coords, const SQLTypeInfo &ti)
 
std::tuple< int, SQLTypes,
std::string > 
Importer_NS::anonymous_namespace{Importer.cpp}::explode_collections_step1 (const std::list< const ColumnDescriptor * > &col_descs)
 
int64_t Importer_NS::anonymous_namespace{Importer.cpp}::explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
 
static ImportStatus Importer_NS::import_thread_delimited (int thread_id, Importer *importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer)
 
static ImportStatus Importer_NS::import_thread_shapefile (int thread_id, Importer *importer, OGRSpatialReference *poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap)
 
int64_t Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
template<class T >
bool Importer_NS::try_cast (const std::string &str)
 
char * Importer_NS::try_strptimes (const char *str, const std::vector< std::string > &formats)
 
void Importer_NS::GDALErrorHandler (CPLErr eErrClass, int err_no, const char *msg)
 
OGRLayer & Importer_NS::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName (const std::string &geo_layer_name, const OGRDataSourceUqPtr &poDS, const std::string &file_name)
 
std::pair< SQLTypes, bool > Importer_NS::ogr_to_type (const OGRFieldType &ogr_type)
 
SQLTypes Importer_NS::ogr_to_type (const OGRwkbGeometryType &ogr_type)
 
void Importer_NS::gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files)
 

Variables

size_t g_archive_read_buf_size = 1 << 20
 
static constexpr bool Importer_NS::PROMOTE_POLYGON_TO_MULTIPOLYGON = true
 
static mapd_shared_mutex Importer_NS::status_mutex
 
static std::map< std::string,
ImportStatus > 
Importer_NS::import_status_map
 

Macro Definition Documentation

#define DEBUG_AWS_AUTHENTICATION   0

Definition at line 140 of file Importer.cpp.

#define DEBUG_RENDER_GROUP_ANALYZER   0
#define DISABLE_MULTI_THREADED_SHAPEFILE_IMPORT   0

Definition at line 142 of file Importer.cpp.

Function Documentation

auto get_filesize ( const std::string &  file_path)
inline

Definition at line 77 of file Importer.cpp.

References error_code, and anonymous_namespace{StringDictionary.cpp}::file_size().

Referenced by Importer_NS::DataStreamSink::archivePlumber().

77  {
78  boost::filesystem::path boost_file_path{file_path};
80  const auto filesize = boost::filesystem::file_size(boost_file_path, ec);
81  return ec ? 0 : filesize;
82 }
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t uint32_t const int64_t int32_t * error_code

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

size_t g_archive_read_buf_size = 1 << 20