#include <QueryExporterGDAL.h>

Inheritance diagram for import_export::QueryExporterGDAL:

Collaboration diagram for import_export::QueryExporterGDAL:

Public Member Functions
	QueryExporterGDAL (const FileType file_type)

	QueryExporterGDAL ()=delete

	~QueryExporterGDAL () override

void	beginExport (const std::string &file_path, const std::string &layer_name, const CopyParams &copy_params, const std::vector< TargetMetaInfo > &column_infos, const FileCompression file_compression, const ArrayNullHandling array_null_handling) final

void	exportResults (const std::vector< AggregatedResult > &query_results) final

void	endExport () final

Public Member Functions inherited from import_export::QueryExporter
	QueryExporter (const FileType file_type)

	QueryExporter ()=delete

virtual	~QueryExporter ()

virtual void	beginExport (const std::string &file_path, const std::string &layer_name, const CopyParams &copy_params, const std::vector< TargetMetaInfo > &column_info, const FileCompression file_compression, const ArrayNullHandling array_null_handling)=0

Private Member Functions
void	cleanUp ()

Private Attributes
CopyParams	copy_params_

GDALDataset *	gdal_dataset_

OGRLayer *	ogr_layer_

std::vector< int >	field_indices_

ArrayNullHandling	array_null_handling_

Additional Inherited Members
Public Types inherited from import_export::QueryExporter
enum	FileType { FileType::kCSV, FileType::kGeoJSON, FileType::kGeoJSONL, FileType::kShapefile, FileType::kFlatGeobuf }

enum	FileCompression { FileCompression::kNone, FileCompression::kGZip, FileCompression::kZip }

enum	ArrayNullHandling { ArrayNullHandling::kAbortWithWarning, ArrayNullHandling::kExportSentinels, ArrayNullHandling::kExportZeros, ArrayNullHandling::kNullEntireField }

Static Public Member Functions inherited from import_export::QueryExporter
static std::unique_ptr < QueryExporter >	create (const FileType file_type)

Protected Member Functions inherited from import_export::QueryExporter
void	validateFileExtensions (const std::string &file_path, const std::string &file_type, const std::unordered_set< std::string > &valid_extensions) const

std::string	safeColumnName (const std::string &resname, const int column_index)

Protected Attributes inherited from import_export::QueryExporter
const FileType	file_type_

Detailed Description

Definition at line 31 of file QueryExporterGDAL.h.

Constructor & Destructor Documentation

import_export::QueryExporterGDAL::QueryExporterGDAL ( const FileType file_type )

explicit

Definition at line 36 of file QueryExporterGDAL.cpp.

     : QueryExporter(file_type)
     , gdal_dataset_{nullptr}
     , ogr_layer_{nullptr}
     , array_null_handling_{ArrayNullHandling::kAbortWithWarning} {}

import_export::QueryExporterGDAL::QueryExporterGDAL ( )

delete

import_export::QueryExporterGDAL::~QueryExporterGDAL ( )

override

Definition at line 42 of file QueryExporterGDAL.cpp.

References cleanUp().

                                       {
   cleanUp();
 }

Here is the call graph for this function:

Member Function Documentation

void import_export::QueryExporterGDAL::beginExport	(	const std::string &	file_path,
		const std::string &	layer_name,
		const CopyParams &	copy_params,
		const std::vector< TargetMetaInfo > &	column_infos,
		const FileCompression	file_compression,
		const ArrayNullHandling	array_null_handling
	)

final

Definition at line 163 of file QueryExporterGDAL.cpp.

                                                                                  {
   validateFileExtensions(file_path,
                          file_type_names[SCI(file_type_)],
                          file_type_valid_extensions[SCI(file_type_)]);
 
   // lazy init GDAL
   Geospatial::GDAL::init();
 
   // capture these
   copy_params_ = copy_params;
   array_null_handling_ = array_null_handling;
 
   try {
     // determine OGR geometry type and SRID and validate other column types
     OGRwkbGeometryType ogr_geometry_type = wkbUnknown;
     int num_geo_columns = 0;
     int geo_column_srid = 0;
     uint32_t num_columns = 0;
     std::string geo_column_name;
     for (auto const& column_info : column_infos) {
       auto const& type_info = column_info.get_type_info();
       if (type_info.is_geometry()) {
         switch (type_info.get_type()) {
           case kPOINT:
             ogr_geometry_type = wkbPoint;
             break;
           case kMULTIPOINT:
             ogr_geometry_type = wkbMultiPoint;
             break;
           case kLINESTRING:
             ogr_geometry_type = wkbLineString;
             break;
           case kMULTILINESTRING:
             ogr_geometry_type = wkbMultiLineString;
             break;
           case kPOLYGON:
             ogr_geometry_type = wkbPolygon;
             break;
           case kMULTIPOLYGON:
             ogr_geometry_type = wkbMultiPolygon;
             break;
           default:
             CHECK(false);
         }
         geo_column_srid = type_info.get_output_srid();
         geo_column_name = safeColumnName(column_info.get_resname(), num_columns + 1);
         num_geo_columns++;
       } else {
         auto column_name = safeColumnName(column_info.get_resname(), num_columns + 1);
         // this will throw if column type is unsupported for this file type
         sql_type_info_to_ogr_field_type(column_name, type_info, file_type_);
       }
       num_columns++;
     }
     if (num_geo_columns != 1) {
       throw std::runtime_error("File type '" +
                                std::string(file_type_names[SCI(file_type_)]) +
                                "' requires exactly one geo column in query results");
     }
 
     // validate SRID
     if (geo_column_srid <= 0) {
       throw std::runtime_error("Geo column '" + geo_column_name + "' has invalid SRID (" +
                                std::to_string(geo_column_srid) +
                                "). Use ST_SetSRID() in query to override.");
     }
 
     // get driver
     auto const& driver_name = driver_names[SCI(file_type_)];
     auto gdal_driver = GetGDALDriverManager()->GetDriverByName(driver_name);
     if (gdal_driver == nullptr) {
       throw std::runtime_error("Failed to find Driver '" + std::string(driver_name) +
                                "'");
     }
 
     // compression?
     auto gdal_file_path{file_path};
     auto user_file_path{file_path};
     if (file_compression != FileCompression::kNone) {
       auto impl = compression_implemented[SCI(file_type_)][SCI(file_compression)];
       if (!impl) {
         // @TODO(se) implement more compression options
         throw std::runtime_error(
             "Selected file compression option not yet supported for file type '" +
             std::string(file_type_names[SCI(file_type_)]) + "'");
       }
       gdal_file_path.insert(0, compression_prefix[SCI(file_compression)]);
       gdal_file_path.append(compression_suffix[SCI(file_compression)]);
       user_file_path.append(compression_suffix[SCI(file_compression)]);
     }
 
     // delete any existing file(s) (with and without compression suffix)
     // GeoJSON driver occasionally refuses to overwrite
     auto remove_file = [](const std::string& filename) {
       if (boost::filesystem::exists(filename)) {
         LOG(INFO) << "Deleting existing file '" << filename << "'";
         boost::filesystem::remove(filename);
       }
     };
     remove_file(file_path);
     remove_file(user_file_path);
 
     LOG(INFO) << "Exporting to file '" << user_file_path << "'";
 
     // create dataset
     gdal_dataset_ =
         gdal_driver->Create(gdal_file_path.c_str(), 0, 0, 0, GDT_Unknown, NULL);
     if (gdal_dataset_ == nullptr) {
       throw std::runtime_error("Failed to create File '" + file_path + "'");
     }
 
     // create spatial reference
     OGRSpatialReference ogr_spatial_reference;
     if (ogr_spatial_reference.importFromEPSG(geo_column_srid)) {
       throw std::runtime_error("Failed to create Spatial Reference for SRID " +
                                std::to_string(geo_column_srid) + "");
     }
 #if GDAL_VERSION_MAJOR >= 3
     ogr_spatial_reference.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
 #endif
 
     // create layer
     ogr_layer_ = gdal_dataset_->CreateLayer(
         layer_name.c_str(), &ogr_spatial_reference, ogr_geometry_type, NULL);
     if (ogr_layer_ == nullptr) {
       throw std::runtime_error("Failed to create Layer '" + layer_name + "'");
     }
 
     // create fields
     int column_index = 0;
     int field_index = 0;
     field_indices_.resize(num_columns);
     for (auto const& column_info : column_infos) {
       auto column_name = safeColumnName(column_info.get_resname(), column_index + 1);
       // create fields for non-geo columns
       auto const& type_info = column_info.get_type_info();
       if (!type_info.is_geometry()) {
         OGRFieldDefn field_defn(
             column_name.c_str(),
             sql_type_info_to_ogr_field_type(column_name, type_info, file_type_));
         if (ogr_layer_->CreateField(&field_defn) != OGRERR_NONE) {
           throw std::runtime_error("Failed to create Field '" + column_name + "'");
         }
         field_indices_[column_index] = field_index;
         field_index++;
       } else {
         field_indices_[column_index] = -1;
       }
       column_index++;
     }
   } catch (std::exception& e) {
     LOG(INFO) << "GDAL Query Export failed to start: " << e.what();
     cleanUp();
     throw;
   }
 }

Here is the call graph for this function:

void import_export::QueryExporterGDAL::cleanUp ( )

private

Definition at line 46 of file QueryExporterGDAL.cpp.

References field_indices_, gdal_dataset_, and ogr_layer_.

Referenced by beginExport(), endExport(), exportResults(), and ~QueryExporterGDAL().

                                 {
   // close dataset
   if (gdal_dataset_) {
     GDALClose(gdal_dataset_);
     gdal_dataset_ = nullptr;
   }
 
   // forget layer
   ogr_layer_ = nullptr;
 
   // forget field indices
   field_indices_.clear();
 }

Here is the caller graph for this function:

void import_export::QueryExporterGDAL::endExport ( )

finalvirtual

Implements import_export::QueryExporter.

Definition at line 728 of file QueryExporterGDAL.cpp.

References cleanUp().

                                   {
   cleanUp();
 }

Here is the call graph for this function:

void import_export::QueryExporterGDAL::exportResults ( const std::vector< AggregatedResult > & query_results )

finalvirtual

Implements import_export::QueryExporter.

Definition at line 660 of file QueryExporterGDAL.cpp.

References array_null_handling_, CHECK, cleanUp(), field_indices_, ResultSet::GeoTargetValue, logger::INFO, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_array_column(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_geo_column(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_scalar_column(), LOG, ogr_layer_, and import_export::QueryExporter::safeColumnName().

                                                       {
   try {
     for (auto const& agg_result : query_results) {
       auto results = agg_result.rs;
       auto const& targets = agg_result.targets_meta;
 
       // configure ResultSet to return geo as raw data
       results->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
 
       while (true) {
         auto const crt_row = results->getNextRow(true, true);
         if (crt_row.empty()) {
           break;
         }
 
         // create feature for this row
         auto ogr_feature = OGRFeature::CreateFeature(ogr_layer_->GetLayerDefn());
         CHECK(ogr_feature);
 
         // destroy feature on exiting this scope
         ScopeGuard destroy_feature = [ogr_feature] {
           OGRFeature::DestroyFeature(ogr_feature);
         };
 
         for (size_t i = 0; i < results->colCount(); ++i) {
           auto const tv = crt_row[i];
           auto const& ti = targets[i].get_type_info();
           auto const column_name = safeColumnName(targets[i].get_resname(), i + 1);
           auto const field_index = field_indices_[i];
 
           // insert this column into the feature
           auto const scalar_tv = boost::get<ScalarTargetValue>(&tv);
           if (scalar_tv) {
             insert_scalar_column(scalar_tv, ti, field_index, ogr_feature);
           } else {
             auto const array_tv = boost::get<ArrayTargetValue>(&tv);
             if (array_tv) {
               insert_array_column(array_tv,
                                   ti,
                                   field_index,
                                   ogr_feature,
                                   column_name,
                                   array_null_handling_);
             } else {
               auto const geo_tv = boost::get<GeoTargetValue>(&tv);
               if (geo_tv && geo_tv->is_initialized()) {
                 insert_geo_column(geo_tv, ti, field_index, ogr_feature);
               } else {
                 ogr_feature->SetGeometry(nullptr);
               }
             }
           }
         }
 
         // add feature to layer
         if (ogr_layer_->CreateFeature(ogr_feature) != OGRERR_NONE) {
           throw std::runtime_error("Failed to create Feature");
         }
       }
     }
   } catch (std::exception& e) {
     LOG(INFO) << "GDAL Query Export failed: " << e.what();
     cleanUp();
     throw;
   }
 }

Here is the call graph for this function:

Member Data Documentation

ArrayNullHandling import_export::QueryExporterGDAL::array_null_handling_

private

Definition at line 51 of file QueryExporterGDAL.h.

Referenced by beginExport(), and exportResults().

CopyParams import_export::QueryExporterGDAL::copy_params_

private

Definition at line 47 of file QueryExporterGDAL.h.

Referenced by beginExport().

std::vector<int> import_export::QueryExporterGDAL::field_indices_

private

Definition at line 50 of file QueryExporterGDAL.h.

Referenced by beginExport(), cleanUp(), and exportResults().

GDALDataset* import_export::QueryExporterGDAL::gdal_dataset_

private

Definition at line 48 of file QueryExporterGDAL.h.

Referenced by beginExport(), and cleanUp().

OGRLayer* import_export::QueryExporterGDAL::ogr_layer_

private

Definition at line 49 of file QueryExporterGDAL.h.

Referenced by beginExport(), cleanUp(), and exportResults().

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/ImportExport/QueryExporterGDAL.h
/home/jenkins-slave/workspace/core-os-doxygen/ImportExport/QueryExporterGDAL.cpp

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation