OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
import_export::QueryExporterGDAL Class Reference

#include <QueryExporterGDAL.h>

+ Inheritance diagram for import_export::QueryExporterGDAL:
+ Collaboration diagram for import_export::QueryExporterGDAL:

Public Member Functions

 QueryExporterGDAL (const FileType file_type)
 
 QueryExporterGDAL ()=delete
 
 ~QueryExporterGDAL () override
 
void beginExport (const std::string &file_path, const std::string &layer_name, const CopyParams &copy_params, const std::vector< TargetMetaInfo > &column_infos, const FileCompression file_compression, const ArrayNullHandling array_null_handling) final
 
void exportResults (const std::vector< AggregatedResult > &query_results) final
 
void endExport () final
 
- Public Member Functions inherited from import_export::QueryExporter
 QueryExporter (const FileType file_type)
 
 QueryExporter ()=delete
 
virtual ~QueryExporter ()
 

Private Member Functions

void cleanUp ()
 

Private Attributes

CopyParams copy_params_
 
GDALDataset * gdal_dataset_
 
OGRLayer * ogr_layer_
 
std::vector< int > field_indices_
 
ArrayNullHandling array_null_handling_
 

Additional Inherited Members

- Public Types inherited from import_export::QueryExporter
enum  FileType {
  FileType::kCSV, FileType::kGeoJSON, FileType::kGeoJSONL, FileType::kShapefile,
  FileType::kFlatGeobuf
}
 
enum  FileCompression { FileCompression::kNone, FileCompression::kGZip, FileCompression::kZip }
 
enum  ArrayNullHandling { ArrayNullHandling::kAbortWithWarning, ArrayNullHandling::kExportSentinels, ArrayNullHandling::kExportZeros, ArrayNullHandling::kNullEntireField }
 
- Static Public Member Functions inherited from import_export::QueryExporter
static std::unique_ptr
< QueryExporter
create (const FileType file_type)
 
- Protected Member Functions inherited from import_export::QueryExporter
void validateFileExtensions (const std::string &file_path, const std::string &file_type, const std::unordered_set< std::string > &valid_extensions) const
 
std::string safeColumnName (const std::string &resname, const int column_index)
 
- Protected Attributes inherited from import_export::QueryExporter
const FileType file_type_
 

Detailed Description

Definition at line 31 of file QueryExporterGDAL.h.

Constructor & Destructor Documentation

import_export::QueryExporterGDAL::QueryExporterGDAL ( const FileType  file_type)
explicit
import_export::QueryExporterGDAL::QueryExporterGDAL ( )
delete
import_export::QueryExporterGDAL::~QueryExporterGDAL ( )
override

Definition at line 42 of file QueryExporterGDAL.cpp.

References cleanUp().

42  {
43  cleanUp();
44 }

+ Here is the call graph for this function:

Member Function Documentation

void import_export::QueryExporterGDAL::beginExport ( const std::string &  file_path,
const std::string &  layer_name,
const CopyParams copy_params,
const std::vector< TargetMetaInfo > &  column_infos,
const FileCompression  file_compression,
const ArrayNullHandling  array_null_handling 
)
finalvirtual

Implements import_export::QueryExporter.

Definition at line 163 of file QueryExporterGDAL.cpp.

References array_null_handling_, CHECK, cleanUp(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_implemented, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_prefix, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_suffix, copy_params_, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::driver_names, field_indices_, import_export::QueryExporter::file_type_, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_names, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_valid_extensions, nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename(), gdal_dataset_, logger::INFO, Geospatial::GDAL::init(), kLINESTRING, kMULTIPOLYGON, import_export::QueryExporter::kNone, kPOINT, kPOLYGON, LOG, ogr_layer_, import_export::QueryExporter::safeColumnName(), SCI, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::sql_type_info_to_ogr_field_type(), to_string(), and import_export::QueryExporter::validateFileExtensions().

168  {
169  validateFileExtensions(file_path,
172 
173  // lazy init GDAL
175 
176  // capture these
177  copy_params_ = copy_params;
178  array_null_handling_ = array_null_handling;
179 
180  try {
181  // determine OGR geometry type and SRID and validate other column types
182  OGRwkbGeometryType ogr_geometry_type = wkbUnknown;
183  int num_geo_columns = 0;
184  int geo_column_srid = 0;
185  uint32_t num_columns = 0;
186  std::string geo_column_name;
187  for (auto const& column_info : column_infos) {
188  auto const& type_info = column_info.get_type_info();
189  if (type_info.is_geometry()) {
190  switch (type_info.get_type()) {
191  case kPOINT:
192  ogr_geometry_type = wkbPoint;
193  break;
194  case kLINESTRING:
195  ogr_geometry_type = wkbLineString;
196  break;
197  case kPOLYGON:
198  ogr_geometry_type = wkbPolygon;
199  break;
200  case kMULTIPOLYGON:
201  ogr_geometry_type = wkbMultiPolygon;
202  break;
203  default:
204  CHECK(false);
205  }
206  geo_column_srid = type_info.get_output_srid();
207  geo_column_name = safeColumnName(column_info.get_resname(), num_columns + 1);
208  num_geo_columns++;
209  } else {
210  auto column_name = safeColumnName(column_info.get_resname(), num_columns + 1);
211  // this will throw if column type is unsupported for this file type
212  sql_type_info_to_ogr_field_type(column_name, type_info, file_type_);
213  }
214  num_columns++;
215  }
216  if (num_geo_columns != 1) {
217  throw std::runtime_error("File type '" +
218  std::string(file_type_names[SCI(file_type_)]) +
219  "' requires exactly one geo column in query results");
220  }
221 
222  // validate SRID
223  if (geo_column_srid <= 0) {
224  throw std::runtime_error("Geo column '" + geo_column_name + "' has invalid SRID (" +
225  std::to_string(geo_column_srid) +
226  "). Use ST_SetSRID() in query to override.");
227  }
228 
229  // get driver
230  auto const& driver_name = driver_names[SCI(file_type_)];
231  auto gdal_driver = GetGDALDriverManager()->GetDriverByName(driver_name);
232  if (gdal_driver == nullptr) {
233  throw std::runtime_error("Failed to find Driver '" + std::string(driver_name) +
234  "'");
235  }
236 
237  // compression?
238  auto gdal_file_path{file_path};
239  auto user_file_path{file_path};
240  if (file_compression != FileCompression::kNone) {
241  auto impl = compression_implemented[SCI(file_type_)][SCI(file_compression)];
242  if (!impl) {
243  // @TODO(se) implement more compression options
244  throw std::runtime_error(
245  "Selected file compression option not yet supported for file type '" +
246  std::string(file_type_names[SCI(file_type_)]) + "'");
247  }
248  gdal_file_path.insert(0, compression_prefix[SCI(file_compression)]);
249  gdal_file_path.append(compression_suffix[SCI(file_compression)]);
250  user_file_path.append(compression_suffix[SCI(file_compression)]);
251  }
252 
253  // delete any existing file(s) (with and without compression suffix)
254  // GeoJSON driver occasionally refuses to overwrite
255  auto remove_file = [](const std::string& filename) {
256  if (boost::filesystem::exists(filename)) {
257  LOG(INFO) << "Deleting existing file '" << filename << "'";
258  boost::filesystem::remove(filename);
259  }
260  };
261  remove_file(file_path);
262  remove_file(user_file_path);
263 
264  LOG(INFO) << "Exporting to file '" << user_file_path << "'";
265 
266  // create dataset
267  gdal_dataset_ =
268  gdal_driver->Create(gdal_file_path.c_str(), 0, 0, 0, GDT_Unknown, NULL);
269  if (gdal_dataset_ == nullptr) {
270  throw std::runtime_error("Failed to create File '" + file_path + "'");
271  }
272 
273  // create spatial reference
274  OGRSpatialReference ogr_spatial_reference;
275  if (ogr_spatial_reference.importFromEPSG(geo_column_srid)) {
276  throw std::runtime_error("Failed to create Spatial Reference for SRID " +
277  std::to_string(geo_column_srid) + "");
278  }
279 #if GDAL_VERSION_MAJOR >= 3
280  ogr_spatial_reference.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
281 #endif
282 
283  // create layer
284  ogr_layer_ = gdal_dataset_->CreateLayer(
285  layer_name.c_str(), &ogr_spatial_reference, ogr_geometry_type, NULL);
286  if (ogr_layer_ == nullptr) {
287  throw std::runtime_error("Failed to create Layer '" + layer_name + "'");
288  }
289 
290  // create fields
291  int column_index = 0;
292  int field_index = 0;
293  field_indices_.resize(num_columns);
294  for (auto const& column_info : column_infos) {
295  auto column_name = safeColumnName(column_info.get_resname(), column_index + 1);
296  // create fields for non-geo columns
297  auto const& type_info = column_info.get_type_info();
298  if (!type_info.is_geometry()) {
299  OGRFieldDefn field_defn(
300  column_name.c_str(),
301  sql_type_info_to_ogr_field_type(column_name, type_info, file_type_));
302  if (ogr_layer_->CreateField(&field_defn) != OGRERR_NONE) {
303  throw std::runtime_error("Failed to create Field '" + column_name + "'");
304  }
305  field_indices_[column_index] = field_index;
306  field_index++;
307  } else {
308  field_indices_[column_index] = -1;
309  }
310  column_index++;
311  }
312  } catch (std::exception& e) {
313  LOG(INFO) << "GDAL Query Export failed to start: " << e.what();
314  cleanUp();
315  throw;
316  }
317 }
static constexpr std::array< const char *, 5 > file_type_names
static constexpr std::array< std::array< bool, 3 >, 5 > compression_implemented
#define LOG(tag)
Definition: Logger.h:203
std::string safeColumnName(const std::string &resname, const int column_index)
static void init()
Definition: GDAL.cpp:59
#define SCI(x)
std::string to_string(char const *&&v)
static std::array< std::unordered_set< std::string >, 5 > file_type_valid_extensions
static constexpr std::array< const char *, 3 > compression_prefix
OGRFieldType sql_type_info_to_ogr_field_type(const std::string &name, const SQLTypeInfo &type_info, const QueryExporter::FileType file_type)
static constexpr std::array< const char *, 3 > compression_suffix
#define CHECK(condition)
Definition: Logger.h:209
void validateFileExtensions(const std::string &file_path, const std::string &file_type, const std::unordered_set< std::string > &valid_extensions) const
static constexpr std::array< const char *, 5 > driver_names

+ Here is the call graph for this function:

void import_export::QueryExporterGDAL::cleanUp ( )
private

Definition at line 46 of file QueryExporterGDAL.cpp.

References field_indices_, gdal_dataset_, and ogr_layer_.

Referenced by beginExport(), endExport(), exportResults(), and ~QueryExporterGDAL().

46  {
47  // close dataset
48  if (gdal_dataset_) {
49  GDALClose(gdal_dataset_);
50  gdal_dataset_ = nullptr;
51  }
52 
53  // forget layer
54  ogr_layer_ = nullptr;
55 
56  // forget field indices
57  field_indices_.clear();
58 }

+ Here is the caller graph for this function:

void import_export::QueryExporterGDAL::endExport ( )
finalvirtual

Implements import_export::QueryExporter.

Definition at line 711 of file QueryExporterGDAL.cpp.

References cleanUp().

711  {
712  cleanUp();
713 }

+ Here is the call graph for this function:

void import_export::QueryExporterGDAL::exportResults ( const std::vector< AggregatedResult > &  query_results)
finalvirtual

Implements import_export::QueryExporter.

Definition at line 643 of file QueryExporterGDAL.cpp.

References array_null_handling_, CHECK, cleanUp(), field_indices_, ResultSet::GeoTargetValue, i, logger::INFO, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_array_column(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_geo_column(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_scalar_column(), LOG, ogr_layer_, and import_export::QueryExporter::safeColumnName().

644  {
645  try {
646  for (auto const& agg_result : query_results) {
647  auto results = agg_result.rs;
648  auto const& targets = agg_result.targets_meta;
649 
650  // configure ResultSet to return geo as raw data
651  results->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
652 
653  while (true) {
654  auto const crt_row = results->getNextRow(true, true);
655  if (crt_row.empty()) {
656  break;
657  }
658 
659  // create feature for this row
660  auto ogr_feature = OGRFeature::CreateFeature(ogr_layer_->GetLayerDefn());
661  CHECK(ogr_feature);
662 
663  // destroy feature on exiting this scope
664  ScopeGuard destroy_feature = [ogr_feature] {
665  OGRFeature::DestroyFeature(ogr_feature);
666  };
667 
668  for (size_t i = 0; i < results->colCount(); ++i) {
669  auto const tv = crt_row[i];
670  auto const& ti = targets[i].get_type_info();
671  auto const column_name = safeColumnName(targets[i].get_resname(), i + 1);
672  auto const field_index = field_indices_[i];
673 
674  // insert this column into the feature
675  auto const scalar_tv = boost::get<ScalarTargetValue>(&tv);
676  if (scalar_tv) {
677  insert_scalar_column(scalar_tv, ti, field_index, ogr_feature);
678  } else {
679  auto const array_tv = boost::get<ArrayTargetValue>(&tv);
680  if (array_tv) {
681  insert_array_column(array_tv,
682  ti,
683  field_index,
684  ogr_feature,
685  column_name,
687  } else {
688  auto const geo_tv = boost::get<GeoTargetValue>(&tv);
689  if (geo_tv && geo_tv->is_initialized()) {
690  insert_geo_column(geo_tv, ti, field_index, ogr_feature);
691  } else {
692  ogr_feature->SetGeometry(nullptr);
693  }
694  }
695  }
696  }
697 
698  // add feature to layer
699  if (ogr_layer_->CreateFeature(ogr_feature) != OGRERR_NONE) {
700  throw std::runtime_error("Failed to create Feature");
701  }
702  }
703  }
704  } catch (std::exception& e) {
705  LOG(INFO) << "GDAL Query Export failed: " << e.what();
706  cleanUp();
707  throw;
708  }
709 }
#define LOG(tag)
Definition: Logger.h:203
void insert_array_column(const ArrayTargetValue *array_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature, const std::string &column_name, QueryExporter::ArrayNullHandling array_null_handling)
std::string safeColumnName(const std::string &resname, const int column_index)
void insert_geo_column(const GeoTargetValue *geo_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
void insert_scalar_column(const ScalarTargetValue *scalar_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

Member Data Documentation

ArrayNullHandling import_export::QueryExporterGDAL::array_null_handling_
private

Definition at line 51 of file QueryExporterGDAL.h.

Referenced by beginExport(), and exportResults().

CopyParams import_export::QueryExporterGDAL::copy_params_
private

Definition at line 47 of file QueryExporterGDAL.h.

Referenced by beginExport().

std::vector<int> import_export::QueryExporterGDAL::field_indices_
private

Definition at line 50 of file QueryExporterGDAL.h.

Referenced by beginExport(), cleanUp(), and exportResults().

GDALDataset* import_export::QueryExporterGDAL::gdal_dataset_
private

Definition at line 48 of file QueryExporterGDAL.h.

Referenced by beginExport(), and cleanUp().

OGRLayer* import_export::QueryExporterGDAL::ogr_layer_
private

Definition at line 49 of file QueryExporterGDAL.h.

Referenced by beginExport(), cleanUp(), and exportResults().


The documentation for this class was generated from the following files: