import_export::QueryExporterGDAL Class Reference

#include <QueryExporterGDAL.h>

Inheritance diagram for import_export::QueryExporterGDAL:

Collaboration diagram for import_export::QueryExporterGDAL:

Public Member Functions
	QueryExporterGDAL (const FileType file_type)
	QueryExporterGDAL ()=delete
	~QueryExporterGDAL () override
void	beginExport (const std::string &file_path, const std::string &layer_name, const CopyParams &copy_params, const std::vector< TargetMetaInfo > &column_infos, const FileCompression file_compression, const ArrayNullHandling array_null_handling) final
void	exportResults (const std::vector< AggregatedResult > &query_results) final
void	endExport () final
Public Member Functions inherited from import_export::QueryExporter
	QueryExporter (const FileType file_type)
	QueryExporter ()=delete
virtual	~QueryExporter ()
virtual void	beginExport (const std::string &file_path, const std::string &layer_name, const CopyParams &copy_params, const std::vector< TargetMetaInfo > &column_info, const FileCompression file_compression, const ArrayNullHandling array_null_handling)=0

Private Member Functions
void	cleanUp ()

Private Attributes
CopyParams	copy_params_
GDALDataset *	gdal_dataset_
OGRLayer *	ogr_layer_
std::vector< int >	field_indices_
ArrayNullHandling	array_null_handling_

Additional Inherited Members
Public Types inherited from import_export::QueryExporter
enum	FileType {   FileType::kCSV, FileType::kGeoJSON, FileType::kGeoJSONL, FileType::kShapefile,   FileType::kFlatGeobuf }
enum	FileCompression { FileCompression::kNone, FileCompression::kGZip, FileCompression::kZip }
enum	ArrayNullHandling { ArrayNullHandling::kAbortWithWarning, ArrayNullHandling::kExportSentinels, ArrayNullHandling::kExportZeros, ArrayNullHandling::kNullEntireField }
Static Public Member Functions inherited from import_export::QueryExporter
static std::unique_ptr < QueryExporter >	create (const FileType file_type)
Protected Member Functions inherited from import_export::QueryExporter
void	validateFileExtensions (const std::string &file_path, const std::string &file_type, const std::unordered_set< std::string > &valid_extensions) const
std::string	safeColumnName (const std::string &resname, const int column_index)
Protected Attributes inherited from import_export::QueryExporter
const FileType	file_type_

Detailed Description

Definition at line 31 of file QueryExporterGDAL.h.

Constructor & Destructor Documentation

import_export::QueryExporterGDAL::QueryExporterGDAL ( const FileType  file_type )

explicit

Definition at line 36 of file QueryExporterGDAL.cpp.

    : QueryExporter(file_type)
    , gdal_dataset_{nullptr}
    , ogr_layer_{nullptr}
    , array_null_handling_{ArrayNullHandling::kAbortWithWarning} {}

import_export::QueryExporterGDAL::QueryExporterGDAL ( )

delete

import_export::QueryExporterGDAL::~QueryExporterGDAL ( )

override

Definition at line 42 of file QueryExporterGDAL.cpp.

References cleanUp().

                                      {
  cleanUp();
}

Here is the call graph for this function:

Member Function Documentation

void import_export::QueryExporterGDAL::beginExport ( const std::string &  file_path, const std::string &  layer_name, const CopyParams &  copy_params, const std::vector< TargetMetaInfo > &  column_infos, const FileCompression  file_compression, const ArrayNullHandling  array_null_handling  )

final

Definition at line 163 of file QueryExporterGDAL.cpp.

References array_null_handling_, CHECK, cleanUp(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_implemented, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_prefix, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_suffix, copy_params_, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::driver_names, field_indices_, import_export::QueryExporter::file_type_, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_names, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_valid_extensions, nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename(), gdal_dataset_, logger::INFO, Geospatial::GDAL::init(), kLINESTRING, kMULTIPOLYGON, import_export::QueryExporter::kNone, kPOINT, kPOLYGON, LOG, ogr_layer_, import_export::QueryExporter::safeColumnName(), SCI, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::sql_type_info_to_ogr_field_type(), to_string(), and import_export::QueryExporter::validateFileExtensions().

                                                                                 {
  validateFileExtensions(file_path,
                         file_type_names[SCI(file_type_)],
                         file_type_valid_extensions[SCI(file_type_)]);

  // lazy init GDAL
  Geospatial::GDAL::init();

  // capture these
  copy_params_ = copy_params;
  array_null_handling_ = array_null_handling;

  try {
    // determine OGR geometry type and SRID and validate other column types
    OGRwkbGeometryType ogr_geometry_type = wkbUnknown;
    int num_geo_columns = 0;
    int geo_column_srid = 0;
    uint32_t num_columns = 0;
    std::string geo_column_name;
    for (auto const& column_info : column_infos) {
      auto const& type_info = column_info.get_type_info();
      if (type_info.is_geometry()) {
        switch (type_info.get_type()) {
          case kPOINT:
            ogr_geometry_type = wkbPoint;
            break;
          case kLINESTRING:
            ogr_geometry_type = wkbLineString;
            break;
          case kPOLYGON:
            ogr_geometry_type = wkbPolygon;
            break;
          case kMULTIPOLYGON:
            ogr_geometry_type = wkbMultiPolygon;
            break;
          default:
            CHECK(false);
        }
        geo_column_srid = type_info.get_output_srid();
        geo_column_name = safeColumnName(column_info.get_resname(), num_columns + 1);
        num_geo_columns++;
      } else {
        auto column_name = safeColumnName(column_info.get_resname(), num_columns + 1);
        // this will throw if column type is unsupported for this file type
        sql_type_info_to_ogr_field_type(column_name, type_info, file_type_);
      }
      num_columns++;
    }
    if (num_geo_columns != 1) {
      throw std::runtime_error("File type '" +
                               std::string(file_type_names[SCI(file_type_)]) +
                               "' requires exactly one geo column in query results");
    }

    // validate SRID
    if (geo_column_srid <= 0) {
      throw std::runtime_error("Geo column '" + geo_column_name + "' has invalid SRID (" +
                               std::to_string(geo_column_srid) +
                               "). Use ST_SetSRID() in query to override.");
    }

    // get driver
    auto const& driver_name = driver_names[SCI(file_type_)];
    auto gdal_driver = GetGDALDriverManager()->GetDriverByName(driver_name);
    if (gdal_driver == nullptr) {
      throw std::runtime_error("Failed to find Driver '" + std::string(driver_name) +
                               "'");
    }

    // compression?
    auto gdal_file_path{file_path};
    auto user_file_path{file_path};
    if (file_compression != FileCompression::kNone) {
      auto impl = compression_implemented[SCI(file_type_)][SCI(file_compression)];
      if (!impl) {
        // @TODO(se) implement more compression options
        throw std::runtime_error(
            "Selected file compression option not yet supported for file type '" +
            std::string(file_type_names[SCI(file_type_)]) + "'");
      }
      gdal_file_path.insert(0, compression_prefix[SCI(file_compression)]);
      gdal_file_path.append(compression_suffix[SCI(file_compression)]);
      user_file_path.append(compression_suffix[SCI(file_compression)]);
    }

    // delete any existing file(s) (with and without compression suffix)
    // GeoJSON driver occasionally refuses to overwrite
    auto remove_file = [](const std::string& filename) {
      if (boost::filesystem::exists(filename)) {
        LOG(INFO) << "Deleting existing file '" << filename << "'";
        boost::filesystem::remove(filename);
      }
    };
    remove_file(file_path);
    remove_file(user_file_path);

    LOG(INFO) << "Exporting to file '" << user_file_path << "'";

    // create dataset
    gdal_dataset_ =
        gdal_driver->Create(gdal_file_path.c_str(), 0, 0, 0, GDT_Unknown, NULL);
    if (gdal_dataset_ == nullptr) {
      throw std::runtime_error("Failed to create File '" + file_path + "'");
    }

    // create spatial reference
    OGRSpatialReference ogr_spatial_reference;
    if (ogr_spatial_reference.importFromEPSG(geo_column_srid)) {
      throw std::runtime_error("Failed to create Spatial Reference for SRID " +
                               std::to_string(geo_column_srid) + "");
    }
#if GDAL_VERSION_MAJOR >= 3
    ogr_spatial_reference.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
#endif

    // create layer
    ogr_layer_ = gdal_dataset_->CreateLayer(
        layer_name.c_str(), &ogr_spatial_reference, ogr_geometry_type, NULL);
    if (ogr_layer_ == nullptr) {
      throw std::runtime_error("Failed to create Layer '" + layer_name + "'");
    }

    // create fields
    int column_index = 0;
    int field_index = 0;
    field_indices_.resize(num_columns);
    for (auto const& column_info : column_infos) {
      auto column_name = safeColumnName(column_info.get_resname(), column_index + 1);
      // create fields for non-geo columns
      auto const& type_info = column_info.get_type_info();
      if (!type_info.is_geometry()) {
        OGRFieldDefn field_defn(
            column_name.c_str(),
            sql_type_info_to_ogr_field_type(column_name, type_info, file_type_));
        if (ogr_layer_->CreateField(&field_defn) != OGRERR_NONE) {
          throw std::runtime_error("Failed to create Field '" + column_name + "'");
        }
        field_indices_[column_index] = field_index;
        field_index++;
      } else {
        field_indices_[column_index] = -1;
      }
      column_index++;
    }
  } catch (std::exception& e) {
    LOG(INFO) << "GDAL Query Export failed to start: " << e.what();
    cleanUp();
    throw;
  }
}

Here is the call graph for this function:

void import_export::QueryExporterGDAL::cleanUp ( )

private

Definition at line 46 of file QueryExporterGDAL.cpp.

References field_indices_, gdal_dataset_, and ogr_layer_.

Referenced by beginExport(), endExport(), exportResults(), and ~QueryExporterGDAL().

                                {
  // close dataset
  if (gdal_dataset_) {
    GDALClose(gdal_dataset_);
    gdal_dataset_ = nullptr;
  }

  // forget layer
  ogr_layer_ = nullptr;

  // forget field indices
  field_indices_.clear();
}

Here is the caller graph for this function:

void import_export::QueryExporterGDAL::endExport ( )

finalvirtual

Implements import_export::QueryExporter.

Definition at line 705 of file QueryExporterGDAL.cpp.

References cleanUp().

                                  {
  cleanUp();
}

Here is the call graph for this function:

void import_export::QueryExporterGDAL::exportResults ( const std::vector< AggregatedResult > &  query_results )

finalvirtual

Implements import_export::QueryExporter.

Definition at line 637 of file QueryExporterGDAL.cpp.

References array_null_handling_, CHECK, cleanUp(), field_indices_, ResultSet::GeoTargetValue, logger::INFO, import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_array_column(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_geo_column(), import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_scalar_column(), LOG, ogr_layer_, and import_export::QueryExporter::safeColumnName().

                                                      {
  try {
    for (auto const& agg_result : query_results) {
      auto results = agg_result.rs;
      auto const& targets = agg_result.targets_meta;

      // configure ResultSet to return geo as raw data
      results->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);

      while (true) {
        auto const crt_row = results->getNextRow(true, true);
        if (crt_row.empty()) {
          break;
        }

        // create feature for this row
        auto ogr_feature = OGRFeature::CreateFeature(ogr_layer_->GetLayerDefn());
        CHECK(ogr_feature);

        // destroy feature on exiting this scope
        ScopeGuard destroy_feature = [ogr_feature] {
          OGRFeature::DestroyFeature(ogr_feature);
        };

        for (size_t i = 0; i < results->colCount(); ++i) {
          auto const tv = crt_row[i];
          auto const& ti = targets[i].get_type_info();
          auto const column_name = safeColumnName(targets[i].get_resname(), i + 1);
          auto const field_index = field_indices_[i];

          // insert this column into the feature
          auto const scalar_tv = boost::get<ScalarTargetValue>(&tv);
          if (scalar_tv) {
            insert_scalar_column(scalar_tv, ti, field_index, ogr_feature);
          } else {
            auto const array_tv = boost::get<ArrayTargetValue>(&tv);
            if (array_tv) {
              insert_array_column(array_tv,
                                  ti,
                                  field_index,
                                  ogr_feature,
                                  column_name,
                                  array_null_handling_);
            } else {
              auto const geo_tv = boost::get<GeoTargetValue>(&tv);
              if (geo_tv && geo_tv->is_initialized()) {
                insert_geo_column(geo_tv, ti, field_index, ogr_feature);
              } else {
                ogr_feature->SetGeometry(nullptr);
              }
            }
          }
        }

        // add feature to layer
        if (ogr_layer_->CreateFeature(ogr_feature) != OGRERR_NONE) {
          throw std::runtime_error("Failed to create Feature");
        }
      }
    }
  } catch (std::exception& e) {
    LOG(INFO) << "GDAL Query Export failed: " << e.what();
    cleanUp();
    throw;
  }
}

Here is the call graph for this function:

Member Data Documentation

ArrayNullHandling import_export::QueryExporterGDAL::array_null_handling_

private

Definition at line 51 of file QueryExporterGDAL.h.

Referenced by beginExport(), and exportResults().

CopyParams import_export::QueryExporterGDAL::copy_params_

private

Definition at line 47 of file QueryExporterGDAL.h.

Referenced by beginExport().

std::vector<int> import_export::QueryExporterGDAL::field_indices_

private

Definition at line 50 of file QueryExporterGDAL.h.

Referenced by beginExport(), cleanUp(), and exportResults().

GDALDataset* import_export::QueryExporterGDAL::gdal_dataset_

private

Definition at line 48 of file QueryExporterGDAL.h.

Referenced by beginExport(), and cleanUp().

OGRLayer* import_export::QueryExporterGDAL::ogr_layer_

private

Definition at line 49 of file QueryExporterGDAL.h.

Referenced by beginExport(), cleanUp(), and exportResults().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/QueryExporterGDAL.h
• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/QueryExporterGDAL.cpp