21 #include <unordered_set>
23 #include <boost/filesystem.hpp>
25 #include <ogrsf_frmts.h>
34 namespace import_export {
38 , gdal_dataset_{
nullptr}
40 , array_null_handling_{ArrayNullHandling::kAbortWithWarning} {}
60 #define SCI(x) static_cast<int>(x)
64 static constexpr std::array<const char*, 4>
driver_names = {
"INVALID",
83 {{
true,
false,
false},
86 {
true,
false,
false}}};
89 {{
".csv",
".tsv"}, {
".geojson",
".json"}, {
".geojson",
".json"}, {
".shp"}}};
127 return OFTIntegerList;
137 return OFTStringList;
143 return OFTInteger64List;
152 throw std::runtime_error(
"Column '" + name +
"' has unsupported type '" +
154 file_type_names[
SCI(file_type)] +
"'");
160 const std::string& layer_name,
162 const std::vector<TargetMetaInfo>& column_infos,
178 OGRwkbGeometryType ogr_geometry_type = wkbUnknown;
179 int num_geo_columns = 0;
180 int geo_column_srid = 0;
181 uint32_t num_columns = 0;
182 std::string geo_column_name;
183 for (
auto const& column_info : column_infos) {
184 auto const& type_info = column_info.get_type_info();
185 if (type_info.is_geometry()) {
186 switch (type_info.get_type()) {
188 ogr_geometry_type = wkbPoint;
191 ogr_geometry_type = wkbLineString;
194 ogr_geometry_type = wkbPolygon;
197 ogr_geometry_type = wkbMultiPolygon;
202 geo_column_srid = type_info.get_output_srid();
203 geo_column_name =
safeColumnName(column_info.get_resname(), num_columns + 1);
206 auto column_name =
safeColumnName(column_info.get_resname(), num_columns + 1);
212 if (num_geo_columns != 1) {
213 throw std::runtime_error(
"File type '" +
215 "' requires exactly one geo column in query results");
219 if (geo_column_srid <= 0) {
220 throw std::runtime_error(
"Geo column '" + geo_column_name +
"' has invalid SRID (" +
222 "). Use ST_SetSRID() in query to override.");
227 auto gdal_driver = GetGDALDriverManager()->GetDriverByName(driver_name);
228 if (gdal_driver ==
nullptr) {
229 throw std::runtime_error(
"Failed to find Driver '" + std::string(driver_name) +
234 auto gdal_file_path{file_path};
235 auto user_file_path{file_path};
240 throw std::runtime_error(
241 "Selected file compression option not yet supported for file type '" +
251 auto remove_file = [](
const std::string&
filename) {
252 if (boost::filesystem::exists(
filename)) {
254 boost::filesystem::remove(
filename);
257 remove_file(file_path);
258 remove_file(user_file_path);
260 LOG(
INFO) <<
"Exporting to file '" << user_file_path <<
"'";
264 gdal_driver->Create(gdal_file_path.c_str(), 0, 0, 0, GDT_Unknown, NULL);
266 throw std::runtime_error(
"Failed to create File '" + file_path +
"'");
270 OGRSpatialReference ogr_spatial_reference;
271 if (ogr_spatial_reference.importFromEPSG(geo_column_srid)) {
272 throw std::runtime_error(
"Failed to create Spatial Reference for SRID " +
275 #if GDAL_VERSION_MAJOR >= 3
276 ogr_spatial_reference.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
281 layer_name.c_str(), &ogr_spatial_reference, ogr_geometry_type, NULL);
283 throw std::runtime_error(
"Failed to create Layer '" + layer_name +
"'");
287 int column_index = 0;
290 for (
auto const& column_info : column_infos) {
291 auto column_name =
safeColumnName(column_info.get_resname(), column_index + 1);
293 auto const& type_info = column_info.get_type_info();
294 if (!type_info.is_geometry()) {
295 OGRFieldDefn field_defn(
298 if (
ogr_layer_->CreateField(&field_defn) != OGRERR_NONE) {
299 throw std::runtime_error(
"Failed to create Field '" + column_name +
"'");
308 }
catch (std::exception& e) {
309 LOG(
INFO) <<
"GDAL Query Export failed to start: " << e.what();
319 const int field_index,
320 OGRFeature* ogr_feature) {
328 auto const point_tv = boost::get<GeoPointTargetValue>(geo_tv->get());
329 auto* coords = point_tv.coords.get();
335 auto const linestring_tv = boost::get<GeoLineStringTargetValue>(geo_tv->get());
336 auto* coords = linestring_tv.coords.get();
342 auto const polygon_tv = boost::get<GeoPolyTargetValue>(geo_tv->get());
343 auto* coords = polygon_tv.coords.get();
345 auto* ring_sizes = polygon_tv.ring_sizes.get();
351 auto const multipolygon_tv = boost::get<GeoMultiPolyTargetValue>(geo_tv->get());
352 auto* coords = multipolygon_tv.coords.get();
354 auto* ring_sizes = multipolygon_tv.ring_sizes.get();
356 auto* poly_rings = multipolygon_tv.poly_rings.get();
368 const int field_index,
369 OGRFeature* ogr_feature) {
373 auto field_type = ogr_feature->GetFieldDefnRef(field_index)->GetType();
376 if (boost::get<int64_t>(scalar_tv)) {
377 auto int_val = *(boost::get<int64_t>(scalar_tv));
378 bool is_int64 =
false;
406 ogr_feature->SetFieldNull(field_index);
409 constexpr
size_t buf_size = 9;
413 ogr_feature->SetField(field_index, buf);
414 }
else if (is_int64) {
416 ogr_feature->SetField(field_index, static_cast<GIntBig>(int_val));
419 ogr_feature->SetField(field_index,
SCI(int_val));
421 }
else if (boost::get<double>(scalar_tv)) {
422 auto real_val = *(boost::get<double>(scalar_tv));
429 ogr_feature->SetFieldNull(field_index);
432 ogr_feature->SetField(field_index, real_val);
434 }
else if (boost::get<float>(scalar_tv)) {
436 auto real_val = *(boost::get<float>(scalar_tv));
438 ogr_feature->SetFieldNull(field_index);
441 ogr_feature->SetField(field_index, real_val);
444 auto s = boost::get<NullableString>(scalar_tv);
445 is_null = !s || boost::get<void*>(s);
447 ogr_feature->SetFieldNull(field_index);
450 auto s_notnull = boost::get<std::string>(s);
452 ogr_feature->SetField(field_index, s_notnull->c_str());
459 const int field_index,
460 OGRFeature* ogr_feature,
461 const std::string& column_name,
466 if (!array_tv->is_initialized()) {
468 ogr_feature->SetFieldNull(field_index);
472 auto const& scalar_tvs = array_tv->get();
474 auto field_type = ogr_feature->GetFieldDefnRef(field_index)->GetType();
480 std::vector<int> int_values;
481 std::vector<GIntBig> int64_values;
482 std::vector<std::string> string_values;
483 std::vector<double> real_values;
484 switch (field_type) {
486 int_values.reserve(scalar_tvs.size());
488 case OFTInteger64List:
489 int64_values.reserve(scalar_tvs.size());
492 real_values.reserve(scalar_tvs.size());
495 string_values.reserve(scalar_tvs.size());
501 bool force_null_to_zero =
505 bool any_null =
false;
506 for (uint32_t i = 0; i < scalar_tvs.size(); i++) {
508 auto const scalar_tv = &scalar_tvs[i];
509 if (boost::get<int64_t>(scalar_tv)) {
510 auto int_val = *(boost::get<int64_t>(scalar_tv));
511 bool is_int64 =
false;
540 string_values.emplace_back(
"");
542 constexpr
size_t buf_size = 9;
546 string_values.emplace_back(buf);
548 }
else if (is_int64) {
549 if (is_null && force_null_to_zero) {
550 int64_values.push_back(0);
552 int64_values.push_back(int_val);
555 if (is_null && force_null_to_zero) {
556 int_values.push_back(0);
558 int_values.push_back(int_val);
561 }
else if (boost::get<double>(scalar_tv)) {
562 auto real_val = *(boost::get<double>(scalar_tv));
568 if (is_null && force_null_to_zero) {
569 real_values.push_back(0.0);
571 real_values.push_back(real_val);
573 }
else if (boost::get<float>(scalar_tv)) {
575 auto real_val = *(boost::get<float>(scalar_tv));
577 if (is_null && force_null_to_zero) {
578 real_values.push_back(0.0);
580 real_values.push_back(static_cast<double>(real_val));
583 auto s = boost::get<NullableString>(scalar_tv);
584 is_null = !s || boost::get<void*>(s);
586 string_values.emplace_back(
"");
588 auto s_notnull = boost::get<std::string>(s);
590 string_values.emplace_back(s_notnull->c_str());
598 switch (array_null_handling) {
600 throw std::runtime_error(
601 "Found individual nulls in Array Column '" + column_name +
"' of type '" +
603 "'. Use 'array_null_handling' Export Option to specify behaviour.");
605 ogr_feature->SetFieldNull(field_index);
613 switch (field_type) {
615 ogr_feature->SetField(field_index, int_values.size(), int_values.data());
617 case OFTInteger64List:
618 ogr_feature->SetField(field_index, int64_values.size(), int64_values.data());
621 ogr_feature->SetField(field_index, real_values.size(), real_values.data());
623 case OFTStringList: {
624 std::vector<const char*> raw_strings;
625 raw_strings.reserve(string_values.size() + 1);
626 for (
auto const& string_value : string_values) {
627 raw_strings.push_back(string_value.c_str());
629 raw_strings.push_back(
nullptr);
630 ogr_feature->SetField(field_index, raw_strings.data());
640 const std::vector<AggregatedResult>& query_results) {
642 for (
auto const& agg_result : query_results) {
643 auto results = agg_result.rs;
644 auto const& targets = agg_result.targets_meta;
650 auto const crt_row = results->getNextRow(
true,
true);
651 if (crt_row.empty()) {
656 auto ogr_feature = OGRFeature::CreateFeature(
ogr_layer_->GetLayerDefn());
661 OGRFeature::DestroyFeature(ogr_feature);
664 for (
size_t i = 0; i < results->colCount(); ++i) {
665 auto const tv = crt_row[i];
666 auto const& ti = targets[i].get_type_info();
667 auto const column_name =
safeColumnName(targets[i].get_resname(), i + 1);
671 auto const scalar_tv = boost::get<ScalarTargetValue>(&tv);
675 auto const array_tv = boost::get<ArrayTargetValue>(&tv);
684 auto const geo_tv = boost::get<GeoTargetValue>(&tv);
685 if (geo_tv && geo_tv->is_initialized()) {
688 ogr_feature->SetGeometry(
nullptr);
695 if (
ogr_layer_->CreateFeature(ogr_feature) != OGRERR_NONE) {
696 throw std::runtime_error(
"Failed to create Feature");
700 }
catch (std::exception& e) {
701 LOG(
INFO) <<
"GDAL Query Export failed: " << e.what();
HOST DEVICE SQLTypes get_subtype() const
std::string filename(char const *path)
const FileType file_type_
const OGRGeometry * getOGRGeometry() const
static constexpr std::array< std::array< bool, 3 >, 4 > compression_implemented
void insert_array_column(const ArrayTargetValue *array_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature, const std::string &column_name, QueryExporter::ArrayNullHandling array_null_handling)
std::string safeColumnName(const std::string &resname, const int column_index)
void beginExport(const std::string &file_path, const std::string &layer_name, const CopyParams ©_params, const std::vector< TargetMetaInfo > &column_infos, const FileCompression file_compression, const ArrayNullHandling array_null_handling) final
HOST DEVICE SQLTypes get_type() const
size_t formatHMS(char *buf, size_t const max, int64_t const unixtime)
ArrayNullHandling array_null_handling_
static constexpr std::array< const char *, 3 > compression_prefix
OGRFieldType sql_type_info_to_ogr_field_type(const std::string &name, const SQLTypeInfo &type_info, const QueryExporter::FileType file_type)
CONSTEXPR DEVICE bool is_null(const T &value)
boost::optional< std::vector< ScalarTargetValue >> ArrayTargetValue
void insert_geo_column(const GeoTargetValue *geo_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
static constexpr std::array< const char *, 4 > driver_names
void exportResults(const std::vector< AggregatedResult > &query_results) final
boost::optional< boost::variant< GeoPointTargetValue, GeoLineStringTargetValue, GeoPolyTargetValue, GeoMultiPolyTargetValue >> GeoTargetValue
static constexpr std::array< const char *, 3 > compression_suffix
void insert_scalar_column(const ScalarTargetValue *scalar_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
std::string get_type_name() const
std::vector< int > field_indices_
static constexpr std::array< const char *, 4 > file_type_names
QueryExporterGDAL()=delete
Basic constructors and methods of the row set interface.
static std::array< std::unordered_set< std::string >, 4 > file_type_valid_extensions
void validateFileExtensions(const std::string &file_path, const std::string &file_type, const std::unordered_set< std::string > &valid_extensions) const
boost::variant< int64_t, double, float, NullableString > ScalarTargetValue
GDALDataset * gdal_dataset_