OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
import_export::anonymous_namespace{QueryExporterGDAL.cpp} Namespace Reference

Functions

OGRFieldType sql_type_info_to_ogr_field_type (const std::string &name, const SQLTypeInfo &type_info, const QueryExporter::FileType file_type)
 
void insert_geo_column (const GeoTargetValue *geo_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
 
void insert_scalar_column (const ScalarTargetValue *scalar_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
 
void insert_array_column (const ArrayTargetValue *array_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature, const std::string &column_name, QueryExporter::ArrayNullHandling array_null_handling)
 

Variables

static constexpr std::array
< const char *, 5 > 
driver_names
 
static constexpr std::array
< const char *, 5 > 
file_type_names
 
static constexpr std::array
< const char *, 3 > 
compression_prefix
 
static constexpr std::array
< const char *, 3 > 
compression_suffix = {"", ".gz", ".zip"}
 
static constexpr std::array
< std::array< bool, 3 >, 5 > 
compression_implemented
 
static std::array
< std::unordered_set
< std::string >, 5 > 
file_type_valid_extensions
 

Function Documentation

void import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_array_column ( const ArrayTargetValue array_tv,
const SQLTypeInfo ti,
const int  field_index,
OGRFeature *  ogr_feature,
const std::string &  column_name,
QueryExporter::ArrayNullHandling  array_null_handling 
)

Definition at line 481 of file QueryExporterGDAL.cpp.

References CHECK, CHECK_EQ, CHECK_GE, shared::convert_temporal_to_iso_format(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type_name(), SQLTypeInfo::is_geometry(), is_null(), SQLTypeInfo::is_time(), import_export::QueryExporter::kAbortWithWarning, kBIGINT, kBOOLEAN, kDATE, import_export::QueryExporter::kExportZeros, kFLOAT, kINT, import_export::QueryExporter::kNullEntireField, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_BIGINT, NULL_BOOLEAN, NULL_DOUBLE, NULL_FLOAT, NULL_INT, NULL_SMALLINT, and NULL_TINYINT.

Referenced by import_export::QueryExporterGDAL::exportResults().

486  {
487  CHECK_GE(field_index, 0);
488  CHECK(!ti.is_geometry());
489 
490  if (!array_tv->is_initialized()) {
491  // entire array is null
492  ogr_feature->SetFieldNull(field_index);
493  return;
494  }
495 
496  auto const& scalar_tvs = array_tv->get();
497 
498  auto field_type = ogr_feature->GetFieldDefnRef(field_index)->GetType();
499 
500  // only one of these will get used
501  // could use a std::vector<ScalarTargetValue> but need raw data at the end
502  // so we would have to extract it there anyway, so not sure it's worthwhile
503  // we can, at least, pre-reserve whichever array we're going to use
504  std::vector<int> int_values;
505  std::vector<GIntBig> int64_values;
506  std::vector<std::string> string_values;
507  std::vector<double> real_values;
508  switch (field_type) {
509  case OFTIntegerList:
510  int_values.reserve(scalar_tvs.size());
511  break;
512  case OFTInteger64List:
513  int64_values.reserve(scalar_tvs.size());
514  break;
515  case OFTRealList:
516  real_values.reserve(scalar_tvs.size());
517  break;
518  case OFTStringList:
519  string_values.reserve(scalar_tvs.size());
520  break;
521  default:
522  CHECK(false);
523  }
524 
525  bool force_null_to_zero =
526  (array_null_handling == QueryExporter::ArrayNullHandling::kExportZeros);
527 
528  // now extract the data
529  bool any_null = false;
530  for (uint32_t i = 0; i < scalar_tvs.size(); i++) {
531  bool is_null = false;
532  auto const scalar_tv = &scalar_tvs[i];
533  if (boost::get<int64_t>(scalar_tv)) {
534  auto int_val = *(boost::get<int64_t>(scalar_tv));
535  bool is_int64 = false;
536  switch (ti.get_subtype()) {
537  case kBOOLEAN:
538  is_null = (int_val == NULL_BOOLEAN);
539  break;
540  case kTINYINT:
541  is_null = (int_val == NULL_TINYINT);
542  break;
543  case kSMALLINT:
544  is_null = (int_val == NULL_SMALLINT);
545  break;
546  case kINT:
547  is_null = (int_val == NULL_INT);
548  break;
549  case kBIGINT:
550  is_null = (int_val == NULL_BIGINT);
551  is_int64 = true;
552  break;
553  case kTIME:
554  case kTIMESTAMP:
555  case kDATE:
556  is_null = (int_val == NULL_BIGINT);
557  is_int64 = true;
558  break;
559  default:
560  is_null = false;
561  }
562  if (ti.get_elem_type().is_time()) {
563  if (is_null) {
564  string_values.emplace_back("");
565  } else {
566  string_values.emplace_back(
568  }
569  } else if (is_int64) {
570  if (is_null && force_null_to_zero) {
571  int64_values.push_back(0);
572  } else {
573  int64_values.push_back(int_val);
574  }
575  } else {
576  if (is_null && force_null_to_zero) {
577  int_values.push_back(0);
578  } else {
579  int_values.push_back(int_val);
580  }
581  }
582  } else if (boost::get<double>(scalar_tv)) {
583  auto real_val = *(boost::get<double>(scalar_tv));
584  if (ti.get_subtype() == kFLOAT) {
585  is_null = (real_val == NULL_FLOAT);
586  } else {
587  is_null = (real_val == NULL_DOUBLE);
588  }
589  if (is_null && force_null_to_zero) {
590  real_values.push_back(0.0);
591  } else {
592  real_values.push_back(real_val);
593  }
594  } else if (boost::get<float>(scalar_tv)) {
595  CHECK_EQ(kFLOAT, ti.get_subtype());
596  auto real_val = *(boost::get<float>(scalar_tv));
597  is_null = (real_val == NULL_FLOAT);
598  if (is_null && force_null_to_zero) {
599  real_values.push_back(0.0);
600  } else {
601  real_values.push_back(static_cast<double>(real_val));
602  }
603  } else {
604  auto s = boost::get<NullableString>(scalar_tv);
605  is_null = !s || boost::get<void*>(s);
606  if (is_null) {
607  string_values.emplace_back("");
608  } else {
609  auto s_notnull = boost::get<std::string>(s);
610  CHECK(s_notnull);
611  string_values.emplace_back(s_notnull->c_str());
612  }
613  }
614  any_null |= is_null;
615  }
616 
617  // special behaviour if we found any individual nulls?
618  if (any_null) {
619  switch (array_null_handling) {
620  case QueryExporter::ArrayNullHandling::kAbortWithWarning:
621  throw std::runtime_error(
622  "Found individual nulls in Array Column '" + column_name + "' of type '" +
623  ti.get_type_name() +
624  "'. Use 'array_null_handling' Export Option to specify behaviour.");
625  case QueryExporter::ArrayNullHandling::kNullEntireField:
626  ogr_feature->SetFieldNull(field_index);
627  return;
628  default:
629  break;
630  }
631  }
632 
633  // store the captured array in the feature
634  switch (field_type) {
635  case OFTIntegerList:
636  ogr_feature->SetField(field_index, int_values.size(), int_values.data());
637  break;
638  case OFTInteger64List:
639  ogr_feature->SetField(field_index, int64_values.size(), int64_values.data());
640  break;
641  case OFTRealList:
642  ogr_feature->SetField(field_index, real_values.size(), real_values.data());
643  break;
644  case OFTStringList: {
645  std::vector<const char*> raw_strings;
646  raw_strings.reserve(string_values.size() + 1);
647  for (auto const& string_value : string_values) {
648  raw_strings.push_back(string_value.c_str());
649  }
650  raw_strings.push_back(nullptr);
651  ogr_feature->SetField(field_index, raw_strings.data());
652  } break;
653  default:
654  CHECK(false);
655  }
656 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
#define CHECK_EQ(x, y)
Definition: Logger.h:301
#define NULL_DOUBLE
Definition: sqltypes.h:76
#define NULL_FLOAT
#define NULL_BIGINT
std::string convert_temporal_to_iso_format(const SQLTypeInfo &type_info, int64_t unix_time)
Definition: misc.cpp:109
#define CHECK_GE(x, y)
Definition: Logger.h:306
bool is_time() const
Definition: sqltypes.h:577
#define NULL_INT
CONSTEXPR DEVICE bool is_null(const T &value)
#define NULL_BOOLEAN
Definition: sqltypes.h:80
std::string get_type_name() const
Definition: sqltypes.h:482
#define NULL_TINYINT
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:595
#define NULL_SMALLINT
Definition: sqltypes.h:72
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:975

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_geo_column ( const GeoTargetValue geo_tv,
const SQLTypeInfo ti,
const int  field_index,
OGRFeature *  ogr_feature 
)

Definition at line 327 of file QueryExporterGDAL.cpp.

References CHECK, CHECK_EQ, SQLTypeInfo::get_type(), Geospatial::GeoBase::getOGRGeometry(), SQLTypeInfo::is_geometry(), kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kPOINT, and kPOLYGON.

Referenced by import_export::QueryExporterGDAL::exportResults().

330  {
331  CHECK_EQ(field_index, -1);
332  CHECK(ti.is_geometry());
333 
334  // use Geo classes to convert to OGRGeometry
335  // and destroy when Geo object goes out of scope
336  switch (ti.get_type()) {
337  case kPOINT: {
338  auto const point_tv = boost::get<GeoPointTargetValue>(geo_tv->get());
339  auto* coords = point_tv.coords.get();
340  CHECK(coords);
341  Geospatial::GeoPoint point(*coords);
342  ogr_feature->SetGeometry(point.getOGRGeometry());
343  } break;
344  case kMULTIPOINT: {
345  auto const multipoint_tv = boost::get<GeoMultiPointTargetValue>(geo_tv->get());
346  auto* coords = multipoint_tv.coords.get();
347  CHECK(coords);
348  Geospatial::GeoMultiPoint multipoint(*coords);
349  ogr_feature->SetGeometry(multipoint.getOGRGeometry());
350  } break;
351  case kLINESTRING: {
352  auto const linestring_tv = boost::get<GeoLineStringTargetValue>(geo_tv->get());
353  auto* coords = linestring_tv.coords.get();
354  CHECK(coords);
355  Geospatial::GeoLineString linestring(*coords);
356  ogr_feature->SetGeometry(linestring.getOGRGeometry());
357  } break;
358  case kMULTILINESTRING: {
359  auto const multilinestring_tv =
360  boost::get<GeoMultiLineStringTargetValue>(geo_tv->get());
361  auto* coords = multilinestring_tv.coords.get();
362  CHECK(coords);
363  auto* linestring_sizes = multilinestring_tv.linestring_sizes.get();
364  CHECK(linestring_sizes);
365  Geospatial::GeoMultiLineString multilinestring(*coords, *linestring_sizes);
366  ogr_feature->SetGeometry(multilinestring.getOGRGeometry());
367  } break;
368  case kPOLYGON: {
369  auto const polygon_tv = boost::get<GeoPolyTargetValue>(geo_tv->get());
370  auto* coords = polygon_tv.coords.get();
371  CHECK(coords);
372  auto* ring_sizes = polygon_tv.ring_sizes.get();
373  CHECK(ring_sizes);
374  Geospatial::GeoPolygon polygon(*coords, *ring_sizes);
375  ogr_feature->SetGeometry(polygon.getOGRGeometry());
376  } break;
377  case kMULTIPOLYGON: {
378  auto const multipolygon_tv = boost::get<GeoMultiPolyTargetValue>(geo_tv->get());
379  auto* coords = multipolygon_tv.coords.get();
380  CHECK(coords);
381  auto* ring_sizes = multipolygon_tv.ring_sizes.get();
382  CHECK(ring_sizes);
383  auto* poly_rings = multipolygon_tv.poly_rings.get();
384  CHECK(poly_rings);
385  Geospatial::GeoMultiPolygon multipolygon(*coords, *ring_sizes, *poly_rings);
386  ogr_feature->SetGeometry(multipolygon.getOGRGeometry());
387  } break;
388  default:
389  CHECK(false);
390  }
391 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:595

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_scalar_column ( const ScalarTargetValue scalar_tv,
const SQLTypeInfo ti,
const int  field_index,
OGRFeature *  ogr_feature 
)

Definition at line 393 of file QueryExporterGDAL.cpp.

References CHECK, CHECK_EQ, CHECK_GE, shared::convert_temporal_to_iso_format(), SQLTypeInfo::get_type(), SQLTypeInfo::is_geometry(), is_null(), SQLTypeInfo::is_time(), kBIGINT, kBOOLEAN, kDATE, kFLOAT, kINT, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_BIGINT, NULL_BOOLEAN, NULL_DOUBLE, NULL_FLOAT, NULL_INT, NULL_SMALLINT, NULL_TINYINT, and SCI.

Referenced by import_export::QueryExporterGDAL::exportResults().

396  {
397  CHECK_GE(field_index, 0);
398  CHECK(!ti.is_geometry());
399 
400  auto field_type = ogr_feature->GetFieldDefnRef(field_index)->GetType();
401 
402  bool is_null{false};
403  if (boost::get<int64_t>(scalar_tv)) {
404  auto int_val = *(boost::get<int64_t>(scalar_tv));
405  bool is_int64 = false;
406  switch (ti.get_type()) {
407  case kBOOLEAN:
408  is_null = (int_val == NULL_BOOLEAN);
409  break;
410  case kTINYINT:
411  is_null = (int_val == NULL_TINYINT);
412  break;
413  case kSMALLINT:
414  is_null = (int_val == NULL_SMALLINT);
415  break;
416  case kINT:
417  is_null = (int_val == NULL_INT);
418  break;
419  case kBIGINT:
420  is_null = (int_val == NULL_BIGINT);
421  is_int64 = true;
422  break;
423  case kTIME:
424  case kTIMESTAMP:
425  case kDATE:
426  is_null = (int_val == NULL_BIGINT);
427  is_int64 = true;
428  break;
429  default:
430  is_null = false;
431  }
432  if (is_null) {
433  ogr_feature->SetFieldNull(field_index);
434  } else if (ti.is_time()) {
435  CHECK_EQ(field_type, OFTString);
436  auto str = shared::convert_temporal_to_iso_format(ti, int_val);
437  ogr_feature->SetField(field_index, str.c_str());
438  } else if (is_int64) {
439  CHECK_EQ(field_type, OFTInteger64);
440  ogr_feature->SetField(field_index, static_cast<GIntBig>(int_val));
441  } else {
442  CHECK_EQ(field_type, OFTInteger);
443  ogr_feature->SetField(field_index, SCI(int_val));
444  }
445  } else if (boost::get<double>(scalar_tv)) {
446  auto real_val = *(boost::get<double>(scalar_tv));
447  if (ti.get_type() == kFLOAT) {
448  is_null = (real_val == NULL_FLOAT);
449  } else {
450  is_null = (real_val == NULL_DOUBLE);
451  }
452  if (is_null) {
453  ogr_feature->SetFieldNull(field_index);
454  } else {
455  CHECK_EQ(field_type, OFTReal);
456  ogr_feature->SetField(field_index, real_val);
457  }
458  } else if (boost::get<float>(scalar_tv)) {
459  CHECK_EQ(kFLOAT, ti.get_type());
460  auto real_val = *(boost::get<float>(scalar_tv));
461  if (real_val == NULL_FLOAT) {
462  ogr_feature->SetFieldNull(field_index);
463  } else {
464  CHECK_EQ(field_type, OFTReal);
465  ogr_feature->SetField(field_index, real_val);
466  }
467  } else {
468  auto s = boost::get<NullableString>(scalar_tv);
469  is_null = !s || boost::get<void*>(s);
470  if (is_null) {
471  ogr_feature->SetFieldNull(field_index);
472  } else {
473  CHECK_EQ(field_type, OFTString);
474  auto s_notnull = boost::get<std::string>(s);
475  CHECK(s_notnull);
476  ogr_feature->SetField(field_index, s_notnull->c_str());
477  }
478  }
479 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
#define NULL_DOUBLE
Definition: sqltypes.h:76
#define NULL_FLOAT
#define NULL_BIGINT
std::string convert_temporal_to_iso_format(const SQLTypeInfo &type_info, int64_t unix_time)
Definition: misc.cpp:109
#define CHECK_GE(x, y)
Definition: Logger.h:306
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
bool is_time() const
Definition: sqltypes.h:577
#define SCI(x)
#define NULL_INT
CONSTEXPR DEVICE bool is_null(const T &value)
#define NULL_BOOLEAN
Definition: sqltypes.h:80
#define NULL_TINYINT
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:595
#define NULL_SMALLINT
Definition: sqltypes.h:72

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

OGRFieldType import_export::anonymous_namespace{QueryExporterGDAL.cpp}::sql_type_info_to_ogr_field_type ( const std::string &  name,
const SQLTypeInfo type_info,
const QueryExporter::FileType  file_type 
)

Definition at line 94 of file QueryExporterGDAL.cpp.

References SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, import_export::QueryExporter::kFlatGeobuf, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kNUMERIC, import_export::QueryExporter::kShapefile, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and SCI.

Referenced by import_export::QueryExporterGDAL::beginExport().

96  {
97  // store BOOLEAN as int
98  // store TIME as string (no OFTTimeList and Shapefiles reject OFTTime anyway)
99  // store all other time/date types as int64
100  // Shapefiles cannot store arrays of any type
101  switch (type_info.get_type()) {
102  case kBOOLEAN:
103  case kTINYINT:
104  case kINT:
105  case kSMALLINT:
106  return OFTInteger;
107  case kFLOAT:
108  case kDOUBLE:
109  case kNUMERIC:
110  case kDECIMAL:
111  return OFTReal;
112  case kCHAR:
113  case kVARCHAR:
114  case kTEXT:
115  case kTIME:
116  case kTIMESTAMP:
117  case kDATE:
118  return OFTString;
119  case kBIGINT:
120  case kINTERVAL_DAY_TIME:
122  return OFTInteger64;
123  case kARRAY:
124  if (file_type != QueryExporter::FileType::kShapefile &&
125  file_type != QueryExporter::FileType::kFlatGeobuf) {
126  switch (type_info.get_subtype()) {
127  case kBOOLEAN:
128  case kTINYINT:
129  case kINT:
130  case kSMALLINT:
131  return OFTIntegerList;
132  case kFLOAT:
133  case kDOUBLE:
134  case kNUMERIC:
135  case kDECIMAL:
136  return OFTRealList;
137  case kCHAR:
138  case kVARCHAR:
139  case kTEXT:
140  case kTIME:
141  case kTIMESTAMP:
142  case kDATE:
143  return OFTStringList;
144  case kBIGINT:
145  case kINTERVAL_DAY_TIME:
147  return OFTInteger64List;
148  default:
149  break;
150  }
151  }
152  break;
153  default:
154  break;
155  }
156  throw std::runtime_error("Column '" + name + "' has unsupported type '" +
157  type_info.get_type_name() + "' for file type '" +
158  file_type_names[SCI(file_type)] + "'");
159 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
static constexpr std::array< const char *, 5 > file_type_names
Definition: sqltypes.h:76
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
#define SCI(x)
Definition: sqltypes.h:79
Definition: sqltypes.h:80
std::string get_type_name() const
Definition: sqltypes.h:482
Definition: sqltypes.h:68
Definition: sqltypes.h:72
string name
Definition: setup.in.py:72

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

constexpr std::array<std::array<bool, 3>, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_implemented
static
Initial value:
= {
{{true, false, false},
{true, true, false},
{true, true, false},
{true, false, false},
{true, false, false}}}

Definition at line 84 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 3> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_prefix
static
Initial value:
= {"",
"/vsigzip/",
"/vsizip/"}

Definition at line 76 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 3> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_suffix = {"", ".gz", ".zip"}
static

Definition at line 80 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::driver_names
static
Initial value:
= {"INVALID",
"GeoJSON",
"GeoJSONSeq",
"ESRI Shapefile",
"FlatGeobuf"}

Definition at line 64 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_names
static
Initial value:
= {"CSV",
"GeoJSON",
"GeoJSONL",
"Shapefile",
"FlatGeobuf"}

Definition at line 70 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

std::array<std::unordered_set<std::string>, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_valid_extensions
static
Initial value:
= {
{{".csv", ".tsv"}, {".geojson", ".json"}, {".geojson", ".json"}, {".shp"}, {".fgb"}}}

Definition at line 91 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().