OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
import_export::anonymous_namespace{QueryExporterGDAL.cpp} Namespace Reference

Functions

OGRFieldType sql_type_info_to_ogr_field_type (const std::string &name, const SQLTypeInfo &type_info, const QueryExporter::FileType file_type)
 
void insert_geo_column (const GeoTargetValue *geo_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
 
void insert_scalar_column (const ScalarTargetValue *scalar_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature)
 
void insert_array_column (const ArrayTargetValue *array_tv, const SQLTypeInfo &ti, const int field_index, OGRFeature *ogr_feature, const std::string &column_name, QueryExporter::ArrayNullHandling array_null_handling)
 

Variables

static constexpr std::array
< const char *, 5 > 
driver_names
 
static constexpr std::array
< const char *, 5 > 
file_type_names
 
static constexpr std::array
< const char *, 3 > 
compression_prefix
 
static constexpr std::array
< const char *, 3 > 
compression_suffix = {"", ".gz", ".zip"}
 
static constexpr std::array
< std::array< bool, 3 >, 5 > 
compression_implemented
 
static std::array
< std::unordered_set
< std::string >, 5 > 
file_type_valid_extensions
 

Function Documentation

void import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_array_column ( const ArrayTargetValue array_tv,
const SQLTypeInfo ti,
const int  field_index,
OGRFeature *  ogr_feature,
const std::string &  column_name,
QueryExporter::ArrayNullHandling  array_null_handling 
)

Definition at line 461 of file QueryExporterGDAL.cpp.

References CHECK, CHECK_EQ, CHECK_GE, shared::formatHMS(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type_name(), i, SQLTypeInfo::is_geometry(), is_null(), import_export::QueryExporter::kAbortWithWarning, kBIGINT, kBOOLEAN, kDATE, import_export::QueryExporter::kExportZeros, kFLOAT, kINT, import_export::QueryExporter::kNullEntireField, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_BIGINT, NULL_BOOLEAN, NULL_DOUBLE, NULL_FLOAT, NULL_INT, NULL_SMALLINT, and NULL_TINYINT.

Referenced by import_export::QueryExporterGDAL::exportResults().

466  {
467  CHECK_GE(field_index, 0);
468  CHECK(!ti.is_geometry());
469 
470  if (!array_tv->is_initialized()) {
471  // entire array is null
472  ogr_feature->SetFieldNull(field_index);
473  return;
474  }
475 
476  auto const& scalar_tvs = array_tv->get();
477 
478  auto field_type = ogr_feature->GetFieldDefnRef(field_index)->GetType();
479 
480  // only one of these will get used
481  // could use a std::vector<ScalarTargetValue> but need raw data at the end
482  // so we would have to extract it there anyway, so not sure it's worthwhile
483  // we can, at least, pre-reserve whichever array we're going to use
484  std::vector<int> int_values;
485  std::vector<GIntBig> int64_values;
486  std::vector<std::string> string_values;
487  std::vector<double> real_values;
488  switch (field_type) {
489  case OFTIntegerList:
490  int_values.reserve(scalar_tvs.size());
491  break;
492  case OFTInteger64List:
493  int64_values.reserve(scalar_tvs.size());
494  break;
495  case OFTRealList:
496  real_values.reserve(scalar_tvs.size());
497  break;
498  case OFTStringList:
499  string_values.reserve(scalar_tvs.size());
500  break;
501  default:
502  CHECK(false);
503  }
504 
505  bool force_null_to_zero =
506  (array_null_handling == QueryExporter::ArrayNullHandling::kExportZeros);
507 
508  // now extract the data
509  bool any_null = false;
510  for (uint32_t i = 0; i < scalar_tvs.size(); i++) {
511  bool is_null = false;
512  auto const scalar_tv = &scalar_tvs[i];
513  if (boost::get<int64_t>(scalar_tv)) {
514  auto int_val = *(boost::get<int64_t>(scalar_tv));
515  bool is_int64 = false;
516  switch (ti.get_subtype()) {
517  case kBOOLEAN:
518  is_null = (int_val == NULL_BOOLEAN);
519  break;
520  case kTINYINT:
521  is_null = (int_val == NULL_TINYINT);
522  break;
523  case kSMALLINT:
524  is_null = (int_val == NULL_SMALLINT);
525  break;
526  case kINT:
527  is_null = (int_val == NULL_INT);
528  break;
529  case kBIGINT:
530  is_null = (int_val == NULL_BIGINT);
531  is_int64 = true;
532  break;
533  case kTIME:
534  case kTIMESTAMP:
535  case kDATE:
536  is_null = (int_val == NULL_BIGINT);
537  is_int64 = true;
538  break;
539  default:
540  is_null = false;
541  }
542  if (ti.get_subtype() == kTIME) {
543  if (is_null) {
544  string_values.emplace_back("");
545  } else {
546  constexpr size_t buf_size = 9;
547  char buf[buf_size];
548  size_t const len = shared::formatHMS(buf, buf_size, int_val);
549  CHECK_EQ(8u, len); // 8 == strlen("HH:MM:SS")
550  string_values.emplace_back(buf);
551  }
552  } else if (is_int64) {
553  if (is_null && force_null_to_zero) {
554  int64_values.push_back(0);
555  } else {
556  int64_values.push_back(int_val);
557  }
558  } else {
559  if (is_null && force_null_to_zero) {
560  int_values.push_back(0);
561  } else {
562  int_values.push_back(int_val);
563  }
564  }
565  } else if (boost::get<double>(scalar_tv)) {
566  auto real_val = *(boost::get<double>(scalar_tv));
567  if (ti.get_subtype() == kFLOAT) {
568  is_null = (real_val == NULL_FLOAT);
569  } else {
570  is_null = (real_val == NULL_DOUBLE);
571  }
572  if (is_null && force_null_to_zero) {
573  real_values.push_back(0.0);
574  } else {
575  real_values.push_back(real_val);
576  }
577  } else if (boost::get<float>(scalar_tv)) {
578  CHECK_EQ(kFLOAT, ti.get_subtype());
579  auto real_val = *(boost::get<float>(scalar_tv));
580  is_null = (real_val == NULL_FLOAT);
581  if (is_null && force_null_to_zero) {
582  real_values.push_back(0.0);
583  } else {
584  real_values.push_back(static_cast<double>(real_val));
585  }
586  } else {
587  auto s = boost::get<NullableString>(scalar_tv);
588  is_null = !s || boost::get<void*>(s);
589  if (is_null) {
590  string_values.emplace_back("");
591  } else {
592  auto s_notnull = boost::get<std::string>(s);
593  CHECK(s_notnull);
594  string_values.emplace_back(s_notnull->c_str());
595  }
596  }
597  any_null |= is_null;
598  }
599 
600  // special behaviour if we found any individual nulls?
601  if (any_null) {
602  switch (array_null_handling) {
603  case QueryExporter::ArrayNullHandling::kAbortWithWarning:
604  throw std::runtime_error(
605  "Found individual nulls in Array Column '" + column_name + "' of type '" +
606  ti.get_type_name() +
607  "'. Use 'array_null_handling' Export Option to specify behaviour.");
608  case QueryExporter::ArrayNullHandling::kNullEntireField:
609  ogr_feature->SetFieldNull(field_index);
610  return;
611  default:
612  break;
613  }
614  }
615 
616  // store the captured array in the feature
617  switch (field_type) {
618  case OFTIntegerList:
619  ogr_feature->SetField(field_index, int_values.size(), int_values.data());
620  break;
621  case OFTInteger64List:
622  ogr_feature->SetField(field_index, int64_values.size(), int64_values.data());
623  break;
624  case OFTRealList:
625  ogr_feature->SetField(field_index, real_values.size(), real_values.data());
626  break;
627  case OFTStringList: {
628  std::vector<const char*> raw_strings;
629  raw_strings.reserve(string_values.size() + 1);
630  for (auto const& string_value : string_values) {
631  raw_strings.push_back(string_value.c_str());
632  }
633  raw_strings.push_back(nullptr);
634  ogr_feature->SetField(field_index, raw_strings.data());
635  } break;
636  default:
637  CHECK(false);
638  }
639 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
#define CHECK_EQ(x, y)
Definition: Logger.h:217
#define NULL_DOUBLE
Definition: sqltypes.h:49
#define NULL_FLOAT
#define NULL_BIGINT
#define CHECK_GE(x, y)
Definition: Logger.h:222
size_t formatHMS(char *buf, size_t const max, int64_t const unixtime)
Definition: misc.cpp:80
#define NULL_INT
CONSTEXPR DEVICE bool is_null(const T &value)
#define NULL_BOOLEAN
Definition: sqltypes.h:53
std::string get_type_name() const
Definition: sqltypes.h:432
#define NULL_TINYINT
#define CHECK(condition)
Definition: Logger.h:209
bool is_geometry() const
Definition: sqltypes.h:521
#define NULL_SMALLINT
Definition: sqltypes.h:45

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_geo_column ( const GeoTargetValue geo_tv,
const SQLTypeInfo ti,
const int  field_index,
OGRFeature *  ogr_feature 
)

Definition at line 321 of file QueryExporterGDAL.cpp.

References CHECK, CHECK_EQ, SQLTypeInfo::get_type(), Geospatial::GeoBase::getOGRGeometry(), SQLTypeInfo::is_geometry(), kLINESTRING, kMULTIPOLYGON, kPOINT, and kPOLYGON.

Referenced by import_export::QueryExporterGDAL::exportResults().

324  {
325  CHECK_EQ(field_index, -1);
326  CHECK(ti.is_geometry());
327 
328  // use Geo classes to convert to OGRGeometry
329  // and destroy when Geo object goes out of scope
330  switch (ti.get_type()) {
331  case kPOINT: {
332  auto const point_tv = boost::get<GeoPointTargetValue>(geo_tv->get());
333  auto* coords = point_tv.coords.get();
334  CHECK(coords);
335  Geospatial::GeoPoint point(*coords);
336  ogr_feature->SetGeometry(point.getOGRGeometry());
337  } break;
338  case kLINESTRING: {
339  auto const linestring_tv = boost::get<GeoLineStringTargetValue>(geo_tv->get());
340  auto* coords = linestring_tv.coords.get();
341  CHECK(coords);
342  Geospatial::GeoLineString linestring(*coords);
343  ogr_feature->SetGeometry(linestring.getOGRGeometry());
344  } break;
345  case kPOLYGON: {
346  auto const polygon_tv = boost::get<GeoPolyTargetValue>(geo_tv->get());
347  auto* coords = polygon_tv.coords.get();
348  CHECK(coords);
349  auto* ring_sizes = polygon_tv.ring_sizes.get();
350  CHECK(ring_sizes);
351  Geospatial::GeoPolygon polygon(*coords, *ring_sizes);
352  ogr_feature->SetGeometry(polygon.getOGRGeometry());
353  } break;
354  case kMULTIPOLYGON: {
355  auto const multipolygon_tv = boost::get<GeoMultiPolyTargetValue>(geo_tv->get());
356  auto* coords = multipolygon_tv.coords.get();
357  CHECK(coords);
358  auto* ring_sizes = multipolygon_tv.ring_sizes.get();
359  CHECK(ring_sizes);
360  auto* poly_rings = multipolygon_tv.poly_rings.get();
361  CHECK(poly_rings);
362  Geospatial::GeoMultiPolygon multipolygon(*coords, *ring_sizes, *poly_rings);
363  ogr_feature->SetGeometry(multipolygon.getOGRGeometry());
364  } break;
365  default:
366  CHECK(false);
367  }
368 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
#define CHECK(condition)
Definition: Logger.h:209
bool is_geometry() const
Definition: sqltypes.h:521

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void import_export::anonymous_namespace{QueryExporterGDAL.cpp}::insert_scalar_column ( const ScalarTargetValue scalar_tv,
const SQLTypeInfo ti,
const int  field_index,
OGRFeature *  ogr_feature 
)

Definition at line 370 of file QueryExporterGDAL.cpp.

References CHECK, CHECK_EQ, CHECK_GE, shared::formatHMS(), SQLTypeInfo::get_type(), SQLTypeInfo::is_geometry(), is_null(), kBIGINT, kBOOLEAN, kDATE, kFLOAT, kINT, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_BIGINT, NULL_BOOLEAN, NULL_DOUBLE, NULL_FLOAT, NULL_INT, NULL_SMALLINT, NULL_TINYINT, and SCI.

Referenced by import_export::QueryExporterGDAL::exportResults().

373  {
374  CHECK_GE(field_index, 0);
375  CHECK(!ti.is_geometry());
376 
377  auto field_type = ogr_feature->GetFieldDefnRef(field_index)->GetType();
378 
379  bool is_null{false};
380  if (boost::get<int64_t>(scalar_tv)) {
381  auto int_val = *(boost::get<int64_t>(scalar_tv));
382  bool is_int64 = false;
383  switch (ti.get_type()) {
384  case kBOOLEAN:
385  is_null = (int_val == NULL_BOOLEAN);
386  break;
387  case kTINYINT:
388  is_null = (int_val == NULL_TINYINT);
389  break;
390  case kSMALLINT:
391  is_null = (int_val == NULL_SMALLINT);
392  break;
393  case kINT:
394  is_null = (int_val == NULL_INT);
395  break;
396  case kBIGINT:
397  is_null = (int_val == NULL_BIGINT);
398  is_int64 = true;
399  break;
400  case kTIME:
401  case kTIMESTAMP:
402  case kDATE:
403  is_null = (int_val == NULL_BIGINT);
404  is_int64 = true;
405  break;
406  default:
407  is_null = false;
408  }
409  if (is_null) {
410  ogr_feature->SetFieldNull(field_index);
411  } else if (ti.get_type() == kTIME) {
412  CHECK_EQ(field_type, OFTString);
413  constexpr size_t buf_size = 9;
414  char buf[buf_size];
415  size_t const len = shared::formatHMS(buf, buf_size, int_val);
416  CHECK_EQ(8u, len); // 8 == strlen("HH:MM:SS")
417  ogr_feature->SetField(field_index, buf);
418  } else if (is_int64) {
419  CHECK_EQ(field_type, OFTInteger64);
420  ogr_feature->SetField(field_index, static_cast<GIntBig>(int_val));
421  } else {
422  CHECK_EQ(field_type, OFTInteger);
423  ogr_feature->SetField(field_index, SCI(int_val));
424  }
425  } else if (boost::get<double>(scalar_tv)) {
426  auto real_val = *(boost::get<double>(scalar_tv));
427  if (ti.get_type() == kFLOAT) {
428  is_null = (real_val == NULL_FLOAT);
429  } else {
430  is_null = (real_val == NULL_DOUBLE);
431  }
432  if (is_null) {
433  ogr_feature->SetFieldNull(field_index);
434  } else {
435  CHECK_EQ(field_type, OFTReal);
436  ogr_feature->SetField(field_index, real_val);
437  }
438  } else if (boost::get<float>(scalar_tv)) {
439  CHECK_EQ(kFLOAT, ti.get_type());
440  auto real_val = *(boost::get<float>(scalar_tv));
441  if (real_val == NULL_FLOAT) {
442  ogr_feature->SetFieldNull(field_index);
443  } else {
444  CHECK_EQ(field_type, OFTReal);
445  ogr_feature->SetField(field_index, real_val);
446  }
447  } else {
448  auto s = boost::get<NullableString>(scalar_tv);
449  is_null = !s || boost::get<void*>(s);
450  if (is_null) {
451  ogr_feature->SetFieldNull(field_index);
452  } else {
453  CHECK_EQ(field_type, OFTString);
454  auto s_notnull = boost::get<std::string>(s);
455  CHECK(s_notnull);
456  ogr_feature->SetField(field_index, s_notnull->c_str());
457  }
458  }
459 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217
#define NULL_DOUBLE
Definition: sqltypes.h:49
#define NULL_FLOAT
#define NULL_BIGINT
#define CHECK_GE(x, y)
Definition: Logger.h:222
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
#define SCI(x)
size_t formatHMS(char *buf, size_t const max, int64_t const unixtime)
Definition: misc.cpp:80
#define NULL_INT
CONSTEXPR DEVICE bool is_null(const T &value)
#define NULL_BOOLEAN
Definition: sqltypes.h:53
#define NULL_TINYINT
#define CHECK(condition)
Definition: Logger.h:209
bool is_geometry() const
Definition: sqltypes.h:521
#define NULL_SMALLINT
Definition: sqltypes.h:45

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

OGRFieldType import_export::anonymous_namespace{QueryExporterGDAL.cpp}::sql_type_info_to_ogr_field_type ( const std::string &  name,
const SQLTypeInfo type_info,
const QueryExporter::FileType  file_type 
)

Definition at line 94 of file QueryExporterGDAL.cpp.

References SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SQLTypeInfo::get_type_name(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, import_export::QueryExporter::kFlatGeobuf, kFLOAT, kINT, kINTERVAL_DAY_TIME, kINTERVAL_YEAR_MONTH, kNUMERIC, import_export::QueryExporter::kShapefile, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and SCI.

Referenced by import_export::QueryExporterGDAL::beginExport().

96  {
97  // store BOOLEAN as int
98  // store TIME as string (no OFTTimeList and Shapefiles reject OFTTime anyway)
99  // store all other time/date types as int64
100  // Shapefiles cannot store arrays of any type
101  switch (type_info.get_type()) {
102  case kBOOLEAN:
103  case kTINYINT:
104  case kINT:
105  case kSMALLINT:
106  return OFTInteger;
107  case kFLOAT:
108  case kDOUBLE:
109  case kNUMERIC:
110  case kDECIMAL:
111  return OFTReal;
112  case kCHAR:
113  case kVARCHAR:
114  case kTEXT:
115  case kTIME:
116  return OFTString;
117  case kBIGINT:
118  case kTIMESTAMP:
119  case kDATE:
120  case kINTERVAL_DAY_TIME:
122  return OFTInteger64;
123  case kARRAY:
124  if (file_type != QueryExporter::FileType::kShapefile &&
125  file_type != QueryExporter::FileType::kFlatGeobuf) {
126  switch (type_info.get_subtype()) {
127  case kBOOLEAN:
128  case kTINYINT:
129  case kINT:
130  case kSMALLINT:
131  return OFTIntegerList;
132  case kFLOAT:
133  case kDOUBLE:
134  case kNUMERIC:
135  case kDECIMAL:
136  return OFTRealList;
137  case kCHAR:
138  case kVARCHAR:
139  case kTEXT:
140  case kTIME:
141  return OFTStringList;
142  case kBIGINT:
143  case kTIMESTAMP:
144  case kDATE:
145  case kINTERVAL_DAY_TIME:
147  return OFTInteger64List;
148  default:
149  break;
150  }
151  }
152  break;
153  default:
154  break;
155  }
156  throw std::runtime_error("Column '" + name + "' has unsupported type '" +
157  type_info.get_type_name() + "' for file type '" +
158  file_type_names[SCI(file_type)] + "'");
159 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
static constexpr std::array< const char *, 5 > file_type_names
Definition: sqltypes.h:49
string name
Definition: setup.in.py:72
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
#define SCI(x)
Definition: sqltypes.h:52
Definition: sqltypes.h:53
std::string get_type_name() const
Definition: sqltypes.h:432
Definition: sqltypes.h:41
Definition: sqltypes.h:45

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

constexpr std::array<std::array<bool, 3>, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_implemented
static
Initial value:
= {
{{true, false, false},
{true, true, false},
{true, true, false},
{true, false, false},
{true, false, false}}}

Definition at line 84 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 3> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_prefix
static
Initial value:
= {"",
"/vsigzip/",
"/vsizip/"}

Definition at line 76 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 3> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::compression_suffix = {"", ".gz", ".zip"}
static

Definition at line 80 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::driver_names
static
Initial value:
= {"INVALID",
"GeoJSON",
"GeoJSONSeq",
"ESRI Shapefile",
"FlatGeobuf"}

Definition at line 64 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

constexpr std::array<const char*, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_names
static
Initial value:
= {"CSV",
"GeoJSON",
"GeoJSONL",
"Shapefile",
"FlatGeobuf"}

Definition at line 70 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().

std::array<std::unordered_set<std::string>, 5> import_export::anonymous_namespace{QueryExporterGDAL.cpp}::file_type_valid_extensions
static
Initial value:
= {
{{".csv", ".tsv"}, {".geojson", ".json"}, {".geojson", ".json"}, {".shp"}, {".fgb"}}}

Definition at line 91 of file QueryExporterGDAL.cpp.

Referenced by import_export::QueryExporterGDAL::beginExport().