OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
GDAL.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "Geospatial/GDAL.h"
18 
19 #include <array>
20 #include <string>
21 
22 #include <gdal.h>
23 #include <gdal_priv.h>
24 #include <ogrsf_frmts.h>
25 
26 #include <boost/filesystem.hpp>
27 #include "Logger/Logger.h"
29 #include "Shared/scope.h"
30 
31 #ifdef _WIN32
32 #include "Shared/clean_windows.h"
33 #endif
34 
35 namespace Geospatial {
36 
37 namespace {
38 
39 void gdal_error_handler(CPLErr err_class, int err_no, const char* err_msg) {
40  CHECK(err_class >= CE_None && err_class <= CE_Fatal);
41  if (err_no == CPLE_NotSupported) {
42  // squash these
43  return;
44  }
45  static constexpr std::array<const char*, 5> err_class_strings{
46  "Info",
47  "Debug",
48  "Warning",
49  "Failure",
50  "Fatal",
51  };
52  std::string log_msg = std::string("GDAL ") + err_class_strings[err_class] + ": " +
53  err_msg + " (" + std::to_string(err_no) + ")";
54  if (err_class >= CE_Failure) {
55  LOG(ERROR) << log_msg;
56  } else {
57  LOG(INFO) << log_msg;
58  }
59 }
60 
61 } // namespace
62 
63 bool GDAL::initialized_ = false;
64 
65 std::mutex GDAL::init_mutex_;
66 
67 void GDAL::init() {
68  // this should not be called from multiple threads, but...
69  std::lock_guard<std::mutex> guard(init_mutex_);
70 
71  // init under mutex
72  if (!initialized_) {
73  // FIXME(andrewseidl): investigate if CPLPushFinderLocation can be public
74 #ifdef _WIN32
75  _putenv_s(
76  "GDAL_DATA",
77  std::string(heavyai::get_root_abs_path() + "/ThirdParty/gdal-data").c_str());
78  _putenv_s(
79  "PROJ_LIB",
80  std::string(heavyai::get_root_abs_path() + "/ThirdParty/gdal-data/proj").c_str());
81 #else
82  setenv("GDAL_DATA",
83  std::string(heavyai::get_root_abs_path() + "/ThirdParty/gdal-data").c_str(),
84  true);
85  setenv(
86  "PROJ_LIB",
87  std::string(heavyai::get_root_abs_path() + "/ThirdParty/gdal-data/proj").c_str(),
88  true);
89 #endif
90 
91 #ifndef _MSC_VER // TODO
92  // configure SSL certificate path (per S3Archive::init_for_read)
93  // in a production build, GDAL and Curl will have been built on
94  // CentOS, so the baked-in system path will be wrong for Ubuntu
95  // and other Linux distros. Unless the user is deliberately
96  // overriding it by setting SSL_CERT_FILE explicitly in the server
97  // environment, we set it to whichever CA bundle directory exists
98  // on the machine we're running on
99  static constexpr std::array<const char*, 6> known_ca_paths{
100  "/etc/ssl/certs/ca-certificates.crt",
101  "/etc/pki/tls/certs/ca-bundle.crt",
102  "/usr/share/ssl/certs/ca-bundle.crt",
103  "/usr/local/share/certs/ca-root.crt",
104  "/etc/ssl/cert.pem",
105  "/etc/ssl/ca-bundle.pem"};
106  for (const auto& known_ca_path : known_ca_paths) {
107  if (boost::filesystem::exists(known_ca_path)) {
108  LOG(INFO) << "GDAL SSL Certificate path: " << known_ca_path;
109  setenv("SSL_CERT_FILE", known_ca_path, false); // no overwrite
110  break;
111  }
112  }
113 #endif
114 
115  GDALAllRegister();
116  OGRRegisterAll();
117  CPLSetErrorHandler(*gdal_error_handler);
118  LOG(INFO) << "GDAL Initialized: " << GDALVersionInfo("--version");
119  initialized_ = true;
120  }
121 }
122 
124 #if (GDAL_VERSION_MAJOR > 2) || (GDAL_VERSION_MAJOR == 2 && GDAL_VERSION_MINOR >= 2)
125  return true;
126 #else
127  return false;
128 #endif
129 }
130 
131 bool GDAL::supportsDriver(const std::string& driver_name) {
132  // lazy init
133  init();
134 
135  return GetGDALDriverManager()->GetDriverByName(driver_name.c_str()) != nullptr;
136 }
137 
138 void GDAL::setAuthorizationTokens(const std::string& s3_region,
139  const std::string& s3_endpoint,
140  const std::string& s3_access_key,
141  const std::string& s3_secret_key,
142  const std::string& s3_session_token) {
143  // lazy init
144  init();
145 
146  // set tokens
147  if (s3_region.size()) {
148  CPLSetConfigOption("AWS_REGION", s3_region.c_str());
149  } else {
150  CPLSetConfigOption("AWS_REGION", nullptr);
151  }
152  if (s3_endpoint.size()) {
153  CPLSetConfigOption("AWS_S3_ENDPOINT", s3_endpoint.c_str());
154  } else {
155  CPLSetConfigOption("AWS_S3_ENDPOINT", nullptr);
156  }
157  if (s3_access_key.size()) {
158  CPLSetConfigOption("AWS_ACCESS_KEY_ID", s3_access_key.c_str());
159  } else {
160  CPLSetConfigOption("AWS_ACCESS_KEY_ID", nullptr);
161  }
162  if (s3_secret_key.size()) {
163  CPLSetConfigOption("AWS_SECRET_ACCESS_KEY", s3_secret_key.c_str());
164  } else {
165  CPLSetConfigOption("AWS_SECRET_ACCESS_KEY", nullptr);
166  }
167  if (s3_session_token.size()) {
168  CPLSetConfigOption("AWS_SESSION_TOKEN", s3_session_token.c_str());
169  } else {
170  CPLSetConfigOption("AWS_SESSION_TOKEN", nullptr);
171  }
172 
173  // if we haven't set keys, we need to disable signed access
174  if (s3_access_key.size() || s3_secret_key.size()) {
175  CPLSetConfigOption("AWS_NO_SIGN_REQUEST", nullptr);
176  } else {
177  CPLSetConfigOption("AWS_NO_SIGN_REQUEST", "YES");
178  }
179 }
180 
182  const import_export::SourceType source_type) {
183  // lazy init
184  init();
185 
186  // how should we try to open it?
187  unsigned int open_flags{0u};
188  switch (source_type) {
190  open_flags = GDAL_OF_VECTOR | GDAL_OF_RASTER;
191  break;
193  open_flags = GDAL_OF_VECTOR;
194  break;
196  open_flags = GDAL_OF_RASTER;
197  break;
198  default:
199  CHECK(false) << "Invalid datasource source type";
200  }
201 
202  // attempt to open datasource
203  // error will simply log and return null to be trapped later
204  auto* datasource = static_cast<OGRDataSource*>(
205  GDALOpenEx(name.c_str(), open_flags | GDAL_OF_READONLY, nullptr, nullptr, nullptr));
206 
207  // done
208  return DataSourceUqPtr(datasource);
209 }
210 
212  // lazy init
213  init();
214 
215  // attempt to open datasource as either vector or raster
217 
219 
220  // couldn't open it at all
221  if (!datasource) {
222  return source_type;
223  }
224 
225  // get the driver
226  auto* driver = datasource->GetDriver();
227  if (!driver) {
228  return source_type;
229  }
230 
231  // get capabilities
232  auto const is_vector = getMetadataString(driver->GetMetadata(), "DCAP_VECTOR") == "YES";
233  auto const is_raster = getMetadataString(driver->GetMetadata(), "DCAP_RASTER") == "YES";
234 
235  // analyze
236  if (is_vector && !is_raster) {
238  } else if (is_raster && !is_vector) {
240  }
241 
242  // done
243  return source_type;
244 }
245 
246 std::vector<std::string> GDAL::unpackMetadata(char** metadata) {
247  std::vector<std::string> strings;
248  if (metadata) {
249  while (*metadata) {
250  strings.emplace_back(*metadata);
251  metadata++;
252  }
253  }
254  return strings;
255 }
256 
257 void GDAL::logMetadata(GDALMajorObject* object) {
258  CHECK(initialized_) << "GDAL not initialized!";
259  CHECK(object);
260  LOG(INFO) << "DEBUG: Metadata domains for object '" << object->GetDescription() << "'";
261  LOG(INFO) << "DEBUG: (default)";
262  auto const default_metadata = unpackMetadata(object->GetMetadata());
263  for (auto const& str : default_metadata) {
264  LOG(INFO) << "DEBUG: " << str;
265  }
266  auto const metadata_domains = unpackMetadata(object->GetMetadataDomainList());
267  for (auto const& domain : metadata_domains) {
268  LOG(INFO) << "DEBUG: " << domain;
269  auto const metadata = unpackMetadata(object->GetMetadata(domain.c_str()));
270  for (auto const& str : metadata) {
271  LOG(INFO) << "DEBUG: " << str;
272  }
273  }
274 }
275 
276 std::string GDAL::getMetadataString(char** metadata, const std::string& key) {
277  CHECK(initialized_) << "GDAL not initialized!";
278  auto const key_len = key.length();
279  auto const strings = unpackMetadata(metadata);
280  for (auto const& str : strings) {
281  if (str.substr(0, key_len) == key) {
282  return str.substr(key_len + 1, std::string::npos);
283  }
284  }
285  return std::string();
286 }
287 
288 void GDAL::DataSourceDeleter::operator()(OGRDataSource* datasource) {
289  if (datasource) {
290  GDALClose(datasource);
291  }
292 }
293 
294 void GDAL::FeatureDeleter::operator()(OGRFeature* feature) {
295  if (feature) {
296  OGRFeature::DestroyFeature(feature);
297  }
298 }
299 
300 void GDAL::SpatialReferenceDeleter::operator()(OGRSpatialReference* reference) {
301  if (reference) {
302  OGRSpatialReference::DestroySpatialReference(reference);
303  }
304 }
305 
307  OGRCoordinateTransformation* transformation) {
308  if (transformation) {
309  delete transformation;
310  }
311 }
312 
313 } // namespace Geospatial
void gdal_error_handler(CPLErr err_class, int err_no, const char *err_msg)
Definition: GDAL.cpp:39
std::string get_root_abs_path()
static std::vector< std::string > unpackMetadata(char **metadata)
Definition: GDAL.cpp:246
static void logMetadata(GDALMajorObject *object)
Definition: GDAL.cpp:257
#define LOG(tag)
Definition: Logger.h:216
static void init()
Definition: GDAL.cpp:67
void operator()(OGRDataSource *datasource)
Definition: GDAL.cpp:288
std::string to_string(char const *&&v)
void operator()(OGRFeature *feature)
Definition: GDAL.cpp:294
void operator()(OGRSpatialReference *ref)
Definition: GDAL.cpp:300
static bool initialized_
Definition: GDAL.h:76
static void setAuthorizationTokens(const std::string &s3_region, const std::string &s3_endpoint, const std::string &s3_access_key, const std::string &s3_secret_key, const std::string &s3_session_token)
Definition: GDAL.cpp:138
static bool supportsNetworkFileAccess()
Definition: GDAL.cpp:123
static std::mutex init_mutex_
Definition: GDAL.h:77
std::unique_ptr< OGRDataSource, DataSourceDeleter > DataSourceUqPtr
Definition: GDAL.h:48
static import_export::SourceType getDataSourceType(const std::string &name)
Definition: GDAL.cpp:211
static bool supportsDriver(const std::string &driver_name)
Definition: GDAL.cpp:131
#define CHECK(condition)
Definition: Logger.h:222
void operator()(OGRCoordinateTransformation *transformation)
Definition: GDAL.cpp:306
string name
Definition: setup.in.py:72
static DataSourceUqPtr openDataSource(const std::string &name, const import_export::SourceType source_type)
Definition: GDAL.cpp:181
static std::string getMetadataString(char **metadata, const std::string &key)
Definition: GDAL.cpp:276