OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
PointCloudTableFunctions.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  */
4 
5 #ifdef HAVE_POINT_CLOUD_TFS
6 #ifndef __CUDACC__
7 
8 #include <tbb/blocked_range.h>
9 #include <tbb/parallel_for.h>
10 
11 #include "QueryEngine/TableFunctions/SystemFunctions/os/Shared/Loaders/PdalLoader.h"
12 #include "QueryEngine/TableFunctions/SystemFunctions/os/Shared/TableFunctionsCommon.hpp"
13 #include "QueryEngine/TableFunctions/SystemFunctions/os/Shared/TableFunctionsDataCache.h"
14 #include "Utils/DdlUtils.h"
15 
16 #include "PointCloudTableFunctions.h"
17 
18 EXTENSION_NOINLINE_HOST
19 __attribute__((__used__)) int32_t tf_point_cloud_metadata__cpu_(
20  TableFunctionManager& mgr,
21  const TextEncodingNone& path,
22  const double x_min,
23  const double x_max,
24  const double y_min,
25  const double y_max,
26  Column<TextEncodingDict>& file_path,
27  Column<TextEncodingDict>& file_name,
28  Column<int32_t>& file_source_id,
29  Column<int16_t>& version_major,
30  Column<int16_t>& version_minor,
31  Column<int16_t>& creation_year,
32  Column<bool>& is_compressed,
33  Column<int64_t>& num_points,
34  Column<int16_t>& num_dims,
35  Column<int16_t>& point_len,
36  Column<bool>& has_time,
37  Column<bool>& has_color,
38  Column<bool>& has_wave,
39  Column<bool>& has_infrared,
40  Column<bool>& has_14_point_format,
41  Column<int32_t>& specified_utm_zone,
42  Column<double>& source_x_min,
43  Column<double>& source_x_max,
44  Column<double>& source_y_min,
45  Column<double>& source_y_max,
46  Column<double>& source_z_min,
47  Column<double>& source_z_max,
48  Column<double>& transformed_x_min,
49  Column<double>& transformed_x_max,
50  Column<double>& transformed_y_min,
51  Column<double>& transformed_y_max,
52  Column<double>& transformed_z_min,
53  Column<double>& transformed_z_max) {
54  auto timer = DEBUG_TIMER(__func__);
55  // try {
56  // ddl_utils::validate_allowed_file_path(
57  // path, ddl_utils::DataTransferType::IMPORT, true);
58  //} catch (const std::runtime_error& e) {
59  // return mgr.ERROR_MESSAGE("File path is not whitelisted");
60  //}
61 
62  if (x_min >= x_max) {
63  return mgr.ERROR_MESSAGE("x_min must be less than x_max");
64  }
65 
66  if (y_min >= y_max) {
67  return mgr.ERROR_MESSAGE("y_min must be less than y_max");
68  }
69 
70  const std::string las_path(path.getString());
71  const std::vector<std::filesystem::path> file_paths =
72  FileUtilities::get_fs_paths(las_path);
73  for (const auto& file_path : file_paths) {
74  try {
75  ddl_utils::validate_allowed_file_path(
76  file_path, ddl_utils::DataTransferType::IMPORT, true);
77  } catch (const std::runtime_error& e) {
78  const auto error_msg{"File path " + std::string(file_path) + " is not whitelisted"};
79  return mgr.ERROR_MESSAGE(error_msg);
80  }
81  }
82  const std::string out_srs("EPSG:4326");
83  const auto& lidar_file_info = PdalLoader::get_metadata_for_files(file_paths, out_srs);
84  const auto& filtered_lidar_file_info =
85  filter_files_by_bounds(lidar_file_info.second, x_min, x_max, y_min, y_max);
86  const size_t num_filtered_files(filtered_lidar_file_info.second.size());
87  mgr.set_output_row_size(num_filtered_files);
88 
89  for (size_t file_idx = 0; file_idx < num_filtered_files; ++file_idx) {
90  const std::shared_ptr<PdalLoader::LidarMetadata>& lidar_metadata =
91  filtered_lidar_file_info.second[file_idx];
92  file_path[file_idx] =
93  file_path.string_dict_proxy_->getOrAddTransient(lidar_metadata->filename);
94  auto last_slash_pos =
95  lidar_metadata->filename.rfind("/", lidar_metadata->filename.size() - 1UL);
96  if (last_slash_pos == std::string::npos) {
97  last_slash_pos = 0;
98  } else {
99  last_slash_pos++;
100  }
101  file_name[file_idx] =
102  file_name.string_dict_proxy_->getOrAddTransient(lidar_metadata->filename.substr(
103  last_slash_pos, lidar_metadata->filename.size() - last_slash_pos));
104  file_source_id[file_idx] = lidar_metadata->file_source_id;
105  version_major[file_idx] = lidar_metadata->version_major;
106  version_minor[file_idx] = lidar_metadata->version_minor;
107  creation_year[file_idx] = lidar_metadata->creation_year;
108  is_compressed[file_idx] = lidar_metadata->is_compressed;
109  num_points[file_idx] = lidar_metadata->num_points;
110  num_dims[file_idx] = lidar_metadata->num_dims;
111  point_len[file_idx] = lidar_metadata->point_len;
112  has_time[file_idx] = lidar_metadata->has_time;
113  has_color[file_idx] = lidar_metadata->has_color;
114  has_wave[file_idx] = lidar_metadata->has_wave;
115  has_infrared[file_idx] = lidar_metadata->has_infrared;
116  has_14_point_format[file_idx] = lidar_metadata->has_14_point_format;
117  specified_utm_zone[file_idx] = lidar_metadata->specified_utm_zone;
118  source_x_min[file_idx] = lidar_metadata->source_x_bounds.first;
119  source_x_max[file_idx] = lidar_metadata->source_x_bounds.second;
120  source_y_min[file_idx] = lidar_metadata->source_y_bounds.first;
121  source_y_max[file_idx] = lidar_metadata->source_y_bounds.second;
122  source_z_min[file_idx] = lidar_metadata->source_z_bounds.first;
123  source_z_max[file_idx] = lidar_metadata->source_z_bounds.second;
124  transformed_x_min[file_idx] = lidar_metadata->transformed_x_bounds.first;
125  transformed_x_max[file_idx] = lidar_metadata->transformed_x_bounds.second;
126  transformed_y_min[file_idx] = lidar_metadata->transformed_y_bounds.first;
127  transformed_y_max[file_idx] = lidar_metadata->transformed_y_bounds.second;
128  transformed_z_min[file_idx] = lidar_metadata->transformed_z_bounds.first;
129  transformed_z_max[file_idx] = lidar_metadata->transformed_z_bounds.second;
130  }
131  return num_filtered_files;
132 }
133 
134 EXTENSION_NOINLINE_HOST
135 __attribute__((__used__)) int32_t tf_point_cloud_metadata__cpu_2(
136  TableFunctionManager& mgr,
137  const TextEncodingNone& path,
138  Column<TextEncodingDict>& file_path,
139  Column<TextEncodingDict>& file_name,
140  Column<int32_t>& file_source_id,
141  Column<int16_t>& version_major,
142  Column<int16_t>& version_minor,
143  Column<int16_t>& creation_year,
144  Column<bool>& is_compressed,
145  Column<int64_t>& num_points,
146  Column<int16_t>& num_dims,
147  Column<int16_t>& point_len,
148  Column<bool>& has_time,
149  Column<bool>& has_color,
150  Column<bool>& has_wave,
151  Column<bool>& has_infrared,
152  Column<bool>& has_14_point_format,
153  Column<int32_t>& specified_utm_zone,
154  Column<double>& source_x_min,
155  Column<double>& source_x_max,
156  Column<double>& source_y_min,
157  Column<double>& source_y_max,
158  Column<double>& source_z_min,
159  Column<double>& source_z_max,
160  Column<double>& transformed_x_min,
161  Column<double>& transformed_x_max,
162  Column<double>& transformed_y_min,
163  Column<double>& transformed_y_max,
164  Column<double>& transformed_z_min,
165  Column<double>& transformed_z_max) {
166  const double x_min = std::numeric_limits<double>::lowest();
167  const double x_max = std::numeric_limits<double>::max();
168  const double y_min = std::numeric_limits<double>::lowest();
169  const double y_max = std::numeric_limits<double>::max();
170 
171  return tf_point_cloud_metadata__cpu_(mgr,
172  path,
173  x_min,
174  x_max,
175  y_min,
176  y_max,
177  file_path,
178  file_name,
179  file_source_id,
180  version_major,
181  version_minor,
182  creation_year,
183  is_compressed,
184  num_points,
185  num_dims,
186  point_len,
187  has_time,
188  has_color,
189  has_wave,
190  has_infrared,
191  has_14_point_format,
192  specified_utm_zone,
193  source_x_min,
194  source_x_max,
195  source_y_min,
196  source_y_max,
197  source_z_min,
198  source_z_max,
199  transformed_x_min,
200  transformed_x_max,
201  transformed_y_min,
202  transformed_y_max,
203  transformed_z_min,
204  transformed_z_max);
205 }
206 
207 EXTENSION_NOINLINE_HOST
208 __attribute__((__used__)) int32_t tf_load_point_cloud__cpu_(
209  TableFunctionManager& mgr,
210  const TextEncodingNone& path,
211  const TextEncodingNone& out_srs,
212  const bool use_cache,
213  const double x_min,
214  const double x_max,
215  const double y_min,
216  const double y_max,
217  Column<double>& x,
218  Column<double>& y,
219  Column<double>& z,
220  Column<int32_t>& intensity,
221  Column<int8_t>& return_num,
222  Column<int8_t>& num_returns,
223  Column<int8_t>& scan_direction_flag,
224  Column<int8_t>& edge_of_flight_line_flag,
225  Column<int16_t>& classification,
226  Column<int8_t>& scan_angle_rank) {
227  auto timer = DEBUG_TIMER(__func__);
228 
229  if (x_min >= x_max) {
230  return mgr.ERROR_MESSAGE("x_min must be less than x_max");
231  }
232 
233  if (y_min >= y_max) {
234  return mgr.ERROR_MESSAGE("y_min must be less than y_max");
235  }
236 
237  const std::string las_path(path.getString());
238  const std::string las_out_srs(out_srs.getString());
239 
240  const std::vector<std::filesystem::path> file_paths =
241  FileUtilities::get_fs_paths(las_path);
242  for (const auto& file_path : file_paths) {
243  try {
244  ddl_utils::validate_allowed_file_path(
245  file_path, ddl_utils::DataTransferType::IMPORT, true);
246  } catch (const std::runtime_error& e) {
247  const auto error_msg{"File path " + std::string(file_path) + " is not whitelisted"};
248  return mgr.ERROR_MESSAGE(error_msg);
249  }
250  }
251  const auto& lidar_file_info =
252  PdalLoader::get_metadata_for_files(file_paths, las_out_srs);
253  const auto& filtered_lidar_file_info =
254  filter_files_by_bounds(lidar_file_info.second, x_min, x_max, y_min, y_max);
255  const size_t num_filtered_files(filtered_lidar_file_info.second.size());
256  mgr.set_output_row_size(filtered_lidar_file_info.first); // contains number of rows
257 
258  std::atomic<int64_t> row_offset = 0;
259  tbb::parallel_for(
260  tbb::blocked_range<size_t>(0, num_filtered_files),
261  [&](const tbb::blocked_range<size_t>& r) {
262  const size_t start_file_idx = r.begin();
263  const size_t end_file_idx = r.end();
264  for (size_t file_idx = start_file_idx; file_idx != end_file_idx; ++file_idx) {
265  const std::shared_ptr<PdalLoader::LidarMetadata>& lidar_metadata =
266  filtered_lidar_file_info.second[file_idx];
267  const int64_t num_points = lidar_metadata->num_points;
268  const auto& las_fs_path = std::filesystem::path(lidar_metadata->filename);
269  const auto file_status = std::filesystem::status(las_fs_path);
270  if (!std::filesystem::is_regular_file(file_status)) {
271  continue;
272  }
273 
274  const std::vector<std::pair<std::string, size_t>> sub_keys_and_byte_sizes = {
275  std::make_pair("x", 8),
276  std::make_pair("y", 8),
277  std::make_pair("z", 8),
278  std::make_pair("intensity", 4),
279  std::make_pair("return_num", 1),
280  std::make_pair("num_returns", 1),
281  std::make_pair("scan_direction_flag", 1),
282  std::make_pair("edge_of_flight_line_flag", 1),
283  std::make_pair("classification", 2),
284  std::make_pair("scan_angle_rank", 1)};
285 
286  if (use_cache && PdalLoader::keys_are_cached(lidar_metadata->filename,
287  las_out_srs,
288  num_points,
289  sub_keys_and_byte_sizes)) {
290  const int64_t offset =
291  row_offset.fetch_add(num_points, std::memory_order_relaxed);
292  PdalLoader::data_cache.getDataForKey(
293  PdalLoader::make_cache_key(lidar_metadata->filename, las_out_srs, "x"),
294  x.ptr_ + offset);
295  PdalLoader::data_cache.getDataForKey(
296  PdalLoader::make_cache_key(lidar_metadata->filename, las_out_srs, "y"),
297  y.ptr_ + offset);
298  PdalLoader::data_cache.getDataForKey(
299  PdalLoader::make_cache_key(lidar_metadata->filename, las_out_srs, "z"),
300  z.ptr_ + offset);
301  PdalLoader::data_cache.getDataForKey(
302  PdalLoader::make_cache_key(
303  lidar_metadata->filename, las_out_srs, "intensity"),
304  intensity.ptr_ + offset);
305  PdalLoader::data_cache.getDataForKey(
306  PdalLoader::make_cache_key(
307  lidar_metadata->filename, las_out_srs, "return_num"),
308  return_num.ptr_ + offset);
309  PdalLoader::data_cache.getDataForKey(
310  PdalLoader::make_cache_key(
311  lidar_metadata->filename, las_out_srs, "num_returns"),
312  num_returns.ptr_ + offset);
313  PdalLoader::data_cache.getDataForKey(
314  PdalLoader::make_cache_key(
315  lidar_metadata->filename, las_out_srs, "scan_direction_flag"),
316  scan_direction_flag.ptr_ + offset);
317  PdalLoader::data_cache.getDataForKey(
318  PdalLoader::make_cache_key(
319  lidar_metadata->filename, las_out_srs, "edge_of_flight_line_flag"),
320  edge_of_flight_line_flag.ptr_ + offset);
321  PdalLoader::data_cache.getDataForKey(
322  PdalLoader::make_cache_key(
323  lidar_metadata->filename, las_out_srs, "classification"),
324  classification.ptr_ + offset);
325  PdalLoader::data_cache.getDataForKey(
326  PdalLoader::make_cache_key(
327  lidar_metadata->filename, las_out_srs, "scan_angle_rank"),
328  scan_angle_rank.ptr_ + offset);
329  continue;
330  }
331 
332  PdalLoader::PdalFileMgr pdal_file_mgr(lidar_metadata, las_out_srs);
333  pdal_file_mgr.openAndPrepareFile();
334  if (!pdal_file_mgr.isReady()) {
335  continue;
336  // const std::string error_msg = "tf_load_point_cloud: PDAL error opening
337  // file " + lidar_metadata->filename; return
338  // mgr.ERROR_MESSAGE(error_msg.c_str());
339  }
340  const int64_t offset =
341  row_offset.fetch_add(lidar_metadata->num_points, std::memory_order_relaxed);
342  pdal_file_mgr.readData(x.ptr_ + offset,
343  y.ptr_ + offset,
344  z.ptr_ + offset,
345  intensity.ptr_ + offset,
346  return_num.ptr_ + offset,
347  num_returns.ptr_ + offset,
348  scan_direction_flag.ptr_ + offset,
349  edge_of_flight_line_flag.ptr_ + offset,
350  classification.ptr_ + offset,
351  scan_angle_rank.ptr_ + offset);
352 
353  if (use_cache) {
354  try {
355  PdalLoader::data_cache.putDataForKey(
356  PdalLoader::make_cache_key(lidar_metadata->filename, las_out_srs, "x"),
357  x.ptr_ + offset,
358  lidar_metadata->num_points);
359  PdalLoader::data_cache.putDataForKey(
360  PdalLoader::make_cache_key(lidar_metadata->filename, las_out_srs, "y"),
361  y.ptr_ + offset,
362  lidar_metadata->num_points);
363  PdalLoader::data_cache.putDataForKey(
364  PdalLoader::make_cache_key(lidar_metadata->filename, las_out_srs, "z"),
365  z.ptr_ + offset,
366  lidar_metadata->num_points);
367  PdalLoader::data_cache.putDataForKey(
368  PdalLoader::make_cache_key(
369  lidar_metadata->filename, las_out_srs, "intensity"),
370  intensity.ptr_ + offset,
371  lidar_metadata->num_points);
372  PdalLoader::data_cache.putDataForKey(
373  PdalLoader::make_cache_key(
374  lidar_metadata->filename, las_out_srs, "return_num"),
375  return_num.ptr_ + offset,
376  lidar_metadata->num_points);
377  PdalLoader::data_cache.putDataForKey(
378  PdalLoader::make_cache_key(
379  lidar_metadata->filename, las_out_srs, "num_returns"),
380  num_returns.ptr_ + offset,
381  lidar_metadata->num_points);
382  PdalLoader::data_cache.putDataForKey(
383  PdalLoader::make_cache_key(
384  lidar_metadata->filename, las_out_srs, "scan_direction_flag"),
385  scan_direction_flag.ptr_ + offset,
386  lidar_metadata->num_points);
387  PdalLoader::data_cache.putDataForKey(
388  PdalLoader::make_cache_key(
389  lidar_metadata->filename, las_out_srs, "edge_of_flight_line_flag"),
390  edge_of_flight_line_flag.ptr_ + offset,
391  lidar_metadata->num_points);
392  PdalLoader::data_cache.putDataForKey(
393  PdalLoader::make_cache_key(
394  lidar_metadata->filename, las_out_srs, "classification"),
395  classification.ptr_ + offset,
396  lidar_metadata->num_points);
397  PdalLoader::data_cache.putDataForKey(
398  PdalLoader::make_cache_key(
399  lidar_metadata->filename, las_out_srs, "scan_angle_rank"),
400  scan_angle_rank.ptr_ + offset,
401  lidar_metadata->num_points);
402  } catch (std::runtime_error& e) {
403  continue;
404  // return mgr.ERROR_MESSAGE("tf_tf_load_point_cloud: Error writing data to
405  // cache");
406  }
407  }
408  }
409  });
410  return row_offset;
411 }
412 
413 EXTENSION_NOINLINE_HOST
414 __attribute__((__used__)) int32_t tf_load_point_cloud__cpu_2(
415  TableFunctionManager& mgr,
416  const TextEncodingNone& filename,
417  Column<double>& x,
418  Column<double>& y,
419  Column<double>& z,
420  Column<int32_t>& intensity,
421  Column<int8_t>& return_num,
422  Column<int8_t>& num_returns,
423  Column<int8_t>& scan_direction_flag,
424  Column<int8_t>& edge_of_flight_line_flag,
425  Column<int16_t>& classification,
426  Column<int8_t>& scan_angle_rank) {
427  std::string pdal_out_srs("EPSG:4326");
428  TextEncodingNone out_srs;
429  out_srs.ptr_ = pdal_out_srs.data();
430  out_srs.size_ = pdal_out_srs.size();
431  const double x_min = std::numeric_limits<double>::lowest();
432  const double x_max = std::numeric_limits<double>::max();
433  const double y_min = std::numeric_limits<double>::lowest();
434  const double y_max = std::numeric_limits<double>::max();
435  return tf_load_point_cloud__cpu_(mgr,
436  filename,
437  out_srs,
438  true,
439  x_min,
440  x_max,
441  y_min,
442  y_max,
443  x,
444  y,
445  z,
446  intensity,
447  return_num,
448  num_returns,
449  scan_direction_flag,
450  edge_of_flight_line_flag,
451  classification,
452  scan_angle_rank);
453 }
454 
455 EXTENSION_NOINLINE_HOST
456 __attribute__((__used__)) int32_t tf_load_point_cloud__cpu_3(
457  TableFunctionManager& mgr,
458  const TextEncodingNone& filename,
459  const double x_min,
460  const double x_max,
461  const double y_min,
462  const double y_max,
463  Column<double>& x,
464  Column<double>& y,
465  Column<double>& z,
466  Column<int32_t>& intensity,
467  Column<int8_t>& return_num,
468  Column<int8_t>& num_returns,
469  Column<int8_t>& scan_direction_flag,
470  Column<int8_t>& edge_of_flight_line_flag,
471  Column<int16_t>& classification,
472  Column<int8_t>& scan_angle_rank) {
473  std::string pdal_out_srs("EPSG:4326");
474  TextEncodingNone out_srs;
475  out_srs.ptr_ = pdal_out_srs.data();
476  out_srs.size_ = pdal_out_srs.size();
477  return tf_load_point_cloud__cpu_(mgr,
478  filename,
479  out_srs,
480  true,
481  x_min,
482  x_max,
483  y_min,
484  y_max,
485  x,
486  y,
487  z,
488  intensity,
489  return_num,
490  num_returns,
491  scan_direction_flag,
492  edge_of_flight_line_flag,
493  classification,
494  scan_angle_rank);
495 }
496 
497 EXTENSION_NOINLINE_HOST
498 __attribute__((__used__)) int32_t tf_load_point_cloud__cpu_4(
499  TableFunctionManager& mgr,
500  const TextEncodingNone& filename,
501  const bool use_cache,
502  Column<double>& x,
503  Column<double>& y,
504  Column<double>& z,
505  Column<int32_t>& intensity,
506  Column<int8_t>& return_num,
507  Column<int8_t>& num_returns,
508  Column<int8_t>& scan_direction_flag,
509  Column<int8_t>& edge_of_flight_line_flag,
510  Column<int16_t>& classification,
511  Column<int8_t>& scan_angle_rank) {
512  std::string pdal_out_srs("EPSG:4326");
513  TextEncodingNone out_srs;
514  out_srs.ptr_ = pdal_out_srs.data();
515  out_srs.size_ = pdal_out_srs.size();
516  const double x_min = std::numeric_limits<double>::lowest();
517  const double x_max = std::numeric_limits<double>::max();
518  const double y_min = std::numeric_limits<double>::lowest();
519  const double y_max = std::numeric_limits<double>::max();
520  return tf_load_point_cloud__cpu_(mgr,
521  filename,
522  out_srs,
523  use_cache,
524  x_min,
525  x_max,
526  y_min,
527  y_max,
528  x,
529  y,
530  z,
531  intensity,
532  return_num,
533  num_returns,
534  scan_direction_flag,
535  edge_of_flight_line_flag,
536  classification,
537  scan_angle_rank);
538 }
539 
540 #endif // __CUDACC__
541 #endif // HAVE_TF_POINT_CLOUD_TFS
DdlUtils.h
FileUtilities::get_fs_paths
std::vector< std::filesystem::path > get_fs_paths(const std::string &file_or_directory)
Definition: TableFunctionsCommon.cpp:515
TableFunctionsCommon.hpp
TableFunctionsDataCache.h
TextEncodingDict
Definition: heavydbTypes.h:215
PointCloudTableFunctions.h
EXTENSION_NOINLINE_HOST
#define EXTENSION_NOINLINE_HOST
Definition: heavydbTypes.h:55
PdalLoader.h
ddl_utils::validate_allowed_file_path
void validate_allowed_file_path(const std::string &file_path, const DataTransferType data_transfer_type, const bool allow_wildcards)
Definition: DdlUtils.cpp:822
__attribute__
__attribute__((__used__)) ModelInfo get_model_info_from_file(const std
Definition: TorchWrapper.cpp:469
TextEncodingNone::size_
int64_t size_
Definition: heavydbTypes.h:609
TextEncodingNone
Definition: heavydbTypes.h:607
threading_serial::parallel_for
void parallel_for(const blocked_range< Int > &range, const Body &body, const Partitioner &p=Partitioner())
Definition: threading_serial.h:34
nvtx_helpers::anonymous_namespace{nvtx_helpers.cpp}::filename
std::string filename(char const *path)
Definition: nvtx_helpers.cpp:98
DEBUG_TIMER
#define DEBUG_TIMER(name)
Definition: Logger.h:412
TextEncodingNone::ptr_
char * ptr_
Definition: heavydbTypes.h:608
ddl_utils::DataTransferType::IMPORT