OmniSciDB  2e3a973ef4
LazyParquetImporter.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <limits>
20 #include <list>
21 
22 #include <arrow/filesystem/filesystem.h>
23 
24 #include "ArrayMetadataStats.h"
25 #include "ForeignTableSchema.h"
26 #include "ImportExport/Importer.h"
27 #include "Interval.h"
28 #include "ParquetShared.h"
29 
30 namespace foreign_storage {
31 
38  bool has_nulls;
39  size_t num_elements;
42 };
43 
45  std::string file_path;
46  std::vector<RowGroupMetadata> row_group_metadata_vector;
47 };
48 
53  public:
55  const std::set<std::string>& file_paths,
56  std::shared_ptr<arrow::fs::FileSystem> file_system,
57  const import_export::CopyParams& copy_params,
58  ParquetLoaderMetadata& parquet_loader_metadata,
59  ForeignTableSchema& schema);
60 
71  void partialImport(const std::vector<RowGroupInterval>& row_group_intervals,
72  const Interval<ColumnType>& column_interval,
73  const bool is_metadata_scan = false);
74 
78  void metadataScan();
79 
80  private:
83  const std::set<std::string>& file_paths_;
84  std::shared_ptr<arrow::fs::FileSystem> file_system_;
85 };
86 
87 } // namespace foreign_storage
ParquetLoaderMetadata & parquet_loader_metadata_
std::vector< RowGroupMetadata > row_group_metadata_vector
std::shared_ptr< arrow::fs::FileSystem > file_system_
const ForeignTableSchema & schema_
const std::set< std::string > & file_paths_