OmniSciDB  16c4e035a1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
CopyParams.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file CopyParams.h
19  * @author Mehmet Sariyuce <mehmet.sariyuce@omnisci.com>
20  * @brief CopyParams struct
21  */
22 
23 #pragma once
24 
25 #include <string>
26 
28 #include "Shared/sqltypes.h"
29 
30 namespace import_export {
31 
32 // not too big (need much memory) but not too small (many thread forks)
33 constexpr static size_t kImportFileBufferSize = (1 << 23);
34 
35 // import buffers may grow to this size if necessary
36 constexpr static size_t max_import_buffer_resize_byte_size = 1024 * 1024 * 1024;
37 
41 
42 struct CopyParams {
43  char delimiter;
44  std::string null_str;
46  bool quoted; // does the input have any quoted fields, default to false
47  char quote;
48  char escape;
49  char line_delim;
52  char array_end;
53  int threads;
54  size_t
55  max_reject; // maximum number of records that can be rejected before copy is failed
57  bool plain_text = false;
58  // s3/parquet related params
59  std::string s3_access_key; // per-query credentials to override the
60  std::string s3_secret_key; // settings in ~/.aws/credentials or environment
61  std::string s3_session_token = "";
62  std::string s3_region;
63  std::string s3_endpoint;
64  // kafka related params
65  size_t retry_count;
66  size_t retry_wait;
67  size_t batch_size;
68  size_t buffer_size;
69  // geospatial params
70  bool lonlat;
74  int32_t geo_coords_srid;
76  std::string geo_layer_name;
79  int32_t source_srid;
80  std::optional<std::string> regex_path_filter;
81  std::optional<std::string> file_sort_order_by;
82  std::optional<std::string> file_sort_regex;
84  std::string raster_import_bands;
89 
91  : delimiter(',')
92  , null_str("\\N")
94  , quoted(true)
95  , quote('"')
96  , escape('"')
97  , line_delim('\n')
98  , array_delim(',')
99  , array_begin('{')
100  , array_end('}')
101  , threads(0)
102  , max_reject(100000)
103  , source_type(import_export::SourceType::kDelimitedFile)
104  , retry_count(100)
105  , retry_wait(5)
106  , batch_size(1000)
108  , lonlat(true)
112  , geo_coords_srid(4326)
116  , source_srid(0)
120  , raster_point_compute_angle{false} {}
121 
122  CopyParams(char d, const std::string& n, char l, size_t b, size_t retries, size_t wait)
123  : delimiter(d)
124  , null_str(n)
126  , quoted(true)
127  , quote('"')
128  , escape('"')
129  , line_delim(l)
130  , array_delim(',')
131  , array_begin('{')
132  , array_end('}')
133  , threads(0)
134  , max_reject(100000)
135  , source_type(import_export::SourceType::kDelimitedFile)
136  , retry_count(retries)
137  , retry_wait(wait)
138  , batch_size(b)
140  , lonlat(true)
144  , geo_coords_srid(4326)
148  , source_srid(0)
152  , raster_point_compute_angle{false} {}
153 };
154 
155 } // namespace import_export
std::string s3_secret_key
Definition: CopyParams.h:60
int32_t raster_scanlines_per_thread
Definition: CopyParams.h:85
SQLTypes
Definition: sqltypes.h:38
Constants for Builtin SQL Types supported by OmniSci.
std::string raster_import_dimensions
Definition: CopyParams.h:88
ImportHeaderRow has_header
Definition: CopyParams.h:45
EncodingType
Definition: sqltypes.h:233
std::optional< std::string > regex_path_filter
Definition: CopyParams.h:80
RasterPointType raster_point_type
Definition: CopyParams.h:83
import_export::SourceType source_type
Definition: CopyParams.h:56
bool g_enable_smem_group_by true
std::string geo_layer_name
Definition: CopyParams.h:76
std::string s3_session_token
Definition: CopyParams.h:61
CopyParams(char d, const std::string &n, char l, size_t b, size_t retries, size_t wait)
Definition: CopyParams.h:122
std::string raster_import_bands
Definition: CopyParams.h:84
bool g_enable_watchdog false
Definition: Execute.cpp:76
static constexpr size_t max_import_buffer_resize_byte_size
Definition: CopyParams.h:36
static constexpr size_t kImportFileBufferSize
Definition: CopyParams.h:33
constexpr double n
Definition: Utm.h:38
std::string s3_access_key
Definition: CopyParams.h:59
RasterPointTransform raster_point_transform
Definition: CopyParams.h:86
std::optional< std::string > file_sort_order_by
Definition: CopyParams.h:81
Shared Enum.
std::optional< std::string > file_sort_regex
Definition: CopyParams.h:82
EncodingType geo_coords_encoding
Definition: CopyParams.h:71