OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
CopyParams.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file CopyParams.h
19  * @author Mehmet Sariyuce <mehmet.sariyuce@omnisci.com>
20  * @brief CopyParams struct
21  */
22 
23 #pragma once
24 
25 #include <string>
26 
27 #include "Shared/sqltypes.h"
28 
29 namespace import_export {
30 
31 // not too big (need much memory) but not too small (many thread forks)
32 constexpr static size_t kImportFileBufferSize = (1 << 23);
33 
34 // import buffers may grow to this size if necessary
35 constexpr static size_t max_import_buffer_resize_byte_size = 1024 * 1024 * 1024;
36 
37 enum class FileType {
38  DELIMITED,
39  POLYGON
40 #ifdef ENABLE_IMPORT_PARQUET
41  ,
42  PARQUET
43 #endif
44 };
45 
47 
48 struct CopyParams {
49  char delimiter;
50  std::string null_str;
52  bool quoted; // does the input have any quoted fields, default to false
53  char quote;
54  char escape;
55  char line_delim;
58  char array_end;
59  int threads;
60  size_t
61  max_reject; // maximum number of records that can be rejected before copy is failed
63  bool plain_text = false;
64  // s3/parquet related params
65  std::string s3_access_key; // per-query credentials to override the
66  std::string s3_secret_key; // settings in ~/.aws/credentials or environment
67  std::string s3_session_token = "";
68  std::string s3_region;
69  std::string s3_endpoint;
70  // kafka related params
71  size_t retry_count;
72  size_t retry_wait;
73  size_t batch_size;
74  size_t buffer_size;
75  // geospatial params
76  bool lonlat;
80  int32_t geo_coords_srid;
82  std::string geo_layer_name;
85  int32_t source_srid;
86  std::optional<std::string> regex_path_filter;
87  std::optional<std::string> file_sort_order_by;
88  std::optional<std::string> file_sort_regex;
89 
91  : delimiter(',')
92  , null_str("\\N")
94  , quoted(true)
95  , quote('"')
96  , escape('"')
97  , line_delim('\n')
98  , array_delim(',')
99  , array_begin('{')
100  , array_end('}')
101  , threads(0)
102  , max_reject(100000)
104  , retry_count(100)
105  , retry_wait(5)
106  , batch_size(1000)
108  , lonlat(true)
112  , geo_coords_srid(4326)
116  , source_srid(0) {}
117 
118  CopyParams(char d, const std::string& n, char l, size_t b, size_t retries, size_t wait)
119  : delimiter(d)
120  , null_str(n)
122  , quoted(true)
123  , quote('"')
124  , escape('"')
125  , line_delim(l)
126  , array_delim(',')
127  , array_begin('{')
128  , array_end('}')
129  , threads(0)
130  , max_reject(100000)
132  , retry_count(retries)
133  , retry_wait(wait)
134  , batch_size(b)
136  , lonlat(true)
140  , geo_coords_srid(4326)
144  , source_srid(0) {}
145 };
146 } // namespace import_export
std::string s3_secret_key
Definition: CopyParams.h:66
SQLTypes
Definition: sqltypes.h:38
Constants for Builtin SQL Types supported by OmniSci.
ImportHeaderRow has_header
Definition: CopyParams.h:51
EncodingType
Definition: sqltypes.h:233
std::optional< std::string > regex_path_filter
Definition: CopyParams.h:86
bool g_enable_smem_group_by true
std::string geo_layer_name
Definition: CopyParams.h:82
std::string s3_session_token
Definition: CopyParams.h:67
CopyParams(char d, const std::string &n, char l, size_t b, size_t retries, size_t wait)
Definition: CopyParams.h:118
bool g_enable_watchdog false
Definition: Execute.cpp:76
static constexpr size_t max_import_buffer_resize_byte_size
Definition: CopyParams.h:35
static constexpr size_t kImportFileBufferSize
Definition: CopyParams.h:32
constexpr double n
Definition: Utm.h:46
std::string s3_access_key
Definition: CopyParams.h:65
std::optional< std::string > file_sort_order_by
Definition: CopyParams.h:87
std::optional< std::string > file_sort_regex
Definition: CopyParams.h:88
EncodingType geo_coords_encoding
Definition: CopyParams.h:77