OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
CopyParams.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file CopyParams.h
19  * @author Mehmet Sariyuce <mehmet.sariyuce@omnisci.com>
20  * @brief CopyParams struct
21  */
22 
23 #pragma once
24 
25 #include <string>
26 
27 #include "Shared/sqltypes.h"
28 
29 namespace import_export {
30 
31 // not too big (need much memory) but not too small (many thread forks)
32 constexpr static size_t kImportFileBufferSize = (1 << 23);
33 
34 enum class FileType {
35  DELIMITED,
36  POLYGON
37 #ifdef ENABLE_IMPORT_PARQUET
38  ,
39  PARQUET
40 #endif
41 };
42 
44 
45 struct CopyParams {
46  char delimiter;
47  std::string null_str;
49  bool quoted; // does the input have any quoted fields, default to false
50  char quote;
51  char escape;
52  char line_delim;
55  char array_end;
56  int threads;
57  size_t
58  max_reject; // maximum number of records that can be rejected before copy is failed
60  bool plain_text = false;
61  // s3/parquet related params
62  std::string s3_access_key; // per-query credentials to override the
63  std::string s3_secret_key; // settings in ~/.aws/credentials or environment
64  std::string s3_session_token = "";
65  std::string s3_region;
66  std::string s3_endpoint;
67  // kafka related params
68  size_t retry_count;
69  size_t retry_wait;
70  size_t batch_size;
71  size_t buffer_size;
72  // geospatial params
73  bool lonlat;
77  int32_t geo_coords_srid;
79  std::string geo_layer_name;
82  int32_t source_srid;
83 
85  : delimiter(',')
86  , null_str("\\N")
88  , quoted(true)
89  , quote('"')
90  , escape('"')
91  , line_delim('\n')
92  , array_delim(',')
93  , array_begin('{')
94  , array_end('}')
95  , threads(0)
96  , max_reject(100000)
98  , retry_count(100)
99  , retry_wait(5)
100  , batch_size(1000)
102  , lonlat(true)
106  , geo_coords_srid(4326)
110  , source_srid(0) {}
111 
112  CopyParams(char d, const std::string& n, char l, size_t b, size_t retries, size_t wait)
113  : delimiter(d)
114  , null_str(n)
116  , quoted(true)
117  , quote('"')
118  , escape('"')
119  , line_delim(l)
120  , array_delim(',')
121  , array_begin('{')
122  , array_end('}')
123  , threads(0)
124  , max_reject(100000)
126  , retry_count(retries)
127  , retry_wait(wait)
128  , batch_size(b)
130  , lonlat(true)
134  , geo_coords_srid(4326)
138  , source_srid(0) {}
139 };
140 } // namespace import_export
std::string s3_secret_key
Definition: CopyParams.h:63
SQLTypes
Definition: sqltypes.h:37
tuple d
Definition: test_fsi.py:9
Constants for Builtin SQL Types supported by OmniSci.
ImportHeaderRow has_header
Definition: CopyParams.h:48
EncodingType
Definition: sqltypes.h:227
bool g_enable_smem_group_by true
std::string geo_layer_name
Definition: CopyParams.h:79
std::string s3_session_token
Definition: CopyParams.h:64
CopyParams(char d, const std::string &n, char l, size_t b, size_t retries, size_t wait)
Definition: CopyParams.h:112
bool g_enable_watchdog false
Definition: Execute.cpp:76
static constexpr size_t kImportFileBufferSize
Definition: CopyParams.h:32
std::string s3_access_key
Definition: CopyParams.h:62
EncodingType geo_coords_encoding
Definition: CopyParams.h:74