OmniSciDB  0264ff685a
CopyParams.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file CopyParams.h
19  * @author Mehmet Sariyuce <mehmet.sariyuce@omnisci.com>
20  * @brief CopyParams struct
21  */
22 
23 #pragma once
24 
25 #include <string>
26 
27 #include "Shared/sqltypes.h"
28 
29 namespace import_export {
30 
31 // not too big (need much memory) but not too small (many thread forks)
32 constexpr static size_t kImportFileBufferSize = (1 << 23);
33 
34 enum class FileType {
35  DELIMITED,
36  POLYGON
37 #ifdef ENABLE_IMPORT_PARQUET
38  ,
39  PARQUET
40 #endif
41 };
42 
44 
45 struct CopyParams {
46  char delimiter;
47  std::string null_str;
49  bool quoted; // does the input have any quoted fields, default to false
50  char quote;
51  char escape;
52  char line_delim;
55  char array_end;
56  int threads;
57  size_t
58  max_reject; // maximum number of records that can be rejected before copy is failed
60  bool plain_text = false;
61  // s3/parquet related params
62  std::string s3_access_key; // per-query credentials to override the
63  std::string s3_secret_key; // settings in ~/.aws/credentials or environment
64  std::string s3_region;
65  std::string s3_endpoint;
66  // kafka related params
67  size_t retry_count;
68  size_t retry_wait;
69  size_t batch_size;
70  size_t buffer_size;
71  // geospatial params
72  bool lonlat;
76  int32_t geo_coords_srid;
78  std::string geo_layer_name;
81  int32_t source_srid;
82 
84  : delimiter(',')
85  , null_str("\\N")
86  , has_header(ImportHeaderRow::AUTODETECT)
87  , quoted(true)
88  , quote('"')
89  , escape('"')
90  , line_delim('\n')
91  , array_delim(',')
92  , array_begin('{')
93  , array_end('}')
94  , threads(0)
95  , max_reject(100000)
96  , file_type(FileType::DELIMITED)
97  , retry_count(100)
98  , retry_wait(5)
99  , batch_size(1000)
100  , buffer_size(kImportFileBufferSize)
101  , lonlat(true)
102  , geo_coords_encoding(kENCODING_GEOINT)
103  , geo_coords_comp_param(32)
104  , geo_coords_type(kGEOMETRY)
105  , geo_coords_srid(4326)
106  , sanitize_column_names(true)
107  , geo_assign_render_groups(true)
108  , geo_explode_collections(false)
109  , source_srid(0) {}
110 
111  CopyParams(char d, const std::string& n, char l, size_t b, size_t retries, size_t wait)
112  : delimiter(d)
113  , null_str(n)
114  , has_header(ImportHeaderRow::AUTODETECT)
115  , quoted(true)
116  , quote('"')
117  , escape('"')
118  , line_delim(l)
119  , array_delim(',')
120  , array_begin('{')
121  , array_end('}')
122  , threads(0)
123  , max_reject(100000)
124  , file_type(FileType::DELIMITED)
125  , retry_count(retries)
126  , retry_wait(wait)
127  , batch_size(b)
128  , buffer_size(kImportFileBufferSize)
129  , lonlat(true)
130  , geo_coords_encoding(kENCODING_GEOINT)
131  , geo_coords_comp_param(32)
132  , geo_coords_type(kGEOMETRY)
133  , geo_coords_srid(4326)
134  , sanitize_column_names(true)
135  , geo_assign_render_groups(true)
136  , geo_explode_collections(false)
137  , source_srid(0) {}
138 };
139 } // namespace import_export
std::string s3_secret_key
Definition: CopyParams.h:63
SQLTypes
Definition: sqltypes.h:37
Constants for Builtin SQL Types supported by OmniSci.
ImportHeaderRow has_header
Definition: CopyParams.h:48
EncodingType
Definition: sqltypes.h:224
std::string geo_layer_name
Definition: CopyParams.h:78
CopyParams(char d, const std::string &n, char l, size_t b, size_t retries, size_t wait)
Definition: CopyParams.h:111
static constexpr size_t kImportFileBufferSize
Definition: CopyParams.h:32
std::string s3_access_key
Definition: CopyParams.h:62
EncodingType geo_coords_encoding
Definition: CopyParams.h:73