OmniSciDB  f632821e96
CopyParams.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file CopyParams.h
19  * @author Mehmet Sariyuce <mehmet.sariyuce@omnisci.com>
20  * @brief CopyParams struct
21  */
22 
23 #pragma once
24 
25 #include <string>
26 
27 #include "Shared/sqltypes.h"
28 
29 namespace import_export {
30 
31 // not too big (need much memory) but not too small (many thread forks)
32 constexpr static size_t kImportFileBufferSize = (1 << 23);
33 
34 enum class FileType {
35  DELIMITED,
36  POLYGON
37 #ifdef ENABLE_IMPORT_PARQUET
38  ,
39  PARQUET
40 #endif
41 };
42 
44 
45 struct CopyParams {
46  char delimiter;
47  std::string null_str;
49  bool quoted; // does the input have any quoted fields, default to false
50  char quote;
51  char escape;
52  char line_delim;
55  char array_end;
56  int threads;
57  size_t
58  max_reject; // maximum number of records that can be rejected before copy is failed
60  bool plain_text = false;
61  // s3/parquet related params
62  std::string s3_access_key; // per-query credentials to override the
63  std::string s3_secret_key; // settings in ~/.aws/credentials or environment
64  std::string s3_region;
65  std::string s3_endpoint;
66  // kafka related params
67  size_t retry_count;
68  size_t retry_wait;
69  size_t batch_size;
70  size_t buffer_size;
71  // geospatial params
72  bool lonlat;
76  int32_t geo_coords_srid;
78  std::string geo_layer_name;
81 
83  : delimiter(',')
84  , null_str("\\N")
85  , has_header(ImportHeaderRow::AUTODETECT)
86  , quoted(true)
87  , quote('"')
88  , escape('"')
89  , line_delim('\n')
90  , array_delim(',')
91  , array_begin('{')
92  , array_end('}')
93  , threads(0)
94  , max_reject(100000)
95  , file_type(FileType::DELIMITED)
96  , retry_count(100)
97  , retry_wait(5)
98  , batch_size(1000)
99  , buffer_size(kImportFileBufferSize)
100  , lonlat(true)
101  , geo_coords_encoding(kENCODING_GEOINT)
102  , geo_coords_comp_param(32)
103  , geo_coords_type(kGEOMETRY)
104  , geo_coords_srid(4326)
105  , sanitize_column_names(true)
106  , geo_assign_render_groups(true)
107  , geo_explode_collections(false) {}
108 
109  CopyParams(char d, const std::string& n, char l, size_t b, size_t retries, size_t wait)
110  : delimiter(d)
111  , null_str(n)
112  , has_header(ImportHeaderRow::AUTODETECT)
113  , quoted(true)
114  , quote('"')
115  , escape('"')
116  , line_delim(l)
117  , array_delim(',')
118  , array_begin('{')
119  , array_end('}')
120  , threads(0)
121  , max_reject(100000)
122  , file_type(FileType::DELIMITED)
123  , retry_count(retries)
124  , retry_wait(wait)
125  , batch_size(b)
126  , buffer_size(kImportFileBufferSize)
127  , lonlat(true)
128  , geo_coords_encoding(kENCODING_GEOINT)
129  , geo_coords_comp_param(32)
130  , geo_coords_type(kGEOMETRY)
131  , geo_coords_srid(4326)
132  , sanitize_column_names(true)
133  , geo_assign_render_groups(true)
134  , geo_explode_collections(false) {}
135 };
136 } // namespace import_export
std::string s3_secret_key
Definition: CopyParams.h:63
SQLTypes
Definition: sqltypes.h:40
Constants for Builtin SQL Types supported by OmniSci.
ImportHeaderRow has_header
Definition: CopyParams.h:48
EncodingType
Definition: sqltypes.h:156
std::string geo_layer_name
Definition: CopyParams.h:78
CopyParams(char d, const std::string &n, char l, size_t b, size_t retries, size_t wait)
Definition: CopyParams.h:109
static constexpr size_t kImportFileBufferSize
Definition: CopyParams.h:32
std::string s3_access_key
Definition: CopyParams.h:62
EncodingType geo_coords_encoding
Definition: CopyParams.h:73