OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
CopyParams.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file CopyParams.h
19  * @author Mehmet Sariyuce <mehmet.sariyuce@omnisci.com>
20  * @brief CopyParams struct
21  */
22 
23 #pragma once
24 
25 #include <string>
26 
27 #include "Shared/sqltypes.h"
28 
29 namespace Importer_NS {
30 
31 // not too big (need much memory) but not too small (many thread forks)
32 constexpr static size_t kImportFileBufferSize = (1 << 23);
33 
34 enum class FileType {
35  DELIMITED,
36  POLYGON
37 #ifdef ENABLE_IMPORT_PARQUET
38  ,
39  PARQUET
40 #endif
41 };
42 
44 
45 struct CopyParams {
46  char delimiter;
47  std::string null_str;
49  bool quoted; // does the input have any quoted fields, default to false
50  char quote;
51  char escape;
52  char line_delim;
55  char array_end;
56  int threads;
57  size_t
58  max_reject; // maximum number of records that can be rejected before copy is failed
60  bool plain_text = false;
61  // s3/parquet related params
62  std::string s3_access_key; // per-query credentials to override the
63  std::string s3_secret_key; // settings in ~/.aws/credentials or environment
64  std::string s3_region;
65  std::string s3_endpoint;
66  // kafka related params
67  size_t retry_count;
68  size_t retry_wait;
69  size_t batch_size;
70  size_t buffer_size;
71  // geospatial params
72  bool lonlat;
76  int32_t geo_coords_srid;
78  std::string geo_layer_name;
81 
83  : delimiter(',')
84  , null_str("\\N")
86  , quoted(true)
87  , quote('"')
88  , escape('"')
89  , line_delim('\n')
90  , array_delim(',')
91  , array_begin('{')
92  , array_end('}')
93  , threads(0)
94  , max_reject(100000)
96  , retry_count(100)
97  , retry_wait(5)
98  , batch_size(1000)
100  , lonlat(true)
104  , geo_coords_srid(4326)
108 
109  CopyParams(char d, const std::string& n, char l, size_t b, size_t retries, size_t wait)
110  : delimiter(d)
111  , null_str(n)
113  , quoted(true)
114  , quote('"')
115  , escape('"')
116  , line_delim(l)
117  , array_delim(',')
118  , array_begin('{')
119  , array_end('}')
120  , threads(0)
121  , max_reject(100000)
123  , retry_count(retries)
124  , retry_wait(wait)
125  , batch_size(b)
127  , lonlat(true)
131  , geo_coords_srid(4326)
135 };
136 } // namespace Importer_NS
std::string null_str
Definition: CopyParams.h:47
SQLTypes
Definition: sqltypes.h:41
EncodingType
Definition: encodetypes.h:22
Constants for Builtin SQL Types supported by MapD.
EncodingType geo_coords_encoding
Definition: CopyParams.h:73
CopyParams(char d, const std::string &n, char l, size_t b, size_t retries, size_t wait)
Definition: CopyParams.h:109
std::string s3_access_key
Definition: CopyParams.h:62
ImportHeaderRow has_header
Definition: CopyParams.h:48
bool g_enable_smem_group_by true
std::string s3_endpoint
Definition: CopyParams.h:65
std::string s3_region
Definition: CopyParams.h:64
std::string geo_layer_name
Definition: CopyParams.h:78
bool g_enable_watchdog false
Definition: Execute.cpp:71
std::string s3_secret_key
Definition: CopyParams.h:63
static constexpr size_t kImportFileBufferSize
Definition: CopyParams.h:32
int32_t geo_coords_comp_param
Definition: CopyParams.h:74