OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
RegexFileBufferParser.h
Go to the documentation of this file.
1 /*
2  * Copyright 2021 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #if defined(_WIN32) && !defined(WIN32_LEAN_AND_MEAN)
20 // boost/regex.hpp on win32 includes Windows.h
21 // and we need to clean up macros such as ERROR and GetObject
22 #define WIN32_LEAN_AND_MEAN
23 #endif
24 #include <boost/regex.hpp>
25 
26 #if defined(_WIN32) && defined(WIN32_LEAN_AND_MEAN)
28 #undef WIN32_LEAN_AND_MEAN
29 #endif
30 
32 
33 namespace foreign_storage {
35  public:
36  RegexFileBufferParser(const ForeignTable* foreign_table);
37 
39  bool convert_data_blocks,
40  bool columns_are_pre_filtered = false) const override;
41 
43  const ForeignTable* foreign_table) const override;
44 
45  size_t findRowEndPosition(size_t& alloc_size,
46  std::unique_ptr<char[]>& buffer,
47  size_t& buffer_size,
48  const import_export::CopyParams& copy_params,
49  const size_t buffer_first_row_index,
50  unsigned int& num_rows_in_buffer,
51  FileReader* file_reader) const override;
52 
53  void validateFiles(const FileReader* file_reader,
54  const ForeignTable* foreign_table) const override;
55 
56  // For testing purposes only
57  static void setSkipFirstLineForTesting(bool skip);
58  static void setMaxBufferResize(size_t max_buffer_resize);
59 
60  inline static const std::string LINE_REGEX_KEY = "LINE_REGEX";
61  inline static const std::string LINE_START_REGEX_KEY = "LINE_START_REGEX";
62  inline static const std::string BUFFER_SIZE_KEY = "BUFFER_SIZE";
63 
64  private:
65  static size_t getMaxBufferResize();
66 
67  inline static size_t max_buffer_resize_{
69 
70  // Flag added for testing purposes only
71  inline static bool skip_first_line_{false};
72 
73  boost::regex line_regex_;
74  std::optional<boost::regex> line_start_regex_;
75 };
76 } // namespace foreign_storage
import_export::CopyParams validateAndGetCopyParams(const ForeignTable *foreign_table) const override
RegexFileBufferParser(const ForeignTable *foreign_table)
static void setMaxBufferResize(size_t max_buffer_resize)
size_t findRowEndPosition(size_t &alloc_size, std::unique_ptr< char[]> &buffer, size_t &buffer_size, const import_export::CopyParams &copy_params, const size_t buffer_first_row_index, unsigned int &num_rows_in_buffer, FileReader *file_reader) const override
std::optional< boost::regex > line_start_regex_
ParseBufferResult parseBuffer(ParseBufferRequest &request, bool convert_data_blocks, bool columns_are_pre_filtered=false) const override
static constexpr size_t max_import_buffer_resize_byte_size
Definition: CopyParams.h:35
void validateFiles(const FileReader *file_reader, const ForeignTable *foreign_table) const override