DelimitedParserUtils.h

Go to the documentation of this file.

/*
 * Copyright 2022 HEAVY.AI, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * @file DelimitedParserUtils.h
 * @brief utilities for parsing delimited data
 *
 */

#pragma once

#include <string>
#include <vector>

#include "DataMgr/ForeignStorage/FileReader.h"
#include "ImportExport/CopyParams.h"

namespace import_export {
namespace delimited_parser {

class InsufficientBufferSizeException : public std::runtime_error {
 public:
  InsufficientBufferSizeException(const std::string& message)
      : std::runtime_error(message) {}
};

size_t find_beginning(const char* buffer,
                      size_t begin,
                      size_t end,
                      const CopyParams& copy_params);

size_t get_max_buffer_resize();

void set_max_buffer_resize(const size_t max_buffer_resize);

size_t find_row_end_pos(size_t& alloc_size,
                        std::unique_ptr<char[]>& buffer,
                        size_t& buffer_size,
                        const CopyParams& copy_params,
                        const size_t buffer_first_row_index,
                        unsigned int& num_rows_in_buffer,
                        FILE* file,
                        foreign_storage::FileReader* file_reader = nullptr);

template <typename T>
const char* get_row(const char* buf,
                    const char* buf_end,
                    const char* entire_buf_end,
                    const import_export::CopyParams& copy_params,
                    const bool* is_array,
                    std::vector<T>& row,
                    std::vector<std::unique_ptr<char[]>>& tmp_buffers,
                    bool& try_single_thread,
                    bool filter_empty_lines);

void parse_string_array(const std::string& s,
                        const import_export::CopyParams& copy_params,
                        std::vector<std::string>& string_vec,
                        bool truncate_values = false);

void extend_buffer(std::unique_ptr<char[]>& buffer,
                   size_t& buffer_size,
                   size_t& alloc_size,
                   FILE* file,
                   foreign_storage::FileReader* file_reader,
                   size_t max_buffer_resize);
}  // namespace delimited_parser

}  // namespace import_export