#include <DelimitedParserUtils.h>

Static Public Member Functions
static size_t	find_beginning (const char *buffer, size_t begin, size_t end, const CopyParams &copy_params)
	Finds the closest possible row beginning in the given buffer. More...

static size_t	find_end (const char *buffer, size_t size, const CopyParams &copy_params, unsigned int &num_rows_this_buffer)
	Finds the closest possible row ending to the end of the given buffer. More...

static const char *	get_row (const char buf, const char buf_end, const char entire_buf_end, const Importer_NS::CopyParams &copy_params, const bool is_array, std::vector< std::string > &row, bool &try_single_thread)
	Parses the first row in the given buffer and inserts fields into given vector. More...

static void	parseStringArray (const std::string &s, const Importer_NS::CopyParams &copy_params, std::vector< std::string > &string_vec)
	Parses given string array and inserts into given vector of strings. More...

Detailed Description

Definition at line 31 of file DelimitedParserUtils.h.

Member Function Documentation

size_t Importer_NS::DelimitedParserUtils::find_beginning	(	const char *	buffer,
		size_t	begin,
		size_t	end,
		const CopyParams &	copy_params
	)

static

Finds the closest possible row beginning in the given buffer.

Parameters

buffer	Given buffer which has the rows in csv format. (NOT OWN)
begin	Start index of buffer to look for the beginning.
end	End index of buffer to look for the beginning.
copy_params	Copy params for the table.

Returns: The position of the closest possible row beginning to the start of the given buffer.

Definition at line 58 of file DelimitedParserUtils.cpp.

References Importer_NS::CopyParams::line_delim.

Referenced by Importer_NS::import_thread_delimited().

                                                                                       {
   // @TODO(wei) line_delim is in quotes note supported
   if (begin == 0 || (begin > 0 && buffer[begin - 1] == copy_params.line_delim)) {
     return 0;
   }
   size_t i;
   const char* buf = buffer + begin;
   for (i = 0; i < end - begin; i++) {
     if (buf[i] == copy_params.line_delim) {
       return i + 1;
     }
   }
   return i;
 }

Here is the caller graph for this function:

size_t Importer_NS::DelimitedParserUtils::find_end	(	const char *	buffer,
		size_t	size,
		const CopyParams &	copy_params,
		unsigned int &	num_rows_this_buffer
	)

static

Finds the closest possible row ending to the end of the given buffer.

Parameters

buffer	Given buffer which has the rows in csv format. (NOT OWN)
size	Size of the buffer.
copy_params	Copy params for the table.
num_rows_this_buffer	Number of rows until the closest possible row ending.

Returns: The position of the closest possible row ending to the end of the given buffer.

Definition at line 76 of file DelimitedParserUtils.cpp.

References logger::ERROR, Importer_NS::CopyParams::escape, Importer_NS::CopyParams::line_delim, LOG, Importer_NS::CopyParams::quote, and Importer_NS::CopyParams::quoted.

Referenced by Importer_NS::Importer::importDelimited().

                                                                           {
   size_t last_line_delim_pos = 0;
   if (copy_params.quoted) {
     const char* current = buffer;
     bool in_quote = false;
 
     while (current < buffer + size) {
       while (!in_quote && current < buffer + size) {
         // We are outside of quotes. We have to find the last possible line delimiter.
         if (*current == copy_params.line_delim) {
           last_line_delim_pos = current - buffer;
           ++num_rows_this_buffer;
         } else if (*current == copy_params.quote) {
           in_quote = true;
         }
         ++current;
       }
 
       while (in_quote && current < buffer + size) {
         // We are in a quoted field. We have to find the ending quote.
         if ((*current == copy_params.escape) && (current < buffer + size - 1) &&
             (*(current + 1) == copy_params.quote)) {
           ++current;
         } else if (*current == copy_params.quote) {
           in_quote = false;
         }
         ++current;
       }
     }
   } else {
     const char* current = buffer;
     while (current < buffer + size) {
       if (*current == copy_params.line_delim) {
         last_line_delim_pos = current - buffer;
         ++num_rows_this_buffer;
       }
       ++current;
     }
   }
 
   if (last_line_delim_pos <= 0) {
     size_t slen = size < 50 ? size : 50;
     std::string showMsgStr(buffer, buffer + slen);
     LOG(ERROR) << "No line delimiter in block. Block was of size " << size
                << " bytes, first few characters " << showMsgStr;
     return size;
   }
 
   return last_line_delim_pos + 1;
 }

Here is the caller graph for this function:

const char * Importer_NS::DelimitedParserUtils::get_row	(	const char *	buf,
		const char *	buf_end,
		const char *	entire_buf_end,
		const Importer_NS::CopyParams &	copy_params,
		const bool *	is_array,
		std::vector< std::string > &	row,
		bool &	try_single_thread
	)

static

Parses the first row in the given buffer and inserts fields into given vector.

Parameters

buf	Given buffer which has the rows in csv format. (NOT OWN)
buf_end	End of the sliced buffer for the thread. (NOT OWN)
entire_buf_end	End of the entire buffer. (NOT OWN)
copy_params	Copy params for the table.
is_array	Array of bools which tells if a column is an array type.
row	Given vector to be populated with parsed fields.
try_single_thread	In case of parse errors, this will tell if parsing should continue with single thread.

Returns: Pointer to the next row after the first row is parsed.

Definition at line 130 of file DelimitedParserUtils.cpp.

References Importer_NS::CopyParams::array_begin, Importer_NS::CopyParams::array_end, Importer_NS::CopyParams::delimiter, logger::ERROR, Importer_NS::CopyParams::escape, field(), anonymous_namespace{DelimitedParserUtils.cpp}::is_eol(), LOG, Importer_NS::CopyParams::quote, Importer_NS::CopyParams::quoted, anonymous_namespace{DelimitedParserUtils.cpp}::trim_quotes(), and Importer_NS::trim_space().

Referenced by Importer_NS::import_thread_delimited(), parseStringArray(), and Importer_NS::Detector::split_raw_data().

                                                                    {
   const char* field = buf;
   const char* p;
   bool in_quote = false;
   bool in_array = false;
   bool has_escape = false;
   bool strip_quotes = false;
   try_single_thread = false;
   for (p = buf; p < entire_buf_end; ++p) {
     if (*p == copy_params.escape && p < entire_buf_end - 1 &&
         *(p + 1) == copy_params.quote) {
       p++;
       has_escape = true;
     } else if (copy_params.quoted && *p == copy_params.quote) {
       in_quote = !in_quote;
       if (in_quote) {
         strip_quotes = true;
       }
     } else if (!in_quote && is_array != nullptr && *p == copy_params.array_begin &&
                is_array[row.size()]) {
       in_array = true;
       while (p < entire_buf_end - 1) {  // Array type will be parsed separately.
         ++p;
         if (*p == copy_params.array_end) {
           in_array = false;
           break;
         }
       }
     } else if (*p == copy_params.delimiter || is_eol(*p, copy_params)) {
       if (!in_quote) {
         if (!has_escape && !strip_quotes) {
           const char* field_end = p;
           trim_space(field, field_end);
           row.emplace_back(field, field_end - field);
         } else {
           auto field_buf = std::make_unique<char[]>(p - field + 1);
           int j = 0, i = 0;
           for (; i < p - field; i++, j++) {
             if (has_escape && field[i] == copy_params.escape &&
                 field[i + 1] == copy_params.quote) {
               field_buf[j] = copy_params.quote;
               i++;
             } else {
               field_buf[j] = field[i];
             }
           }
           const char* field_begin = field_buf.get();
           const char* field_end = field_buf.get() + j;
           trim_space(field_begin, field_end);
           trim_quotes(field_begin, field_end, copy_params);
           row.emplace_back(field_begin, field_end - field_begin);
         }
         field = p + 1;
         has_escape = false;
         strip_quotes = false;
 
         if (is_eol(*p, copy_params)) {
           // We are at the end of the row. Skip the line endings now.
           while (p + 1 < buf_end && is_eol(*(p + 1), copy_params)) {
             p++;
           }
           break;
         }
       }
     }
   }
   /*
   @TODO(wei) do error handling
   */
   if (in_quote) {
     LOG(ERROR) << "Unmatched quote.";
     try_single_thread = true;
   }
   if (in_array) {
     LOG(ERROR) << "Unmatched array.";
     try_single_thread = true;
   }
   return p;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Importer_NS::DelimitedParserUtils::parseStringArray	(	const std::string &	s,
		const Importer_NS::CopyParams &	copy_params,
		std::vector< std::string > &	string_vec
	)

static

Parses given string array and inserts into given vector of strings.

Parameters

s	Given string array
copy_params	Copy params for the table.
string_vec	Given vector to be populated with parsed fields.

Definition at line 216 of file DelimitedParserUtils.cpp.

References Importer_NS::CopyParams::array_begin, Importer_NS::CopyParams::array_delim, Importer_NS::CopyParams::array_end, Importer_NS::CopyParams::delimiter, get_row(), StringDictionary::MAX_STRLEN, Importer_NS::CopyParams::null_str, and to_string().

Referenced by Importer_NS::TypedImportBuffer::add_value(), and RowToColumnLoader::convert_string_to_column().

                                                                               {
   if (s == copy_params.null_str || s == "NULL" || s.size() < 1 || s.empty()) {
     // TODO: should not convert NULL, empty arrays to {"NULL"},
     //       need to support NULL, empty properly
     string_vec.emplace_back("NULL");
     return;
   }
   if (s[0] != copy_params.array_begin || s[s.size() - 1] != copy_params.array_end) {
     throw std::runtime_error("Malformed Array :" + s);
   }
 
   std::string row(s.c_str() + 1, s.length() - 2);
   row.push_back('\n');
   bool try_single_thread = false;
   Importer_NS::CopyParams array_params = copy_params;
   array_params.delimiter = copy_params.array_delim;
   get_row(row.c_str(),
           row.c_str() + row.length(),
           row.c_str() + row.length(),
           array_params,
           nullptr,
           string_vec,
           try_single_thread);
 
   for (size_t i = 0; i < string_vec.size(); ++i) {
     if (string_vec[i].empty()) {  // Disallow empty strings for now
       string_vec.erase(string_vec.begin() + i);
       --i;
     } else if (string_vec[i].size() > StringDictionary::MAX_STRLEN) {
       throw std::runtime_error("Array String too long : " + string_vec[i] + " max is " +
                                std::to_string(StringDictionary::MAX_STRLEN));
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

The documentation for this class was generated from the following files:

/home/jenkins-slave/workspace/core-os-doxygen/Import/DelimitedParserUtils.h
/home/jenkins-slave/workspace/core-os-doxygen/Import/DelimitedParserUtils.cpp

Static Public Member Functions

Detailed Description

Member Function Documentation