OmniSciDB  c07336695a
QuotedInputSupportParser< TOKEN_LIST_TYPE > Class Template Reference

#include <InputTokenizers.h>

Public Member Functions

template<typename INPUT_TYPE >
TOKEN_LIST_TYPE extract_tokens (INPUT_TYPE &input_stream, std::ostream &error_stream=std::cout)
 

Detailed Description

template<typename TOKEN_LIST_TYPE>
class QuotedInputSupportParser< TOKEN_LIST_TYPE >

Definition at line 26 of file InputTokenizers.h.

Member Function Documentation

◆ extract_tokens()

template<typename TOKEN_LIST_TYPE >
template<typename INPUT_TYPE >
TOKEN_LIST_TYPE QuotedInputSupportParser< TOKEN_LIST_TYPE >::extract_tokens ( INPUT_TYPE &  input_stream,
std::ostream &  error_stream = std::cout 
)
inline

Definition at line 29 of file InputTokenizers.h.

30  {
31  using Tokenizer = boost::tokenizer<boost::escaped_list_separator<char>>;
32  using TokenIterator = Tokenizer::iterator;
33  TOKEN_LIST_TYPE token_list;
34 
35  std::istringstream input_resolver(input_stream);
36  std::string input_token;
37  if (input_resolver >> input_token) {
38  token_list.push_back(input_token); // First token
39  } else {
40  return token_list; // No token, no more business, return nothing.
41  }
42 
43  // The reason the tokenizing is broken into two portions is because
44  // boost tokenizer thinks the backslash commands are escape
45  // sequences. They aren't. We extract the command first, then
46  // extract the portion sent to the tokenizer
47  std::string second_half;
48  std::getline(input_resolver, second_half);
49  if (!input_resolver) {
50  return token_list;
51  }
52 
53  try {
54  boost::escaped_list_separator<char> els("\\", " ", "\"");
55  Tokenizer splitter(second_half, els);
56 
57  for (TokenIterator i = splitter.begin(); i != splitter.end(); i++) {
58  if (!i->empty()) {
59  token_list.push_back(*i);
60  }
61  }
62  } catch (boost::exception& e) {
63  // Silently fail; odds are this input stream will be processed by a parallel input
64  // tokenizer (like for Regex commands)
65  } // Best efforts
66 
67  return token_list;
68  }

The documentation for this class was generated from the following file: