OmniSciDB  04ee39c94c
InputTokenizers.h
Go to the documentation of this file.
1 #ifndef INPUTTOKENIZERS_H
2 #define INPUTTOKENIZERS_H
3 
4 #include <boost/tokenizer.hpp>
5 #include <iostream>
6 #include <sstream>
7 
8 template <typename TOKEN_LIST_TYPE>
10  public:
11  template <typename INPUT_TYPE>
12  TOKEN_LIST_TYPE extract_tokens(INPUT_TYPE& input_stream) {
13  TOKEN_LIST_TYPE token_list;
14 
15  std::istringstream input_resolver(input_stream);
16  std::string input_token;
17  while (input_resolver >> input_token) {
18  token_list.push_back(input_token);
19  }
20 
21  return token_list;
22  }
23 };
24 
25 template <typename TOKEN_LIST_TYPE>
27  public:
28  template <typename INPUT_TYPE>
29  TOKEN_LIST_TYPE extract_tokens(INPUT_TYPE& input_stream,
30  std::ostream& error_stream = std::cout) {
31  using Tokenizer = boost::tokenizer<boost::escaped_list_separator<char>>;
32  using TokenIterator = Tokenizer::iterator;
33  TOKEN_LIST_TYPE token_list;
34 
35  std::istringstream input_resolver(input_stream);
36  std::string input_token;
37  if (input_resolver >> input_token) {
38  token_list.push_back(input_token); // First token
39  } else {
40  return token_list; // No token, no more business, return nothing.
41  }
42 
43  // The reason the tokenizing is broken into two portions is because
44  // boost tokenizer thinks the backslash commands are escape
45  // sequences. They aren't. We extract the command first, then
46  // extract the portion sent to the tokenizer
47  std::string second_half;
48  std::getline(input_resolver, second_half);
49  if (!input_resolver) {
50  return token_list;
51  }
52 
53  try {
54  boost::escaped_list_separator<char> els("\\", " ", "\"");
55  Tokenizer splitter(second_half, els);
56 
57  for (TokenIterator i = splitter.begin(); i != splitter.end(); i++) {
58  if (!i->empty()) {
59  token_list.push_back(*i);
60  }
61  }
62  } catch (boost::exception& e) {
63  // Silently fail; odds are this input stream will be processed by a parallel input
64  // tokenizer (like for Regex commands)
65  } // Best efforts
66 
67  return token_list;
68  }
69 };
70 
71 template <typename TOKEN_LIST_TYPE>
73 
74 #endif
TOKEN_LIST_TYPE extract_tokens(INPUT_TYPE &input_stream)
TOKEN_LIST_TYPE extract_tokens(INPUT_TYPE &input_stream, std::ostream &error_stream=std::cout)