OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
StringDictionary.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef STRINGDICTIONARY_STRINGDICTIONARY_H
18 #define STRINGDICTIONARY_STRINGDICTIONARY_H
19 
20 #include "DictRef.h"
21 #include "DictionaryCache.hpp"
22 #include "StringOps/StringOpInfo.h"
23 
24 #include <functional>
25 #include <future>
26 #include <map>
27 #include <shared_mutex>
28 #include <string>
29 #include <string_view>
30 #include <tuple>
31 #include <vector>
32 
34 
36 
37 namespace StringOps_Namespace {
38 struct StringOpInfo;
39 }
40 
41 class DictPayloadUnavailable : public std::runtime_error {
42  public:
43  DictPayloadUnavailable() : std::runtime_error("DictPayloadUnavailable") {}
44 
45  DictPayloadUnavailable(const std::string& err) : std::runtime_error(err) {}
46 };
47 
48 class LeafHostInfo;
49 
50 using string_dict_hash_t = uint32_t;
51 
52 using StringLookupCallback = std::function<bool(std::string_view, int32_t string_id)>;
53 
55  public:
56  StringDictionary(const DictRef& dict_ref,
57  const std::string& folder,
58  const bool isTemp,
59  const bool recover,
60  const bool materializeHashes = false,
61  size_t initial_capacity = 256);
62  StringDictionary(const LeafHostInfo& host, const DictRef dict_ref);
63  ~StringDictionary() noexcept;
64 
65  int32_t getDbId() const noexcept;
66  int32_t getDictId() const noexcept;
67 
69  public:
70  virtual ~StringCallback() = default;
71  virtual void operator()(std::string const&, int32_t const string_id) = 0;
72  virtual void operator()(std::string_view const, int32_t const string_id) = 0;
73  };
74 
75  // Functors passed to eachStringSerially() must derive from StringCallback.
76  // Each std::string const& (if isClient()) or std::string_view (if !isClient())
77  // plus string_id is passed to the callback functor.
78  void eachStringSerially(int64_t const generation, StringCallback&) const;
79  std::function<int32_t(std::string const&)> makeLambdaStringToId() const;
80  friend class StringLocalCallback;
81 
82  int32_t getOrAdd(const std::string& str) noexcept;
83  template <class T, class String>
84  size_t getBulk(const std::vector<String>& string_vec, T* encoded_vec) const;
85  template <class T, class String>
86  size_t getBulk(const std::vector<String>& string_vec,
87  T* encoded_vec,
88  const int64_t generation) const;
89  template <class T, class String>
90  void getOrAddBulk(const std::vector<String>& string_vec, T* encoded_vec);
91  template <class T, class String>
92  void getOrAddBulkParallel(const std::vector<String>& string_vec, T* encoded_vec);
93  template <class String>
94  void getOrAddBulkArray(const std::vector<std::vector<String>>& string_array_vec,
95  std::vector<std::vector<int32_t>>& ids_array_vec);
96  template <class String>
97  int32_t getIdOfString(const String&) const;
98  std::string getString(int32_t string_id) const;
99  std::pair<char*, size_t> getStringBytes(int32_t string_id) const noexcept;
100  size_t storageEntryCount() const;
101 
102  std::vector<int32_t> getLike(const std::string& pattern,
103  const bool icase,
104  const bool is_simple,
105  const char escape,
106  const size_t generation) const;
107 
108  std::vector<int32_t> getCompare(const std::string& pattern,
109  const std::string& comp_operator,
110  const size_t generation);
111 
112  std::vector<int32_t> getRegexpLike(const std::string& pattern,
113  const char escape,
114  const size_t generation) const;
115 
116  std::vector<std::string> copyStrings() const;
117 
118  std::vector<std::string_view> getStringViews() const;
119  std::vector<std::string_view> getStringViews(const size_t generation) const;
120 
121  std::vector<int32_t> buildDictionaryTranslationMap(
122  const std::shared_ptr<StringDictionary> dest_dict,
123  StringLookupCallback const& dest_transient_lookup_callback) const;
124 
126  const StringDictionary* dest_dict,
127  int32_t* translated_ids,
128  const int64_t source_generation,
129  const int64_t dest_generation,
130  const bool dest_has_transients,
131  StringLookupCallback const& dest_transient_lookup_callback,
132  const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos) const;
133 
135  Datum* translated_ids,
136  const int64_t source_generation,
137  const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos) const;
138 
139  bool checkpoint() noexcept;
140 
141  bool isClient() const noexcept;
142 
160  static void populate_string_ids(
161  std::vector<int32_t>& dest_ids,
162  StringDictionary* dest_dict,
163  const std::vector<int32_t>& source_ids,
164  const StringDictionary* source_dict,
165  const std::vector<std::string const*>& transient_string_vec = {});
166 
167  static void populate_string_array_ids(
168  std::vector<std::vector<int32_t>>& dest_array_ids,
169  StringDictionary* dest_dict,
170  const std::vector<std::vector<int32_t>>& source_array_ids,
171  const StringDictionary* source_dict);
172 
173  static constexpr int32_t INVALID_STR_ID = -1;
174  static constexpr size_t MAX_STRLEN = (1 << 15) - 1;
175  static constexpr size_t MAX_STRCOUNT = (1U << 31) - 1;
176 
177  void update_leaf(const LeafHostInfo& host_info);
178 
179  private:
180  struct StringIdxEntry {
181  uint64_t off : 48;
182  uint64_t size : 16;
183  };
184 
185  // In the compare_cache_value_t index represents the index of the sorted cache.
186  // The diff component represents whether the index the cache is pointing to is equal to
187  // the pattern it is cached for. We want to use diff so we don't have compare string
188  // again when we are retrieving it from the cache.
190  int32_t index;
191  int32_t diff;
192  };
193 
194  struct PayloadString {
195  char* c_str_ptr;
196  size_t size;
197  bool canary;
198  };
199 
201  std::vector<std::future<std::vector<std::pair<string_dict_hash_t, unsigned int>>>>&
202  dictionary_futures);
203  size_t getNumStringsFromStorage(const size_t storage_slots) const noexcept;
204  bool fillRateIsHigh(const size_t num_strings) const noexcept;
205  void increaseHashTableCapacity() noexcept;
206  template <class String>
208  const size_t str_count,
209  const size_t storage_high_water_mark,
210  const std::vector<String>& input_strings,
211  const std::vector<size_t>& string_memory_ids,
212  const std::vector<string_dict_hash_t>& input_strings_hashes) noexcept;
213  int32_t getOrAddImpl(const std::string_view& str) noexcept;
214  template <class String>
215  void hashStrings(const std::vector<String>& string_vec,
216  std::vector<string_dict_hash_t>& hashes) const noexcept;
217 
218  int32_t getUnlocked(const std::string_view sv) const noexcept;
219  std::string getStringUnlocked(int32_t string_id) const noexcept;
220  std::string getStringChecked(const int string_id) const noexcept;
221  std::pair<char*, size_t> getStringBytesChecked(const int string_id) const noexcept;
222  template <class String>
223  uint32_t computeBucket(
224  const string_dict_hash_t hash,
225  const String& input_string,
226  const std::vector<int32_t>& string_id_string_dict_hash_table) const noexcept;
227  template <class String>
229  const string_dict_hash_t input_string_hash,
230  const String& input_string,
231  const std::vector<int32_t>& string_id_string_dict_hash_table,
232  const size_t storage_high_water_mark,
233  const std::vector<String>& input_strings,
234  const std::vector<size_t>& string_memory_ids) const noexcept;
236  const string_dict_hash_t hash,
237  const std::vector<int32_t>& string_id_string_dict_hash_table) noexcept;
238  void checkAndConditionallyIncreasePayloadCapacity(const size_t write_length);
239  void checkAndConditionallyIncreaseOffsetCapacity(const size_t write_length);
240 
241  template <class String>
242  void appendToStorage(const String str) noexcept;
243  template <class String>
244  void appendToStorageBulk(const std::vector<String>& input_strings,
245  const std::vector<size_t>& string_memory_ids,
246  const size_t sum_new_strings_lengths) noexcept;
247  PayloadString getStringFromStorage(const int string_id) const noexcept;
248  std::string_view getStringFromStorageFast(const int string_id) const noexcept;
249  void addPayloadCapacity(const size_t min_capacity_requested = 0) noexcept;
250  void addOffsetCapacity(const size_t min_capacity_requested = 0) noexcept;
251  size_t addStorageCapacity(int fd, const size_t min_capacity_requested = 0) noexcept;
252  void* addMemoryCapacity(void* addr,
253  size_t& mem_size,
254  const size_t min_capacity_requested = 0) noexcept;
255  void invalidateInvertedIndex() noexcept;
256  std::vector<int32_t> getEquals(std::string pattern,
257  std::string comp_operator,
258  size_t generation);
259  void buildSortedCache();
260  void insertInSortedCache(std::string str, int32_t str_id);
261  void sortCache(std::vector<int32_t>& cache);
262  void mergeSortedCache(std::vector<int32_t>& temp_sorted_cache);
263  compare_cache_value_t* binary_search_cache(const std::string& pattern) const;
264 
266  const std::string folder_;
267  size_t str_count_;
268  size_t collisions_;
269  std::vector<int32_t> string_id_string_dict_hash_table_;
271  std::vector<int32_t> sorted_cache;
272  bool isTemp_;
274  std::string offsets_path_;
282  mutable std::shared_mutex rw_mutex_;
283  mutable std::map<std::tuple<std::string, bool, bool, char>, std::vector<int32_t>>
285  mutable std::map<std::pair<std::string, char>, std::vector<int32_t>> regex_cache_;
286  mutable std::map<std::string, int32_t> equal_cache_;
288  mutable std::shared_ptr<std::vector<std::string>> strings_cache_;
289  mutable std::unique_ptr<StringDictionaryClient> client_;
290  mutable std::unique_ptr<StringDictionaryClient> client_no_timeout_;
291 
292  char* CANARY_BUFFER{nullptr};
293  size_t canary_buffer_size = 0;
294 };
295 
296 int32_t truncate_to_generation(const int32_t id, const size_t generation);
297 
298 void translate_string_ids(std::vector<int32_t>& dest_ids,
299  const LeafHostInfo& dict_server_host,
300  const DictRef dest_dict_ref,
301  const std::vector<int32_t>& source_ids,
302  const DictRef source_dict_ref,
303  const int32_t dest_generation);
304 
305 #endif // STRINGDICTIONARY_STRINGDICTIONARY_H
StringIdxEntry * offset_map_
bool isClient() const noexcept
void increaseHashTableCapacity() noexcept
void checkAndConditionallyIncreasePayloadCapacity(const size_t write_length)
size_t addStorageCapacity(int fd, const size_t min_capacity_requested=0) noexcept
std::vector< int32_t > getRegexpLike(const std::string &pattern, const char escape, const size_t generation) const
size_t getBulk(const std::vector< String > &string_vec, T *encoded_vec) const
std::vector< int32_t > buildDictionaryTranslationMap(const std::shared_ptr< StringDictionary > dest_dict, StringLookupCallback const &dest_transient_lookup_callback) const
std::vector< std::string > copyStrings() const
void buildDictionaryNumericTranslationMap(Datum *translated_ids, const int64_t source_generation, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos) const
uint64_t off
std::pair< char *, size_t > getStringBytesChecked(const int string_id) const noexcept
uint64_t size
size_t storageEntryCount() const
void addOffsetCapacity(const size_t min_capacity_requested=0) noexcept
uint32_t computeBucketFromStorageAndMemory(const string_dict_hash_t input_string_hash, const String &input_string, const std::vector< int32_t > &string_id_string_dict_hash_table, const size_t storage_high_water_mark, const std::vector< String > &input_strings, const std::vector< size_t > &string_memory_ids) const noexcept
std::string getStringChecked(const int string_id) const noexcept
std::vector< string_dict_hash_t > hash_cache_
DictionaryCache< std::string, compare_cache_value_t > compare_cache_
bool fillRateIsHigh(const size_t num_strings) const noexcept
void * addMemoryCapacity(void *addr, size_t &mem_size, const size_t min_capacity_requested=0) noexcept
std::string offsets_path_
static void populate_string_ids(std::vector< int32_t > &dest_ids, StringDictionary *dest_dict, const std::vector< int32_t > &source_ids, const StringDictionary *source_dict, const std::vector< std::string const * > &transient_string_vec={})
Populates provided dest_ids vector with string ids corresponding to given source strings.
const DictRef dict_ref_
std::shared_mutex rw_mutex_
std::string_view getStringFromStorageFast(const int string_id) const noexcept
int32_t getOrAdd(const std::string &str) noexcept
std::map< std::pair< std::string, char >, std::vector< int32_t > > regex_cache_
std::unique_ptr< StringDictionaryClient > client_
std::string getStringUnlocked(int32_t string_id) const noexcept
int32_t getIdOfString(const String &) const
static constexpr size_t MAX_STRCOUNT
std::vector< int32_t > getEquals(std::string pattern, std::string comp_operator, size_t generation)
uint32_t computeBucket(const string_dict_hash_t hash, const String &input_string, const std::vector< int32_t > &string_id_string_dict_hash_table) const noexcept
static constexpr int32_t INVALID_STR_ID
std::shared_ptr< std::vector< std::string > > strings_cache_
virtual void operator()(std::string const &, int32_t const string_id)=0
std::vector< int32_t > getCompare(const std::string &pattern, const std::string &comp_operator, const size_t generation)
StringDictionary(const DictRef &dict_ref, const std::string &folder, const bool isTemp, const bool recover, const bool materializeHashes=false, size_t initial_capacity=256)
const std::string folder_
void appendToStorageBulk(const std::vector< String > &input_strings, const std::vector< size_t > &string_memory_ids, const size_t sum_new_strings_lengths) noexcept
void addPayloadCapacity(const size_t min_capacity_requested=0) noexcept
std::map< std::string, int32_t > equal_cache_
uint32_t computeUniqueBucketWithHash(const string_dict_hash_t hash, const std::vector< int32_t > &string_id_string_dict_hash_table) noexcept
void getOrAddBulkArray(const std::vector< std::vector< String >> &string_array_vec, std::vector< std::vector< int32_t >> &ids_array_vec)
void translate_string_ids(std::vector< int32_t > &dest_ids, const LeafHostInfo &dict_server_host, const DictRef dest_dict_ref, const std::vector< int32_t > &source_ids, const DictRef source_dict_ref, const int32_t dest_generation)
int32_t getUnlocked(const std::string_view sv) const noexcept
void appendToStorage(const String str) noexcept
void getOrAddBulk(const std::vector< String > &string_vec, T *encoded_vec)
std::pair< char *, size_t > getStringBytes(int32_t string_id) const noexcept
DictPayloadUnavailable(const std::string &err)
void processDictionaryFutures(std::vector< std::future< std::vector< std::pair< string_dict_hash_t, unsigned int >>>> &dictionary_futures)
bool checkpoint() noexcept
std::vector< int32_t > getLike(const std::string &pattern, const bool icase, const bool is_simple, const char escape, const size_t generation) const
void mergeSortedCache(std::vector< int32_t > &temp_sorted_cache)
void eachStringSerially(int64_t const generation, StringCallback &) const
int32_t getDictId() const noexcept
size_t getNumStringsFromStorage(const size_t storage_slots) const noexcept
void update_leaf(const LeafHostInfo &host_info)
std::string getString(int32_t string_id) const
void hashStrings(const std::vector< String > &string_vec, std::vector< string_dict_hash_t > &hashes) const noexcept
std::vector< std::string_view > getStringViews() const
int32_t getDbId() const noexcept
std::unique_ptr< StringDictionaryClient > client_no_timeout_
virtual ~StringCallback()=default
void increaseHashTableCapacityFromStorageAndMemory(const size_t str_count, const size_t storage_high_water_mark, const std::vector< String > &input_strings, const std::vector< size_t > &string_memory_ids, const std::vector< string_dict_hash_t > &input_strings_hashes) noexcept
std::function< int32_t(std::string const &)> makeLambdaStringToId() const
void checkAndConditionallyIncreaseOffsetCapacity(const size_t write_length)
static void populate_string_array_ids(std::vector< std::vector< int32_t >> &dest_array_ids, StringDictionary *dest_dict, const std::vector< std::vector< int32_t >> &source_array_ids, const StringDictionary *source_dict)
void invalidateInvertedIndex() noexcept
uint32_t string_dict_hash_t
std::vector< int32_t > string_id_string_dict_hash_table_
void sortCache(std::vector< int32_t > &cache)
static constexpr size_t MAX_STRLEN
void getOrAddBulkParallel(const std::vector< String > &string_vec, T *encoded_vec)
std::function< bool(std::string_view, int32_t string_id)> StringLookupCallback
std::map< std::tuple< std::string, bool, bool, char >, std::vector< int32_t > > like_cache_
bool g_enable_stringdict_parallel
PayloadString getStringFromStorage(const int string_id) const noexcept
std::shared_timed_mutex shared_mutex
compare_cache_value_t * binary_search_cache(const std::string &pattern) const
void insertInSortedCache(std::string str, int32_t str_id)
Definition: Datum.h:44
std::vector< int32_t > sorted_cache
int32_t getOrAddImpl(const std::string_view &str) noexcept
int32_t truncate_to_generation(const int32_t id, const size_t generation)
~StringDictionary() noexcept