OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
StringDictionary.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef STRINGDICTIONARY_STRINGDICTIONARY_H
18 #define STRINGDICTIONARY_STRINGDICTIONARY_H
19 
20 #include "../Shared/mapd_shared_mutex.h"
21 #include "DictRef.h"
22 #include "DictionaryCache.hpp"
23 
24 #include <sys/mman.h>
25 #include <sys/stat.h>
26 #include <sys/types.h>
27 #include <unistd.h>
28 
29 #include <future>
30 #include <map>
31 #include <string>
32 #include <tuple>
33 #include <vector>
34 
36 
37 class DictPayloadUnavailable : public std::runtime_error {
38  public:
39  DictPayloadUnavailable() : std::runtime_error("DictPayloadUnavailable") {}
40 
41  DictPayloadUnavailable(const std::string& err) : std::runtime_error(err) {}
42 };
43 
44 class LeafHostInfo;
45 
47  public:
48  StringDictionary(const std::string& folder,
49  const bool isTemp,
50  const bool recover,
51  const bool materializeHashes = false,
52  size_t initial_capacity = 256);
53  StringDictionary(const LeafHostInfo& host, const DictRef dict_ref);
54  ~StringDictionary() noexcept;
55 
56  int32_t getOrAdd(const std::string& str) noexcept;
57  template <class T>
58  void getOrAddBulk(const std::vector<std::string>& string_vec, T* encoded_vec);
59  void getOrAddBulkArray(const std::vector<std::vector<std::string>>& string_array_vec,
60  std::vector<std::vector<int32_t>>& ids_array_vec);
61  int32_t getIdOfString(const std::string& str) const;
62  std::string getString(int32_t string_id) const;
63  std::pair<char*, size_t> getStringBytes(int32_t string_id) const noexcept;
64  size_t storageEntryCount() const;
65 
66  std::vector<int32_t> getLike(const std::string& pattern,
67  const bool icase,
68  const bool is_simple,
69  const char escape,
70  const size_t generation) const;
71 
72  std::vector<int32_t> getCompare(const std::string& pattern,
73  const std::string& comp_operator,
74  const size_t generation);
75 
76  std::vector<int32_t> getRegexpLike(const std::string& pattern,
77  const char escape,
78  const size_t generation) const;
79 
80  std::shared_ptr<const std::vector<std::string>> copyStrings() const;
81 
82  bool checkpoint() noexcept;
83 
101  static void populate_string_ids(
102  std::vector<int32_t>& dest_ids,
103  StringDictionary* dest_dict,
104  const std::vector<int32_t>& source_ids,
105  const StringDictionary* source_dict,
106  const std::map<int32_t, std::string> transient_mapping = {});
107 
108  static void populate_string_array_ids(
109  std::vector<std::vector<int32_t>>& dest_array_ids,
110  StringDictionary* dest_dict,
111  const std::vector<std::vector<int32_t>>& source_array_ids,
112  const StringDictionary* source_dict);
113 
114  static constexpr int32_t INVALID_STR_ID = -1;
115  static constexpr size_t MAX_STRLEN = (1 << 15) - 1;
116  static constexpr size_t MAX_STRCOUNT = (1U << 31) - 1;
117 
118  private:
119  struct StringIdxEntry {
120  uint64_t off : 48;
121  uint64_t size : 16;
122  };
123 
124  // In the compare_cache_value_t index represents the index of the sorted cache.
125  // The diff component represents whether the index the cache is pointing to is equal to
126  // the pattern it is cached for. We want to use diff so we don't have compare string
127  // again when we are retrieving it from the cache.
129  int32_t index;
130  int32_t diff;
131  };
132 
133  struct PayloadString {
134  char* c_str_ptr;
135  size_t size;
136  bool canary;
137  };
138 
140  std::vector<std::future<std::vector<std::pair<uint32_t, unsigned int>>>>&
141  dictionary_futures);
142  bool fillRateIsHigh() const noexcept;
143  void increaseCapacity() noexcept;
144  int32_t getOrAddImpl(const std::string& str) noexcept;
145  template <class T>
146  void getOrAddBulkRemote(const std::vector<std::string>& string_vec, T* encoded_vec);
147  int32_t getUnlocked(const std::string& str) const noexcept;
148  std::string getStringUnlocked(int32_t string_id) const noexcept;
149  std::string getStringChecked(const int string_id) const noexcept;
150  std::pair<char*, size_t> getStringBytesChecked(const int string_id) const noexcept;
151  uint32_t computeBucket(const uint32_t hash,
152  const std::string str,
153  const std::vector<int32_t>& data,
154  const bool unique) const noexcept;
155  uint32_t computeUniqueBucketWithHash(const uint32_t hash,
156  const std::vector<int32_t>& data) const noexcept;
157  void appendToStorage(const std::string& str) noexcept;
158  PayloadString getStringFromStorage(const int string_id) const noexcept;
159  void addPayloadCapacity() noexcept;
160  void addOffsetCapacity() noexcept;
161  size_t addStorageCapacity(int fd) noexcept;
162  void* addMemoryCapacity(void* addr, size_t& mem_size) noexcept;
163  void invalidateInvertedIndex() noexcept;
164  std::vector<int32_t> getEquals(std::string pattern,
165  std::string comp_operator,
166  size_t generation);
167  void buildSortedCache();
168  void insertInSortedCache(std::string str, int32_t str_id);
169  void sortCache(std::vector<int32_t>& cache);
170  void mergeSortedCache(std::vector<int32_t>& temp_sorted_cache);
171  compare_cache_value_t* binary_search_cache(const std::string& pattern) const;
172 
173  size_t str_count_;
174  std::vector<int32_t> str_ids_;
175  std::vector<uint32_t> rk_hashes_;
176  std::vector<int32_t> sorted_cache;
177  bool isTemp_;
179  std::string offsets_path_;
188  mutable std::map<std::tuple<std::string, bool, bool, char>, std::vector<int32_t>>
190  mutable std::map<std::pair<std::string, char>, std::vector<int32_t>> regex_cache_;
191  mutable std::map<std::string, int32_t> equal_cache_;
193  mutable std::shared_ptr<std::vector<std::string>> strings_cache_;
194  std::unique_ptr<StringDictionaryClient> client_;
196 
197  static char* CANARY_BUFFER;
198 };
199 
200 int32_t truncate_to_generation(const int32_t id, const size_t generation);
201 
202 void translate_string_ids(std::vector<int32_t>& dest_ids,
203  const LeafHostInfo& dict_server_host,
204  const DictRef dest_dict_ref,
205  const std::vector<int32_t>& source_ids,
206  const DictRef source_dict_ref,
207  const int32_t dest_generation);
208 
209 #endif // STRINGDICTIONARY_STRINGDICTIONARY_H
StringIdxEntry * offset_map_
void * addMemoryCapacity(void *addr, size_t &mem_size) noexcept
std::vector< int32_t > getRegexpLike(const std::string &pattern, const char escape, const size_t generation) const
uint64_t off
std::pair< char *, size_t > getStringBytesChecked(const int string_id) const noexcept
uint64_t size
size_t storageEntryCount() const
std::string getStringChecked(const int string_id) const noexcept
DictionaryCache< std::string, compare_cache_value_t > compare_cache_
static void populate_string_ids(std::vector< int32_t > &dest_ids, StringDictionary *dest_dict, const std::vector< int32_t > &source_ids, const StringDictionary *source_dict, const std::map< int32_t, std::string > transient_mapping={})
Populates provided dest_ids vector with string ids corresponding to given source strings.
std::string offsets_path_
int32_t getIdOfString(const std::string &str) const
int32_t getOrAddImpl(const std::string &str) noexcept
int32_t getOrAdd(const std::string &str) noexcept
int32_t getUnlocked(const std::string &str) const noexcept
std::map< std::pair< std::string, char >, std::vector< int32_t > > regex_cache_
void increaseCapacity() noexcept
std::unique_ptr< StringDictionaryClient > client_
std::string getStringUnlocked(int32_t string_id) const noexcept
StringDictionary(const std::string &folder, const bool isTemp, const bool recover, const bool materializeHashes=false, size_t initial_capacity=256)
static constexpr size_t MAX_STRCOUNT
std::vector< int32_t > getEquals(std::string pattern, std::string comp_operator, size_t generation)
static constexpr int32_t INVALID_STR_ID
std::shared_ptr< std::vector< std::string > > strings_cache_
std::vector< int32_t > getCompare(const std::string &pattern, const std::string &comp_operator, const size_t generation)
std::shared_timed_mutex mapd_shared_mutex
void addPayloadCapacity() noexcept
mapd_shared_mutex rw_mutex_
void getOrAddBulk(const std::vector< std::string > &string_vec, T *encoded_vec)
size_t addStorageCapacity(int fd) noexcept
std::map< std::string, int32_t > equal_cache_
void translate_string_ids(std::vector< int32_t > &dest_ids, const LeafHostInfo &dict_server_host, const DictRef dest_dict_ref, const std::vector< int32_t > &source_ids, const DictRef source_dict_ref, const int32_t dest_generation)
void addOffsetCapacity() noexcept
std::pair< char *, size_t > getStringBytes(int32_t string_id) const noexcept
DictPayloadUnavailable(const std::string &err)
void getOrAddBulkArray(const std::vector< std::vector< std::string >> &string_array_vec, std::vector< std::vector< int32_t >> &ids_array_vec)
bool checkpoint() noexcept
std::vector< int32_t > getLike(const std::string &pattern, const bool icase, const bool is_simple, const char escape, const size_t generation) const
static char * CANARY_BUFFER
void mergeSortedCache(std::vector< int32_t > &temp_sorted_cache)
void processDictionaryFutures(std::vector< std::future< std::vector< std::pair< uint32_t, unsigned int >>>> &dictionary_futures)
std::string getString(int32_t string_id) const
std::unique_ptr< StringDictionaryClient > client_no_timeout_
uint32_t computeUniqueBucketWithHash(const uint32_t hash, const std::vector< int32_t > &data) const noexcept
void getOrAddBulkRemote(const std::vector< std::string > &string_vec, T *encoded_vec)
static void populate_string_array_ids(std::vector< std::vector< int32_t >> &dest_array_ids, StringDictionary *dest_dict, const std::vector< std::vector< int32_t >> &source_array_ids, const StringDictionary *source_dict)
void invalidateInvertedIndex() noexcept
std::vector< int32_t > str_ids_
void sortCache(std::vector< int32_t > &cache)
static constexpr size_t MAX_STRLEN
void appendToStorage(const std::string &str) noexcept
uint32_t computeBucket(const uint32_t hash, const std::string str, const std::vector< int32_t > &data, const bool unique) const noexcept
std::map< std::tuple< std::string, bool, bool, char >, std::vector< int32_t > > like_cache_
PayloadString getStringFromStorage(const int string_id) const noexcept
compare_cache_value_t * binary_search_cache(const std::string &pattern) const
void insertInSortedCache(std::string str, int32_t str_id)
std::vector< int32_t > sorted_cache
int32_t truncate_to_generation(const int32_t id, const size_t generation)
~StringDictionary() noexcept
std::shared_ptr< const std::vector< std::string > > copyStrings() const
std::vector< uint32_t > rk_hashes_
bool fillRateIsHigh() const noexcept