17 #ifndef STRINGDICTIONARY_STRINGDICTIONARY_H
18 #define STRINGDICTIONARY_STRINGDICTIONARY_H
27 #include <shared_mutex>
29 #include <string_view>
37 namespace StringOps_Namespace {
57 const std::string& folder,
60 const bool materializeHashes =
false,
61 size_t initial_capacity = 256);
65 int32_t
getDbId()
const noexcept;
71 virtual void operator()(std::string
const&, int32_t
const string_id) = 0;
72 virtual void operator()(std::string_view
const, int32_t
const string_id) = 0;
82 int32_t
getOrAdd(
const std::string& str) noexcept;
83 template <
class T,
class String>
84 size_t getBulk(
const std::vector<String>& string_vec,
T* encoded_vec)
const;
85 template <
class T,
class String>
86 size_t getBulk(
const std::vector<String>& string_vec,
88 const int64_t generation)
const;
89 template <
class T,
class String>
90 void getOrAddBulk(
const std::vector<String>& string_vec,
T* encoded_vec);
91 template <
class T,
class String>
93 template <
class String>
95 std::vector<std::vector<int32_t>>& ids_array_vec);
96 template <
class String>
98 std::string
getString(int32_t string_id)
const;
99 std::pair<char*, size_t>
getStringBytes(int32_t string_id)
const noexcept;
102 std::vector<int32_t>
getLike(
const std::string& pattern,
104 const bool is_simple,
106 const size_t generation)
const;
108 std::vector<int32_t>
getCompare(
const std::string& pattern,
109 const std::string& comp_operator,
110 const size_t generation);
112 std::vector<int32_t>
getRegexpLike(
const std::string& pattern,
114 const size_t generation)
const;
119 std::vector<std::string_view>
getStringViews(
const size_t generation)
const;
122 const std::shared_ptr<StringDictionary> dest_dict,
127 int32_t* translated_ids,
128 const int64_t source_generation,
129 const int64_t dest_generation,
130 const bool dest_has_transients,
132 const std::vector<StringOps_Namespace::StringOpInfo>& string_op_infos)
const;
156 std::vector<int32_t>& dest_ids,
158 const std::vector<int32_t>& source_ids,
160 const std::vector<std::
string const*>& transient_string_vec = {});
163 std::vector<std::vector<int32_t>>& dest_array_ids,
165 const std::vector<std::vector<int32_t>>& source_array_ids,
196 std::vector<std::future<std::vector<std::pair<string_dict_hash_t, unsigned int>>>>&
201 template <class String>
203 const
size_t str_count,
204 const
size_t storage_high_water_mark,
205 const std::vector<String>& input_strings,
206 const std::vector<
size_t>& string_memory_ids,
208 int32_t
getOrAddImpl(const std::string_view& str) noexcept;
209 template <class String>
210 void hashStrings(const std::vector<String>& string_vec,
213 int32_t
getUnlocked(const std::string_view sv) const noexcept;
217 template <class String>
220 const String& input_string,
221 const std::vector<int32_t>& string_id_string_dict_hash_table) const noexcept;
222 template <class String>
225 const String& input_string,
226 const std::vector<int32_t>& string_id_string_dict_hash_table,
227 const
size_t storage_high_water_mark,
228 const std::vector<String>& input_strings,
229 const std::vector<
size_t>& string_memory_ids) const noexcept;
232 const std::vector<int32_t>& string_id_string_dict_hash_table) noexcept;
236 template <class String>
238 template <class String>
240 const std::vector<
size_t>& string_memory_ids,
241 const
size_t sum_new_strings_lengths) noexcept;
249 const
size_t min_capacity_requested = 0) noexcept;
251 std::vector<int32_t>
getEquals(std::
string pattern,
252 std::
string comp_operator,
256 void sortCache(std::vector<int32_t>& cache);
278 mutable std::map<std::tuple<std::
string,
bool,
bool,
char>, std::vector<int32_t>>
280 mutable std::map<std::pair<std::
string,
char>, std::vector<int32_t>>
regex_cache_;
296 const std::vector<int32_t>& source_ids,
298 const int32_t dest_generation);
300 #endif // STRINGDICTIONARY_STRINGDICTIONARY_H
StringIdxEntry * offset_map_
size_t payload_file_size_
bool isClient() const noexcept
void increaseHashTableCapacity() noexcept
void checkAndConditionallyIncreasePayloadCapacity(const size_t write_length)
size_t addStorageCapacity(int fd, const size_t min_capacity_requested=0) noexcept
std::vector< int32_t > getRegexpLike(const std::string &pattern, const char escape, const size_t generation) const
size_t getBulk(const std::vector< String > &string_vec, T *encoded_vec) const
std::vector< int32_t > buildDictionaryTranslationMap(const std::shared_ptr< StringDictionary > dest_dict, StringLookupCallback const &dest_transient_lookup_callback) const
std::vector< std::string > copyStrings() const
std::pair< char *, size_t > getStringBytesChecked(const int string_id) const noexcept
size_t storageEntryCount() const
void addOffsetCapacity(const size_t min_capacity_requested=0) noexcept
uint32_t computeBucketFromStorageAndMemory(const string_dict_hash_t input_string_hash, const String &input_string, const std::vector< int32_t > &string_id_string_dict_hash_table, const size_t storage_high_water_mark, const std::vector< String > &input_strings, const std::vector< size_t > &string_memory_ids) const noexcept
std::string getStringChecked(const int string_id) const noexcept
size_t canary_buffer_size
std::vector< string_dict_hash_t > hash_cache_
DictionaryCache< std::string, compare_cache_value_t > compare_cache_
bool fillRateIsHigh(const size_t num_strings) const noexcept
void * addMemoryCapacity(void *addr, size_t &mem_size, const size_t min_capacity_requested=0) noexcept
std::string offsets_path_
static void populate_string_ids(std::vector< int32_t > &dest_ids, StringDictionary *dest_dict, const std::vector< int32_t > &source_ids, const StringDictionary *source_dict, const std::vector< std::string const * > &transient_string_vec={})
Populates provided dest_ids vector with string ids corresponding to given source strings.
std::shared_mutex rw_mutex_
std::string_view getStringFromStorageFast(const int string_id) const noexcept
int32_t getOrAdd(const std::string &str) noexcept
std::map< std::pair< std::string, char >, std::vector< int32_t > > regex_cache_
std::unique_ptr< StringDictionaryClient > client_
std::string getStringUnlocked(int32_t string_id) const noexcept
int32_t getIdOfString(const String &) const
static constexpr size_t MAX_STRCOUNT
std::vector< int32_t > getEquals(std::string pattern, std::string comp_operator, size_t generation)
uint32_t computeBucket(const string_dict_hash_t hash, const String &input_string, const std::vector< int32_t > &string_id_string_dict_hash_table) const noexcept
static constexpr int32_t INVALID_STR_ID
std::shared_ptr< std::vector< std::string > > strings_cache_
virtual void operator()(std::string const &, int32_t const string_id)=0
std::vector< int32_t > getCompare(const std::string &pattern, const std::string &comp_operator, const size_t generation)
StringDictionary(const DictRef &dict_ref, const std::string &folder, const bool isTemp, const bool recover, const bool materializeHashes=false, size_t initial_capacity=256)
const std::string folder_
void appendToStorageBulk(const std::vector< String > &input_strings, const std::vector< size_t > &string_memory_ids, const size_t sum_new_strings_lengths) noexcept
void addPayloadCapacity(const size_t min_capacity_requested=0) noexcept
std::map< std::string, int32_t > equal_cache_
uint32_t computeUniqueBucketWithHash(const string_dict_hash_t hash, const std::vector< int32_t > &string_id_string_dict_hash_table) noexcept
void getOrAddBulkArray(const std::vector< std::vector< String >> &string_array_vec, std::vector< std::vector< int32_t >> &ids_array_vec)
void translate_string_ids(std::vector< int32_t > &dest_ids, const LeafHostInfo &dict_server_host, const DictRef dest_dict_ref, const std::vector< int32_t > &source_ids, const DictRef source_dict_ref, const int32_t dest_generation)
int32_t getUnlocked(const std::string_view sv) const noexcept
void appendToStorage(const String str) noexcept
void getOrAddBulk(const std::vector< String > &string_vec, T *encoded_vec)
std::pair< char *, size_t > getStringBytes(int32_t string_id) const noexcept
DictPayloadUnavailable(const std::string &err)
void processDictionaryFutures(std::vector< std::future< std::vector< std::pair< string_dict_hash_t, unsigned int >>>> &dictionary_futures)
bool checkpoint() noexcept
std::vector< int32_t > getLike(const std::string &pattern, const bool icase, const bool is_simple, const char escape, const size_t generation) const
void mergeSortedCache(std::vector< int32_t > &temp_sorted_cache)
void eachStringSerially(int64_t const generation, StringCallback &) const
int32_t getDictId() const noexcept
size_t getNumStringsFromStorage(const size_t storage_slots) const noexcept
void update_leaf(const LeafHostInfo &host_info)
std::string getString(int32_t string_id) const
void hashStrings(const std::vector< String > &string_vec, std::vector< string_dict_hash_t > &hashes) const noexcept
std::vector< std::string_view > getStringViews() const
int32_t getDbId() const noexcept
std::unique_ptr< StringDictionaryClient > client_no_timeout_
virtual ~StringCallback()=default
void increaseHashTableCapacityFromStorageAndMemory(const size_t str_count, const size_t storage_high_water_mark, const std::vector< String > &input_strings, const std::vector< size_t > &string_memory_ids, const std::vector< string_dict_hash_t > &input_strings_hashes) noexcept
std::function< int32_t(std::string const &)> makeLambdaStringToId() const
void checkAndConditionallyIncreaseOffsetCapacity(const size_t write_length)
static void populate_string_array_ids(std::vector< std::vector< int32_t >> &dest_array_ids, StringDictionary *dest_dict, const std::vector< std::vector< int32_t >> &source_array_ids, const StringDictionary *source_dict)
void invalidateInvertedIndex() noexcept
uint32_t string_dict_hash_t
std::vector< int32_t > string_id_string_dict_hash_table_
void sortCache(std::vector< int32_t > &cache)
static constexpr size_t MAX_STRLEN
void getOrAddBulkParallel(const std::vector< String > &string_vec, T *encoded_vec)
std::function< bool(std::string_view, int32_t string_id)> StringLookupCallback
std::map< std::tuple< std::string, bool, bool, char >, std::vector< int32_t > > like_cache_
bool g_enable_stringdict_parallel
PayloadString getStringFromStorage(const int string_id) const noexcept
std::shared_timed_mutex shared_mutex
compare_cache_value_t * binary_search_cache(const std::string &pattern) const
void insertInSortedCache(std::string str, int32_t str_id)
std::vector< int32_t > sorted_cache
int32_t getOrAddImpl(const std::string_view &str) noexcept
int32_t truncate_to_generation(const int32_t id, const size_t generation)
~StringDictionary() noexcept