OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
FsiChunkUtils.h
Go to the documentation of this file.
1 /*
2  * Copyright 2021 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <map>
20 
21 #include "Catalog/CatalogFwd.h"
22 #include "DataMgr/Chunk/Chunk.h"
23 #include "DataMgr/ChunkMetadata.h"
24 #include "Shared/distributed.h"
25 
26 namespace foreign_storage {
28  const ChunkKey& chunk_key,
29  const std::map<ChunkKey, std::shared_ptr<ChunkMetadata>>& chunk_metadata_map,
30  const std::map<ChunkKey, AbstractBuffer*>& buffers,
31  Chunk_NS::Chunk& chunk);
32 
33 // Construct default metadata for given column descriptor with num_elements
34 std::shared_ptr<ChunkMetadata> get_placeholder_metadata(const ColumnDescriptor* column,
35  size_t num_elements);
36 /*
37  Splits up a set of items to be processed into multiple partitions, with the intention
38  that each thread will process a separate part.
39  */
40 // TODO(Misiu): Change this to return a list of Views/Ranges when we support c++20.
41 template <typename T>
42 auto partition_for_threads(const std::set<T>& items, size_t max_threads) {
43  const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
44  std::list<std::set<T>> items_by_thread;
45  auto i = 0U;
46  for (auto item : items) {
47  if (i++ % items_per_thread == 0) {
48  items_by_thread.emplace_back(std::set<T>{});
49  }
50  items_by_thread.back().emplace(item);
51  }
52  return items_by_thread;
53 }
54 
55 /*
56  Splits up a vector of items to be processed into multiple partitions, with the intention
57  that each thread will process a separate part.
58  */
59 // TODO: refactor partition_for_threads to use Requires when we support c++20
60 template <typename T>
61 auto partition_for_threads(const std::vector<T>& items, size_t max_threads) {
62  const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
63  std::list<std::vector<T>> items_by_thread;
64  auto i = 0U;
65  for (auto item : items) {
66  if (i++ % items_per_thread == 0) {
67  items_by_thread.emplace_back(std::vector<T>{});
68  }
69  items_by_thread.back().emplace_back(item);
70  }
71  return items_by_thread;
72 }
73 
74 template <typename Container>
75 std::vector<std::future<void>> create_futures_for_workers(
76  const Container& items,
77  size_t max_threads,
78  std::function<void(const Container&)> lambda) {
79  auto items_per_thread = partition_for_threads(items, max_threads);
80  std::vector<std::future<void>> futures;
81  for (const auto& items : items_per_thread) {
82  futures.emplace_back(std::async(std::launch::async, lambda, items));
83  }
84 
85  return futures;
86 }
87 
89 
90 bool is_system_table_chunk_key(const ChunkKey& chunk_key);
91 
92 bool is_replicated_table_chunk_key(const ChunkKey& chunk_key);
93 
94 bool is_append_table_chunk_key(const ChunkKey& chunk_key);
95 
96 bool is_shardable_key(const ChunkKey& key);
97 
98 bool fragment_maps_to_leaf(const ChunkKey& key);
99 
100 bool key_does_not_shard_to_leaf(const ChunkKey& key);
101 } // namespace foreign_storage
std::vector< int > ChunkKey
Definition: types.h:37
bool is_system_table_chunk_key(const ChunkKey &chunk_key)
auto partition_for_threads(const std::set< T > &items, size_t max_threads)
Definition: FsiChunkUtils.h:42
bool is_append_table_chunk_key(const ChunkKey &chunk_key)
bool is_replicated_table_chunk_key(const ChunkKey &chunk_key)
bool key_does_not_shard_to_leaf(const ChunkKey &key)
future< Result > async(Fn &&fn, Args &&...args)
std::vector< std::future< void > > create_futures_for_workers(const Container &items, size_t max_threads, std::function< void(const Container &)> lambda)
Definition: FsiChunkUtils.h:75
std::shared_ptr< ChunkMetadata > get_placeholder_metadata(const ColumnDescriptor *column, size_t num_elements)
void init_chunk_for_column(const ChunkKey &chunk_key, const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map, const std::map< ChunkKey, AbstractBuffer * > &buffers, Chunk_NS::Chunk &chunk)
specifies the content in-memory of a row in the column metadata table
bool fragment_maps_to_leaf(const ChunkKey &key)
bool is_shardable_key(const ChunkKey &key)
const foreign_storage::ForeignTable & get_foreign_table_for_key(const ChunkKey &key)