OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Fragmenter.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef FRAGMENTER_H
18 #define FRAGMENTER_H
19 
20 #include <deque>
21 #include <list>
22 #include <map>
23 #include <mutex>
24 #include "../Catalog/ColumnDescriptor.h"
25 #include "../DataMgr/ChunkMetadata.h"
26 #include "../Shared/mapd_shared_mutex.h"
27 #include "../Shared/types.h"
28 
29 namespace Data_Namespace {
30 class AbstractBuffer;
31 }
32 
33 class ResultSet;
34 
35 namespace Fragmenter_Namespace {
36 class InsertOrderFragmenter;
37 
45  INSERT_ORDER = 0 // these values persist in catalog. make explicit
46 };
47 
59 struct InsertData {
60  int databaseId;
61  int tableId;
62  std::vector<int> columnIds;
63  size_t numRows;
64  std::vector<DataBlockPtr> data;
65  int64_t replicate_count =
67  0;
68  std::vector<bool> bypass; // bypass corresponding columnIds[]
69  std::map<int, const ColumnDescriptor*> columnDescriptors;
70 };
71 
79 class FragmentInfo {
80  public:
82  : fragmentId(-1)
83  , shadowNumTuples(0)
84  , physicalTableId(-1)
85  , shard(-1)
86  , mutex_access_inmem_states(new std::mutex)
88  , numTuples(0)
91 
92  void setChunkMetadataMap(const std::map<int, ChunkMetadata>& chunkMetadataMap) {
93  this->chunkMetadataMap = chunkMetadataMap;
94  }
95 
96  void setChunkMetadata(const int col, const ChunkMetadata& chunkMetadata) {
97  chunkMetadataMap[col] = chunkMetadata;
98  }
99 
100  const std::map<int, ChunkMetadata>& getChunkMetadataMap() const;
101 
102  const std::map<int, ChunkMetadata>& getChunkMetadataMapPhysical() const {
103  return chunkMetadataMap;
104  }
105 
106  size_t getNumTuples() const;
107 
108  size_t getPhysicalNumTuples() const { return numTuples; }
109 
110  bool isEmptyPhysicalFragment() const { return physicalTableId >= 0 && !numTuples; }
111 
112  void setPhysicalNumTuples(const size_t physNumTuples) { numTuples = physNumTuples; }
113 
116 
117  // for unit tests
118  static void setUnconditionalVacuum(const double unconditionalVacuum) {
119  unconditionalVacuum_ = unconditionalVacuum;
120  }
121 
124  std::vector<int> deviceIds;
126  int shard;
127  std::map<int, ChunkMetadata> shadowChunkMetadataMap;
128  mutable std::shared_ptr<std::mutex> mutex_access_inmem_states;
130  mutable std::shared_ptr<std::mutex> resultSetMutex;
131 
132  private:
133  mutable size_t numTuples;
134  mutable std::map<int, ChunkMetadata> chunkMetadataMap;
137 
138  friend class InsertOrderFragmenter;
139  mutable std::shared_ptr<std::mutex> updateMutex_{new std::mutex};
140  static bool unconditionalVacuum_;
141 };
142 
152 class TableInfo {
153  public:
155 
156  size_t getNumTuples() const;
157 
158  size_t getNumTuplesUpperBound() const;
159 
160  size_t getPhysicalNumTuples() const { return numTuples; }
161 
162  void setPhysicalNumTuples(const size_t physNumTuples) { numTuples = physNumTuples; }
163 
164  size_t getFragmentNumTuplesUpperBound() const;
165 
166  std::vector<int> chunkKeyPrefix;
167  std::deque<FragmentInfo> fragments;
168 
169  private:
170  mutable size_t numTuples;
171 };
172 
173 } // namespace Fragmenter_Namespace
174 
175 #endif // FRAGMENTER_H
static void setUnconditionalVacuum(const double unconditionalVacuum)
Definition: Fragmenter.h:118
size_t getPhysicalNumTuples() const
Definition: Fragmenter.h:108
std::map< int, ChunkMetadata > shadowChunkMetadataMap
Definition: Fragmenter.h:127
void setChunkMetadataMap(const std::map< int, ChunkMetadata > &chunkMetadataMap)
Definition: Fragmenter.h:92
std::deque< FragmentInfo > fragments
Definition: Fragmenter.h:167
std::shared_ptr< std::mutex > resultSetMutex
Definition: Fragmenter.h:130
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
std::vector< int > chunkKeyPrefix
Definition: Fragmenter.h:166
std::map< int, const ColumnDescriptor * > columnDescriptors
Definition: Fragmenter.h:69
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:61
size_t getPhysicalNumTuples() const
Definition: Fragmenter.h:160
size_t numRows
a vector of column ids for the row(s) being inserted
Definition: Fragmenter.h:63
std::vector< bool > bypass
count to replicate values of column(s); used only for ALTER ADD column
Definition: Fragmenter.h:68
Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(r...
Definition: Fragmenter.h:79
std::shared_ptr< std::mutex > mutex_access_inmem_states
Definition: Fragmenter.h:128
size_t getFragmentNumTuplesUpperBound() const
std::shared_ptr< std::mutex > updateMutex_
Definition: Fragmenter.h:139
const int32_t groups_buffer_size return nullptr
std::map< int, ChunkMetadata > chunkMetadataMap
Definition: Fragmenter.h:134
void invalidateChunkMetadataMap() const
Definition: Fragmenter.h:114
const std::map< int, ChunkMetadata > & getChunkMetadataMap() const
std::vector< DataBlockPtr > data
the number of rows being inserted
Definition: Fragmenter.h:64
const std::map< int, ChunkMetadata > & getChunkMetadataMapPhysical() const
Definition: Fragmenter.h:102
void setChunkMetadata(const int col, const ChunkMetadata &chunkMetadata)
Definition: Fragmenter.h:96
bool g_enable_watchdog false
Definition: Execute.cpp:71
void setPhysicalNumTuples(const size_t physNumTuples)
Definition: Fragmenter.h:162
void setPhysicalNumTuples(const size_t physNumTuples)
Definition: Fragmenter.h:112
The data to be inserted using the fragment manager.
Definition: Fragmenter.h:59
std::vector< int > columnIds
identifies the table into which the data is being inserted
Definition: Fragmenter.h:62