OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ChunkIter.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * @file ChunkIter.cpp
19  * @author Wei Hong <wei@mapd.com>
20  */
21 
22 #include "ChunkIter.h"
23 
24 #include <cstdlib>
25 
26 DEVICE static void decompress(const SQLTypeInfo& ti,
27  int8_t* compressed,
29  Datum* datum) {
30  switch (ti.get_type()) {
31  case kSMALLINT:
32  result->length = sizeof(int16_t);
33  result->pointer = (int8_t*)&datum->smallintval;
34  switch (ti.get_compression()) {
35  case kENCODING_FIXED:
36  assert(ti.get_comp_param() == 8);
37  datum->smallintval = (int16_t) * (int8_t*)compressed;
38  break;
39  case kENCODING_RL:
40  case kENCODING_DIFF:
41  case kENCODING_SPARSE:
42  assert(false);
43  break;
44  default:
45  assert(false);
46  }
47  break;
48  case kINT:
49  result->length = sizeof(int32_t);
50  result->pointer = (int8_t*)&datum->intval;
51  switch (ti.get_compression()) {
52  case kENCODING_FIXED:
53  switch (ti.get_comp_param()) {
54  case 8:
55  datum->intval = (int32_t) * (int8_t*)compressed;
56  break;
57  case 16:
58  datum->intval = (int32_t) * (int16_t*)compressed;
59  break;
60  default:
61  assert(false);
62  }
63  break;
64  case kENCODING_RL:
65  case kENCODING_DIFF:
66  case kENCODING_SPARSE:
67  assert(false);
68  break;
69  default:
70  assert(false);
71  }
72  break;
73  case kBIGINT:
74  case kNUMERIC:
75  case kDECIMAL:
76  result->length = sizeof(int64_t);
77  result->pointer = (int8_t*)&datum->bigintval;
78  switch (ti.get_compression()) {
79  case kENCODING_FIXED:
80  switch (ti.get_comp_param()) {
81  case 8:
82  datum->bigintval = (int64_t) * (int8_t*)compressed;
83  break;
84  case 16:
85  datum->bigintval = (int64_t) * (int16_t*)compressed;
86  break;
87  case 32:
88  datum->bigintval = (int64_t) * (int32_t*)compressed;
89  break;
90  default:
91  assert(false);
92  }
93  break;
94  case kENCODING_RL:
95  case kENCODING_DIFF:
96  case kENCODING_SPARSE:
97  assert(false);
98  break;
99  default:
100  assert(false);
101  }
102  break;
103  case kTIME:
104  case kTIMESTAMP:
105  case kDATE:
106  switch (ti.get_compression()) {
107  case kENCODING_FIXED:
108  datum->bigintval = (int64_t) * (int32_t*)compressed;
109  break;
111  switch (ti.get_comp_param()) {
112  case 0:
113  case 32:
114  datum->bigintval = (int64_t) * (int32_t*)compressed;
115  break;
116  case 16:
117  datum->bigintval = (int64_t) * (int16_t*)compressed;
118  break;
119  default:
120  assert(false);
121  break;
122  }
123  break;
124  case kENCODING_RL:
125  case kENCODING_DIFF:
126  case kENCODING_DICT:
127  case kENCODING_SPARSE:
128  case kENCODING_NONE:
129  assert(false);
130  break;
131  default:
132  assert(false);
133  }
134  result->length = sizeof(int64_t);
135  result->pointer = (int8_t*)&datum->bigintval;
136  break;
137  default:
138  assert(false);
139  }
140  result->is_null = ti.is_null(*datum);
141 }
142 
144  it->current_pos = it->start_pos;
145 }
146 
148  bool uncompress,
150  bool* is_end) {
151  if (it->current_pos >= it->end_pos) {
152  *is_end = true;
153  result->length = 0;
154  result->pointer = NULL;
155  result->is_null = true;
156  return;
157  }
158  *is_end = false;
159 
160  if (it->skip_size > 0) {
161  // for fixed-size
162  if (uncompress && (it->type_info.get_compression() != kENCODING_NONE)) {
163  decompress(it->type_info, it->current_pos, result, &it->datum);
164  } else {
165  result->length = static_cast<size_t>(it->skip_size);
166  result->pointer = it->current_pos;
167  result->is_null = it->type_info.is_null(result->pointer);
168  }
169  it->current_pos += it->skip * it->skip_size;
170  } else {
171  StringOffsetT offset = *(StringOffsetT*)it->current_pos;
172  result->length = static_cast<size_t>(*((StringOffsetT*)it->current_pos + 1) - offset);
173  result->pointer = it->second_buf + offset;
174  // @TODO(wei) treat zero length as null for now
175  result->is_null = (result->length == 0);
176  it->current_pos += it->skip * sizeof(StringOffsetT);
177  }
178 }
179 
180 // @brief get nth element in Chunk. Does not change ChunkIter state
182  int n,
183  bool uncompress,
185  bool* is_end) {
186  if (static_cast<size_t>(n) >= it->num_elems || n < 0) {
187  *is_end = true;
188  result->length = 0;
189  result->pointer = NULL;
190  result->is_null = true;
191  return;
192  }
193  *is_end = false;
194 
195  if (it->skip_size > 0) {
196  // for fixed-size
197  int8_t* current_pos = it->start_pos + n * it->skip_size;
198  if (uncompress && (it->type_info.get_compression() != kENCODING_NONE)) {
199  decompress(it->type_info, current_pos, result, &it->datum);
200  } else {
201  result->length = static_cast<size_t>(it->skip_size);
202  result->pointer = current_pos;
203  result->is_null = it->type_info.is_null(result->pointer);
204  }
205  } else {
206  int8_t* current_pos = it->start_pos + n * sizeof(StringOffsetT);
207  StringOffsetT offset = *(StringOffsetT*)current_pos;
208  result->length = static_cast<size_t>(*((StringOffsetT*)current_pos + 1) - offset);
209  result->pointer = it->second_buf + offset;
210  // @TODO(wei) treat zero length as null for now
211  result->is_null = (result->length == 0);
212  }
213 }
214 
215 // @brief get nth element in Chunk. Does not change ChunkIter state
216 DEVICE void ChunkIter_get_nth(ChunkIter* it, int n, ArrayDatum* result, bool* is_end) {
217  if (static_cast<size_t>(n) >= it->num_elems || n < 0) {
218  *is_end = true;
219  result->length = 0;
220  result->pointer = NULL;
221  result->is_null = true;
222  return;
223  }
224  *is_end = false;
225 
226  if (it->skip_size > 0) {
227  // for fixed-size
228  int8_t* current_pos = it->start_pos + n * it->skip_size;
229  result->length = static_cast<size_t>(it->skip_size);
230  result->pointer = current_pos;
231  bool is_null = false;
232  if (!it->type_info.get_notnull()) {
233  // Nulls can only be recognized when iterating over a !notnull-typed chunk
234  is_null = it->type_info.is_null_fixlen_array(result->pointer, result->length);
235  }
236  result->is_null = is_null;
237  } else {
238  int8_t* current_pos = it->start_pos + n * sizeof(ArrayOffsetT);
239  int8_t* next_pos = current_pos + sizeof(ArrayOffsetT);
240  ArrayOffsetT offset = *(ArrayOffsetT*)current_pos;
241  ArrayOffsetT next_offset = *(ArrayOffsetT*)next_pos;
242  if (next_offset < 0) { // Encoded NULL array
243  result->length = 0;
244  result->pointer = NULL;
245  result->is_null = true;
246  } else {
247  if (offset < 0) {
248  offset = -offset; // Previous array may have been NULL, remove negativity
249  }
250  result->length = static_cast<size_t>(next_offset - offset);
251  result->pointer = it->second_buf + offset;
252  result->is_null = false;
253  }
254  }
255 }
256 
257 // @brief get nth varlen array element in Chunk. Does not change ChunkIter state
259  int n,
261  bool* is_end) {
262  *is_end = (static_cast<size_t>(n) >= it->num_elems || n < 0);
263 
264  if (!*is_end) {
265  int8_t* current_pos = it->start_pos + n * sizeof(ArrayOffsetT);
266  int8_t* next_pos = current_pos + sizeof(ArrayOffsetT);
267  ArrayOffsetT offset = *(ArrayOffsetT*)current_pos;
268  ArrayOffsetT next_offset = *(ArrayOffsetT*)next_pos;
269 
270  if (next_offset >= 0) {
271  // Previous array may have been NULL, remove offset negativity
272  if (offset < 0) {
273  offset = -offset;
274  }
275  result->length = static_cast<size_t>(next_offset - offset);
276  result->pointer = it->second_buf + offset;
277  result->is_null = false;
278  return;
279  }
280  }
281  // Encoded NULL array or out of bounds
282  result->length = 0;
283  result->pointer = NULL;
284  result->is_null = true;
285 }
286 
287 // @brief get nth varlen notnull array element in Chunk. Does not change ChunkIter state
289  int n,
291  bool* is_end) {
292  *is_end = (static_cast<size_t>(n) >= it->num_elems || n < 0);
293 
294  int8_t* current_pos = it->start_pos + n * sizeof(ArrayOffsetT);
295  int8_t* next_pos = current_pos + sizeof(ArrayOffsetT);
296  ArrayOffsetT offset = *(ArrayOffsetT*)current_pos;
297  ArrayOffsetT next_offset = *(ArrayOffsetT*)next_pos;
298 
299  result->length = static_cast<size_t>(next_offset - offset);
300  result->pointer = it->second_buf + offset;
301  result->is_null = false;
302 }
int8_t * start_pos
Definition: ChunkIter.h:33
int8_t * current_pos
Definition: ChunkIter.h:32
SQLTypeInfo type_info
Definition: ChunkIter.h:30
Definition: sqltypes.h:52
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
static DEVICE void decompress(const SQLTypeInfo &ti, int8_t *compressed, VarlenDatum *result, Datum *datum)
Definition: ChunkIter.cpp:26
Datum datum
Definition: ChunkIter.h:38
bool is_null
Definition: sqltypes.h:76
DEVICE void ChunkIter_get_nth(ChunkIter *it, int n, bool uncompress, VarlenDatum *result, bool *is_end)
Definition: ChunkIter.cpp:181
int32_t intval
Definition: sqltypes.h:128
int8_t * pointer
Definition: sqltypes.h:75
int32_t StringOffsetT
Definition: sqltypes.h:912
#define DEVICE
int8_t * end_pos
Definition: ChunkIter.h:34
size_t num_elems
Definition: ChunkIter.h:37
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
int64_t bigintval
Definition: sqltypes.h:129
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
int16_t smallintval
Definition: sqltypes.h:127
DEVICE void ChunkIter_get_nth_varlen(ChunkIter *it, int n, ArrayDatum *result, bool *is_end)
Definition: ChunkIter.cpp:258
int skip_size
Definition: ChunkIter.h:36
Definition: sqltypes.h:56
HOST DEVICE bool is_null(const Datum &d) const
Definition: sqltypes.h:564
int64_t const int32_t sz assert(dest)
void ChunkIter_reset(ChunkIter *it)
Definition: ChunkIter.cpp:143
bool is_null(const T &v, const SQLTypeInfo &t)
int32_t ArrayOffsetT
Definition: sqltypes.h:913
int8_t * second_buf
Definition: ChunkIter.h:31
DEVICE void ChunkIter_get_next(ChunkIter *it, bool uncompress, VarlenDatum *result, bool *is_end)
Definition: ChunkIter.cpp:147
int skip
Definition: ChunkIter.h:35
Definition: sqltypes.h:48
HOST DEVICE bool is_null_fixlen_array(const int8_t *val, int array_size) const
Definition: sqltypes.h:626
DEVICE void ChunkIter_get_nth_varlen_notnull(ChunkIter *it, int n, ArrayDatum *result, bool *is_end)
Definition: ChunkIter.cpp:288
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:122
size_t length
Definition: sqltypes.h:74