OmniSciDB  c07336695a
StringNoneEncoder Class Reference

#include <StringNoneEncoder.h>

+ Inheritance diagram for StringNoneEncoder:
+ Collaboration diagram for StringNoneEncoder:

Public Member Functions

 StringNoneEncoder (AbstractBuffer *buffer)
 
size_t getNumElemsForBytesInsertData (const std::vector< std::string > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)
 
ChunkMetadata appendData (int8_t *&srcData, const size_t numAppendElems, const SQLTypeInfo &, const bool replicating=false) override
 
ChunkMetadata appendData (const std::vector< std::string > *srcData, const int start_idx, const size_t numAppendElems, const bool replicating=false)
 
void getMetadata (ChunkMetadata &chunkMetadata) override
 
ChunkMetadata getMetadata (const SQLTypeInfo &ti) override
 
void updateStats (const int64_t, const bool) override
 
void updateStats (const double, const bool) override
 
void reduceStats (const Encoder &) override
 
void writeMetadata (FILE *f) override
 
void readMetadata (FILE *f) override
 
void copyMetadata (const Encoder *copyFromEncoder) override
 
AbstractBufferget_index_buf () const
 
void set_index_buf (AbstractBuffer *buf)
 
- Public Member Functions inherited from Encoder
 Encoder (Data_Namespace::AbstractBuffer *buffer)
 
virtual ~Encoder ()
 
virtual bool resetChunkStats (const ChunkStats &)
 : Reset chunk level stats (min, max, nulls) using new values from the argument. More...
 
size_t getNumElems () const
 
void setNumElems (const size_t num_elems)
 

Private Attributes

AbstractBufferindex_buf
 
StringOffsetT last_offset
 
bool has_nulls
 

Additional Inherited Members

- Static Public Member Functions inherited from Encoder
static EncoderCreate (Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
 
- Protected Attributes inherited from Encoder
size_t num_elems_
 
Data_Namespace::AbstractBufferbuffer_
 
DecimalOverflowValidator decimal_overflow_validator_
 
DateDaysOverflowValidator date_days_overflow_validator_
 

Detailed Description

Definition at line 37 of file StringNoneEncoder.h.

Constructor & Destructor Documentation

◆ StringNoneEncoder()

StringNoneEncoder::StringNoneEncoder ( AbstractBuffer buffer)
inline

Definition at line 39 of file StringNoneEncoder.h.

References getNumElemsForBytesInsertData().

40  : Encoder(buffer), index_buf(nullptr), last_offset(-1), has_nulls(false) {}
AbstractBuffer * index_buf
Encoder(Data_Namespace::AbstractBuffer *buffer)
Definition: Encoder.cpp:221
StringOffsetT last_offset
+ Here is the call graph for this function:

Member Function Documentation

◆ appendData() [1/2]

ChunkMetadata StringNoneEncoder::appendData ( int8_t *&  srcData,
const size_t  numAppendElems,
const SQLTypeInfo ,
const bool  replicating = false 
)
inlineoverridevirtual

Implements Encoder.

Definition at line 48 of file StringNoneEncoder.h.

References CHECK.

Referenced by Chunk_NS::Chunk::appendData().

51  {
52  CHECK(false); // should never be called for strings
53  return ChunkMetadata{};
54  }
#define CHECK(condition)
Definition: Logger.h:187
+ Here is the caller graph for this function:

◆ appendData() [2/2]

ChunkMetadata StringNoneEncoder::appendData ( const std::vector< std::string > *  srcData,
const int  start_idx,
const size_t  numAppendElems,
const bool  replicating = false 
)

Definition at line 51 of file StringNoneEncoder.cpp.

References Data_Namespace::AbstractBuffer::append(), Encoder::buffer_, Data_Namespace::CPU_LEVEL, run-benchmark-import::dest, getMetadata(), has_nulls, index_buf, Data_Namespace::AbstractBuffer::isDirty(), last_offset, MAX_INPUT_BUF_SIZE, Encoder::num_elems_, Data_Namespace::AbstractBuffer::read(), Data_Namespace::AbstractBuffer::reserve(), Data_Namespace::AbstractBuffer::setDirty(), and Data_Namespace::AbstractBuffer::size().

54  {
55  assert(index_buf != nullptr); // index_buf must be set before this.
56  size_t index_size = numAppendElems * sizeof(StringOffsetT);
57  if (num_elems_ == 0) {
58  index_size += sizeof(StringOffsetT); // plus one for the initial offset of 0.
59  }
60  index_buf->reserve(index_size);
61  StringOffsetT offset = 0;
62  if (num_elems_ == 0) {
63  index_buf->append((int8_t*)&offset,
64  sizeof(StringOffsetT)); // write the inital 0 offset
65  last_offset = 0;
66  } else {
67  // always need to read a valid last offset from buffer/disk
68  // b/c now due to vacuum "last offset" may go backward and if
69  // index chunk was not reloaded last_offset would go way off!
70  index_buf->read((int8_t*)&last_offset,
71  sizeof(StringOffsetT),
72  index_buf->size() - sizeof(StringOffsetT),
74  assert(last_offset >= 0);
75  }
76  size_t data_size = 0;
77  for (size_t n = start_idx; n < start_idx + numAppendElems; n++) {
78  size_t len = (*srcData)[replicating ? 0 : n].length();
79  data_size += len;
80  }
81  buffer_->reserve(data_size);
82 
83  size_t inbuf_size =
84  std::min(std::max(index_size, data_size), (size_t)MAX_INPUT_BUF_SIZE);
85  auto inbuf = new int8_t[inbuf_size];
86  std::unique_ptr<int8_t[]> gc_inbuf(inbuf);
87  for (size_t num_appended = 0; num_appended < numAppendElems;) {
88  StringOffsetT* p = (StringOffsetT*)inbuf;
89  size_t i;
90  for (i = 0; num_appended < numAppendElems && i < inbuf_size / sizeof(StringOffsetT);
91  i++, num_appended++) {
92  p[i] =
93  last_offset + (*srcData)[replicating ? 0 : num_appended + start_idx].length();
94  last_offset = p[i];
95  }
96  index_buf->append(inbuf, i * sizeof(StringOffsetT));
97  }
98 
99  for (size_t num_appended = 0; num_appended < numAppendElems;) {
100  size_t size = 0;
101  for (int i = start_idx + num_appended;
102  num_appended < numAppendElems && size < inbuf_size;
103  i++, num_appended++) {
104  size_t len = (*srcData)[replicating ? 0 : i].length();
105  if (len > inbuf_size) {
106  // for large strings, append on its own
107  if (size > 0) {
108  buffer_->append(inbuf, size);
109  }
110  size = 0;
111  buffer_->append((int8_t*)(*srcData)[replicating ? 0 : i].data(), len);
112  num_appended++;
113  break;
114  } else if (size + len > inbuf_size) {
115  break;
116  }
117  char* dest = (char*)inbuf + size;
118  if (len > 0) {
119  (*srcData)[replicating ? 0 : i].copy(dest, len);
120  size += len;
121  } else {
122  has_nulls = true;
123  }
124  }
125  if (size > 0) {
126  buffer_->append(inbuf, size);
127  }
128  }
129  // make sure buffer_ is flushed even if no new data is appended to it
130  // (e.g. empty strings) because the metadata needs to be flushed.
131  if (!buffer_->isDirty()) {
132  buffer_->setDirty();
133  }
134 
135  num_elems_ += numAppendElems;
136  ChunkMetadata chunkMetadata;
137  getMetadata(chunkMetadata);
138  return chunkMetadata;
139 }
size_t num_elems_
Definition: Encoder.h:179
#define MAX_INPUT_BUF_SIZE
Definition: Encoder.h:36
virtual size_t size() const =0
virtual void reserve(size_t numBytes)=0
int32_t StringOffsetT
Definition: sqltypes.h:873
AbstractBuffer * index_buf
Data_Namespace::AbstractBuffer * buffer_
Definition: Encoder.h:181
virtual void append(int8_t *src, const size_t numBytes, const MemoryLevel srcBufferType=CPU_LEVEL, const int deviceId=-1)=0
StringOffsetT last_offset
virtual void read(int8_t *const dst, const size_t numBytes, const size_t offset=0, const MemoryLevel dstBufferType=CPU_LEVEL, const int dstDeviceId=-1)=0
void getMetadata(ChunkMetadata &chunkMetadata) override
virtual bool isDirty() const
+ Here is the call graph for this function:

◆ copyMetadata()

void StringNoneEncoder::copyMetadata ( const Encoder copyFromEncoder)
inlineoverridevirtual

Implements Encoder.

Definition at line 96 of file StringNoneEncoder.h.

References Encoder::getNumElems(), has_nulls, and Encoder::num_elems_.

96  {
97  num_elems_ = copyFromEncoder->getNumElems();
98  has_nulls = static_cast<const StringNoneEncoder*>(copyFromEncoder)->has_nulls;
99  }
size_t num_elems_
Definition: Encoder.h:179
size_t getNumElems() const
Definition: Encoder.h:175
+ Here is the call graph for this function:

◆ get_index_buf()

AbstractBuffer* StringNoneEncoder::get_index_buf ( ) const
inline

Definition at line 101 of file StringNoneEncoder.h.

References index_buf.

101 { return index_buf; }
AbstractBuffer * index_buf

◆ getMetadata() [1/2]

void StringNoneEncoder::getMetadata ( ChunkMetadata chunkMetadata)
inlineoverridevirtual

Reimplemented from Encoder.

Definition at line 61 of file StringNoneEncoder.h.

References ChunkMetadata::chunkStats, Encoder::getMetadata(), ChunkStats::has_nulls, has_nulls, ChunkStats::max, ChunkStats::min, and Datum::stringval.

Referenced by appendData().

61  {
62  Encoder::getMetadata(chunkMetadata); // call on parent class
63  chunkMetadata.chunkStats.min.stringval = nullptr;
64  chunkMetadata.chunkStats.max.stringval = nullptr;
65  chunkMetadata.chunkStats.has_nulls = has_nulls;
66  }
bool has_nulls
Definition: ChunkMetadata.h:28
ChunkStats chunkStats
Definition: ChunkMetadata.h:35
virtual void getMetadata(ChunkMetadata &chunkMetadata)
Definition: Encoder.cpp:227
std::string * stringval
Definition: sqltypes.h:131
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadata() [2/2]

ChunkMetadata StringNoneEncoder::getMetadata ( const SQLTypeInfo ti)
inlineoverridevirtual

Implements Encoder.

Definition at line 69 of file StringNoneEncoder.h.

References has_nulls, ChunkStats::min, and Datum::stringval.

69  {
70  auto chunk_stats = ChunkStats{};
71  chunk_stats.min.stringval = nullptr;
72  chunk_stats.max.stringval = nullptr;
73  chunk_stats.has_nulls = has_nulls;
74  ChunkMetadata chunk_metadata{ti, 0, 0, chunk_stats};
75  return chunk_metadata;
76  }
std::string * stringval
Definition: sqltypes.h:131

◆ getNumElemsForBytesInsertData()

size_t StringNoneEncoder::getNumElemsForBytesInsertData ( const std::vector< std::string > *  srcData,
const int  start_idx,
const size_t  numAppendElems,
const size_t  byteLimit,
const bool  replicating = false 
)

Definition at line 33 of file StringNoneEncoder.cpp.

Referenced by Chunk_NS::Chunk::getNumElemsForBytesInsertData(), and StringNoneEncoder().

38  {
39  size_t dataSize = 0;
40  size_t n = start_idx;
41  for (; n < start_idx + numAppendElems; n++) {
42  size_t len = (*srcData)[replicating ? 0 : n].length();
43  if (dataSize + len > byteLimit) {
44  break;
45  }
46  dataSize += len;
47  }
48  return n - start_idx;
49 }
+ Here is the caller graph for this function:

◆ readMetadata()

void StringNoneEncoder::readMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 90 of file StringNoneEncoder.h.

References CHECK_NE, has_nulls, and Encoder::num_elems_.

90  {
91  // assumes pointer is already in right place
92  CHECK_NE(fread((int8_t*)&num_elems_, sizeof(size_t), size_t(1), f), size_t(0));
93  CHECK_NE(fread((int8_t*)&has_nulls, sizeof(bool), size_t(1), f), size_t(0));
94  }
size_t num_elems_
Definition: Encoder.h:179
#define CHECK_NE(x, y)
Definition: Logger.h:196

◆ reduceStats()

void StringNoneEncoder::reduceStats ( const Encoder )
inlineoverridevirtual

Implements Encoder.

Definition at line 82 of file StringNoneEncoder.h.

References CHECK.

82 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:187

◆ set_index_buf()

void StringNoneEncoder::set_index_buf ( AbstractBuffer buf)
inline

Definition at line 102 of file StringNoneEncoder.h.

References index_buf.

Referenced by Chunk_NS::Chunk::getChunkBuffer(), and Chunk_NS::Chunk::init_encoder().

102 { index_buf = buf; }
AbstractBuffer * index_buf
+ Here is the caller graph for this function:

◆ updateStats() [1/2]

void StringNoneEncoder::updateStats ( const int64_t  ,
const bool   
)
inlineoverridevirtual

Implements Encoder.

Definition at line 78 of file StringNoneEncoder.h.

References CHECK.

78 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:187

◆ updateStats() [2/2]

void StringNoneEncoder::updateStats ( const double  ,
const bool   
)
inlineoverridevirtual

Implements Encoder.

Definition at line 80 of file StringNoneEncoder.h.

References CHECK.

80 { CHECK(false); }
#define CHECK(condition)
Definition: Logger.h:187

◆ writeMetadata()

void StringNoneEncoder::writeMetadata ( FILE *  f)
inlineoverridevirtual

Implements Encoder.

Definition at line 84 of file StringNoneEncoder.h.

References has_nulls, and Encoder::num_elems_.

84  {
85  // assumes pointer is already in right place
86  fwrite((int8_t*)&num_elems_, sizeof(size_t), 1, f);
87  fwrite((int8_t*)&has_nulls, sizeof(bool), 1, f);
88  }
size_t num_elems_
Definition: Encoder.h:179

Member Data Documentation

◆ has_nulls

bool StringNoneEncoder::has_nulls
private

◆ index_buf

AbstractBuffer* StringNoneEncoder::index_buf
private

Definition at line 105 of file StringNoneEncoder.h.

Referenced by appendData(), get_index_buf(), and set_index_buf().

◆ last_offset

StringOffsetT StringNoneEncoder::last_offset
private

Definition at line 106 of file StringNoneEncoder.h.

Referenced by appendData().


The documentation for this class was generated from the following files: