34 const std::vector<std::string>* srcData,
36 const size_t numAppendElems,
37 const size_t byteLimit,
38 const bool replicating) {
41 for (; n < start_idx + numAppendElems; n++) {
42 size_t len = (*srcData)[replicating ? 0 : n].length();
43 if (dataSize + len > byteLimit) {
52 const std::vector<std::string>* srcData,
54 const size_t numAppendElems,
55 const bool replicating) {
78 for (
size_t n = start_idx; n < start_idx + numAppendElems; n++) {
79 size_t len = (*srcData)[replicating ? 0 : n].length();
86 auto inbuf = std::make_unique<int8_t[]>(inbuf_size);
87 for (
size_t num_appended = 0; num_appended < numAppendElems;) {
90 for (i = 0; num_appended < numAppendElems && i < inbuf_size /
sizeof(
StringOffsetT);
91 i++, num_appended++) {
93 last_offset + (*srcData)[replicating ? 0 : num_appended + start_idx].length();
99 for (
size_t num_appended = 0; num_appended < numAppendElems;) {
101 for (
int i = start_idx + num_appended;
102 num_appended < numAppendElems && size < inbuf_size;
103 i++, num_appended++) {
104 size_t len = (*srcData)[replicating ? 0 : i].length();
105 if (len > inbuf_size) {
111 buffer_->
append((int8_t*)(*srcData)[replicating ? 0 : i].data(), len);
114 }
else if (size + len > inbuf_size) {
117 char*
dest =
reinterpret_cast<char*
>(inbuf.get()) + size;
119 (*srcData)[replicating ? 0 : i].copy(dest, len);
135 auto chunk_metadata = std::make_shared<ChunkMetadata>();
137 return chunk_metadata;
141 const size_t start_idx,
142 const size_t num_elements) {
143 for (
size_t n = start_idx; n < start_idx + num_elements; n++) {
void updateStats(const int64_t, const bool) override
#define MAX_INPUT_BUF_SIZE
std::shared_ptr< ChunkMetadata > appendData(int8_t *&src_data, const size_t num_elems_to_append, const SQLTypeInfo &ti, const bool replicating=false, const int64_t offset=-1) override
size_t getNumElemsForBytesInsertData(const std::vector< std::string > *srcData, const int start_idx, const size_t numAppendElems, const size_t byteLimit, const bool replicating=false)
virtual void read(int8_t *const dst, const size_t num_bytes, const size_t offset=0, const MemoryLevel dst_buffer_type=CPU_LEVEL, const int dst_device_id=-1)=0
AbstractBuffer * index_buf
Data_Namespace::AbstractBuffer * buffer_
An AbstractBuffer is a unit of data management for a data manager.
void update_elem_stats(const std::string &elem)
StringOffsetT last_offset
void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata) override
virtual void append(int8_t *src, const size_t num_bytes, const MemoryLevel src_buffer_type=CPU_LEVEL, const int device_id=-1)=0
virtual void reserve(size_t num_bytes)=0