AbstractFragmenter.h

Go to the documentation of this file.

/*
 * Copyright 2022 HEAVY.AI, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#pragma once

#include "Fragmenter/Fragmenter.h"

#include <boost/variant.hpp>
#include <string>
#include <vector>

#include "QueryEngine/TargetMetaInfo.h"
#include "QueryEngine/TargetValue.h"
#include "Shared/UpdelRoll.h"
#include "Shared/sqltypes.h"
#include "StringDictionary/StringDictionaryProxy.h"

// Should the ColumnInfo and FragmentInfo structs be in
// AbstractFragmenter?

class Executor;

namespace Chunk_NS {
class Chunk;
};

namespace Data_Namespace {
class AbstractBuffer;
class AbstractDataMgr;
}  // namespace Data_Namespace

namespace import_export {
class TypedImportBuffer;
}

namespace Catalog_Namespace {
class Catalog;
}
struct TableDescriptor;
struct ColumnDescriptor;

namespace Fragmenter_Namespace {

class RowDataProvider {
 public:
  virtual size_t const getRowCount() const = 0;
  virtual size_t const getEntryCount() const = 0;
  virtual StringDictionaryProxy* getLiteralDictionary() const = 0;
  virtual std::vector<TargetValue> getEntryAt(const size_t index) const = 0;
  virtual std::vector<TargetValue> getTranslatedEntryAt(const size_t index) const = 0;
};

struct UpdateValuesStats {
  bool has_null{false};
  double max_double{std::numeric_limits<double>::lowest()};
  double min_double{std::numeric_limits<double>::max()};
  int64_t max_int64t{std::numeric_limits<int64_t>::min()};
  int64_t min_int64t{std::numeric_limits<int64_t>::max()};
};

/*
 * @type ChunkUpdateStats
 * @brief struct containing stats from a column chunk update.
 * `new_values_stats` represents aggregate stats for the new
 * values that were put into the chunk. `old_values_stats`
 * represents aggregate stats for chunk values that were
 * replaced.
 */
struct ChunkUpdateStats {
  UpdateValuesStats new_values_stats;
  UpdateValuesStats old_values_stats;
  int64_t updated_rows_count{0};
  int64_t fragment_rows_count{0};
  std::shared_ptr<Chunk_NS::Chunk> chunk;
};

/*
 * @type AbstractFragmenter
 * @brief abstract base class for all table partitioners
 *
 * The virtual methods of this class provide an interface
 * for an interface for getting the id and type of a
 * partitioner, inserting data into a partitioner, and
 * getting the partitions (fragments) managed by a
 * partitioner that must be queried given a predicate
 */

class AbstractFragmenter {
 public:
  virtual ~AbstractFragmenter() {}

  // virtual void getFragmentsForQuery(QueryInfo &queryInfo, const void *predicate = 0) =
  // 0

  virtual size_t getNumFragments() = 0;

  virtual TableInfo getFragmentsForQuery() = 0;

  virtual void insertData(InsertData& insert_data_struct) = 0;

  virtual void insertChunks(const InsertChunks& insert_chunk) = 0;

  virtual void insertDataNoCheckpoint(InsertData& insert_data_struct) = 0;

  virtual void insertChunksNoCheckpoint(const InsertChunks& insert_chunk) = 0;

  virtual void dropFragmentsToSize(const size_t maxRows) = 0;

  virtual void updateChunkStats(
      const ColumnDescriptor* cd,
      std::unordered_map</*fragment_id*/ int, ChunkStats>& stats_map,
      std::optional<Data_Namespace::MemoryLevel> memory_level) = 0;

  virtual FragmentInfo* getFragmentInfo(const int fragment_id) const = 0;

  virtual int getFragmenterId() = 0;

  virtual std::string getFragmenterType() = 0;

  virtual size_t getNumRows() = 0;
  virtual void setNumRows(const size_t numTuples) = 0;

  virtual std::optional<ChunkUpdateStats> updateColumn(
      const Catalog_Namespace::Catalog* catalog,
      const TableDescriptor* td,
      const ColumnDescriptor* cd,
      const int fragment_id,
      const std::vector<uint64_t>& frag_offsets,
      const std::vector<ScalarTargetValue>& rhs_values,
      const SQLTypeInfo& rhs_type,
      const Data_Namespace::MemoryLevel memory_level,
      UpdelRoll& updel_roll) = 0;

  virtual void updateColumns(const Catalog_Namespace::Catalog* catalog,
                             const TableDescriptor* td,
                             const int fragmentId,
                             const std::vector<TargetMetaInfo> sourceMetaInfo,
                             const std::vector<const ColumnDescriptor*> columnDescriptors,
                             const RowDataProvider& sourceDataProvider,
                             const size_t indexOffFragmentOffsetColumn,
                             const Data_Namespace::MemoryLevel memoryLevel,
                             UpdelRoll& updelRoll,
                             Executor* executor) = 0;

  virtual void updateColumn(const Catalog_Namespace::Catalog* catalog,
                            const TableDescriptor* td,
                            const ColumnDescriptor* cd,
                            const int fragment_id,
                            const std::vector<uint64_t>& frag_offsets,
                            const ScalarTargetValue& rhs_value,
                            const SQLTypeInfo& rhs_type,
                            const Data_Namespace::MemoryLevel memory_level,
                            UpdelRoll& updel_roll) = 0;

  virtual void updateColumnMetadata(const ColumnDescriptor* cd,
                                    FragmentInfo& fragment,
                                    std::shared_ptr<Chunk_NS::Chunk> chunk,
                                    const UpdateValuesStats& update_values_stats,
                                    const SQLTypeInfo& rhs_type,
                                    UpdelRoll& updel_roll) = 0;

  virtual void updateMetadata(const Catalog_Namespace::Catalog* catalog,
                              const MetaDataKey& key,
                              UpdelRoll& updel_roll) = 0;

  virtual void compactRows(const Catalog_Namespace::Catalog* catalog,
                           const TableDescriptor* td,
                           const int fragmentId,
                           const std::vector<uint64_t>& fragOffsets,
                           const Data_Namespace::MemoryLevel memoryLevel,
                           UpdelRoll& updelRoll) = 0;

  virtual const std::vector<uint64_t> getVacuumOffsets(
      const std::shared_ptr<Chunk_NS::Chunk>& chunk) = 0;

  virtual void dropColumns(const std::vector<int>& columnIds) = 0;

  virtual bool hasDeletedRows(const int delete_column_id) = 0;

  virtual void updateColumnChunkMetadata(
      const ColumnDescriptor* cd,
      const int fragment_id,
      const std::shared_ptr<ChunkMetadata> metadata) = 0;

  virtual void resetSizesFromFragments() = 0;
};

}  // namespace Fragmenter_Namespace