OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
S3Archive Class Reference

#include <S3Archive.h>

+ Inheritance diagram for S3Archive:
+ Collaboration diagram for S3Archive:

Public Member Functions

 S3Archive (const std::string &url, const bool plain_text)
 
 S3Archive (const std::string &url, const std::string &s3_access_key, const std::string &s3_secret_key, const std::string &s3_session_token, const std::string &s3_region, const std::string &s3_endpoint, const bool plain_text, const std::optional< std::string > &regex_path_filter, const std::optional< std::string > &file_sort_order_by, const std::optional< std::string > &file_sort_regex, const std::string &s3_temp_dir_path={})
 
 ~S3Archive () override
 
void init_for_read () override
 
const std::vector< std::string > & get_objkeys ()
 
const std::string land (const std::string &objkey, std::exception_ptr &teptr, const bool for_detection)
 
void vacuum (const std::string &objkey)
 
size_t get_total_file_size () const
 
- Public Member Functions inherited from Archive
 Archive (const std::string url, const bool plain_text)
 
virtual ~Archive ()
 
virtual std::string archive_error (int err)
 
virtual bool read_next_header ()
 
virtual bool read_data_block (const void **buff, size_t *size, int64_t *offset)
 
virtual int64_t get_position_compressed () const
 
virtual int open ()
 
virtual int close ()
 
virtual ptrdiff_t read (const void **buff)
 
const std::string url_part (const int i)
 
std::string entryName ()
 

Private Attributes

std::string s3_access_key
 
std::string s3_secret_key
 
std::string s3_session_token
 
std::string s3_region
 
std::string s3_endpoint
 
std::string s3_temp_dir
 
std::string bucket_name
 
std::string prefix_name
 
std::optional< std::string > regex_path_filter
 
std::optional< std::string > file_sort_order_by
 
std::optional< std::string > file_sort_regex
 
std::vector< std::string > objkeys
 
std::map< const std::string,
const std::string > 
file_paths
 
size_t total_file_size {0}
 

Additional Inherited Members

- Static Public Member Functions inherited from Archive
static ptrdiff_t read (struct archive *a, void *client_data, const void **buff)
 
static int open (struct archive *a, void *client_data)
 
static int close (struct archive *a, void *client_data)
 
static void parse_url (const std::string url, std::map< int, std::string > &url_parts)
 
- Protected Attributes inherited from Archive
std::string url
 
std::map< int, std::string > url_parts
 
archive * ar = 0
 
archive_entry * entry
 
bool plain_text
 

Detailed Description

Definition at line 41 of file S3Archive.h.

Constructor & Destructor Documentation

S3Archive::S3Archive ( const std::string &  url,
const bool  plain_text 
)
inline

Definition at line 43 of file S3Archive.h.

References s3_access_key, s3_endpoint, s3_region, s3_secret_key, and s3_session_token.

43  : Archive(url, plain_text) {
44  // these envs are on server side so are global settings
45  // which make few senses in case of private s3 resources
46  char* env;
47  if (0 != (env = getenv("AWS_REGION"))) {
48  s3_region = env;
49  }
50  if (0 != (env = getenv("AWS_ACCESS_KEY_ID"))) {
51  s3_access_key = env;
52  }
53  if (0 != (env = getenv("AWS_SECRET_ACCESS_KEY"))) {
54  s3_secret_key = env;
55  }
56  if (0 != (env = getenv("AWS_SESSION_TOKEN"))) {
57  s3_session_token = env;
58  }
59 
60  if (0 != (env = getenv("AWS_ENDPOINT"))) {
61  s3_endpoint = env;
62  }
63  }
std::string s3_endpoint
Definition: S3Archive.h:150
std::string s3_region
Definition: S3Archive.h:149
std::string s3_access_key
Definition: S3Archive.h:146
std::string s3_session_token
Definition: S3Archive.h:148
std::string url
Definition: Archive.h:202
std::string s3_secret_key
Definition: S3Archive.h:147
bool plain_text
Definition: Archive.h:206
Archive(const std::string url, const bool plain_text)
Definition: Archive.h:34
S3Archive::S3Archive ( const std::string &  url,
const std::string &  s3_access_key,
const std::string &  s3_secret_key,
const std::string &  s3_session_token,
const std::string &  s3_region,
const std::string &  s3_endpoint,
const bool  plain_text,
const std::optional< std::string > &  regex_path_filter,
const std::optional< std::string > &  file_sort_order_by,
const std::optional< std::string > &  file_sort_regex,
const std::string &  s3_temp_dir_path = {} 
)
inline

Definition at line 65 of file S3Archive.h.

75  {})
80  this->s3_region = s3_region;
81  this->s3_endpoint = s3_endpoint;
85 
86  if (s3_temp_dir_path.empty()) {
87  // this must be local to heavydb not client
88  // or posix dir path accessible to heavydb
89  auto env_s3_temp_dir = getenv("TMPDIR");
90  s3_temp_dir = env_s3_temp_dir ? env_s3_temp_dir : "/tmp";
91  } else {
92  s3_temp_dir = s3_temp_dir_path;
93  }
94  }
std::string s3_endpoint
Definition: S3Archive.h:150
std::string s3_region
Definition: S3Archive.h:149
std::string s3_access_key
Definition: S3Archive.h:146
std::optional< std::string > file_sort_regex
Definition: S3Archive.h:157
std::optional< std::string > regex_path_filter
Definition: S3Archive.h:155
std::optional< std::string > file_sort_order_by
Definition: S3Archive.h:156
std::string s3_session_token
Definition: S3Archive.h:148
std::string s3_temp_dir
Definition: S3Archive.h:151
std::string url
Definition: Archive.h:202
std::string s3_secret_key
Definition: S3Archive.h:147
S3Archive(const std::string &url, const bool plain_text)
Definition: S3Archive.h:43
bool plain_text
Definition: Archive.h:206
S3Archive::~S3Archive ( )
inlineoverride

Definition at line 96 of file S3Archive.h.

96  {
97 #ifdef HAVE_AWS_S3
98  for (auto& thread : threads) {
99  if (thread.joinable()) {
100  thread.join();
101  }
102  }
103 #endif // HAVE_AWS_S3
104  }

Member Function Documentation

const std::vector<std::string>& S3Archive::get_objkeys ( )
inline

Definition at line 113 of file S3Archive.h.

References objkeys.

113  {
114  return objkeys;
115  }
std::vector< std::string > objkeys
Definition: S3Archive.h:158
size_t S3Archive::get_total_file_size ( ) const
inline

Definition at line 133 of file S3Archive.h.

References total_file_size.

133  {
134  return total_file_size;
135  }
size_t total_file_size
Definition: S3Archive.h:160
void S3Archive::init_for_read ( )
inlineoverridevirtual

Reimplemented from Archive.

Definition at line 109 of file S3Archive.h.

109  {
110  throw std::runtime_error("AWS S3 support not available");
111  }
const std::string S3Archive::land ( const std::string &  objkey,
std::exception_ptr &  teptr,
const bool  for_detection 
)
inline

Definition at line 124 of file S3Archive.h.

126  {
127  throw std::runtime_error("AWS S3 support not available");
128  }
void S3Archive::vacuum ( const std::string &  objkey)
inline

Definition at line 129 of file S3Archive.h.

129  {
130  throw std::runtime_error("AWS S3 support not available");
131  }

Member Data Documentation

std::string S3Archive::bucket_name
private

Definition at line 153 of file S3Archive.h.

std::map<const std::string, const std::string> S3Archive::file_paths
private

Definition at line 159 of file S3Archive.h.

std::optional<std::string> S3Archive::file_sort_order_by
private

Definition at line 156 of file S3Archive.h.

std::optional<std::string> S3Archive::file_sort_regex
private

Definition at line 157 of file S3Archive.h.

std::vector<std::string> S3Archive::objkeys
private

Definition at line 158 of file S3Archive.h.

Referenced by get_objkeys().

std::string S3Archive::prefix_name
private

Definition at line 154 of file S3Archive.h.

std::optional<std::string> S3Archive::regex_path_filter
private

Definition at line 155 of file S3Archive.h.

std::string S3Archive::s3_access_key
private
std::string S3Archive::s3_endpoint
private
std::string S3Archive::s3_region
private
std::string S3Archive::s3_secret_key
private
std::string S3Archive::s3_session_token
private
std::string S3Archive::s3_temp_dir
private

Definition at line 151 of file S3Archive.h.

size_t S3Archive::total_file_size {0}
private

Definition at line 160 of file S3Archive.h.

Referenced by get_total_file_size().


The documentation for this class was generated from the following files: