OmniSciDB  d2f719934e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
S3Archive Class Reference

#include <S3Archive.h>

+ Inheritance diagram for S3Archive:
+ Collaboration diagram for S3Archive:

Public Member Functions

 S3Archive (const std::string &url, const bool plain_text)
 
 S3Archive (const std::string &url, const std::string &s3_access_key, const std::string &s3_secret_key, const std::string &s3_session_token, const std::string &s3_region, const std::string &s3_endpoint, const bool plain_text, const std::optional< std::string > &regex_path_filter, const std::optional< std::string > &file_sort_order_by, const std::optional< std::string > &file_sort_regex)
 
 ~S3Archive () override
 
void init_for_read () override
 
const std::vector< std::string > & get_objkeys ()
 
const std::string land (const std::string &objkey, std::exception_ptr &teptr, const bool for_detection)
 
void vacuum (const std::string &objkey)
 
size_t get_total_file_size () const
 
- Public Member Functions inherited from Archive
 Archive (const std::string url, const bool plain_text)
 
virtual ~Archive ()
 
virtual std::string archive_error (int err)
 
virtual bool read_next_header ()
 
virtual bool read_data_block (const void **buff, size_t *size, int64_t *offset)
 
virtual int64_t get_position_compressed () const
 
virtual int open ()
 
virtual int close ()
 
virtual ptrdiff_t read (const void **buff)
 
const std::string url_part (const int i)
 
std::string entryName ()
 

Private Attributes

std::string s3_access_key
 
std::string s3_secret_key
 
std::string s3_session_token
 
std::string s3_region
 
std::string s3_endpoint
 
std::string s3_temp_dir
 
std::string bucket_name
 
std::string prefix_name
 
std::optional< std::string > regex_path_filter
 
std::optional< std::string > file_sort_order_by
 
std::optional< std::string > file_sort_regex
 
std::vector< std::string > objkeys
 
std::map< const std::string,
const std::string > 
file_paths
 
size_t total_file_size {0}
 

Additional Inherited Members

- Static Public Member Functions inherited from Archive
static ptrdiff_t read (struct archive *a, void *client_data, const void **buff)
 
static int open (struct archive *a, void *client_data)
 
static int close (struct archive *a, void *client_data)
 
static void parse_url (const std::string url, std::map< int, std::string > &url_parts)
 
- Protected Attributes inherited from Archive
std::string url
 
std::map< int, std::string > url_parts
 
archive * ar = 0
 
archive_entry * entry
 
bool plain_text
 

Detailed Description

Definition at line 41 of file S3Archive.h.

Constructor & Destructor Documentation

S3Archive::S3Archive ( const std::string &  url,
const bool  plain_text 
)
inline

Definition at line 43 of file S3Archive.h.

References s3_access_key, s3_endpoint, s3_region, s3_secret_key, and s3_session_token.

43  : Archive(url, plain_text) {
44  // these envs are on server side so are global settings
45  // which make few senses in case of private s3 resources
46  char* env;
47  if (0 != (env = getenv("AWS_REGION"))) {
48  s3_region = env;
49  }
50  if (0 != (env = getenv("AWS_ACCESS_KEY_ID"))) {
51  s3_access_key = env;
52  }
53  if (0 != (env = getenv("AWS_SECRET_ACCESS_KEY"))) {
54  s3_secret_key = env;
55  }
56  if (0 != (env = getenv("AWS_SESSION_TOKEN"))) {
57  s3_session_token = env;
58  }
59 
60  if (0 != (env = getenv("AWS_ENDPOINT"))) {
61  s3_endpoint = env;
62  }
63  }
std::string s3_endpoint
Definition: S3Archive.h:139
std::string s3_region
Definition: S3Archive.h:138
std::string s3_access_key
Definition: S3Archive.h:135
std::string s3_session_token
Definition: S3Archive.h:137
std::string url
Definition: Archive.h:201
std::string s3_secret_key
Definition: S3Archive.h:136
bool plain_text
Definition: Archive.h:205
Archive(const std::string url, const bool plain_text)
Definition: Archive.h:45
S3Archive::S3Archive ( const std::string &  url,
const std::string &  s3_access_key,
const std::string &  s3_secret_key,
const std::string &  s3_session_token,
const std::string &  s3_region,
const std::string &  s3_endpoint,
const bool  plain_text,
const std::optional< std::string > &  regex_path_filter,
const std::optional< std::string > &  file_sort_order_by,
const std::optional< std::string > &  file_sort_regex 
)
inline

Definition at line 65 of file S3Archive.h.

References file_sort_order_by, file_sort_regex, regex_path_filter, s3_access_key, s3_endpoint, s3_region, s3_secret_key, s3_session_token, and s3_temp_dir.

79  this->s3_region = s3_region;
80  this->s3_endpoint = s3_endpoint;
84 
85  // this must be local to omnisci_server not client
86  // or posix dir path accessible to omnisci_server
87  auto env_s3_temp_dir = getenv("TMPDIR");
88  s3_temp_dir = env_s3_temp_dir ? env_s3_temp_dir : "/tmp";
89  }
std::string s3_endpoint
Definition: S3Archive.h:139
std::string s3_region
Definition: S3Archive.h:138
std::string s3_access_key
Definition: S3Archive.h:135
std::optional< std::string > file_sort_regex
Definition: S3Archive.h:146
std::optional< std::string > regex_path_filter
Definition: S3Archive.h:144
std::optional< std::string > file_sort_order_by
Definition: S3Archive.h:145
std::string s3_session_token
Definition: S3Archive.h:137
std::string s3_temp_dir
Definition: S3Archive.h:140
std::string url
Definition: Archive.h:201
std::string s3_secret_key
Definition: S3Archive.h:136
S3Archive(const std::string &url, const bool plain_text)
Definition: S3Archive.h:43
bool plain_text
Definition: Archive.h:205
S3Archive::~S3Archive ( )
inlineoverride

Definition at line 91 of file S3Archive.h.

91  {
92 #ifdef HAVE_AWS_S3
93  for (auto& thread : threads) {
94  if (thread.joinable()) {
95  thread.join();
96  }
97  }
98 #endif // HAVE_AWS_S3
99  }

Member Function Documentation

const std::vector<std::string>& S3Archive::get_objkeys ( )
inline

Definition at line 108 of file S3Archive.h.

References objkeys.

108 { return objkeys; }
std::vector< std::string > objkeys
Definition: S3Archive.h:147
size_t S3Archive::get_total_file_size ( ) const
inline

Definition at line 124 of file S3Archive.h.

References total_file_size.

124 { return total_file_size; }
size_t total_file_size
Definition: S3Archive.h:149
void S3Archive::init_for_read ( )
inlineoverridevirtual

Reimplemented from Archive.

Definition at line 104 of file S3Archive.h.

104  {
105  throw std::runtime_error("AWS S3 support not available");
106  }
const std::string S3Archive::land ( const std::string &  objkey,
std::exception_ptr &  teptr,
const bool  for_detection 
)
inline

Definition at line 115 of file S3Archive.h.

117  {
118  throw std::runtime_error("AWS S3 support not available");
119  }
void S3Archive::vacuum ( const std::string &  objkey)
inline

Definition at line 120 of file S3Archive.h.

120  {
121  throw std::runtime_error("AWS S3 support not available");
122  }

Member Data Documentation

std::string S3Archive::bucket_name
private

Definition at line 142 of file S3Archive.h.

std::map<const std::string, const std::string> S3Archive::file_paths
private

Definition at line 148 of file S3Archive.h.

std::optional<std::string> S3Archive::file_sort_order_by
private

Definition at line 145 of file S3Archive.h.

Referenced by S3Archive().

std::optional<std::string> S3Archive::file_sort_regex
private

Definition at line 146 of file S3Archive.h.

Referenced by S3Archive().

std::vector<std::string> S3Archive::objkeys
private

Definition at line 147 of file S3Archive.h.

Referenced by get_objkeys().

std::string S3Archive::prefix_name
private

Definition at line 143 of file S3Archive.h.

std::optional<std::string> S3Archive::regex_path_filter
private

Definition at line 144 of file S3Archive.h.

Referenced by S3Archive().

std::string S3Archive::s3_access_key
private
std::string S3Archive::s3_endpoint
private
std::string S3Archive::s3_region
private
std::string S3Archive::s3_secret_key
private
std::string S3Archive::s3_session_token
private
std::string S3Archive::s3_temp_dir
private

Definition at line 140 of file S3Archive.h.

Referenced by S3Archive().

size_t S3Archive::total_file_size {0}
private

Definition at line 149 of file S3Archive.h.

Referenced by get_total_file_size().


The documentation for this class was generated from the following files: