OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
com.mapd.tests.ForeignTableRefreshConcurrencyTest Class Reference
+ Collaboration diagram for com.mapd.tests.ForeignTableRefreshConcurrencyTest:

Public Member Functions

void runSqlAsUser (String sql, MapdTestClient user, String logPrefix) throws Exception
 
void testConcurrency () throws Exception
 

Static Public Member Functions

static String getTimeStamp (int sec_from_now)
 
static void main (String[] args) throws Exception
 

Static Package Attributes

static final Logger logger
 

Private Member Functions

Path getAbsolutePath (String path)
 
void runTest (String db, String userName, String userPassword, int num_foreign_manual_refresh_threads, int num_foreign_scheduled_refresh_threads, int num_table_join_threads, int num_runs) throws Exception
 
void createForeignTestTableScheduledRefresh (MapdTestClient dba, String foreign_table_name) throws Exception
 
void createForeignTestTableManualRefresh (MapdTestClient dba, String foreign_table_name) throws Exception
 
void createTestTable (MapdTestClient dba, String table_name, Path copy_from_path) throws Exception
 
void createTestTables (MapdTestClient dba, String table_name, Path copy_from_path) throws Exception
 

Detailed Description

Definition at line 30 of file ForeignTableRefreshConcurrencyTest.java.

Member Function Documentation

void com.mapd.tests.ForeignTableRefreshConcurrencyTest.createForeignTestTableManualRefresh ( MapdTestClient  dba,
String  foreign_table_name 
) throws Exception
inlineprivate

Definition at line 206 of file ForeignTableRefreshConcurrencyTest.java.

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.testConcurrency().

207  {
208  dba.runSql("CREATE FOREIGN TABLE " + foreign_table_name + " "
209  + "(b BOOLEAN, t TINYINT, s SMALLINT, i INTEGER, bi BIGINT, f FLOAT, "
210  + "dc DECIMAL(10, 5), tm TIME, tp TIMESTAMP, d DATE, txt TEXT, "
211  + "txt_2 TEXT ENCODING NONE) "
212  + "SERVER test_server WITH "
213  + "(file_path = 'scalar_types.csv', "
214  + "fragment_size = 2);");
215  }

+ Here is the caller graph for this function:

void com.mapd.tests.ForeignTableRefreshConcurrencyTest.createForeignTestTableScheduledRefresh ( MapdTestClient  dba,
String  foreign_table_name 
) throws Exception
inlineprivate

Definition at line 192 of file ForeignTableRefreshConcurrencyTest.java.

References com.mapd.tests.ForeignTableRefreshConcurrencyTest.getTimeStamp().

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.testConcurrency().

193  {
194  dba.runSql("CREATE FOREIGN TABLE " + foreign_table_name + " "
195  + "(b BOOLEAN, t TINYINT, s SMALLINT, i INTEGER, bi BIGINT, f FLOAT, "
196  + "dc DECIMAL(10, 5), tm TIME, tp TIMESTAMP, d DATE, txt TEXT, "
197  + "txt_2 TEXT ENCODING NONE) "
198  + "SERVER test_server WITH "
199  + "(file_path = 'scalar_types.csv', "
200  + "refresh_timing_type = 'scheduled', "
201  + "refresh_start_date_time = '" + getTimeStamp(1) + "', "
202  + "refresh_interval = '1S', "
203  + "fragment_size = 2);");
204  }

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void com.mapd.tests.ForeignTableRefreshConcurrencyTest.createTestTable ( MapdTestClient  dba,
String  table_name,
Path  copy_from_path 
) throws Exception
inlineprivate

Definition at line 217 of file ForeignTableRefreshConcurrencyTest.java.

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.createTestTables().

218  {
219  dba.runSql("CREATE TABLE " + table_name
220  + " (id INTEGER, str TEXT ENCODING DICT(32), x DOUBLE, y BIGINT) WITH (FRAGMENT_SIZE=1)");
221  dba.runSql("COPY " + table_name + " FROM '" + copy_from_path.toString()
222  + "' WITH (header='false');");
223  }

+ Here is the caller graph for this function:

void com.mapd.tests.ForeignTableRefreshConcurrencyTest.createTestTables ( MapdTestClient  dba,
String  table_name,
Path  copy_from_path 
) throws Exception
inlineprivate

Definition at line 225 of file ForeignTableRefreshConcurrencyTest.java.

References com.mapd.tests.ForeignTableRefreshConcurrencyTest.createTestTable().

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.testConcurrency().

226  {
227  createTestTable(dba, table_name + "_left", copy_from_path);
228  createTestTable(dba, table_name + "_right", copy_from_path);
229  }
void createTestTable(MapdTestClient dba, String table_name, Path copy_from_path)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Path com.mapd.tests.ForeignTableRefreshConcurrencyTest.getAbsolutePath ( String  path)
inlineprivate

Definition at line 48 of file ForeignTableRefreshConcurrencyTest.java.

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.testConcurrency().

48  {
49  Path path_obj = Paths.get(path).toAbsolutePath();
50  assert Files.exists(path_obj);
51  return path_obj;
52  }

+ Here is the caller graph for this function:

static String com.mapd.tests.ForeignTableRefreshConcurrencyTest.getTimeStamp ( int  sec_from_now)
inlinestatic

Definition at line 34 of file ForeignTableRefreshConcurrencyTest.java.

References omnisci.dtypes.Date.

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.createForeignTestTableScheduledRefresh().

34  {
35  Calendar cal = Calendar.getInstance();
36  cal.setTime(new Date());
37  cal.add(Calendar.SECOND, sec_from_now);
38  SimpleDateFormat date_format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
39  date_format.setTimeZone(TimeZone.getTimeZone("GMT"));
40  return date_format.format(cal.getTime());
41  }

+ Here is the caller graph for this function:

static void com.mapd.tests.ForeignTableRefreshConcurrencyTest.main ( String[]  args) throws Exception
inlinestatic

Definition at line 43 of file ForeignTableRefreshConcurrencyTest.java.

43  {
44  ForeignTableRefreshConcurrencyTest test = new ForeignTableRefreshConcurrencyTest();
45  test.testConcurrency();
46  }
void com.mapd.tests.ForeignTableRefreshConcurrencyTest.runSqlAsUser ( String  sql,
MapdTestClient  user,
String  logPrefix 
) throws Exception
inline

Definition at line 186 of file ForeignTableRefreshConcurrencyTest.java.

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.runTest().

187  {
188  logger.info(logPrefix + " " + sql);
189  user.runSql(sql);
190  }

+ Here is the caller graph for this function:

void com.mapd.tests.ForeignTableRefreshConcurrencyTest.runTest ( String  db,
String  userName,
String  userPassword,
int  num_foreign_manual_refresh_threads,
int  num_foreign_scheduled_refresh_threads,
int  num_table_join_threads,
int  num_runs 
) throws Exception
inlineprivate

Definition at line 60 of file ForeignTableRefreshConcurrencyTest.java.

References i, run, com.mapd.tests.ForeignTableRefreshConcurrencyTest.runSqlAsUser(), and t.

Referenced by com.mapd.tests.ForeignTableRefreshConcurrencyTest.testConcurrency().

66  {
67  ArrayList<Exception> exceptions = new ArrayList<Exception>();
68 
69  Thread[] foreign_table_manual_refresh_threads =
70  new Thread[num_foreign_manual_refresh_threads];
71  Thread[] foreign_table_scheduled_refresh_threads =
72  new Thread[num_foreign_scheduled_refresh_threads];
73  Thread[] table_join_threads = new Thread[num_table_join_threads];
74 
75  for (int i = 0; i < num_foreign_manual_refresh_threads; ++i) {
76  final int tid = i;
77  foreign_table_manual_refresh_threads[tid] = new Thread(new Runnable() {
78  @Override
79  public void run() {
80  final String thread_name = "F[" + tid + "]";
81  try {
82  logger.info("Starting manual foreign table refresh thread " + thread_name);
83  MapdTestClient user = MapdTestClient.getClient(
84  "localhost", 6274, db, userName, userPassword);
85  for (int irun = 0; irun < num_runs; ++irun) {
86  runSqlAsUser("SELECT * FROM test_foreign_table_manual_refresh_" + tid + ";",
87  user,
88  thread_name);
89  runSqlAsUser("REFRESH FOREIGN TABLES test_foreign_table_manual_refresh_"
90  + tid + ";",
91  user,
92  thread_name);
93  }
94  logger.info("Finished foreign table manual refresh " + thread_name);
95  } catch (Exception e) {
96  logger.error("Foreign table refresh " + thread_name
97  + " Caught Exception: " + e.getMessage(),
98  e);
99  exceptions.add(e);
100  }
101  }
102  });
103  foreign_table_manual_refresh_threads[tid].start();
104  }
105 
106  for (int i = 0; i < num_foreign_scheduled_refresh_threads; ++i) {
107  final int tid = i;
108  foreign_table_scheduled_refresh_threads[tid] = new Thread(new Runnable() {
109  @Override
110  public void run() {
111  final String thread_name = "S[" + tid + "]";
112  try {
113  logger.info("Starting scheduled foreign table refresh thread " + thread_name);
114  MapdTestClient user = MapdTestClient.getClient(
115  "localhost", 6274, db, userName, userPassword);
116  for (int irun = 0; irun < num_runs; ++irun) {
117  runSqlAsUser(
118  "SELECT * FROM test_foreign_table_scheduled_refresh_" + tid + ";",
119  user,
120  thread_name);
121  Thread.sleep(1000); // sleep 1s between queries to allow
122  // refreshes to occur, which occur at 1s intervals
123  }
124  logger.info("Finished foreign table scheduled refresh " + thread_name);
125  } catch (Exception e) {
126  logger.error("Foreign table scheduled refresh " + thread_name
127  + " Caught Exception: " + e.getMessage(),
128  e);
129  exceptions.add(e);
130  }
131  }
132  });
133  foreign_table_scheduled_refresh_threads[tid].start();
134  }
135 
136  for (int i = 0; i < num_table_join_threads; ++i) {
137  final int tid = i;
138  table_join_threads[tid] = new Thread(new Runnable() {
139  @Override
140  public void run() {
141  final String thread_name = "T[" + tid + "]";
142  try {
143  logger.info("Starting table join " + thread_name);
144  MapdTestClient user = MapdTestClient.getClient(
145  "localhost", 6274, db, userName, userPassword);
146  for (int irun = 0; irun < num_runs; ++irun) {
147  runSqlAsUser("SELECT * FROM test_table_" + tid
148  + "_left AS l JOIN test_table_" + tid
149  + "_right AS r ON l.id = r.id;",
150  user,
151  thread_name);
152  Thread.sleep(1000); // sleep 1s between queries to allow more
153  // contention between foreign table refreshes;
154  // especially in the case of scheduled refreshes
155  }
156  logger.info("Finished table join thread T[" + tid + "]");
157  } catch (Exception e) {
158  logger.error(
159  "Table join " + thread_name + " Caught Exception: " + e.getMessage(),
160  e);
161  exceptions.add(e);
162  }
163  }
164  });
165  table_join_threads[tid].start();
166  }
167 
168  for (Thread t : foreign_table_manual_refresh_threads) {
169  t.join();
170  }
171  for (Thread t : foreign_table_scheduled_refresh_threads) {
172  t.join();
173  }
174  for (Thread t : table_join_threads) {
175  t.join();
176  }
177 
178  for (Exception e : exceptions) {
179  if (null != e) {
180  logger.error("Exception: " + e.getMessage(), e);
181  throw e;
182  }
183  }
184  }
char * t
static bool run
void runSqlAsUser(String sql, MapdTestClient user, String logPrefix)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void com.mapd.tests.ForeignTableRefreshConcurrencyTest.testConcurrency ( ) throws Exception
inline

Definition at line 231 of file ForeignTableRefreshConcurrencyTest.java.

References com.mapd.tests.ForeignTableRefreshConcurrencyTest.createForeignTestTableManualRefresh(), com.mapd.tests.ForeignTableRefreshConcurrencyTest.createForeignTestTableScheduledRefresh(), com.mapd.tests.ForeignTableRefreshConcurrencyTest.createTestTables(), com.mapd.tests.ForeignTableRefreshConcurrencyTest.getAbsolutePath(), i, and com.mapd.tests.ForeignTableRefreshConcurrencyTest.runTest().

231  {
232  logger.info("ForeignTableRefreshConcurrencyTest()");
233 
234  MapdTestClient su = MapdTestClient.getClient(
235  "localhost", 6274, "omnisci", "admin", "HyperInteractive");
236 
237  // initialize
238  su.runSql("DROP DATABASE IF EXISTS db1;");
239  su.runSql("CREATE DATABASE db1;");
240 
241  // create tables for use
242  final int num_foreign_manual_refresh_threads = 2;
243  final int num_foreign_scheduled_refresh_threads = 2;
244  final int num_table_join_threads = 2;
245  Path table_import_path = getAbsolutePath(
246  "../java/utility/src/main/java/com/mapd/tests/data/simple_test.csv");
247  Path foreign_server_path = getAbsolutePath("../Tests/FsiDataFiles/");
248  MapdTestClient dba = MapdTestClient.getClient(
249  "localhost", 6274, "db1", "admin", "HyperInteractive");
250  dba.runSql("CREATE SERVER test_server "
251  + "FOREIGN DATA WRAPPER omnisci_csv WITH (storage_type = 'LOCAL_FILE', "
252  + "base_path = '" + foreign_server_path.toString() + "');");
253  for (int i = 0; i < num_foreign_manual_refresh_threads; ++i) {
254  createForeignTestTableManualRefresh(dba, "test_foreign_table_manual_refresh_" + i);
255  }
256  for (int i = 0; i < num_foreign_scheduled_refresh_threads; ++i) {
258  dba, "test_foreign_table_scheduled_refresh_" + i);
259  }
260  for (int i = 0; i < num_table_join_threads; ++i) {
261  createTestTables(dba, "test_table_" + i, table_import_path);
262  }
263 
264  runTest("db1",
265  "admin",
266  "HyperInteractive",
267  num_foreign_manual_refresh_threads,
268  num_foreign_scheduled_refresh_threads,
269  num_table_join_threads,
270  25);
271 
272  // cleanup
273  for (int i = 0; i < num_foreign_manual_refresh_threads; ++i) {
274  dba.runSql("DROP FOREIGN TABLE test_foreign_table_manual_refresh_" + i + ";");
275  }
276  for (int i = 0; i < num_foreign_manual_refresh_threads; ++i) {
277  dba.runSql("DROP FOREIGN TABLE test_foreign_table_scheduled_refresh_" + i + ";");
278  }
279  for (int i = 0; i < num_table_join_threads; ++i) {
280  dba.runSql("DROP TABLE test_table_" + i + "_left ;");
281  dba.runSql("DROP TABLE test_table_" + i + "_right ;");
282  }
283  dba.runSql("DROP SERVER test_server;");
284  su.runSql("DROP DATABASE db1;");
285 
286  logger.info("ForeignTableRefreshConcurrencyTest() done");
287  }
void createForeignTestTableScheduledRefresh(MapdTestClient dba, String foreign_table_name)
void createForeignTestTableManualRefresh(MapdTestClient dba, String foreign_table_name)
void runTest(String db, String userName, String userPassword, int num_foreign_manual_refresh_threads, int num_foreign_scheduled_refresh_threads, int num_table_join_threads, int num_runs)
void createTestTables(MapdTestClient dba, String table_name, Path copy_from_path)

+ Here is the call graph for this function:

Member Data Documentation

final Logger com.mapd.tests.ForeignTableRefreshConcurrencyTest.logger
staticpackage
Initial value:
=
LoggerFactory.getLogger(ForeignTableRefreshConcurrencyTest.class)

Definition at line 31 of file ForeignTableRefreshConcurrencyTest.java.


The documentation for this class was generated from the following file: