This source file includes following definitions.
- mode_cb
- mainloop_test_done
- dispatch_helper
- st_callback
- st_global_callback
- passive_test
- run_fence_failure_test
- run_fence_failure_rollover_test
- run_standard_test
- sanity_tests
- standard_dev_test
- mainloop_callback
- register_callback_helper
- test_async_fence_pass
- test_async_fence_custom_timeout
- test_async_fence_timeout
- test_async_monitor
- test_register_async_devices
- try_mainloop_connect
- iterate_mainloop_tests
- trigger_iterate_mainloop_tests
- test_shutdown
- mainloop_tests
- build_arg_context
- main
1
2
3
4
5
6
7
8 #include <crm_internal.h>
9
10 #include <sys/param.h>
11 #include <stdio.h>
12 #include <sys/time.h>
13 #include <sys/types.h>
14 #include <sys/stat.h>
15 #include <unistd.h>
16 #include <sys/utsname.h>
17
18 #include <stdlib.h>
19 #include <errno.h>
20 #include <fcntl.h>
21
22 #include <crm/crm.h>
23 #include <crm/msg_xml.h>
24 #include <crm/common/ipc.h>
25 #include <crm/cluster/internal.h>
26
27 #include <crm/stonith-ng.h>
28 #include <crm/fencing/internal.h>
29 #include <crm/common/agents.h>
30 #include <crm/common/cmdline_internal.h>
31 #include <crm/common/xml.h>
32
33 #include <crm/common/mainloop.h>
34
35 #define SUMMARY "cts-fence-helper - inject commands into the Pacemaker fencer and watch for events"
36
37 static GMainLoop *mainloop = NULL;
38 static crm_trigger_t *trig = NULL;
39 static int mainloop_iter = 0;
40 static pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
41
42 typedef void (*mainloop_test_iteration_cb) (int check_event);
43
44 #define MAINLOOP_DEFAULT_TIMEOUT 2
45
46 enum test_modes {
47 test_standard = 0,
48 test_passive,
49 test_api_sanity,
50 test_api_mainloop,
51 };
52
53 struct {
54 enum test_modes mode;
55 } options = {
56 .mode = test_standard
57 };
58
59 static gboolean
60 mode_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
61 if (pcmk__str_any_of(option_name, "--mainloop_api_test", "-m", NULL)) {
62 options.mode = test_api_mainloop;
63 } else if (pcmk__str_any_of(option_name, "--api_test", "-t", NULL)) {
64 options.mode = test_api_sanity;
65 } else if (pcmk__str_any_of(option_name, "--passive", "-p", NULL)) {
66 options.mode = test_passive;
67 }
68
69 return TRUE;
70 }
71
72 static GOptionEntry entries[] = {
73 { "mainloop_api_test", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
74 NULL, NULL,
75 },
76
77 { "api_test", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
78 NULL, NULL,
79 },
80
81 { "passive", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
82 NULL, NULL,
83 },
84
85 { NULL }
86 };
87
88 static stonith_t *st = NULL;
89 static struct pollfd pollfd;
90 static const int st_opts = st_opt_sync_call;
91 static int expected_notifications = 0;
92 static int verbose = 0;
93
94 static void
95 mainloop_test_done(const char *origin, bool pass)
96 {
97 if (pass) {
98 crm_info("SUCCESS - %s", origin);
99 mainloop_iter++;
100 mainloop_set_trigger(trig);
101 result.execution_status = PCMK_EXEC_DONE;
102 result.exit_status = CRM_EX_OK;
103 } else {
104 crm_err("FAILURE - %s (%d: %s)", origin, result.exit_status,
105 pcmk_exec_status_str(result.execution_status));
106 crm_exit(CRM_EX_ERROR);
107 }
108 }
109
110
111 static void
112 dispatch_helper(int timeout)
113 {
114 int rc;
115
116 crm_debug("Looking for notification");
117 pollfd.events = POLLIN;
118 while (true) {
119 rc = poll(&pollfd, 1, timeout);
120 if (rc > 0) {
121 if (!stonith_dispatch(st)) {
122 break;
123 }
124 } else {
125 break;
126 }
127 }
128 }
129
130 static void
131 st_callback(stonith_t * st, stonith_event_t * e)
132 {
133 char *desc = NULL;
134
135 if (st->state == stonith_disconnected) {
136 crm_exit(CRM_EX_DISCONNECT);
137 }
138
139 desc = stonith__event_description(e);
140 crm_notice("%s", desc);
141 free(desc);
142
143 if (expected_notifications) {
144 expected_notifications--;
145 }
146 }
147
148 static void
149 st_global_callback(stonith_t * stonith, stonith_callback_data_t * data)
150 {
151 crm_notice("Call %d exited %d: %s (%s)",
152 data->call_id, stonith__exit_status(data),
153 stonith__execution_status(data),
154 pcmk__s(stonith__exit_reason(data), "unspecified reason"));
155 }
156
157 static void
158 passive_test(void)
159 {
160 int rc = 0;
161
162 rc = st->cmds->connect(st, crm_system_name, &pollfd.fd);
163 if (rc != pcmk_ok) {
164 stonith_api_delete(st);
165 crm_exit(CRM_EX_DISCONNECT);
166 }
167 st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback);
168 st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback);
169 st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback);
170 st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback);
171 st->cmds->register_callback(st, 0, 120, st_opt_timeout_updates, NULL, "st_global_callback",
172 st_global_callback);
173
174 dispatch_helper(600 * 1000);
175 }
176
177 #define single_test(cmd, str, num_notifications, expected_rc) \
178 { \
179 int rc = 0; \
180 rc = cmd; \
181 expected_notifications = 0; \
182 if (num_notifications) { \
183 expected_notifications = num_notifications; \
184 dispatch_helper(500); \
185 } \
186 if (rc != expected_rc) { \
187 crm_err("FAILURE - expected rc %d != %d(%s) for cmd - %s", expected_rc, rc, pcmk_strerror(rc), str); \
188 crm_exit(CRM_EX_ERROR); \
189 } else if (expected_notifications) { \
190 crm_err("FAILURE - expected %d notifications, got only %d for cmd - %s", \
191 num_notifications, num_notifications - expected_notifications, str); \
192 crm_exit(CRM_EX_ERROR); \
193 } else { \
194 if (verbose) { \
195 crm_info("SUCCESS - %s: %d", str, rc); \
196 } else { \
197 crm_debug("SUCCESS - %s: %d", str, rc); \
198 } \
199 } \
200 }\
201
202 static void
203 run_fence_failure_test(void)
204 {
205 stonith_key_value_t *params = NULL;
206
207 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
208 "false_1_node1=1,2 false_1_node2=3,4");
209 params = stonith_key_value_add(params, "mode", "fail");
210
211 single_test(st->
212 cmds->register_device(st, st_opts, "test-id1", "stonith-ng", "fence_dummy", params),
213 "Register device1 for failure test", 1, 0);
214
215 single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0),
216 "Fence failure results off", 1, -ENODATA);
217
218 single_test(st->cmds->fence(st, st_opts, "false_1_node2", "reboot", 3, 0),
219 "Fence failure results reboot", 1, -ENODATA);
220
221 single_test(st->cmds->remove_device(st, st_opts, "test-id1"),
222 "Remove device1 for failure test", 1, 0);
223
224 stonith_key_value_freeall(params, 1, 1);
225 }
226
227 static void
228 run_fence_failure_rollover_test(void)
229 {
230 stonith_key_value_t *params = NULL;
231
232 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
233 "false_1_node1=1,2 false_1_node2=3,4");
234 params = stonith_key_value_add(params, "mode", "fail");
235
236 single_test(st->
237 cmds->register_device(st, st_opts, "test-id1", "stonith-ng", "fence_dummy", params),
238 "Register device1 for rollover test", 1, 0);
239 stonith_key_value_freeall(params, 1, 1);
240 params = NULL;
241 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
242 "false_1_node1=1,2 false_1_node2=3,4");
243 params = stonith_key_value_add(params, "mode", "pass");
244
245 single_test(st->
246 cmds->register_device(st, st_opts, "test-id2", "stonith-ng", "fence_dummy", params),
247 "Register device2 for rollover test", 1, 0);
248
249 single_test(st->cmds->fence(st, st_opts, "false_1_node2", "off", 3, 0),
250 "Fence rollover results off", 1, 0);
251
252
253 single_test(st->cmds->fence(st, st_opts, "false_1_node2", "on", 3, 0),
254 "Fence rollover results on", 1, -ENODEV);
255
256 single_test(st->cmds->remove_device(st, st_opts, "test-id1"),
257 "Remove device1 for rollover tests", 1, 0);
258
259 single_test(st->cmds->remove_device(st, st_opts, "test-id2"),
260 "Remove device2 for rollover tests", 1, 0);
261
262 stonith_key_value_freeall(params, 1, 1);
263 }
264
265 static void
266 run_standard_test(void)
267 {
268 stonith_key_value_t *params = NULL;
269
270 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
271 "false_1_node1=1,2 false_1_node2=3,4");
272 params = stonith_key_value_add(params, "mode", "pass");
273 params = stonith_key_value_add(params, "mock_dynamic_hosts", "false_1_node1 false_1_node2");
274
275 single_test(st->
276 cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_dummy", params),
277 "Register", 1, 0);
278 stonith_key_value_freeall(params, 1, 1);
279 params = NULL;
280
281 single_test(st->cmds->list(st, st_opts, "test-id", NULL, 1), "list", 1, 0);
282
283 single_test(st->cmds->monitor(st, st_opts, "test-id", 1), "Monitor", 1, 0);
284
285 single_test(st->cmds->status(st, st_opts, "test-id", "false_1_node2", 1),
286 "Status false_1_node2", 1, 0);
287
288 single_test(st->cmds->status(st, st_opts, "test-id", "false_1_node1", 1),
289 "Status false_1_node1", 1, 0);
290
291 single_test(st->cmds->fence(st, st_opts, "unknown-host", "off", 1, 0),
292 "Fence unknown-host (expected failure)", 0, -ENODEV);
293
294 single_test(st->cmds->fence(st, st_opts, "false_1_node1", "off", 1, 0),
295 "Fence false_1_node1", 1, 0);
296
297
298 single_test(st->cmds->fence(st, st_opts, "false_1_node1", "on", 1, 0),
299 "Unfence false_1_node1", 1, -ENODEV);
300
301
302 single_test(st->cmds->register_level(st, st_opts, "node1", 999, params),
303 "Attempt to register an invalid level index", 0, -EINVAL);
304
305 single_test(st->cmds->remove_device(st, st_opts, "test-id"), "Remove test-id", 1, 0);
306
307 stonith_key_value_freeall(params, 1, 1);
308 }
309
310 static void
311 sanity_tests(void)
312 {
313 int rc = 0;
314
315 rc = st->cmds->connect(st, crm_system_name, &pollfd.fd);
316 if (rc != pcmk_ok) {
317 stonith_api_delete(st);
318 crm_exit(CRM_EX_DISCONNECT);
319 }
320 st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT, st_callback);
321 st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback);
322 st->cmds->register_notification(st, STONITH_OP_DEVICE_ADD, st_callback);
323 st->cmds->register_notification(st, STONITH_OP_DEVICE_DEL, st_callback);
324 st->cmds->register_callback(st, 0, 120, st_opt_timeout_updates, NULL, "st_global_callback",
325 st_global_callback);
326
327 crm_info("Starting API Sanity Tests");
328 run_standard_test();
329 run_fence_failure_test();
330 run_fence_failure_rollover_test();
331 crm_info("Sanity Tests Passed");
332 }
333
334 static void
335 standard_dev_test(void)
336 {
337 int rc = 0;
338 char *tmp = NULL;
339 stonith_key_value_t *params = NULL;
340
341 rc = st->cmds->connect(st, crm_system_name, &pollfd.fd);
342 if (rc != pcmk_ok) {
343 stonith_api_delete(st);
344 crm_exit(CRM_EX_DISCONNECT);
345 }
346
347 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
348 "some-host=pcmk-7 true_1_node1=3,4");
349
350 rc = st->cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_xvm", params);
351 crm_debug("Register: %d", rc);
352
353 rc = st->cmds->list(st, st_opts, "test-id", &tmp, 10);
354 crm_debug("List: %d output: %s", rc, tmp ? tmp : "<none>");
355
356 rc = st->cmds->monitor(st, st_opts, "test-id", 10);
357 crm_debug("Monitor: %d", rc);
358
359 rc = st->cmds->status(st, st_opts, "test-id", "false_1_node2", 10);
360 crm_debug("Status false_1_node2: %d", rc);
361
362 rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
363 crm_debug("Status false_1_node1: %d", rc);
364
365 rc = st->cmds->fence(st, st_opts, "unknown-host", "off", 60, 0);
366 crm_debug("Fence unknown-host: %d", rc);
367
368 rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
369 crm_debug("Status false_1_node1: %d", rc);
370
371 rc = st->cmds->fence(st, st_opts, "false_1_node1", "off", 60, 0);
372 crm_debug("Fence false_1_node1: %d", rc);
373
374 rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
375 crm_debug("Status false_1_node1: %d", rc);
376
377 rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0);
378 crm_debug("Unfence false_1_node1: %d", rc);
379
380 rc = st->cmds->status(st, st_opts, "test-id", "false_1_node1", 10);
381 crm_debug("Status false_1_node1: %d", rc);
382
383 rc = st->cmds->fence(st, st_opts, "some-host", "off", 10, 0);
384 crm_debug("Fence alias: %d", rc);
385
386 rc = st->cmds->status(st, st_opts, "test-id", "some-host", 10);
387 crm_debug("Status alias: %d", rc);
388
389 rc = st->cmds->fence(st, st_opts, "false_1_node1", "on", 10, 0);
390 crm_debug("Unfence false_1_node1: %d", rc);
391
392 rc = st->cmds->remove_device(st, st_opts, "test-id");
393 crm_debug("Remove test-id: %d", rc);
394
395 stonith_key_value_freeall(params, 1, 1);
396 }
397
398 static void
399 iterate_mainloop_tests(gboolean event_ready);
400
401 static void
402 mainloop_callback(stonith_t * stonith, stonith_callback_data_t * data)
403 {
404 pcmk__set_result(&result, stonith__exit_status(data),
405 stonith__execution_status(data),
406 stonith__exit_reason(data));
407 iterate_mainloop_tests(TRUE);
408 }
409
410 static int
411 register_callback_helper(int callid)
412 {
413 return st->cmds->register_callback(st,
414 callid,
415 MAINLOOP_DEFAULT_TIMEOUT,
416 st_opt_timeout_updates, NULL, "callback", mainloop_callback);
417 }
418
419 static void
420 test_async_fence_pass(int check_event)
421 {
422 int rc = 0;
423
424 if (check_event) {
425 mainloop_test_done(__func__, (result.exit_status == CRM_EX_OK));
426 return;
427 }
428
429 rc = st->cmds->fence(st, 0, "true_1_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
430 if (rc < 0) {
431 crm_err("fence failed with rc %d", rc);
432 mainloop_test_done(__func__, false);
433 }
434 register_callback_helper(rc);
435
436 }
437
438 #define CUSTOM_TIMEOUT_ADDITION 10
439 static void
440 test_async_fence_custom_timeout(int check_event)
441 {
442 int rc = 0;
443 static time_t begin = 0;
444
445 if (check_event) {
446 uint32_t diff = (time(NULL) - begin);
447
448 if (result.execution_status != PCMK_EXEC_TIMEOUT) {
449 mainloop_test_done(__func__, false);
450 } else if (diff < CUSTOM_TIMEOUT_ADDITION + MAINLOOP_DEFAULT_TIMEOUT) {
451 crm_err
452 ("Custom timeout test failed, callback expiration should be updated to %d, actual timeout was %d",
453 CUSTOM_TIMEOUT_ADDITION + MAINLOOP_DEFAULT_TIMEOUT, diff);
454 mainloop_test_done(__func__, false);
455 } else {
456 mainloop_test_done(__func__, true);
457 }
458 return;
459 }
460 begin = time(NULL);
461
462 rc = st->cmds->fence(st, 0, "custom_timeout_node1", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
463 if (rc < 0) {
464 crm_err("fence failed with rc %d", rc);
465 mainloop_test_done(__func__, false);
466 }
467 register_callback_helper(rc);
468
469 }
470
471 static void
472 test_async_fence_timeout(int check_event)
473 {
474 int rc = 0;
475
476 if (check_event) {
477 mainloop_test_done(__func__,
478 (result.execution_status == PCMK_EXEC_NO_FENCE_DEVICE));
479 return;
480 }
481
482 rc = st->cmds->fence(st, 0, "false_1_node2", "off", MAINLOOP_DEFAULT_TIMEOUT, 0);
483 if (rc < 0) {
484 crm_err("fence failed with rc %d", rc);
485 mainloop_test_done(__func__, false);
486 }
487 register_callback_helper(rc);
488
489 }
490
491 static void
492 test_async_monitor(int check_event)
493 {
494 int rc = 0;
495
496 if (check_event) {
497 mainloop_test_done(__func__, (result.exit_status == CRM_EX_OK));
498 return;
499 }
500
501 rc = st->cmds->monitor(st, 0, "false_1", MAINLOOP_DEFAULT_TIMEOUT);
502 if (rc < 0) {
503 crm_err("monitor failed with rc %d", rc);
504 mainloop_test_done(__func__, false);
505 }
506
507 register_callback_helper(rc);
508
509 }
510
511 static void
512 test_register_async_devices(int check_event)
513 {
514 char buf[16] = { 0, };
515 stonith_key_value_t *params = NULL;
516
517 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
518 "false_1_node1=1,2");
519 params = stonith_key_value_add(params, "mode", "fail");
520 st->cmds->register_device(st, st_opts, "false_1", "stonith-ng", "fence_dummy", params);
521 stonith_key_value_freeall(params, 1, 1);
522
523 params = NULL;
524 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
525 "true_1_node1=1,2");
526 params = stonith_key_value_add(params, "mode", "pass");
527 st->cmds->register_device(st, st_opts, "true_1", "stonith-ng", "fence_dummy", params);
528 stonith_key_value_freeall(params, 1, 1);
529
530 params = NULL;
531 params = stonith_key_value_add(params, PCMK_STONITH_HOST_MAP,
532 "custom_timeout_node1=1,2");
533 params = stonith_key_value_add(params, "mode", "fail");
534 params = stonith_key_value_add(params, "delay", "1000");
535 snprintf(buf, sizeof(buf) - 1, "%d", MAINLOOP_DEFAULT_TIMEOUT + CUSTOM_TIMEOUT_ADDITION);
536 params = stonith_key_value_add(params, "pcmk_off_timeout", buf);
537 st->cmds->register_device(st, st_opts, "false_custom_timeout", "stonith-ng", "fence_dummy",
538 params);
539 stonith_key_value_freeall(params, 1, 1);
540
541 mainloop_test_done(__func__, true);
542 }
543
544 static void
545 try_mainloop_connect(int check_event)
546 {
547 int rc = stonith_api_connect_retry(st, crm_system_name, 10);
548
549 if (rc == pcmk_ok) {
550 mainloop_test_done(__func__, true);
551 return;
552 }
553 crm_err("API CONNECTION FAILURE");
554 mainloop_test_done(__func__, false);
555 }
556
557 static void
558 iterate_mainloop_tests(gboolean event_ready)
559 {
560 static mainloop_test_iteration_cb callbacks[] = {
561 try_mainloop_connect,
562 test_register_async_devices,
563 test_async_monitor,
564 test_async_fence_pass,
565 test_async_fence_timeout,
566 test_async_fence_custom_timeout,
567 };
568
569 if (mainloop_iter == (sizeof(callbacks) / sizeof(mainloop_test_iteration_cb))) {
570
571 crm_info("ALL MAINLOOP TESTS PASSED!");
572 crm_exit(CRM_EX_OK);
573 }
574
575 callbacks[mainloop_iter] (event_ready);
576 }
577
578 static gboolean
579 trigger_iterate_mainloop_tests(gpointer user_data)
580 {
581 iterate_mainloop_tests(FALSE);
582 return TRUE;
583 }
584
585 static void
586 test_shutdown(int nsig)
587 {
588 int rc = 0;
589
590 if (st) {
591 rc = st->cmds->disconnect(st);
592 crm_info("Disconnect: %d", rc);
593
594 crm_debug("Destroy");
595 stonith_api_delete(st);
596 }
597
598 if (rc) {
599 crm_exit(CRM_EX_ERROR);
600 }
601 }
602
603 static void
604 mainloop_tests(void)
605 {
606 trig = mainloop_add_trigger(G_PRIORITY_HIGH, trigger_iterate_mainloop_tests, NULL);
607 mainloop_set_trigger(trig);
608 mainloop_add_signal(SIGTERM, test_shutdown);
609
610 crm_info("Starting");
611 mainloop = g_main_loop_new(NULL, FALSE);
612 g_main_loop_run(mainloop);
613 }
614
615 static GOptionContext *
616 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
617 GOptionContext *context = NULL;
618
619 context = pcmk__build_arg_context(args, NULL, group, NULL);
620 pcmk__add_main_args(context, entries);
621 return context;
622 }
623
624 int
625 main(int argc, char **argv)
626 {
627 GError *error = NULL;
628 crm_exit_t exit_code = CRM_EX_OK;
629
630 pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
631 gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
632 GOptionContext *context = build_arg_context(args, NULL);
633
634 if (!g_option_context_parse_strv(context, &processed_args, &error)) {
635 exit_code = CRM_EX_USAGE;
636 goto done;
637 }
638
639
640
641
642
643 crm_log_init(NULL, LOG_INFO, TRUE, (verbose? TRUE : FALSE), argc, argv,
644 FALSE);
645
646 for (int i = 0; i < args->verbosity; i++) {
647 crm_bump_log_level(argc, argv);
648 }
649
650 st = stonith_api_new();
651 if (st == NULL) {
652 exit_code = CRM_EX_DISCONNECT;
653 g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
654 "Could not connect to fencer: API memory allocation failed");
655 goto done;
656 }
657
658 switch (options.mode) {
659 case test_standard:
660 standard_dev_test();
661 break;
662 case test_passive:
663 passive_test();
664 break;
665 case test_api_sanity:
666 sanity_tests();
667 break;
668 case test_api_mainloop:
669 mainloop_tests();
670 break;
671 }
672
673 test_shutdown(0);
674
675 done:
676 g_strfreev(processed_args);
677 pcmk__free_arg_context(context);
678
679 pcmk__output_and_clear_error(error, NULL);
680 crm_exit(exit_code);
681 }
![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
![[previous]](../icons/left.png)
![[last]](../icons/n_last.png)
![[bottom]](../icons/n_bottom.png){kind=icon}