This source file includes following definitions.
- is_action_required
- get_action_delay_max
- get_action_delay_base
- get_action_timeout
- cmd_device
- fenced_device_reboot_action
- fenced_device_supports_on
- free_async_command
- create_async_command
- get_action_limit
- get_active_cmds
- fork_cb
- get_agent_metadata_cb
- report_internal_result
- stonith_device_execute
- stonith_device_dispatch
- start_delay_helper
- schedule_stonith_command
- free_device
- free_device_list
- init_device_list
- build_port_aliases
- free_metadata_cache
- init_metadata_cache
- get_agent_metadata
- is_nodeid_required
- read_action_metadata
- target_list_type
- build_device_from_xml
- schedule_internal_command
- status_search_cb
- dynamic_list_search_cb
- device_params_diff
- device_has_duplicate
- stonith_device_register
- stonith_device_remove
- count_active_levels
- free_topology_entry
- free_topology_list
- init_topology_list
- stonith_level_key
- unpack_level_kind
- parse_device_list
- unpack_level_request
- fenced_register_level
- fenced_unregister_level
- list_to_string
- execute_agent_action
- search_devices_record_result
- localhost_is_eligible
- localhost_is_eligible_with_remap
- can_fence_host_with_device
- search_devices
- get_capable_devices
- add_action_specific_attributes
- add_disallowed
- add_action_reply
- stonith_send_reply
- stonith_query_capable_device_cb
- log_async_result
- send_async_reply
- cancel_stonith_command
- reply_to_duplicates
- next_required_device
- st_child_done
- sort_device_priority
- stonith_fence_get_devices_cb
- fence_locally
- fenced_construct_reply
- construct_async_reply
- fencing_peer_active
- set_fencing_completed
- check_alternate_host
- remove_relay_op
- is_privileged
- handle_register_request
- handle_agent_request
- handle_update_timeout_request
- handle_query_request
- handle_notify_request
- handle_relay_request
- handle_fence_request
- handle_history_request
- handle_device_add_request
- handle_device_delete_request
- handle_level_add_request
- handle_level_delete_request
- handle_cache_request
- handle_unknown_request
- fenced_register_handlers
- fenced_unregister_handlers
- handle_request
- handle_reply
- stonith_command
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <sys/param.h>
13 #include <stdio.h>
14 #include <sys/types.h>
15 #include <sys/wait.h>
16 #include <sys/stat.h>
17 #include <unistd.h>
18 #include <sys/utsname.h>
19
20 #include <stdlib.h>
21 #include <errno.h>
22 #include <fcntl.h>
23 #include <ctype.h>
24
25 #include <crm/crm.h>
26 #include <crm/common/ipc.h>
27 #include <crm/common/ipc_internal.h>
28 #include <crm/cluster/internal.h>
29 #include <crm/common/mainloop.h>
30
31 #include <crm/stonith-ng.h>
32 #include <crm/fencing/internal.h>
33 #include <crm/common/xml.h>
34
35 #include <pacemaker-fenced.h>
36
37 GHashTable *device_list = NULL;
38 GHashTable *topology = NULL;
39 static GList *cmd_list = NULL;
40
41 static GHashTable *fenced_handlers = NULL;
42
43 struct device_search_s {
44
45 char *host;
46
47 char *action;
48
49 int per_device_timeout;
50
51 int replies_needed;
52
53 int replies_received;
54
55 bool allow_self;
56
57
58 void *user_data;
59
60 void (*callback) (GList * devices, void *user_data);
61
62 GList *capable;
63
64 uint32_t support_action_only;
65 };
66
67 static gboolean stonith_device_dispatch(gpointer user_data);
68 static void st_child_done(int pid, const pcmk__action_result_t *result,
69 void *user_data);
70
71 static void search_devices_record_result(struct device_search_s *search, const char *device,
72 gboolean can_fence);
73
74 static int get_agent_metadata(const char *agent, xmlNode **metadata);
75 static void read_action_metadata(stonith_device_t *device);
76 static enum fenced_target_by unpack_level_kind(const xmlNode *level);
77
78 typedef struct async_command_s {
79
80 int id;
81 int pid;
82 int fd_stdout;
83 uint32_t options;
84 int default_timeout;
85 int timeout;
86
87 int start_delay;
88 int delay_id;
89
90 char *op;
91 char *origin;
92 char *client;
93 char *client_name;
94 char *remote_op_id;
95
96 char *target;
97 uint32_t target_nodeid;
98 char *action;
99 char *device;
100
101 GList *device_list;
102 GList *next_device_iter;
103
104 void *internal_user_data;
105 void (*done_cb) (int pid, const pcmk__action_result_t *result,
106 void *user_data);
107 guint timer_sigterm;
108 guint timer_sigkill;
109
110
111 int last_timeout_signo;
112
113 stonith_device_t *active_on;
114 stonith_device_t *activating_on;
115 } async_command_t;
116
117 static xmlNode *construct_async_reply(const async_command_t *cmd,
118 const pcmk__action_result_t *result);
119
120 static gboolean
121 is_action_required(const char *action, const stonith_device_t *device)
122 {
123 return (device != NULL) && device->automatic_unfencing
124 && pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none);
125 }
126
127 static int
128 get_action_delay_max(const stonith_device_t *device, const char *action)
129 {
130 const char *value = NULL;
131 guint delay_max = 0U;
132
133 if (!pcmk__is_fencing_action(action)) {
134 return 0;
135 }
136
137 value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
138 if (value) {
139 pcmk_parse_interval_spec(value, &delay_max);
140 delay_max /= 1000;
141 }
142
143 return (int) delay_max;
144 }
145
146 static int
147 get_action_delay_base(const stonith_device_t *device, const char *action,
148 const char *target)
149 {
150 char *hash_value = NULL;
151 guint delay_base = 0U;
152
153 if (!pcmk__is_fencing_action(action)) {
154 return 0;
155 }
156
157 hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);
158
159 if (hash_value) {
160 char *value = pcmk__str_copy(hash_value);
161 char *valptr = value;
162
163 if (target != NULL) {
164 for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
165 char *mapval = strchr(val, ':');
166
167 if (mapval == NULL || mapval[1] == 0) {
168 crm_err("pcmk_delay_base: empty value in mapping", val);
169 continue;
170 }
171
172 if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
173 value = mapval + 1;
174 crm_debug("pcmk_delay_base mapped to %s for %s",
175 value, target);
176 break;
177 }
178 }
179 }
180
181 if (strchr(value, ':') == 0) {
182 pcmk_parse_interval_spec(value, &delay_base);
183 delay_base /= 1000;
184 }
185
186 free(valptr);
187 }
188
189 return (int) delay_base;
190 }
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212 static int
213 get_action_timeout(const stonith_device_t *device, const char *action,
214 int default_timeout)
215 {
216 if (action && device && device->params) {
217 char buffer[64] = { 0, };
218 const char *value = NULL;
219
220
221
222
223 if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)
224 && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
225 crm_trace("%s doesn't support reboot, using timeout for off instead",
226 device->id);
227 action = PCMK_ACTION_OFF;
228 }
229
230
231 snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
232 value = g_hash_table_lookup(device->params, buffer);
233 if (value) {
234 long long timeout_ms = crm_get_msec(value);
235 return (int) QB_MIN(pcmk__timeout_ms2s(timeout_ms), INT_MAX);
236 }
237 }
238 return default_timeout;
239 }
240
241
242
243
244
245
246
247
248
249 static stonith_device_t *
250 cmd_device(const async_command_t *cmd)
251 {
252 if ((cmd == NULL) || (cmd->device == NULL) || (device_list == NULL)) {
253 return NULL;
254 }
255 return g_hash_table_lookup(device_list, cmd->device);
256 }
257
258
259
260
261
262
263
264
265
266 const char *
267 fenced_device_reboot_action(const char *device_id)
268 {
269 const char *action = NULL;
270
271 if ((device_list != NULL) && (device_id != NULL)) {
272 stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
273
274 if ((device != NULL) && (device->params != NULL)) {
275 action = g_hash_table_lookup(device->params, "pcmk_reboot_action");
276 }
277 }
278 return pcmk__s(action, PCMK_ACTION_REBOOT);
279 }
280
281
282
283
284
285
286
287
288
289 bool
290 fenced_device_supports_on(const char *device_id)
291 {
292 if ((device_list != NULL) && (device_id != NULL)) {
293 stonith_device_t *device = g_hash_table_lookup(device_list, device_id);
294
295 if (device != NULL) {
296 return pcmk_is_set(device->flags, st_device_supports_on);
297 }
298 }
299 return false;
300 }
301
302 static void
303 free_async_command(async_command_t * cmd)
304 {
305 if (!cmd) {
306 return;
307 }
308
309 if (cmd->delay_id) {
310 g_source_remove(cmd->delay_id);
311 }
312
313 cmd_list = g_list_remove(cmd_list, cmd);
314
315 g_list_free_full(cmd->device_list, free);
316 free(cmd->device);
317 free(cmd->action);
318 free(cmd->target);
319 free(cmd->remote_op_id);
320 free(cmd->client);
321 free(cmd->client_name);
322 free(cmd->origin);
323 free(cmd->op);
324 free(cmd);
325 }
326
327
328
329
330
331
332
333
334
335
336
337
338 static async_command_t *
339 create_async_command(xmlNode *msg)
340 {
341 xmlNode *op = NULL;
342 async_command_t *cmd = NULL;
343 int rc = pcmk_rc_ok;
344
345 if (msg == NULL) {
346 return NULL;
347 }
348
349 op = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, msg, LOG_ERR);
350 if (op == NULL) {
351 return NULL;
352 }
353
354 cmd = pcmk__assert_alloc(1, sizeof(async_command_t));
355
356
357 cmd->action = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ACTION);
358 cmd->op = crm_element_value_copy(msg, PCMK__XA_ST_OP);
359 cmd->client = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTID);
360 if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) {
361 free_async_command(cmd);
362 return NULL;
363 }
364
365 crm_element_value_int(msg, PCMK__XA_ST_CALLID, &(cmd->id));
366 crm_element_value_int(msg, PCMK__XA_ST_DELAY, &(cmd->start_delay));
367 crm_element_value_int(msg, PCMK__XA_ST_TIMEOUT, &(cmd->default_timeout));
368 cmd->timeout = cmd->default_timeout;
369
370 rc = pcmk__xe_get_flags(msg, PCMK__XA_ST_CALLOPT, &(cmd->options),
371 st_opt_none);
372 if (rc != pcmk_rc_ok) {
373 crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
374 }
375
376 cmd->origin = crm_element_value_copy(msg, PCMK__XA_SRC);
377 cmd->remote_op_id = crm_element_value_copy(msg, PCMK__XA_ST_REMOTE_OP);
378 cmd->client_name = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTNAME);
379 cmd->target = crm_element_value_copy(op, PCMK__XA_ST_TARGET);
380 cmd->device = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ID);
381
382 cmd->done_cb = st_child_done;
383
384
385 cmd_list = g_list_append(cmd_list, cmd);
386
387 return cmd;
388 }
389
390 static int
391 get_action_limit(stonith_device_t * device)
392 {
393 const char *value = NULL;
394 int action_limit = 1;
395
396 value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
397 if ((value == NULL)
398 || (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
399 || (action_limit == 0)) {
400 action_limit = 1;
401 }
402 return action_limit;
403 }
404
405 static int
406 get_active_cmds(stonith_device_t * device)
407 {
408 int counter = 0;
409 GList *gIter = NULL;
410 GList *gIterNext = NULL;
411
412 CRM_CHECK(device != NULL, return 0);
413
414 for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
415 async_command_t *cmd = gIter->data;
416
417 gIterNext = gIter->next;
418
419 if (cmd->active_on == device) {
420 counter++;
421 }
422 }
423
424 return counter;
425 }
426
427 static void
428 fork_cb(int pid, void *user_data)
429 {
430 async_command_t *cmd = (async_command_t *) user_data;
431 stonith_device_t * device =
432
433
434
435 cmd->activating_on?cmd->activating_on:cmd->active_on;
436
437 pcmk__assert(device != NULL);
438 crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
439 cmd->action, pid,
440 ((cmd->target == NULL)? "" : " targeting "),
441 pcmk__s(cmd->target, ""), device->id, cmd->timeout);
442 cmd->active_on = device;
443 cmd->activating_on = NULL;
444 }
445
446 static int
447 get_agent_metadata_cb(gpointer data) {
448 stonith_device_t *device = data;
449 guint period_ms;
450
451 switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
452 case pcmk_rc_ok:
453 if (device->agent_metadata) {
454 read_action_metadata(device);
455 stonith__device_parameter_flags(&(device->flags), device->id,
456 device->agent_metadata);
457 }
458 return G_SOURCE_REMOVE;
459
460 case EAGAIN:
461 period_ms = pcmk__mainloop_timer_get_period(device->timer);
462 if (period_ms < 160 * 1000) {
463 mainloop_timer_set_period(device->timer, 2 * period_ms);
464 }
465 return G_SOURCE_CONTINUE;
466
467 default:
468 return G_SOURCE_REMOVE;
469 }
470 }
471
472
473
474
475
476
477
478
479
480
481 static void
482 report_internal_result(async_command_t *cmd, int exit_status,
483 int execution_status, const char *exit_reason)
484 {
485 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
486
487 pcmk__set_result(&result, exit_status, execution_status, exit_reason);
488 cmd->done_cb(0, &result, cmd);
489 pcmk__reset_result(&result);
490 }
491
492 static gboolean
493 stonith_device_execute(stonith_device_t * device)
494 {
495 int exec_rc = 0;
496 const char *action_str = NULL;
497 const char *host_arg = NULL;
498 async_command_t *cmd = NULL;
499 stonith_action_t *action = NULL;
500 int active_cmds = 0;
501 int action_limit = 0;
502 GList *gIter = NULL;
503 GList *gIterNext = NULL;
504
505 CRM_CHECK(device != NULL, return FALSE);
506
507 active_cmds = get_active_cmds(device);
508 action_limit = get_action_limit(device);
509 if (action_limit > -1 && active_cmds >= action_limit) {
510 crm_trace("%s is over its action limit of %d (%u active action%s)",
511 device->id, action_limit, active_cmds,
512 pcmk__plural_s(active_cmds));
513 return TRUE;
514 }
515
516 for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
517 async_command_t *pending_op = gIter->data;
518
519 gIterNext = gIter->next;
520
521 if (pending_op && pending_op->delay_id) {
522 crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
523 "waiting for start delay of %ds",
524 pending_op->action,
525 ((pending_op->target == NULL)? "" : " targeting "),
526 pcmk__s(pending_op->target, ""),
527 device->id, pending_op->start_delay);
528 continue;
529 }
530
531 device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
532 g_list_free_1(gIter);
533
534 cmd = pending_op;
535 break;
536 }
537
538 if (cmd == NULL) {
539 crm_trace("No actions using %s are needed", device->id);
540 return TRUE;
541 }
542
543 if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
544 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
545 if (pcmk__is_fencing_action(cmd->action)) {
546 if (node_does_watchdog_fencing(fenced_get_local_node())) {
547 pcmk__panic("Watchdog self-fencing required");
548 goto done;
549 }
550 } else {
551 crm_info("Faking success for %s watchdog operation", cmd->action);
552 report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
553 goto done;
554 }
555 }
556
557 #if PCMK__ENABLE_CIBSECRETS
558 exec_rc = pcmk__substitute_secrets(device->id, device->params);
559 if (exec_rc != pcmk_rc_ok) {
560 if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_none)) {
561 crm_info("Proceeding with stop operation for %s "
562 "despite being unable to load CIB secrets (%s)",
563 device->id, pcmk_rc_str(exec_rc));
564 } else {
565 crm_err("Considering %s unconfigured "
566 "because unable to load CIB secrets: %s",
567 device->id, pcmk_rc_str(exec_rc));
568 report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
569 "Failed to get CIB secrets");
570 goto done;
571 }
572 }
573 #endif
574
575 action_str = cmd->action;
576 if (pcmk__str_eq(cmd->action, PCMK_ACTION_REBOOT, pcmk__str_none)
577 && !pcmk_is_set(device->flags, st_device_supports_reboot)) {
578
579 crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
580 "because agent '%s' does not support reboot",
581 ((cmd->target == NULL)? "" : " targeting "),
582 pcmk__s(cmd->target, ""), device->id, device->agent);
583 action_str = PCMK_ACTION_OFF;
584 }
585
586 if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
587 host_arg = "port";
588
589 } else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
590 host_arg = "plug";
591 }
592
593 action = stonith__action_create(device->agent, action_str, cmd->target,
594 cmd->target_nodeid, cmd->timeout,
595 device->params, device->aliases, host_arg);
596
597
598
599 cmd->activating_on = device;
600 exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
601 fork_cb);
602 if (exec_rc < 0) {
603 cmd->activating_on = NULL;
604 cmd->done_cb(0, stonith__action_result(action), cmd);
605 stonith__destroy_action(action);
606 }
607
608 done:
609
610
611
612 if (device->pending_ops) {
613 mainloop_set_trigger(device->work);
614 }
615 return TRUE;
616 }
617
618 static gboolean
619 stonith_device_dispatch(gpointer user_data)
620 {
621 return stonith_device_execute(user_data);
622 }
623
624 static gboolean
625 start_delay_helper(gpointer data)
626 {
627 async_command_t *cmd = data;
628 stonith_device_t *device = cmd_device(cmd);
629
630 cmd->delay_id = 0;
631 if (device) {
632 mainloop_set_trigger(device->work);
633 }
634
635 return FALSE;
636 }
637
638 static void
639 schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
640 {
641 int delay_max = 0;
642 int delay_base = 0;
643 int requested_delay = cmd->start_delay;
644
645 CRM_CHECK(cmd != NULL, return);
646 CRM_CHECK(device != NULL, return);
647
648 if (cmd->device) {
649 free(cmd->device);
650 }
651
652 if (device->include_nodeid && (cmd->target != NULL)) {
653 pcmk__node_status_t *node =
654 pcmk__get_node(0, cmd->target, NULL,
655 pcmk__node_search_cluster_member);
656
657 cmd->target_nodeid = node->cluster_layer_id;
658 }
659
660 cmd->device = pcmk__str_copy(device->id);
661 cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
662
663 if (cmd->remote_op_id) {
664 crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
665 "with op id %.8s and timeout %ds",
666 cmd->action,
667 (cmd->target == NULL)? "" : " targeting ",
668 pcmk__s(cmd->target, ""),
669 device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
670 } else {
671 crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
672 cmd->action,
673 (cmd->target == NULL)? "" : " targeting ",
674 pcmk__s(cmd->target, ""),
675 device->id, cmd->client, cmd->timeout);
676 }
677
678 device->pending_ops = g_list_append(device->pending_ops, cmd);
679 mainloop_set_trigger(device->work);
680
681
682 if (requested_delay < 0) {
683 return;
684 }
685
686 delay_max = get_action_delay_max(device, cmd->action);
687 delay_base = get_action_delay_base(device, cmd->action, cmd->target);
688 if (delay_max == 0) {
689 delay_max = delay_base;
690 }
691 if (delay_max < delay_base) {
692 crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
693 PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
694 "(limiting to maximum delay)",
695 delay_base, delay_max, cmd->action, device->id);
696 delay_base = delay_max;
697 }
698 if (delay_max > 0) {
699
700 cmd->start_delay +=
701 ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
702 + delay_base;
703 }
704
705 if (cmd->start_delay > 0) {
706 crm_notice("Delaying '%s' action%s%s using %s for %ds " QB_XS
707 " timeout=%ds requested_delay=%ds base=%ds max=%ds",
708 cmd->action,
709 (cmd->target == NULL)? "" : " targeting ",
710 pcmk__s(cmd->target, ""),
711 device->id, cmd->start_delay, cmd->timeout,
712 requested_delay, delay_base, delay_max);
713 cmd->delay_id =
714 pcmk__create_timer(cmd->start_delay * 1000, start_delay_helper, cmd);
715 }
716 }
717
718 static void
719 free_device(gpointer data)
720 {
721 GList *gIter = NULL;
722 stonith_device_t *device = data;
723
724 g_hash_table_destroy(device->params);
725 g_hash_table_destroy(device->aliases);
726
727 for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
728 async_command_t *cmd = gIter->data;
729
730 crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
731 report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
732 "Device was removed before action could be executed");
733 }
734 g_list_free(device->pending_ops);
735
736 g_list_free_full(device->targets, free);
737
738 if (device->timer) {
739 mainloop_timer_stop(device->timer);
740 mainloop_timer_del(device->timer);
741 }
742
743 mainloop_destroy_trigger(device->work);
744
745 pcmk__xml_free(device->agent_metadata);
746 free(device->namespace);
747 if (device->on_target_actions != NULL) {
748 g_string_free(device->on_target_actions, TRUE);
749 }
750 free(device->agent);
751 free(device->id);
752 free(device);
753 }
754
755 void free_device_list(void)
756 {
757 if (device_list != NULL) {
758 g_hash_table_destroy(device_list);
759 device_list = NULL;
760 }
761 }
762
763 void
764 init_device_list(void)
765 {
766 if (device_list == NULL) {
767 device_list = pcmk__strkey_table(NULL, free_device);
768 }
769 }
770
771 static GHashTable *
772 build_port_aliases(const char *hostmap, GList ** targets)
773 {
774 char *name = NULL;
775 int last = 0, lpc = 0, max = 0, added = 0;
776 GHashTable *aliases = pcmk__strikey_table(free, free);
777
778 if (hostmap == NULL) {
779 return aliases;
780 }
781
782 max = strlen(hostmap);
783 for (; lpc <= max; lpc++) {
784 switch (hostmap[lpc]) {
785
786 case '\\':
787 lpc++;
788 break;
789
790
791 case '=':
792 case ':':
793 if (lpc > last) {
794 free(name);
795 name = pcmk__assert_alloc(1, 1 + lpc - last);
796 memcpy(name, hostmap + last, lpc - last);
797 }
798 last = lpc + 1;
799 break;
800
801
802
803 case 0:
804 case ';':
805 case ' ':
806 case '\t':
807 if (name) {
808 char *value = NULL;
809 int k = 0;
810
811 value = pcmk__assert_alloc(1, 1 + lpc - last);
812 memcpy(value, hostmap + last, lpc - last);
813
814 for (int i = 0; value[i] != '\0'; i++) {
815 if (value[i] != '\\') {
816 value[k++] = value[i];
817 }
818 }
819 value[k] = '\0';
820
821 crm_debug("Adding alias '%s'='%s'", name, value);
822 g_hash_table_replace(aliases, name, value);
823 if (targets) {
824 *targets = g_list_append(*targets, pcmk__str_copy(value));
825 }
826 value = NULL;
827 name = NULL;
828 added++;
829
830 } else if (lpc > last) {
831 crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
832 }
833
834 last = lpc + 1;
835 break;
836 }
837
838 if (hostmap[lpc] == 0) {
839 break;
840 }
841 }
842
843 if (added == 0) {
844 crm_info("No host mappings detected in '%s'", hostmap);
845 }
846
847 free(name);
848 return aliases;
849 }
850
851 GHashTable *metadata_cache = NULL;
852
853 void
854 free_metadata_cache(void) {
855 if (metadata_cache != NULL) {
856 g_hash_table_destroy(metadata_cache);
857 metadata_cache = NULL;
858 }
859 }
860
861 static void
862 init_metadata_cache(void) {
863 if (metadata_cache == NULL) {
864 metadata_cache = pcmk__strkey_table(free, free);
865 }
866 }
867
868 int
869 get_agent_metadata(const char *agent, xmlNode ** metadata)
870 {
871 char *buffer = NULL;
872
873 if (metadata == NULL) {
874 return EINVAL;
875 }
876 *metadata = NULL;
877 if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
878 return pcmk_rc_ok;
879 }
880 init_metadata_cache();
881 buffer = g_hash_table_lookup(metadata_cache, agent);
882 if (buffer == NULL) {
883 stonith_t *st = stonith_api_new();
884 int rc;
885
886 if (st == NULL) {
887 crm_warn("Could not get agent meta-data: "
888 "API memory allocation failed");
889 return EAGAIN;
890 }
891 rc = st->cmds->metadata(st, st_opt_sync_call, agent,
892 NULL, &buffer, 10);
893 stonith_api_delete(st);
894 if (rc || !buffer) {
895 crm_err("Could not retrieve metadata for fencing agent %s", agent);
896 return EAGAIN;
897 }
898 g_hash_table_replace(metadata_cache, pcmk__str_copy(agent), buffer);
899 }
900
901 *metadata = pcmk__xml_parse(buffer);
902 return pcmk_rc_ok;
903 }
904
905 static gboolean
906 is_nodeid_required(xmlNode * xml)
907 {
908 xmlXPathObjectPtr xpath = NULL;
909
910 if (!xml) {
911 return FALSE;
912 }
913
914 xpath = xpath_search(xml,
915 "//" PCMK_XE_PARAMETER "[@" PCMK_XA_NAME "='nodeid']");
916 if (numXpathResults(xpath) <= 0) {
917 freeXpathObject(xpath);
918 return FALSE;
919 }
920
921 freeXpathObject(xpath);
922 return TRUE;
923 }
924
925 static void
926 read_action_metadata(stonith_device_t *device)
927 {
928 xmlXPathObjectPtr xpath = NULL;
929 int max = 0;
930 int lpc = 0;
931
932 if (device->agent_metadata == NULL) {
933 return;
934 }
935
936 xpath = xpath_search(device->agent_metadata, "//action");
937 max = numXpathResults(xpath);
938
939 if (max <= 0) {
940 freeXpathObject(xpath);
941 return;
942 }
943
944 for (lpc = 0; lpc < max; lpc++) {
945 const char *action = NULL;
946 xmlNode *match = getXpathResult(xpath, lpc);
947
948 CRM_LOG_ASSERT(match != NULL);
949 if(match == NULL) { continue; };
950
951 action = crm_element_value(match, PCMK_XA_NAME);
952
953 if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
954 stonith__set_device_flags(device->flags, device->id,
955 st_device_supports_list);
956 } else if (pcmk__str_eq(action, PCMK_ACTION_STATUS, pcmk__str_none)) {
957 stonith__set_device_flags(device->flags, device->id,
958 st_device_supports_status);
959 } else if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
960 stonith__set_device_flags(device->flags, device->id,
961 st_device_supports_reboot);
962 } else if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)) {
963
964
965
966
967
968
969 if (pcmk__xe_attr_is_true(match, PCMK_XA_AUTOMATIC)
970 || pcmk__xe_attr_is_true(match, PCMK__XA_REQUIRED)) {
971 device->automatic_unfencing = TRUE;
972 }
973 stonith__set_device_flags(device->flags, device->id,
974 st_device_supports_on);
975 }
976
977 if ((action != NULL)
978 && pcmk__xe_attr_is_true(match, PCMK_XA_ON_TARGET)) {
979
980 pcmk__add_word(&(device->on_target_actions), 64, action);
981 }
982 }
983
984 freeXpathObject(xpath);
985 }
986
987 static const char *
988 target_list_type(stonith_device_t * dev)
989 {
990 const char *check_type = NULL;
991
992 check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
993
994 if (check_type == NULL) {
995
996 if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
997 check_type = PCMK_VALUE_STATIC_LIST;
998 } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
999 check_type = PCMK_VALUE_STATIC_LIST;
1000 } else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
1001 check_type = PCMK_VALUE_DYNAMIC_LIST;
1002 } else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
1003 check_type = PCMK_VALUE_STATUS;
1004 } else {
1005 check_type = PCMK_VALUE_NONE;
1006 }
1007 }
1008
1009 return check_type;
1010 }
1011
1012 static stonith_device_t *
1013 build_device_from_xml(xmlNode *dev)
1014 {
1015 const char *value;
1016 stonith_device_t *device = NULL;
1017 char *agent = crm_element_value_copy(dev, PCMK_XA_AGENT);
1018
1019 CRM_CHECK(agent != NULL, return device);
1020
1021 device = pcmk__assert_alloc(1, sizeof(stonith_device_t));
1022
1023 device->id = crm_element_value_copy(dev, PCMK_XA_ID);
1024 device->agent = agent;
1025 device->namespace = crm_element_value_copy(dev, PCMK__XA_NAMESPACE);
1026 device->params = xml2list(dev);
1027
1028 value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
1029 if (value) {
1030 device->targets = stonith__parse_targets(value);
1031 }
1032
1033 value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
1034 device->aliases = build_port_aliases(value, &(device->targets));
1035
1036 value = target_list_type(device);
1037 if (!pcmk__str_eq(value, PCMK_VALUE_STATIC_LIST, pcmk__str_casei)
1038 && (device->targets != NULL)) {
1039
1040
1041 g_list_free_full(device->targets, free);
1042 device->targets = NULL;
1043 }
1044 switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
1045 case pcmk_rc_ok:
1046 if (device->agent_metadata) {
1047 read_action_metadata(device);
1048 stonith__device_parameter_flags(&(device->flags), device->id,
1049 device->agent_metadata);
1050 }
1051 break;
1052
1053 case EAGAIN:
1054 if (device->timer == NULL) {
1055 device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
1056 TRUE, get_agent_metadata_cb, device);
1057 }
1058 if (!mainloop_timer_running(device->timer)) {
1059 mainloop_timer_start(device->timer);
1060 }
1061 break;
1062
1063 default:
1064 break;
1065 }
1066
1067 value = g_hash_table_lookup(device->params, "nodeid");
1068 if (!value) {
1069 device->include_nodeid = is_nodeid_required(device->agent_metadata);
1070 }
1071
1072 value = crm_element_value(dev, PCMK__XA_RSC_PROVIDES);
1073 if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
1074 device->automatic_unfencing = TRUE;
1075 }
1076
1077 if (is_action_required(PCMK_ACTION_ON, device)) {
1078 crm_info("Fencing device '%s' requires unfencing", device->id);
1079 }
1080
1081 if (device->on_target_actions != NULL) {
1082 crm_info("Fencing device '%s' requires actions (%s) to be executed "
1083 "on target", device->id,
1084 (const char *) device->on_target_actions->str);
1085 }
1086
1087 device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
1088
1089
1090 return device;
1091 }
1092
1093 static void
1094 schedule_internal_command(const char *origin,
1095 stonith_device_t * device,
1096 const char *action,
1097 const char *target,
1098 int timeout,
1099 void *internal_user_data,
1100 void (*done_cb) (int pid,
1101 const pcmk__action_result_t *result,
1102 void *user_data))
1103 {
1104 async_command_t *cmd = NULL;
1105
1106 cmd = pcmk__assert_alloc(1, sizeof(async_command_t));
1107
1108 cmd->id = -1;
1109 cmd->default_timeout = timeout ? timeout : 60;
1110 cmd->timeout = cmd->default_timeout;
1111 cmd->action = pcmk__str_copy(action);
1112 cmd->target = pcmk__str_copy(target);
1113 cmd->device = pcmk__str_copy(device->id);
1114 cmd->origin = pcmk__str_copy(origin);
1115 cmd->client = pcmk__str_copy(crm_system_name);
1116 cmd->client_name = pcmk__str_copy(crm_system_name);
1117
1118 cmd->internal_user_data = internal_user_data;
1119 cmd->done_cb = done_cb;
1120
1121 schedule_stonith_command(cmd, device);
1122 }
1123
1124
1125 enum fence_status_code {
1126 fence_status_invalid = -1,
1127 fence_status_active = 0,
1128 fence_status_unknown = 1,
1129 fence_status_inactive = 2,
1130 };
1131
1132 static void
1133 status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data)
1134 {
1135 async_command_t *cmd = user_data;
1136 struct device_search_s *search = cmd->internal_user_data;
1137 stonith_device_t *dev = cmd_device(cmd);
1138 gboolean can = FALSE;
1139
1140 free_async_command(cmd);
1141
1142 if (!dev) {
1143 search_devices_record_result(search, NULL, FALSE);
1144 return;
1145 }
1146
1147 mainloop_set_trigger(dev->work);
1148
1149 if (result->execution_status != PCMK_EXEC_DONE) {
1150 crm_warn("Assuming %s cannot fence %s "
1151 "because status could not be executed: %s%s%s%s",
1152 dev->id, search->host,
1153 pcmk_exec_status_str(result->execution_status),
1154 ((result->exit_reason == NULL)? "" : " ("),
1155 ((result->exit_reason == NULL)? "" : result->exit_reason),
1156 ((result->exit_reason == NULL)? "" : ")"));
1157 search_devices_record_result(search, dev->id, FALSE);
1158 return;
1159 }
1160
1161 switch (result->exit_status) {
1162 case fence_status_unknown:
1163 crm_trace("%s reported it cannot fence %s", dev->id, search->host);
1164 break;
1165
1166 case fence_status_active:
1167 case fence_status_inactive:
1168 crm_trace("%s reported it can fence %s", dev->id, search->host);
1169 can = TRUE;
1170 break;
1171
1172 default:
1173 crm_warn("Assuming %s cannot fence %s "
1174 "(status returned unknown code %d)",
1175 dev->id, search->host, result->exit_status);
1176 break;
1177 }
1178 search_devices_record_result(search, dev->id, can);
1179 }
1180
1181 static void
1182 dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
1183 void *user_data)
1184 {
1185 async_command_t *cmd = user_data;
1186 struct device_search_s *search = cmd->internal_user_data;
1187 stonith_device_t *dev = cmd_device(cmd);
1188 gboolean can_fence = FALSE;
1189
1190 free_async_command(cmd);
1191
1192
1193
1194
1195
1196
1197 if (!dev) {
1198 search_devices_record_result(search, NULL, FALSE);
1199 return;
1200 }
1201
1202 mainloop_set_trigger(dev->work);
1203
1204 if (pcmk__result_ok(result)) {
1205 crm_info("Refreshing target list for %s", dev->id);
1206 g_list_free_full(dev->targets, free);
1207 dev->targets = stonith__parse_targets(result->action_stdout);
1208 dev->targets_age = time(NULL);
1209
1210 } else if (dev->targets != NULL) {
1211 if (result->execution_status == PCMK_EXEC_DONE) {
1212 crm_info("Reusing most recent target list for %s "
1213 "because list returned error code %d",
1214 dev->id, result->exit_status);
1215 } else {
1216 crm_info("Reusing most recent target list for %s "
1217 "because list could not be executed: %s%s%s%s",
1218 dev->id, pcmk_exec_status_str(result->execution_status),
1219 ((result->exit_reason == NULL)? "" : " ("),
1220 ((result->exit_reason == NULL)? "" : result->exit_reason),
1221 ((result->exit_reason == NULL)? "" : ")"));
1222 }
1223
1224 } else {
1225 if (result->execution_status == PCMK_EXEC_DONE) {
1226 crm_warn("Assuming %s cannot fence %s "
1227 "because list returned error code %d",
1228 dev->id, search->host, result->exit_status);
1229 } else {
1230 crm_warn("Assuming %s cannot fence %s "
1231 "because list could not be executed: %s%s%s%s",
1232 dev->id, search->host,
1233 pcmk_exec_status_str(result->execution_status),
1234 ((result->exit_reason == NULL)? "" : " ("),
1235 ((result->exit_reason == NULL)? "" : result->exit_reason),
1236 ((result->exit_reason == NULL)? "" : ")"));
1237 }
1238
1239
1240
1241
1242 if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
1243 crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
1244 pcmk__insert_dup(dev->params, PCMK_STONITH_HOST_CHECK,
1245 PCMK_VALUE_STATUS);
1246 }
1247 }
1248
1249 if (dev->targets) {
1250 const char *alias = g_hash_table_lookup(dev->aliases, search->host);
1251
1252 if (!alias) {
1253 alias = search->host;
1254 }
1255 if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
1256 can_fence = TRUE;
1257 }
1258 }
1259 search_devices_record_result(search, dev->id, can_fence);
1260 }
1261
1262
1263
1264
1265
1266 static int
1267 device_params_diff(GHashTable *first, GHashTable *second) {
1268 char *key = NULL;
1269 char *value = NULL;
1270 GHashTableIter gIter;
1271
1272 g_hash_table_iter_init(&gIter, first);
1273 while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
1274
1275 if(strstr(key, "CRM_meta") == key) {
1276 continue;
1277 } else if (strcmp(key, PCMK_XA_CRM_FEATURE_SET) == 0) {
1278 continue;
1279 } else {
1280 char *other_value = g_hash_table_lookup(second, key);
1281
1282 if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
1283 crm_trace("Different value for %s: %s != %s", key, other_value, value);
1284 return 1;
1285 }
1286 }
1287 }
1288
1289 return 0;
1290 }
1291
1292
1293
1294
1295
1296 static stonith_device_t *
1297 device_has_duplicate(const stonith_device_t *device)
1298 {
1299 stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);
1300
1301 if (!dup) {
1302 crm_trace("No match for %s", device->id);
1303 return NULL;
1304
1305 } else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
1306 crm_trace("Different agent: %s != %s", dup->agent, device->agent);
1307 return NULL;
1308 }
1309
1310
1311 if (device_params_diff(device->params, dup->params) ||
1312 device_params_diff(dup->params, device->params)) {
1313 return NULL;
1314 }
1315
1316 crm_trace("Match");
1317 return dup;
1318 }
1319
1320 int
1321 stonith_device_register(xmlNode *dev, gboolean from_cib)
1322 {
1323 stonith_device_t *dup = NULL;
1324 stonith_device_t *device = build_device_from_xml(dev);
1325 guint ndevices = 0;
1326 int rv = pcmk_ok;
1327
1328 CRM_CHECK(device != NULL, return -ENOMEM);
1329
1330
1331 if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) ||
1332 pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
1333 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do {
1334 if (stonith_watchdog_timeout_ms <= 0) {
1335 crm_err("Ignoring watchdog fence device without "
1336 PCMK_OPT_STONITH_WATCHDOG_TIMEOUT " set.");
1337 rv = -ENODEV;
1338
1339 } else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
1340 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
1341 crm_err("Ignoring watchdog fence device with unknown "
1342 "agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.",
1343 device->agent?device->agent:"");
1344 rv = -ENODEV;
1345
1346 } else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID,
1347 pcmk__str_none)) {
1348 crm_err("Ignoring watchdog fence device "
1349 "named %s !='"STONITH_WATCHDOG_ID"'.",
1350 device->id?device->id:"");
1351 rv = -ENODEV;
1352
1353 } else {
1354 const char *local_node_name = fenced_get_local_node();
1355
1356 if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
1357 pcmk__str_none)) {
1358
1359
1360
1361 g_list_free_full(stonith_watchdog_targets, free);
1362 stonith_watchdog_targets = device->targets;
1363 device->targets = NULL;
1364 }
1365 if (node_does_watchdog_fencing(local_node_name)) {
1366 g_list_free_full(device->targets, free);
1367 device->targets = stonith__parse_targets(local_node_name);
1368 pcmk__insert_dup(device->params,
1369 PCMK_STONITH_HOST_LIST, local_node_name);
1370
1371 break;
1372 }
1373
1374 crm_debug("Skip registration of watchdog fence device on node not in host-list.");
1375
1376 device->targets = NULL;
1377 stonith_device_remove(device->id, from_cib);
1378 }
1379 free_device(device);
1380 return rv;
1381 } while (0);
1382
1383 dup = device_has_duplicate(device);
1384 if (dup) {
1385 ndevices = g_hash_table_size(device_list);
1386 crm_debug("Device '%s' already in device list (%d active device%s)",
1387 device->id, ndevices, pcmk__plural_s(ndevices));
1388 free_device(device);
1389 device = dup;
1390 dup = g_hash_table_lookup(device_list, device->id);
1391 dup->dirty = FALSE;
1392
1393 } else {
1394 stonith_device_t *old = g_hash_table_lookup(device_list, device->id);
1395
1396 if (from_cib && old && old->api_registered) {
1397
1398
1399
1400
1401 crm_info("Overwriting existing entry for %s from CIB", device->id);
1402 device->pending_ops = old->pending_ops;
1403 device->api_registered = TRUE;
1404 old->pending_ops = NULL;
1405 if (device->pending_ops) {
1406 mainloop_set_trigger(device->work);
1407 }
1408 }
1409 g_hash_table_replace(device_list, device->id, device);
1410
1411 ndevices = g_hash_table_size(device_list);
1412 crm_notice("Added '%s' to device list (%d active device%s)",
1413 device->id, ndevices, pcmk__plural_s(ndevices));
1414 }
1415
1416 if (from_cib) {
1417 device->cib_registered = TRUE;
1418 } else {
1419 device->api_registered = TRUE;
1420 }
1421
1422 return pcmk_ok;
1423 }
1424
1425 void
1426 stonith_device_remove(const char *id, bool from_cib)
1427 {
1428 stonith_device_t *device = g_hash_table_lookup(device_list, id);
1429 guint ndevices = 0;
1430
1431 if (!device) {
1432 ndevices = g_hash_table_size(device_list);
1433 crm_info("Device '%s' not found (%d active device%s)",
1434 id, ndevices, pcmk__plural_s(ndevices));
1435 return;
1436 }
1437
1438 if (from_cib) {
1439 device->cib_registered = FALSE;
1440 } else {
1441 device->verified = FALSE;
1442 device->api_registered = FALSE;
1443 }
1444
1445 if (!device->cib_registered && !device->api_registered) {
1446 g_hash_table_remove(device_list, id);
1447 ndevices = g_hash_table_size(device_list);
1448 crm_info("Removed '%s' from device list (%d active device%s)",
1449 id, ndevices, pcmk__plural_s(ndevices));
1450 } else {
1451 crm_trace("Not removing '%s' from device list (%d active) because "
1452 "still registered via:%s%s",
1453 id, g_hash_table_size(device_list),
1454 (device->cib_registered? " cib" : ""),
1455 (device->api_registered? " api" : ""));
1456 }
1457 }
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468 static int
1469 count_active_levels(const stonith_topology_t *tp)
1470 {
1471 int lpc = 0;
1472 int count = 0;
1473
1474 for (lpc = 0; lpc < ST__LEVEL_COUNT; lpc++) {
1475 if (tp->levels[lpc] != NULL) {
1476 count++;
1477 }
1478 }
1479 return count;
1480 }
1481
1482 static void
1483 free_topology_entry(gpointer data)
1484 {
1485 stonith_topology_t *tp = data;
1486
1487 int lpc = 0;
1488
1489 for (lpc = 0; lpc < ST__LEVEL_COUNT; lpc++) {
1490 if (tp->levels[lpc] != NULL) {
1491 g_list_free_full(tp->levels[lpc], free);
1492 }
1493 }
1494 free(tp->target);
1495 free(tp->target_value);
1496 free(tp->target_pattern);
1497 free(tp->target_attribute);
1498 free(tp);
1499 }
1500
1501 void
1502 free_topology_list(void)
1503 {
1504 if (topology != NULL) {
1505 g_hash_table_destroy(topology);
1506 topology = NULL;
1507 }
1508 }
1509
1510 void
1511 init_topology_list(void)
1512 {
1513 if (topology == NULL) {
1514 topology = pcmk__strkey_table(NULL, free_topology_entry);
1515 }
1516 }
1517
1518 char *
1519 stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
1520 {
1521 if (mode == fenced_target_by_unknown) {
1522 mode = unpack_level_kind(level);
1523 }
1524 switch (mode) {
1525 case fenced_target_by_name:
1526 return crm_element_value_copy(level, PCMK_XA_TARGET);
1527
1528 case fenced_target_by_pattern:
1529 return crm_element_value_copy(level, PCMK_XA_TARGET_PATTERN);
1530
1531 case fenced_target_by_attribute:
1532 return crm_strdup_printf("%s=%s",
1533 crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE),
1534 crm_element_value(level, PCMK_XA_TARGET_VALUE));
1535
1536 default:
1537 return crm_strdup_printf("unknown-%s", pcmk__xe_id(level));
1538 }
1539 }
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549 static enum fenced_target_by
1550 unpack_level_kind(const xmlNode *level)
1551 {
1552 if (crm_element_value(level, PCMK_XA_TARGET) != NULL) {
1553 return fenced_target_by_name;
1554 }
1555 if (crm_element_value(level, PCMK_XA_TARGET_PATTERN) != NULL) {
1556 return fenced_target_by_pattern;
1557 }
1558 if ((crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE) != NULL)
1559 && (crm_element_value(level, PCMK_XA_TARGET_VALUE) != NULL)) {
1560 return fenced_target_by_attribute;
1561 }
1562 return fenced_target_by_unknown;
1563 }
1564
1565 static stonith_key_value_t *
1566 parse_device_list(const char *devices)
1567 {
1568 int lpc = 0;
1569 int max = 0;
1570 int last = 0;
1571 stonith_key_value_t *output = NULL;
1572
1573 if (devices == NULL) {
1574 return output;
1575 }
1576
1577 max = strlen(devices);
1578 for (lpc = 0; lpc <= max; lpc++) {
1579 if (devices[lpc] == ',' || devices[lpc] == 0) {
1580 char *line = strndup(devices + last, lpc - last);
1581
1582 output = stonith_key_value_add(output, NULL, line);
1583 free(line);
1584
1585 last = lpc + 1;
1586 }
1587 }
1588
1589 return output;
1590 }
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605 static xmlNode *
1606 unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target,
1607 int *id, char **desc)
1608 {
1609 enum fenced_target_by local_mode = fenced_target_by_unknown;
1610 char *local_target = NULL;
1611 int local_id = 0;
1612
1613
1614
1615
1616
1617 if ((xml != NULL) && !pcmk__xe_is(xml, PCMK_XE_FENCING_LEVEL)) {
1618 xml = get_xpath_object("//" PCMK_XE_FENCING_LEVEL, xml, LOG_WARNING);
1619 }
1620
1621 if (xml == NULL) {
1622 if (desc != NULL) {
1623 *desc = crm_strdup_printf("missing");
1624 }
1625 } else {
1626 local_mode = unpack_level_kind(xml);
1627 local_target = stonith_level_key(xml, local_mode);
1628 crm_element_value_int(xml, PCMK_XA_INDEX, &local_id);
1629 if (desc != NULL) {
1630 *desc = crm_strdup_printf("%s[%d]", local_target, local_id);
1631 }
1632 }
1633
1634 if (mode != NULL) {
1635 *mode = local_mode;
1636 }
1637 if (id != NULL) {
1638 *id = local_id;
1639 }
1640
1641 if (target != NULL) {
1642 *target = local_target;
1643 } else {
1644 free(local_target);
1645 }
1646
1647 return xml;
1648 }
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663 void
1664 fenced_register_level(xmlNode *msg, char **desc, pcmk__action_result_t *result)
1665 {
1666 int id = 0;
1667 xmlNode *level;
1668 enum fenced_target_by mode;
1669 char *target;
1670
1671 stonith_topology_t *tp;
1672 stonith_key_value_t *dIter = NULL;
1673 stonith_key_value_t *devices = NULL;
1674
1675 CRM_CHECK((msg != NULL) && (result != NULL), return);
1676
1677 level = unpack_level_request(msg, &mode, &target, &id, desc);
1678 if (level == NULL) {
1679 fenced_set_protocol_error(result);
1680 return;
1681 }
1682
1683
1684 if (pcmk__str_empty(pcmk__xe_id(level))) {
1685 crm_warn("Ignoring registration for topology level without ID");
1686 free(target);
1687 crm_log_xml_trace(level, "Bad level");
1688 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1689 "Topology level is invalid without ID");
1690 return;
1691 }
1692
1693
1694 if (mode == fenced_target_by_unknown) {
1695 crm_warn("Ignoring registration for topology level '%s' "
1696 "without valid target", pcmk__xe_id(level));
1697 free(target);
1698 crm_log_xml_trace(level, "Bad level");
1699 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1700 "Invalid target for topology level '%s'",
1701 pcmk__xe_id(level));
1702 return;
1703 }
1704
1705
1706 if ((id < ST__LEVEL_MIN) || (id > ST__LEVEL_MAX)) {
1707 crm_warn("Ignoring topology registration for %s with invalid level %d",
1708 target, id);
1709 free(target);
1710 crm_log_xml_trace(level, "Bad level");
1711 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1712 "Invalid level number '%s' for topology level '%s'",
1713 pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
1714 ""),
1715 pcmk__xe_id(level));
1716 return;
1717 }
1718
1719
1720 tp = g_hash_table_lookup(topology, target);
1721 if (tp == NULL) {
1722 tp = pcmk__assert_alloc(1, sizeof(stonith_topology_t));
1723
1724 tp->kind = mode;
1725 tp->target = target;
1726 tp->target_value = crm_element_value_copy(level, PCMK_XA_TARGET_VALUE);
1727 tp->target_pattern = crm_element_value_copy(level,
1728 PCMK_XA_TARGET_PATTERN);
1729 tp->target_attribute = crm_element_value_copy(level,
1730 PCMK_XA_TARGET_ATTRIBUTE);
1731
1732 g_hash_table_replace(topology, tp->target, tp);
1733 crm_trace("Added %s (%d) to the topology (%d active entries)",
1734 target, (int) mode, g_hash_table_size(topology));
1735 } else {
1736 free(target);
1737 }
1738
1739 if (tp->levels[id] != NULL) {
1740 crm_info("Adding to the existing %s[%d] topology entry",
1741 tp->target, id);
1742 }
1743
1744 devices = parse_device_list(crm_element_value(level, PCMK_XA_DEVICES));
1745 for (dIter = devices; dIter; dIter = dIter->next) {
1746 const char *device = dIter->value;
1747
1748 crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
1749 tp->levels[id] = g_list_append(tp->levels[id], pcmk__str_copy(device));
1750 }
1751 stonith_key_value_freeall(devices, 1, 1);
1752
1753 {
1754 int nlevels = count_active_levels(tp);
1755
1756 crm_info("Target %s has %d active fencing level%s",
1757 tp->target, nlevels, pcmk__plural_s(nlevels));
1758 }
1759
1760 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1761 }
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775 void
1776 fenced_unregister_level(xmlNode *msg, char **desc,
1777 pcmk__action_result_t *result)
1778 {
1779 int id = -1;
1780 stonith_topology_t *tp;
1781 char *target;
1782 xmlNode *level = NULL;
1783
1784 CRM_CHECK(result != NULL, return);
1785
1786 level = unpack_level_request(msg, NULL, &target, &id, desc);
1787 if (level == NULL) {
1788 fenced_set_protocol_error(result);
1789 return;
1790 }
1791
1792
1793 if ((id < 0) || (id >= ST__LEVEL_COUNT)) {
1794 crm_warn("Ignoring topology unregistration for %s with invalid level %d",
1795 target, id);
1796 free(target);
1797 crm_log_xml_trace(level, "Bad level");
1798 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1799 "Invalid level number '%s' for topology level %s",
1800 pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
1801 "<null>"),
1802
1803
1804 pcmk__s(pcmk__xe_id(level), ""));
1805 return;
1806 }
1807
1808 tp = g_hash_table_lookup(topology, target);
1809 if (tp == NULL) {
1810 guint nentries = g_hash_table_size(topology);
1811
1812 crm_info("No fencing topology found for %s (%d active %s)",
1813 target, nentries,
1814 pcmk__plural_alt(nentries, "entry", "entries"));
1815
1816 } else if (id == 0 && g_hash_table_remove(topology, target)) {
1817 guint nentries = g_hash_table_size(topology);
1818
1819 crm_info("Removed all fencing topology entries related to %s "
1820 "(%d active %s remaining)", target, nentries,
1821 pcmk__plural_alt(nentries, "entry", "entries"));
1822
1823 } else if (tp->levels[id] != NULL) {
1824 guint nlevels;
1825
1826 g_list_free_full(tp->levels[id], free);
1827 tp->levels[id] = NULL;
1828
1829 nlevels = count_active_levels(tp);
1830 crm_info("Removed level %d from fencing topology for %s "
1831 "(%d active level%s remaining)",
1832 id, target, nlevels, pcmk__plural_s(nlevels));
1833 }
1834
1835 free(target);
1836 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1837 }
1838
1839 static char *
1840 list_to_string(GList *list, const char *delim, gboolean terminate_with_delim)
1841 {
1842 int max = g_list_length(list);
1843 size_t delim_len = delim?strlen(delim):0;
1844 size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
1845 char *rv;
1846 GList *gIter;
1847
1848 char *pos = NULL;
1849 const char *lead_delim = "";
1850
1851 for (gIter = list; gIter != NULL; gIter = gIter->next) {
1852 const char *value = (const char *) gIter->data;
1853
1854 alloc_size += strlen(value);
1855 }
1856
1857 rv = pcmk__assert_alloc(alloc_size, sizeof(char));
1858 pos = rv;
1859
1860 for (gIter = list; gIter != NULL; gIter = gIter->next) {
1861 const char *value = (const char *) gIter->data;
1862
1863 pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
1864 lead_delim = delim;
1865 }
1866
1867 if (max && terminate_with_delim) {
1868 sprintf(pos, "%s", delim);
1869 }
1870
1871 return rv;
1872 }
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889 static void
1890 execute_agent_action(xmlNode *msg, pcmk__action_result_t *result)
1891 {
1892 xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, msg, LOG_ERR);
1893 xmlNode *op = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, msg,
1894 LOG_ERR);
1895 const char *id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
1896 const char *action = crm_element_value(op, PCMK__XA_ST_DEVICE_ACTION);
1897 async_command_t *cmd = NULL;
1898 stonith_device_t *device = NULL;
1899
1900 if ((id == NULL) || (action == NULL)) {
1901 crm_info("Malformed API action request: device %s, action %s",
1902 (id? id : "not specified"),
1903 (action? action : "not specified"));
1904 fenced_set_protocol_error(result);
1905 return;
1906 }
1907
1908 if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
1909
1910 if (stonith_watchdog_timeout_ms <= 0) {
1911 pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1912 "Watchdog fence device not configured");
1913 return;
1914
1915 } else if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
1916 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1917 pcmk__set_result_output(result,
1918 list_to_string(stonith_watchdog_targets,
1919 "\n", TRUE),
1920 NULL);
1921 return;
1922
1923 } else if (pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_none)) {
1924 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1925 return;
1926 }
1927 }
1928
1929 device = g_hash_table_lookup(device_list, id);
1930 if (device == NULL) {
1931 crm_info("Ignoring API '%s' action request because device %s not found",
1932 action, id);
1933 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1934 "'%s' not found", id);
1935 return;
1936
1937 } else if (!device->api_registered
1938 && (strcmp(action, PCMK_ACTION_MONITOR) == 0)) {
1939
1940 crm_info("Ignoring API '%s' action request because device %s not active",
1941 action, id);
1942 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1943 "'%s' not active", id);
1944 return;
1945 }
1946
1947 cmd = create_async_command(msg);
1948 if (cmd == NULL) {
1949 crm_log_xml_warn(msg, "invalid");
1950 fenced_set_protocol_error(result);
1951 return;
1952 }
1953
1954 schedule_stonith_command(cmd, device);
1955 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
1956 }
1957
1958 static void
1959 search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
1960 {
1961 search->replies_received++;
1962 if (can_fence && device) {
1963 if (search->support_action_only != st_device_supports_none) {
1964 stonith_device_t *dev = g_hash_table_lookup(device_list, device);
1965 if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
1966 return;
1967 }
1968 }
1969 search->capable = g_list_append(search->capable,
1970 pcmk__str_copy(device));
1971 }
1972
1973 if (search->replies_needed == search->replies_received) {
1974
1975 guint ndevices = g_list_length(search->capable);
1976
1977 crm_debug("Search found %d device%s that can perform '%s' targeting %s",
1978 ndevices, pcmk__plural_s(ndevices),
1979 (search->action? search->action : "unknown action"),
1980 (search->host? search->host : "any node"));
1981
1982 search->callback(search->capable, search->user_data);
1983 free(search->host);
1984 free(search->action);
1985 free(search);
1986 }
1987 }
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000 static gboolean
2001 localhost_is_eligible(const stonith_device_t *device, const char *action,
2002 const char *target, gboolean allow_self)
2003 {
2004 gboolean localhost_is_target = pcmk__str_eq(target, fenced_get_local_node(),
2005 pcmk__str_casei);
2006
2007 if ((device != NULL) && (action != NULL)
2008 && (device->on_target_actions != NULL)
2009 && (strstr((const char*) device->on_target_actions->str,
2010 action) != NULL)) {
2011
2012 if (!localhost_is_target) {
2013 crm_trace("Operation '%s' using %s can only be executed for local "
2014 "host, not %s", action, device->id, target);
2015 return FALSE;
2016 }
2017
2018 } else if (localhost_is_target && !allow_self) {
2019 crm_trace("'%s' operation does not support self-fencing", action);
2020 return FALSE;
2021 }
2022 return TRUE;
2023 }
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037 static bool
2038 localhost_is_eligible_with_remap(const stonith_device_t *device,
2039 const char *action, const char *target,
2040 gboolean allow_self)
2041 {
2042
2043 if (localhost_is_eligible(device, action, target, allow_self)) {
2044 return true;
2045 }
2046
2047
2048
2049 if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
2050
2051
2052
2053
2054
2055 if (localhost_is_eligible(device, PCMK_ACTION_OFF, target, allow_self)
2056 || localhost_is_eligible(device, PCMK_ACTION_ON, target, FALSE)) {
2057 return true;
2058 }
2059 }
2060
2061 return false;
2062 }
2063
2064 static void
2065 can_fence_host_with_device(stonith_device_t *dev,
2066 struct device_search_s *search)
2067 {
2068 gboolean can = FALSE;
2069 const char *check_type = "Internal bug";
2070 const char *target = NULL;
2071 const char *alias = NULL;
2072 const char *dev_id = "Unspecified device";
2073 const char *action = (search == NULL)? NULL : search->action;
2074
2075 CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results);
2076
2077 if (dev->id != NULL) {
2078 dev_id = dev->id;
2079 }
2080
2081 target = search->host;
2082 if (target == NULL) {
2083 can = TRUE;
2084 check_type = "No target";
2085 goto search_report_results;
2086 }
2087
2088
2089
2090
2091 if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)
2092 && !pcmk_is_set(dev->flags, st_device_supports_on)) {
2093 check_type = "Agent does not support 'on'";
2094 goto search_report_results;
2095
2096 } else if (!localhost_is_eligible_with_remap(dev, action, target,
2097 search->allow_self)) {
2098 check_type = "This node is not allowed to execute action";
2099 goto search_report_results;
2100 }
2101
2102
2103 check_type = target_list_type(dev);
2104 alias = g_hash_table_lookup(dev->aliases, target);
2105 if (pcmk__str_eq(check_type, PCMK_VALUE_NONE, pcmk__str_casei)) {
2106 can = TRUE;
2107
2108 } else if (pcmk__str_eq(check_type, PCMK_VALUE_STATIC_LIST,
2109 pcmk__str_casei)) {
2110
2111 if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) {
2112 can = TRUE;
2113 } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
2114 && g_hash_table_lookup(dev->aliases, target)) {
2115 can = TRUE;
2116 }
2117
2118 } else if (pcmk__str_eq(check_type, PCMK_VALUE_DYNAMIC_LIST,
2119 pcmk__str_casei)) {
2120 time_t now = time(NULL);
2121
2122 if (dev->targets == NULL || dev->targets_age + 60 < now) {
2123 int device_timeout = get_action_timeout(dev, PCMK_ACTION_LIST,
2124 search->per_device_timeout);
2125
2126 if (device_timeout > search->per_device_timeout) {
2127 crm_notice("Since the pcmk_list_timeout (%ds) parameter of %s "
2128 "is larger than " PCMK_OPT_STONITH_TIMEOUT
2129 " (%ds), timeout may occur",
2130 device_timeout, dev_id, search->per_device_timeout);
2131 }
2132
2133 crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
2134 check_type, dev_id, target, action);
2135
2136 schedule_internal_command(__func__, dev, PCMK_ACTION_LIST, NULL,
2137 search->per_device_timeout, search, dynamic_list_search_cb);
2138
2139
2140 return;
2141 }
2142
2143 if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets,
2144 pcmk__str_casei)) {
2145 can = TRUE;
2146 }
2147
2148 } else if (pcmk__str_eq(check_type, PCMK_VALUE_STATUS, pcmk__str_casei)) {
2149 int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);
2150
2151 if (device_timeout > search->per_device_timeout) {
2152 crm_notice("Since the pcmk_status_timeout (%ds) parameter of %s is "
2153 "larger than " PCMK_OPT_STONITH_TIMEOUT " (%ds), "
2154 "timeout may occur",
2155 device_timeout, dev_id, search->per_device_timeout);
2156 }
2157
2158 crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
2159 check_type, dev_id, target, action);
2160 schedule_internal_command(__func__, dev, PCMK_ACTION_STATUS, target,
2161 search->per_device_timeout, search, status_search_cb);
2162
2163 return;
2164 } else {
2165 crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
2166 check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
2167 }
2168
2169 search_report_results:
2170 crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s",
2171 dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"),
2172 pcmk__s(target, "unspecified target"),
2173 (alias == NULL)? "" : " (as '", pcmk__s(alias, ""),
2174 (alias == NULL)? "" : "')", check_type);
2175 search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can);
2176 }
2177
2178 static void
2179 search_devices(gpointer key, gpointer value, gpointer user_data)
2180 {
2181 stonith_device_t *dev = value;
2182 struct device_search_s *search = user_data;
2183
2184 can_fence_host_with_device(dev, search);
2185 }
2186
2187 #define DEFAULT_QUERY_TIMEOUT 20
2188 static void
2189 get_capable_devices(const char *host, const char *action, int timeout,
2190 bool allow_self, void *user_data,
2191 void (*callback) (GList * devices, void *user_data),
2192 uint32_t support_action_only)
2193 {
2194 struct device_search_s *search;
2195 guint ndevices = g_hash_table_size(device_list);
2196
2197 if (ndevices == 0) {
2198 callback(NULL, user_data);
2199 return;
2200 }
2201
2202 search = pcmk__assert_alloc(1, sizeof(struct device_search_s));
2203
2204 search->host = pcmk__str_copy(host);
2205 search->action = pcmk__str_copy(action);
2206 search->per_device_timeout = timeout;
2207 search->allow_self = allow_self;
2208 search->callback = callback;
2209 search->user_data = user_data;
2210 search->support_action_only = support_action_only;
2211
2212
2213
2214
2215 search->replies_needed = ndevices;
2216
2217 crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
2218 ndevices, pcmk__plural_s(ndevices),
2219 (search->action? search->action : "unknown action"),
2220 (search->host? search->host : "any node"));
2221 g_hash_table_foreach(device_list, search_devices, search);
2222 }
2223
2224 struct st_query_data {
2225 xmlNode *reply;
2226 char *remote_peer;
2227 char *client_id;
2228 char *target;
2229 char *action;
2230 int call_options;
2231 };
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242 static void
2243 add_action_specific_attributes(xmlNode *xml, const char *action,
2244 const stonith_device_t *device,
2245 const char *target)
2246 {
2247 int action_specific_timeout;
2248 int delay_max;
2249 int delay_base;
2250
2251 CRM_CHECK(xml && action && device, return);
2252
2253
2254 if (is_action_required(action, device)) {
2255 crm_trace("Action '%s' is required using %s", action, device->id);
2256 crm_xml_add_int(xml, PCMK__XA_ST_REQUIRED, 1);
2257 }
2258
2259
2260 action_specific_timeout = get_action_timeout(device, action, 0);
2261 if (action_specific_timeout) {
2262 crm_trace("Action '%s' has timeout %ds using %s",
2263 action, action_specific_timeout, device->id);
2264 crm_xml_add_int(xml, PCMK__XA_ST_ACTION_TIMEOUT,
2265 action_specific_timeout);
2266 }
2267
2268 delay_max = get_action_delay_max(device, action);
2269 if (delay_max > 0) {
2270 crm_trace("Action '%s' has maximum random delay %ds using %s",
2271 action, delay_max, device->id);
2272 crm_xml_add_int(xml, PCMK__XA_ST_DELAY_MAX, delay_max);
2273 }
2274
2275 delay_base = get_action_delay_base(device, action, target);
2276 if (delay_base > 0) {
2277 crm_xml_add_int(xml, PCMK__XA_ST_DELAY_BASE, delay_base);
2278 }
2279
2280 if ((delay_max > 0) && (delay_base == 0)) {
2281 crm_trace("Action '%s' has maximum random delay %ds using %s",
2282 action, delay_max, device->id);
2283 } else if ((delay_max == 0) && (delay_base > 0)) {
2284 crm_trace("Action '%s' has a static delay of %ds using %s",
2285 action, delay_base, device->id);
2286 } else if ((delay_max > 0) && (delay_base > 0)) {
2287 crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen "
2288 "maximum delay of %ds using %s",
2289 action, delay_base, delay_max, device->id);
2290 }
2291 }
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303 static void
2304 add_disallowed(xmlNode *xml, const char *action, const stonith_device_t *device,
2305 const char *target, gboolean allow_self)
2306 {
2307 if (!localhost_is_eligible(device, action, target, allow_self)) {
2308 crm_trace("Action '%s' using %s is disallowed for local host",
2309 action, device->id);
2310 pcmk__xe_set_bool_attr(xml, PCMK__XA_ST_ACTION_DISALLOWED, true);
2311 }
2312 }
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324 static void
2325 add_action_reply(xmlNode *xml, const char *action,
2326 const stonith_device_t *device, const char *target,
2327 gboolean allow_self)
2328 {
2329 xmlNode *child = pcmk__xe_create(xml, PCMK__XE_ST_DEVICE_ACTION);
2330
2331 crm_xml_add(child, PCMK_XA_ID, action);
2332 add_action_specific_attributes(child, action, device, target);
2333 add_disallowed(child, action, device, target, allow_self);
2334 }
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345 static void
2346 stonith_send_reply(const xmlNode *reply, int call_options,
2347 const char *remote_peer, pcmk__client_t *client)
2348 {
2349 CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)),
2350 return);
2351
2352 if (remote_peer == NULL) {
2353 do_local_reply(reply, client, call_options);
2354 } else {
2355 const pcmk__node_status_t *node =
2356 pcmk__get_node(0, remote_peer, NULL,
2357 pcmk__node_search_cluster_member);
2358
2359 pcmk__cluster_send_message(node, pcmk_ipc_fenced, reply);
2360 }
2361 }
2362
2363 static void
2364 stonith_query_capable_device_cb(GList * devices, void *user_data)
2365 {
2366 struct st_query_data *query = user_data;
2367 int available_devices = 0;
2368 xmlNode *wrapper = NULL;
2369 xmlNode *list = NULL;
2370 GList *lpc = NULL;
2371 pcmk__client_t *client = NULL;
2372
2373 if (query->client_id != NULL) {
2374 client = pcmk__find_client_by_id(query->client_id);
2375 if ((client == NULL) && (query->remote_peer == NULL)) {
2376 crm_trace("Skipping reply to %s: no longer a client",
2377 query->client_id);
2378 goto done;
2379 }
2380 }
2381
2382
2383 wrapper = pcmk__xe_create(query->reply, PCMK__XE_ST_CALLDATA);
2384 list = pcmk__xe_create(wrapper, __func__);
2385 crm_xml_add(list, PCMK__XA_ST_TARGET, query->target);
2386
2387 for (lpc = devices; lpc != NULL; lpc = lpc->next) {
2388 stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
2389 const char *action = query->action;
2390 xmlNode *dev = NULL;
2391
2392 if (!device) {
2393
2394
2395 continue;
2396 }
2397
2398 available_devices++;
2399
2400 dev = pcmk__xe_create(list, PCMK__XE_ST_DEVICE_ID);
2401 crm_xml_add(dev, PCMK_XA_ID, device->id);
2402 crm_xml_add(dev, PCMK__XA_NAMESPACE, device->namespace);
2403 crm_xml_add(dev, PCMK_XA_AGENT, device->agent);
2404
2405
2406 crm_xml_add_int(dev, PCMK__XA_ST_MONITOR_VERIFIED, device->verified);
2407
2408 crm_xml_add_int(dev, PCMK__XA_ST_DEVICE_SUPPORT_FLAGS, device->flags);
2409
2410
2411
2412
2413 if (!pcmk_is_set(device->flags, st_device_supports_reboot)
2414 && pcmk__str_eq(query->action, PCMK_ACTION_REBOOT,
2415 pcmk__str_none)) {
2416 crm_trace("%s doesn't support reboot, using values for off instead",
2417 device->id);
2418 action = PCMK_ACTION_OFF;
2419 }
2420
2421
2422 add_action_specific_attributes(dev, action, device, query->target);
2423 if (pcmk__str_eq(query->action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435 add_disallowed(dev, action, device, query->target,
2436 pcmk_is_set(query->call_options,
2437 st_opt_allow_self_fencing));
2438 add_action_reply(dev, PCMK_ACTION_OFF, device, query->target,
2439 pcmk_is_set(query->call_options,
2440 st_opt_allow_self_fencing));
2441 add_action_reply(dev, PCMK_ACTION_ON, device, query->target, FALSE);
2442 }
2443
2444
2445 if (query->target == NULL) {
2446 xmlNode *attrs = pcmk__xe_create(dev, PCMK__XE_ATTRIBUTES);
2447
2448 g_hash_table_foreach(device->params, hash2field, attrs);
2449 }
2450 }
2451
2452 crm_xml_add_int(list, PCMK__XA_ST_AVAILABLE_DEVICES, available_devices);
2453 if (query->target) {
2454 crm_debug("Found %d matching device%s for target '%s'",
2455 available_devices, pcmk__plural_s(available_devices),
2456 query->target);
2457 } else {
2458 crm_debug("%d device%s installed",
2459 available_devices, pcmk__plural_s(available_devices));
2460 }
2461
2462 crm_log_xml_trace(list, "query-result");
2463
2464 stonith_send_reply(query->reply, query->call_options, query->remote_peer,
2465 client);
2466
2467 done:
2468 pcmk__xml_free(query->reply);
2469 free(query->remote_peer);
2470 free(query->client_id);
2471 free(query->target);
2472 free(query->action);
2473 free(query);
2474 g_list_free_full(devices, free);
2475 }
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487 static void
2488 log_async_result(const async_command_t *cmd,
2489 const pcmk__action_result_t *result,
2490 int pid, const char *next, bool op_merged)
2491 {
2492 int log_level = LOG_ERR;
2493 int output_log_level = LOG_NEVER;
2494 guint devices_remaining = g_list_length(cmd->next_device_iter);
2495
2496 GString *msg = g_string_sized_new(80);
2497
2498
2499 if (pcmk__result_ok(result)) {
2500 log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
2501 if ((result->action_stdout != NULL)
2502 && !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
2503 pcmk__str_none)) {
2504 output_log_level = LOG_DEBUG;
2505 }
2506 next = NULL;
2507 } else {
2508 log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
2509 if ((result->action_stdout != NULL)
2510 && !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
2511 pcmk__str_none)) {
2512 output_log_level = LOG_WARNING;
2513 }
2514 }
2515
2516
2517 pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL);
2518 if (pid != 0) {
2519 g_string_append_printf(msg, "[%d] ", pid);
2520 }
2521 if (cmd->target != NULL) {
2522 pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL);
2523 }
2524 if (cmd->device != NULL) {
2525 pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL);
2526 }
2527
2528
2529 if (result->execution_status == PCMK_EXEC_DONE) {
2530 g_string_append_printf(msg, "returned %d", result->exit_status);
2531 } else {
2532 pcmk__g_strcat(msg, "could not be executed: ",
2533 pcmk_exec_status_str(result->execution_status), NULL);
2534 }
2535
2536
2537 if (result->exit_reason != NULL) {
2538 pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL);
2539 }
2540 if (next != NULL) {
2541 pcmk__g_strcat(msg, ", retrying with ", next, NULL);
2542 }
2543 if (devices_remaining > 0) {
2544 g_string_append_printf(msg, " (%u device%s remaining)",
2545 (unsigned int) devices_remaining,
2546 pcmk__plural_s(devices_remaining));
2547 }
2548 g_string_append_printf(msg, " " QB_XS " %scall %d from %s",
2549 (op_merged? "merged " : ""), cmd->id,
2550 cmd->client_name);
2551
2552
2553 do_crm_log(log_level, "%s", msg->str);
2554 g_string_free(msg, TRUE);
2555
2556
2557 if (output_log_level != LOG_NEVER) {
2558 char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);
2559
2560 crm_log_output(output_log_level, prefix, result->action_stdout);
2561 free(prefix);
2562 }
2563 }
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574 static void
2575 send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result,
2576 int pid, bool merged)
2577 {
2578 xmlNode *reply = NULL;
2579 pcmk__client_t *client = NULL;
2580
2581 CRM_CHECK((cmd != NULL) && (result != NULL), return);
2582
2583 log_async_result(cmd, result, pid, NULL, merged);
2584
2585 if (cmd->client != NULL) {
2586 client = pcmk__find_client_by_id(cmd->client);
2587 if ((client == NULL) && (cmd->origin == NULL)) {
2588 crm_trace("Skipping reply to %s: no longer a client", cmd->client);
2589 return;
2590 }
2591 }
2592
2593 reply = construct_async_reply(cmd, result);
2594 if (merged) {
2595 pcmk__xe_set_bool_attr(reply, PCMK__XA_ST_OP_MERGED, true);
2596 }
2597
2598 if (pcmk__is_fencing_action(cmd->action)
2599 && pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
2600
2601
2602
2603 crm_trace("Broadcast '%s' result for %s (target was also originator)",
2604 cmd->action, cmd->target);
2605 crm_xml_add(reply, PCMK__XA_SUBT, PCMK__VALUE_BROADCAST);
2606 crm_xml_add(reply, PCMK__XA_ST_OP, STONITH_OP_NOTIFY);
2607 pcmk__cluster_send_message(NULL, pcmk_ipc_fenced, reply);
2608 } else {
2609
2610 stonith_send_reply(reply, cmd->options, cmd->origin, client);
2611 }
2612
2613 crm_log_xml_trace(reply, "Reply");
2614 pcmk__xml_free(reply);
2615 }
2616
2617 static void
2618 cancel_stonith_command(async_command_t * cmd)
2619 {
2620 stonith_device_t *device = cmd_device(cmd);
2621
2622 if (device) {
2623 crm_trace("Cancel scheduled '%s' action using %s",
2624 cmd->action, device->id);
2625 device->pending_ops = g_list_remove(device->pending_ops, cmd);
2626 }
2627 }
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648 static void
2649 reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result,
2650 int pid)
2651 {
2652 GList *next = NULL;
2653
2654 for (GList *iter = cmd_list; iter != NULL; iter = next) {
2655 async_command_t *cmd_other = iter->data;
2656
2657 next = iter->next;
2658
2659 if (cmd == cmd_other) {
2660 continue;
2661 }
2662
2663
2664
2665
2666
2667
2668
2669 if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) ||
2670 !pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) ||
2671 !pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) ||
2672 !pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {
2673
2674 continue;
2675 }
2676
2677 crm_notice("Merging fencing action '%s'%s%s originating from "
2678 "client %s with identical fencing request from client %s",
2679 cmd_other->action,
2680 (cmd_other->target == NULL)? "" : " targeting ",
2681 pcmk__s(cmd_other->target, ""), cmd_other->client_name,
2682 cmd->client_name);
2683
2684
2685 cmd_list = g_list_remove_link(cmd_list, iter);
2686 send_async_reply(cmd_other, result, pid, true);
2687 cancel_stonith_command(cmd_other);
2688
2689 free_async_command(cmd_other);
2690 g_list_free_1(iter);
2691 }
2692 }
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702 static stonith_device_t *
2703 next_required_device(async_command_t *cmd)
2704 {
2705 for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
2706 stonith_device_t *next_device = g_hash_table_lookup(device_list,
2707 iter->data);
2708
2709 if (is_action_required(cmd->action, next_device)) {
2710
2711
2712
2713 cmd->next_device_iter = iter->next;
2714 return next_device;
2715 }
2716 }
2717 return NULL;
2718 }
2719
2720 static void
2721 st_child_done(int pid, const pcmk__action_result_t *result, void *user_data)
2722 {
2723 async_command_t *cmd = user_data;
2724
2725 stonith_device_t *device = NULL;
2726 stonith_device_t *next_device = NULL;
2727
2728 CRM_CHECK(cmd != NULL, return);
2729
2730 device = cmd_device(cmd);
2731 cmd->active_on = NULL;
2732
2733
2734 if (device) {
2735 if (!device->verified && pcmk__result_ok(result)
2736 && pcmk__strcase_any_of(cmd->action, PCMK_ACTION_LIST,
2737 PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS,
2738 NULL)) {
2739
2740 device->verified = TRUE;
2741 }
2742
2743 mainloop_set_trigger(device->work);
2744 }
2745
2746 if (pcmk__result_ok(result)) {
2747 next_device = next_required_device(cmd);
2748
2749 } else if ((cmd->next_device_iter != NULL)
2750 && !is_action_required(cmd->action, device)) {
2751
2752
2753 next_device = g_hash_table_lookup(device_list,
2754 cmd->next_device_iter->data);
2755 cmd->next_device_iter = cmd->next_device_iter->next;
2756 }
2757
2758 if (next_device == NULL) {
2759 send_async_reply(cmd, result, pid, false);
2760 if (pcmk__result_ok(result)) {
2761 reply_to_duplicates(cmd, result, pid);
2762 }
2763 free_async_command(cmd);
2764
2765 } else {
2766 log_async_result(cmd, result, pid, next_device->id, false);
2767 schedule_stonith_command(cmd, next_device);
2768 }
2769 }
2770
2771 static gint
2772 sort_device_priority(gconstpointer a, gconstpointer b)
2773 {
2774 const stonith_device_t *dev_a = a;
2775 const stonith_device_t *dev_b = b;
2776
2777 if (dev_a->priority > dev_b->priority) {
2778 return -1;
2779 } else if (dev_a->priority < dev_b->priority) {
2780 return 1;
2781 }
2782 return 0;
2783 }
2784
2785 static void
2786 stonith_fence_get_devices_cb(GList * devices, void *user_data)
2787 {
2788 async_command_t *cmd = user_data;
2789 stonith_device_t *device = NULL;
2790 guint ndevices = g_list_length(devices);
2791
2792 crm_info("Found %d matching device%s for target '%s'",
2793 ndevices, pcmk__plural_s(ndevices), cmd->target);
2794
2795 if (devices != NULL) {
2796
2797 devices = g_list_sort(devices, sort_device_priority);
2798 device = g_hash_table_lookup(device_list, devices->data);
2799 }
2800
2801 if (device == NULL) {
2802 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2803
2804 pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
2805 "No device configured for target '%s'",
2806 cmd->target);
2807 send_async_reply(cmd, &result, 0, false);
2808 pcmk__reset_result(&result);
2809 free_async_command(cmd);
2810 g_list_free_full(devices, free);
2811
2812 } else {
2813 cmd->device_list = devices;
2814 cmd->next_device_iter = devices->next;
2815 schedule_stonith_command(cmd, device);
2816 }
2817 }
2818
2819
2820
2821
2822
2823
2824
2825
2826 static void
2827 fence_locally(xmlNode *msg, pcmk__action_result_t *result)
2828 {
2829 const char *device_id = NULL;
2830 stonith_device_t *device = NULL;
2831 async_command_t *cmd = NULL;
2832 xmlNode *dev = NULL;
2833
2834 CRM_CHECK((msg != NULL) && (result != NULL), return);
2835
2836 dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, msg, LOG_ERR);
2837
2838 cmd = create_async_command(msg);
2839 if (cmd == NULL) {
2840 crm_log_xml_warn(msg, "invalid");
2841 fenced_set_protocol_error(result);
2842 return;
2843 }
2844
2845 device_id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
2846 if (device_id != NULL) {
2847 device = g_hash_table_lookup(device_list, device_id);
2848 if (device == NULL) {
2849 crm_err("Requested device '%s' is not available", device_id);
2850 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
2851 "Requested device '%s' not found", device_id);
2852 return;
2853 }
2854 schedule_stonith_command(cmd, device);
2855
2856 } else {
2857 const char *host = crm_element_value(dev, PCMK__XA_ST_TARGET);
2858
2859 if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
2860 int nodeid = 0;
2861 pcmk__node_status_t *node = NULL;
2862
2863 pcmk__scan_min_int(host, &nodeid, 0);
2864 node = pcmk__search_node_caches(nodeid, NULL,
2865 pcmk__node_search_any
2866 |pcmk__node_search_cluster_cib);
2867 if (node != NULL) {
2868 host = node->name;
2869 }
2870 }
2871
2872
2873 get_capable_devices(host, cmd->action, cmd->default_timeout,
2874 TRUE, cmd, stonith_fence_get_devices_cb,
2875 fenced_support_flag(cmd->action));
2876 }
2877
2878 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
2879 }
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895 xmlNode *
2896 fenced_construct_reply(const xmlNode *request, xmlNode *data,
2897 const pcmk__action_result_t *result)
2898 {
2899 xmlNode *reply = NULL;
2900
2901 reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY);
2902
2903 crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
2904 crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
2905 stonith__xe_set_result(reply, result);
2906
2907 if (request == NULL) {
2908
2909
2910
2911
2912
2913
2914 crm_warn("Missing request information for client notifications for "
2915 "operation with result '%s' (initiated before we came up?)",
2916 pcmk_exec_status_str(result->execution_status));
2917
2918 } else {
2919 const char *name = NULL;
2920 const char *value = NULL;
2921
2922
2923 const char *names[] = {
2924 PCMK__XA_ST_OP,
2925 PCMK__XA_ST_CALLID,
2926 PCMK__XA_ST_CLIENTID,
2927 PCMK__XA_ST_CLIENTNAME,
2928 PCMK__XA_ST_REMOTE_OP,
2929 PCMK__XA_ST_CALLOPT,
2930 };
2931
2932 for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
2933 name = names[lpc];
2934 value = crm_element_value(request, name);
2935 crm_xml_add(reply, name, value);
2936 }
2937 if (data != NULL) {
2938 xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_ST_CALLDATA);
2939
2940 pcmk__xml_copy(wrapper, data);
2941 }
2942 }
2943 return reply;
2944 }
2945
2946
2947
2948
2949
2950
2951
2952
2953 static xmlNode *
2954 construct_async_reply(const async_command_t *cmd,
2955 const pcmk__action_result_t *result)
2956 {
2957 xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY);
2958
2959 crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
2960 crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
2961 crm_xml_add(reply, PCMK__XA_ST_OP, cmd->op);
2962 crm_xml_add(reply, PCMK__XA_ST_DEVICE_ID, cmd->device);
2963 crm_xml_add(reply, PCMK__XA_ST_REMOTE_OP, cmd->remote_op_id);
2964 crm_xml_add(reply, PCMK__XA_ST_CLIENTID, cmd->client);
2965 crm_xml_add(reply, PCMK__XA_ST_CLIENTNAME, cmd->client_name);
2966 crm_xml_add(reply, PCMK__XA_ST_TARGET, cmd->target);
2967 crm_xml_add(reply, PCMK__XA_ST_DEVICE_ACTION, cmd->op);
2968 crm_xml_add(reply, PCMK__XA_ST_ORIGIN, cmd->origin);
2969 crm_xml_add_int(reply, PCMK__XA_ST_CALLID, cmd->id);
2970 crm_xml_add_int(reply, PCMK__XA_ST_CALLOPT, cmd->options);
2971
2972 stonith__xe_set_result(reply, result);
2973 return reply;
2974 }
2975
2976 bool
2977 fencing_peer_active(pcmk__node_status_t *peer)
2978 {
2979 return (peer != NULL) && (peer->name != NULL)
2980 && pcmk_is_set(peer->processes, crm_get_cluster_proc());
2981 }
2982
2983 void
2984 set_fencing_completed(remote_fencing_op_t *op)
2985 {
2986 struct timespec tv;
2987
2988 qb_util_timespec_from_epoch_get(&tv);
2989 op->completed = tv.tv_sec;
2990 op->completed_nsec = tv.tv_nsec;
2991 }
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002 static const char *
3003 check_alternate_host(const char *target)
3004 {
3005 if (pcmk__str_eq(target, fenced_get_local_node(), pcmk__str_casei)) {
3006 GHashTableIter gIter;
3007 pcmk__node_status_t *entry = NULL;
3008
3009 g_hash_table_iter_init(&gIter, pcmk__peer_cache);
3010 while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
3011 if (fencing_peer_active(entry)
3012 && !pcmk__str_eq(entry->name, target, pcmk__str_casei)) {
3013 crm_notice("Forwarding self-fencing request to %s",
3014 entry->name);
3015 return entry->name;
3016 }
3017 }
3018 crm_warn("Will handle own fencing because no peer can");
3019 }
3020 return NULL;
3021 }
3022
3023 static void
3024 remove_relay_op(xmlNode * request)
3025 {
3026 xmlNode *dev = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, request,
3027 LOG_TRACE);
3028 const char *relay_op_id = NULL;
3029 const char *op_id = NULL;
3030 const char *client_name = NULL;
3031 const char *target = NULL;
3032 remote_fencing_op_t *relay_op = NULL;
3033
3034 if (dev) {
3035 target = crm_element_value(dev, PCMK__XA_ST_TARGET);
3036 }
3037
3038 relay_op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP_RELAY);
3039 op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP);
3040 client_name = crm_element_value(request, PCMK__XA_ST_CLIENTNAME);
3041
3042
3043 if ((relay_op_id != NULL) && (target != NULL)
3044 && pcmk__str_eq(target, fenced_get_local_node(), pcmk__str_casei)) {
3045 relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);
3046
3047 if (relay_op) {
3048 GHashTableIter iter;
3049 remote_fencing_op_t *list_op = NULL;
3050 g_hash_table_iter_init(&iter, stonith_remote_op_list);
3051
3052
3053 while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
3054 GList *dup_iter = NULL;
3055 if (list_op != relay_op) {
3056 for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
3057 remote_fencing_op_t *other = dup_iter->data;
3058 if (other == relay_op) {
3059 other->duplicates = g_list_remove(other->duplicates, relay_op);
3060 break;
3061 }
3062 }
3063 }
3064 }
3065 crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
3066 "replaced by op %s ('%s'%s%s for %s)",
3067 relay_op->id, relay_op->action,
3068 (relay_op->target == NULL)? "" : " targeting ",
3069 pcmk__s(relay_op->target, ""),
3070 relay_op->client_name, op_id, relay_op->action,
3071 (target == NULL)? "" : " targeting ", pcmk__s(target, ""),
3072 client_name);
3073
3074 g_hash_table_remove(stonith_remote_op_list, relay_op_id);
3075 }
3076 }
3077 }
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093 static inline bool
3094 is_privileged(const pcmk__client_t *c, const char *op)
3095 {
3096 if ((c == NULL) || pcmk_is_set(c->flags, pcmk__client_privileged)) {
3097 return true;
3098 } else {
3099 crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
3100 pcmk__s(op, ""), pcmk__client_name(c));
3101 return false;
3102 }
3103 }
3104
3105
3106 static xmlNode *
3107 handle_register_request(pcmk__request_t *request)
3108 {
3109 xmlNode *reply = pcmk__xe_create(NULL, "reply");
3110
3111 pcmk__assert(request->ipc_client != NULL);
3112 crm_xml_add(reply, PCMK__XA_ST_OP, CRM_OP_REGISTER);
3113 crm_xml_add(reply, PCMK__XA_ST_CLIENTID, request->ipc_client->id);
3114 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3115 pcmk__set_request_flags(request, pcmk__request_reuse_options);
3116 return reply;
3117 }
3118
3119
3120 static xmlNode *
3121 handle_agent_request(pcmk__request_t *request)
3122 {
3123 execute_agent_action(request->xml, &request->result);
3124 if (request->result.execution_status == PCMK_EXEC_PENDING) {
3125 return NULL;
3126 }
3127 return fenced_construct_reply(request->xml, NULL, &request->result);
3128 }
3129
3130
3131 static xmlNode *
3132 handle_update_timeout_request(pcmk__request_t *request)
3133 {
3134 const char *call_id = crm_element_value(request->xml, PCMK__XA_ST_CALLID);
3135 const char *client_id = crm_element_value(request->xml,
3136 PCMK__XA_ST_CLIENTID);
3137 int op_timeout = 0;
3138
3139 crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &op_timeout);
3140 do_stonith_async_timeout_update(client_id, call_id, op_timeout);
3141 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3142 return NULL;
3143 }
3144
3145
3146 static xmlNode *
3147 handle_query_request(pcmk__request_t *request)
3148 {
3149 int timeout = 0;
3150 xmlNode *dev = NULL;
3151 const char *action = NULL;
3152 const char *target = NULL;
3153 const char *client_id = crm_element_value(request->xml,
3154 PCMK__XA_ST_CLIENTID);
3155 struct st_query_data *query = NULL;
3156
3157 if (request->peer != NULL) {
3158
3159 create_remote_stonith_op(client_id, request->xml, TRUE);
3160 }
3161
3162
3163 remove_relay_op(request->xml);
3164
3165 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3166
3167 dev = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, request->xml,
3168 LOG_NEVER);
3169 if (dev != NULL) {
3170 const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
3171
3172 if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
3173 return NULL;
3174 }
3175 target = crm_element_value(dev, PCMK__XA_ST_TARGET);
3176 action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
3177 }
3178
3179 crm_log_xml_trace(request->xml, "Query");
3180
3181 query = pcmk__assert_alloc(1, sizeof(struct st_query_data));
3182
3183 query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
3184 query->remote_peer = pcmk__str_copy(request->peer);
3185 query->client_id = pcmk__str_copy(client_id);
3186 query->target = pcmk__str_copy(target);
3187 query->action = pcmk__str_copy(action);
3188 query->call_options = request->call_options;
3189
3190 crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &timeout);
3191 get_capable_devices(target, action, timeout,
3192 pcmk_is_set(query->call_options,
3193 st_opt_allow_self_fencing),
3194 query, stonith_query_capable_device_cb, st_device_supports_none);
3195 return NULL;
3196 }
3197
3198
3199 static xmlNode *
3200 handle_notify_request(pcmk__request_t *request)
3201 {
3202 const char *flag_name = NULL;
3203
3204 pcmk__assert(request->ipc_client != NULL);
3205 flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_ACTIVATE);
3206 if (flag_name != NULL) {
3207 crm_debug("Enabling %s callbacks for client %s",
3208 flag_name, pcmk__request_origin(request));
3209 pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name));
3210 }
3211
3212 flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_DEACTIVATE);
3213 if (flag_name != NULL) {
3214 crm_debug("Disabling %s callbacks for client %s",
3215 flag_name, pcmk__request_origin(request));
3216 pcmk__clear_client_flags(request->ipc_client,
3217 get_stonith_flag(flag_name));
3218 }
3219
3220 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3221 pcmk__set_request_flags(request, pcmk__request_reuse_options);
3222
3223 return pcmk__ipc_create_ack(request->ipc_flags, PCMK__XE_ACK, NULL,
3224 CRM_EX_OK);
3225 }
3226
3227
3228 static xmlNode *
3229 handle_relay_request(pcmk__request_t *request)
3230 {
3231 xmlNode *dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, request->xml,
3232 LOG_TRACE);
3233
3234 crm_notice("Received forwarded fencing request from "
3235 "%s %s to fence (%s) peer %s",
3236 pcmk__request_origin_type(request),
3237 pcmk__request_origin(request),
3238 crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION),
3239 crm_element_value(dev, PCMK__XA_ST_TARGET));
3240
3241 if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
3242 fenced_set_protocol_error(&request->result);
3243 return fenced_construct_reply(request->xml, NULL, &request->result);
3244 }
3245
3246 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
3247 return NULL;
3248 }
3249
3250
3251 static xmlNode *
3252 handle_fence_request(pcmk__request_t *request)
3253 {
3254 if (request->peer != NULL) {
3255 fence_locally(request->xml, &request->result);
3256
3257 } else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
3258 switch (fenced_handle_manual_confirmation(request->ipc_client,
3259 request->xml)) {
3260 case pcmk_rc_ok:
3261 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
3262 NULL);
3263 break;
3264 case EINPROGRESS:
3265 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3266 NULL);
3267 break;
3268 default:
3269 fenced_set_protocol_error(&request->result);
3270 break;
3271 }
3272
3273 } else {
3274 const char *alternate_host = NULL;
3275 xmlNode *dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, request->xml,
3276 LOG_TRACE);
3277 const char *target = crm_element_value(dev, PCMK__XA_ST_TARGET);
3278 const char *action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
3279 const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
3280
3281 if (request->ipc_client != NULL) {
3282 int tolerance = 0;
3283
3284 crm_notice("Client %s wants to fence (%s) %s using %s",
3285 pcmk__request_origin(request), action,
3286 target, (device? device : "any device"));
3287 crm_element_value_int(dev, PCMK__XA_ST_TOLERANCE, &tolerance);
3288 if (stonith_check_fence_tolerance(tolerance, target, action)) {
3289 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
3290 NULL);
3291 return fenced_construct_reply(request->xml, NULL,
3292 &request->result);
3293 }
3294 alternate_host = check_alternate_host(target);
3295
3296 } else {
3297 crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
3298 request->peer, action, target,
3299 (device == NULL)? "(any)" : device);
3300 }
3301
3302 if (alternate_host != NULL) {
3303 const char *client_id = NULL;
3304 remote_fencing_op_t *op = NULL;
3305 pcmk__node_status_t *node =
3306 pcmk__get_node(0, alternate_host, NULL,
3307 pcmk__node_search_cluster_member);
3308
3309 if (request->ipc_client->id == 0) {
3310 client_id = crm_element_value(request->xml,
3311 PCMK__XA_ST_CLIENTID);
3312 } else {
3313 client_id = request->ipc_client->id;
3314 }
3315
3316
3317
3318
3319
3320 op = create_remote_stonith_op(client_id, request->xml, FALSE);
3321
3322 crm_xml_add(request->xml, PCMK__XA_ST_OP, STONITH_OP_RELAY);
3323 crm_xml_add(request->xml, PCMK__XA_ST_CLIENTID,
3324 request->ipc_client->id);
3325 crm_xml_add(request->xml, PCMK__XA_ST_REMOTE_OP, op->id);
3326 pcmk__cluster_send_message(node, pcmk_ipc_fenced, request->xml);
3327 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3328 NULL);
3329
3330 } else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
3331 FALSE) == NULL) {
3332 fenced_set_protocol_error(&request->result);
3333
3334 } else {
3335 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3336 NULL);
3337 }
3338 }
3339
3340 if (request->result.execution_status == PCMK_EXEC_PENDING) {
3341 return NULL;
3342 }
3343 return fenced_construct_reply(request->xml, NULL, &request->result);
3344 }
3345
3346
3347 static xmlNode *
3348 handle_history_request(pcmk__request_t *request)
3349 {
3350 xmlNode *reply = NULL;
3351 xmlNode *data = NULL;
3352
3353 stonith_fence_history(request->xml, &data, request->peer,
3354 request->call_options);
3355 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3356 if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
3357
3358
3359
3360 reply = fenced_construct_reply(request->xml, data, &request->result);
3361 }
3362 pcmk__xml_free(data);
3363 return reply;
3364 }
3365
3366
3367 static xmlNode *
3368 handle_device_add_request(pcmk__request_t *request)
3369 {
3370 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3371 xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, request->xml,
3372 LOG_ERR);
3373
3374 if (is_privileged(request->ipc_client, op)) {
3375 int rc = stonith_device_register(dev, FALSE);
3376
3377 pcmk__set_result(&request->result,
3378 ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
3379 stonith__legacy2status(rc),
3380 ((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
3381 } else {
3382 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3383 PCMK_EXEC_INVALID,
3384 "Unprivileged users must register device via CIB");
3385 }
3386 fenced_send_config_notification(op, &request->result,
3387 (dev == NULL)? NULL : pcmk__xe_id(dev));
3388 return fenced_construct_reply(request->xml, NULL, &request->result);
3389 }
3390
3391
3392 static xmlNode *
3393 handle_device_delete_request(pcmk__request_t *request)
3394 {
3395 xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, request->xml,
3396 LOG_ERR);
3397 const char *device_id = crm_element_value(dev, PCMK_XA_ID);
3398 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3399
3400 if (is_privileged(request->ipc_client, op)) {
3401 stonith_device_remove(device_id, false);
3402 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3403 } else {
3404 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3405 PCMK_EXEC_INVALID,
3406 "Unprivileged users must delete device via CIB");
3407 }
3408 fenced_send_config_notification(op, &request->result, device_id);
3409 return fenced_construct_reply(request->xml, NULL, &request->result);
3410 }
3411
3412
3413 static xmlNode *
3414 handle_level_add_request(pcmk__request_t *request)
3415 {
3416 char *desc = NULL;
3417 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3418
3419 if (is_privileged(request->ipc_client, op)) {
3420 fenced_register_level(request->xml, &desc, &request->result);
3421 } else {
3422 unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
3423 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3424 PCMK_EXEC_INVALID,
3425 "Unprivileged users must add level via CIB");
3426 }
3427 fenced_send_config_notification(op, &request->result, desc);
3428 free(desc);
3429 return fenced_construct_reply(request->xml, NULL, &request->result);
3430 }
3431
3432
3433 static xmlNode *
3434 handle_level_delete_request(pcmk__request_t *request)
3435 {
3436 char *desc = NULL;
3437 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3438
3439 if (is_privileged(request->ipc_client, op)) {
3440 fenced_unregister_level(request->xml, &desc, &request->result);
3441 } else {
3442 unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
3443 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3444 PCMK_EXEC_INVALID,
3445 "Unprivileged users must delete level via CIB");
3446 }
3447 fenced_send_config_notification(op, &request->result, desc);
3448 free(desc);
3449 return fenced_construct_reply(request->xml, NULL, &request->result);
3450 }
3451
3452
3453 static xmlNode *
3454 handle_cache_request(pcmk__request_t *request)
3455 {
3456 int node_id = 0;
3457 const char *name = NULL;
3458
3459 crm_element_value_int(request->xml, PCMK_XA_ID, &node_id);
3460 name = crm_element_value(request->xml, PCMK_XA_UNAME);
3461 pcmk__cluster_forget_cluster_node(node_id, name);
3462 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3463 return NULL;
3464 }
3465
3466 static xmlNode *
3467 handle_unknown_request(pcmk__request_t *request)
3468 {
3469 crm_err("Unknown IPC request %s from %s %s",
3470 request->op, pcmk__request_origin_type(request),
3471 pcmk__request_origin(request));
3472 pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
3473 "Unknown IPC request type '%s' (bug?)", request->op);
3474 return fenced_construct_reply(request->xml, NULL, &request->result);
3475 }
3476
3477 static void
3478 fenced_register_handlers(void)
3479 {
3480 pcmk__server_command_t handlers[] = {
3481 { CRM_OP_REGISTER, handle_register_request },
3482 { STONITH_OP_EXEC, handle_agent_request },
3483 { STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
3484 { STONITH_OP_QUERY, handle_query_request },
3485 { STONITH_OP_NOTIFY, handle_notify_request },
3486 { STONITH_OP_RELAY, handle_relay_request },
3487 { STONITH_OP_FENCE, handle_fence_request },
3488 { STONITH_OP_FENCE_HISTORY, handle_history_request },
3489 { STONITH_OP_DEVICE_ADD, handle_device_add_request },
3490 { STONITH_OP_DEVICE_DEL, handle_device_delete_request },
3491 { STONITH_OP_LEVEL_ADD, handle_level_add_request },
3492 { STONITH_OP_LEVEL_DEL, handle_level_delete_request },
3493 { CRM_OP_RM_NODE_CACHE, handle_cache_request },
3494 { NULL, handle_unknown_request },
3495 };
3496
3497 fenced_handlers = pcmk__register_handlers(handlers);
3498 }
3499
3500 void
3501 fenced_unregister_handlers(void)
3502 {
3503 if (fenced_handlers != NULL) {
3504 g_hash_table_destroy(fenced_handlers);
3505 fenced_handlers = NULL;
3506 }
3507 }
3508
3509 static void
3510 handle_request(pcmk__request_t *request)
3511 {
3512 xmlNode *reply = NULL;
3513 const char *reason = NULL;
3514
3515 if (fenced_handlers == NULL) {
3516 fenced_register_handlers();
3517 }
3518 reply = pcmk__process_request(request, fenced_handlers);
3519 if (reply != NULL) {
3520 if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
3521 && (request->ipc_client != NULL)) {
3522
3523
3524
3525
3526 pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
3527 request->ipc_flags);
3528 request->ipc_client->request_id = 0;
3529 } else {
3530 stonith_send_reply(reply, request->call_options,
3531 request->peer, request->ipc_client);
3532 }
3533 pcmk__xml_free(reply);
3534 }
3535
3536 reason = request->result.exit_reason;
3537 crm_debug("Processed %s request from %s %s: %s%s%s%s",
3538 request->op, pcmk__request_origin_type(request),
3539 pcmk__request_origin(request),
3540 pcmk_exec_status_str(request->result.execution_status),
3541 (reason == NULL)? "" : " (",
3542 (reason == NULL)? "" : reason,
3543 (reason == NULL)? "" : ")");
3544 }
3545
3546 static void
3547 handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer)
3548 {
3549
3550 char *op = crm_element_value_copy(request, PCMK__XA_ST_OP);
3551
3552 if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
3553 process_remote_stonith_query(request);
3554
3555 } else if (pcmk__str_any_of(op, STONITH_OP_NOTIFY, STONITH_OP_FENCE,
3556 NULL)) {
3557 fenced_process_fencing_reply(request);
3558
3559 } else {
3560 crm_err("Ignoring unknown %s reply from %s %s",
3561 pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
3562 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3563 crm_log_xml_warn(request, "UnknownOp");
3564 free(op);
3565 return;
3566 }
3567 crm_debug("Processed %s reply from %s %s",
3568 op, ((client == NULL)? "peer" : "client"),
3569 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3570 free(op);
3571 }
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583 void
3584 stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
3585 xmlNode *message, const char *remote_peer)
3586 {
3587 uint32_t call_options = st_opt_none;
3588 int rc = pcmk_rc_ok;
3589 bool is_reply = false;
3590
3591 CRM_CHECK(message != NULL, return);
3592
3593 if (get_xpath_object("//" PCMK__XE_ST_REPLY, message, LOG_NEVER) != NULL) {
3594 is_reply = true;
3595 }
3596
3597 rc = pcmk__xe_get_flags(message, PCMK__XA_ST_CALLOPT, &call_options,
3598 st_opt_none);
3599 if (rc != pcmk_rc_ok) {
3600 crm_warn("Couldn't parse options from message: %s", pcmk_rc_str(rc));
3601 }
3602
3603 crm_debug("Processing %ssynchronous %s %s %u from %s %s",
3604 pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
3605 crm_element_value(message, PCMK__XA_ST_OP),
3606 (is_reply? "reply" : "request"), id,
3607 ((client == NULL)? "peer" : "client"),
3608 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3609
3610 if (pcmk_is_set(call_options, st_opt_sync_call)) {
3611 pcmk__assert((client == NULL) || (client->request_id == id));
3612 }
3613
3614 if (is_reply) {
3615 handle_reply(client, message, remote_peer);
3616 } else {
3617 pcmk__request_t request = {
3618 .ipc_client = client,
3619 .ipc_id = id,
3620 .ipc_flags = flags,
3621 .peer = remote_peer,
3622 .xml = message,
3623 .call_options = call_options,
3624 .result = PCMK__UNKNOWN_RESULT,
3625 };
3626
3627 request.op = crm_element_value_copy(request.xml, PCMK__XA_ST_OP);
3628 CRM_CHECK(request.op != NULL, return);
3629
3630 if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
3631 pcmk__set_request_flags(&request, pcmk__request_sync);
3632 }
3633
3634 handle_request(&request);
3635 pcmk__reset_request(&request);
3636 }
3637 }
![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
![[previous]](../icons/left.png)
![[last]](../icons/n_last.png)
![[bottom]](../icons/n_bottom.png){kind=icon}