This source file includes following definitions.
- set_bad_request_result
- fenced_has_watchdog_device
- fenced_foreach_device
- fenced_foreach_device_remove
- is_action_required
- get_action_delay_max
- get_action_delay_base
- get_action_timeout
- cmd_device
- fenced_device_reboot_action
- fenced_device_supports_on
- free_async_command
- create_async_command
- get_action_limit
- get_active_cmds
- fork_cb
- get_agent_metadata_cb
- report_internal_result
- stonith_device_execute
- stonith_device_dispatch
- start_delay_helper
- schedule_stonith_command
- free_device
- fenced_init_device_table
- fenced_free_device_table
- build_port_aliases
- free_metadata_cache
- init_metadata_cache
- get_agent_metadata
- read_action_metadata
- target_list_type
- build_device_from_xml
- schedule_internal_command
- status_search_cb
- dynamic_list_search_cb
- device_params_diff
- device_has_duplicate
- fenced_device_register
- stonith_device_remove
- count_active_levels
- free_topology_entry
- free_topology_list
- init_topology_list
- stonith_level_key
- unpack_level_kind
- unpack_level_request
- fenced_register_level
- fenced_unregister_level
- list_to_string
- execute_agent_action
- search_devices_record_result
- localhost_is_eligible
- localhost_is_eligible_with_remap
- can_use_target_cache
- can_fence_host_with_device
- search_devices
- get_capable_devices
- add_action_specific_attributes
- add_disallowed
- add_action_reply
- stonith_send_reply
- stonith_query_capable_device_cb
- log_async_result
- send_async_reply
- cancel_stonith_command
- reply_to_duplicates
- next_required_device
- st_child_done
- stonith_fence_get_devices_cb
- fence_locally
- fenced_construct_reply
- construct_async_reply
- fencing_peer_active
- set_fencing_completed
- check_alternate_host
- remove_relay_op
- is_privileged
- handle_register_request
- handle_agent_request
- handle_update_timeout_request
- handle_query_request
- handle_notify_request
- handle_relay_request
- handle_fence_request
- handle_history_request
- handle_device_add_request
- handle_device_delete_request
- handle_level_add_request
- handle_level_delete_request
- handle_cache_request
- handle_unknown_request
- fenced_register_handlers
- fenced_unregister_handlers
- handle_request
- handle_reply
- stonith_command
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <sys/param.h>
13 #include <stdbool.h>
14 #include <stdio.h>
15 #include <sys/types.h>
16 #include <sys/wait.h>
17 #include <sys/stat.h>
18 #include <unistd.h>
19 #include <sys/utsname.h>
20
21 #include <stdlib.h>
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <ctype.h>
25
26 #include <libxml/tree.h>
27 #include <libxml/xpath.h>
28
29 #include <crm/crm.h>
30 #include <crm/common/ipc.h>
31 #include <crm/common/ipc_internal.h>
32 #include <crm/cluster/internal.h>
33 #include <crm/common/mainloop.h>
34
35 #include <crm/stonith-ng.h>
36 #include <crm/fencing/internal.h>
37 #include <crm/common/xml.h>
38
39 #include <pacemaker-fenced.h>
40
41 static GHashTable *device_table = NULL;
42
43 GHashTable *topology = NULL;
44 static GList *cmd_list = NULL;
45
46 static GHashTable *fenced_handlers = NULL;
47
48 struct device_search_s {
49
50 char *host;
51
52 char *action;
53
54
55 int per_device_timeout;
56
57 int replies_needed;
58
59 int replies_received;
60
61 bool allow_self;
62
63
64 void *user_data;
65
66 void (*callback) (GList * devices, void *user_data);
67
68 GList *capable;
69
70 uint32_t support_action_only;
71 };
72
73 static gboolean stonith_device_dispatch(gpointer user_data);
74 static void st_child_done(int pid, const pcmk__action_result_t *result,
75 void *user_data);
76
77 static void search_devices_record_result(struct device_search_s *search, const char *device,
78 gboolean can_fence);
79
80 static int get_agent_metadata(const char *agent, xmlNode **metadata);
81 static void read_action_metadata(fenced_device_t *device);
82 static enum fenced_target_by unpack_level_kind(const xmlNode *level);
83
84 typedef struct {
85 int id;
86 uint32_t options;
87 int default_timeout;
88 int timeout;
89
90 int start_delay;
91 int delay_id;
92
93 char *op;
94 char *origin;
95 char *client;
96 char *client_name;
97 char *remote_op_id;
98
99 char *target;
100 char *action;
101 char *device;
102
103
104 GList *device_list;
105
106
107 GList *next_device_iter;
108
109 void *internal_user_data;
110 void (*done_cb) (int pid, const pcmk__action_result_t *result,
111 void *user_data);
112
113 fenced_device_t *active_on;
114 fenced_device_t *activating_on;
115 } async_command_t;
116
117 static xmlNode *construct_async_reply(const async_command_t *cmd,
118 const pcmk__action_result_t *result);
119
120
121
122
123
124
125
126 static inline void
127 set_bad_request_result(pcmk__action_result_t *result)
128 {
129 pcmk__set_result(result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
130 "Fencer API request missing required information (bug?)");
131 }
132
133
134
135
136
137
138
139
140 bool
141 fenced_has_watchdog_device(void)
142 {
143 return (device_table != NULL)
144 && (g_hash_table_lookup(device_table, STONITH_WATCHDOG_ID) != NULL);
145 }
146
147
148
149
150
151
152
153
154 void
155 fenced_foreach_device(GHFunc fn, gpointer user_data)
156 {
157 if (device_table != NULL) {
158 g_hash_table_foreach(device_table, fn, user_data);
159 }
160 }
161
162
163
164
165
166
167
168
169 void
170 fenced_foreach_device_remove(GHRFunc fn)
171 {
172 if (device_table != NULL) {
173 g_hash_table_foreach_remove(device_table, fn, NULL);
174 }
175 }
176
177 static gboolean
178 is_action_required(const char *action, const fenced_device_t *device)
179 {
180 return (device != NULL)
181 && pcmk_is_set(device->flags, fenced_df_auto_unfence)
182 && pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none);
183 }
184
185 static int
186 get_action_delay_max(const fenced_device_t *device, const char *action)
187 {
188 const char *value = NULL;
189 guint delay_max = 0U;
190
191 if (!pcmk__is_fencing_action(action)) {
192 return 0;
193 }
194
195 value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
196 if (value) {
197 pcmk_parse_interval_spec(value, &delay_max);
198 delay_max /= 1000;
199 }
200
201 return (int) delay_max;
202 }
203
204 static int
205 get_action_delay_base(const fenced_device_t *device, const char *action,
206 const char *target)
207 {
208 char *hash_value = NULL;
209 guint delay_base = 0U;
210
211 if (!pcmk__is_fencing_action(action)) {
212 return 0;
213 }
214
215 hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);
216
217 if (hash_value) {
218 char *value = pcmk__str_copy(hash_value);
219 char *valptr = value;
220
221 if (target != NULL) {
222 for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
223 char *mapval = strchr(val, ':');
224
225 if (mapval == NULL || mapval[1] == 0) {
226 crm_err("pcmk_delay_base: empty value in mapping", val);
227 continue;
228 }
229
230 if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
231 value = mapval + 1;
232 crm_debug("pcmk_delay_base mapped to %s for %s",
233 value, target);
234 break;
235 }
236 }
237 }
238
239 if (strchr(value, ':') == 0) {
240 pcmk_parse_interval_spec(value, &delay_base);
241 delay_base /= 1000;
242 }
243
244 free(valptr);
245 }
246
247 return (int) delay_base;
248 }
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270 static int
271 get_action_timeout(const fenced_device_t *device, const char *action,
272 int default_timeout)
273 {
274 if (action && device && device->params) {
275 char buffer[64] = { 0, };
276 const char *value = NULL;
277
278
279
280
281 if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)
282 && !pcmk_is_set(device->flags, fenced_df_supports_reboot)) {
283 crm_trace("%s doesn't support reboot, using timeout for off instead",
284 device->id);
285 action = PCMK_ACTION_OFF;
286 }
287
288
289 snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
290 value = g_hash_table_lookup(device->params, buffer);
291 if (value) {
292 long long timeout_ms = crm_get_msec(value);
293 return (int) QB_MIN(pcmk__timeout_ms2s(timeout_ms), INT_MAX);
294 }
295 }
296 return default_timeout;
297 }
298
299
300
301
302
303
304
305
306
307 static fenced_device_t *
308 cmd_device(const async_command_t *cmd)
309 {
310 if ((cmd == NULL) || (cmd->device == NULL) || (device_table == NULL)) {
311 return NULL;
312 }
313 return g_hash_table_lookup(device_table, cmd->device);
314 }
315
316
317
318
319
320
321
322
323
324 const char *
325 fenced_device_reboot_action(const char *device_id)
326 {
327 const char *action = NULL;
328
329 if ((device_table != NULL) && (device_id != NULL)) {
330 fenced_device_t *device = g_hash_table_lookup(device_table, device_id);
331
332 if ((device != NULL) && (device->params != NULL)) {
333 action = g_hash_table_lookup(device->params, "pcmk_reboot_action");
334 }
335 }
336 return pcmk__s(action, PCMK_ACTION_REBOOT);
337 }
338
339
340
341
342
343
344
345
346
347 bool
348 fenced_device_supports_on(const char *device_id)
349 {
350 if ((device_table != NULL) && (device_id != NULL)) {
351 fenced_device_t *device = g_hash_table_lookup(device_table, device_id);
352
353 if (device != NULL) {
354 return pcmk_is_set(device->flags, fenced_df_supports_on);
355 }
356 }
357 return false;
358 }
359
360 static void
361 free_async_command(async_command_t * cmd)
362 {
363 if (!cmd) {
364 return;
365 }
366
367 if (cmd->delay_id) {
368 g_source_remove(cmd->delay_id);
369 }
370
371 cmd_list = g_list_remove(cmd_list, cmd);
372
373 g_list_free_full(cmd->device_list, free);
374 free(cmd->device);
375 free(cmd->action);
376 free(cmd->target);
377 free(cmd->remote_op_id);
378 free(cmd->client);
379 free(cmd->client_name);
380 free(cmd->origin);
381 free(cmd->op);
382 free(cmd);
383 }
384
385
386
387
388
389
390
391
392
393
394
395
396 static async_command_t *
397 create_async_command(xmlNode *msg)
398 {
399 xmlNode *op = NULL;
400 async_command_t *cmd = NULL;
401 int rc = pcmk_rc_ok;
402
403 if (msg == NULL) {
404 return NULL;
405 }
406
407 op = pcmk__xpath_find_one(msg->doc, "//*[@" PCMK__XA_ST_DEVICE_ACTION "]",
408 LOG_ERR);
409 if (op == NULL) {
410 return NULL;
411 }
412
413 cmd = pcmk__assert_alloc(1, sizeof(async_command_t));
414
415
416 cmd->action = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ACTION);
417 cmd->op = crm_element_value_copy(msg, PCMK__XA_ST_OP);
418 cmd->client = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTID);
419 if ((cmd->action == NULL) || (cmd->op == NULL) || (cmd->client == NULL)) {
420 free_async_command(cmd);
421 return NULL;
422 }
423
424 crm_element_value_int(msg, PCMK__XA_ST_CALLID, &(cmd->id));
425 crm_element_value_int(msg, PCMK__XA_ST_DELAY, &(cmd->start_delay));
426 crm_element_value_int(msg, PCMK__XA_ST_TIMEOUT, &(cmd->default_timeout));
427 cmd->timeout = cmd->default_timeout;
428
429 rc = pcmk__xe_get_flags(msg, PCMK__XA_ST_CALLOPT, &(cmd->options),
430 st_opt_none);
431 if (rc != pcmk_rc_ok) {
432 crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
433 }
434
435 cmd->origin = crm_element_value_copy(msg, PCMK__XA_SRC);
436 cmd->remote_op_id = crm_element_value_copy(msg, PCMK__XA_ST_REMOTE_OP);
437 cmd->client_name = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTNAME);
438 cmd->target = crm_element_value_copy(op, PCMK__XA_ST_TARGET);
439 cmd->device = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ID);
440
441 cmd->done_cb = st_child_done;
442
443
444 cmd_list = g_list_append(cmd_list, cmd);
445
446 return cmd;
447 }
448
449 static int
450 get_action_limit(fenced_device_t *device)
451 {
452 const char *value = NULL;
453 int action_limit = 1;
454
455 value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
456 if ((value == NULL)
457 || (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
458 || (action_limit == 0)) {
459 action_limit = 1;
460 }
461 return action_limit;
462 }
463
464 static int
465 get_active_cmds(fenced_device_t *device)
466 {
467 int counter = 0;
468 GList *gIter = NULL;
469 GList *gIterNext = NULL;
470
471 CRM_CHECK(device != NULL, return 0);
472
473 for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
474 async_command_t *cmd = gIter->data;
475
476 gIterNext = gIter->next;
477
478 if (cmd->active_on == device) {
479 counter++;
480 }
481 }
482
483 return counter;
484 }
485
486 static void
487 fork_cb(int pid, void *user_data)
488 {
489 async_command_t *cmd = (async_command_t *) user_data;
490 fenced_device_t *device = cmd->activating_on;
491
492 if (device == NULL) {
493
494
495
496 device = cmd->active_on;
497 }
498
499 pcmk__assert(device != NULL);
500 crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
501 cmd->action, pid,
502 ((cmd->target == NULL)? "" : " targeting "),
503 pcmk__s(cmd->target, ""), device->id, cmd->timeout);
504 cmd->active_on = device;
505 cmd->activating_on = NULL;
506 }
507
508 static int
509 get_agent_metadata_cb(gpointer data) {
510 fenced_device_t *device = data;
511 guint period_ms;
512
513 switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
514 case pcmk_rc_ok:
515 if (device->agent_metadata) {
516 read_action_metadata(device);
517 device->default_host_arg =
518 stonith__default_host_arg(device->agent_metadata);
519 }
520 return G_SOURCE_REMOVE;
521
522 case EAGAIN:
523 period_ms = pcmk__mainloop_timer_get_period(device->timer);
524 if (period_ms < 160 * 1000) {
525 mainloop_timer_set_period(device->timer, 2 * period_ms);
526 }
527 return G_SOURCE_CONTINUE;
528
529 default:
530 return G_SOURCE_REMOVE;
531 }
532 }
533
534
535
536
537
538
539
540
541
542
543 static void
544 report_internal_result(async_command_t *cmd, int exit_status,
545 int execution_status, const char *exit_reason)
546 {
547 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
548
549 pcmk__set_result(&result, exit_status, execution_status, exit_reason);
550 cmd->done_cb(0, &result, cmd);
551 pcmk__reset_result(&result);
552 }
553
554 static gboolean
555 stonith_device_execute(fenced_device_t *device)
556 {
557 int exec_rc = 0;
558 const char *action_str = NULL;
559 async_command_t *cmd = NULL;
560 stonith_action_t *action = NULL;
561 int active_cmds = 0;
562 int action_limit = 0;
563 GList *gIter = NULL;
564 GList *gIterNext = NULL;
565
566 CRM_CHECK(device != NULL, return FALSE);
567
568 active_cmds = get_active_cmds(device);
569 action_limit = get_action_limit(device);
570 if (action_limit > -1 && active_cmds >= action_limit) {
571 crm_trace("%s is over its action limit of %d (%u active action%s)",
572 device->id, action_limit, active_cmds,
573 pcmk__plural_s(active_cmds));
574 return TRUE;
575 }
576
577 for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
578 async_command_t *pending_op = gIter->data;
579
580 gIterNext = gIter->next;
581
582 if (pending_op && pending_op->delay_id) {
583 crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
584 "waiting for start delay of %ds",
585 pending_op->action,
586 ((pending_op->target == NULL)? "" : " targeting "),
587 pcmk__s(pending_op->target, ""),
588 device->id, pending_op->start_delay);
589 continue;
590 }
591
592 device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
593 g_list_free_1(gIter);
594
595 cmd = pending_op;
596 break;
597 }
598
599 if (cmd == NULL) {
600 crm_trace("No actions using %s are needed", device->id);
601 return TRUE;
602 }
603
604 if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
605 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
606 if (pcmk__is_fencing_action(cmd->action)) {
607 if (node_does_watchdog_fencing(fenced_get_local_node())) {
608 pcmk__panic("Watchdog self-fencing required");
609 goto done;
610 }
611 } else {
612 crm_info("Faking success for %s watchdog operation", cmd->action);
613 report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
614 goto done;
615 }
616 }
617
618 #if PCMK__ENABLE_CIBSECRETS
619 exec_rc = pcmk__substitute_secrets(device->id, device->params);
620 if (exec_rc != pcmk_rc_ok) {
621 if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_none)) {
622 crm_info("Proceeding with stop operation for %s "
623 "despite being unable to load CIB secrets (%s)",
624 device->id, pcmk_rc_str(exec_rc));
625 } else {
626 crm_err("Considering %s unconfigured "
627 "because unable to load CIB secrets: %s",
628 device->id, pcmk_rc_str(exec_rc));
629 report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
630 "Failed to get CIB secrets");
631 goto done;
632 }
633 }
634 #endif
635
636 action_str = cmd->action;
637 if (pcmk__str_eq(cmd->action, PCMK_ACTION_REBOOT, pcmk__str_none)
638 && !pcmk_is_set(device->flags, fenced_df_supports_reboot)) {
639
640 crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
641 "because agent '%s' does not support reboot",
642 ((cmd->target == NULL)? "" : " targeting "),
643 pcmk__s(cmd->target, ""), device->id, device->agent);
644 action_str = PCMK_ACTION_OFF;
645 }
646
647 action = stonith__action_create(device->agent, action_str, cmd->target,
648 cmd->timeout, device->params,
649 device->aliases, device->default_host_arg);
650
651
652
653 cmd->activating_on = device;
654 exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
655 fork_cb);
656 if (exec_rc < 0) {
657 cmd->activating_on = NULL;
658 cmd->done_cb(0, stonith__action_result(action), cmd);
659 stonith__destroy_action(action);
660 }
661
662 done:
663
664
665
666 if (device->pending_ops) {
667 mainloop_set_trigger(device->work);
668 }
669 return TRUE;
670 }
671
672 static gboolean
673 stonith_device_dispatch(gpointer user_data)
674 {
675 return stonith_device_execute(user_data);
676 }
677
678 static gboolean
679 start_delay_helper(gpointer data)
680 {
681 async_command_t *cmd = data;
682 fenced_device_t *device = cmd_device(cmd);
683
684 cmd->delay_id = 0;
685 if (device) {
686 mainloop_set_trigger(device->work);
687 }
688
689 return FALSE;
690 }
691
692 static void
693 schedule_stonith_command(async_command_t *cmd, fenced_device_t *device)
694 {
695 int delay_max = 0;
696 int delay_base = 0;
697 int requested_delay = cmd->start_delay;
698
699 CRM_CHECK(cmd != NULL, return);
700 CRM_CHECK(device != NULL, return);
701
702 if (cmd->device) {
703 free(cmd->device);
704 }
705
706 cmd->device = pcmk__str_copy(device->id);
707 cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
708
709 if (cmd->remote_op_id) {
710 crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
711 "with op id %.8s and timeout %ds",
712 cmd->action,
713 (cmd->target == NULL)? "" : " targeting ",
714 pcmk__s(cmd->target, ""),
715 device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
716 } else {
717 crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
718 cmd->action,
719 (cmd->target == NULL)? "" : " targeting ",
720 pcmk__s(cmd->target, ""),
721 device->id, cmd->client, cmd->timeout);
722 }
723
724 device->pending_ops = g_list_append(device->pending_ops, cmd);
725 mainloop_set_trigger(device->work);
726
727
728 if (requested_delay < 0) {
729 return;
730 }
731
732 delay_max = get_action_delay_max(device, cmd->action);
733 delay_base = get_action_delay_base(device, cmd->action, cmd->target);
734 if (delay_max == 0) {
735 delay_max = delay_base;
736 }
737 if (delay_max < delay_base) {
738 crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
739 PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
740 "(limiting to maximum delay)",
741 delay_base, delay_max, cmd->action, device->id);
742 delay_base = delay_max;
743 }
744 if (delay_max > 0) {
745 cmd->start_delay +=
746
747 ((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
748 + delay_base;
749 }
750
751 if (cmd->start_delay > 0) {
752 crm_notice("Delaying '%s' action%s%s using %s for %ds " QB_XS
753 " timeout=%ds requested_delay=%ds base=%ds max=%ds",
754 cmd->action,
755 (cmd->target == NULL)? "" : " targeting ",
756 pcmk__s(cmd->target, ""),
757 device->id, cmd->start_delay, cmd->timeout,
758 requested_delay, delay_base, delay_max);
759 cmd->delay_id =
760 pcmk__create_timer(cmd->start_delay * 1000, start_delay_helper, cmd);
761 }
762 }
763
764 static void
765 free_device(gpointer data)
766 {
767 GList *gIter = NULL;
768 fenced_device_t *device = data;
769
770 g_hash_table_destroy(device->params);
771 g_hash_table_destroy(device->aliases);
772
773 for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
774 async_command_t *cmd = gIter->data;
775
776 crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
777 report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
778 "Device was removed before action could be executed");
779 }
780 g_list_free(device->pending_ops);
781
782 g_list_free_full(device->targets, free);
783
784 if (device->timer) {
785 mainloop_timer_stop(device->timer);
786 mainloop_timer_del(device->timer);
787 }
788
789 mainloop_destroy_trigger(device->work);
790
791 pcmk__xml_free(device->agent_metadata);
792 free(device->namespace);
793 if (device->on_target_actions != NULL) {
794 g_string_free(device->on_target_actions, TRUE);
795 }
796 free(device->agent);
797 free(device->id);
798 free(device);
799 }
800
801
802
803
804
805 void
806 fenced_init_device_table(void)
807 {
808 if (device_table == NULL) {
809 device_table = pcmk__strkey_table(NULL, free_device);
810 }
811 }
812
813
814
815
816
817 void
818 fenced_free_device_table(void)
819 {
820 if (device_table != NULL) {
821 g_hash_table_destroy(device_table);
822 device_table = NULL;
823 }
824 }
825
826 static GHashTable *
827 build_port_aliases(const char *hostmap, GList ** targets)
828 {
829 char *name = NULL;
830 int last = 0, lpc = 0, max = 0, added = 0;
831 GHashTable *aliases = pcmk__strikey_table(free, free);
832
833 if (hostmap == NULL) {
834 return aliases;
835 }
836
837 max = strlen(hostmap);
838 for (; lpc <= max; lpc++) {
839 switch (hostmap[lpc]) {
840
841 case '\\':
842 lpc++;
843 break;
844
845
846 case '=':
847 case ':':
848 if (lpc > last) {
849 free(name);
850 name = pcmk__assert_alloc(1, 1 + lpc - last);
851 memcpy(name, hostmap + last, lpc - last);
852 }
853 last = lpc + 1;
854 break;
855
856
857
858 case 0:
859 case ';':
860 case ' ':
861 case '\t':
862 if (name) {
863 char *value = NULL;
864 int k = 0;
865
866 value = pcmk__assert_alloc(1, 1 + lpc - last);
867 memcpy(value, hostmap + last, lpc - last);
868
869 for (int i = 0; value[i] != '\0'; i++) {
870 if (value[i] != '\\') {
871 value[k++] = value[i];
872 }
873 }
874 value[k] = '\0';
875
876 crm_debug("Adding alias '%s'='%s'", name, value);
877 g_hash_table_replace(aliases, name, value);
878 if (targets) {
879 *targets = g_list_append(*targets, pcmk__str_copy(value));
880 }
881 value = NULL;
882 name = NULL;
883 added++;
884
885 } else if (lpc > last) {
886 crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
887 }
888
889 last = lpc + 1;
890 break;
891 }
892
893 if (hostmap[lpc] == 0) {
894 break;
895 }
896 }
897
898 if (added == 0) {
899 crm_info("No host mappings detected in '%s'", hostmap);
900 }
901
902 free(name);
903 return aliases;
904 }
905
906 GHashTable *metadata_cache = NULL;
907
908 void
909 free_metadata_cache(void) {
910 if (metadata_cache != NULL) {
911 g_hash_table_destroy(metadata_cache);
912 metadata_cache = NULL;
913 }
914 }
915
916 static void
917 init_metadata_cache(void) {
918 if (metadata_cache == NULL) {
919 metadata_cache = pcmk__strkey_table(free, free);
920 }
921 }
922
923 int
924 get_agent_metadata(const char *agent, xmlNode ** metadata)
925 {
926 char *buffer = NULL;
927
928 if (metadata == NULL) {
929 return EINVAL;
930 }
931 *metadata = NULL;
932 if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
933 return pcmk_rc_ok;
934 }
935 init_metadata_cache();
936 buffer = g_hash_table_lookup(metadata_cache, agent);
937 if (buffer == NULL) {
938 stonith_t *st = stonith__api_new();
939 int rc;
940
941 if (st == NULL) {
942 crm_warn("Could not get agent meta-data: "
943 "API memory allocation failed");
944 return EAGAIN;
945 }
946 rc = st->cmds->metadata(st, st_opt_sync_call, agent,
947 NULL, &buffer, 10);
948 stonith__api_free(st);
949 if (rc || !buffer) {
950 crm_err("Could not retrieve metadata for fencing agent %s", agent);
951 return EAGAIN;
952 }
953 g_hash_table_replace(metadata_cache, pcmk__str_copy(agent), buffer);
954 }
955
956 *metadata = pcmk__xml_parse(buffer);
957 return pcmk_rc_ok;
958 }
959
960 static void
961 read_action_metadata(fenced_device_t *device)
962 {
963 xmlXPathObject *xpath = NULL;
964 int max = 0;
965 int lpc = 0;
966
967 if (device->agent_metadata == NULL) {
968 return;
969 }
970
971 xpath = pcmk__xpath_search(device->agent_metadata->doc,
972 "//" PCMK_XE_ACTION);
973 max = pcmk__xpath_num_results(xpath);
974
975 if (max == 0) {
976 xmlXPathFreeObject(xpath);
977 return;
978 }
979
980 for (lpc = 0; lpc < max; lpc++) {
981 const char *action = NULL;
982 xmlNode *match = pcmk__xpath_result(xpath, lpc);
983
984 CRM_LOG_ASSERT(match != NULL);
985 if(match == NULL) { continue; };
986
987 action = crm_element_value(match, PCMK_XA_NAME);
988
989 if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
990 fenced_device_set_flags(device, fenced_df_supports_list);
991
992 } else if (pcmk__str_eq(action, PCMK_ACTION_STATUS, pcmk__str_none)) {
993 fenced_device_set_flags(device, fenced_df_supports_status);
994
995 } else if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
996 fenced_device_set_flags(device, fenced_df_supports_reboot);
997
998 } else if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)) {
999
1000
1001
1002
1003
1004
1005 if (pcmk__xe_attr_is_true(match, PCMK_XA_AUTOMATIC)
1006 || pcmk__xe_attr_is_true(match, PCMK__XA_REQUIRED)) {
1007
1008 fenced_device_set_flags(device, fenced_df_auto_unfence);
1009 }
1010 fenced_device_set_flags(device, fenced_df_supports_on);
1011 }
1012
1013 if ((action != NULL)
1014 && pcmk__xe_attr_is_true(match, PCMK_XA_ON_TARGET)) {
1015
1016 pcmk__add_word(&(device->on_target_actions), 64, action);
1017 }
1018 }
1019
1020 xmlXPathFreeObject(xpath);
1021 }
1022
1023 static const char *
1024 target_list_type(fenced_device_t *dev)
1025 {
1026 const char *check_type = NULL;
1027
1028 check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);
1029
1030 if (check_type == NULL) {
1031
1032 if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
1033 check_type = PCMK_VALUE_STATIC_LIST;
1034 } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
1035 check_type = PCMK_VALUE_STATIC_LIST;
1036 } else if (pcmk_is_set(dev->flags, fenced_df_supports_list)) {
1037 check_type = PCMK_VALUE_DYNAMIC_LIST;
1038 } else if (pcmk_is_set(dev->flags, fenced_df_supports_status)) {
1039 check_type = PCMK_VALUE_STATUS;
1040 } else {
1041 check_type = PCMK_VALUE_NONE;
1042 }
1043 }
1044
1045 return check_type;
1046 }
1047
1048 static fenced_device_t *
1049 build_device_from_xml(const xmlNode *dev)
1050 {
1051 const char *value;
1052 fenced_device_t *device = NULL;
1053 char *agent = crm_element_value_copy(dev, PCMK_XA_AGENT);
1054
1055 CRM_CHECK(agent != NULL, return device);
1056
1057 device = pcmk__assert_alloc(1, sizeof(fenced_device_t));
1058
1059 device->id = crm_element_value_copy(dev, PCMK_XA_ID);
1060 device->agent = agent;
1061 device->namespace = crm_element_value_copy(dev, PCMK__XA_NAMESPACE);
1062 device->params = xml2list(dev);
1063
1064 value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
1065 if (value) {
1066 device->targets = stonith__parse_targets(value);
1067 }
1068
1069 value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
1070 device->aliases = build_port_aliases(value, &(device->targets));
1071
1072 value = target_list_type(device);
1073 if (!pcmk__str_eq(value, PCMK_VALUE_STATIC_LIST, pcmk__str_casei)
1074 && (device->targets != NULL)) {
1075
1076
1077 g_list_free_full(device->targets, free);
1078 device->targets = NULL;
1079 }
1080 switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
1081 case pcmk_rc_ok:
1082 if (device->agent_metadata) {
1083 read_action_metadata(device);
1084 device->default_host_arg =
1085 stonith__default_host_arg(device->agent_metadata);
1086 }
1087 break;
1088
1089 case EAGAIN:
1090 if (device->timer == NULL) {
1091 device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
1092 TRUE, get_agent_metadata_cb, device);
1093 }
1094 if (!mainloop_timer_running(device->timer)) {
1095 mainloop_timer_start(device->timer);
1096 }
1097 break;
1098
1099 default:
1100 break;
1101 }
1102
1103 value = crm_element_value(dev, PCMK__XA_RSC_PROVIDES);
1104 if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
1105 fenced_device_set_flags(device, fenced_df_auto_unfence);
1106 }
1107
1108 if (is_action_required(PCMK_ACTION_ON, device)) {
1109 crm_info("Fencing device '%s' requires unfencing", device->id);
1110 }
1111
1112 if (device->on_target_actions != NULL) {
1113 crm_info("Fencing device '%s' requires actions (%s) to be executed "
1114 "on target", device->id,
1115 (const char *) device->on_target_actions->str);
1116 }
1117
1118 device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
1119
1120 return device;
1121 }
1122
1123 static void
1124 schedule_internal_command(const char *origin, fenced_device_t *device,
1125 const char *action, const char *target, int timeout,
1126 void *internal_user_data,
1127 void (*done_cb) (int pid,
1128 const pcmk__action_result_t *result,
1129 void *user_data))
1130 {
1131 async_command_t *cmd = NULL;
1132
1133 cmd = pcmk__assert_alloc(1, sizeof(async_command_t));
1134
1135 cmd->id = -1;
1136 cmd->default_timeout = timeout ? timeout : 60;
1137 cmd->timeout = cmd->default_timeout;
1138 cmd->action = pcmk__str_copy(action);
1139 cmd->target = pcmk__str_copy(target);
1140 cmd->device = pcmk__str_copy(device->id);
1141 cmd->origin = pcmk__str_copy(origin);
1142 cmd->client = pcmk__str_copy(crm_system_name);
1143 cmd->client_name = pcmk__str_copy(crm_system_name);
1144
1145 cmd->internal_user_data = internal_user_data;
1146 cmd->done_cb = done_cb;
1147
1148 schedule_stonith_command(cmd, device);
1149 }
1150
1151
1152 enum fence_status_code {
1153 fence_status_invalid = -1,
1154 fence_status_active = 0,
1155 fence_status_unknown = 1,
1156 fence_status_inactive = 2,
1157 };
1158
1159 static void
1160 status_search_cb(int pid, const pcmk__action_result_t *result, void *user_data)
1161 {
1162 async_command_t *cmd = user_data;
1163 struct device_search_s *search = cmd->internal_user_data;
1164 fenced_device_t *dev = cmd_device(cmd);
1165 gboolean can = FALSE;
1166
1167 free_async_command(cmd);
1168
1169 if (!dev) {
1170 search_devices_record_result(search, NULL, FALSE);
1171 return;
1172 }
1173
1174 mainloop_set_trigger(dev->work);
1175
1176 if (result->execution_status != PCMK_EXEC_DONE) {
1177 crm_warn("Assuming %s cannot fence %s "
1178 "because status could not be executed: %s%s%s%s",
1179 dev->id, search->host,
1180 pcmk_exec_status_str(result->execution_status),
1181 ((result->exit_reason == NULL)? "" : " ("),
1182 ((result->exit_reason == NULL)? "" : result->exit_reason),
1183 ((result->exit_reason == NULL)? "" : ")"));
1184 search_devices_record_result(search, dev->id, FALSE);
1185 return;
1186 }
1187
1188 switch (result->exit_status) {
1189 case fence_status_unknown:
1190 crm_trace("%s reported it cannot fence %s", dev->id, search->host);
1191 break;
1192
1193 case fence_status_active:
1194 case fence_status_inactive:
1195 crm_trace("%s reported it can fence %s", dev->id, search->host);
1196 can = TRUE;
1197 break;
1198
1199 default:
1200 crm_warn("Assuming %s cannot fence %s "
1201 "(status returned unknown code %d)",
1202 dev->id, search->host, result->exit_status);
1203 break;
1204 }
1205 search_devices_record_result(search, dev->id, can);
1206 }
1207
1208 static void
1209 dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
1210 void *user_data)
1211 {
1212 async_command_t *cmd = user_data;
1213 struct device_search_s *search = cmd->internal_user_data;
1214 fenced_device_t *dev = cmd_device(cmd);
1215 gboolean can_fence = FALSE;
1216
1217 free_async_command(cmd);
1218
1219
1220
1221
1222
1223
1224 if (!dev) {
1225 search_devices_record_result(search, NULL, FALSE);
1226 return;
1227 }
1228
1229 mainloop_set_trigger(dev->work);
1230
1231 if (pcmk__result_ok(result)) {
1232 crm_info("Refreshing target list for %s", dev->id);
1233 g_list_free_full(dev->targets, free);
1234 dev->targets = stonith__parse_targets(result->action_stdout);
1235 dev->targets_age = time(NULL);
1236
1237 } else if (dev->targets != NULL) {
1238 if (result->execution_status == PCMK_EXEC_DONE) {
1239 crm_info("Reusing most recent target list for %s "
1240 "because list returned error code %d",
1241 dev->id, result->exit_status);
1242 } else {
1243 crm_info("Reusing most recent target list for %s "
1244 "because list could not be executed: %s%s%s%s",
1245 dev->id, pcmk_exec_status_str(result->execution_status),
1246 ((result->exit_reason == NULL)? "" : " ("),
1247 ((result->exit_reason == NULL)? "" : result->exit_reason),
1248 ((result->exit_reason == NULL)? "" : ")"));
1249 }
1250
1251 } else {
1252 if (result->execution_status == PCMK_EXEC_DONE) {
1253 crm_warn("Assuming %s cannot fence %s "
1254 "because list returned error code %d",
1255 dev->id, search->host, result->exit_status);
1256 } else {
1257 crm_warn("Assuming %s cannot fence %s "
1258 "because list could not be executed: %s%s%s%s",
1259 dev->id, search->host,
1260 pcmk_exec_status_str(result->execution_status),
1261 ((result->exit_reason == NULL)? "" : " ("),
1262 ((result->exit_reason == NULL)? "" : result->exit_reason),
1263 ((result->exit_reason == NULL)? "" : ")"));
1264 }
1265
1266
1267
1268
1269 if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
1270 crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
1271 pcmk__insert_dup(dev->params, PCMK_STONITH_HOST_CHECK,
1272 PCMK_VALUE_STATUS);
1273 }
1274 }
1275
1276 if (dev->targets) {
1277 const char *alias = g_hash_table_lookup(dev->aliases, search->host);
1278
1279 if (!alias) {
1280 alias = search->host;
1281 }
1282 if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
1283 can_fence = TRUE;
1284 }
1285 }
1286 search_devices_record_result(search, dev->id, can_fence);
1287 }
1288
1289
1290
1291
1292
1293 static int
1294 device_params_diff(GHashTable *first, GHashTable *second) {
1295 char *key = NULL;
1296 char *value = NULL;
1297 GHashTableIter gIter;
1298
1299 g_hash_table_iter_init(&gIter, first);
1300 while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
1301
1302 if(strstr(key, "CRM_meta") == key) {
1303 continue;
1304 } else if (strcmp(key, PCMK_XA_CRM_FEATURE_SET) == 0) {
1305 continue;
1306 } else {
1307 char *other_value = g_hash_table_lookup(second, key);
1308
1309 if (!other_value || !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
1310 crm_trace("Different value for %s: %s != %s", key, other_value, value);
1311 return 1;
1312 }
1313 }
1314 }
1315
1316 return 0;
1317 }
1318
1319
1320
1321
1322
1323 static fenced_device_t *
1324 device_has_duplicate(const fenced_device_t *device)
1325 {
1326 fenced_device_t *dup = g_hash_table_lookup(device_table, device->id);
1327
1328 if (!dup) {
1329 crm_trace("No match for %s", device->id);
1330 return NULL;
1331
1332 } else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
1333 crm_trace("Different agent: %s != %s", dup->agent, device->agent);
1334 return NULL;
1335 }
1336
1337
1338 if (device_params_diff(device->params, dup->params) ||
1339 device_params_diff(dup->params, device->params)) {
1340 return NULL;
1341 }
1342
1343 crm_trace("Match");
1344 return dup;
1345 }
1346
1347 int
1348 fenced_device_register(const xmlNode *dev, bool from_cib)
1349 {
1350 const char *local_node_name = fenced_get_local_node();
1351 fenced_device_t *dup = NULL;
1352 fenced_device_t *device = build_device_from_xml(dev);
1353 int rc = pcmk_rc_ok;
1354
1355 CRM_CHECK(device != NULL, return ENOMEM);
1356
1357
1358 if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none)
1359 || pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
1360 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
1361
1362 if (stonith_watchdog_timeout_ms <= 0) {
1363 crm_err("Ignoring watchdog fence device without "
1364 PCMK_OPT_STONITH_WATCHDOG_TIMEOUT " set");
1365 rc = ENODEV;
1366 goto done;
1367 }
1368 if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
1369 STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
1370 crm_err("Ignoring watchdog fence device with unknown agent '%s' "
1371 "rather than '" STONITH_WATCHDOG_AGENT "'",
1372 pcmk__s(device->agent, ""));
1373 rc = ENODEV;
1374 goto done;
1375 }
1376 if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
1377 crm_err("Ignoring watchdog fence device named '%s' rather than "
1378 "'" STONITH_WATCHDOG_ID "'",
1379 pcmk__s(device->id, ""));
1380 rc = ENODEV;
1381 goto done;
1382 }
1383
1384 if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
1385 pcmk__str_none)) {
1386
1387
1388
1389 g_list_free_full(stonith_watchdog_targets, free);
1390 stonith_watchdog_targets = device->targets;
1391 device->targets = NULL;
1392 }
1393
1394 if (!node_does_watchdog_fencing(local_node_name)) {
1395 crm_debug("Skip registration of watchdog fence device on node not "
1396 "in host list");
1397 device->targets = NULL;
1398 stonith_device_remove(device->id, from_cib);
1399 goto done;
1400 }
1401
1402
1403 g_list_free_full(device->targets, free);
1404 device->targets = stonith__parse_targets(local_node_name);
1405 pcmk__insert_dup(device->params, PCMK_STONITH_HOST_LIST,
1406 local_node_name);
1407 }
1408
1409 dup = device_has_duplicate(device);
1410 if (dup != NULL) {
1411 guint ndevices = g_hash_table_size(device_table);
1412
1413 crm_debug("Device '%s' already in device list (%d active device%s)",
1414 device->id, ndevices, pcmk__plural_s(ndevices));
1415 free_device(device);
1416 device = dup;
1417 fenced_device_clear_flags(device, fenced_df_dirty);
1418
1419 } else {
1420 guint ndevices = 0;
1421 fenced_device_t *old = g_hash_table_lookup(device_table, device->id);
1422
1423 if (from_cib && (old != NULL)
1424 && pcmk_is_set(old->flags, fenced_df_api_registered)) {
1425
1426
1427
1428
1429
1430 crm_info("Overwriting existing entry for %s from CIB", device->id);
1431 device->pending_ops = old->pending_ops;
1432 fenced_device_set_flags(device, fenced_df_api_registered);
1433 old->pending_ops = NULL;
1434 if (device->pending_ops != NULL) {
1435 mainloop_set_trigger(device->work);
1436 }
1437 }
1438 g_hash_table_replace(device_table, device->id, device);
1439
1440 ndevices = g_hash_table_size(device_table);
1441 crm_notice("Added '%s' to device list (%d active device%s)",
1442 device->id, ndevices, pcmk__plural_s(ndevices));
1443 }
1444
1445 if (from_cib) {
1446 fenced_device_set_flags(device, fenced_df_cib_registered);
1447 } else {
1448 fenced_device_set_flags(device, fenced_df_api_registered);
1449 }
1450
1451 done:
1452 if (rc != pcmk_rc_ok) {
1453 free_device(device);
1454 }
1455 return rc;
1456 }
1457
1458 void
1459 stonith_device_remove(const char *id, bool from_cib)
1460 {
1461 fenced_device_t *device = g_hash_table_lookup(device_table, id);
1462 guint ndevices = 0;
1463
1464 if (device == NULL) {
1465 ndevices = g_hash_table_size(device_table);
1466 crm_info("Device '%s' not found (%u active device%s)", id, ndevices,
1467 pcmk__plural_s(ndevices));
1468 return;
1469 }
1470
1471 if (from_cib) {
1472 fenced_device_clear_flags(device, fenced_df_cib_registered);
1473 } else {
1474 fenced_device_clear_flags(device,
1475 fenced_df_api_registered|fenced_df_verified);
1476 }
1477
1478 if (!pcmk_any_flags_set(device->flags,
1479 fenced_df_api_registered
1480 |fenced_df_cib_registered)) {
1481 g_hash_table_remove(device_table, id);
1482 ndevices = g_hash_table_size(device_table);
1483 crm_info("Removed '%s' from device list (%u active device%s)",
1484 id, ndevices, pcmk__plural_s(ndevices));
1485 } else {
1486
1487 const bool cib_registered = pcmk_is_set(device->flags,
1488 fenced_df_cib_registered);
1489
1490 crm_trace("Not removing '%s' from device list (%u active) because "
1491 "still registered via %s",
1492 id, g_hash_table_size(device_table),
1493 (cib_registered? "CIB" : "API"));
1494 }
1495 }
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506 static int
1507 count_active_levels(const stonith_topology_t *tp)
1508 {
1509 int lpc = 0;
1510 int count = 0;
1511
1512 for (lpc = 0; lpc < ST__LEVEL_COUNT; lpc++) {
1513 if (tp->levels[lpc] != NULL) {
1514 count++;
1515 }
1516 }
1517 return count;
1518 }
1519
1520 static void
1521 free_topology_entry(gpointer data)
1522 {
1523 stonith_topology_t *tp = data;
1524
1525 int lpc = 0;
1526
1527 for (lpc = 0; lpc < ST__LEVEL_COUNT; lpc++) {
1528 if (tp->levels[lpc] != NULL) {
1529 g_list_free_full(tp->levels[lpc], free);
1530 }
1531 }
1532 free(tp->target);
1533 free(tp->target_value);
1534 free(tp->target_pattern);
1535 free(tp->target_attribute);
1536 free(tp);
1537 }
1538
1539 void
1540 free_topology_list(void)
1541 {
1542 if (topology != NULL) {
1543 g_hash_table_destroy(topology);
1544 topology = NULL;
1545 }
1546 }
1547
1548 void
1549 init_topology_list(void)
1550 {
1551 if (topology == NULL) {
1552 topology = pcmk__strkey_table(NULL, free_topology_entry);
1553 }
1554 }
1555
1556 char *
1557 stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
1558 {
1559 if (mode == fenced_target_by_unknown) {
1560 mode = unpack_level_kind(level);
1561 }
1562 switch (mode) {
1563 case fenced_target_by_name:
1564 return crm_element_value_copy(level, PCMK_XA_TARGET);
1565
1566 case fenced_target_by_pattern:
1567 return crm_element_value_copy(level, PCMK_XA_TARGET_PATTERN);
1568
1569 case fenced_target_by_attribute:
1570 return crm_strdup_printf("%s=%s",
1571 crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE),
1572 crm_element_value(level, PCMK_XA_TARGET_VALUE));
1573
1574 default:
1575 return crm_strdup_printf("unknown-%s", pcmk__xe_id(level));
1576 }
1577 }
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587 static enum fenced_target_by
1588 unpack_level_kind(const xmlNode *level)
1589 {
1590 if (crm_element_value(level, PCMK_XA_TARGET) != NULL) {
1591 return fenced_target_by_name;
1592 }
1593 if (crm_element_value(level, PCMK_XA_TARGET_PATTERN) != NULL) {
1594 return fenced_target_by_pattern;
1595 }
1596 if ((crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE) != NULL)
1597 && (crm_element_value(level, PCMK_XA_TARGET_VALUE) != NULL)) {
1598 return fenced_target_by_attribute;
1599 }
1600 return fenced_target_by_unknown;
1601 }
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615 static xmlNode *
1616 unpack_level_request(xmlNode *xml, enum fenced_target_by *mode, char **target,
1617 int *id)
1618 {
1619 enum fenced_target_by local_mode = fenced_target_by_unknown;
1620 char *local_target = NULL;
1621 int local_id = 0;
1622
1623
1624
1625
1626
1627 if ((xml != NULL) && !pcmk__xe_is(xml, PCMK_XE_FENCING_LEVEL)) {
1628 xml = pcmk__xpath_find_one(xml->doc, "//" PCMK_XE_FENCING_LEVEL,
1629 LOG_WARNING);
1630 }
1631
1632 if (xml != NULL) {
1633 local_mode = unpack_level_kind(xml);
1634 local_target = stonith_level_key(xml, local_mode);
1635 crm_element_value_int(xml, PCMK_XA_INDEX, &local_id);
1636 }
1637
1638 if (mode != NULL) {
1639 *mode = local_mode;
1640 }
1641 if (id != NULL) {
1642 *id = local_id;
1643 }
1644
1645 if (target != NULL) {
1646 *target = local_target;
1647 } else {
1648 free(local_target);
1649 }
1650
1651 return xml;
1652 }
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666 void
1667 fenced_register_level(xmlNode *msg, pcmk__action_result_t *result)
1668 {
1669 int id = 0;
1670 xmlNode *level;
1671 enum fenced_target_by mode;
1672 char *target;
1673
1674 stonith_topology_t *tp;
1675 const char *value = NULL;
1676
1677 CRM_CHECK(msg != NULL, return);
1678
1679 level = unpack_level_request(msg, &mode, &target, &id);
1680 if (level == NULL) {
1681 set_bad_request_result(result);
1682 return;
1683 }
1684
1685
1686 if (pcmk__str_empty(pcmk__xe_id(level))) {
1687 crm_warn("Ignoring registration for topology level without ID");
1688 free(target);
1689 crm_log_xml_trace(level, "Bad level");
1690 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1691 "Topology level is invalid without ID");
1692 return;
1693 }
1694
1695
1696 if (mode == fenced_target_by_unknown) {
1697 crm_warn("Ignoring registration for topology level '%s' "
1698 "without valid target", pcmk__xe_id(level));
1699 free(target);
1700 crm_log_xml_trace(level, "Bad level");
1701 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1702 "Invalid target for topology level '%s'",
1703 pcmk__xe_id(level));
1704 return;
1705 }
1706
1707
1708 if ((id < ST__LEVEL_MIN) || (id > ST__LEVEL_MAX)) {
1709 crm_warn("Ignoring topology registration for %s with invalid level %d",
1710 target, id);
1711 free(target);
1712 crm_log_xml_trace(level, "Bad level");
1713 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1714 "Invalid level number '%s' for topology level '%s'",
1715 pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
1716 ""),
1717 pcmk__xe_id(level));
1718 return;
1719 }
1720
1721
1722 tp = g_hash_table_lookup(topology, target);
1723 if (tp == NULL) {
1724 tp = pcmk__assert_alloc(1, sizeof(stonith_topology_t));
1725
1726 tp->kind = mode;
1727 tp->target = target;
1728 tp->target_value = crm_element_value_copy(level, PCMK_XA_TARGET_VALUE);
1729 tp->target_pattern = crm_element_value_copy(level,
1730 PCMK_XA_TARGET_PATTERN);
1731 tp->target_attribute = crm_element_value_copy(level,
1732 PCMK_XA_TARGET_ATTRIBUTE);
1733
1734 g_hash_table_replace(topology, tp->target, tp);
1735 crm_trace("Added %s (%d) to the topology (%d active entries)",
1736 target, (int) mode, g_hash_table_size(topology));
1737 } else {
1738 free(target);
1739 }
1740
1741 if (tp->levels[id] != NULL) {
1742 crm_info("Adding to the existing %s[%d] topology entry",
1743 tp->target, id);
1744 }
1745
1746 value = crm_element_value(level, PCMK_XA_DEVICES);
1747 if (value != NULL) {
1748
1749
1750
1751 gchar **devices = g_strsplit(value, ",", 0);
1752
1753 for (char **dev = devices; (dev != NULL) && (*dev != NULL); dev++) {
1754 crm_trace("Adding device '%s' for %s[%d]", *dev, tp->target, id);
1755 tp->levels[id] = g_list_append(tp->levels[id],
1756 pcmk__str_copy(*dev));
1757 }
1758 g_strfreev(devices);
1759 }
1760
1761 {
1762 int nlevels = count_active_levels(tp);
1763
1764 crm_info("Target %s has %d active fencing level%s",
1765 tp->target, nlevels, pcmk__plural_s(nlevels));
1766 }
1767
1768 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1769 }
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782 void
1783 fenced_unregister_level(xmlNode *msg, pcmk__action_result_t *result)
1784 {
1785 int id = -1;
1786 stonith_topology_t *tp;
1787 char *target;
1788 xmlNode *level = NULL;
1789
1790 level = unpack_level_request(msg, NULL, &target, &id);
1791 if (level == NULL) {
1792 set_bad_request_result(result);
1793 return;
1794 }
1795
1796
1797 if ((id < 0) || (id >= ST__LEVEL_COUNT)) {
1798 crm_warn("Ignoring topology unregistration for %s with invalid level %d",
1799 target, id);
1800 free(target);
1801 crm_log_xml_trace(level, "Bad level");
1802 pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
1803 "Invalid level number '%s' for topology level %s",
1804 pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
1805 "<null>"),
1806
1807
1808 pcmk__s(pcmk__xe_id(level), ""));
1809 return;
1810 }
1811
1812 tp = g_hash_table_lookup(topology, target);
1813 if (tp == NULL) {
1814 guint nentries = g_hash_table_size(topology);
1815
1816 crm_info("No fencing topology found for %s (%d active %s)",
1817 target, nentries,
1818 pcmk__plural_alt(nentries, "entry", "entries"));
1819
1820 } else if (id == 0 && g_hash_table_remove(topology, target)) {
1821 guint nentries = g_hash_table_size(topology);
1822
1823 crm_info("Removed all fencing topology entries related to %s "
1824 "(%d active %s remaining)", target, nentries,
1825 pcmk__plural_alt(nentries, "entry", "entries"));
1826
1827 } else if (tp->levels[id] != NULL) {
1828 guint nlevels;
1829
1830 g_list_free_full(tp->levels[id], free);
1831 tp->levels[id] = NULL;
1832
1833 nlevels = count_active_levels(tp);
1834 crm_info("Removed level %d from fencing topology for %s "
1835 "(%d active level%s remaining)",
1836 id, target, nlevels, pcmk__plural_s(nlevels));
1837 }
1838
1839 free(target);
1840 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1841 }
1842
1843 static char *
1844 list_to_string(GList *list, const char *delim, gboolean terminate_with_delim)
1845 {
1846 int max = g_list_length(list);
1847 size_t delim_len = delim?strlen(delim):0;
1848 size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
1849 char *rv;
1850 GList *gIter;
1851
1852 char *pos = NULL;
1853 const char *lead_delim = "";
1854
1855 for (gIter = list; gIter != NULL; gIter = gIter->next) {
1856 const char *value = (const char *) gIter->data;
1857
1858 alloc_size += strlen(value);
1859 }
1860
1861 rv = pcmk__assert_alloc(alloc_size, sizeof(char));
1862 pos = rv;
1863
1864 for (gIter = list; gIter != NULL; gIter = gIter->next) {
1865 const char *value = (const char *) gIter->data;
1866
1867 pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
1868 lead_delim = delim;
1869 }
1870
1871 if (max && terminate_with_delim) {
1872 sprintf(pos, "%s", delim);
1873 }
1874
1875 return rv;
1876 }
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893 static void
1894 execute_agent_action(xmlNode *msg, pcmk__action_result_t *result)
1895 {
1896 xmlNode *dev = pcmk__xpath_find_one(msg->doc, "//" PCMK__XE_ST_DEVICE_ID,
1897 LOG_ERR);
1898 xmlNode *op = pcmk__xpath_find_one(msg->doc,
1899 "//*[@" PCMK__XA_ST_DEVICE_ACTION "]",
1900 LOG_ERR);
1901 const char *id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
1902 const char *action = crm_element_value(op, PCMK__XA_ST_DEVICE_ACTION);
1903 async_command_t *cmd = NULL;
1904 fenced_device_t *device = NULL;
1905
1906 if ((id == NULL) || (action == NULL)) {
1907 crm_info("Malformed API action request: device %s, action %s",
1908 (id? id : "not specified"),
1909 (action? action : "not specified"));
1910 set_bad_request_result(result);
1911 return;
1912 }
1913
1914 if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
1915
1916 if (stonith_watchdog_timeout_ms <= 0) {
1917 pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1918 "Watchdog fence device not configured");
1919 return;
1920
1921 } else if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
1922 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1923 pcmk__set_result_output(result,
1924 list_to_string(stonith_watchdog_targets,
1925 "\n", TRUE),
1926 NULL);
1927 return;
1928
1929 } else if (pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_none)) {
1930 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
1931 return;
1932 }
1933 }
1934
1935 device = g_hash_table_lookup(device_table, id);
1936 if (device == NULL) {
1937 crm_info("Ignoring API '%s' action request because device %s not found",
1938 action, id);
1939 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1940 "'%s' not found", id);
1941 return;
1942
1943 } else if (!pcmk_is_set(device->flags, fenced_df_api_registered)
1944 && (strcmp(action, PCMK_ACTION_MONITOR) == 0)) {
1945
1946 crm_info("Ignoring API '%s' action request because device %s not active",
1947 action, id);
1948 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
1949 "'%s' not active", id);
1950 return;
1951 }
1952
1953 cmd = create_async_command(msg);
1954 if (cmd == NULL) {
1955 crm_log_xml_warn(msg, "invalid");
1956 set_bad_request_result(result);
1957 return;
1958 }
1959
1960 schedule_stonith_command(cmd, device);
1961 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
1962 }
1963
1964 static void
1965 search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
1966 {
1967 search->replies_received++;
1968 if (can_fence && device) {
1969 if (search->support_action_only != fenced_df_none) {
1970 fenced_device_t *dev = g_hash_table_lookup(device_table, device);
1971 if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
1972 return;
1973 }
1974 }
1975 search->capable = g_list_append(search->capable,
1976 pcmk__str_copy(device));
1977 }
1978
1979 if (search->replies_needed == search->replies_received) {
1980
1981 guint ndevices = g_list_length(search->capable);
1982
1983 crm_debug("Search found %d device%s that can perform '%s' targeting %s",
1984 ndevices, pcmk__plural_s(ndevices),
1985 (search->action? search->action : "unknown action"),
1986 (search->host? search->host : "any node"));
1987
1988 search->callback(search->capable, search->user_data);
1989 free(search->host);
1990 free(search->action);
1991 free(search);
1992 }
1993 }
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006 static gboolean
2007 localhost_is_eligible(const fenced_device_t *device, const char *action,
2008 const char *target, gboolean allow_self)
2009 {
2010 gboolean localhost_is_target = pcmk__str_eq(target, fenced_get_local_node(),
2011 pcmk__str_casei);
2012
2013 if ((device != NULL) && (action != NULL)
2014 && (device->on_target_actions != NULL)
2015 && (strstr((const char*) device->on_target_actions->str,
2016 action) != NULL)) {
2017
2018 if (!localhost_is_target) {
2019 crm_trace("Operation '%s' using %s can only be executed for local "
2020 "host, not %s", action, device->id, target);
2021 return FALSE;
2022 }
2023
2024 } else if (localhost_is_target && !allow_self) {
2025 crm_trace("'%s' operation does not support self-fencing", action);
2026 return FALSE;
2027 }
2028 return TRUE;
2029 }
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043 static bool
2044 localhost_is_eligible_with_remap(const fenced_device_t *device,
2045 const char *action, const char *target,
2046 gboolean allow_self)
2047 {
2048
2049 if (localhost_is_eligible(device, action, target, allow_self)) {
2050 return true;
2051 }
2052
2053
2054
2055 if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
2056
2057
2058
2059
2060
2061 if (localhost_is_eligible(device, PCMK_ACTION_OFF, target, allow_self)
2062 || localhost_is_eligible(device, PCMK_ACTION_ON, target, FALSE)) {
2063 return true;
2064 }
2065 }
2066
2067 return false;
2068 }
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079 static inline bool
2080 can_use_target_cache(const fenced_device_t *dev)
2081 {
2082 return (dev->targets != NULL) && (time(NULL) < (dev->targets_age + 60));
2083 }
2084
2085 static void
2086 can_fence_host_with_device(fenced_device_t *dev,
2087 struct device_search_s *search)
2088 {
2089 gboolean can = FALSE;
2090 const char *check_type = "Internal bug";
2091 const char *target = NULL;
2092 const char *alias = NULL;
2093 const char *dev_id = "Unspecified device";
2094 const char *action = (search == NULL)? NULL : search->action;
2095
2096 CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results);
2097
2098 if (dev->id != NULL) {
2099 dev_id = dev->id;
2100 }
2101
2102 target = search->host;
2103 if (target == NULL) {
2104 can = TRUE;
2105 check_type = "No target";
2106 goto search_report_results;
2107 }
2108
2109
2110
2111
2112 if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)
2113 && !pcmk_is_set(dev->flags, fenced_df_supports_on)) {
2114 check_type = "Agent does not support 'on'";
2115 goto search_report_results;
2116
2117 } else if (!localhost_is_eligible_with_remap(dev, action, target,
2118 search->allow_self)) {
2119 check_type = "This node is not allowed to execute action";
2120 goto search_report_results;
2121 }
2122
2123
2124 check_type = target_list_type(dev);
2125 alias = g_hash_table_lookup(dev->aliases, target);
2126 if (pcmk__str_eq(check_type, PCMK_VALUE_NONE, pcmk__str_casei)) {
2127 can = TRUE;
2128
2129 } else if (pcmk__str_eq(check_type, PCMK_VALUE_STATIC_LIST,
2130 pcmk__str_casei)) {
2131
2132 if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) {
2133 can = TRUE;
2134 } else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
2135 && g_hash_table_lookup(dev->aliases, target)) {
2136 can = TRUE;
2137 }
2138
2139 } else if (pcmk__str_eq(check_type, PCMK_VALUE_DYNAMIC_LIST,
2140 pcmk__str_casei)) {
2141 if (!can_use_target_cache(dev)) {
2142 int device_timeout = get_action_timeout(dev, PCMK_ACTION_LIST,
2143 search->per_device_timeout);
2144
2145 if (device_timeout > search->per_device_timeout) {
2146 crm_notice("Since the pcmk_list_timeout (%ds) parameter of %s "
2147 "is larger than " PCMK_OPT_STONITH_TIMEOUT
2148 " (%ds), timeout may occur",
2149 device_timeout, dev_id, search->per_device_timeout);
2150 }
2151
2152 crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
2153 check_type, dev_id, target, action);
2154
2155 schedule_internal_command(__func__, dev, PCMK_ACTION_LIST, NULL,
2156 search->per_device_timeout, search, dynamic_list_search_cb);
2157
2158
2159 return;
2160 }
2161
2162 if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets,
2163 pcmk__str_casei)) {
2164 can = TRUE;
2165 }
2166
2167 } else if (pcmk__str_eq(check_type, PCMK_VALUE_STATUS, pcmk__str_casei)) {
2168 int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);
2169
2170 if (device_timeout > search->per_device_timeout) {
2171 crm_notice("Since the pcmk_status_timeout (%ds) parameter of %s is "
2172 "larger than " PCMK_OPT_STONITH_TIMEOUT " (%ds), "
2173 "timeout may occur",
2174 device_timeout, dev_id, search->per_device_timeout);
2175 }
2176
2177 crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
2178 check_type, dev_id, target, action);
2179 schedule_internal_command(__func__, dev, PCMK_ACTION_STATUS, target,
2180 search->per_device_timeout, search, status_search_cb);
2181
2182 return;
2183 } else {
2184 crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
2185 check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
2186 }
2187
2188 search_report_results:
2189 crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s",
2190 dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"),
2191 pcmk__s(target, "unspecified target"),
2192 (alias == NULL)? "" : " (as '", pcmk__s(alias, ""),
2193 (alias == NULL)? "" : "')", check_type);
2194 search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can);
2195 }
2196
2197 static void
2198 search_devices(gpointer key, gpointer value, gpointer user_data)
2199 {
2200 fenced_device_t *dev = value;
2201 struct device_search_s *search = user_data;
2202
2203 can_fence_host_with_device(dev, search);
2204 }
2205
2206 #define DEFAULT_QUERY_TIMEOUT 20
2207 static void
2208 get_capable_devices(const char *host, const char *action, int timeout,
2209 bool allow_self, void *user_data,
2210 void (*callback) (GList * devices, void *user_data),
2211 uint32_t support_action_only)
2212 {
2213 struct device_search_s *search;
2214 guint ndevices = g_hash_table_size(device_table);
2215
2216 if (ndevices == 0) {
2217 callback(NULL, user_data);
2218 return;
2219 }
2220
2221 search = pcmk__assert_alloc(1, sizeof(struct device_search_s));
2222
2223 search->host = pcmk__str_copy(host);
2224 search->action = pcmk__str_copy(action);
2225 search->per_device_timeout = timeout;
2226 search->allow_self = allow_self;
2227 search->callback = callback;
2228 search->user_data = user_data;
2229 search->support_action_only = support_action_only;
2230
2231
2232
2233
2234 search->replies_needed = ndevices;
2235
2236 crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
2237 ndevices, pcmk__plural_s(ndevices),
2238 (search->action? search->action : "unknown action"),
2239 (search->host? search->host : "any node"));
2240 fenced_foreach_device(search_devices, search);
2241 }
2242
2243 struct st_query_data {
2244 xmlNode *reply;
2245 char *remote_peer;
2246 char *client_id;
2247 char *target;
2248 char *action;
2249 int call_options;
2250 };
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261 static void
2262 add_action_specific_attributes(xmlNode *xml, const char *action,
2263 const fenced_device_t *device,
2264 const char *target)
2265 {
2266 int action_specific_timeout;
2267 int delay_max;
2268 int delay_base;
2269
2270 CRM_CHECK(xml && action && device, return);
2271
2272
2273 if (is_action_required(action, device)) {
2274 crm_trace("Action '%s' is required using %s", action, device->id);
2275 crm_xml_add_int(xml, PCMK__XA_ST_REQUIRED, 1);
2276 }
2277
2278
2279 action_specific_timeout = get_action_timeout(device, action, 0);
2280 if (action_specific_timeout) {
2281 crm_trace("Action '%s' has timeout %ds using %s",
2282 action, action_specific_timeout, device->id);
2283 crm_xml_add_int(xml, PCMK__XA_ST_ACTION_TIMEOUT,
2284 action_specific_timeout);
2285 }
2286
2287 delay_max = get_action_delay_max(device, action);
2288 if (delay_max > 0) {
2289 crm_trace("Action '%s' has maximum random delay %ds using %s",
2290 action, delay_max, device->id);
2291 crm_xml_add_int(xml, PCMK__XA_ST_DELAY_MAX, delay_max);
2292 }
2293
2294 delay_base = get_action_delay_base(device, action, target);
2295 if (delay_base > 0) {
2296 crm_xml_add_int(xml, PCMK__XA_ST_DELAY_BASE, delay_base);
2297 }
2298
2299 if ((delay_max > 0) && (delay_base == 0)) {
2300 crm_trace("Action '%s' has maximum random delay %ds using %s",
2301 action, delay_max, device->id);
2302 } else if ((delay_max == 0) && (delay_base > 0)) {
2303 crm_trace("Action '%s' has a static delay of %ds using %s",
2304 action, delay_base, device->id);
2305 } else if ((delay_max > 0) && (delay_base > 0)) {
2306 crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen "
2307 "maximum delay of %ds using %s",
2308 action, delay_base, delay_max, device->id);
2309 }
2310 }
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322 static void
2323 add_disallowed(xmlNode *xml, const char *action, const fenced_device_t *device,
2324 const char *target, gboolean allow_self)
2325 {
2326 if (!localhost_is_eligible(device, action, target, allow_self)) {
2327 crm_trace("Action '%s' using %s is disallowed for local host",
2328 action, device->id);
2329 pcmk__xe_set_bool_attr(xml, PCMK__XA_ST_ACTION_DISALLOWED, true);
2330 }
2331 }
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343 static void
2344 add_action_reply(xmlNode *xml, const char *action,
2345 const fenced_device_t *device, const char *target,
2346 gboolean allow_self)
2347 {
2348 xmlNode *child = pcmk__xe_create(xml, PCMK__XE_ST_DEVICE_ACTION);
2349
2350 crm_xml_add(child, PCMK_XA_ID, action);
2351 add_action_specific_attributes(child, action, device, target);
2352 add_disallowed(child, action, device, target, allow_self);
2353 }
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364 static void
2365 stonith_send_reply(const xmlNode *reply, int call_options,
2366 const char *remote_peer, pcmk__client_t *client)
2367 {
2368 CRM_CHECK((reply != NULL) && ((remote_peer != NULL) || (client != NULL)),
2369 return);
2370
2371 if (remote_peer == NULL) {
2372 do_local_reply(reply, client, call_options);
2373 } else {
2374 const pcmk__node_status_t *node =
2375 pcmk__get_node(0, remote_peer, NULL,
2376 pcmk__node_search_cluster_member);
2377
2378 pcmk__cluster_send_message(node, pcmk_ipc_fenced, reply);
2379 }
2380 }
2381
2382 static void
2383 stonith_query_capable_device_cb(GList * devices, void *user_data)
2384 {
2385 struct st_query_data *query = user_data;
2386 int available_devices = 0;
2387 xmlNode *wrapper = NULL;
2388 xmlNode *list = NULL;
2389 GList *lpc = NULL;
2390 pcmk__client_t *client = NULL;
2391
2392 if (query->client_id != NULL) {
2393 client = pcmk__find_client_by_id(query->client_id);
2394 if ((client == NULL) && (query->remote_peer == NULL)) {
2395 crm_trace("Skipping reply to %s: no longer a client",
2396 query->client_id);
2397 goto done;
2398 }
2399 }
2400
2401
2402 wrapper = pcmk__xe_create(query->reply, PCMK__XE_ST_CALLDATA);
2403 list = pcmk__xe_create(wrapper, __func__);
2404 crm_xml_add(list, PCMK__XA_ST_TARGET, query->target);
2405
2406 for (lpc = devices; lpc != NULL; lpc = lpc->next) {
2407 fenced_device_t *device = g_hash_table_lookup(device_table, lpc->data);
2408 const char *action = query->action;
2409 xmlNode *dev = NULL;
2410
2411 if (!device) {
2412
2413
2414 continue;
2415 }
2416
2417 available_devices++;
2418
2419 dev = pcmk__xe_create(list, PCMK__XE_ST_DEVICE_ID);
2420 crm_xml_add(dev, PCMK_XA_ID, device->id);
2421 crm_xml_add(dev, PCMK__XA_NAMESPACE, device->namespace);
2422 crm_xml_add(dev, PCMK_XA_AGENT, device->agent);
2423
2424
2425 crm_xml_add_int(dev, PCMK__XA_ST_MONITOR_VERIFIED,
2426 pcmk_is_set(device->flags, fenced_df_verified));
2427
2428 crm_xml_add_int(dev, PCMK__XA_ST_DEVICE_SUPPORT_FLAGS, device->flags);
2429
2430
2431
2432
2433 if (!pcmk_is_set(device->flags, fenced_df_supports_reboot)
2434 && pcmk__str_eq(query->action, PCMK_ACTION_REBOOT,
2435 pcmk__str_none)) {
2436 crm_trace("%s doesn't support reboot, using values for off instead",
2437 device->id);
2438 action = PCMK_ACTION_OFF;
2439 }
2440
2441
2442 add_action_specific_attributes(dev, action, device, query->target);
2443 if (pcmk__str_eq(query->action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455 add_disallowed(dev, action, device, query->target,
2456 pcmk_is_set(query->call_options,
2457 st_opt_allow_self_fencing));
2458 add_action_reply(dev, PCMK_ACTION_OFF, device, query->target,
2459 pcmk_is_set(query->call_options,
2460 st_opt_allow_self_fencing));
2461 add_action_reply(dev, PCMK_ACTION_ON, device, query->target, FALSE);
2462 }
2463
2464
2465 if (query->target == NULL) {
2466 xmlNode *attrs = pcmk__xe_create(dev, PCMK__XE_ATTRIBUTES);
2467
2468 g_hash_table_foreach(device->params, hash2field, attrs);
2469 }
2470 }
2471
2472 crm_xml_add_int(list, PCMK__XA_ST_AVAILABLE_DEVICES, available_devices);
2473 if (query->target) {
2474 crm_debug("Found %d matching device%s for target '%s'",
2475 available_devices, pcmk__plural_s(available_devices),
2476 query->target);
2477 } else {
2478 crm_debug("%d device%s installed",
2479 available_devices, pcmk__plural_s(available_devices));
2480 }
2481
2482 crm_log_xml_trace(list, "query-result");
2483
2484 stonith_send_reply(query->reply, query->call_options, query->remote_peer,
2485 client);
2486
2487 done:
2488 pcmk__xml_free(query->reply);
2489 free(query->remote_peer);
2490 free(query->client_id);
2491 free(query->target);
2492 free(query->action);
2493 free(query);
2494 g_list_free_full(devices, free);
2495 }
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507 static void
2508 log_async_result(const async_command_t *cmd,
2509 const pcmk__action_result_t *result,
2510 int pid, const char *next, bool op_merged)
2511 {
2512 int log_level = LOG_ERR;
2513 int output_log_level = LOG_NEVER;
2514 guint devices_remaining = g_list_length(cmd->next_device_iter);
2515
2516 GString *msg = g_string_sized_new(80);
2517
2518
2519 if (pcmk__result_ok(result)) {
2520 log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
2521 if ((result->action_stdout != NULL)
2522 && !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
2523 pcmk__str_none)) {
2524 output_log_level = LOG_DEBUG;
2525 }
2526 next = NULL;
2527 } else {
2528 log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
2529 if ((result->action_stdout != NULL)
2530 && !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
2531 pcmk__str_none)) {
2532 output_log_level = LOG_WARNING;
2533 }
2534 }
2535
2536
2537 pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL);
2538 if (pid != 0) {
2539 g_string_append_printf(msg, "[%d] ", pid);
2540 }
2541 if (cmd->target != NULL) {
2542 pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL);
2543 }
2544 if (cmd->device != NULL) {
2545 pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL);
2546 }
2547
2548
2549 if (result->execution_status == PCMK_EXEC_DONE) {
2550 g_string_append_printf(msg, "returned %d", result->exit_status);
2551 } else {
2552 pcmk__g_strcat(msg, "could not be executed: ",
2553 pcmk_exec_status_str(result->execution_status), NULL);
2554 }
2555
2556
2557 if (result->exit_reason != NULL) {
2558 pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL);
2559 }
2560 if (next != NULL) {
2561 pcmk__g_strcat(msg, ", retrying with ", next, NULL);
2562 }
2563 if (devices_remaining > 0) {
2564 g_string_append_printf(msg, " (%u device%s remaining)",
2565 (unsigned int) devices_remaining,
2566 pcmk__plural_s(devices_remaining));
2567 }
2568 g_string_append_printf(msg, " " QB_XS " %scall %d from %s",
2569 (op_merged? "merged " : ""), cmd->id,
2570 cmd->client_name);
2571
2572
2573 do_crm_log(log_level, "%s", msg->str);
2574 g_string_free(msg, TRUE);
2575
2576
2577 if (output_log_level != LOG_NEVER) {
2578 char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);
2579
2580 crm_log_output(output_log_level, prefix, result->action_stdout);
2581 free(prefix);
2582 }
2583 }
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594 static void
2595 send_async_reply(const async_command_t *cmd, const pcmk__action_result_t *result,
2596 int pid, bool merged)
2597 {
2598 xmlNode *reply = NULL;
2599 pcmk__client_t *client = NULL;
2600
2601 CRM_CHECK((cmd != NULL) && (result != NULL), return);
2602
2603 log_async_result(cmd, result, pid, NULL, merged);
2604
2605 if (cmd->client != NULL) {
2606 client = pcmk__find_client_by_id(cmd->client);
2607 if ((client == NULL) && (cmd->origin == NULL)) {
2608 crm_trace("Skipping reply to %s: no longer a client", cmd->client);
2609 return;
2610 }
2611 }
2612
2613 reply = construct_async_reply(cmd, result);
2614 if (merged) {
2615 pcmk__xe_set_bool_attr(reply, PCMK__XA_ST_OP_MERGED, true);
2616 }
2617
2618 if (pcmk__is_fencing_action(cmd->action)
2619 && pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
2620
2621
2622
2623 crm_trace("Broadcast '%s' result for %s (target was also originator)",
2624 cmd->action, cmd->target);
2625 crm_xml_add(reply, PCMK__XA_SUBT, PCMK__VALUE_BROADCAST);
2626 crm_xml_add(reply, PCMK__XA_ST_OP, STONITH_OP_NOTIFY);
2627 pcmk__cluster_send_message(NULL, pcmk_ipc_fenced, reply);
2628 } else {
2629
2630 stonith_send_reply(reply, cmd->options, cmd->origin, client);
2631 }
2632
2633 crm_log_xml_trace(reply, "Reply");
2634 pcmk__xml_free(reply);
2635 }
2636
2637 static void
2638 cancel_stonith_command(async_command_t * cmd)
2639 {
2640 fenced_device_t *device = cmd_device(cmd);
2641
2642 if (device) {
2643 crm_trace("Cancel scheduled '%s' action using %s",
2644 cmd->action, device->id);
2645 device->pending_ops = g_list_remove(device->pending_ops, cmd);
2646 }
2647 }
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668 static void
2669 reply_to_duplicates(async_command_t *cmd, const pcmk__action_result_t *result,
2670 int pid)
2671 {
2672 GList *next = NULL;
2673
2674 for (GList *iter = cmd_list; iter != NULL; iter = next) {
2675 async_command_t *cmd_other = iter->data;
2676
2677 next = iter->next;
2678
2679 if (cmd == cmd_other) {
2680 continue;
2681 }
2682
2683
2684
2685
2686
2687
2688
2689 if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) ||
2690 !pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) ||
2691 !pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) ||
2692 !pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {
2693
2694 continue;
2695 }
2696
2697 crm_notice("Merging fencing action '%s'%s%s originating from "
2698 "client %s with identical fencing request from client %s",
2699 cmd_other->action,
2700 (cmd_other->target == NULL)? "" : " targeting ",
2701 pcmk__s(cmd_other->target, ""), cmd_other->client_name,
2702 cmd->client_name);
2703
2704
2705 cmd_list = g_list_remove_link(cmd_list, iter);
2706 send_async_reply(cmd_other, result, pid, true);
2707 cancel_stonith_command(cmd_other);
2708
2709 free_async_command(cmd_other);
2710 g_list_free_1(iter);
2711 }
2712 }
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722 static fenced_device_t *
2723 next_required_device(async_command_t *cmd)
2724 {
2725 for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
2726 fenced_device_t *next_device = g_hash_table_lookup(device_table,
2727 iter->data);
2728
2729 if (is_action_required(cmd->action, next_device)) {
2730
2731
2732
2733 cmd->next_device_iter = iter->next;
2734 return next_device;
2735 }
2736 }
2737 return NULL;
2738 }
2739
2740 static void
2741 st_child_done(int pid, const pcmk__action_result_t *result, void *user_data)
2742 {
2743 async_command_t *cmd = user_data;
2744
2745 fenced_device_t *device = NULL;
2746 fenced_device_t *next_device = NULL;
2747
2748 CRM_CHECK(cmd != NULL, return);
2749
2750 device = cmd_device(cmd);
2751 cmd->active_on = NULL;
2752
2753
2754 if (device) {
2755 if (!pcmk_is_set(device->flags, fenced_df_verified)
2756 && pcmk__result_ok(result)
2757 && pcmk__strcase_any_of(cmd->action, PCMK_ACTION_LIST,
2758 PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS,
2759 NULL)) {
2760
2761 fenced_device_set_flags(device, fenced_df_verified);
2762 }
2763
2764 mainloop_set_trigger(device->work);
2765 }
2766
2767 if (pcmk__result_ok(result)) {
2768 next_device = next_required_device(cmd);
2769
2770 } else if ((cmd->next_device_iter != NULL)
2771 && !is_action_required(cmd->action, device)) {
2772
2773
2774 next_device = g_hash_table_lookup(device_table,
2775 cmd->next_device_iter->data);
2776 cmd->next_device_iter = cmd->next_device_iter->next;
2777 }
2778
2779 if (next_device == NULL) {
2780 send_async_reply(cmd, result, pid, false);
2781 if (pcmk__result_ok(result)) {
2782 reply_to_duplicates(cmd, result, pid);
2783 }
2784 free_async_command(cmd);
2785
2786 } else {
2787 log_async_result(cmd, result, pid, next_device->id, false);
2788 schedule_stonith_command(cmd, next_device);
2789 }
2790 }
2791
2792 static void
2793 stonith_fence_get_devices_cb(GList * devices, void *user_data)
2794 {
2795 async_command_t *cmd = user_data;
2796 fenced_device_t *device = NULL;
2797 guint ndevices = g_list_length(devices);
2798
2799 crm_info("Found %d matching device%s for target '%s'",
2800 ndevices, pcmk__plural_s(ndevices), cmd->target);
2801
2802 if (devices != NULL) {
2803 device = g_hash_table_lookup(device_table, devices->data);
2804 }
2805
2806 if (device == NULL) {
2807 pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2808
2809 pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
2810 "No device configured for target '%s'",
2811 cmd->target);
2812 send_async_reply(cmd, &result, 0, false);
2813 pcmk__reset_result(&result);
2814 free_async_command(cmd);
2815 g_list_free_full(devices, free);
2816
2817 } else {
2818
2819
2820
2821
2822 cmd->device_list = devices;
2823 cmd->next_device_iter = devices->next;
2824 schedule_stonith_command(cmd, device);
2825 }
2826 }
2827
2828
2829
2830
2831
2832
2833
2834
2835 static void
2836 fence_locally(xmlNode *msg, pcmk__action_result_t *result)
2837 {
2838 const char *device_id = NULL;
2839 fenced_device_t *device = NULL;
2840 async_command_t *cmd = NULL;
2841 xmlNode *dev = NULL;
2842
2843 CRM_CHECK((msg != NULL) && (result != NULL), return);
2844
2845 dev = pcmk__xpath_find_one(msg->doc, "//*[@" PCMK__XA_ST_TARGET "]",
2846 LOG_ERR);
2847
2848 cmd = create_async_command(msg);
2849 if (cmd == NULL) {
2850 crm_log_xml_warn(msg, "invalid");
2851 set_bad_request_result(result);
2852 return;
2853 }
2854
2855 device_id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
2856 if (device_id != NULL) {
2857 device = g_hash_table_lookup(device_table, device_id);
2858 if (device == NULL) {
2859 crm_err("Requested device '%s' is not available", device_id);
2860 pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
2861 "Requested device '%s' not found", device_id);
2862 return;
2863 }
2864 schedule_stonith_command(cmd, device);
2865
2866 } else {
2867 const char *host = crm_element_value(dev, PCMK__XA_ST_TARGET);
2868
2869 if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
2870 int nodeid = 0;
2871 pcmk__node_status_t *node = NULL;
2872
2873 pcmk__scan_min_int(host, &nodeid, 0);
2874 node = pcmk__search_node_caches(nodeid, NULL, NULL,
2875 pcmk__node_search_any
2876 |pcmk__node_search_cluster_cib);
2877 if (node != NULL) {
2878 host = node->name;
2879 }
2880 }
2881
2882
2883 get_capable_devices(host, cmd->action, cmd->default_timeout,
2884 TRUE, cmd, stonith_fence_get_devices_cb,
2885 fenced_support_flag(cmd->action));
2886 }
2887
2888 pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
2889 }
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905 xmlNode *
2906 fenced_construct_reply(const xmlNode *request, xmlNode *data,
2907 const pcmk__action_result_t *result)
2908 {
2909 xmlNode *reply = NULL;
2910
2911 reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY);
2912
2913 crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
2914 crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
2915 stonith__xe_set_result(reply, result);
2916
2917 if (request == NULL) {
2918
2919
2920
2921
2922
2923
2924 crm_warn("Missing request information for client notifications for "
2925 "operation with result '%s' (initiated before we came up?)",
2926 pcmk_exec_status_str(result->execution_status));
2927
2928 } else {
2929 const char *name = NULL;
2930 const char *value = NULL;
2931
2932
2933 const char *names[] = {
2934 PCMK__XA_ST_OP,
2935 PCMK__XA_ST_CALLID,
2936 PCMK__XA_ST_CLIENTID,
2937 PCMK__XA_ST_CLIENTNAME,
2938 PCMK__XA_ST_REMOTE_OP,
2939 PCMK__XA_ST_CALLOPT,
2940 };
2941
2942 for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
2943 name = names[lpc];
2944 value = crm_element_value(request, name);
2945 crm_xml_add(reply, name, value);
2946 }
2947 if (data != NULL) {
2948 xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_ST_CALLDATA);
2949
2950 pcmk__xml_copy(wrapper, data);
2951 }
2952 }
2953 return reply;
2954 }
2955
2956
2957
2958
2959
2960
2961
2962
2963 static xmlNode *
2964 construct_async_reply(const async_command_t *cmd,
2965 const pcmk__action_result_t *result)
2966 {
2967 xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY);
2968
2969 crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
2970 crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
2971 crm_xml_add(reply, PCMK__XA_ST_OP, cmd->op);
2972 crm_xml_add(reply, PCMK__XA_ST_DEVICE_ID, cmd->device);
2973 crm_xml_add(reply, PCMK__XA_ST_REMOTE_OP, cmd->remote_op_id);
2974 crm_xml_add(reply, PCMK__XA_ST_CLIENTID, cmd->client);
2975 crm_xml_add(reply, PCMK__XA_ST_CLIENTNAME, cmd->client_name);
2976 crm_xml_add(reply, PCMK__XA_ST_TARGET, cmd->target);
2977 crm_xml_add(reply, PCMK__XA_ST_DEVICE_ACTION, cmd->op);
2978 crm_xml_add(reply, PCMK__XA_ST_ORIGIN, cmd->origin);
2979 crm_xml_add_int(reply, PCMK__XA_ST_CALLID, cmd->id);
2980 crm_xml_add_int(reply, PCMK__XA_ST_CALLOPT, cmd->options);
2981
2982 stonith__xe_set_result(reply, result);
2983 return reply;
2984 }
2985
2986 bool
2987 fencing_peer_active(pcmk__node_status_t *peer)
2988 {
2989 return (peer != NULL) && (peer->name != NULL)
2990 && pcmk_is_set(peer->processes, crm_get_cluster_proc());
2991 }
2992
2993 void
2994 set_fencing_completed(remote_fencing_op_t *op)
2995 {
2996 struct timespec tv;
2997
2998 qb_util_timespec_from_epoch_get(&tv);
2999 op->completed = tv.tv_sec;
3000 op->completed_nsec = tv.tv_nsec;
3001 }
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012 static const char *
3013 check_alternate_host(const char *target)
3014 {
3015 if (pcmk__str_eq(target, fenced_get_local_node(), pcmk__str_casei)) {
3016 GHashTableIter gIter;
3017 pcmk__node_status_t *entry = NULL;
3018
3019 g_hash_table_iter_init(&gIter, pcmk__peer_cache);
3020 while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
3021 if (fencing_peer_active(entry)
3022 && !pcmk__str_eq(entry->name, target, pcmk__str_casei)) {
3023 crm_notice("Forwarding self-fencing request to %s",
3024 entry->name);
3025 return entry->name;
3026 }
3027 }
3028 crm_warn("Will handle own fencing because no peer can");
3029 }
3030 return NULL;
3031 }
3032
3033 static void
3034 remove_relay_op(xmlNode * request)
3035 {
3036 xmlNode *dev = pcmk__xpath_find_one(request->doc,
3037 "//*[@" PCMK__XA_ST_DEVICE_ACTION "]",
3038 LOG_TRACE);
3039 const char *relay_op_id = NULL;
3040 const char *op_id = NULL;
3041 const char *client_name = NULL;
3042 const char *target = NULL;
3043 remote_fencing_op_t *relay_op = NULL;
3044
3045 if (dev) {
3046 target = crm_element_value(dev, PCMK__XA_ST_TARGET);
3047 }
3048
3049 relay_op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP_RELAY);
3050 op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP);
3051 client_name = crm_element_value(request, PCMK__XA_ST_CLIENTNAME);
3052
3053
3054 if ((relay_op_id != NULL) && (target != NULL)
3055 && pcmk__str_eq(target, fenced_get_local_node(), pcmk__str_casei)) {
3056 relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);
3057
3058 if (relay_op) {
3059 GHashTableIter iter;
3060 remote_fencing_op_t *list_op = NULL;
3061 g_hash_table_iter_init(&iter, stonith_remote_op_list);
3062
3063
3064 while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
3065 GList *dup_iter = NULL;
3066 if (list_op != relay_op) {
3067 for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
3068 remote_fencing_op_t *other = dup_iter->data;
3069 if (other == relay_op) {
3070 other->duplicates = g_list_remove(other->duplicates, relay_op);
3071 break;
3072 }
3073 }
3074 }
3075 }
3076 crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
3077 "replaced by op %s ('%s'%s%s for %s)",
3078 relay_op->id, relay_op->action,
3079 (relay_op->target == NULL)? "" : " targeting ",
3080 pcmk__s(relay_op->target, ""),
3081 relay_op->client_name, op_id, relay_op->action,
3082 (target == NULL)? "" : " targeting ", pcmk__s(target, ""),
3083 client_name);
3084
3085 g_hash_table_remove(stonith_remote_op_list, relay_op_id);
3086 }
3087 }
3088 }
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104 static inline bool
3105 is_privileged(const pcmk__client_t *c, const char *op)
3106 {
3107 if ((c == NULL) || pcmk_is_set(c->flags, pcmk__client_privileged)) {
3108 return true;
3109 } else {
3110 crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
3111 pcmk__s(op, ""), pcmk__client_name(c));
3112 return false;
3113 }
3114 }
3115
3116
3117 static xmlNode *
3118 handle_register_request(pcmk__request_t *request)
3119 {
3120 xmlNode *reply = pcmk__xe_create(NULL, "reply");
3121
3122 pcmk__assert(request->ipc_client != NULL);
3123 crm_xml_add(reply, PCMK__XA_ST_OP, CRM_OP_REGISTER);
3124 crm_xml_add(reply, PCMK__XA_ST_CLIENTID, request->ipc_client->id);
3125 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3126 pcmk__set_request_flags(request, pcmk__request_reuse_options);
3127 return reply;
3128 }
3129
3130
3131 static xmlNode *
3132 handle_agent_request(pcmk__request_t *request)
3133 {
3134 execute_agent_action(request->xml, &request->result);
3135 if (request->result.execution_status == PCMK_EXEC_PENDING) {
3136 return NULL;
3137 }
3138 return fenced_construct_reply(request->xml, NULL, &request->result);
3139 }
3140
3141
3142 static xmlNode *
3143 handle_update_timeout_request(pcmk__request_t *request)
3144 {
3145 const char *call_id = crm_element_value(request->xml, PCMK__XA_ST_CALLID);
3146 const char *client_id = crm_element_value(request->xml,
3147 PCMK__XA_ST_CLIENTID);
3148 int op_timeout = 0;
3149
3150 crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &op_timeout);
3151 do_stonith_async_timeout_update(client_id, call_id, op_timeout);
3152 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3153 return NULL;
3154 }
3155
3156
3157 static xmlNode *
3158 handle_query_request(pcmk__request_t *request)
3159 {
3160 int timeout = 0;
3161 xmlNode *dev = NULL;
3162 const char *action = NULL;
3163 const char *target = NULL;
3164 const char *client_id = crm_element_value(request->xml,
3165 PCMK__XA_ST_CLIENTID);
3166 struct st_query_data *query = NULL;
3167
3168 if (request->peer != NULL) {
3169
3170 create_remote_stonith_op(client_id, request->xml, TRUE);
3171 }
3172
3173
3174 remove_relay_op(request->xml);
3175
3176 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3177
3178 dev = pcmk__xpath_find_one(request->xml->doc,
3179 "//*[@" PCMK__XA_ST_DEVICE_ACTION "]",
3180 LOG_NEVER);
3181 if (dev != NULL) {
3182 const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
3183
3184 if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
3185 return NULL;
3186 }
3187 target = crm_element_value(dev, PCMK__XA_ST_TARGET);
3188 action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
3189 }
3190
3191 crm_log_xml_trace(request->xml, "Query");
3192
3193 query = pcmk__assert_alloc(1, sizeof(struct st_query_data));
3194
3195 query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
3196 query->remote_peer = pcmk__str_copy(request->peer);
3197 query->client_id = pcmk__str_copy(client_id);
3198 query->target = pcmk__str_copy(target);
3199 query->action = pcmk__str_copy(action);
3200 query->call_options = request->call_options;
3201
3202 crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &timeout);
3203 get_capable_devices(target, action, timeout,
3204 pcmk_is_set(query->call_options,
3205 st_opt_allow_self_fencing),
3206 query, stonith_query_capable_device_cb, fenced_df_none);
3207 return NULL;
3208 }
3209
3210
3211 static xmlNode *
3212 handle_notify_request(pcmk__request_t *request)
3213 {
3214 const char *flag_name = NULL;
3215
3216 pcmk__assert(request->ipc_client != NULL);
3217 flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_ACTIVATE);
3218 if (flag_name != NULL) {
3219 crm_debug("Enabling %s callbacks for client %s",
3220 flag_name, pcmk__request_origin(request));
3221 pcmk__set_client_flags(request->ipc_client,
3222 fenced_parse_notify_flag(flag_name));
3223 }
3224
3225 flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_DEACTIVATE);
3226 if (flag_name != NULL) {
3227 crm_debug("Disabling %s callbacks for client %s",
3228 flag_name, pcmk__request_origin(request));
3229 pcmk__clear_client_flags(request->ipc_client,
3230 fenced_parse_notify_flag(flag_name));
3231 }
3232
3233 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3234 pcmk__set_request_flags(request, pcmk__request_reuse_options);
3235
3236 return pcmk__ipc_create_ack(request->ipc_flags, PCMK__XE_ACK, NULL,
3237 CRM_EX_OK);
3238 }
3239
3240
3241 static xmlNode *
3242 handle_relay_request(pcmk__request_t *request)
3243 {
3244 xmlNode *dev = pcmk__xpath_find_one(request->xml->doc,
3245 "//*[@" PCMK__XA_ST_TARGET "]",
3246 LOG_TRACE);
3247
3248 crm_notice("Received forwarded fencing request from "
3249 "%s %s to fence (%s) peer %s",
3250 pcmk__request_origin_type(request),
3251 pcmk__request_origin(request),
3252 crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION),
3253 crm_element_value(dev, PCMK__XA_ST_TARGET));
3254
3255 if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
3256 set_bad_request_result(&request->result);
3257 return fenced_construct_reply(request->xml, NULL, &request->result);
3258 }
3259
3260 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
3261 return NULL;
3262 }
3263
3264
3265 static xmlNode *
3266 handle_fence_request(pcmk__request_t *request)
3267 {
3268 if (request->peer != NULL) {
3269 fence_locally(request->xml, &request->result);
3270
3271 } else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
3272 switch (fenced_handle_manual_confirmation(request->ipc_client,
3273 request->xml)) {
3274 case pcmk_rc_ok:
3275 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
3276 NULL);
3277 break;
3278 case EINPROGRESS:
3279 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3280 NULL);
3281 break;
3282 default:
3283 set_bad_request_result(&request->result);
3284 break;
3285 }
3286
3287 } else {
3288 const char *alternate_host = NULL;
3289 xmlNode *dev = pcmk__xpath_find_one(request->xml->doc,
3290 "//*[@" PCMK__XA_ST_TARGET "]",
3291 LOG_TRACE);
3292 const char *target = crm_element_value(dev, PCMK__XA_ST_TARGET);
3293 const char *action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
3294 const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
3295
3296 if (request->ipc_client != NULL) {
3297 int tolerance = 0;
3298
3299 crm_notice("Client %s wants to fence (%s) %s using %s",
3300 pcmk__request_origin(request), action,
3301 target, (device? device : "any device"));
3302 crm_element_value_int(dev, PCMK__XA_ST_TOLERANCE, &tolerance);
3303 if (stonith_check_fence_tolerance(tolerance, target, action)) {
3304 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
3305 NULL);
3306 return fenced_construct_reply(request->xml, NULL,
3307 &request->result);
3308 }
3309 alternate_host = check_alternate_host(target);
3310
3311 } else {
3312 crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
3313 request->peer, action, target,
3314 (device == NULL)? "(any)" : device);
3315 }
3316
3317 if (alternate_host != NULL) {
3318 const char *client_id = NULL;
3319 remote_fencing_op_t *op = NULL;
3320 pcmk__node_status_t *node =
3321 pcmk__get_node(0, alternate_host, NULL,
3322 pcmk__node_search_cluster_member);
3323
3324 if (request->ipc_client->id == 0) {
3325 client_id = crm_element_value(request->xml,
3326 PCMK__XA_ST_CLIENTID);
3327 } else {
3328 client_id = request->ipc_client->id;
3329 }
3330
3331
3332
3333
3334
3335 op = create_remote_stonith_op(client_id, request->xml, FALSE);
3336
3337 crm_xml_add(request->xml, PCMK__XA_ST_OP, STONITH_OP_RELAY);
3338 crm_xml_add(request->xml, PCMK__XA_ST_CLIENTID,
3339 request->ipc_client->id);
3340 crm_xml_add(request->xml, PCMK__XA_ST_REMOTE_OP, op->id);
3341
3342
3343 pcmk__cluster_send_message(node, pcmk_ipc_fenced, request->xml);
3344
3345 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3346 NULL);
3347
3348 } else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
3349 FALSE) == NULL) {
3350 set_bad_request_result(&request->result);
3351
3352 } else {
3353 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
3354 NULL);
3355 }
3356 }
3357
3358 if (request->result.execution_status == PCMK_EXEC_PENDING) {
3359 return NULL;
3360 }
3361 return fenced_construct_reply(request->xml, NULL, &request->result);
3362 }
3363
3364
3365 static xmlNode *
3366 handle_history_request(pcmk__request_t *request)
3367 {
3368 xmlNode *reply = NULL;
3369 xmlNode *data = NULL;
3370
3371 stonith_fence_history(request->xml, &data, request->peer,
3372 request->call_options);
3373 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3374 if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
3375
3376
3377
3378 reply = fenced_construct_reply(request->xml, data, &request->result);
3379 }
3380 pcmk__xml_free(data);
3381 return reply;
3382 }
3383
3384
3385 static xmlNode *
3386 handle_device_add_request(pcmk__request_t *request)
3387 {
3388 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3389 xmlNode *dev = pcmk__xpath_find_one(request->xml->doc,
3390 "//" PCMK__XE_ST_DEVICE_ID, LOG_ERR);
3391
3392 if (is_privileged(request->ipc_client, op)) {
3393 int rc = fenced_device_register(dev, false);
3394
3395 rc = pcmk_rc2legacy(rc);
3396 pcmk__set_result(&request->result,
3397 ((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
3398 stonith__legacy2status(rc),
3399 ((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
3400 } else {
3401 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3402 PCMK_EXEC_INVALID,
3403 "Unprivileged users must register device via CIB");
3404 }
3405 fenced_send_config_notification(op, &request->result,
3406 (dev == NULL)? NULL : pcmk__xe_id(dev));
3407 return fenced_construct_reply(request->xml, NULL, &request->result);
3408 }
3409
3410
3411 static xmlNode *
3412 handle_device_delete_request(pcmk__request_t *request)
3413 {
3414 xmlNode *dev = pcmk__xpath_find_one(request->xml->doc,
3415 "//" PCMK__XE_ST_DEVICE_ID, LOG_ERR);
3416 const char *device_id = crm_element_value(dev, PCMK_XA_ID);
3417 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3418
3419 if (is_privileged(request->ipc_client, op)) {
3420 stonith_device_remove(device_id, false);
3421 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3422 } else {
3423 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3424 PCMK_EXEC_INVALID,
3425 "Unprivileged users must delete device via CIB");
3426 }
3427 fenced_send_config_notification(op, &request->result, device_id);
3428 return fenced_construct_reply(request->xml, NULL, &request->result);
3429 }
3430
3431
3432 static xmlNode *
3433 handle_level_add_request(pcmk__request_t *request)
3434 {
3435 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3436
3437 if (is_privileged(request->ipc_client, op)) {
3438 fenced_register_level(request->xml, &request->result);
3439 } else {
3440 unpack_level_request(request->xml, NULL, NULL, NULL);
3441 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3442 PCMK_EXEC_INVALID,
3443 "Unprivileged users must add level via CIB");
3444 }
3445 return fenced_construct_reply(request->xml, NULL, &request->result);
3446 }
3447
3448
3449 static xmlNode *
3450 handle_level_delete_request(pcmk__request_t *request)
3451 {
3452 const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
3453
3454 if (is_privileged(request->ipc_client, op)) {
3455 fenced_unregister_level(request->xml, &request->result);
3456 } else {
3457 unpack_level_request(request->xml, NULL, NULL, NULL);
3458 pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
3459 PCMK_EXEC_INVALID,
3460 "Unprivileged users must delete level via CIB");
3461 }
3462 return fenced_construct_reply(request->xml, NULL, &request->result);
3463 }
3464
3465
3466 static xmlNode *
3467 handle_cache_request(pcmk__request_t *request)
3468 {
3469 int node_id = 0;
3470 const char *name = NULL;
3471
3472 crm_element_value_int(request->xml, PCMK_XA_ID, &node_id);
3473 name = crm_element_value(request->xml, PCMK_XA_UNAME);
3474 pcmk__cluster_forget_cluster_node(node_id, name);
3475 pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
3476 return NULL;
3477 }
3478
3479 static xmlNode *
3480 handle_unknown_request(pcmk__request_t *request)
3481 {
3482 crm_err("Unknown IPC request %s from %s %s",
3483 request->op, pcmk__request_origin_type(request),
3484 pcmk__request_origin(request));
3485 pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
3486 "Unknown IPC request type '%s' (bug?)", request->op);
3487 return fenced_construct_reply(request->xml, NULL, &request->result);
3488 }
3489
3490 static void
3491 fenced_register_handlers(void)
3492 {
3493 pcmk__server_command_t handlers[] = {
3494 { CRM_OP_REGISTER, handle_register_request },
3495 { STONITH_OP_EXEC, handle_agent_request },
3496 { STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
3497 { STONITH_OP_QUERY, handle_query_request },
3498 { STONITH_OP_NOTIFY, handle_notify_request },
3499 { STONITH_OP_RELAY, handle_relay_request },
3500 { STONITH_OP_FENCE, handle_fence_request },
3501 { STONITH_OP_FENCE_HISTORY, handle_history_request },
3502 { STONITH_OP_DEVICE_ADD, handle_device_add_request },
3503 { STONITH_OP_DEVICE_DEL, handle_device_delete_request },
3504 { STONITH_OP_LEVEL_ADD, handle_level_add_request },
3505 { STONITH_OP_LEVEL_DEL, handle_level_delete_request },
3506 { CRM_OP_RM_NODE_CACHE, handle_cache_request },
3507 { NULL, handle_unknown_request },
3508 };
3509
3510 fenced_handlers = pcmk__register_handlers(handlers);
3511 }
3512
3513 void
3514 fenced_unregister_handlers(void)
3515 {
3516 if (fenced_handlers != NULL) {
3517 g_hash_table_destroy(fenced_handlers);
3518 fenced_handlers = NULL;
3519 }
3520 }
3521
3522 static void
3523 handle_request(pcmk__request_t *request)
3524 {
3525 xmlNode *reply = NULL;
3526 const char *reason = NULL;
3527
3528 if (fenced_handlers == NULL) {
3529 fenced_register_handlers();
3530 }
3531 reply = pcmk__process_request(request, fenced_handlers);
3532 if (reply != NULL) {
3533 if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
3534 && (request->ipc_client != NULL)) {
3535
3536
3537
3538
3539 pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
3540 request->ipc_flags);
3541 request->ipc_client->request_id = 0;
3542 } else {
3543 stonith_send_reply(reply, request->call_options,
3544 request->peer, request->ipc_client);
3545 }
3546 pcmk__xml_free(reply);
3547 }
3548
3549 reason = request->result.exit_reason;
3550 crm_debug("Processed %s request from %s %s: %s%s%s%s",
3551 request->op, pcmk__request_origin_type(request),
3552 pcmk__request_origin(request),
3553 pcmk_exec_status_str(request->result.execution_status),
3554 (reason == NULL)? "" : " (",
3555 (reason == NULL)? "" : reason,
3556 (reason == NULL)? "" : ")");
3557 }
3558
3559 static void
3560 handle_reply(pcmk__client_t *client, xmlNode *request, const char *remote_peer)
3561 {
3562
3563 char *op = crm_element_value_copy(request, PCMK__XA_ST_OP);
3564
3565 if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
3566 process_remote_stonith_query(request);
3567
3568 } else if (pcmk__str_any_of(op, STONITH_OP_NOTIFY, STONITH_OP_FENCE,
3569 NULL)) {
3570 fenced_process_fencing_reply(request);
3571
3572 } else {
3573 crm_err("Ignoring unknown %s reply from %s %s",
3574 pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
3575 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3576 crm_log_xml_warn(request, "UnknownOp");
3577 free(op);
3578 return;
3579 }
3580 crm_debug("Processed %s reply from %s %s",
3581 op, ((client == NULL)? "peer" : "client"),
3582 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3583 free(op);
3584 }
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596 void
3597 stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
3598 xmlNode *message, const char *remote_peer)
3599 {
3600 uint32_t call_options = st_opt_none;
3601 int rc = pcmk_rc_ok;
3602 bool is_reply = false;
3603
3604 CRM_CHECK(message != NULL, return);
3605
3606 if (pcmk__xpath_find_one(message->doc, "//" PCMK__XE_ST_REPLY,
3607 LOG_NEVER) != NULL) {
3608 is_reply = true;
3609 }
3610
3611 rc = pcmk__xe_get_flags(message, PCMK__XA_ST_CALLOPT, &call_options,
3612 st_opt_none);
3613 if (rc != pcmk_rc_ok) {
3614 crm_warn("Couldn't parse options from message: %s", pcmk_rc_str(rc));
3615 }
3616
3617 crm_debug("Processing %ssynchronous %s %s %u from %s %s",
3618 pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
3619 crm_element_value(message, PCMK__XA_ST_OP),
3620 (is_reply? "reply" : "request"), id,
3621 ((client == NULL)? "peer" : "client"),
3622 ((client == NULL)? remote_peer : pcmk__client_name(client)));
3623
3624 if (pcmk_is_set(call_options, st_opt_sync_call)) {
3625 pcmk__assert((client == NULL) || (client->request_id == id));
3626 }
3627
3628 if (is_reply) {
3629 handle_reply(client, message, remote_peer);
3630 } else {
3631 pcmk__request_t request = {
3632 .ipc_client = client,
3633 .ipc_id = id,
3634 .ipc_flags = flags,
3635 .peer = remote_peer,
3636 .xml = message,
3637 .call_options = call_options,
3638 .result = PCMK__UNKNOWN_RESULT,
3639 };
3640
3641 request.op = crm_element_value_copy(request.xml, PCMK__XA_ST_OP);
3642 CRM_CHECK(request.op != NULL, return);
3643
3644 if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
3645 pcmk__set_request_flags(&request, pcmk__request_sync);
3646 }
3647
3648 handle_request(&request);
3649 pcmk__reset_request(&request);
3650 }
3651 }
![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
![[previous]](../icons/left.png)
![[last]](../icons/n_last.png)
![[bottom]](../icons/n_bottom.png){kind=icon}