![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
1 /*
2 * Copyright 2004-2022 the Pacemaker project contributors
3 *
4 * The version control history for this file may have further details.
5 *
6 * This source code is licensed under the GNU General Public License version 2
7 * or later (GPLv2+) WITHOUT ANY WARRANTY.
8 */
9
10 #include <crm_internal.h>
11
12 #include <glib.h>
13
14 #include <crm/crm.h>
15 #include <crm/pengine/status.h>
16 #include <pacemaker-internal.h>
17 #include "libpacemaker_private.h"
18
19 /*!
20 * \internal
21 * \brief Check whether a resource is known on a particular node
22 *
23 * \param[in] rsc Resource to check
24 * \param[in] node Node to check
25 *
26 * \return TRUE if resource (or parent if an anonymous clone) is known
27 */
28 static bool
29 rsc_is_known_on(pe_resource_t *rsc, const pe_node_t *node)
/* ![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
*/
30 {
31 if (pe_hash_table_lookup(rsc->known_on, node->details->id)) {
32 return TRUE;
33
34 } else if ((rsc->variant == pe_native)
35 && pe_rsc_is_anon_clone(rsc->parent)
36 && pe_hash_table_lookup(rsc->parent->known_on, node->details->id)) {
37 /* We check only the parent, not the uber-parent, because we cannot
38 * assume that the resource is known if it is in an anonymously cloned
39 * group (which may be only partially known).
40 */
41 return TRUE;
42 }
43 return FALSE;
44 }
45
46 /*!
47 * \internal
48 * \brief Order a resource's start and promote actions relative to fencing
49 *
50 * \param[in] rsc Resource to be ordered
51 * \param[in] stonith_op Fence action
52 * \param[in] data_set Cluster working set
53 */
54 static void
55 order_start_vs_fencing(pe_resource_t *rsc, pe_action_t *stonith_op,
/* ![[previous]](../icons/left.png)
*/
56 pe_working_set_t *data_set)
57 {
58 pe_node_t *target;
59 GList *gIter = NULL;
60
61 CRM_CHECK(stonith_op && stonith_op->node, return);
62 target = stonith_op->node;
63
64 for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
65 pe_action_t *action = (pe_action_t *) gIter->data;
66
67 switch (action->needs) {
68 case rsc_req_nothing:
69 // Anything other than start or promote requires nothing
70 break;
71
72 case rsc_req_stonith:
73 order_actions(stonith_op, action, pe_order_optional);
74 break;
75
76 case rsc_req_quorum:
77 if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
78 && pe_hash_table_lookup(rsc->allowed_nodes, target->details->id)
79 && !rsc_is_known_on(rsc, target)) {
80
81 /* If we don't know the status of the resource on the node
82 * we're about to shoot, we have to assume it may be active
83 * there. Order the resource start after the fencing. This
84 * is analogous to waiting for all the probes for a resource
85 * to complete before starting it.
86 *
87 * The most likely explanation is that the DC died and took
88 * its status with it.
89 */
90 pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
91 pe__node_name(target));
92 order_actions(stonith_op, action,
93 pe_order_optional | pe_order_runnable_left);
94 }
95 break;
96 }
97 }
98 }
99
100 /*!
101 * \internal
102 * \brief Order a resource's stop and demote actions relative to fencing
103 *
104 * \param[in] rsc Resource to be ordered
105 * \param[in] stonith_op Fence action
106 * \param[in] data_set Cluster working set
107 */
108 static void
109 order_stop_vs_fencing(pe_resource_t *rsc, pe_action_t *stonith_op,
/* */
110 pe_working_set_t *data_set)
111 {
112 GList *gIter = NULL;
113 GList *action_list = NULL;
114 bool order_implicit = false;
115
116 pe_resource_t *top = uber_parent(rsc);
117 pe_action_t *parent_stop = NULL;
118 pe_node_t *target;
119
120 CRM_CHECK(stonith_op && stonith_op->node, return);
121 target = stonith_op->node;
122
123 /* Get a list of stop actions potentially implied by the fencing */
124 action_list = pe__resource_actions(rsc, target, RSC_STOP, FALSE);
125
126 /* If resource requires fencing, implicit actions must occur after fencing.
127 *
128 * Implied stops and demotes of resources running on guest nodes are always
129 * ordered after fencing, even if the resource does not require fencing,
130 * because guest node "fencing" is actually just a resource stop.
131 */
132 if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)
133 || pe__is_guest_node(target)) {
134
135 order_implicit = true;
136 }
137
138 if (action_list && order_implicit) {
139 parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
140 }
141
142 for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
143 pe_action_t *action = (pe_action_t *) gIter->data;
144
145 // The stop would never complete, so convert it into a pseudo-action.
146 pe__set_action_flags(action, pe_action_pseudo|pe_action_runnable);
147
148 if (order_implicit) {
149 pe__set_action_flags(action, pe_action_implied_by_stonith);
150
151 /* Order the stonith before the parent stop (if any).
152 *
153 * Also order the stonith before the resource stop, unless the
154 * resource is inside a bundle -- that would cause a graph loop.
155 * We can rely on the parent stop's ordering instead.
156 *
157 * User constraints must not order a resource in a guest node
158 * relative to the guest node container resource. The
159 * pe_order_preserve flag marks constraints as generated by the
160 * cluster and thus immune to that check (and is irrelevant if
161 * target is not a guest).
162 */
163 if (!pe_rsc_is_bundled(rsc)) {
164 order_actions(stonith_op, action, pe_order_preserve);
165 }
166 order_actions(stonith_op, parent_stop, pe_order_preserve);
167 }
168
169 if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
170 crm_notice("Stop of failed resource %s is implicit %s %s is fenced",
171 rsc->id, (order_implicit? "after" : "because"),
172 pe__node_name(target));
173 } else {
174 crm_info("%s is implicit %s %s is fenced",
175 action->uuid, (order_implicit? "after" : "because"),
176 pe__node_name(target));
177 }
178
179 if (pcmk_is_set(rsc->flags, pe_rsc_notify)) {
180 pe__order_notifs_after_fencing(action, rsc, stonith_op);
181 }
182
183 #if 0
184 /* It might be a good idea to stop healthy resources on a node about to
185 * be fenced, when possible.
186 *
187 * However, fencing must be done before a failed resource's
188 * (pseudo-)stop action, so that could create a loop. For example, given
189 * a group of A and B running on node N with a failed stop of B:
190 *
191 * fence N -> stop B (pseudo-op) -> stop A -> fence N
192 *
193 * The block below creates the stop A -> fence N ordering and therefore
194 * must (at least for now) be disabled. Instead, run the block above and
195 * treat all resources on N as B would be (i.e., as a pseudo-op after
196 * the fencing).
197 *
198 * @TODO Maybe break the "A requires B" dependency in
199 * pcmk__update_action_for_orderings() and use this block for healthy
200 * resources instead of the above.
201 */
202 crm_info("Moving healthy resource %s off %s before fencing",
203 rsc->id, pe__node_name(node));
204 pcmk__new_ordering(rsc, stop_key(rsc), NULL, NULL,
205 strdup(CRM_OP_FENCE), stonith_op,
206 pe_order_optional, data_set);
207 #endif
208 }
209
210 g_list_free(action_list);
211
212 /* Get a list of demote actions potentially implied by the fencing */
213 action_list = pe__resource_actions(rsc, target, RSC_DEMOTE, FALSE);
214
215 for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
216 pe_action_t *action = (pe_action_t *) gIter->data;
217
218 if (!(action->node->details->online) || action->node->details->unclean
219 || pcmk_is_set(rsc->flags, pe_rsc_failed)) {
220
221 if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
222 pe_rsc_info(rsc,
223 "Demote of failed resource %s is implicit after %s is fenced",
224 rsc->id, pe__node_name(target));
225 } else {
226 pe_rsc_info(rsc, "%s is implicit after %s is fenced",
227 action->uuid, pe__node_name(target));
228 }
229
230 /* The demote would never complete and is now implied by the
231 * fencing, so convert it into a pseudo-action.
232 */
233 pe__set_action_flags(action, pe_action_pseudo|pe_action_runnable);
234
235 if (pe_rsc_is_bundled(rsc)) {
236 // Do nothing, let recovery be ordered after parent's implied stop
237
238 } else if (order_implicit) {
239 order_actions(stonith_op, action, pe_order_preserve|pe_order_optional);
240 }
241 }
242 }
243
244 g_list_free(action_list);
245 }
246
247 /*!
248 * \internal
249 * \brief Order resource actions properly relative to fencing
250 *
251 * \param[in] rsc Resource whose actions should be ordered
252 * \param[in] stonith_op Fencing operation to be ordered against
253 * \param[in] data_set Cluster working set
254 */
255 static void
256 rsc_stonith_ordering(pe_resource_t *rsc, pe_action_t *stonith_op,
/* */
257 pe_working_set_t *data_set)
258 {
259 if (rsc->children) {
260 GList *gIter = NULL;
261
262 for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
263 pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
264
265 rsc_stonith_ordering(child_rsc, stonith_op, data_set);
266 }
267
268 } else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
269 pe_rsc_trace(rsc,
270 "Skipping fencing constraints for unmanaged resource: %s",
271 rsc->id);
272
273 } else {
274 order_start_vs_fencing(rsc, stonith_op, data_set);
275 order_stop_vs_fencing(rsc, stonith_op, data_set);
276 }
277 }
278
279 /*!
280 * \internal
281 * \brief Order all actions appropriately relative to a fencing operation
282 *
283 * Ensure start operations of affected resources are ordered after fencing,
284 * imply stop and demote operations of affected resources by marking them as
285 * pseudo-actions, etc.
286 *
287 * \param[in] stonith_op Fencing operation
288 * \param[in,out] data_set Working set of cluster
289 */
290 void
291 pcmk__order_vs_fence(pe_action_t *stonith_op, pe_working_set_t *data_set)
/* */
292 {
293 CRM_CHECK(stonith_op && data_set, return);
294 for (GList *r = data_set->resources; r != NULL; r = r->next) {
295 rsc_stonith_ordering((pe_resource_t *) r->data, stonith_op, data_set);
296 }
297 }
298
299 /*!
300 * \internal
301 * \brief Order an action after unfencing
302 *
303 * \param[in] rsc Resource that action is for
304 * \param[in] node Node that action is on
305 * \param[in] action Action to be ordered after unfencing
306 * \param[in] order Ordering flags
307 */
308 void
309 pcmk__order_vs_unfence(pe_resource_t *rsc, pe_node_t *node, pe_action_t *action,
/* */
310 enum pe_ordering order)
311 {
312 /* When unfencing is in use, we order unfence actions before any probe or
313 * start of resources that require unfencing, and also of fence devices.
314 *
315 * This might seem to violate the principle that fence devices require
316 * only quorum. However, fence agents that unfence often don't have enough
317 * information to even probe or start unless the node is first unfenced.
318 */
319 if ((pcmk_is_set(rsc->flags, pe_rsc_fence_device)
320 && pcmk_is_set(rsc->cluster->flags, pe_flag_enable_unfencing))
321 || pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
322
323 /* Start with an optional ordering. Requiring unfencing would result in
324 * the node being unfenced, and all its resources being stopped,
325 * whenever a new resource is added -- which would be highly suboptimal.
326 */
327 pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE,
328 rsc->cluster);
329
330 order_actions(unfence, action, order);
331
332 if (!pcmk__node_unfenced(node)) {
333 // But unfencing is required if it has never been done
334 char *reason = crm_strdup_printf("required by %s %s",
335 rsc->id, action->task);
336
337 trigger_unfencing(NULL, node, reason, NULL, rsc->cluster);
338 free(reason);
339 }
340 }
341 }
342
343 /*!
344 * \internal
345 * \brief Create pseudo-op for guest node fence, and order relative to it
346 *
347 * \param[in] node Guest node to fence
348 */
349 void
350 pcmk__fence_guest(pe_node_t *node)
/* */
351 {
352 pe_resource_t *container = NULL;
353 pe_action_t *stop = NULL;
354 pe_action_t *stonith_op = NULL;
355
356 /* The fence action is just a label; we don't do anything differently for
357 * off vs. reboot. We specify it explicitly, rather than let it default to
358 * cluster's default action, because we are not _initiating_ fencing -- we
359 * are creating a pseudo-event to describe fencing that is already occurring
360 * by other means (container recovery).
361 */
362 const char *fence_action = "off";
363
364 CRM_ASSERT(node != NULL);
365
366 /* Check whether guest's container resource has any explicit stop or
367 * start (the stop may be implied by fencing of the guest's host).
368 */
369 container = node->details->remote_rsc->container;
370 if (container) {
371 stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP,
372 NULL);
373
374 if (find_first_action(container->actions, NULL, CRMD_ACTION_START,
375 NULL)) {
376 fence_action = "reboot";
377 }
378 }
379
380 /* Create a fence pseudo-event, so we have an event to order actions
381 * against, and the controller can always detect it.
382 */
383 stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean",
384 FALSE, node->details->data_set);
385 pe__set_action_flags(stonith_op, pe_action_pseudo|pe_action_runnable);
386
387 /* We want to imply stops/demotes after the guest is stopped, not wait until
388 * it is restarted, so we always order pseudo-fencing after stop, not start
389 * (even though start might be closer to what is done for a real reboot).
390 */
391 if ((stop != NULL) && pcmk_is_set(stop->flags, pe_action_pseudo)) {
392 pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE,
393 NULL, FALSE,
394 node->details->data_set);
395
396 crm_info("Implying guest %s is down (action %d) after %s fencing",
397 pe__node_name(node), stonith_op->id,
398 pe__node_name(stop->node));
399 order_actions(parent_stonith_op, stonith_op,
400 pe_order_runnable_left|pe_order_implies_then);
401
402 } else if (stop) {
403 order_actions(stop, stonith_op,
404 pe_order_runnable_left|pe_order_implies_then);
405 crm_info("Implying guest %s is down (action %d) "
406 "after container %s is stopped (action %d)",
407 pe__node_name(node), stonith_op->id,
408 container->id, stop->id);
409 } else {
410 /* If we're fencing the guest node but there's no stop for the guest
411 * resource, we must think the guest is already stopped. However, we may
412 * think so because its resource history was just cleaned. To avoid
413 * unnecessarily considering the guest node down if it's really up,
414 * order the pseudo-fencing after any stop of the connection resource,
415 * which will be ordered after any container (re-)probe.
416 */
417 stop = find_first_action(node->details->remote_rsc->actions, NULL,
418 RSC_STOP, NULL);
419
420 if (stop) {
421 order_actions(stop, stonith_op, pe_order_optional);
422 crm_info("Implying guest %s is down (action %d) "
423 "after connection is stopped (action %d)",
424 pe__node_name(node), stonith_op->id, stop->id);
425 } else {
426 /* Not sure why we're fencing, but everything must already be
427 * cleanly stopped.
428 */
429 crm_info("Implying guest %s is down (action %d) ",
430 pe__node_name(node), stonith_op->id);
431 }
432 }
433
434 // Order/imply other actions relative to pseudo-fence as with real fence
435 pcmk__order_vs_fence(stonith_op, node->details->data_set);
436 }
437
438 /*!
439 * \internal
440 * \brief Check whether node has already been unfenced
441 *
442 * \param[in] node Node to check
443 *
444 * \return true if node has a nonzero #node-unfenced attribute (or none),
445 * otherwise false
446 */
447 bool
448 pcmk__node_unfenced(pe_node_t *node)
/* */
449 {
450 const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED);
451
452 return !pcmk__str_eq(unfenced, "0", pcmk__str_null_matches);
453 }
454
455 /*!
456 * \internal
457 * \brief Order a resource's start and stop relative to unfencing of a node
458 *
459 * \param[in] data Node that could be unfenced
460 * \param[in,out] user_data Resource to order
461 */
462 void
463 pcmk__order_restart_vs_unfence(gpointer data, gpointer user_data)
/* ![[last]](../icons/n_last.png)
*/
464 {
465 pe_node_t *node = (pe_node_t *) data;
466 pe_resource_t *rsc = (pe_resource_t *) user_data;
467
468 pe_action_t *unfence = pe_fence_op(node, "on", true, NULL, false,
469 rsc->cluster);
470
471 crm_debug("Ordering any stops of %s before %s, and any starts after",
472 rsc->id, unfence->uuid);
473
474 /*
475 * It would be more efficient to order clone resources once,
476 * rather than order each instance, but ordering the instance
477 * allows us to avoid unnecessary dependencies that might conflict
478 * with user constraints.
479 *
480 * @TODO: This constraint can still produce a transition loop if the
481 * resource has a stop scheduled on the node being unfenced, and
482 * there is a user ordering constraint to start some other resource
483 * (which will be ordered after the unfence) before stopping this
484 * resource. An example is "start some slow-starting cloned service
485 * before stopping an associated virtual IP that may be moving to
486 * it":
487 * stop this -> unfencing -> start that -> stop this
488 */
489 pcmk__new_ordering(rsc, stop_key(rsc), NULL,
490 NULL, strdup(unfence->uuid), unfence,
491 pe_order_optional|pe_order_same_node,
492 rsc->cluster);
493
494 pcmk__new_ordering(NULL, strdup(unfence->uuid), unfence,
495 rsc, start_key(rsc), NULL,
496 pe_order_implies_then_on_node|pe_order_same_node,
497 rsc->cluster);
498 }
![[bottom]](../icons/n_bottom.png){kind=icon}
*/