![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
1 /*
2 * Copyright 2004-2022 the Pacemaker project contributors
3 *
4 * The version control history for this file may have further details.
5 *
6 * This source code is licensed under the GNU General Public License version 2
7 * or later (GPLv2+) WITHOUT ANY WARRANTY.
8 */
9
10 #include <crm_internal.h>
11
12 #include <glib.h>
13
14 #include <crm/crm.h>
15 #include <crm/pengine/status.h>
16 #include <pacemaker-internal.h>
17 #include "libpacemaker_private.h"
18
19 /*!
20 * \internal
21 * \brief Check whether a resource is known on a particular node
22 *
23 * \param[in] rsc Resource to check
24 * \param[in] node Node to check
25 *
26 * \return TRUE if resource (or parent if an anonymous clone) is known
27 */
28 static bool
29 rsc_is_known_on(pe_resource_t *rsc, const pe_node_t *node)
/* ![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
*/
30 {
31 if (pe_hash_table_lookup(rsc->known_on, node->details->id)) {
32 return TRUE;
33
34 } else if ((rsc->variant == pe_native)
35 && pe_rsc_is_anon_clone(rsc->parent)
36 && pe_hash_table_lookup(rsc->parent->known_on, node->details->id)) {
37 /* We check only the parent, not the uber-parent, because we cannot
38 * assume that the resource is known if it is in an anonymously cloned
39 * group (which may be only partially known).
40 */
41 return TRUE;
42 }
43 return FALSE;
44 }
45
46 /*!
47 * \internal
48 * \brief Order a resource's start and promote actions relative to fencing
49 *
50 * \param[in] rsc Resource to be ordered
51 * \param[in] stonith_op Fence action
52 * \param[in] data_set Cluster working set
53 */
54 static void
55 order_start_vs_fencing(pe_resource_t *rsc, pe_action_t *stonith_op,
/* ![[previous]](../icons/left.png)
*/
56 pe_working_set_t *data_set)
57 {
58 pe_node_t *target;
59 GList *gIter = NULL;
60
61 CRM_CHECK(stonith_op && stonith_op->node, return);
62 target = stonith_op->node;
63
64 for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
65 pe_action_t *action = (pe_action_t *) gIter->data;
66
67 switch (action->needs) {
68 case rsc_req_nothing:
69 // Anything other than start or promote requires nothing
70 break;
71
72 case rsc_req_stonith:
73 order_actions(stonith_op, action, pe_order_optional);
74 break;
75
76 case rsc_req_quorum:
77 if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)
78 && pe_hash_table_lookup(rsc->allowed_nodes, target->details->id)
79 && !rsc_is_known_on(rsc, target)) {
80
81 /* If we don't know the status of the resource on the node
82 * we're about to shoot, we have to assume it may be active
83 * there. Order the resource start after the fencing. This
84 * is analogous to waiting for all the probes for a resource
85 * to complete before starting it.
86 *
87 * The most likely explanation is that the DC died and took
88 * its status with it.
89 */
90 pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
91 target->details->uname);
92 order_actions(stonith_op, action,
93 pe_order_optional | pe_order_runnable_left);
94 }
95 break;
96 }
97 }
98 }
99
100 /*!
101 * \internal
102 * \brief Order a resource's stop and demote actions relative to fencing
103 *
104 * \param[in] rsc Resource to be ordered
105 * \param[in] stonith_op Fence action
106 * \param[in] data_set Cluster working set
107 */
108 static void
109 order_stop_vs_fencing(pe_resource_t *rsc, pe_action_t *stonith_op,
/* */
110 pe_working_set_t *data_set)
111 {
112 GList *gIter = NULL;
113 GList *action_list = NULL;
114 bool order_implicit = false;
115
116 pe_resource_t *top = uber_parent(rsc);
117 pe_action_t *parent_stop = NULL;
118 pe_node_t *target;
119
120 CRM_CHECK(stonith_op && stonith_op->node, return);
121 target = stonith_op->node;
122
123 /* Get a list of stop actions potentially implied by the fencing */
124 action_list = pe__resource_actions(rsc, target, RSC_STOP, FALSE);
125
126 /* If resource requires fencing, implicit actions must occur after fencing.
127 *
128 * Implied stops and demotes of resources running on guest nodes are always
129 * ordered after fencing, even if the resource does not require fencing,
130 * because guest node "fencing" is actually just a resource stop.
131 */
132 if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)
133 || pe__is_guest_node(target)) {
134
135 order_implicit = true;
136 }
137
138 if (action_list && order_implicit) {
139 parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
140 }
141
142 for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
143 pe_action_t *action = (pe_action_t *) gIter->data;
144
145 // The stop would never complete, so convert it into a pseudo-action.
146 pe__set_action_flags(action, pe_action_pseudo|pe_action_runnable);
147
148 if (order_implicit) {
149 pe__set_action_flags(action, pe_action_implied_by_stonith);
150
151 /* Order the stonith before the parent stop (if any).
152 *
153 * Also order the stonith before the resource stop, unless the
154 * resource is inside a bundle -- that would cause a graph loop.
155 * We can rely on the parent stop's ordering instead.
156 *
157 * User constraints must not order a resource in a guest node
158 * relative to the guest node container resource. The
159 * pe_order_preserve flag marks constraints as generated by the
160 * cluster and thus immune to that check (and is irrelevant if
161 * target is not a guest).
162 */
163 if (!pe_rsc_is_bundled(rsc)) {
164 order_actions(stonith_op, action, pe_order_preserve);
165 }
166 order_actions(stonith_op, parent_stop, pe_order_preserve);
167 }
168
169 if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
170 crm_notice("Stop of failed resource %s is implicit %s %s is fenced",
171 rsc->id, (order_implicit? "after" : "because"),
172 target->details->uname);
173 } else {
174 crm_info("%s is implicit %s %s is fenced",
175 action->uuid, (order_implicit? "after" : "because"),
176 target->details->uname);
177 }
178
179 if (pcmk_is_set(rsc->flags, pe_rsc_notify)) {
180 pcmk__order_notifs_after_fencing(action, rsc, stonith_op);
181 }
182
183 #if 0
184 /* It might be a good idea to stop healthy resources on a node about to
185 * be fenced, when possible.
186 *
187 * However, fencing must be done before a failed resource's
188 * (pseudo-)stop action, so that could create a loop. For example, given
189 * a group of A and B running on node N with a failed stop of B:
190 *
191 * fence N -> stop B (pseudo-op) -> stop A -> fence N
192 *
193 * The block below creates the stop A -> fence N ordering and therefore
194 * must (at least for now) be disabled. Instead, run the block above and
195 * treat all resources on N as B would be (i.e., as a pseudo-op after
196 * the fencing).
197 *
198 * @TODO Maybe break the "A requires B" dependency in
199 * pcmk__update_action_for_orderings() and use this block for healthy
200 * resources instead of the above.
201 */
202 crm_info("Moving healthy resource %s off %s before fencing",
203 rsc->id, node->details->uname);
204 pcmk__new_ordering(rsc, stop_key(rsc), NULL, NULL,
205 strdup(CRM_OP_FENCE), stonith_op,
206 pe_order_optional, data_set);
207 #endif
208 }
209
210 g_list_free(action_list);
211
212 /* Get a list of demote actions potentially implied by the fencing */
213 action_list = pe__resource_actions(rsc, target, RSC_DEMOTE, FALSE);
214
215 for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
216 pe_action_t *action = (pe_action_t *) gIter->data;
217
218 if (!(action->node->details->online) || action->node->details->unclean
219 || pcmk_is_set(rsc->flags, pe_rsc_failed)) {
220
221 if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
222 pe_rsc_info(rsc,
223 "Demote of failed resource %s is implicit after %s is fenced",
224 rsc->id, target->details->uname);
225 } else {
226 pe_rsc_info(rsc, "%s is implicit after %s is fenced",
227 action->uuid, target->details->uname);
228 }
229
230 /* The demote would never complete and is now implied by the
231 * fencing, so convert it into a pseudo-action.
232 */
233 pe__set_action_flags(action, pe_action_pseudo|pe_action_runnable);
234
235 if (pe_rsc_is_bundled(rsc)) {
236 // Do nothing, let recovery be ordered after parent's implied stop
237
238 } else if (order_implicit) {
239 order_actions(stonith_op, action, pe_order_preserve|pe_order_optional);
240 }
241 }
242 }
243
244 g_list_free(action_list);
245 }
246
247 /*!
248 * \internal
249 * \brief Order resource actions properly relative to fencing
250 *
251 * \param[in] rsc Resource whose actions should be ordered
252 * \param[in] stonith_op Fencing operation to be ordered against
253 * \param[in] data_set Cluster working set
254 */
255 static void
256 rsc_stonith_ordering(pe_resource_t *rsc, pe_action_t *stonith_op,
/* */
257 pe_working_set_t *data_set)
258 {
259 if (rsc->children) {
260 GList *gIter = NULL;
261
262 for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
263 pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
264
265 rsc_stonith_ordering(child_rsc, stonith_op, data_set);
266 }
267
268 } else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
269 pe_rsc_trace(rsc,
270 "Skipping fencing constraints for unmanaged resource: %s",
271 rsc->id);
272
273 } else {
274 order_start_vs_fencing(rsc, stonith_op, data_set);
275 order_stop_vs_fencing(rsc, stonith_op, data_set);
276 }
277 }
278
279 /*!
280 * \internal
281 * \brief Order all actions appropriately relative to a fencing operation
282 *
283 * Ensure start operations of affected resources are ordered after fencing,
284 * imply stop and demote operations of affected resources by marking them as
285 * pseudo-actions, etc.
286 *
287 * \param[in] stonith_op Fencing operation
288 * \param[in,out] data_set Working set of cluster
289 */
290 void
291 pcmk__order_vs_fence(pe_action_t *stonith_op, pe_working_set_t *data_set)
/* */
292 {
293 CRM_CHECK(stonith_op && data_set, return);
294 for (GList *r = data_set->resources; r != NULL; r = r->next) {
295 rsc_stonith_ordering((pe_resource_t *) r->data, stonith_op, data_set);
296 }
297 }
298
299 /*!
300 * \internal
301 * \brief Order an action after unfencing
302 *
303 * \param[in] rsc Resource that action is for
304 * \param[in] node Node that action is on
305 * \param[in] action Action to be ordered after unfencing
306 * \param[in] order Ordering flags
307 * \param[in] data_set Cluster working set
308 */
309 void
310 pcmk__order_vs_unfence(pe_resource_t *rsc, pe_node_t *node, pe_action_t *action,
/* */
311 enum pe_ordering order, pe_working_set_t *data_set)
312 {
313 /* When unfencing is in use, we order unfence actions before any probe or
314 * start of resources that require unfencing, and also of fence devices.
315 *
316 * This might seem to violate the principle that fence devices require
317 * only quorum. However, fence agents that unfence often don't have enough
318 * information to even probe or start unless the node is first unfenced.
319 */
320 if (pcmk__is_unfence_device(rsc, data_set)
321 || pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
322
323 /* Start with an optional ordering. Requiring unfencing would result in
324 * the node being unfenced, and all its resources being stopped,
325 * whenever a new resource is added -- which would be highly suboptimal.
326 */
327 pe_action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, FALSE, data_set);
328
329 order_actions(unfence, action, order);
330
331 if (!pcmk__node_unfenced(node)) {
332 // But unfencing is required if it has never been done
333 char *reason = crm_strdup_printf("required by %s %s",
334 rsc->id, action->task);
335
336 trigger_unfencing(NULL, node, reason, NULL, data_set);
337 free(reason);
338 }
339 }
340 }
341
342 /*!
343 * \internal
344 * \brief Create pseudo-op for guest node fence, and order relative to it
345 *
346 * \param[in] node Guest node to fence
347 * \param[in] data_set Working set of CIB state
348 */
349 void
350 pcmk__fence_guest(pe_node_t *node, pe_working_set_t *data_set)
/* */
351 {
352 pe_resource_t *container = node->details->remote_rsc->container;
353 pe_action_t *stop = NULL;
354 pe_action_t *stonith_op = NULL;
355
356 /* The fence action is just a label; we don't do anything differently for
357 * off vs. reboot. We specify it explicitly, rather than let it default to
358 * cluster's default action, because we are not _initiating_ fencing -- we
359 * are creating a pseudo-event to describe fencing that is already occurring
360 * by other means (container recovery).
361 */
362 const char *fence_action = "off";
363
364 /* Check whether guest's container resource has any explicit stop or
365 * start (the stop may be implied by fencing of the guest's host).
366 */
367 if (container) {
368 stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP,
369 NULL);
370
371 if (find_first_action(container->actions, NULL, CRMD_ACTION_START,
372 NULL)) {
373 fence_action = "reboot";
374 }
375 }
376
377 /* Create a fence pseudo-event, so we have an event to order actions
378 * against, and the controller can always detect it.
379 */
380 stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean",
381 FALSE, data_set);
382 pe__set_action_flags(stonith_op, pe_action_pseudo|pe_action_runnable);
383
384 /* We want to imply stops/demotes after the guest is stopped, not wait until
385 * it is restarted, so we always order pseudo-fencing after stop, not start
386 * (even though start might be closer to what is done for a real reboot).
387 */
388 if ((stop != NULL) && pcmk_is_set(stop->flags, pe_action_pseudo)) {
389 pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE,
390 NULL, FALSE, data_set);
391
392 crm_info("Implying guest node %s is down (action %d) after %s fencing",
393 node->details->uname, stonith_op->id,
394 stop->node->details->uname);
395 order_actions(parent_stonith_op, stonith_op,
396 pe_order_runnable_left|pe_order_implies_then);
397
398 } else if (stop) {
399 order_actions(stop, stonith_op,
400 pe_order_runnable_left|pe_order_implies_then);
401 crm_info("Implying guest node %s is down (action %d) "
402 "after container %s is stopped (action %d)",
403 node->details->uname, stonith_op->id,
404 container->id, stop->id);
405 } else {
406 /* If we're fencing the guest node but there's no stop for the guest
407 * resource, we must think the guest is already stopped. However, we may
408 * think so because its resource history was just cleaned. To avoid
409 * unnecessarily considering the guest node down if it's really up,
410 * order the pseudo-fencing after any stop of the connection resource,
411 * which will be ordered after any container (re-)probe.
412 */
413 stop = find_first_action(node->details->remote_rsc->actions, NULL,
414 RSC_STOP, NULL);
415
416 if (stop) {
417 order_actions(stop, stonith_op, pe_order_optional);
418 crm_info("Implying guest node %s is down (action %d) "
419 "after connection is stopped (action %d)",
420 node->details->uname, stonith_op->id, stop->id);
421 } else {
422 /* Not sure why we're fencing, but everything must already be
423 * cleanly stopped.
424 */
425 crm_info("Implying guest node %s is down (action %d) ",
426 node->details->uname, stonith_op->id);
427 }
428 }
429
430 // Order/imply other actions relative to pseudo-fence as with real fence
431 pcmk__order_vs_fence(stonith_op, data_set);
432 }
433
434 /*!
435 * \internal
436 * \brief Check whether node has already been unfenced
437 *
438 * \param[in] node Node to check
439 *
440 * \return true if node has a nonzero #node-unfenced attribute (or none),
441 * otherwise false
442 */
443 bool
444 pcmk__node_unfenced(pe_node_t *node)
/* */
445 {
446 const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED);
447
448 return !pcmk__str_eq(unfenced, "0", pcmk__str_null_matches);
449 }
450
451 /*!
452 * \internal
453 * \brief Check whether a resource is a fencing device that supports unfencing
454 *
455 * \param[in] rsc Resource to check
456 * \param[in] data_set Cluster working set
457 *
458 * \return true if \p rsc is a fencing device that supports unfencing,
459 * otherwise false
460 */
461 bool
462 pcmk__is_unfence_device(const pe_resource_t *rsc,
/* ![[last]](../icons/n_last.png)
*/
463 const pe_working_set_t *data_set)
464 {
465 return pcmk_is_set(rsc->flags, pe_rsc_fence_device)
466 && pcmk_is_set(data_set->flags, pe_flag_enable_unfencing);
467 }
![[bottom]](../icons/n_bottom.png){kind=icon}
*/