This source file includes following definitions.
- shutdown_lock_cleared
- te_update_diff_v1
- process_lrm_resource_diff
- process_resource_updates
- extract_node_uuid
- abort_unless_down
- process_op_deletion
- process_delete_diff
- process_node_state_diff
- process_status_diff
- process_cib_diff
- te_update_diff_v2
- te_update_diff
- process_te_message
- cib_action_updated
- action_timer_callback
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <sys/stat.h>
13
14 #include <crm/crm.h>
15 #include <crm/common/xml.h>
16 #include <crm/common/xml_internal.h>
17 #include <crm/msg_xml.h>
18 #include <crm/cluster.h>
19
20 #include <pacemaker-controld.h>
21
22 void te_update_confirm(const char *event, xmlNode * msg);
23
24 extern char *te_uuid;
25 gboolean shuttingdown = FALSE;
26 crm_graph_t *transition_graph;
27 crm_trigger_t *transition_trigger = NULL;
28
29
30 #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']"
31
32
33 static bool
34 shutdown_lock_cleared(xmlNode *lrm_resource)
35 {
36 time_t shutdown_lock = 0;
37
38 return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
39 &shutdown_lock) == pcmk_ok)
40 && (shutdown_lock == 0);
41 }
42
43 static void
44 te_update_diff_v1(const char *event, xmlNode *diff)
45 {
46 int lpc, max;
47 xmlXPathObject *xpathObj = NULL;
48
49 CRM_CHECK(diff != NULL, return);
50
51 xml_log_patchset(LOG_TRACE, __func__, diff);
52 if (cib_config_changed(NULL, NULL, &diff)) {
53 abort_transition(INFINITY, tg_restart, "Non-status change", diff);
54 goto bail;
55 }
56
57
58 xpathObj =
59 xpath_search(diff,
60 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
61 if (numXpathResults(xpathObj) > 0) {
62 xmlNode *aborted = getXpathResult(xpathObj, 0);
63
64 abort_transition(INFINITY, tg_restart, "Ticket attribute: update", aborted);
65 goto bail;
66
67 }
68 freeXpathObject(xpathObj);
69
70
71 xpathObj =
72 xpath_search(diff,
73 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
74 if (numXpathResults(xpathObj) > 0) {
75 xmlNode *aborted = getXpathResult(xpathObj, 0);
76
77 abort_transition(INFINITY, tg_restart, "Ticket attribute: removal", aborted);
78 goto bail;
79 }
80 freeXpathObject(xpathObj);
81
82
83 xpathObj =
84 xpath_search(diff,
85 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
86 XML_TAG_TRANSIENT_NODEATTRS);
87 if (numXpathResults(xpathObj) > 0) {
88 xmlNode *aborted = getXpathResult(xpathObj, 0);
89
90 abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
91 goto bail;
92
93 }
94 freeXpathObject(xpathObj);
95
96
97 xpathObj = xpath_search(diff,
98 "//" F_CIB_UPDATE_RESULT
99 "//" XML_TAG_DIFF_ADDED
100 "//" XML_LRM_TAG_RESOURCE);
101 max = numXpathResults(xpathObj);
102
103
104
105
106
107
108
109
110
111
112
113 if ((transition_graph->pending == 0) && (max > 1)) {
114 crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
115 max);
116 crm_log_xml_trace(diff, "lrm-refresh");
117 abort_transition(INFINITY, tg_restart, "History refresh", NULL);
118 goto bail;
119 }
120
121 if (max == 1) {
122 xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
123
124 if (shutdown_lock_cleared(lrm_resource)) {
125
126 abort_transition(INFINITY, tg_restart, "Shutdown lock cleared",
127 lrm_resource);
128
129 }
130 }
131 freeXpathObject(xpathObj);
132
133
134 xpathObj =
135 xpath_search(diff,
136 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
137 max = numXpathResults(xpathObj);
138 if (max > 0) {
139 int lpc = 0;
140
141 for (lpc = 0; lpc < max; lpc++) {
142 xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
143 const char *node = get_node_id(rsc_op);
144
145 process_graph_event(rsc_op, node);
146 }
147 }
148 freeXpathObject(xpathObj);
149
150
151 xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
152 max = numXpathResults(xpathObj);
153 for (lpc = 0; lpc < max; lpc++) {
154 int path_max = 0;
155 const char *op_id = NULL;
156 char *rsc_op_xpath = NULL;
157 xmlXPathObject *op_match = NULL;
158 xmlNode *match = getXpathResult(xpathObj, lpc);
159
160 CRM_LOG_ASSERT(match != NULL);
161 if(match == NULL) { continue; };
162
163 op_id = ID(match);
164
165 path_max = strlen(RSC_OP_TEMPLATE) + strlen(op_id) + 1;
166 rsc_op_xpath = calloc(1, path_max);
167 snprintf(rsc_op_xpath, path_max, RSC_OP_TEMPLATE, op_id);
168
169 op_match = xpath_search(diff, rsc_op_xpath);
170 if (numXpathResults(op_match) == 0) {
171
172 const char *node = get_node_id(match);
173 crm_action_t *cancelled = get_cancel_action(op_id, node);
174
175 if (cancelled == NULL) {
176 crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
177 node);
178 abort_transition(INFINITY, tg_restart, "Resource op removal", match);
179 freeXpathObject(op_match);
180 free(rsc_op_xpath);
181 goto bail;
182
183 } else {
184 crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
185 op_id, node, cancelled->id);
186 }
187 }
188
189 freeXpathObject(op_match);
190 free(rsc_op_xpath);
191 }
192
193 bail:
194 freeXpathObject(xpathObj);
195 }
196
197 static void
198 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
199 {
200 for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
201 rsc_op = pcmk__xml_next(rsc_op)) {
202 process_graph_event(rsc_op, node);
203 }
204 if (shutdown_lock_cleared(lrm_resource)) {
205
206 abort_transition(INFINITY, tg_restart, "Shutdown lock cleared",
207 lrm_resource);
208 }
209 }
210
211 static void
212 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
213 const char *op, const char *xpath)
214 {
215 xmlNode *rsc = NULL;
216
217 if (xml == NULL) {
218 return;
219 }
220
221 if (strcmp(TYPE(xml), XML_CIB_TAG_LRM) == 0) {
222 xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
223 CRM_CHECK(xml != NULL, return);
224 }
225
226 CRM_CHECK(strcmp(TYPE(xml), XML_LRM_TAG_RESOURCES) == 0, return);
227
228
229
230
231
232
233
234
235
236
237
238
239
240 if ((transition_graph->pending == 0)
241 && xml->children && xml->children->next) {
242
243 crm_log_xml_trace(change, "lrm-refresh");
244 abort_transition(INFINITY, tg_restart, "History refresh", NULL);
245 return;
246 }
247
248 for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
249 rsc = pcmk__xml_next(rsc)) {
250 crm_trace("Processing %s", ID(rsc));
251 process_lrm_resource_diff(rsc, node);
252 }
253 }
254
255 static char *extract_node_uuid(const char *xpath)
256 {
257 char *mutable_path = strdup(xpath);
258 char *node_uuid = NULL;
259 char *search = NULL;
260 char *match = NULL;
261
262 match = strstr(mutable_path, "node_state[@id=\'");
263 if (match == NULL) {
264 free(mutable_path);
265 return NULL;
266 }
267 match += strlen("node_state[@id=\'");
268
269 search = strchr(match, '\'');
270 if (search == NULL) {
271 free(mutable_path);
272 return NULL;
273 }
274 search[0] = 0;
275
276 node_uuid = strdup(match);
277 free(mutable_path);
278 return node_uuid;
279 }
280
281 static void
282 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
283 const char *reason)
284 {
285 char *node_uuid = NULL;
286 crm_action_t *down = NULL;
287
288 if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
289 abort_transition(INFINITY, tg_restart, reason, change);
290 return;
291 }
292
293 node_uuid = extract_node_uuid(xpath);
294 if(node_uuid == NULL) {
295 crm_err("Could not extract node ID from %s", xpath);
296 abort_transition(INFINITY, tg_restart, reason, change);
297 return;
298 }
299
300 down = match_down_event(node_uuid);
301 if (down == NULL) {
302 crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
303 abort_transition(INFINITY, tg_restart, reason, change);
304 } else {
305 crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
306 }
307 free(node_uuid);
308 }
309
310 static void
311 process_op_deletion(const char *xpath, xmlNode *change)
312 {
313 char *mutable_key = strdup(xpath);
314 char *key;
315 char *node_uuid;
316
317
318 key = strrchr(mutable_key, '\'');
319 if (key != NULL) {
320 *key = '\0';
321 key = strrchr(mutable_key, '\'');
322 }
323 if (key == NULL) {
324 crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
325 xpath);
326 free(mutable_key);
327 return;
328 }
329 ++key;
330
331 node_uuid = extract_node_uuid(xpath);
332 if (confirm_cancel_action(key, node_uuid) == FALSE) {
333 abort_transition(INFINITY, tg_restart, "Resource operation removal",
334 change);
335 }
336 free(mutable_key);
337 free(node_uuid);
338 }
339
340 static void
341 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
342 {
343 if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
344 process_op_deletion(xpath, change);
345
346 } else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
347 abort_unless_down(xpath, op, change, "Resource state removal");
348
349 } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
350 abort_unless_down(xpath, op, change, "Node state removal");
351
352 } else {
353 crm_trace("Ignoring delete of %s", xpath);
354 }
355 }
356
357 static void
358 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
359 const char *xpath)
360 {
361 xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
362
363 process_resource_updates(ID(state), lrm, change, op, xpath);
364 }
365
366 static void
367 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
368 const char *xpath)
369 {
370 for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
371 state = pcmk__xml_next(state)) {
372 process_node_state_diff(state, change, op, xpath);
373 }
374 }
375
376 static void
377 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
378 const char *xpath)
379 {
380 xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
381 xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);
382
383 if (status) {
384 process_status_diff(status, change, op, xpath);
385 }
386 if (config) {
387 abort_transition(INFINITY, tg_restart,
388 "Non-status-only change", change);
389 }
390 }
391
392 static void
393 te_update_diff_v2(xmlNode *diff)
394 {
395 crm_log_xml_trace(diff, "Patch:Raw");
396
397 for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
398 change = pcmk__xml_next(change)) {
399
400 xmlNode *match = NULL;
401 const char *name = NULL;
402 const char *xpath = crm_element_value(change, XML_DIFF_PATH);
403
404
405 const char *op = crm_element_value(change, XML_DIFF_OP);
406
407
408 if (op == NULL) {
409 continue;
410
411 } else if (xpath == NULL) {
412 crm_trace("Ignoring %s change for version field", op);
413 continue;
414
415 } else if (strcmp(op, "move") == 0) {
416 crm_trace("Ignoring move change at %s", xpath);
417 continue;
418 }
419
420
421 if (strcmp(op, "create") == 0) {
422 match = change->children;
423
424 } else if (strcmp(op, "modify") == 0) {
425 match = first_named_child(change, XML_DIFF_RESULT);
426 if(match) {
427 match = match->children;
428 }
429
430 } else if (strcmp(op, "delete") != 0) {
431 crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
432 op, xpath);
433 continue;
434 }
435
436 if (match) {
437 if (match->type == XML_COMMENT_NODE) {
438 crm_trace("Ignoring %s operation for comment at %s", op, xpath);
439 continue;
440 }
441 name = (const char *)match->name;
442 }
443
444 crm_trace("Handling %s operation for %s%s%s",
445 op, (xpath? xpath : "CIB"),
446 (name? " matched by " : ""), (name? name : ""));
447
448 if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
449 abort_transition(INFINITY, tg_restart, "Configuration change",
450 change);
451 break;
452
453 } else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
454 || pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
455 abort_transition(INFINITY, tg_restart, "Ticket attribute change", change);
456 break;
457
458 } else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
459 || pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS, pcmk__str_casei)) {
460 abort_unless_down(xpath, op, change, "Transient attribute change");
461 break;
462
463 } else if (strcmp(op, "delete") == 0) {
464 process_delete_diff(xpath, op, change);
465
466 } else if (name == NULL) {
467 crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
468 op, xpath);
469
470 } else if (strcmp(name, XML_TAG_CIB) == 0) {
471 process_cib_diff(match, change, op, xpath);
472
473 } else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
474 process_status_diff(match, change, op, xpath);
475
476 } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
477 process_node_state_diff(match, change, op, xpath);
478
479 } else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
480 process_resource_updates(ID(match), match, change, op, xpath);
481
482 } else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
483 char *local_node = pcmk__xpath_node_id(xpath, "lrm");
484
485 process_resource_updates(local_node, match, change, op, xpath);
486 free(local_node);
487
488 } else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
489 char *local_node = pcmk__xpath_node_id(xpath, "lrm");
490
491 process_lrm_resource_diff(match, local_node);
492 free(local_node);
493
494 } else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
495 char *local_node = pcmk__xpath_node_id(xpath, "lrm");
496
497 process_graph_event(match, local_node);
498 free(local_node);
499
500 } else {
501 crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
502 op, xpath, name);
503 }
504 }
505 }
506
507 void
508 te_update_diff(const char *event, xmlNode * msg)
509 {
510 xmlNode *diff = NULL;
511 const char *op = NULL;
512 int rc = -EINVAL;
513 int format = 1;
514 int p_add[] = { 0, 0, 0 };
515 int p_del[] = { 0, 0, 0 };
516
517 CRM_CHECK(msg != NULL, return);
518 crm_element_value_int(msg, F_CIB_RC, &rc);
519
520 if (transition_graph == NULL) {
521 crm_trace("No graph");
522 return;
523
524 } else if (rc < pcmk_ok) {
525 crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
526 return;
527
528 } else if (transition_graph->complete
529 && fsa_state != S_IDLE
530 && fsa_state != S_TRANSITION_ENGINE
531 && fsa_state != S_POLICY_ENGINE) {
532 crm_trace("Filter state=%s, complete=%d", fsa_state2string(fsa_state),
533 transition_graph->complete);
534 return;
535 }
536
537 op = crm_element_value(msg, F_CIB_OPERATION);
538 diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
539
540 xml_patch_versions(diff, p_add, p_del);
541 crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
542 p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
543 fsa_state2string(fsa_state));
544
545 crm_element_value_int(diff, "format", &format);
546 switch (format) {
547 case 1:
548 te_update_diff_v1(event, diff);
549 break;
550 case 2:
551 te_update_diff_v2(diff);
552 break;
553 default:
554 crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
555 format);
556 }
557 }
558
559 gboolean
560 process_te_message(xmlNode * msg, xmlNode * xml_data)
561 {
562 const char *from = crm_element_value(msg, F_ORIG);
563 const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
564 const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
565 const char *ref = crm_element_value(msg, F_CRM_REFERENCE);
566 const char *op = crm_element_value(msg, F_CRM_TASK);
567 const char *type = crm_element_value(msg, F_CRM_MSG_TYPE);
568
569 crm_trace("Processing %s (%s) message", op, ref);
570 crm_log_xml_trace(msg, "ipc");
571
572 if (op == NULL) {
573
574
575 } else if (sys_to == NULL || strcasecmp(sys_to, CRM_SYSTEM_TENGINE) != 0) {
576 crm_trace("Bad sys-to %s", crm_str(sys_to));
577 return FALSE;
578
579 } else if (pcmk__str_eq(op, CRM_OP_INVOKE_LRM, pcmk__str_casei)
580 && pcmk__str_eq(sys_from, CRM_SYSTEM_LRMD, pcmk__str_casei)
581
582 ) {
583 xmlXPathObject *xpathObj = NULL;
584
585 crm_log_xml_trace(msg, "Processing (N)ACK");
586 crm_debug("Processing (N)ACK %s from %s", crm_element_value(msg, F_CRM_REFERENCE), from);
587
588 xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
589 if (numXpathResults(xpathObj)) {
590 int lpc = 0, max = numXpathResults(xpathObj);
591
592 for (lpc = 0; lpc < max; lpc++) {
593 xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
594 const char *node = get_node_id(rsc_op);
595
596 process_graph_event(rsc_op, node);
597 }
598 freeXpathObject(xpathObj);
599
600 } else {
601 crm_log_xml_err(msg, "Invalid (N)ACK");
602 freeXpathObject(xpathObj);
603 return FALSE;
604 }
605
606 } else {
607 crm_err("Unknown command: %s::%s from %s", type, op, sys_from);
608 }
609
610 crm_trace("finished processing message");
611
612 return TRUE;
613 }
614
615 void
616 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
617 {
618 if (rc < pcmk_ok) {
619 crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
620 }
621 }
622
623
624
625
626
627
628
629
630 gboolean
631 action_timer_callback(gpointer data)
632 {
633 crm_action_timer_t *timer = NULL;
634 const char *task = NULL;
635 const char *on_node = NULL;
636 const char *via_node = NULL;
637
638 CRM_CHECK(data != NULL, return FALSE);
639
640 timer = (crm_action_timer_t *) data;
641 stop_te_timer(timer);
642
643 CRM_CHECK(timer->action != NULL, return FALSE);
644
645 task = crm_element_value(timer->action->xml, XML_LRM_ATTR_TASK);
646 on_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_TARGET);
647 via_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_ROUTER_NODE);
648
649 if (transition_graph->complete) {
650 crm_notice("Node %s did not send %s result (via %s) within %dms "
651 "(ignoring because transition not in progress)",
652 (on_node? on_node : ""), (task? task : "unknown action"),
653 (via_node? via_node : "controller"), timer->timeout);
654 } else {
655
656
657 crm_err("Node %s did not send %s result (via %s) within %dms "
658 "(action timeout plus cluster-delay)",
659 (on_node? on_node : ""), (task? task : "unknown action"),
660 (via_node? via_node : "controller"),
661 timer->timeout + transition_graph->network_delay);
662 pcmk__log_graph_action(LOG_ERR, timer->action);
663
664 crm__set_graph_action_flags(timer->action, pcmk__graph_action_failed);
665
666 te_action_confirmed(timer->action, transition_graph);
667 abort_transition(INFINITY, tg_restart, "Action lost", NULL);
668
669
670 if ((timer->action->type == action_type_rsc)
671 && controld_action_is_recordable(task)) {
672 controld_record_action_timeout(timer->action);
673 }
674 }
675
676 return FALSE;
677 }