This source file includes following definitions.
- lrm_connection_destroy
- make_stop_id
- copy_instance_keys
- copy_meta_keys
- history_remove_recurring_op
- history_free_recurring_ops
- history_free
- update_history_cache
- send_task_ok_ack
- op_node_name
- lrm_op_callback
- do_lrm_control
- lrm_state_verify_stopped
- build_parameter_list
- append_restart_list
- append_secure_list
- build_operation_update
- is_rsc_active
- build_active_RAs
- do_lrm_query_internal
- controld_query_executor_state
- controld_rc2event
- controld_trigger_delete_refresh
- notify_deleted
- lrm_remove_deleted_rsc
- lrm_remove_deleted_op
- delete_rsc_entry
- erase_lrm_history_by_op
- erase_lrm_history_by_id
- last_failed_matches_op
- lrm_clear_last_failure
- cancel_op
- cancel_action_by_key
- cancel_op_key
- get_lrm_resource
- delete_resource
- get_fake_call_id
- fake_op_status
- force_reprobe
- synthesize_lrmd_failure
- lrm_op_target
- fail_lrm_resource
- handle_refresh_op
- handle_query_op
- handle_reprobe_op
- do_lrm_cancel
- do_lrm_delete
- do_lrm_invoke
- resolve_versioned_parameters
- construct_op
- controld_ack_event_directly
- verify_stopped
- stop_recurring_action_by_rsc
- stop_recurring_actions
- record_pending_op
- do_lrm_rsc_op
- cib_rsc_callback
- should_preserve_lock
- do_update_resource
- do_lrm_event
- unescape_newlines
- did_lrm_rsc_op_fail
- process_lrm_event
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <regex.h>
13 #include <sys/param.h>
14 #include <sys/types.h>
15 #include <sys/wait.h>
16
17 #include <crm/crm.h>
18 #include <crm/lrmd.h>
19 #include <crm/services.h>
20 #include <crm/msg_xml.h>
21 #include <crm/common/xml.h>
22 #include <crm/pengine/rules.h>
23
24 #include <pacemaker-internal.h>
25 #include <pacemaker-controld.h>
26
27 #define START_DELAY_THRESHOLD 5 * 60 * 1000
28 #define MAX_LRM_REG_FAILS 30
29
30 struct delete_event_s {
31 int rc;
32 const char *rsc;
33 lrm_state_t *lrm_state;
34 };
35
36 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
37 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
38 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
39
40 static lrmd_event_data_t *construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op,
41 const char *rsc_id, const char *operation);
42 static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
43 const char *operation, xmlNode *msg);
44
45 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
46 int log_level);
47 static int do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
48 lrmd_event_data_t *op, time_t lock_time);
49
50 static void
51 lrm_connection_destroy(void)
52 {
53 if (pcmk_is_set(fsa_input_register, R_LRM_CONNECTED)) {
54 crm_crit("Connection to executor failed");
55 register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
56 controld_clear_fsa_input_flags(R_LRM_CONNECTED);
57
58 } else {
59 crm_info("Disconnected from executor");
60 }
61
62 }
63
64 static char *
65 make_stop_id(const char *rsc, int call_id)
66 {
67 return crm_strdup_printf("%s:%d", rsc, call_id);
68 }
69
70 static void
71 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
72 {
73 if (strstr(key, CRM_META "_") == NULL) {
74 g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
75 }
76 }
77
78 static void
79 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
80 {
81 if (strstr(key, CRM_META "_") != NULL) {
82 g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
83 }
84 }
85
86
87
88
89
90
91
92
93
94
95 static gboolean
96 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
97 {
98 GList *iter;
99
100 for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
101 lrmd_event_data_t *existing = iter->data;
102
103 if ((op->interval_ms == existing->interval_ms)
104 && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
105 && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
106
107 history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
108 lrmd_free_event(existing);
109 return TRUE;
110 }
111 }
112 return FALSE;
113 }
114
115
116
117
118
119
120
121 static void
122 history_free_recurring_ops(rsc_history_t *history)
123 {
124 GList *iter;
125
126 for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
127 lrmd_free_event(iter->data);
128 }
129 g_list_free(history->recurring_op_list);
130 history->recurring_op_list = NULL;
131 }
132
133
134
135
136
137
138
139 void
140 history_free(gpointer data)
141 {
142 rsc_history_t *history = (rsc_history_t*)data;
143
144 if (history->stop_params) {
145 g_hash_table_destroy(history->stop_params);
146 }
147
148
149 free(history->rsc.type);
150 free(history->rsc.standard);
151 free(history->rsc.provider);
152
153 lrmd_free_event(history->failed);
154 lrmd_free_event(history->last);
155 free(history->id);
156 history_free_recurring_ops(history);
157 free(history);
158 }
159
160 static void
161 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
162 {
163 int target_rc = 0;
164 rsc_history_t *entry = NULL;
165
166 if (op->rsc_deleted) {
167 crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
168 controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
169 NULL, crmd_cib_smart_opt());
170 return;
171 }
172
173 if (pcmk__str_eq(op->op_type, RSC_NOTIFY, pcmk__str_casei)) {
174 return;
175 }
176
177 crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
178
179 entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
180 if (entry == NULL && rsc) {
181 entry = calloc(1, sizeof(rsc_history_t));
182 entry->id = strdup(op->rsc_id);
183 g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
184
185 entry->rsc.id = entry->id;
186 entry->rsc.type = strdup(rsc->type);
187 entry->rsc.standard = strdup(rsc->standard);
188 if (rsc->provider) {
189 entry->rsc.provider = strdup(rsc->provider);
190 } else {
191 entry->rsc.provider = NULL;
192 }
193
194 } else if (entry == NULL) {
195 crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
196 return;
197 }
198
199 entry->last_callid = op->call_id;
200 target_rc = rsc_op_expected_rc(op);
201 if (op->op_status == PCMK_LRM_OP_CANCELLED) {
202 if (op->interval_ms > 0) {
203 crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
204 op->rsc_id, op->op_type, op->interval_ms);
205 history_remove_recurring_op(entry, op);
206 return;
207 } else {
208 crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
209 op->rsc_id, op->op_type, op->interval_ms, op->rc,
210 op->op_status);
211 }
212
213 } else if (did_rsc_op_fail(op, target_rc)) {
214
215
216
217 if (entry->failed) {
218 lrmd_free_event(entry->failed);
219 }
220 entry->failed = lrmd_copy_event(op);
221
222 } else if (op->interval_ms == 0) {
223 if (entry->last) {
224 lrmd_free_event(entry->last);
225 }
226 entry->last = lrmd_copy_event(op);
227
228 if (op->params && pcmk__strcase_any_of(op->op_type, CRMD_ACTION_START,
229 "reload", CRMD_ACTION_STATUS, NULL)) {
230 if (entry->stop_params) {
231 g_hash_table_destroy(entry->stop_params);
232 }
233 entry->stop_params = crm_str_table_new();
234
235 g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
236 }
237 }
238
239 if (op->interval_ms > 0) {
240
241 history_remove_recurring_op(entry, op);
242
243 crm_trace("Adding recurring op: " PCMK__OP_FMT,
244 op->rsc_id, op->op_type, op->interval_ms);
245 entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
246
247 } else if (entry->recurring_op_list && !pcmk__str_eq(op->op_type, RSC_STATUS, pcmk__str_casei)) {
248 crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
249 g_list_length(entry->recurring_op_list), op->rsc_id,
250 op->op_type, op->interval_ms);
251 history_free_recurring_ops(entry);
252 }
253 }
254
255
256
257
258
259
260
261
262
263
264
265
266
267 static void
268 send_task_ok_ack(lrm_state_t *lrm_state, ha_msg_input_t *input,
269 const char *rsc_id, lrmd_rsc_info_t *rsc, const char *task,
270 const char *ack_host, const char *ack_sys)
271 {
272 lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
273
274 op->rc = PCMK_OCF_OK;
275 op->op_status = PCMK_LRM_OP_DONE;
276 controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
277 lrmd_free_event(op);
278 }
279
280 static inline const char *
281 op_node_name(lrmd_event_data_t *op)
282 {
283 return op->remote_nodename? op->remote_nodename : fsa_our_uname;
284 }
285
286 void
287 lrm_op_callback(lrmd_event_data_t * op)
288 {
289 CRM_CHECK(op != NULL, return);
290 switch (op->type) {
291 case lrmd_event_disconnect:
292 if (op->remote_nodename == NULL) {
293
294
295
296 lrm_connection_destroy();
297 }
298 break;
299
300 case lrmd_event_exec_complete:
301 {
302 lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
303
304 CRM_ASSERT(lrm_state != NULL);
305 process_lrm_event(lrm_state, op, NULL, NULL);
306 }
307 break;
308
309 default:
310 break;
311 }
312 }
313
314
315 void
316 do_lrm_control(long long action,
317 enum crmd_fsa_cause cause,
318 enum crmd_fsa_state cur_state,
319 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
320 {
321
322
323
324
325
326 lrm_state_t *lrm_state = NULL;
327
328 if(fsa_our_uname == NULL) {
329 return;
330 }
331 lrm_state = lrm_state_find_or_create(fsa_our_uname);
332 if (lrm_state == NULL) {
333 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
334 return;
335 }
336
337 if (action & A_LRM_DISCONNECT) {
338 if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
339 if (action == A_LRM_DISCONNECT) {
340 crmd_fsa_stall(FALSE);
341 return;
342 }
343 }
344
345 controld_clear_fsa_input_flags(R_LRM_CONNECTED);
346 crm_info("Disconnecting from the executor");
347 lrm_state_disconnect(lrm_state);
348 lrm_state_reset_tables(lrm_state, FALSE);
349 crm_notice("Disconnected from the executor");
350 }
351
352 if (action & A_LRM_CONNECT) {
353 int ret = pcmk_ok;
354
355 crm_debug("Connecting to the executor");
356 ret = lrm_state_ipc_connect(lrm_state);
357
358 if (ret != pcmk_ok) {
359 if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
360 crm_warn("Failed to connect to the executor %d time%s (%d max)",
361 lrm_state->num_lrm_register_fails,
362 pcmk__plural_s(lrm_state->num_lrm_register_fails),
363 MAX_LRM_REG_FAILS);
364
365 controld_start_timer(wait_timer);
366 crmd_fsa_stall(FALSE);
367 return;
368 }
369 }
370
371 if (ret != pcmk_ok) {
372 crm_err("Failed to connect to the executor the max allowed %d time%s",
373 lrm_state->num_lrm_register_fails,
374 pcmk__plural_s(lrm_state->num_lrm_register_fails));
375 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
376 return;
377 }
378
379 controld_set_fsa_input_flags(R_LRM_CONNECTED);
380 crm_info("Connection to the executor established");
381 }
382
383 if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
384 crm_err("Unexpected action %s in %s", fsa_action2string(action),
385 __func__);
386 }
387 }
388
389 static gboolean
390 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
391 {
392 int counter = 0;
393 gboolean rc = TRUE;
394 const char *when = "lrm disconnect";
395
396 GHashTableIter gIter;
397 const char *key = NULL;
398 rsc_history_t *entry = NULL;
399 active_op_t *pending = NULL;
400
401 crm_debug("Checking for active resources before exit");
402
403 if (cur_state == S_TERMINATE) {
404 log_level = LOG_ERR;
405 when = "shutdown";
406
407 } else if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
408 when = "shutdown... waiting";
409 }
410
411 if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
412 guint removed = g_hash_table_foreach_remove(
413 lrm_state->pending_ops, stop_recurring_actions, lrm_state);
414 guint nremaining = g_hash_table_size(lrm_state->pending_ops);
415
416 if (removed || nremaining) {
417 crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
418 removed, pcmk__plural_s(removed), when, nremaining);
419 }
420 }
421
422 if (lrm_state->pending_ops) {
423 g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
424 while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
425
426 if (pending->interval_ms == 0) {
427 counter++;
428 }
429 }
430 }
431
432 if (counter > 0) {
433 do_crm_log(log_level, "%d pending executor operation%s at %s",
434 counter, pcmk__plural_s(counter), when);
435
436 if ((cur_state == S_TERMINATE)
437 || !pcmk_is_set(fsa_input_register, R_SENT_RSC_STOP)) {
438 g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
439 while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
440 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
441 }
442
443 } else {
444 rc = FALSE;
445 }
446 return rc;
447 }
448
449 if (lrm_state->resource_history == NULL) {
450 return rc;
451 }
452
453 if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
454
455 when = "shutdown";
456 }
457
458 counter = 0;
459 g_hash_table_iter_init(&gIter, lrm_state->resource_history);
460 while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
461 if (is_rsc_active(lrm_state, entry->id) == FALSE) {
462 continue;
463 }
464
465 counter++;
466 if (log_level == LOG_ERR) {
467 crm_info("Found %s active at %s", entry->id, when);
468 } else {
469 crm_trace("Found %s active at %s", entry->id, when);
470 }
471 if (lrm_state->pending_ops) {
472 GHashTableIter hIter;
473
474 g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
475 while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
476 if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
477 crm_notice("%sction %s (%s) incomplete at %s",
478 pending->interval_ms == 0 ? "A" : "Recurring a",
479 key, pending->op_key, when);
480 }
481 }
482 }
483 }
484
485 if (counter) {
486 crm_err("%d resource%s active at %s",
487 counter, (counter == 1)? " was" : "s were", when);
488 }
489
490 return rc;
491 }
492
493 static char *
494 build_parameter_list(const lrmd_event_data_t *op,
495 const struct ra_metadata_s *metadata,
496 xmlNode *result, enum ra_param_flags_e param_type,
497 bool invert_for_xml)
498 {
499 char *list = NULL;
500 size_t len = 0;
501 size_t max = 0;
502
503
504
505
506
507 const char *secure_terms[] = {
508 "password",
509 "passwd",
510 "user",
511 };
512
513 if (!pcmk_is_set(metadata->ra_flags, ra_uses_private)
514 && (param_type == ra_param_private)) {
515
516 max = DIMOF(secure_terms);
517 }
518
519 for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
520 struct ra_param_s *param = (struct ra_param_s *) iter->data;
521 bool accept = FALSE;
522
523 if (pcmk_is_set(param->rap_flags, param_type)) {
524 accept = TRUE;
525
526 } else if (max) {
527 for (int lpc = 0; lpc < max; lpc++) {
528 if (pcmk__str_eq(secure_terms[lpc], param->rap_name, pcmk__str_casei)) {
529 accept = TRUE;
530 break;
531 }
532 }
533 }
534
535 if (accept) {
536 crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
537
538 if (list == NULL) {
539
540 pcmk__add_word(&list, &len, " ");
541 }
542 pcmk__add_word(&list, &len, param->rap_name);
543
544 } else {
545 crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
546 }
547
548 if (result && (invert_for_xml? !accept : accept)) {
549 const char *v = g_hash_table_lookup(op->params, param->rap_name);
550
551 if (v != NULL) {
552 crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
553 crm_xml_add(result, param->rap_name, v);
554 }
555 }
556 }
557
558 if (list != NULL) {
559
560 pcmk__add_word(&list, &len, " ");
561 }
562 return list;
563 }
564
565 static void
566 append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
567 xmlNode *update, const char *version)
568 {
569 char *list = NULL;
570 char *digest = NULL;
571 xmlNode *restart = NULL;
572
573 CRM_LOG_ASSERT(op->params != NULL);
574
575 if (op->interval_ms > 0) {
576
577 return;
578 }
579
580 if (pcmk_is_set(metadata->ra_flags, ra_supports_reload)) {
581 restart = create_xml_node(NULL, XML_TAG_PARAMS);
582
583
584
585
586
587
588 list = build_parameter_list(op, metadata, restart, ra_param_unique,
589 FALSE);
590
591 } else {
592
593 return;
594 }
595
596 digest = calculate_operation_digest(restart, version);
597
598
599 crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list? list: "");
600 crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
601
602 crm_trace("%s: %s, %s", op->rsc_id, digest, list);
603 crm_log_xml_trace(restart, "restart digest source");
604
605 free_xml(restart);
606 free(digest);
607 free(list);
608 }
609
610 static void
611 append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
612 xmlNode *update, const char *version)
613 {
614 char *list = NULL;
615 char *digest = NULL;
616 xmlNode *secure = NULL;
617
618 CRM_LOG_ASSERT(op->params != NULL);
619
620
621
622
623
624
625 secure = create_xml_node(NULL, XML_TAG_PARAMS);
626 list = build_parameter_list(op, metadata, secure, ra_param_private, TRUE);
627
628 if (list != NULL) {
629 digest = calculate_operation_digest(secure, version);
630 crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, list);
631 crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
632
633 crm_trace("%s: %s, %s", op->rsc_id, digest, list);
634 crm_log_xml_trace(secure, "secure digest source");
635 } else {
636 crm_trace("%s: no secure parameters", op->rsc_id);
637 }
638
639 free_xml(secure);
640 free(digest);
641 free(list);
642 }
643
644 static gboolean
645 build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op,
646 const char *node_name, const char *src)
647 {
648 int target_rc = 0;
649 xmlNode *xml_op = NULL;
650 struct ra_metadata_s *metadata = NULL;
651 const char *caller_version = NULL;
652 lrm_state_t *lrm_state = NULL;
653
654 if (op == NULL) {
655 return FALSE;
656 }
657
658 target_rc = rsc_op_expected_rc(op);
659
660
661
662
663
664
665
666 caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
667 CRM_LOG_ASSERT(caller_version != NULL);
668
669 if(caller_version == NULL) {
670 caller_version = CRM_FEATURE_SET;
671 }
672
673 crm_trace("Building %s operation update with originator version: %s", op->rsc_id, caller_version);
674 xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
675 fsa_our_uname, src, LOG_DEBUG);
676 if (xml_op == NULL) {
677 return TRUE;
678 }
679
680 if ((rsc == NULL) || (op->params == NULL)
681 || !crm_op_needs_metadata(rsc->standard, op->op_type)) {
682
683 crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
684 op->op_type, op->rsc_id, op->params, rsc);
685 return TRUE;
686 }
687
688 lrm_state = lrm_state_find(node_name);
689 if (lrm_state == NULL) {
690 crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
691 " because we have no connection to executor for %s",
692 op->rsc_id, op->op_type, op->interval_ms, node_name);
693 return TRUE;
694 }
695
696 metadata = metadata_cache_get(lrm_state->metadata_cache, rsc);
697 if (metadata == NULL) {
698
699
700
701
702
703
704
705
706
707
708
709
710 char *metadata_str = NULL;
711
712 int rc = lrm_state_get_metadata(lrm_state, rsc->standard,
713 rsc->provider, rsc->type,
714 &metadata_str, 0);
715
716 if (rc != pcmk_ok) {
717 crm_warn("Failed to get metadata for %s (%s:%s:%s)",
718 rsc->id, rsc->standard, rsc->provider, rsc->type);
719 return TRUE;
720 }
721
722 metadata = metadata_cache_update(lrm_state->metadata_cache, rsc,
723 metadata_str);
724 free(metadata_str);
725 if (metadata == NULL) {
726 crm_warn("Failed to update metadata for %s (%s:%s:%s)",
727 rsc->id, rsc->standard, rsc->provider, rsc->type);
728 return TRUE;
729 }
730 }
731
732 #if ENABLE_VERSIONED_ATTRS
733 crm_xml_add(xml_op, XML_ATTR_RA_VERSION, metadata->ra_version);
734 #endif
735
736 crm_trace("Including additional digests for %s::%s:%s", rsc->standard, rsc->provider, rsc->type);
737 append_restart_list(op, metadata, xml_op, caller_version);
738 append_secure_list(op, metadata, xml_op, caller_version);
739
740 return TRUE;
741 }
742
743 static gboolean
744 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
745 {
746 rsc_history_t *entry = NULL;
747
748 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
749 if (entry == NULL || entry->last == NULL) {
750 return FALSE;
751 }
752
753 crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
754 entry->last->interval_ms, entry->last->rc);
755 if (entry->last->rc == PCMK_OCF_OK && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_STOP, pcmk__str_casei)) {
756 return FALSE;
757
758 } else if (entry->last->rc == PCMK_OCF_OK
759 && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
760
761 return FALSE;
762
763 } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
764 return FALSE;
765
766 } else if ((entry->last->interval_ms == 0)
767 && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
768
769 return FALSE;
770 }
771
772 return TRUE;
773 }
774
775 static gboolean
776 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
777 {
778 GHashTableIter iter;
779 rsc_history_t *entry = NULL;
780
781 g_hash_table_iter_init(&iter, lrm_state->resource_history);
782 while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
783
784 GList *gIter = NULL;
785 xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
786
787 crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
788 crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
789 crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
790 crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
791
792 if (entry->last && entry->last->params) {
793 const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
794 if (container) {
795 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
796 crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
797 }
798 }
799 build_operation_update(xml_rsc, &(entry->rsc), entry->failed, lrm_state->node_name,
800 __func__);
801 build_operation_update(xml_rsc, &(entry->rsc), entry->last, lrm_state->node_name,
802 __func__);
803 for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
804 build_operation_update(xml_rsc, &(entry->rsc), gIter->data, lrm_state->node_name,
805 __func__);
806 }
807 }
808
809 return FALSE;
810 }
811
812 static xmlNode *
813 do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags)
814 {
815 xmlNode *xml_state = NULL;
816 xmlNode *xml_data = NULL;
817 xmlNode *rsc_list = NULL;
818 crm_node_t *peer = NULL;
819
820 peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
821 CRM_CHECK(peer != NULL, return NULL);
822
823 xml_state = create_node_state_update(peer, update_flags, NULL,
824 __func__);
825 if (xml_state == NULL) {
826 return NULL;
827 }
828
829 xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
830 crm_xml_add(xml_data, XML_ATTR_ID, peer->uuid);
831 rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
832
833
834 build_active_RAs(lrm_state, rsc_list);
835
836 crm_log_xml_trace(xml_state, "Current executor state");
837
838 return xml_state;
839 }
840
841 xmlNode *
842 controld_query_executor_state(const char *node_name)
843 {
844 lrm_state_t *lrm_state = lrm_state_find(node_name);
845
846 if (!lrm_state) {
847 crm_err("Could not find executor state for node %s", node_name);
848 return NULL;
849 }
850 return do_lrm_query_internal(lrm_state,
851 node_update_cluster|node_update_peer);
852 }
853
854
855
856
857
858
859
860
861 void
862 controld_rc2event(lrmd_event_data_t *event, int rc)
863 {
864 switch (rc) {
865 case pcmk_rc_ok:
866 event->rc = PCMK_OCF_OK;
867 event->op_status = PCMK_LRM_OP_DONE;
868 break;
869 case EACCES:
870 event->rc = PCMK_OCF_INSUFFICIENT_PRIV;
871 event->op_status = PCMK_LRM_OP_ERROR;
872 break;
873 default:
874 event->rc = PCMK_OCF_UNKNOWN_ERROR;
875 event->op_status = PCMK_LRM_OP_ERROR;
876 break;
877 }
878 }
879
880
881
882
883
884
885
886
887
888
889
890
891 void
892 controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
893 {
894 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
895 char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
896
897 crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
898 update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG,
899 NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s,
900 FALSE, NULL, NULL);
901 free(now_s);
902 }
903 }
904
905 static void
906 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
907 {
908 lrmd_event_data_t *op = NULL;
909 const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
910 const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
911
912 crm_info("Notifying %s on %s that %s was%s deleted",
913 from_sys, (from_host? from_host : "localhost"), rsc_id,
914 ((rc == pcmk_ok)? "" : " not"));
915 op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
916 controld_rc2event(op, pcmk_legacy2rc(rc));
917 controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
918 lrmd_free_event(op);
919 controld_trigger_delete_refresh(from_sys, rsc_id);
920 }
921
922 static gboolean
923 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
924 {
925 struct delete_event_s *event = user_data;
926 struct pending_deletion_op_s *op = value;
927
928 if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
929 notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
930 return TRUE;
931 }
932 return FALSE;
933 }
934
935 static gboolean
936 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
937 {
938 const char *rsc = user_data;
939 active_op_t *pending = value;
940
941 if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
942 crm_info("Removing op %s:%d for deleted resource %s",
943 pending->op_key, pending->call_id, rsc);
944 return TRUE;
945 }
946 return FALSE;
947 }
948
949 static void
950 delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
951 GHashTableIter * rsc_gIter, int rc, const char *user_name)
952 {
953 struct delete_event_s event;
954
955 CRM_CHECK(rsc_id != NULL, return);
956
957 if (rc == pcmk_ok) {
958 char *rsc_id_copy = strdup(rsc_id);
959
960 if (rsc_gIter) {
961 g_hash_table_iter_remove(rsc_gIter);
962 } else {
963 g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
964 }
965 controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
966 user_name, crmd_cib_smart_opt());
967 g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
968 free(rsc_id_copy);
969 }
970
971 if (input) {
972 notify_deleted(lrm_state, input, rsc_id, rc);
973 }
974
975 event.rc = rc;
976 event.rsc = rsc_id;
977 event.lrm_state = lrm_state;
978 g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
979 }
980
981
982
983
984
985
986
987
988 static void
989 erase_lrm_history_by_op(lrm_state_t *lrm_state, lrmd_event_data_t *op)
990 {
991 xmlNode *xml_top = NULL;
992
993 CRM_CHECK(op != NULL, return);
994
995 xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
996 crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
997 crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
998
999 if (op->interval_ms > 0) {
1000 char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
1001
1002
1003 crm_xml_add(xml_top, XML_ATTR_ID, op_id);
1004 free(op_id);
1005 }
1006
1007 crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
1008 op->rsc_id, op->op_type, op->interval_ms, op->call_id);
1009
1010 fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
1011 cib_quorum_override);
1012
1013 crm_log_xml_trace(xml_top, "op:cancel");
1014 free_xml(xml_top);
1015 }
1016
1017
1018 #define XPATH_HISTORY \
1019 "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
1020 "/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
1021 "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
1022 "/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']" \
1023 "/" XML_LRM_TAG_RSC_OP
1024
1025
1026 #define XPATH_HISTORY_ID XPATH_HISTORY \
1027 "[@" XML_ATTR_ID "='%s']"
1028
1029
1030 #define XPATH_HISTORY_CALL XPATH_HISTORY \
1031 "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
1032
1033
1034 #define XPATH_HISTORY_ORIG XPATH_HISTORY \
1035 "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047 static void
1048 erase_lrm_history_by_id(lrm_state_t *lrm_state, const char *rsc_id,
1049 const char *key, const char *orig_op, int call_id)
1050 {
1051 char *op_xpath = NULL;
1052
1053 CRM_CHECK((rsc_id != NULL) && (key != NULL), return);
1054
1055 if (call_id > 0) {
1056 op_xpath = crm_strdup_printf(XPATH_HISTORY_CALL,
1057 lrm_state->node_name, rsc_id, key,
1058 call_id);
1059
1060 } else if (orig_op) {
1061 op_xpath = crm_strdup_printf(XPATH_HISTORY_ORIG,
1062 lrm_state->node_name, rsc_id, key,
1063 orig_op);
1064 } else {
1065 op_xpath = crm_strdup_printf(XPATH_HISTORY_ID,
1066 lrm_state->node_name, rsc_id, key);
1067 }
1068
1069 crm_debug("Erasing resource operation history for %s on %s (call=%d)",
1070 key, rsc_id, call_id);
1071 fsa_cib_conn->cmds->remove(fsa_cib_conn, op_xpath, NULL,
1072 cib_quorum_override | cib_xpath);
1073 free(op_xpath);
1074 }
1075
1076 static inline gboolean
1077 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
1078 {
1079 if (entry == NULL) {
1080 return FALSE;
1081 }
1082 if (op == NULL) {
1083 return TRUE;
1084 }
1085 return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
1086 && (interval_ms == entry->failed->interval_ms));
1087 }
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102 void
1103 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
1104 const char *operation, guint interval_ms)
1105 {
1106 char *op_key = NULL;
1107 char *orig_op_key = NULL;
1108 lrm_state_t *lrm_state = NULL;
1109
1110 lrm_state = lrm_state_find(node_name);
1111 if (lrm_state == NULL) {
1112 return;
1113 }
1114
1115
1116 op_key = pcmk__op_key(rsc_id, "last_failure", 0);
1117 if (operation) {
1118 orig_op_key = pcmk__op_key(rsc_id, operation, interval_ms);
1119 }
1120 erase_lrm_history_by_id(lrm_state, rsc_id, op_key, orig_op_key, 0);
1121 free(op_key);
1122 free(orig_op_key);
1123
1124
1125 if (lrm_state->resource_history) {
1126 rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
1127 rsc_id);
1128
1129 if (last_failed_matches_op(entry, operation, interval_ms)) {
1130 lrmd_free_event(entry->failed);
1131 entry->failed = NULL;
1132 }
1133 }
1134 }
1135
1136
1137 static gboolean
1138 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
1139 {
1140 int rc = pcmk_ok;
1141 char *local_key = NULL;
1142 active_op_t *pending = NULL;
1143
1144 CRM_CHECK(op != 0, return FALSE);
1145 CRM_CHECK(rsc_id != NULL, return FALSE);
1146 if (key == NULL) {
1147 local_key = make_stop_id(rsc_id, op);
1148 key = local_key;
1149 }
1150 pending = g_hash_table_lookup(lrm_state->pending_ops, key);
1151
1152 if (pending) {
1153 if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
1154 controld_set_active_op_flags(pending, active_op_remove);
1155 crm_debug("Scheduling %s for removal", key);
1156 }
1157
1158 if (pcmk_is_set(pending->flags, active_op_cancelled)) {
1159 crm_debug("Operation %s already cancelled", key);
1160 free(local_key);
1161 return FALSE;
1162 }
1163 controld_set_active_op_flags(pending, active_op_cancelled);
1164
1165 } else {
1166 crm_info("No pending op found for %s", key);
1167 free(local_key);
1168 return FALSE;
1169 }
1170
1171 crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
1172 rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
1173 pending->interval_ms);
1174 if (rc == pcmk_ok) {
1175 crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
1176 free(local_key);
1177 return TRUE;
1178 }
1179
1180 crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190 free(local_key);
1191 return FALSE;
1192 }
1193
1194 struct cancel_data {
1195 gboolean done;
1196 gboolean remove;
1197 const char *key;
1198 lrmd_rsc_info_t *rsc;
1199 lrm_state_t *lrm_state;
1200 };
1201
1202 static gboolean
1203 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
1204 {
1205 gboolean remove = FALSE;
1206 struct cancel_data *data = user_data;
1207 active_op_t *op = value;
1208
1209 if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
1210 data->done = TRUE;
1211 remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
1212 }
1213 return remove;
1214 }
1215
1216 static gboolean
1217 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
1218 {
1219 guint removed = 0;
1220 struct cancel_data data;
1221
1222 CRM_CHECK(rsc != NULL, return FALSE);
1223 CRM_CHECK(key != NULL, return FALSE);
1224
1225 data.key = key;
1226 data.rsc = rsc;
1227 data.done = FALSE;
1228 data.remove = remove;
1229 data.lrm_state = lrm_state;
1230
1231 removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
1232 crm_trace("Removed %u op cache entries, new size: %u",
1233 removed, g_hash_table_size(lrm_state->pending_ops));
1234 return data.done;
1235 }
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254 static int
1255 get_lrm_resource(lrm_state_t *lrm_state, xmlNode *rsc_xml, gboolean do_create,
1256 lrmd_rsc_info_t **rsc_info)
1257 {
1258 const char *id = ID(rsc_xml);
1259
1260 CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
1261 CRM_CHECK(id, return -EINVAL);
1262
1263 if (lrm_state_is_connected(lrm_state) == FALSE) {
1264 return -ENOTCONN;
1265 }
1266
1267 crm_trace("Retrieving resource information for %s from the executor", id);
1268 *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
1269
1270
1271 if (!*rsc_info) {
1272 const char *long_id = crm_element_value(rsc_xml, XML_ATTR_ID_LONG);
1273
1274 if (long_id) {
1275 *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
1276 }
1277 }
1278
1279 if ((*rsc_info == NULL) && do_create) {
1280 const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
1281 const char *provider = crm_element_value(rsc_xml, XML_AGENT_ATTR_PROVIDER);
1282 const char *type = crm_element_value(rsc_xml, XML_ATTR_TYPE);
1283 int rc;
1284
1285 crm_trace("Registering resource %s with the executor", id);
1286 rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
1287 lrmd_opt_drop_recurring);
1288 if (rc != pcmk_ok) {
1289 fsa_data_t *msg_data = NULL;
1290
1291 crm_err("Could not register resource %s with the executor on %s: %s "
1292 CRM_XS " rc=%d",
1293 id, lrm_state->node_name, pcmk_strerror(rc), rc);
1294
1295
1296
1297
1298
1299 if (lrm_state_is_local(lrm_state) == TRUE) {
1300 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
1301 }
1302 return rc;
1303 }
1304
1305 *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
1306 }
1307 return *rsc_info? pcmk_ok : -ENODEV;
1308 }
1309
1310 static void
1311 delete_resource(lrm_state_t * lrm_state,
1312 const char *id,
1313 lrmd_rsc_info_t * rsc,
1314 GHashTableIter * gIter,
1315 const char *sys,
1316 const char *user,
1317 ha_msg_input_t * request,
1318 gboolean unregister)
1319 {
1320 int rc = pcmk_ok;
1321
1322 crm_info("Removing resource %s from executor for %s%s%s",
1323 id, sys, (user? " as " : ""), (user? user : ""));
1324
1325 if (rsc && unregister) {
1326 rc = lrm_state_unregister_rsc(lrm_state, id, 0);
1327 }
1328
1329 if (rc == pcmk_ok) {
1330 crm_trace("Resource %s deleted from executor", id);
1331 } else if (rc == -EINPROGRESS) {
1332 crm_info("Deletion of resource '%s' from executor is pending", id);
1333 if (request) {
1334 struct pending_deletion_op_s *op = NULL;
1335 char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
1336
1337 op = calloc(1, sizeof(struct pending_deletion_op_s));
1338 op->rsc = strdup(rsc->id);
1339 op->input = copy_ha_msg_input(request);
1340 g_hash_table_insert(lrm_state->deletion_ops, ref, op);
1341 }
1342 return;
1343 } else {
1344 crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
1345 CRM_XS " rc=%d", id, sys, (user? " as " : ""),
1346 (user? user : ""), pcmk_strerror(rc), rc);
1347 }
1348
1349 delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
1350 }
1351
1352 static int
1353 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
1354 {
1355 int call_id = 999999999;
1356 rsc_history_t *entry = NULL;
1357
1358 if(lrm_state) {
1359 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
1360 }
1361
1362
1363
1364
1365 if (entry) {
1366 call_id = entry->last_callid + 1;
1367 }
1368
1369 if (call_id < 0) {
1370 call_id = 1;
1371 }
1372 return call_id;
1373 }
1374
1375 static void
1376 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
1377 enum ocf_exitcode op_exitcode)
1378 {
1379 op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
1380 op->t_run = time(NULL);
1381 op->t_rcchange = op->t_run;
1382 op->op_status = op_status;
1383 op->rc = op_exitcode;
1384 }
1385
1386 static void
1387 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
1388 const char *from_host, const char *user_name,
1389 gboolean is_remote_node)
1390 {
1391 GHashTableIter gIter;
1392 rsc_history_t *entry = NULL;
1393
1394 crm_info("Clearing resource history on node %s", lrm_state->node_name);
1395 g_hash_table_iter_init(&gIter, lrm_state->resource_history);
1396 while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
1397
1398
1399
1400 gboolean unregister = TRUE;
1401
1402 if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
1403 lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
1404 if (remote_lrm_state) {
1405
1406
1407 force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE);
1408 }
1409 unregister = FALSE;
1410 }
1411
1412 delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
1413 user_name, NULL, unregister);
1414 }
1415
1416
1417 controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
1418 cib_scope_local);
1419
1420
1421
1422
1423 update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
1424 }
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439 static void
1440 synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action,
1441 int op_status, enum ocf_exitcode rc)
1442 {
1443 lrmd_event_data_t *op = NULL;
1444 const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
1445 const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
1446 xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
1447
1448 if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) {
1449
1450 crm_info("Can't fake %s failure (%d) on %s without resource configuration",
1451 crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
1452 target_node);
1453 return;
1454
1455 } else if(operation == NULL) {
1456
1457 crm_info("Can't fake %s failure (%d) on %s without operation",
1458 ID(xml_rsc), rc, target_node);
1459 return;
1460 }
1461
1462 op = construct_op(lrm_state, action, ID(xml_rsc), operation);
1463
1464 if (pcmk__str_eq(operation, RSC_NOTIFY, pcmk__str_casei)) {
1465 fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_OK);
1466 } else {
1467 fake_op_status(lrm_state, op, op_status, rc);
1468 }
1469
1470 crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
1471 op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
1472
1473
1474 process_lrm_event(lrm_state, op, NULL, action);
1475 lrmd_free_event(op);
1476 }
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486 static const char *
1487 lrm_op_target(xmlNode *xml)
1488 {
1489 const char *target = NULL;
1490
1491 if (xml) {
1492 target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
1493 }
1494 if (target == NULL) {
1495 target = fsa_our_uname;
1496 }
1497 return target;
1498 }
1499
1500 static void
1501 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
1502 const char *from_host, const char *from_sys)
1503 {
1504 lrmd_event_data_t *op = NULL;
1505 lrmd_rsc_info_t *rsc = NULL;
1506 xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);
1507
1508 CRM_CHECK(xml_rsc != NULL, return);
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518 op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");
1519 fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
1520
1521 free((char*) op->user_data);
1522 op->user_data = NULL;
1523 op->interval_ms = 0;
1524
1525 #if ENABLE_ACL
1526 if (user_name && !pcmk__is_privileged(user_name)) {
1527 crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
1528 controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
1529 lrmd_free_event(op);
1530 return;
1531 }
1532 #endif
1533
1534 if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
1535 crm_info("Failing resource %s...", rsc->id);
1536 op->exit_reason = strdup("Simulated failure");
1537 process_lrm_event(lrm_state, op, NULL, xml);
1538 op->op_status = PCMK_LRM_OP_DONE;
1539 op->rc = PCMK_OCF_OK;
1540 lrmd_free_rsc_info(rsc);
1541
1542 } else {
1543 crm_info("Cannot find/create resource in order to fail it...");
1544 crm_log_xml_warn(xml, "bad input");
1545 }
1546
1547 controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
1548 lrmd_free_event(op);
1549 }
1550
1551 static void
1552 handle_refresh_op(lrm_state_t *lrm_state, const char *user_name,
1553 const char *from_host, const char *from_sys)
1554 {
1555 int rc = pcmk_ok;
1556 xmlNode *fragment = do_lrm_query_internal(lrm_state, node_update_all);
1557
1558 fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
1559 crm_info("Forced a local resource history refresh: call=%d", rc);
1560
1561 if (!pcmk__str_eq(CRM_SYSTEM_CRMD, from_sys, pcmk__str_casei)) {
1562 xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, fragment, from_host,
1563 from_sys, CRM_SYSTEM_LRMD,
1564 fsa_our_uuid);
1565
1566 crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);
1567
1568 if (relay_message(reply, TRUE) == FALSE) {
1569 crm_log_xml_err(reply, "Unable to route reply");
1570 }
1571 free_xml(reply);
1572 }
1573
1574 free_xml(fragment);
1575 }
1576
1577 static void
1578 handle_query_op(xmlNode *msg, lrm_state_t *lrm_state)
1579 {
1580 xmlNode *data = do_lrm_query_internal(lrm_state, node_update_all);
1581 xmlNode *reply = create_reply(msg, data);
1582
1583 if (relay_message(reply, TRUE) == FALSE) {
1584 crm_err("Unable to route reply");
1585 crm_log_xml_err(reply, "reply");
1586 }
1587 free_xml(reply);
1588 free_xml(data);
1589 }
1590
1591 static void
1592 handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
1593 const char *from_host, const char *user_name,
1594 gboolean is_remote_node)
1595 {
1596 crm_notice("Forcing the status of all resources to be redetected");
1597 force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node);
1598
1599 if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
1600
1601 xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
1602 from_sys, CRM_SYSTEM_LRMD,
1603 fsa_our_uuid);
1604
1605 crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
1606
1607 if (relay_message(reply, TRUE) == FALSE) {
1608 crm_log_xml_err(reply, "Unable to route reply");
1609 }
1610 free_xml(reply);
1611 }
1612 }
1613
1614 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
1615 lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
1616 {
1617 char *op_key = NULL;
1618 char *meta_key = NULL;
1619 int call = 0;
1620 const char *call_id = NULL;
1621 const char *op_task = NULL;
1622 guint interval_ms = 0;
1623 gboolean in_progress = FALSE;
1624 xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
1625
1626 CRM_CHECK(params != NULL, return FALSE);
1627
1628 meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
1629 op_task = crm_element_value(params, meta_key);
1630 free(meta_key);
1631 CRM_CHECK(op_task != NULL, return FALSE);
1632
1633 meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
1634 if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
1635 free(meta_key);
1636 return FALSE;
1637 }
1638 free(meta_key);
1639
1640 op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
1641
1642 meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
1643 call_id = crm_element_value(params, meta_key);
1644 free(meta_key);
1645
1646 crm_debug("Scheduler requested op %s (call=%s) be cancelled",
1647 op_key, (call_id? call_id : "NA"));
1648 call = crm_parse_int(call_id, "0");
1649 if (call == 0) {
1650
1651 in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
1652
1653 } else {
1654
1655 in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
1656 }
1657
1658
1659 if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1660 char *op_id = make_stop_id(rsc->id, call);
1661
1662 if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
1663 crm_info("Nothing known about operation %d for %s", call, op_key);
1664 }
1665 erase_lrm_history_by_id(lrm_state, rsc->id, op_key, NULL, call);
1666 send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
1667 from_host, from_sys);
1668
1669
1670 g_hash_table_remove(lrm_state->pending_ops, op_id);
1671 free(op_id);
1672
1673 } else {
1674
1675
1676
1677
1678
1679
1680
1681 const char *peer_version = crm_element_value(params, XML_ATTR_CRM_VERSION);
1682
1683 if (compare_version(peer_version, "3.0.8") <= 0) {
1684 crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
1685 op_key, from_host, peer_version);
1686 send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
1687 from_host, from_sys);
1688 }
1689 }
1690
1691 free(op_key);
1692 return TRUE;
1693 }
1694
1695 static void
1696 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
1697 lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
1698 bool crm_rsc_delete, const char *user_name)
1699 {
1700 gboolean unregister = TRUE;
1701
1702 #if ENABLE_ACL
1703 int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
1704 user_name,
1705 cib_dryrun|cib_sync_call);
1706
1707 if (cib_rc != pcmk_rc_ok) {
1708 lrmd_event_data_t *op = NULL;
1709
1710 op = construct_op(lrm_state, input->xml, rsc->id, CRMD_ACTION_DELETE);
1711 op->op_status = PCMK_LRM_OP_ERROR;
1712
1713 if (cib_rc == EACCES) {
1714 op->rc = PCMK_OCF_INSUFFICIENT_PRIV;
1715 } else {
1716 op->rc = PCMK_OCF_UNKNOWN_ERROR;
1717 }
1718 controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
1719 lrmd_free_event(op);
1720 return;
1721 }
1722 #endif
1723
1724 if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1725 unregister = FALSE;
1726 }
1727
1728 delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
1729 user_name, input, unregister);
1730 }
1731
1732
1733 void
1734 do_lrm_invoke(long long action,
1735 enum crmd_fsa_cause cause,
1736 enum crmd_fsa_state cur_state,
1737 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
1738 {
1739 lrm_state_t *lrm_state = NULL;
1740 const char *crm_op = NULL;
1741 const char *from_sys = NULL;
1742 const char *from_host = NULL;
1743 const char *operation = NULL;
1744 ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
1745 const char *user_name = NULL;
1746 const char *target_node = NULL;
1747 gboolean is_remote_node = FALSE;
1748 bool crm_rsc_delete = FALSE;
1749
1750 target_node = lrm_op_target(input->xml);
1751 is_remote_node = !pcmk__str_eq(target_node, fsa_our_uname,
1752 pcmk__str_casei);
1753
1754 lrm_state = lrm_state_find(target_node);
1755 if ((lrm_state == NULL) && is_remote_node) {
1756 crm_err("Failing action because local node has never had connection to remote node %s",
1757 target_node);
1758 synthesize_lrmd_failure(NULL, input->xml, PCMK_LRM_OP_NOT_CONNECTED,
1759 PCMK_OCF_UNKNOWN_ERROR);
1760 return;
1761 }
1762 CRM_ASSERT(lrm_state != NULL);
1763
1764 #if ENABLE_ACL
1765 user_name = pcmk__update_acl_user(input->msg, F_CRM_USER, NULL);
1766 #endif
1767
1768 crm_op = crm_element_value(input->msg, F_CRM_TASK);
1769 from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
1770 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
1771 from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
1772 }
1773 #if ENABLE_ACL
1774 crm_trace("Executor %s command from %s as user %s",
1775 crm_op, from_sys, user_name);
1776 #else
1777 crm_trace("Executor %s command from %s",
1778 crm_op, from_sys);
1779 #endif
1780
1781 if (pcmk__str_eq(crm_op, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
1782 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
1783 crm_rsc_delete = TRUE;
1784 }
1785 operation = CRMD_ACTION_DELETE;
1786
1787 } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_casei)) {
1788 fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
1789 from_sys);
1790 return;
1791
1792 } else if (input->xml != NULL) {
1793 operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
1794 }
1795
1796 if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
1797 handle_refresh_op(lrm_state, user_name, from_host, from_sys);
1798
1799 } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_QUERY, pcmk__str_casei)) {
1800 handle_query_op(input->msg, lrm_state);
1801
1802 } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_casei)) {
1803 update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
1804 user_name, is_remote_node);
1805
1806 } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_casei)
1807 || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_casei)) {
1808 handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
1809 is_remote_node);
1810
1811 } else if (operation != NULL) {
1812 lrmd_rsc_info_t *rsc = NULL;
1813 xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
1814 gboolean create_rsc = !pcmk__str_eq(operation, CRMD_ACTION_DELETE,
1815 pcmk__str_casei);
1816 int rc;
1817
1818
1819 CRM_CHECK(xml_rsc && ID(xml_rsc), return);
1820
1821 rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
1822 if (rc == -ENOTCONN) {
1823 synthesize_lrmd_failure(lrm_state, input->xml,
1824 PCMK_LRM_OP_NOT_CONNECTED,
1825 PCMK_OCF_UNKNOWN_ERROR);
1826 return;
1827
1828 } else if ((rc < 0) && !create_rsc) {
1829
1830
1831
1832 crm_notice("Not registering resource '%s' for a %s event "
1833 CRM_XS " get-rc=%d (%s) transition-key=%s",
1834 ID(xml_rsc), operation,
1835 rc, pcmk_strerror(rc), ID(input->xml));
1836 delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
1837 user_name);
1838 return;
1839
1840 } else if (rc == -EINVAL) {
1841
1842 crm_err("Invalid resource definition for %s", ID(xml_rsc));
1843 crm_log_xml_warn(input->msg, "invalid resource");
1844 synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR,
1845 PCMK_OCF_NOT_CONFIGURED);
1846 return;
1847
1848 } else if (rc < 0) {
1849
1850 crm_err("Could not register resource '%s' with executor: %s "
1851 CRM_XS " rc=%d",
1852 ID(xml_rsc), pcmk_strerror(rc), rc);
1853 crm_log_xml_warn(input->msg, "failed registration");
1854 synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR,
1855 PCMK_OCF_INVALID_PARAM);
1856 return;
1857 }
1858
1859 if (pcmk__str_eq(operation, CRMD_ACTION_CANCEL, pcmk__str_casei)) {
1860 if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
1861 crm_log_xml_warn(input->xml, "Bad command");
1862 }
1863
1864 } else if (pcmk__str_eq(operation, CRMD_ACTION_DELETE, pcmk__str_casei)) {
1865 do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
1866 crm_rsc_delete, user_name);
1867
1868 } else {
1869 do_lrm_rsc_op(lrm_state, rsc, operation, input->xml);
1870 }
1871
1872 lrmd_free_rsc_info(rsc);
1873
1874 } else {
1875 crm_err("Cannot perform operation %s of unknown type", crm_str(crm_op));
1876 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
1877 }
1878 }
1879
1880 #if ENABLE_VERSIONED_ATTRS
1881 static void
1882 resolve_versioned_parameters(lrm_state_t *lrm_state, const char *rsc_id,
1883 const xmlNode *rsc_op, GHashTable *params)
1884 {
1885
1886
1887 lrmd_rsc_info_t *rsc = lrm_state_get_rsc_info(lrm_state, rsc_id, 0);
1888 struct ra_metadata_s *metadata;
1889
1890 metadata = metadata_cache_get(lrm_state->metadata_cache, rsc);
1891 if (metadata) {
1892 xmlNode *versioned_attrs = NULL;
1893 GHashTable *hash = NULL;
1894 char *key = NULL;
1895 char *value = NULL;
1896 GHashTableIter iter;
1897
1898 versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_ATTRS);
1899 hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
1900 g_hash_table_iter_init(&iter, hash);
1901 while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
1902 g_hash_table_iter_steal(&iter);
1903 g_hash_table_replace(params, key, value);
1904 }
1905 g_hash_table_destroy(hash);
1906
1907 versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_META);
1908 hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
1909 g_hash_table_iter_init(&iter, hash);
1910 while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
1911 g_hash_table_replace(params, crm_meta_name(key), strdup(value));
1912
1913 if (pcmk__str_eq(key, XML_ATTR_TIMEOUT, pcmk__str_casei)) {
1914 op->timeout = crm_parse_int(value, "0");
1915 } else if (pcmk__str_eq(key, XML_OP_ATTR_START_DELAY, pcmk__str_casei)) {
1916 op->start_delay = crm_parse_int(value, "0");
1917 }
1918 }
1919 g_hash_table_destroy(hash);
1920
1921 versioned_attrs = first_named_child(rsc_op, XML_TAG_RSC_VER_ATTRS);
1922 hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
1923 g_hash_table_iter_init(&iter, hash);
1924 while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
1925 g_hash_table_iter_steal(&iter);
1926 g_hash_table_replace(params, key, value);
1927 }
1928 g_hash_table_destroy(hash);
1929 }
1930
1931 lrmd_free_rsc_info(rsc);
1932 }
1933 #endif
1934
1935 static lrmd_event_data_t *
1936 construct_op(lrm_state_t *lrm_state, xmlNode *rsc_op, const char *rsc_id,
1937 const char *operation)
1938 {
1939 lrmd_event_data_t *op = NULL;
1940 const char *op_delay = NULL;
1941 const char *op_timeout = NULL;
1942 GHashTable *params = NULL;
1943
1944 xmlNode *primitive = NULL;
1945 const char *class = NULL;
1946
1947 const char *transition = NULL;
1948
1949 CRM_ASSERT(rsc_id && operation);
1950
1951 op = lrmd_new_event(rsc_id, operation, 0);
1952 op->type = lrmd_event_exec_complete;
1953 op->op_status = PCMK_LRM_OP_PENDING;
1954 op->rc = -1;
1955 op->timeout = 0;
1956 op->start_delay = 0;
1957
1958 if (rsc_op == NULL) {
1959 CRM_LOG_ASSERT(pcmk__str_eq(CRMD_ACTION_STOP, operation, pcmk__str_casei));
1960 op->user_data = NULL;
1961
1962
1963
1964
1965
1966 op->params = crm_str_table_new();
1967
1968 g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
1969
1970 crm_trace("Constructed %s op for %s", operation, rsc_id);
1971 return op;
1972 }
1973
1974 params = xml2list(rsc_op);
1975 g_hash_table_remove(params, CRM_META "_op_target_rc");
1976
1977 op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
1978 op->start_delay = crm_parse_int(op_delay, "0");
1979
1980 op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
1981 op->timeout = crm_parse_int(op_timeout, "0");
1982
1983 if (pcmk__guint_from_hash(params, CRM_META "_" XML_LRM_ATTR_INTERVAL_MS, 0,
1984 &(op->interval_ms)) != pcmk_rc_ok) {
1985 op->interval_ms = 0;
1986 }
1987
1988
1989
1990 primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE);
1991 class = crm_element_value(primitive, XML_AGENT_ATTR_CLASS);
1992
1993 if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
1994 && pcmk__str_eq(operation, CRMD_ACTION_STATUS, pcmk__str_casei)
1995 && (op->interval_ms > 0)) {
1996
1997 op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
1998 if (op_timeout != NULL) {
1999 op->timeout = crm_get_msec(op_timeout);
2000 }
2001 }
2002
2003 #if ENABLE_VERSIONED_ATTRS
2004 if (lrm_state && !is_remote_lrmd_ra(NULL, NULL, rsc_id)
2005 && !pcmk__strcase_any_of(op_type, CRMD_ACTION_METADATA, CRMD_ACTION_DELETE,
2006 NULL)) {
2007 resolve_versioned_parameters(lrm_state, rsc_id, rsc_op, params);
2008 }
2009 #endif
2010
2011 if (!pcmk__str_eq(operation, RSC_STOP, pcmk__str_casei)) {
2012 op->params = params;
2013
2014 } else {
2015 rsc_history_t *entry = NULL;
2016
2017 if (lrm_state) {
2018 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
2019 }
2020
2021
2022
2023 if (!entry || !entry->stop_params) {
2024 op->params = params;
2025 } else {
2026
2027
2028 op->params = crm_str_table_new();
2029
2030 g_hash_table_foreach(params, copy_meta_keys, op->params);
2031 g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
2032 g_hash_table_destroy(params);
2033 params = NULL;
2034 }
2035 }
2036
2037
2038 if (op->timeout <= 0) {
2039 op->timeout = op->interval_ms;
2040 }
2041 if (op->start_delay < 0) {
2042 op->start_delay = 0;
2043 }
2044
2045 transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
2046 CRM_CHECK(transition != NULL, return op);
2047
2048 op->user_data = strdup(transition);
2049
2050 if (op->interval_ms != 0) {
2051 if (pcmk__strcase_any_of(operation, CRMD_ACTION_START, CRMD_ACTION_STOP, NULL)) {
2052 crm_err("Start and Stop actions cannot have an interval: %u",
2053 op->interval_ms);
2054 op->interval_ms = 0;
2055 }
2056 }
2057
2058 crm_trace("Constructed %s op for %s: interval=%u",
2059 operation, rsc_id, op->interval_ms);
2060
2061 return op;
2062 }
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077 void
2078 controld_ack_event_directly(const char *to_host, const char *to_sys,
2079 lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
2080 const char *rsc_id)
2081 {
2082 xmlNode *reply = NULL;
2083 xmlNode *update, *iter;
2084 crm_node_t *peer = NULL;
2085
2086 CRM_CHECK(op != NULL, return);
2087 if (op->rsc_id == NULL) {
2088 CRM_ASSERT(rsc_id != NULL);
2089 op->rsc_id = strdup(rsc_id);
2090 }
2091 if (to_sys == NULL) {
2092 to_sys = CRM_SYSTEM_TENGINE;
2093 }
2094
2095 peer = crm_get_peer(0, fsa_our_uname);
2096 update = create_node_state_update(peer, node_update_none, NULL,
2097 __func__);
2098
2099 iter = create_xml_node(update, XML_CIB_TAG_LRM);
2100 crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
2101 iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
2102 iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
2103
2104 crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
2105
2106 build_operation_update(iter, rsc, op, fsa_our_uname, __func__);
2107 reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
2108
2109 crm_log_xml_trace(update, "[direct ACK]");
2110
2111 crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
2112 op->rsc_id, op->op_type, op->interval_ms, op->user_data,
2113 crm_element_value(reply, XML_ATTR_REFERENCE));
2114
2115 if (relay_message(reply, TRUE) == FALSE) {
2116 crm_log_xml_err(reply, "Unable to route reply");
2117 }
2118
2119 free_xml(update);
2120 free_xml(reply);
2121 }
2122
2123 gboolean
2124 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
2125 {
2126 gboolean res = TRUE;
2127 GList *lrm_state_list = lrm_state_get_list();
2128 GList *state_entry;
2129
2130 for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
2131 lrm_state_t *lrm_state = state_entry->data;
2132
2133 if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
2134
2135 res = FALSE;
2136 }
2137 }
2138
2139 controld_set_fsa_input_flags(R_SENT_RSC_STOP);
2140 g_list_free(lrm_state_list); lrm_state_list = NULL;
2141 return res;
2142 }
2143
2144 struct stop_recurring_action_s {
2145 lrmd_rsc_info_t *rsc;
2146 lrm_state_t *lrm_state;
2147 };
2148
2149 static gboolean
2150 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
2151 {
2152 gboolean remove = FALSE;
2153 struct stop_recurring_action_s *event = user_data;
2154 active_op_t *op = value;
2155
2156 if ((op->interval_ms != 0)
2157 && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
2158
2159 crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
2160 remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
2161 }
2162
2163 return remove;
2164 }
2165
2166 static gboolean
2167 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
2168 {
2169 gboolean remove = FALSE;
2170 lrm_state_t *lrm_state = user_data;
2171 active_op_t *op = value;
2172
2173 if (op->interval_ms != 0) {
2174 crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
2175 (const char *) key);
2176 remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
2177 }
2178
2179 return remove;
2180 }
2181
2182 static void
2183 record_pending_op(const char *node_name, lrmd_rsc_info_t *rsc, lrmd_event_data_t *op)
2184 {
2185 const char *record_pending = NULL;
2186
2187 CRM_CHECK(node_name != NULL, return);
2188 CRM_CHECK(rsc != NULL, return);
2189 CRM_CHECK(op != NULL, return);
2190
2191
2192 if ((op->op_type == NULL) || (op->params == NULL)
2193 || !controld_action_is_recordable(op->op_type)) {
2194 return;
2195 }
2196
2197
2198 record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
2199 if (record_pending && !crm_is_true(record_pending)) {
2200 return;
2201 }
2202
2203 op->call_id = -1;
2204 op->op_status = PCMK_LRM_OP_PENDING;
2205 op->rc = PCMK_OCF_UNKNOWN;
2206
2207 op->t_run = time(NULL);
2208 op->t_rcchange = op->t_run;
2209
2210
2211 crm_debug("Recording pending op " PCMK__OP_FMT " on %s in the CIB",
2212 op->rsc_id, op->op_type, op->interval_ms, node_name);
2213
2214 do_update_resource(node_name, rsc, op, 0);
2215 }
2216
2217 static void
2218 do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
2219 const char *operation, xmlNode *msg)
2220 {
2221 int call_id = 0;
2222 char *op_id = NULL;
2223 lrmd_event_data_t *op = NULL;
2224 lrmd_key_value_t *params = NULL;
2225 fsa_data_t *msg_data = NULL;
2226 const char *transition = NULL;
2227 gboolean stop_recurring = FALSE;
2228 bool send_nack = FALSE;
2229
2230 CRM_CHECK(rsc != NULL, return);
2231 CRM_CHECK(operation != NULL, return);
2232
2233 if (msg != NULL) {
2234 transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
2235 if (transition == NULL) {
2236 crm_log_xml_err(msg, "Missing transition number");
2237 }
2238 }
2239
2240 op = construct_op(lrm_state, msg, rsc->id, operation);
2241 CRM_CHECK(op != NULL, return);
2242
2243 if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
2244 && (op->interval_ms == 0)
2245 && strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {
2246
2247
2248
2249
2250
2251 stop_recurring = FALSE;
2252
2253 } else if ((op->interval_ms == 0)
2254 && strcmp(operation, CRMD_ACTION_STATUS) != 0
2255 && strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {
2256
2257
2258 stop_recurring = TRUE;
2259 }
2260
2261 if (stop_recurring == TRUE) {
2262 guint removed = 0;
2263 struct stop_recurring_action_s data;
2264
2265 data.rsc = rsc;
2266 data.lrm_state = lrm_state;
2267 removed = g_hash_table_foreach_remove(
2268 lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);
2269
2270 if (removed) {
2271 crm_debug("Stopped %u recurring operation%s in preparation for "
2272 PCMK__OP_FMT, removed, pcmk__plural_s(removed),
2273 rsc->id, operation, op->interval_ms);
2274 }
2275 }
2276
2277
2278 crm_notice("Requesting local execution of %s operation for %s on %s "
2279 CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
2280 crm_action_str(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name,
2281 transition, rsc->id, operation, op->interval_ms);
2282
2283 if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)
2284 && pcmk__str_eq(operation, RSC_START, pcmk__str_casei)) {
2285
2286 register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
2287 send_nack = TRUE;
2288
2289 } else if (fsa_state != S_NOT_DC
2290 && fsa_state != S_POLICY_ENGINE
2291 && fsa_state != S_TRANSITION_ENGINE
2292 && !pcmk__str_eq(operation, CRMD_ACTION_STOP, pcmk__str_casei)) {
2293 send_nack = TRUE;
2294 }
2295
2296 if(send_nack) {
2297 crm_notice("Discarding attempt to perform action %s on %s in state %s (shutdown=%s)",
2298 operation, rsc->id, fsa_state2string(fsa_state),
2299 pcmk__btoa(pcmk_is_set(fsa_input_register, R_SHUTDOWN)));
2300
2301 op->rc = PCMK_OCF_UNKNOWN_ERROR;
2302 op->op_status = PCMK_LRM_OP_INVALID;
2303 controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
2304 lrmd_free_event(op);
2305 free(op_id);
2306 return;
2307 }
2308
2309 record_pending_op(lrm_state->node_name, rsc, op);
2310
2311 op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
2312
2313 if (op->interval_ms > 0) {
2314
2315 cancel_op_key(lrm_state, rsc, op_id, FALSE);
2316 }
2317
2318 if (op->params) {
2319 char *key = NULL;
2320 char *value = NULL;
2321 GHashTableIter iter;
2322
2323 g_hash_table_iter_init(&iter, op->params);
2324 while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
2325 params = lrmd_key_value_add(params, key, value);
2326 }
2327 }
2328
2329 call_id = lrm_state_exec(lrm_state, rsc->id, op->op_type, op->user_data,
2330 op->interval_ms, op->timeout, op->start_delay,
2331 params);
2332
2333 if (call_id <= 0 && lrm_state_is_local(lrm_state)) {
2334 crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
2335 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
2336
2337 } else if (call_id <= 0) {
2338 crm_err("Operation %s on resource %s failed to execute on remote node %s: %d",
2339 operation, rsc->id, lrm_state->node_name, call_id);
2340 fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
2341 process_lrm_event(lrm_state, op, NULL, NULL);
2342
2343 } else {
2344
2345
2346
2347 char *call_id_s = make_stop_id(rsc->id, call_id);
2348 active_op_t *pending = NULL;
2349
2350 pending = calloc(1, sizeof(active_op_t));
2351 crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
2352
2353 pending->call_id = call_id;
2354 pending->interval_ms = op->interval_ms;
2355 pending->op_type = strdup(operation);
2356 pending->op_key = strdup(op_id);
2357 pending->rsc_id = strdup(rsc->id);
2358 pending->start_time = time(NULL);
2359 pending->user_data = op->user_data? strdup(op->user_data) : NULL;
2360 if (crm_element_value_epoch(msg, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
2361 &(pending->lock_time)) != pcmk_ok) {
2362 pending->lock_time = 0;
2363 }
2364 g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);
2365
2366 if ((op->interval_ms > 0)
2367 && (op->start_delay > START_DELAY_THRESHOLD)) {
2368 int target_rc = 0;
2369
2370 crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
2371 decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
2372 op->rc = target_rc;
2373 op->op_status = PCMK_LRM_OP_DONE;
2374 controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
2375 }
2376
2377 pending->params = op->params;
2378 op->params = NULL;
2379 }
2380
2381 free(op_id);
2382 lrmd_free_event(op);
2383 return;
2384 }
2385
2386 int last_resource_update = 0;
2387
2388 static void
2389 cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
2390 {
2391 switch (rc) {
2392 case pcmk_ok:
2393 case -pcmk_err_diff_failed:
2394 case -pcmk_err_diff_resync:
2395 crm_trace("Resource update %d complete: rc=%d", call_id, rc);
2396 break;
2397 default:
2398 crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
2399 }
2400
2401 if (call_id == last_resource_update) {
2402 last_resource_update = 0;
2403 trigger_fsa();
2404 }
2405 }
2406
2407
2408
2409
2410
2411 static bool
2412 should_preserve_lock(lrmd_event_data_t *op)
2413 {
2414 if (!controld_shutdown_lock_enabled) {
2415 return false;
2416 }
2417 if (!strcmp(op->op_type, RSC_STOP) && (op->rc == PCMK_OCF_OK)) {
2418 return true;
2419 }
2420 if (!strcmp(op->op_type, RSC_STATUS) && (op->rc == PCMK_OCF_NOT_RUNNING)) {
2421 return true;
2422 }
2423 return false;
2424 }
2425
2426 static int
2427 do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
2428 lrmd_event_data_t *op, time_t lock_time)
2429 {
2430
2431
2432
2433
2434
2435
2436
2437
2438 int rc = pcmk_ok;
2439 xmlNode *update, *iter = NULL;
2440 int call_opt = crmd_cib_smart_opt();
2441 const char *uuid = NULL;
2442
2443 CRM_CHECK(op != NULL, return 0);
2444
2445 iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
2446 update = iter;
2447 iter = create_xml_node(iter, XML_CIB_TAG_STATE);
2448
2449 if (pcmk__str_eq(node_name, fsa_our_uname, pcmk__str_casei)) {
2450 uuid = fsa_our_uuid;
2451
2452 } else {
2453
2454 uuid = node_name;
2455 crm_xml_add(iter, XML_NODE_IS_REMOTE, "true");
2456 }
2457
2458 CRM_LOG_ASSERT(uuid != NULL);
2459 if(uuid == NULL) {
2460 rc = -EINVAL;
2461 goto done;
2462 }
2463
2464 crm_xml_add(iter, XML_ATTR_UUID, uuid);
2465 crm_xml_add(iter, XML_ATTR_UNAME, node_name);
2466 crm_xml_add(iter, XML_ATTR_ORIGIN, __func__);
2467
2468 iter = create_xml_node(iter, XML_CIB_TAG_LRM);
2469 crm_xml_add(iter, XML_ATTR_ID, uuid);
2470
2471 iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
2472 iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
2473 crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
2474
2475 build_operation_update(iter, rsc, op, node_name, __func__);
2476
2477 if (rsc) {
2478 const char *container = NULL;
2479
2480 crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
2481 crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->standard);
2482 crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);
2483 if (lock_time != 0) {
2484
2485
2486
2487 if (!should_preserve_lock(op)) {
2488 lock_time = 0;
2489 }
2490 crm_xml_add_ll(iter, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
2491 (long long) lock_time);
2492 }
2493
2494 if (op->params) {
2495 container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
2496 }
2497 if (container) {
2498 crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
2499 crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
2500 }
2501
2502 } else {
2503 crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
2504 controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
2505 goto cleanup;
2506 }
2507
2508 crm_log_xml_trace(update, __func__);
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527 fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);
2528
2529 if (rc > 0) {
2530 last_resource_update = rc;
2531 }
2532 done:
2533
2534 crm_trace("Sent resource state update message: %d for %s=%u on %s",
2535 rc, op->op_type, op->interval_ms, op->rsc_id);
2536 fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);
2537
2538 cleanup:
2539 free_xml(update);
2540 return rc;
2541 }
2542
2543 void
2544 do_lrm_event(long long action,
2545 enum crmd_fsa_cause cause,
2546 enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
2547 {
2548 CRM_CHECK(FALSE, return);
2549 }
2550
2551 static char *
2552 unescape_newlines(const char *string)
2553 {
2554 char *pch = NULL;
2555 char *ret = NULL;
2556 static const char *escaped_newline = "\\n";
2557
2558 if (!string) {
2559 return NULL;
2560 }
2561
2562 ret = strdup(string);
2563 pch = strstr(ret, escaped_newline);
2564 while (pch != NULL) {
2565
2566
2567
2568 pch[0] = '\n';
2569 pch[1] = ' ';
2570 pch = strstr(pch, escaped_newline);
2571 }
2572
2573 return ret;
2574 }
2575
2576 static bool
2577 did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
2578 const char * op_type, guint interval_ms)
2579 {
2580 rsc_history_t *entry = NULL;
2581
2582 CRM_CHECK(lrm_state != NULL, return FALSE);
2583 CRM_CHECK(rsc_id != NULL, return FALSE);
2584 CRM_CHECK(op_type != NULL, return FALSE);
2585
2586 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
2587 if (entry == NULL || entry->failed == NULL) {
2588 return FALSE;
2589 }
2590
2591 if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
2592 && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
2593 && entry->failed->interval_ms == interval_ms) {
2594 return TRUE;
2595 }
2596
2597 return FALSE;
2598 }
2599
2600 void
2601 process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
2602 active_op_t *pending, xmlNode *action_xml)
2603 {
2604 char *op_id = NULL;
2605 char *op_key = NULL;
2606
2607 int update_id = 0;
2608 gboolean remove = FALSE;
2609 gboolean removed = FALSE;
2610 bool need_direct_ack = FALSE;
2611 lrmd_rsc_info_t *rsc = NULL;
2612 const char *node_name = NULL;
2613
2614 CRM_CHECK(op != NULL, return);
2615 CRM_CHECK(op->rsc_id != NULL, return);
2616
2617
2618 if (compare_version(fsa_our_dc_version, "3.2.0") < 0) {
2619 switch (op->op_status) {
2620 case PCMK_LRM_OP_NOT_CONNECTED:
2621 op->op_status = PCMK_LRM_OP_ERROR;
2622 op->rc = PCMK_OCF_CONNECTION_DIED;
2623 break;
2624 case PCMK_LRM_OP_INVALID:
2625 op->op_status = PCMK_LRM_OP_ERROR;
2626 op->rc = CRM_DIRECT_NACK_RC;
2627 break;
2628 default:
2629 break;
2630 }
2631 }
2632
2633 op_id = make_stop_id(op->rsc_id, op->call_id);
2634 op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
2635
2636
2637 if (lrm_state) {
2638 rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
2639 }
2640 if ((rsc == NULL) && action_xml) {
2641 xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
2642
2643 const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
2644 const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
2645 const char *type = crm_element_value(xml, XML_ATTR_TYPE);
2646
2647 if (standard && type) {
2648 crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
2649 op->rsc_id, standard,
2650 (provider? ":" : ""), (provider? provider : ""), type);
2651 rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
2652 } else {
2653 crm_err("Can't process %s result because %s agent information not cached or in XML",
2654 op_key, op->rsc_id);
2655 }
2656 }
2657
2658
2659 if (lrm_state) {
2660 node_name = lrm_state->node_name;
2661 } else if (action_xml) {
2662 node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
2663 }
2664
2665 if(pending == NULL) {
2666 remove = TRUE;
2667 if (lrm_state) {
2668 pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
2669 }
2670 }
2671
2672 if (op->op_status == PCMK_LRM_OP_ERROR) {
2673 switch(op->rc) {
2674 case PCMK_OCF_NOT_RUNNING:
2675 case PCMK_OCF_RUNNING_MASTER:
2676 case PCMK_OCF_DEGRADED:
2677 case PCMK_OCF_DEGRADED_MASTER:
2678
2679 op->op_status = PCMK_LRM_OP_DONE;
2680 break;
2681 default:
2682
2683 break;
2684 }
2685 }
2686
2687 if (op->op_status != PCMK_LRM_OP_CANCELLED) {
2688
2689
2690
2691
2692 need_direct_ack = TRUE;
2693
2694 if (controld_action_is_recordable(op->op_type)) {
2695 if (node_name && rsc) {
2696
2697 update_id = do_update_resource(node_name, rsc, op,
2698 pending? pending->lock_time : 0);
2699 need_direct_ack = FALSE;
2700
2701 } else if (op->rsc_deleted) {
2702
2703
2704
2705
2706 crm_notice("Not recording %s result in CIB because "
2707 "resource information was removed since it was initiated",
2708 op_key);
2709 } else {
2710
2711
2712
2713
2714 crm_err("Unable to record %s result in CIB: %s", op_key,
2715 (node_name? "No resource information" : "No node name"));
2716 }
2717 }
2718
2719 } else if (op->interval_ms == 0) {
2720
2721
2722
2723
2724 need_direct_ack = TRUE;
2725
2726 } else if (pending == NULL) {
2727
2728
2729
2730
2731 } else if (op->user_data == NULL) {
2732
2733
2734
2735 crm_err("Recurring operation %s was cancelled without transition information",
2736 op_key);
2737
2738 } else if (pcmk_is_set(pending->flags, active_op_remove)) {
2739
2740
2741
2742 if (lrm_state) {
2743 erase_lrm_history_by_op(lrm_state, op);
2744 }
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755 if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
2756 pending->op_type, pending->interval_ms)) {
2757 need_direct_ack = TRUE;
2758 }
2759
2760 } else if (op->rsc_deleted) {
2761
2762
2763
2764
2765 crm_debug("Recurring op %s was cancelled due to resource deletion",
2766 op_key);
2767 need_direct_ack = TRUE;
2768
2769 } else {
2770
2771
2772
2773
2774 }
2775
2776 if (need_direct_ack) {
2777 controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
2778 }
2779
2780 if(remove == FALSE) {
2781
2782 removed = TRUE;
2783
2784 } else if (lrm_state && ((op->interval_ms == 0)
2785 || (op->op_status == PCMK_LRM_OP_CANCELLED))) {
2786
2787 gboolean found = g_hash_table_remove(lrm_state->pending_ops, op_id);
2788
2789 if (op->interval_ms != 0) {
2790 removed = TRUE;
2791 } else if (found) {
2792 removed = TRUE;
2793 crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
2794 op_key, op->call_id, op_id,
2795 g_hash_table_size(lrm_state->pending_ops));
2796 }
2797 }
2798
2799 if (node_name == NULL) {
2800 node_name = "unknown node";
2801 }
2802
2803 switch (op->op_status) {
2804 case PCMK_LRM_OP_CANCELLED:
2805 crm_info("Result of %s operation for %s on %s: %s "
2806 CRM_XS " call=%d key=%s confirmed=%s",
2807 crm_action_str(op->op_type, op->interval_ms),
2808 op->rsc_id, node_name,
2809 services_lrm_status_str(op->op_status),
2810 op->call_id, op_key, pcmk__btoa(removed));
2811 break;
2812
2813 case PCMK_LRM_OP_DONE:
2814 crm_notice("Result of %s operation for %s on %s: %s "
2815 CRM_XS " rc=%d call=%d key=%s confirmed=%s cib-update=%d",
2816 crm_action_str(op->op_type, op->interval_ms),
2817 op->rsc_id, node_name,
2818 services_ocf_exitcode_str(op->rc), op->rc,
2819 op->call_id, op_key, pcmk__btoa(removed), update_id);
2820 break;
2821
2822 case PCMK_LRM_OP_TIMEOUT:
2823 crm_err("Result of %s operation for %s on %s: %s "
2824 CRM_XS " call=%d key=%s timeout=%dms",
2825 crm_action_str(op->op_type, op->interval_ms),
2826 op->rsc_id, node_name,
2827 services_lrm_status_str(op->op_status),
2828 op->call_id, op_key, op->timeout);
2829 break;
2830
2831 default:
2832 crm_err("Result of %s operation for %s on %s: %s "
2833 CRM_XS " call=%d key=%s confirmed=%s status=%d cib-update=%d",
2834 crm_action_str(op->op_type, op->interval_ms),
2835 op->rsc_id, node_name,
2836 services_lrm_status_str(op->op_status), op->call_id, op_key,
2837 pcmk__btoa(removed), op->op_status, update_id);
2838 }
2839
2840 if (op->output) {
2841 char *prefix =
2842 crm_strdup_printf("%s-" PCMK__OP_FMT ":%d", node_name,
2843 op->rsc_id, op->op_type, op->interval_ms,
2844 op->call_id);
2845
2846 if (op->rc) {
2847 crm_log_output(LOG_NOTICE, prefix, op->output);
2848 } else {
2849 crm_log_output(LOG_DEBUG, prefix, op->output);
2850 }
2851 free(prefix);
2852 }
2853
2854 if (lrm_state) {
2855 if (!pcmk__str_eq(op->op_type, RSC_METADATA, pcmk__str_casei)) {
2856 crmd_alert_resource_op(lrm_state->node_name, op);
2857 } else if (rsc && (op->rc == PCMK_OCF_OK)) {
2858 char *metadata = unescape_newlines(op->output);
2859
2860 metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
2861 free(metadata);
2862 }
2863 }
2864
2865 if (op->rsc_deleted) {
2866 crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
2867 if (lrm_state) {
2868 delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
2869 }
2870 }
2871
2872
2873
2874
2875 mainloop_set_trigger(fsa_source);
2876 if (lrm_state && rsc) {
2877 update_history_cache(lrm_state, rsc, op);
2878 }
2879
2880 lrmd_free_rsc_info(rsc);
2881 free(op_key);
2882 free(op_id);
2883 }
![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
![[previous]](../icons/left.png)
![[last]](../icons/n_last.png)
![[bottom]](../icons/n_bottom.png){kind=icon}