This source file includes following definitions.
- lrm_connection_destroy
- make_stop_id
- copy_instance_keys
- copy_meta_keys
- history_remove_recurring_op
- history_free_recurring_ops
- history_free
- update_history_cache
- send_task_ok_ack
- op_node_name
- lrm_op_callback
- try_local_executor_connect
- do_lrm_control
- lrm_state_verify_stopped
- is_rsc_active
- build_active_RAs
- controld_query_executor_state
- controld_rc2event
- controld_trigger_delete_refresh
- notify_deleted
- lrm_remove_deleted_rsc
- lrm_remove_deleted_op
- delete_rsc_entry
- last_failed_matches_op
- lrm_clear_last_failure
- cancel_op
- cancel_action_by_key
- cancel_op_key
- get_lrm_resource
- delete_resource
- get_fake_call_id
- fake_op_status
- force_reprobe
- synthesize_lrmd_failure
- lrm_op_target
- fail_lrm_resource
- handle_reprobe_op
- do_lrm_cancel
- do_lrm_delete
- new_metadata_cb_data
- free_metadata_cb_data
- metadata_complete
- do_lrm_invoke
- construct_op
- controld_ack_event_directly
- verify_stopped
- stop_recurring_action_by_rsc
- stop_recurring_actions
- should_cancel_recurring
- should_nack_action
- do_lrm_rsc_op
- do_lrm_event
- unescape_newlines
- did_lrm_rsc_op_fail
- log_executor_event
- process_lrm_event
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <regex.h>
13 #include <sys/param.h>
14 #include <sys/types.h>
15 #include <sys/wait.h>
16
17 #include <crm/crm.h>
18 #include <crm/lrmd.h>
19 #include <crm/services.h>
20 #include <crm/common/xml.h>
21 #include <crm/pengine/rules.h>
22 #include <crm/lrmd_internal.h>
23
24 #include <pacemaker-internal.h>
25 #include <pacemaker-controld.h>
26
27 #define START_DELAY_THRESHOLD 5 * 60 * 1000
28 #define MAX_LRM_REG_FAILS 30
29
30 struct delete_event_s {
31 int rc;
32 const char *rsc;
33 lrm_state_t *lrm_state;
34 };
35
36 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
37 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
38 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
39
40 static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state,
41 const xmlNode *rsc_op,
42 const char *rsc_id,
43 const char *operation);
44 static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
45 xmlNode *msg, struct ra_metadata_s *md);
46
47 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
48 int log_level);
49
50 static void
51 lrm_connection_destroy(void)
52 {
53 if (pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) {
54 crm_crit("Lost connection to local executor");
55 register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
56 controld_clear_fsa_input_flags(R_LRM_CONNECTED);
57 }
58 }
59
60 static char *
61 make_stop_id(const char *rsc, int call_id)
62 {
63 return crm_strdup_printf("%s:%d", rsc, call_id);
64 }
65
66 static void
67 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
68 {
69 if (strstr(key, CRM_META "_") == NULL) {
70 pcmk__insert_dup(user_data, (const char *) key, (const char *) value);
71 }
72 }
73
74 static void
75 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
76 {
77 if (strstr(key, CRM_META "_") != NULL) {
78 pcmk__insert_dup(user_data, (const char *) key, (const char *) value);
79 }
80 }
81
82
83
84
85
86
87
88
89
90
91 static gboolean
92 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
93 {
94 GList *iter;
95
96 for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
97 lrmd_event_data_t *existing = iter->data;
98
99 if ((op->interval_ms == existing->interval_ms)
100 && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
101 && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
102
103 history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
104 lrmd_free_event(existing);
105 return TRUE;
106 }
107 }
108 return FALSE;
109 }
110
111
112
113
114
115
116
117 static void
118 history_free_recurring_ops(rsc_history_t *history)
119 {
120 GList *iter;
121
122 for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
123 lrmd_free_event(iter->data);
124 }
125 g_list_free(history->recurring_op_list);
126 history->recurring_op_list = NULL;
127 }
128
129
130
131
132
133
134
135 void
136 history_free(gpointer data)
137 {
138 rsc_history_t *history = (rsc_history_t*)data;
139
140 if (history->stop_params) {
141 g_hash_table_destroy(history->stop_params);
142 }
143
144
145 free(history->rsc.type);
146 free(history->rsc.standard);
147 free(history->rsc.provider);
148
149 lrmd_free_event(history->failed);
150 lrmd_free_event(history->last);
151 free(history->id);
152 history_free_recurring_ops(history);
153 free(history);
154 }
155
156 static void
157 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
158 {
159 int target_rc = 0;
160 rsc_history_t *entry = NULL;
161
162 if (op->rsc_deleted) {
163 crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
164 controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
165 NULL, crmd_cib_smart_opt());
166 return;
167 }
168
169 if (pcmk__str_eq(op->op_type, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
170 return;
171 }
172
173 crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
174
175 entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
176 if (entry == NULL && rsc) {
177 entry = pcmk__assert_alloc(1, sizeof(rsc_history_t));
178 entry->id = pcmk__str_copy(op->rsc_id);
179 g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
180
181 entry->rsc.id = entry->id;
182 entry->rsc.type = pcmk__str_copy(rsc->type);
183 entry->rsc.standard = pcmk__str_copy(rsc->standard);
184 entry->rsc.provider = pcmk__str_copy(rsc->provider);
185
186 } else if (entry == NULL) {
187 crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
188 return;
189 }
190
191 entry->last_callid = op->call_id;
192 target_rc = rsc_op_expected_rc(op);
193 if (op->op_status == PCMK_EXEC_CANCELLED) {
194 if (op->interval_ms > 0) {
195 crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
196 op->rsc_id, op->op_type, op->interval_ms);
197 history_remove_recurring_op(entry, op);
198 return;
199 } else {
200 crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
201 op->rsc_id, op->op_type, op->interval_ms, op->rc,
202 op->op_status);
203 }
204
205 } else if (did_rsc_op_fail(op, target_rc)) {
206
207
208
209 if (entry->failed) {
210 lrmd_free_event(entry->failed);
211 }
212 entry->failed = lrmd_copy_event(op);
213
214 } else if (op->interval_ms == 0) {
215 if (entry->last) {
216 lrmd_free_event(entry->last);
217 }
218 entry->last = lrmd_copy_event(op);
219
220 if (op->params && pcmk__strcase_any_of(op->op_type, PCMK_ACTION_START,
221 PCMK_ACTION_RELOAD,
222 PCMK_ACTION_RELOAD_AGENT,
223 PCMK_ACTION_MONITOR, NULL)) {
224 if (entry->stop_params) {
225 g_hash_table_destroy(entry->stop_params);
226 }
227 entry->stop_params = pcmk__strkey_table(free, free);
228
229 g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
230 }
231 }
232
233 if (op->interval_ms > 0) {
234
235 history_remove_recurring_op(entry, op);
236
237 crm_trace("Adding recurring op: " PCMK__OP_FMT,
238 op->rsc_id, op->op_type, op->interval_ms);
239 entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
240
241 } else if ((entry->recurring_op_list != NULL)
242 && !pcmk__str_eq(op->op_type, PCMK_ACTION_MONITOR,
243 pcmk__str_casei)) {
244 crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
245 g_list_length(entry->recurring_op_list), op->rsc_id,
246 op->op_type, op->interval_ms);
247 history_free_recurring_ops(entry);
248 }
249 }
250
251
252
253
254
255
256
257
258
259
260
261
262
263 static void
264 send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input,
265 const char *rsc_id, const lrmd_rsc_info_t *rsc,
266 const char *task, const char *ack_host, const char *ack_sys)
267 {
268 lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
269
270 lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
271 controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
272 lrmd_free_event(op);
273 }
274
275 static inline const char *
276 op_node_name(lrmd_event_data_t *op)
277 {
278 return pcmk__s(op->remote_nodename, controld_globals.our_nodename);
279 }
280
281 void
282 lrm_op_callback(lrmd_event_data_t * op)
283 {
284 CRM_CHECK(op != NULL, return);
285 switch (op->type) {
286 case lrmd_event_disconnect:
287 if (op->remote_nodename == NULL) {
288
289
290
291 lrm_connection_destroy();
292 }
293 break;
294
295 case lrmd_event_exec_complete:
296 {
297 lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
298
299 CRM_ASSERT(lrm_state != NULL);
300 process_lrm_event(lrm_state, op, NULL, NULL);
301 }
302 break;
303
304 default:
305 break;
306 }
307 }
308
309 static void
310 try_local_executor_connect(long long action, fsa_data_t *msg_data,
311 lrm_state_t *lrm_state)
312 {
313 int rc = pcmk_rc_ok;
314
315 crm_debug("Connecting to the local executor");
316
317
318 rc = controld_connect_local_executor(lrm_state);
319 if (rc == pcmk_rc_ok) {
320 controld_set_fsa_input_flags(R_LRM_CONNECTED);
321 crm_info("Connection to the local executor established");
322 return;
323 }
324
325
326 if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
327 crm_warn("Failed to connect to the local executor %d time%s "
328 "(%d max): %s", lrm_state->num_lrm_register_fails,
329 pcmk__plural_s(lrm_state->num_lrm_register_fails),
330 MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
331 controld_start_wait_timer();
332 crmd_fsa_stall(FALSE);
333 return;
334 }
335
336
337 crm_err("Failed to connect to the executor the max allowed "
338 "%d time%s: %s", lrm_state->num_lrm_register_fails,
339 pcmk__plural_s(lrm_state->num_lrm_register_fails),
340 pcmk_rc_str(rc));
341 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
342 }
343
344
345 void
346 do_lrm_control(long long action,
347 enum crmd_fsa_cause cause,
348 enum crmd_fsa_state cur_state,
349 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
350 {
351
352
353
354
355
356 lrm_state_t *lrm_state = NULL;
357
358 if (controld_globals.our_nodename == NULL) {
359 return;
360 }
361 lrm_state = lrm_state_find_or_create(controld_globals.our_nodename);
362 if (lrm_state == NULL) {
363 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
364 return;
365 }
366
367 if (action & A_LRM_DISCONNECT) {
368 if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
369 if (action == A_LRM_DISCONNECT) {
370 crmd_fsa_stall(FALSE);
371 return;
372 }
373 }
374
375 controld_clear_fsa_input_flags(R_LRM_CONNECTED);
376 lrm_state_disconnect(lrm_state);
377 lrm_state_reset_tables(lrm_state, FALSE);
378 }
379
380 if (action & A_LRM_CONNECT) {
381 try_local_executor_connect(action, msg_data, lrm_state);
382 }
383
384 if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
385 crm_err("Unexpected action %s in %s", fsa_action2string(action),
386 __func__);
387 }
388 }
389
390 static gboolean
391 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
392 {
393 int counter = 0;
394 gboolean rc = TRUE;
395 const char *when = "lrm disconnect";
396
397 GHashTableIter gIter;
398 const char *key = NULL;
399 rsc_history_t *entry = NULL;
400 active_op_t *pending = NULL;
401
402 crm_debug("Checking for active resources before exit");
403
404 if (cur_state == S_TERMINATE) {
405 log_level = LOG_ERR;
406 when = "shutdown";
407
408 } else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
409 when = "shutdown... waiting";
410 }
411
412 if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) {
413 guint removed = g_hash_table_foreach_remove(lrm_state->active_ops,
414 stop_recurring_actions,
415 lrm_state);
416 guint nremaining = g_hash_table_size(lrm_state->active_ops);
417
418 if (removed || nremaining) {
419 crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
420 removed, pcmk__plural_s(removed), when, nremaining);
421 }
422 }
423
424 if (lrm_state->active_ops != NULL) {
425 g_hash_table_iter_init(&gIter, lrm_state->active_ops);
426 while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
427
428 if (pending->interval_ms == 0) {
429 counter++;
430 }
431 }
432 }
433
434 if (counter > 0) {
435 do_crm_log(log_level, "%d pending executor operation%s at %s",
436 counter, pcmk__plural_s(counter), when);
437
438 if ((cur_state == S_TERMINATE)
439 || !pcmk_is_set(controld_globals.fsa_input_register,
440 R_SENT_RSC_STOP)) {
441 g_hash_table_iter_init(&gIter, lrm_state->active_ops);
442 while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
443 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
444 }
445
446 } else {
447 rc = FALSE;
448 }
449 return rc;
450 }
451
452 if (lrm_state->resource_history == NULL) {
453 return rc;
454 }
455
456 if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
457
458 when = "shutdown";
459 }
460
461 counter = 0;
462 g_hash_table_iter_init(&gIter, lrm_state->resource_history);
463 while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
464 if (is_rsc_active(lrm_state, entry->id) == FALSE) {
465 continue;
466 }
467
468 counter++;
469 if (log_level == LOG_ERR) {
470 crm_info("Found %s active at %s", entry->id, when);
471 } else {
472 crm_trace("Found %s active at %s", entry->id, when);
473 }
474 if (lrm_state->active_ops != NULL) {
475 GHashTableIter hIter;
476
477 g_hash_table_iter_init(&hIter, lrm_state->active_ops);
478 while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
479 if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
480 crm_notice("%sction %s (%s) incomplete at %s",
481 pending->interval_ms == 0 ? "A" : "Recurring a",
482 key, pending->op_key, when);
483 }
484 }
485 }
486 }
487
488 if (counter) {
489 crm_err("%d resource%s active at %s",
490 counter, (counter == 1)? " was" : "s were", when);
491 }
492
493 return rc;
494 }
495
496 static gboolean
497 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
498 {
499 rsc_history_t *entry = NULL;
500
501 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
502 if (entry == NULL || entry->last == NULL) {
503 return FALSE;
504 }
505
506 crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
507 entry->last->interval_ms, entry->last->rc);
508 if ((entry->last->rc == PCMK_OCF_OK)
509 && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_STOP,
510 pcmk__str_casei)) {
511 return FALSE;
512
513 } else if (entry->last->rc == PCMK_OCF_OK
514 && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_MIGRATE_TO,
515 pcmk__str_casei)) {
516
517 return FALSE;
518
519 } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
520 return FALSE;
521
522 } else if ((entry->last->interval_ms == 0)
523 && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
524
525 return FALSE;
526 }
527
528 return TRUE;
529 }
530
531 static gboolean
532 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
533 {
534 GHashTableIter iter;
535 rsc_history_t *entry = NULL;
536
537 g_hash_table_iter_init(&iter, lrm_state->resource_history);
538 while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
539
540 GList *gIter = NULL;
541 xmlNode *xml_rsc = pcmk__xe_create(rsc_list, PCMK__XE_LRM_RESOURCE);
542
543 crm_xml_add(xml_rsc, PCMK_XA_ID, entry->id);
544 crm_xml_add(xml_rsc, PCMK_XA_TYPE, entry->rsc.type);
545 crm_xml_add(xml_rsc, PCMK_XA_CLASS, entry->rsc.standard);
546 crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, entry->rsc.provider);
547
548 if (entry->last && entry->last->params) {
549 static const char *name = CRM_META "_" PCMK__META_CONTAINER;
550 const char *container = g_hash_table_lookup(entry->last->params,
551 name);
552
553 if (container) {
554 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
555 crm_xml_add(xml_rsc, PCMK__META_CONTAINER, container);
556 }
557 }
558 controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed,
559 lrm_state->node_name);
560 controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last,
561 lrm_state->node_name);
562 for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
563 controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data,
564 lrm_state->node_name);
565 }
566 }
567
568 return FALSE;
569 }
570
571 xmlNode *
572 controld_query_executor_state(void)
573 {
574 xmlNode *xml_state = NULL;
575 xmlNode *xml_data = NULL;
576 xmlNode *rsc_list = NULL;
577 crm_node_t *peer = NULL;
578 lrm_state_t *lrm_state = lrm_state_find(controld_globals.our_nodename);
579
580 if (!lrm_state) {
581 crm_err("Could not find executor state for node %s",
582 controld_globals.our_nodename);
583 return NULL;
584 }
585
586 peer = pcmk__get_node(0, lrm_state->node_name, NULL, pcmk__node_search_any);
587 CRM_CHECK(peer != NULL, return NULL);
588
589 xml_state = create_node_state_update(peer,
590 node_update_cluster|node_update_peer,
591 NULL, __func__);
592 if (xml_state == NULL) {
593 return NULL;
594 }
595
596 xml_data = pcmk__xe_create(xml_state, PCMK__XE_LRM);
597 crm_xml_add(xml_data, PCMK_XA_ID, peer->uuid);
598 rsc_list = pcmk__xe_create(xml_data, PCMK__XE_LRM_RESOURCES);
599
600
601 build_active_RAs(lrm_state, rsc_list);
602
603 crm_log_xml_trace(xml_state, "Current executor state");
604
605 return xml_state;
606 }
607
608
609
610
611
612
613
614
615 void
616 controld_rc2event(lrmd_event_data_t *event, int rc)
617 {
618
619
620
621 switch (rc) {
622 case pcmk_rc_ok:
623 lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
624 break;
625 case EACCES:
626 lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
627 PCMK_EXEC_ERROR, NULL);
628 break;
629 default:
630 lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
631 NULL);
632 break;
633 }
634 }
635
636
637
638
639
640
641
642
643
644
645
646
647 void
648 controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
649 {
650 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
651 char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
652
653 crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
654 cib__update_node_attr(controld_globals.logger_out,
655 controld_globals.cib_conn, cib_none,
656 PCMK_XE_CRM_CONFIG, NULL, NULL, NULL, NULL,
657 "last-lrm-refresh", now_s, NULL, NULL);
658 free(now_s);
659 }
660 }
661
662 static void
663 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
664 {
665 lrmd_event_data_t *op = NULL;
666 const char *from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM);
667 const char *from_host = crm_element_value(input->msg, PCMK__XA_SRC);
668
669 crm_info("Notifying %s on %s that %s was%s deleted",
670 from_sys, (from_host? from_host : "localhost"), rsc_id,
671 ((rc == pcmk_ok)? "" : " not"));
672 op = construct_op(lrm_state, input->xml, rsc_id, PCMK_ACTION_DELETE);
673 controld_rc2event(op, pcmk_legacy2rc(rc));
674 controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
675 lrmd_free_event(op);
676 controld_trigger_delete_refresh(from_sys, rsc_id);
677 }
678
679 static gboolean
680 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
681 {
682 struct delete_event_s *event = user_data;
683 struct pending_deletion_op_s *op = value;
684
685 if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
686 notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
687 return TRUE;
688 }
689 return FALSE;
690 }
691
692 static gboolean
693 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
694 {
695 const char *rsc = user_data;
696 active_op_t *pending = value;
697
698 if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
699 crm_info("Removing op %s:%d for deleted resource %s",
700 pending->op_key, pending->call_id, rsc);
701 return TRUE;
702 }
703 return FALSE;
704 }
705
706 static void
707 delete_rsc_entry(lrm_state_t *lrm_state, ha_msg_input_t *input,
708 const char *rsc_id, GHashTableIter *rsc_iter, int rc,
709 const char *user_name, bool from_cib)
710 {
711 struct delete_event_s event;
712
713 CRM_CHECK(rsc_id != NULL, return);
714
715 if (rc == pcmk_ok) {
716 char *rsc_id_copy = pcmk__str_copy(rsc_id);
717
718 if (rsc_iter) {
719 g_hash_table_iter_remove(rsc_iter);
720 } else {
721 g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
722 }
723
724 if (from_cib) {
725 controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
726 user_name, crmd_cib_smart_opt());
727 }
728 g_hash_table_foreach_remove(lrm_state->active_ops,
729 lrm_remove_deleted_op, rsc_id_copy);
730 free(rsc_id_copy);
731 }
732
733 if (input) {
734 notify_deleted(lrm_state, input, rsc_id, rc);
735 }
736
737 event.rc = rc;
738 event.rsc = rsc_id;
739 event.lrm_state = lrm_state;
740 g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
741 }
742
743 static inline gboolean
744 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
745 {
746 if (entry == NULL) {
747 return FALSE;
748 }
749 if (op == NULL) {
750 return TRUE;
751 }
752 return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
753 && (interval_ms == entry->failed->interval_ms));
754 }
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769 void
770 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
771 const char *operation, guint interval_ms)
772 {
773 lrm_state_t *lrm_state = lrm_state_find(node_name);
774
775 if (lrm_state == NULL) {
776 return;
777 }
778 if (lrm_state->resource_history != NULL) {
779 rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
780 rsc_id);
781
782 if (last_failed_matches_op(entry, operation, interval_ms)) {
783 lrmd_free_event(entry->failed);
784 entry->failed = NULL;
785 }
786 }
787 }
788
789
790 static gboolean
791 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
792 {
793 int rc = pcmk_ok;
794 char *local_key = NULL;
795 active_op_t *pending = NULL;
796
797 CRM_CHECK(op != 0, return FALSE);
798 CRM_CHECK(rsc_id != NULL, return FALSE);
799 if (key == NULL) {
800 local_key = make_stop_id(rsc_id, op);
801 key = local_key;
802 }
803 pending = g_hash_table_lookup(lrm_state->active_ops, key);
804
805 if (pending) {
806 if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
807 controld_set_active_op_flags(pending, active_op_remove);
808 crm_debug("Scheduling %s for removal", key);
809 }
810
811 if (pcmk_is_set(pending->flags, active_op_cancelled)) {
812 crm_debug("Operation %s already cancelled", key);
813 free(local_key);
814 return FALSE;
815 }
816 controld_set_active_op_flags(pending, active_op_cancelled);
817
818 } else {
819 crm_info("No pending op found for %s", key);
820 free(local_key);
821 return FALSE;
822 }
823
824 crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
825 rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
826 pending->interval_ms);
827 if (rc == pcmk_ok) {
828 crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
829 free(local_key);
830 return TRUE;
831 }
832
833 crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
834
835
836
837
838
839
840
841
842
843 free(local_key);
844 return FALSE;
845 }
846
847 struct cancel_data {
848 gboolean done;
849 gboolean remove;
850 const char *key;
851 lrmd_rsc_info_t *rsc;
852 lrm_state_t *lrm_state;
853 };
854
855 static gboolean
856 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
857 {
858 gboolean remove = FALSE;
859 struct cancel_data *data = user_data;
860 active_op_t *op = value;
861
862 if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
863 data->done = TRUE;
864 remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
865 }
866 return remove;
867 }
868
869 static gboolean
870 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
871 {
872 guint removed = 0;
873 struct cancel_data data;
874
875 CRM_CHECK(rsc != NULL, return FALSE);
876 CRM_CHECK(key != NULL, return FALSE);
877
878 data.key = key;
879 data.rsc = rsc;
880 data.done = FALSE;
881 data.remove = remove;
882 data.lrm_state = lrm_state;
883
884 removed = g_hash_table_foreach_remove(lrm_state->active_ops,
885 cancel_action_by_key, &data);
886 crm_trace("Removed %u op cache entries, new size: %u",
887 removed, g_hash_table_size(lrm_state->active_ops));
888 return data.done;
889 }
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908 static int
909 get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml,
910 gboolean do_create, lrmd_rsc_info_t **rsc_info)
911 {
912 const char *id = pcmk__xe_id(rsc_xml);
913
914 CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
915 CRM_CHECK(id, return -EINVAL);
916
917 if (lrm_state_is_connected(lrm_state) == FALSE) {
918 return -ENOTCONN;
919 }
920
921 crm_trace("Retrieving resource information for %s from the executor", id);
922 *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
923
924
925 if (!*rsc_info) {
926 const char *long_id = crm_element_value(rsc_xml, PCMK__XA_LONG_ID);
927
928 if (long_id) {
929 *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
930 }
931 }
932
933 if ((*rsc_info == NULL) && do_create) {
934 const char *class = crm_element_value(rsc_xml, PCMK_XA_CLASS);
935 const char *provider = crm_element_value(rsc_xml, PCMK_XA_PROVIDER);
936 const char *type = crm_element_value(rsc_xml, PCMK_XA_TYPE);
937 int rc;
938
939 crm_trace("Registering resource %s with the executor", id);
940 rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
941 lrmd_opt_drop_recurring);
942 if (rc != pcmk_ok) {
943 fsa_data_t *msg_data = NULL;
944
945 crm_err("Could not register resource %s with the executor on %s: %s "
946 CRM_XS " rc=%d",
947 id, lrm_state->node_name, pcmk_strerror(rc), rc);
948
949
950
951
952
953 if (lrm_state_is_local(lrm_state) == TRUE) {
954 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
955 }
956 return rc;
957 }
958
959 *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
960 }
961 return *rsc_info? pcmk_ok : -ENODEV;
962 }
963
964 static void
965 delete_resource(lrm_state_t *lrm_state, const char *id, lrmd_rsc_info_t *rsc,
966 GHashTableIter *iter, const char *sys, const char *user,
967 ha_msg_input_t *request, bool unregister, bool from_cib)
968 {
969 int rc = pcmk_ok;
970
971 crm_info("Removing resource %s from executor for %s%s%s",
972 id, sys, (user? " as " : ""), (user? user : ""));
973
974 if (rsc && unregister) {
975 rc = lrm_state_unregister_rsc(lrm_state, id, 0);
976 }
977
978 if (rc == pcmk_ok) {
979 crm_trace("Resource %s deleted from executor", id);
980 } else if (rc == -EINPROGRESS) {
981 crm_info("Deletion of resource '%s' from executor is pending", id);
982 if (request) {
983 struct pending_deletion_op_s *op = NULL;
984 char *ref = crm_element_value_copy(request->msg, PCMK_XA_REFERENCE);
985
986 op = pcmk__assert_alloc(1, sizeof(struct pending_deletion_op_s));
987 op->rsc = pcmk__str_copy(rsc->id);
988 op->input = copy_ha_msg_input(request);
989 g_hash_table_insert(lrm_state->deletion_ops, ref, op);
990 }
991 return;
992 } else {
993 crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
994 CRM_XS " rc=%d", id, sys, (user? " as " : ""),
995 (user? user : ""), pcmk_strerror(rc), rc);
996 }
997
998 delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib);
999 }
1000
1001 static int
1002 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
1003 {
1004 int call_id = 999999999;
1005 rsc_history_t *entry = NULL;
1006
1007 if(lrm_state) {
1008 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
1009 }
1010
1011
1012
1013
1014 if (entry) {
1015 call_id = entry->last_callid + 1;
1016 }
1017
1018 if (call_id < 0) {
1019 call_id = 1;
1020 }
1021 return call_id;
1022 }
1023
1024 static void
1025 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
1026 enum ocf_exitcode op_exitcode, const char *exit_reason)
1027 {
1028 op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
1029 op->t_run = time(NULL);
1030 op->t_rcchange = op->t_run;
1031 lrmd__set_result(op, op_exitcode, op_status, exit_reason);
1032 }
1033
1034 static void
1035 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
1036 const char *from_host, const char *user_name,
1037 gboolean is_remote_node, bool reprobe_all_nodes)
1038 {
1039 GHashTableIter gIter;
1040 rsc_history_t *entry = NULL;
1041
1042 crm_info("Clearing resource history on node %s", lrm_state->node_name);
1043 g_hash_table_iter_init(&gIter, lrm_state->resource_history);
1044 while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
1045
1046
1047
1048 bool unregister = true;
1049
1050 if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
1051 unregister = false;
1052
1053 if (reprobe_all_nodes) {
1054 lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
1055
1056 if (remote_lrm_state != NULL) {
1057
1058
1059
1060 force_reprobe(remote_lrm_state, from_sys, from_host,
1061 user_name, TRUE, reprobe_all_nodes);
1062 }
1063 }
1064 }
1065
1066
1067
1068
1069 delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
1070 user_name, NULL, unregister, false);
1071 }
1072
1073
1074 controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
1075 cib_scope_local);
1076
1077
1078 update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
1079 }
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095 static void
1096 synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action,
1097 int op_status, enum ocf_exitcode rc,
1098 const char *exit_reason)
1099 {
1100 lrmd_event_data_t *op = NULL;
1101 const char *operation = crm_element_value(action, PCMK_XA_OPERATION);
1102 const char *target_node = crm_element_value(action, PCMK__META_ON_NODE);
1103 xmlNode *xml_rsc = pcmk__xe_first_child(action, PCMK_XE_PRIMITIVE, NULL,
1104 NULL);
1105
1106 if ((xml_rsc == NULL) || (pcmk__xe_id(xml_rsc) == NULL)) {
1107
1108 crm_info("Can't fake %s failure (%d) on %s without resource configuration",
1109 crm_element_value(action, PCMK__XA_OPERATION_KEY), rc,
1110 target_node);
1111 return;
1112
1113 } else if(operation == NULL) {
1114
1115 crm_info("Can't fake %s failure (%d) on %s without operation",
1116 pcmk__xe_id(xml_rsc), rc, target_node);
1117 return;
1118 }
1119
1120 op = construct_op(lrm_state, action, pcmk__xe_id(xml_rsc), operation);
1121
1122 if (pcmk__str_eq(operation, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
1123
1124 fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL);
1125 } else {
1126 fake_op_status(lrm_state, op, op_status, rc, exit_reason);
1127 }
1128
1129 crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
1130 op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
1131
1132
1133 process_lrm_event(lrm_state, op, NULL, action);
1134 lrmd_free_event(op);
1135 }
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146 static const char *
1147 lrm_op_target(const xmlNode *xml)
1148 {
1149 const char *target = NULL;
1150
1151 if (xml) {
1152 target = crm_element_value(xml, PCMK__META_ON_NODE);
1153 }
1154 if (target == NULL) {
1155 target = controld_globals.our_nodename;
1156 }
1157 return target;
1158 }
1159
1160 static void
1161 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
1162 const char *from_host, const char *from_sys)
1163 {
1164 lrmd_event_data_t *op = NULL;
1165 lrmd_rsc_info_t *rsc = NULL;
1166 xmlNode *xml_rsc = pcmk__xe_first_child(xml, PCMK_XE_PRIMITIVE, NULL, NULL);
1167
1168 CRM_CHECK(xml_rsc != NULL, return);
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178 op = construct_op(lrm_state, xml, pcmk__xe_id(xml_rsc), "asyncmon");
1179
1180 free((char*) op->user_data);
1181 op->user_data = NULL;
1182 op->interval_ms = 0;
1183
1184 if (user_name && !pcmk__is_privileged(user_name)) {
1185 crm_err("%s does not have permission to fail %s",
1186 user_name, pcmk__xe_id(xml_rsc));
1187 fake_op_status(lrm_state, op, PCMK_EXEC_ERROR,
1188 PCMK_OCF_INSUFFICIENT_PRIV,
1189 "Unprivileged user cannot fail resources");
1190 controld_ack_event_directly(from_host, from_sys, NULL, op,
1191 pcmk__xe_id(xml_rsc));
1192 lrmd_free_event(op);
1193 return;
1194 }
1195
1196
1197 if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
1198 crm_info("Failing resource %s...", rsc->id);
1199 fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR,
1200 "Simulated failure");
1201 process_lrm_event(lrm_state, op, NULL, xml);
1202 op->rc = PCMK_OCF_OK;
1203 lrmd_free_rsc_info(rsc);
1204
1205 } else {
1206 crm_info("Cannot find/create resource in order to fail it...");
1207 crm_log_xml_warn(xml, "bad input");
1208 fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR,
1209 "Cannot fail unknown resource");
1210 }
1211
1212 controld_ack_event_directly(from_host, from_sys, NULL, op,
1213 pcmk__xe_id(xml_rsc));
1214 lrmd_free_event(op);
1215 }
1216
1217 static void
1218 handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
1219 const char *from_host, const char *user_name,
1220 gboolean is_remote_node, bool reprobe_all_nodes)
1221 {
1222 crm_notice("Forcing the status of all resources to be redetected");
1223 force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node,
1224 reprobe_all_nodes);
1225
1226 if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
1227
1228 xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
1229 from_sys, CRM_SYSTEM_LRMD,
1230 controld_globals.our_uuid);
1231
1232 crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
1233
1234 if (relay_message(reply, TRUE) == FALSE) {
1235 crm_log_xml_err(reply, "Unable to route reply");
1236 }
1237 free_xml(reply);
1238 }
1239 }
1240
1241 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
1242 lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
1243 {
1244 char *op_key = NULL;
1245 char *meta_key = NULL;
1246 int call = 0;
1247 const char *call_id = NULL;
1248 const char *op_task = NULL;
1249 guint interval_ms = 0;
1250 gboolean in_progress = FALSE;
1251 xmlNode *params = pcmk__xe_first_child(input->xml, PCMK__XE_ATTRIBUTES,
1252 NULL, NULL);
1253
1254 CRM_CHECK(params != NULL, return FALSE);
1255
1256 meta_key = crm_meta_name(PCMK_XA_OPERATION);
1257 op_task = crm_element_value(params, meta_key);
1258 free(meta_key);
1259 CRM_CHECK(op_task != NULL, return FALSE);
1260
1261 meta_key = crm_meta_name(PCMK_META_INTERVAL);
1262 if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
1263 free(meta_key);
1264 return FALSE;
1265 }
1266 free(meta_key);
1267
1268 op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
1269
1270 meta_key = crm_meta_name(PCMK__XA_CALL_ID);
1271 call_id = crm_element_value(params, meta_key);
1272 free(meta_key);
1273
1274 crm_debug("Scheduler requested op %s (call=%s) be cancelled",
1275 op_key, (call_id? call_id : "NA"));
1276 pcmk__scan_min_int(call_id, &call, 0);
1277 if (call == 0) {
1278
1279 in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
1280
1281 } else {
1282
1283 in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
1284 }
1285
1286
1287 if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1288 char *op_id = make_stop_id(rsc->id, call);
1289
1290 if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
1291 crm_info("Nothing known about operation %d for %s", call, op_key);
1292 }
1293 controld_delete_action_history_by_key(rsc->id, lrm_state->node_name,
1294 op_key, call);
1295 send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
1296 from_host, from_sys);
1297
1298
1299 if (lrm_state->active_ops != NULL) {
1300 g_hash_table_remove(lrm_state->active_ops, op_id);
1301 }
1302 free(op_id);
1303
1304 } else {
1305
1306
1307
1308
1309
1310
1311
1312 const char *peer_version = crm_element_value(params,
1313 PCMK_XA_CRM_FEATURE_SET);
1314
1315 if (compare_version(peer_version, "3.0.8") <= 0) {
1316 crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
1317 op_key, from_host, peer_version);
1318 send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
1319 from_host, from_sys);
1320 }
1321 }
1322
1323 free(op_key);
1324 return TRUE;
1325 }
1326
1327 static void
1328 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
1329 lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
1330 bool crm_rsc_delete, const char *user_name)
1331 {
1332 bool unregister = true;
1333 int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
1334 user_name,
1335 cib_dryrun|cib_sync_call);
1336
1337 if (cib_rc != pcmk_rc_ok) {
1338 lrmd_event_data_t *op = NULL;
1339
1340 op = construct_op(lrm_state, input->xml, rsc->id, PCMK_ACTION_DELETE);
1341
1342
1343
1344
1345 lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL);
1346 controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
1347 lrmd_free_event(op);
1348 return;
1349 }
1350
1351 if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1352 unregister = false;
1353 }
1354
1355 delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
1356 user_name, input, unregister, true);
1357 }
1358
1359
1360 struct metadata_cb_data {
1361 lrmd_rsc_info_t *rsc;
1362 xmlNode *input_xml;
1363 };
1364
1365 static struct metadata_cb_data *
1366 new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml)
1367 {
1368 struct metadata_cb_data *data = NULL;
1369
1370 data = pcmk__assert_alloc(1, sizeof(struct metadata_cb_data));
1371 data->input_xml = pcmk__xml_copy(NULL, input_xml);
1372 data->rsc = lrmd_copy_rsc_info(rsc);
1373 return data;
1374 }
1375
1376 static void
1377 free_metadata_cb_data(struct metadata_cb_data *data)
1378 {
1379 lrmd_free_rsc_info(data->rsc);
1380 free_xml(data->input_xml);
1381 free(data);
1382 }
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392 static void
1393 metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data)
1394 {
1395 struct metadata_cb_data *data = (struct metadata_cb_data *) user_data;
1396
1397 struct ra_metadata_s *md = NULL;
1398 lrm_state_t *lrm_state = lrm_state_find(lrm_op_target(data->input_xml));
1399
1400 if ((lrm_state != NULL) && pcmk__result_ok(result)) {
1401 md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc,
1402 result->action_stdout);
1403 }
1404 if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
1405 do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md);
1406 }
1407 free_metadata_cb_data(data);
1408 }
1409
1410
1411 void
1412 do_lrm_invoke(long long action,
1413 enum crmd_fsa_cause cause,
1414 enum crmd_fsa_state cur_state,
1415 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
1416 {
1417 lrm_state_t *lrm_state = NULL;
1418 const char *crm_op = NULL;
1419 const char *from_sys = NULL;
1420 const char *from_host = NULL;
1421 const char *operation = NULL;
1422 ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
1423 const char *user_name = NULL;
1424 const char *target_node = lrm_op_target(input->xml);
1425 gboolean is_remote_node = FALSE;
1426 bool crm_rsc_delete = FALSE;
1427
1428
1429 is_remote_node = !pcmk__str_eq(target_node, controld_globals.our_nodename,
1430 pcmk__str_casei);
1431
1432 lrm_state = lrm_state_find(target_node);
1433 if ((lrm_state == NULL) && is_remote_node) {
1434 crm_err("Failing action because local node has never had connection to remote node %s",
1435 target_node);
1436 synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
1437 PCMK_OCF_UNKNOWN_ERROR,
1438 "Local node has no connection to remote");
1439 return;
1440 }
1441 CRM_ASSERT(lrm_state != NULL);
1442
1443 user_name = pcmk__update_acl_user(input->msg, PCMK__XA_CRM_USER, NULL);
1444 crm_op = crm_element_value(input->msg, PCMK__XA_CRM_TASK);
1445 from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM);
1446 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
1447 from_host = crm_element_value(input->msg, PCMK__XA_SRC);
1448 }
1449
1450 if (pcmk__str_eq(crm_op, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
1451 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
1452 crm_rsc_delete = TRUE;
1453 }
1454 operation = PCMK_ACTION_DELETE;
1455
1456 } else if (input->xml != NULL) {
1457 operation = crm_element_value(input->xml, PCMK_XA_OPERATION);
1458 }
1459
1460 CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return);
1461
1462 crm_trace("'%s' execution request from %s as %s user",
1463 pcmk__s(crm_op, operation),
1464 pcmk__s(from_sys, "unknown subsystem"),
1465 pcmk__s(user_name, "current"));
1466
1467 if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) {
1468 fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
1469 from_sys);
1470
1471 } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_none)) {
1472
1473
1474
1475
1476
1477 crm_notice("Ignoring refresh request from Pacemaker Remote 1.1.9 node");
1478
1479
1480 } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_none)) {
1481 update_attrd(lrm_state->node_name, CRM_OP_PROBED, PCMK_VALUE_TRUE,
1482 user_name, is_remote_node);
1483
1484 } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none)
1485 || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
1486 const char *raw_target = NULL;
1487
1488 if (input->xml != NULL) {
1489
1490 raw_target = crm_element_value(input->xml, PCMK__META_ON_NODE);
1491 }
1492 handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
1493 is_remote_node, (raw_target == NULL));
1494
1495 } else if (operation != NULL) {
1496 lrmd_rsc_info_t *rsc = NULL;
1497 xmlNode *xml_rsc = pcmk__xe_first_child(input->xml, PCMK_XE_PRIMITIVE,
1498 NULL, NULL);
1499 gboolean create_rsc = !pcmk__str_eq(operation, PCMK_ACTION_DELETE,
1500 pcmk__str_none);
1501 int rc;
1502
1503
1504 CRM_CHECK((xml_rsc != NULL) && (pcmk__xe_id(xml_rsc) != NULL), return);
1505
1506 rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
1507 if (rc == -ENOTCONN) {
1508 synthesize_lrmd_failure(lrm_state, input->xml,
1509 PCMK_EXEC_NOT_CONNECTED,
1510 PCMK_OCF_UNKNOWN_ERROR,
1511 "Not connected to remote executor");
1512 return;
1513
1514 } else if ((rc < 0) && !create_rsc) {
1515
1516
1517
1518 crm_notice("Not registering resource '%s' for a %s event "
1519 CRM_XS " get-rc=%d (%s) transition-key=%s",
1520 pcmk__xe_id(xml_rsc), operation,
1521 rc, pcmk_strerror(rc), pcmk__xe_id(input->xml));
1522 delete_rsc_entry(lrm_state, input, pcmk__xe_id(xml_rsc), NULL,
1523 pcmk_ok, user_name, true);
1524 return;
1525
1526 } else if (rc == -EINVAL) {
1527
1528 crm_err("Invalid resource definition for %s", pcmk__xe_id(xml_rsc));
1529 crm_log_xml_warn(input->msg, "invalid resource");
1530 synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
1531 PCMK_OCF_NOT_CONFIGURED,
1532 "Invalid resource definition");
1533 return;
1534
1535 } else if (rc < 0) {
1536
1537 crm_err("Could not register resource '%s' with executor: %s "
1538 CRM_XS " rc=%d",
1539 pcmk__xe_id(xml_rsc), pcmk_strerror(rc), rc);
1540 crm_log_xml_warn(input->msg, "failed registration");
1541 synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
1542 PCMK_OCF_INVALID_PARAM,
1543 "Could not register resource with executor");
1544 return;
1545 }
1546
1547 if (pcmk__str_eq(operation, PCMK_ACTION_CANCEL, pcmk__str_none)) {
1548 if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
1549 crm_log_xml_warn(input->xml, "Bad command");
1550 }
1551
1552 } else if (pcmk__str_eq(operation, PCMK_ACTION_DELETE,
1553 pcmk__str_none)) {
1554 do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
1555 crm_rsc_delete, user_name);
1556
1557 } else {
1558 struct ra_metadata_s *md = NULL;
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568 if (strcmp(operation, PCMK_ACTION_START) != 0) {
1569 md = controld_get_rsc_metadata(lrm_state, rsc,
1570 controld_metadata_from_cache);
1571 }
1572
1573 if ((md == NULL) && crm_op_needs_metadata(rsc->standard,
1574 operation)) {
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586 struct metadata_cb_data *data = NULL;
1587
1588 data = new_metadata_cb_data(rsc, input->xml);
1589 crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously",
1590 rsc->id, rsc->standard,
1591 ((rsc->provider == NULL)? "" : ":"),
1592 ((rsc->provider == NULL)? "" : rsc->provider),
1593 rsc->type);
1594 (void) lrmd__metadata_async(rsc, metadata_complete,
1595 (void *) data);
1596 } else {
1597 do_lrm_rsc_op(lrm_state, rsc, input->xml, md);
1598 }
1599 }
1600
1601 lrmd_free_rsc_info(rsc);
1602
1603 } else {
1604 crm_err("Invalid execution request: unknown command '%s' (bug?)",
1605 crm_op);
1606 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
1607 }
1608 }
1609
1610 static lrmd_event_data_t *
1611 construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op,
1612 const char *rsc_id, const char *operation)
1613 {
1614 lrmd_event_data_t *op = NULL;
1615 const char *op_delay = NULL;
1616 const char *op_timeout = NULL;
1617 GHashTable *params = NULL;
1618
1619 xmlNode *primitive = NULL;
1620 const char *class = NULL;
1621
1622 const char *transition = NULL;
1623
1624 CRM_ASSERT(rsc_id && operation);
1625
1626 op = lrmd_new_event(rsc_id, operation, 0);
1627 op->type = lrmd_event_exec_complete;
1628 op->timeout = 0;
1629 op->start_delay = 0;
1630 lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
1631
1632 if (rsc_op == NULL) {
1633 CRM_LOG_ASSERT(pcmk__str_eq(operation, PCMK_ACTION_STOP,
1634 pcmk__str_casei));
1635 op->user_data = NULL;
1636
1637
1638
1639
1640
1641 op->params = pcmk__strkey_table(free, free);
1642
1643 pcmk__insert_dup(op->params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
1644
1645 crm_trace("Constructed %s op for %s", operation, rsc_id);
1646 return op;
1647 }
1648
1649 params = xml2list(rsc_op);
1650 g_hash_table_remove(params, CRM_META "_" PCMK__META_OP_TARGET_RC);
1651
1652 op_delay = crm_meta_value(params, PCMK_META_START_DELAY);
1653 pcmk__scan_min_int(op_delay, &op->start_delay, 0);
1654
1655 op_timeout = crm_meta_value(params, PCMK_META_TIMEOUT);
1656 pcmk__scan_min_int(op_timeout, &op->timeout, 0);
1657
1658 if (pcmk__guint_from_hash(params, CRM_META "_" PCMK_META_INTERVAL, 0,
1659 &(op->interval_ms)) != pcmk_rc_ok) {
1660 op->interval_ms = 0;
1661 }
1662
1663
1664
1665 primitive = pcmk__xe_first_child(rsc_op, PCMK_XE_PRIMITIVE, NULL, NULL);
1666 class = crm_element_value(primitive, PCMK_XA_CLASS);
1667
1668 if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
1669 && pcmk__str_eq(operation, PCMK_ACTION_MONITOR, pcmk__str_casei)
1670 && (op->interval_ms > 0)) {
1671
1672 op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
1673 if (op_timeout != NULL) {
1674 long long timeout_ms = crm_get_msec(op_timeout);
1675
1676 op->timeout = (int) QB_MIN(timeout_ms, INT_MAX);
1677 }
1678 }
1679
1680 if (!pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)) {
1681 op->params = params;
1682
1683 } else {
1684 rsc_history_t *entry = NULL;
1685
1686 if (lrm_state) {
1687 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
1688 }
1689
1690
1691
1692 if (!entry || !entry->stop_params) {
1693 op->params = params;
1694 } else {
1695
1696
1697 op->params = pcmk__strkey_table(free, free);
1698
1699 g_hash_table_foreach(params, copy_meta_keys, op->params);
1700 g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
1701 g_hash_table_destroy(params);
1702 params = NULL;
1703 }
1704 }
1705
1706
1707 if (op->timeout <= 0) {
1708 op->timeout = op->interval_ms;
1709 }
1710 if (op->start_delay < 0) {
1711 op->start_delay = 0;
1712 }
1713
1714 transition = crm_element_value(rsc_op, PCMK__XA_TRANSITION_KEY);
1715 CRM_CHECK(transition != NULL, return op);
1716
1717 op->user_data = pcmk__str_copy(transition);
1718
1719 if (op->interval_ms != 0) {
1720 if (pcmk__strcase_any_of(operation, PCMK_ACTION_START, PCMK_ACTION_STOP,
1721 NULL)) {
1722 crm_err("Start and Stop actions cannot have an interval: %u",
1723 op->interval_ms);
1724 op->interval_ms = 0;
1725 }
1726 }
1727
1728 crm_trace("Constructed %s op for %s: interval=%u",
1729 operation, rsc_id, op->interval_ms);
1730
1731 return op;
1732 }
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747 void
1748 controld_ack_event_directly(const char *to_host, const char *to_sys,
1749 const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
1750 const char *rsc_id)
1751 {
1752 xmlNode *reply = NULL;
1753 xmlNode *update, *iter;
1754 crm_node_t *peer = NULL;
1755
1756 CRM_CHECK(op != NULL, return);
1757 if (op->rsc_id == NULL) {
1758
1759 CRM_ASSERT(rsc_id != NULL);
1760 op->rsc_id = pcmk__str_copy(rsc_id);
1761 }
1762 if (to_sys == NULL) {
1763 to_sys = CRM_SYSTEM_TENGINE;
1764 }
1765
1766 peer = pcmk__get_node(0, controld_globals.our_nodename, NULL,
1767 pcmk__node_search_cluster_member);
1768 update = create_node_state_update(peer, node_update_none, NULL,
1769 __func__);
1770
1771 iter = pcmk__xe_create(update, PCMK__XE_LRM);
1772 crm_xml_add(iter, PCMK_XA_ID, controld_globals.our_uuid);
1773 iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCES);
1774 iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCE);
1775
1776 crm_xml_add(iter, PCMK_XA_ID, op->rsc_id);
1777
1778 controld_add_resource_history_xml(iter, rsc, op,
1779 controld_globals.our_nodename);
1780 reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
1781
1782 crm_log_xml_trace(update, "[direct ACK]");
1783
1784 crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
1785 op->rsc_id, op->op_type, op->interval_ms, op->user_data,
1786 crm_element_value(reply, PCMK_XA_REFERENCE));
1787
1788 if (relay_message(reply, TRUE) == FALSE) {
1789 crm_log_xml_err(reply, "Unable to route reply");
1790 }
1791
1792 free_xml(update);
1793 free_xml(reply);
1794 }
1795
1796 gboolean
1797 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
1798 {
1799 gboolean res = TRUE;
1800 GList *lrm_state_list = lrm_state_get_list();
1801 GList *state_entry;
1802
1803 for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
1804 lrm_state_t *lrm_state = state_entry->data;
1805
1806 if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
1807
1808 res = FALSE;
1809 }
1810 }
1811
1812 controld_set_fsa_input_flags(R_SENT_RSC_STOP);
1813 g_list_free(lrm_state_list); lrm_state_list = NULL;
1814 return res;
1815 }
1816
1817 struct stop_recurring_action_s {
1818 lrmd_rsc_info_t *rsc;
1819 lrm_state_t *lrm_state;
1820 };
1821
1822 static gboolean
1823 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
1824 {
1825 gboolean remove = FALSE;
1826 struct stop_recurring_action_s *event = user_data;
1827 active_op_t *op = value;
1828
1829 if ((op->interval_ms != 0)
1830 && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
1831
1832 crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
1833 remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
1834 }
1835
1836 return remove;
1837 }
1838
1839 static gboolean
1840 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
1841 {
1842 gboolean remove = FALSE;
1843 lrm_state_t *lrm_state = user_data;
1844 active_op_t *op = value;
1845
1846 if (op->interval_ms != 0) {
1847 crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
1848 (const char *) key);
1849 remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
1850 }
1851
1852 return remove;
1853 }
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865 static bool
1866 should_cancel_recurring(const char *rsc_id, const char *action, guint interval_ms)
1867 {
1868 if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0)
1869 && (strcmp(action, PCMK_ACTION_MIGRATE_TO) == 0)) {
1870
1871
1872
1873
1874 return false;
1875 }
1876
1877
1878 return (interval_ms == 0)
1879 && !pcmk__str_any_of(action, PCMK_ACTION_MONITOR,
1880 PCMK_ACTION_NOTIFY, NULL);
1881 }
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892 static const char *
1893 should_nack_action(const char *action)
1894 {
1895 if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)
1896 && pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
1897
1898 register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
1899 return "Not attempting start due to shutdown in progress";
1900 }
1901
1902 switch (controld_globals.fsa_state) {
1903 case S_NOT_DC:
1904 case S_POLICY_ENGINE:
1905 case S_TRANSITION_ENGINE:
1906 break;
1907 default:
1908 if (!pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
1909 return "Controller cannot attempt actions at this time";
1910 }
1911 break;
1912 }
1913 return NULL;
1914 }
1915
1916 static void
1917 do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg,
1918 struct ra_metadata_s *md)
1919 {
1920 int rc;
1921 int call_id = 0;
1922 char *op_id = NULL;
1923 lrmd_event_data_t *op = NULL;
1924 fsa_data_t *msg_data = NULL;
1925 const char *transition = NULL;
1926 const char *operation = NULL;
1927 const char *nack_reason = NULL;
1928
1929 CRM_CHECK((rsc != NULL) && (msg != NULL), return);
1930
1931 operation = crm_element_value(msg, PCMK_XA_OPERATION);
1932 CRM_CHECK(!pcmk__str_empty(operation), return);
1933
1934 transition = crm_element_value(msg, PCMK__XA_TRANSITION_KEY);
1935 if (pcmk__str_empty(transition)) {
1936 crm_log_xml_err(msg, "Missing transition number");
1937 }
1938
1939 if (lrm_state == NULL) {
1940
1941 crm_err("Cannot execute %s of %s: No executor connection "
1942 CRM_XS " transition_key=%s",
1943 operation, rsc->id, pcmk__s(transition, ""));
1944 synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID,
1945 PCMK_OCF_UNKNOWN_ERROR,
1946 "No executor connection");
1947 return;
1948 }
1949
1950 if (pcmk__str_any_of(operation, PCMK_ACTION_RELOAD,
1951 PCMK_ACTION_RELOAD_AGENT, NULL)) {
1952
1953
1954
1955
1956
1957 if ((md != NULL)
1958 && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
1959 operation = PCMK_ACTION_RELOAD;
1960 } else {
1961 operation = PCMK_ACTION_RELOAD_AGENT;
1962 }
1963 }
1964
1965 op = construct_op(lrm_state, msg, rsc->id, operation);
1966 CRM_CHECK(op != NULL, return);
1967
1968 if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) {
1969 guint removed = 0;
1970 struct stop_recurring_action_s data;
1971
1972 data.rsc = rsc;
1973 data.lrm_state = lrm_state;
1974 removed = g_hash_table_foreach_remove(lrm_state->active_ops,
1975 stop_recurring_action_by_rsc,
1976 &data);
1977
1978 if (removed) {
1979 crm_debug("Stopped %u recurring operation%s in preparation for "
1980 PCMK__OP_FMT, removed, pcmk__plural_s(removed),
1981 rsc->id, operation, op->interval_ms);
1982 }
1983 }
1984
1985
1986 crm_notice("Requesting local execution of %s operation for %s on %s "
1987 CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
1988 pcmk__readable_action(op->op_type, op->interval_ms), rsc->id,
1989 lrm_state->node_name, pcmk__s(transition, ""), rsc->id,
1990 operation, op->interval_ms);
1991
1992 nack_reason = should_nack_action(operation);
1993 if (nack_reason != NULL) {
1994 crm_notice("Discarding attempt to perform action %s on %s in state %s "
1995 "(shutdown=%s)", operation, rsc->id,
1996 fsa_state2string(controld_globals.fsa_state),
1997 pcmk__flag_text(controld_globals.fsa_input_register,
1998 R_SHUTDOWN));
1999
2000 lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
2001 nack_reason);
2002 controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
2003 lrmd_free_event(op);
2004 free(op_id);
2005 return;
2006 }
2007
2008 controld_record_pending_op(lrm_state->node_name, rsc, op);
2009
2010 op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
2011
2012 if (op->interval_ms > 0) {
2013
2014 cancel_op_key(lrm_state, rsc, op_id, FALSE);
2015 }
2016
2017 rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
2018 op->user_data, op->interval_ms,
2019 op->timeout, op->start_delay,
2020 op->params, &call_id);
2021 if (rc == pcmk_rc_ok) {
2022
2023
2024
2025 char *call_id_s = make_stop_id(rsc->id, call_id);
2026 active_op_t *pending = NULL;
2027
2028 pending = pcmk__assert_alloc(1, sizeof(active_op_t));
2029 crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
2030
2031 pending->call_id = call_id;
2032 pending->interval_ms = op->interval_ms;
2033 pending->op_type = pcmk__str_copy(operation);
2034 pending->op_key = pcmk__str_copy(op_id);
2035 pending->rsc_id = pcmk__str_copy(rsc->id);
2036 pending->start_time = time(NULL);
2037 pending->user_data = pcmk__str_copy(op->user_data);
2038 if (crm_element_value_epoch(msg, PCMK_OPT_SHUTDOWN_LOCK,
2039 &(pending->lock_time)) != pcmk_ok) {
2040 pending->lock_time = 0;
2041 }
2042 g_hash_table_replace(lrm_state->active_ops, call_id_s, pending);
2043
2044 if ((op->interval_ms > 0)
2045 && (op->start_delay > START_DELAY_THRESHOLD)) {
2046 int target_rc = PCMK_OCF_OK;
2047
2048 crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
2049 decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
2050 lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
2051 controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
2052 }
2053
2054 pending->params = op->params;
2055 op->params = NULL;
2056
2057 } else if (lrm_state_is_local(lrm_state)) {
2058 crm_err("Could not initiate %s action for resource %s locally: %s "
2059 CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
2060 fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
2061 PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
2062 process_lrm_event(lrm_state, op, NULL, NULL);
2063 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
2064
2065 } else {
2066 crm_err("Could not initiate %s action for resource %s remotely on %s: "
2067 "%s " CRM_XS " rc=%d",
2068 operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
2069 fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
2070 PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
2071 process_lrm_event(lrm_state, op, NULL, NULL);
2072 }
2073
2074 free(op_id);
2075 lrmd_free_event(op);
2076 }
2077
2078 void
2079 do_lrm_event(long long action,
2080 enum crmd_fsa_cause cause,
2081 enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
2082 {
2083 CRM_CHECK(FALSE, return);
2084 }
2085
2086 static char *
2087 unescape_newlines(const char *string)
2088 {
2089 char *pch = NULL;
2090 char *ret = NULL;
2091 static const char *escaped_newline = "\\n";
2092
2093 if (!string) {
2094 return NULL;
2095 }
2096
2097 ret = pcmk__str_copy(string);
2098 pch = strstr(ret, escaped_newline);
2099 while (pch != NULL) {
2100
2101
2102
2103 pch[0] = '\n';
2104 pch[1] = ' ';
2105 pch = strstr(pch, escaped_newline);
2106 }
2107
2108 return ret;
2109 }
2110
2111 static bool
2112 did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
2113 const char * op_type, guint interval_ms)
2114 {
2115 rsc_history_t *entry = NULL;
2116
2117 CRM_CHECK(lrm_state != NULL, return FALSE);
2118 CRM_CHECK(rsc_id != NULL, return FALSE);
2119 CRM_CHECK(op_type != NULL, return FALSE);
2120
2121 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
2122 if (entry == NULL || entry->failed == NULL) {
2123 return FALSE;
2124 }
2125
2126 if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
2127 && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
2128 && entry->failed->interval_ms == interval_ms) {
2129 return TRUE;
2130 }
2131
2132 return FALSE;
2133 }
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144 static void
2145 log_executor_event(const lrmd_event_data_t *op, const char *op_key,
2146 const char *node_name, gboolean confirmed)
2147 {
2148 int log_level = LOG_ERR;
2149 GString *str = g_string_sized_new(100);
2150
2151 pcmk__g_strcat(str,
2152 "Result of ",
2153 pcmk__readable_action(op->op_type, op->interval_ms),
2154 " operation for ", op->rsc_id, NULL);
2155
2156 if (node_name != NULL) {
2157 pcmk__g_strcat(str, " on ", node_name, NULL);
2158 }
2159
2160 switch (op->op_status) {
2161 case PCMK_EXEC_DONE:
2162 log_level = LOG_NOTICE;
2163 pcmk__g_strcat(str, ": ", services_ocf_exitcode_str(op->rc), NULL);
2164 break;
2165
2166 case PCMK_EXEC_TIMEOUT:
2167 pcmk__g_strcat(str,
2168 ": ", pcmk_exec_status_str(op->op_status), " after ",
2169 pcmk__readable_interval(op->timeout), NULL);
2170 break;
2171
2172 case PCMK_EXEC_CANCELLED:
2173 log_level = LOG_INFO;
2174
2175
2176
2177
2178
2179
2180
2181
2182 #ifdef __clang__
2183 #ifdef __has_c_attribute
2184 #if __has_attribute(fallthrough)
2185 __attribute__((fallthrough));
2186 #endif
2187 #endif
2188 #endif
2189
2190 default:
2191 pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status),
2192 NULL);
2193 }
2194
2195 if ((op->exit_reason != NULL)
2196 && ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) {
2197
2198 pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL);
2199 }
2200
2201 g_string_append(str, " " CRM_XS);
2202 g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s",
2203 (confirmed? "" : "un"), op->call_id, op_key);
2204 if (op->op_status == PCMK_EXEC_DONE) {
2205 g_string_append_printf(str, " rc=%d", op->rc);
2206 }
2207
2208 do_crm_log(log_level, "%s", str->str);
2209 g_string_free(str, TRUE);
2210
2211
2212
2213
2214 if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) {
2215 char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output",
2216 op->rsc_id, op->op_type,
2217 op->interval_ms, node_name);
2218
2219 crm_log_output(LOG_NOTICE, prefix, op->output);
2220 free(prefix);
2221 }
2222 }
2223
2224 void
2225 process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
2226 active_op_t *pending, const xmlNode *action_xml)
2227 {
2228 char *op_id = NULL;
2229 char *op_key = NULL;
2230
2231 gboolean remove = FALSE;
2232 gboolean removed = FALSE;
2233 bool need_direct_ack = FALSE;
2234 lrmd_rsc_info_t *rsc = NULL;
2235 const char *node_name = NULL;
2236
2237 CRM_CHECK(op != NULL, return);
2238 CRM_CHECK(op->rsc_id != NULL, return);
2239
2240
2241 if (compare_version(controld_globals.dc_version, "3.2.0") < 0) {
2242 switch (op->op_status) {
2243 case PCMK_EXEC_NOT_CONNECTED:
2244 lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
2245 PCMK_EXEC_ERROR, op->exit_reason);
2246 break;
2247 case PCMK_EXEC_INVALID:
2248 lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
2249 op->exit_reason);
2250 break;
2251 default:
2252 break;
2253 }
2254 }
2255
2256 op_id = make_stop_id(op->rsc_id, op->call_id);
2257 op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
2258
2259
2260 if (lrm_state) {
2261 rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
2262 }
2263 if ((rsc == NULL) && action_xml) {
2264 xmlNode *xml = pcmk__xe_first_child(action_xml, PCMK_XE_PRIMITIVE, NULL,
2265 NULL);
2266
2267 const char *standard = crm_element_value(xml, PCMK_XA_CLASS);
2268 const char *provider = crm_element_value(xml, PCMK_XA_PROVIDER);
2269 const char *type = crm_element_value(xml, PCMK_XA_TYPE);
2270
2271 if (standard && type) {
2272 crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
2273 op->rsc_id, standard,
2274 (provider? ":" : ""), (provider? provider : ""), type);
2275 rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
2276 } else {
2277 crm_err("Can't process %s result because %s agent information not cached or in XML",
2278 op_key, op->rsc_id);
2279 }
2280 }
2281
2282
2283 if (lrm_state) {
2284 node_name = lrm_state->node_name;
2285 } else if (action_xml) {
2286 node_name = crm_element_value(action_xml, PCMK__META_ON_NODE);
2287 }
2288
2289 if(pending == NULL) {
2290 remove = TRUE;
2291 if (lrm_state) {
2292 pending = g_hash_table_lookup(lrm_state->active_ops, op_id);
2293 }
2294 }
2295
2296 if (op->op_status == PCMK_EXEC_ERROR) {
2297 switch(op->rc) {
2298 case PCMK_OCF_NOT_RUNNING:
2299 case PCMK_OCF_RUNNING_PROMOTED:
2300 case PCMK_OCF_DEGRADED:
2301 case PCMK_OCF_DEGRADED_PROMOTED:
2302
2303 op->op_status = PCMK_EXEC_DONE;
2304 break;
2305 default:
2306
2307 break;
2308 }
2309 }
2310
2311 if (op->op_status != PCMK_EXEC_CANCELLED) {
2312
2313
2314
2315
2316 need_direct_ack = TRUE;
2317
2318 if (controld_action_is_recordable(op->op_type)) {
2319 if (node_name && rsc) {
2320
2321 time_t lock_time = (pending == NULL)? 0 : pending->lock_time;
2322
2323 controld_update_resource_history(node_name, rsc, op, lock_time);
2324 need_direct_ack = FALSE;
2325
2326 } else if (op->rsc_deleted) {
2327
2328
2329
2330
2331 crm_notice("Not recording %s result in CIB because "
2332 "resource information was removed since it was initiated",
2333 op_key);
2334 } else {
2335
2336
2337
2338
2339 crm_err("Unable to record %s result in CIB: %s", op_key,
2340 (node_name? "No resource information" : "No node name"));
2341 }
2342 }
2343
2344 } else if (op->interval_ms == 0) {
2345
2346
2347
2348
2349 need_direct_ack = TRUE;
2350
2351 } else if (pending == NULL) {
2352
2353
2354
2355
2356 } else if (op->user_data == NULL) {
2357
2358
2359
2360 crm_err("Recurring operation %s was cancelled without transition information",
2361 op_key);
2362
2363 } else if (pcmk_is_set(pending->flags, active_op_remove)) {
2364
2365
2366
2367 if (lrm_state) {
2368 controld_delete_action_history(op);
2369 }
2370
2371
2372
2373
2374
2375
2376
2377 if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
2378 pending->op_type, pending->interval_ms)) {
2379 need_direct_ack = TRUE;
2380 }
2381
2382 } else if (op->rsc_deleted) {
2383
2384
2385
2386
2387 crm_debug("Recurring op %s was cancelled due to resource deletion",
2388 op_key);
2389 need_direct_ack = TRUE;
2390
2391 } else {
2392
2393
2394
2395
2396 }
2397
2398 if (need_direct_ack) {
2399 controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
2400 }
2401
2402 if(remove == FALSE) {
2403
2404 removed = TRUE;
2405
2406 } else if (lrm_state && ((op->interval_ms == 0)
2407 || (op->op_status == PCMK_EXEC_CANCELLED))) {
2408
2409 gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id);
2410
2411 if (op->interval_ms != 0) {
2412 removed = TRUE;
2413 } else if (found) {
2414 removed = TRUE;
2415 crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
2416 op_key, op->call_id, op_id,
2417 g_hash_table_size(lrm_state->active_ops));
2418 }
2419 }
2420
2421 log_executor_event(op, op_key, node_name, removed);
2422
2423 if (lrm_state) {
2424 if (!pcmk__str_eq(op->op_type, PCMK_ACTION_META_DATA,
2425 pcmk__str_casei)) {
2426 crmd_alert_resource_op(lrm_state->node_name, op);
2427 } else if (rsc && (op->rc == PCMK_OCF_OK)) {
2428 char *metadata = unescape_newlines(op->output);
2429
2430 controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata);
2431 free(metadata);
2432 }
2433 }
2434
2435 if (op->rsc_deleted) {
2436 crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
2437 if (lrm_state) {
2438 delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL,
2439 true);
2440 }
2441 }
2442
2443
2444
2445
2446 controld_trigger_fsa();
2447 if (lrm_state && rsc) {
2448 update_history_cache(lrm_state, rsc, op);
2449 }
2450
2451 lrmd_free_rsc_info(rsc);
2452 free(op_key);
2453 free(op_id);
2454 }
![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
![[previous]](../icons/left.png)
![[last]](../icons/n_last.png)
![[bottom]](../icons/n_bottom.png){kind=icon}