This source file includes following definitions.
- lrm_connection_destroy
- make_stop_id
- copy_instance_keys
- copy_meta_keys
- history_remove_recurring_op
- history_free_recurring_ops
- history_free
- update_history_cache
- send_task_ok_ack
- op_node_name
- lrm_op_callback
- try_local_executor_connect
- do_lrm_control
- lrm_state_verify_stopped
- is_rsc_active
- build_active_RAs
- controld_query_executor_state
- controld_rc2event
- controld_trigger_delete_refresh
- notify_deleted
- lrm_remove_deleted_rsc
- lrm_remove_deleted_op
- delete_rsc_entry
- last_failed_matches_op
- lrm_clear_last_failure
- cancel_op
- cancel_action_by_key
- cancel_op_key
- get_lrm_resource
- delete_resource
- get_fake_call_id
- fake_op_status
- force_reprobe
- synthesize_lrmd_failure
- lrm_op_target
- fail_lrm_resource
- handle_reprobe_op
- do_lrm_cancel
- do_lrm_delete
- new_metadata_cb_data
- free_metadata_cb_data
- metadata_complete
- do_lrm_invoke
- construct_op
- controld_ack_event_directly
- verify_stopped
- stop_recurring_action_by_rsc
- stop_recurring_actions
- should_cancel_recurring
- should_nack_action
- do_lrm_rsc_op
- do_lrm_event
- unescape_newlines
- did_lrm_rsc_op_fail
- log_executor_event
- process_lrm_event
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <regex.h>
13 #include <sys/param.h>
14 #include <sys/types.h>
15 #include <sys/wait.h>
16
17 #include <crm/crm.h>
18 #include <crm/lrmd.h>
19 #include <crm/services.h>
20 #include <crm/common/xml.h>
21 #include <crm/pengine/rules.h>
22 #include <crm/lrmd_internal.h>
23
24 #include <pacemaker-internal.h>
25 #include <pacemaker-controld.h>
26
27 #define START_DELAY_THRESHOLD 5 * 60 * 1000
28 #define MAX_LRM_REG_FAILS 30
29
30 struct delete_event_s {
31 int rc;
32 const char *rsc;
33 lrm_state_t *lrm_state;
34 };
35
36 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
37 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
38 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
39
40 static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state,
41 const xmlNode *rsc_op,
42 const char *rsc_id,
43 const char *operation);
44 static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
45 xmlNode *msg, struct ra_metadata_s *md);
46
47 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
48 int log_level);
49
50 static void
51 lrm_connection_destroy(void)
52 {
53 if (pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) {
54 crm_crit("Lost connection to local executor");
55 register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
56 controld_clear_fsa_input_flags(R_LRM_CONNECTED);
57 }
58 }
59
60 static char *
61 make_stop_id(const char *rsc, int call_id)
62 {
63 return crm_strdup_printf("%s:%d", rsc, call_id);
64 }
65
66 static void
67 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
68 {
69 if (strstr(key, CRM_META "_") == NULL) {
70 pcmk__insert_dup(user_data, (const char *) key, (const char *) value);
71 }
72 }
73
74 static void
75 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
76 {
77 if (strstr(key, CRM_META "_") != NULL) {
78 pcmk__insert_dup(user_data, (const char *) key, (const char *) value);
79 }
80 }
81
82
83
84
85
86
87
88
89
90
91 static gboolean
92 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
93 {
94 GList *iter;
95
96 for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
97 lrmd_event_data_t *existing = iter->data;
98
99 if ((op->interval_ms == existing->interval_ms)
100 && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
101 && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
102
103 history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
104 lrmd_free_event(existing);
105 return TRUE;
106 }
107 }
108 return FALSE;
109 }
110
111
112
113
114
115
116
117 static void
118 history_free_recurring_ops(rsc_history_t *history)
119 {
120 GList *iter;
121
122 for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
123 lrmd_free_event(iter->data);
124 }
125 g_list_free(history->recurring_op_list);
126 history->recurring_op_list = NULL;
127 }
128
129
130
131
132
133
134
135 void
136 history_free(gpointer data)
137 {
138 rsc_history_t *history = (rsc_history_t*)data;
139
140 if (history->stop_params) {
141 g_hash_table_destroy(history->stop_params);
142 }
143
144
145 free(history->rsc.type);
146 free(history->rsc.standard);
147 free(history->rsc.provider);
148
149 lrmd_free_event(history->failed);
150 lrmd_free_event(history->last);
151 free(history->id);
152 history_free_recurring_ops(history);
153 free(history);
154 }
155
156 static void
157 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
158 {
159 int target_rc = 0;
160 rsc_history_t *entry = NULL;
161
162 if (op->rsc_deleted) {
163 crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
164 controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
165 NULL, crmd_cib_smart_opt());
166 return;
167 }
168
169 if (pcmk__str_eq(op->op_type, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
170 return;
171 }
172
173 crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
174
175 entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
176 if (entry == NULL && rsc) {
177 entry = pcmk__assert_alloc(1, sizeof(rsc_history_t));
178 entry->id = pcmk__str_copy(op->rsc_id);
179 g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
180
181 entry->rsc.id = entry->id;
182 entry->rsc.type = pcmk__str_copy(rsc->type);
183 entry->rsc.standard = pcmk__str_copy(rsc->standard);
184 entry->rsc.provider = pcmk__str_copy(rsc->provider);
185
186 } else if (entry == NULL) {
187 crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
188 return;
189 }
190
191 entry->last_callid = op->call_id;
192 target_rc = rsc_op_expected_rc(op);
193 if (op->op_status == PCMK_EXEC_CANCELLED) {
194 if (op->interval_ms > 0) {
195 crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
196 op->rsc_id, op->op_type, op->interval_ms);
197 history_remove_recurring_op(entry, op);
198 return;
199 } else {
200 crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
201 op->rsc_id, op->op_type, op->interval_ms, op->rc,
202 op->op_status);
203 }
204
205 } else if (did_rsc_op_fail(op, target_rc)) {
206
207
208
209 if (entry->failed) {
210 lrmd_free_event(entry->failed);
211 }
212 entry->failed = lrmd_copy_event(op);
213
214 } else if (op->interval_ms == 0) {
215 if (entry->last) {
216 lrmd_free_event(entry->last);
217 }
218 entry->last = lrmd_copy_event(op);
219
220 if (op->params && pcmk__strcase_any_of(op->op_type, PCMK_ACTION_START,
221 PCMK_ACTION_RELOAD,
222 PCMK_ACTION_RELOAD_AGENT,
223 PCMK_ACTION_MONITOR, NULL)) {
224 if (entry->stop_params) {
225 g_hash_table_destroy(entry->stop_params);
226 }
227 entry->stop_params = pcmk__strkey_table(free, free);
228
229 g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
230 }
231 }
232
233 if (op->interval_ms > 0) {
234
235 history_remove_recurring_op(entry, op);
236
237 crm_trace("Adding recurring op: " PCMK__OP_FMT,
238 op->rsc_id, op->op_type, op->interval_ms);
239 entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
240
241 } else if ((entry->recurring_op_list != NULL)
242 && !pcmk__str_eq(op->op_type, PCMK_ACTION_MONITOR,
243 pcmk__str_casei)) {
244 crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
245 g_list_length(entry->recurring_op_list), op->rsc_id,
246 op->op_type, op->interval_ms);
247 history_free_recurring_ops(entry);
248 }
249 }
250
251
252
253
254
255
256
257
258
259
260
261
262
263 static void
264 send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input,
265 const char *rsc_id, const lrmd_rsc_info_t *rsc,
266 const char *task, const char *ack_host, const char *ack_sys)
267 {
268 lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
269
270 lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
271 controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
272 lrmd_free_event(op);
273 }
274
275 static inline const char *
276 op_node_name(lrmd_event_data_t *op)
277 {
278 return pcmk__s(op->remote_nodename, controld_globals.our_nodename);
279 }
280
281 void
282 lrm_op_callback(lrmd_event_data_t * op)
283 {
284 CRM_CHECK(op != NULL, return);
285 switch (op->type) {
286 case lrmd_event_disconnect:
287 if (op->remote_nodename == NULL) {
288
289
290
291 lrm_connection_destroy();
292 }
293 break;
294
295 case lrmd_event_exec_complete:
296 {
297 lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
298
299 pcmk__assert(lrm_state != NULL);
300 process_lrm_event(lrm_state, op, NULL, NULL);
301 }
302 break;
303
304 default:
305 break;
306 }
307 }
308
309 static void
310 try_local_executor_connect(long long action, fsa_data_t *msg_data,
311 lrm_state_t *lrm_state)
312 {
313 int rc = pcmk_rc_ok;
314
315 crm_debug("Connecting to the local executor");
316
317
318 rc = controld_connect_local_executor(lrm_state);
319 if (rc == pcmk_rc_ok) {
320 controld_set_fsa_input_flags(R_LRM_CONNECTED);
321 crm_info("Connection to the local executor established");
322 return;
323 }
324
325
326 if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
327 crm_warn("Failed to connect to the local executor %d time%s "
328 "(%d max): %s", lrm_state->num_lrm_register_fails,
329 pcmk__plural_s(lrm_state->num_lrm_register_fails),
330 MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
331 controld_start_wait_timer();
332 crmd_fsa_stall(FALSE);
333 return;
334 }
335
336
337 crm_err("Failed to connect to the executor the max allowed "
338 "%d time%s: %s", lrm_state->num_lrm_register_fails,
339 pcmk__plural_s(lrm_state->num_lrm_register_fails),
340 pcmk_rc_str(rc));
341 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
342 }
343
344
345 void
346 do_lrm_control(long long action,
347 enum crmd_fsa_cause cause,
348 enum crmd_fsa_state cur_state,
349 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
350 {
351
352
353
354
355
356 lrm_state_t *lrm_state = NULL;
357
358 if (controld_globals.our_nodename == NULL) {
359 return;
360 }
361 lrm_state = lrm_state_find_or_create(controld_globals.our_nodename);
362 if (lrm_state == NULL) {
363 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
364 return;
365 }
366
367 if (action & A_LRM_DISCONNECT) {
368 if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
369 if (action == A_LRM_DISCONNECT) {
370 crmd_fsa_stall(FALSE);
371 return;
372 }
373 }
374
375 controld_clear_fsa_input_flags(R_LRM_CONNECTED);
376 lrm_state_disconnect(lrm_state);
377 lrm_state_reset_tables(lrm_state, FALSE);
378 }
379
380 if (action & A_LRM_CONNECT) {
381 try_local_executor_connect(action, msg_data, lrm_state);
382 }
383
384 if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
385 crm_err("Unexpected action %s in %s", fsa_action2string(action),
386 __func__);
387 }
388 }
389
390 static gboolean
391 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
392 {
393 int counter = 0;
394 gboolean rc = TRUE;
395 const char *when = "lrm disconnect";
396
397 GHashTableIter gIter;
398 const char *key = NULL;
399 rsc_history_t *entry = NULL;
400 active_op_t *pending = NULL;
401
402 crm_debug("Checking for active resources before exit");
403
404 if (cur_state == S_TERMINATE) {
405 log_level = LOG_ERR;
406 when = "shutdown";
407
408 } else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
409 when = "shutdown... waiting";
410 }
411
412 if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) {
413 guint removed = g_hash_table_foreach_remove(lrm_state->active_ops,
414 stop_recurring_actions,
415 lrm_state);
416 guint nremaining = g_hash_table_size(lrm_state->active_ops);
417
418 if (removed || nremaining) {
419 crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
420 removed, pcmk__plural_s(removed), when, nremaining);
421 }
422 }
423
424 if (lrm_state->active_ops != NULL) {
425 g_hash_table_iter_init(&gIter, lrm_state->active_ops);
426 while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
427
428 if (pending->interval_ms == 0) {
429 counter++;
430 }
431 }
432 }
433
434 if (counter > 0) {
435 do_crm_log(log_level, "%d pending executor operation%s at %s",
436 counter, pcmk__plural_s(counter), when);
437
438 if ((cur_state == S_TERMINATE)
439 || !pcmk_is_set(controld_globals.fsa_input_register,
440 R_SENT_RSC_STOP)) {
441 g_hash_table_iter_init(&gIter, lrm_state->active_ops);
442 while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
443 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
444 }
445
446 } else {
447 rc = FALSE;
448 }
449 return rc;
450 }
451
452 if (lrm_state->resource_history == NULL) {
453 return rc;
454 }
455
456 if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
457
458 when = "shutdown";
459 }
460
461 counter = 0;
462 g_hash_table_iter_init(&gIter, lrm_state->resource_history);
463 while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
464 if (is_rsc_active(lrm_state, entry->id) == FALSE) {
465 continue;
466 }
467
468 counter++;
469 if (log_level == LOG_ERR) {
470 crm_info("Found %s active at %s", entry->id, when);
471 } else {
472 crm_trace("Found %s active at %s", entry->id, when);
473 }
474 if (lrm_state->active_ops != NULL) {
475 GHashTableIter hIter;
476
477 g_hash_table_iter_init(&hIter, lrm_state->active_ops);
478 while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
479 if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
480 crm_notice("%sction %s (%s) incomplete at %s",
481 pending->interval_ms == 0 ? "A" : "Recurring a",
482 key, pending->op_key, when);
483 }
484 }
485 }
486 }
487
488 if (counter) {
489 crm_err("%d resource%s active at %s",
490 counter, (counter == 1)? " was" : "s were", when);
491 }
492
493 return rc;
494 }
495
496 static gboolean
497 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
498 {
499 rsc_history_t *entry = NULL;
500
501 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
502 if (entry == NULL || entry->last == NULL) {
503 return FALSE;
504 }
505
506 crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
507 entry->last->interval_ms, entry->last->rc);
508 if ((entry->last->rc == PCMK_OCF_OK)
509 && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_STOP,
510 pcmk__str_casei)) {
511 return FALSE;
512
513 } else if (entry->last->rc == PCMK_OCF_OK
514 && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_MIGRATE_TO,
515 pcmk__str_casei)) {
516
517 return FALSE;
518
519 } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
520 return FALSE;
521
522 } else if ((entry->last->interval_ms == 0)
523 && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
524
525 return FALSE;
526 }
527
528 return TRUE;
529 }
530
531 static gboolean
532 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
533 {
534 GHashTableIter iter;
535 rsc_history_t *entry = NULL;
536
537 g_hash_table_iter_init(&iter, lrm_state->resource_history);
538 while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
539
540 GList *gIter = NULL;
541 xmlNode *xml_rsc = pcmk__xe_create(rsc_list, PCMK__XE_LRM_RESOURCE);
542
543 crm_xml_add(xml_rsc, PCMK_XA_ID, entry->id);
544 crm_xml_add(xml_rsc, PCMK_XA_TYPE, entry->rsc.type);
545 crm_xml_add(xml_rsc, PCMK_XA_CLASS, entry->rsc.standard);
546 crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, entry->rsc.provider);
547
548 if (entry->last && entry->last->params) {
549 static const char *name = CRM_META "_" PCMK__META_CONTAINER;
550 const char *container = g_hash_table_lookup(entry->last->params,
551 name);
552
553 if (container) {
554 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
555 crm_xml_add(xml_rsc, PCMK__META_CONTAINER, container);
556 }
557 }
558 controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed,
559 lrm_state->node_name);
560 controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last,
561 lrm_state->node_name);
562 for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
563 controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data,
564 lrm_state->node_name);
565 }
566 }
567
568 return FALSE;
569 }
570
571 xmlNode *
572 controld_query_executor_state(void)
573 {
574 xmlNode *xml_state = NULL;
575 xmlNode *xml_data = NULL;
576 xmlNode *rsc_list = NULL;
577 crm_node_t *peer = NULL;
578 lrm_state_t *lrm_state = lrm_state_find(controld_globals.our_nodename);
579
580 if (!lrm_state) {
581 crm_err("Could not find executor state for node %s",
582 controld_globals.our_nodename);
583 return NULL;
584 }
585
586 peer = pcmk__get_node(0, lrm_state->node_name, NULL, pcmk__node_search_any);
587 CRM_CHECK(peer != NULL, return NULL);
588
589 xml_state = create_node_state_update(peer,
590 node_update_cluster|node_update_peer,
591 NULL, __func__);
592 if (xml_state == NULL) {
593 return NULL;
594 }
595
596 xml_data = pcmk__xe_create(xml_state, PCMK__XE_LRM);
597 crm_xml_add(xml_data, PCMK_XA_ID, peer->uuid);
598 rsc_list = pcmk__xe_create(xml_data, PCMK__XE_LRM_RESOURCES);
599
600
601 build_active_RAs(lrm_state, rsc_list);
602
603 crm_log_xml_trace(xml_state, "Current executor state");
604
605 return xml_state;
606 }
607
608
609
610
611
612
613
614
615 void
616 controld_rc2event(lrmd_event_data_t *event, int rc)
617 {
618
619
620
621 switch (rc) {
622 case pcmk_rc_ok:
623 lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
624 break;
625 case EACCES:
626 lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
627 PCMK_EXEC_ERROR, NULL);
628 break;
629 default:
630 lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
631 NULL);
632 break;
633 }
634 }
635
636
637
638
639
640
641
642
643
644
645
646
647 void
648 controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
649 {
650 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
651 char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
652
653 crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
654 cib__update_node_attr(controld_globals.logger_out,
655 controld_globals.cib_conn, cib_none,
656 PCMK_XE_CRM_CONFIG, NULL, NULL, NULL, NULL,
657 "last-lrm-refresh", now_s, NULL, NULL);
658 free(now_s);
659 }
660 }
661
662 static void
663 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
664 {
665 lrmd_event_data_t *op = NULL;
666 const char *from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM);
667 const char *from_host = crm_element_value(input->msg, PCMK__XA_SRC);
668
669 crm_info("Notifying %s on %s that %s was%s deleted",
670 from_sys, (from_host? from_host : "localhost"), rsc_id,
671 ((rc == pcmk_ok)? "" : " not"));
672 op = construct_op(lrm_state, input->xml, rsc_id, PCMK_ACTION_DELETE);
673 controld_rc2event(op, pcmk_legacy2rc(rc));
674 controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
675 lrmd_free_event(op);
676 controld_trigger_delete_refresh(from_sys, rsc_id);
677 }
678
679 static gboolean
680 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
681 {
682 struct delete_event_s *event = user_data;
683 struct pending_deletion_op_s *op = value;
684
685 if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
686 notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
687 return TRUE;
688 }
689 return FALSE;
690 }
691
692 static gboolean
693 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
694 {
695 const char *rsc = user_data;
696 active_op_t *pending = value;
697
698 if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
699 crm_info("Removing op %s:%d for deleted resource %s",
700 pending->op_key, pending->call_id, rsc);
701 return TRUE;
702 }
703 return FALSE;
704 }
705
706 static void
707 delete_rsc_entry(lrm_state_t *lrm_state, ha_msg_input_t *input,
708 const char *rsc_id, GHashTableIter *rsc_iter, int rc,
709 const char *user_name, bool from_cib)
710 {
711 struct delete_event_s event;
712
713 CRM_CHECK(rsc_id != NULL, return);
714
715 if (rc == pcmk_ok) {
716 char *rsc_id_copy = pcmk__str_copy(rsc_id);
717
718 if (rsc_iter) {
719 g_hash_table_iter_remove(rsc_iter);
720 } else {
721 g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
722 }
723
724 if (from_cib) {
725 controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
726 user_name, crmd_cib_smart_opt());
727 }
728 g_hash_table_foreach_remove(lrm_state->active_ops,
729 lrm_remove_deleted_op, rsc_id_copy);
730 free(rsc_id_copy);
731 }
732
733 if (input) {
734 notify_deleted(lrm_state, input, rsc_id, rc);
735 }
736
737 event.rc = rc;
738 event.rsc = rsc_id;
739 event.lrm_state = lrm_state;
740 g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
741 }
742
743 static inline gboolean
744 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
745 {
746 if (entry == NULL) {
747 return FALSE;
748 }
749 if (op == NULL) {
750 return TRUE;
751 }
752 return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
753 && (interval_ms == entry->failed->interval_ms));
754 }
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769 void
770 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
771 const char *operation, guint interval_ms)
772 {
773 lrm_state_t *lrm_state = lrm_state_find(node_name);
774
775 if (lrm_state == NULL) {
776 return;
777 }
778 if (lrm_state->resource_history != NULL) {
779 rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
780 rsc_id);
781
782 if (last_failed_matches_op(entry, operation, interval_ms)) {
783 lrmd_free_event(entry->failed);
784 entry->failed = NULL;
785 }
786 }
787 }
788
789
790 static gboolean
791 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
792 {
793 int rc = pcmk_ok;
794 char *local_key = NULL;
795 active_op_t *pending = NULL;
796
797 CRM_CHECK(op != 0, return FALSE);
798 CRM_CHECK(rsc_id != NULL, return FALSE);
799 if (key == NULL) {
800 local_key = make_stop_id(rsc_id, op);
801 key = local_key;
802 }
803 pending = g_hash_table_lookup(lrm_state->active_ops, key);
804
805 if (pending) {
806 if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
807 controld_set_active_op_flags(pending, active_op_remove);
808 crm_debug("Scheduling %s for removal", key);
809 }
810
811 if (pcmk_is_set(pending->flags, active_op_cancelled)) {
812 crm_debug("Operation %s already cancelled", key);
813 free(local_key);
814 return FALSE;
815 }
816 controld_set_active_op_flags(pending, active_op_cancelled);
817
818 } else {
819 crm_info("No pending op found for %s", key);
820 free(local_key);
821 return FALSE;
822 }
823
824 crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
825 rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
826 pending->interval_ms);
827 if (rc == pcmk_ok) {
828 crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
829 free(local_key);
830 return TRUE;
831 }
832
833 crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
834
835
836
837
838
839
840
841
842
843 free(local_key);
844 return FALSE;
845 }
846
847 struct cancel_data {
848 gboolean done;
849 gboolean remove;
850 const char *key;
851 lrmd_rsc_info_t *rsc;
852 lrm_state_t *lrm_state;
853 };
854
855 static gboolean
856 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
857 {
858 gboolean remove = FALSE;
859 struct cancel_data *data = user_data;
860 active_op_t *op = value;
861
862 if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
863 data->done = TRUE;
864 remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
865 }
866 return remove;
867 }
868
869 static gboolean
870 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
871 {
872 guint removed = 0;
873 struct cancel_data data;
874
875 CRM_CHECK(rsc != NULL, return FALSE);
876 CRM_CHECK(key != NULL, return FALSE);
877
878 data.key = key;
879 data.rsc = rsc;
880 data.done = FALSE;
881 data.remove = remove;
882 data.lrm_state = lrm_state;
883
884 removed = g_hash_table_foreach_remove(lrm_state->active_ops,
885 cancel_action_by_key, &data);
886 crm_trace("Removed %u op cache entries, new size: %u",
887 removed, g_hash_table_size(lrm_state->active_ops));
888 return data.done;
889 }
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908 static int
909 get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml,
910 gboolean do_create, lrmd_rsc_info_t **rsc_info)
911 {
912 const char *id = pcmk__xe_id(rsc_xml);
913
914 CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
915 CRM_CHECK(id, return -EINVAL);
916
917 if (lrm_state_is_connected(lrm_state) == FALSE) {
918 return -ENOTCONN;
919 }
920
921 crm_trace("Retrieving resource information for %s from the executor", id);
922 *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
923
924
925 if (!*rsc_info) {
926 const char *long_id = crm_element_value(rsc_xml, PCMK__XA_LONG_ID);
927
928 if (long_id) {
929 *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
930 }
931 }
932
933 if ((*rsc_info == NULL) && do_create) {
934 const char *class = crm_element_value(rsc_xml, PCMK_XA_CLASS);
935 const char *provider = crm_element_value(rsc_xml, PCMK_XA_PROVIDER);
936 const char *type = crm_element_value(rsc_xml, PCMK_XA_TYPE);
937 int rc;
938
939 crm_trace("Registering resource %s with the executor", id);
940 rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
941 lrmd_opt_drop_recurring);
942 if (rc != pcmk_ok) {
943 fsa_data_t *msg_data = NULL;
944
945 crm_err("Could not register resource %s with the executor on %s: %s "
946 CRM_XS " rc=%d",
947 id, lrm_state->node_name, pcmk_strerror(rc), rc);
948
949
950
951
952
953 if (lrm_state_is_local(lrm_state) == TRUE) {
954 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
955 }
956 return rc;
957 }
958
959 *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
960 }
961 return *rsc_info? pcmk_ok : -ENODEV;
962 }
963
964 static void
965 delete_resource(lrm_state_t *lrm_state, const char *id, lrmd_rsc_info_t *rsc,
966 GHashTableIter *iter, const char *sys, const char *user,
967 ha_msg_input_t *request, bool unregister, bool from_cib)
968 {
969 int rc = pcmk_ok;
970
971 crm_info("Removing resource %s from executor for %s%s%s",
972 id, sys, (user? " as " : ""), (user? user : ""));
973
974 if (rsc && unregister) {
975 rc = lrm_state_unregister_rsc(lrm_state, id, 0);
976 }
977
978 if (rc == pcmk_ok) {
979 crm_trace("Resource %s deleted from executor", id);
980 } else if (rc == -EINPROGRESS) {
981 crm_info("Deletion of resource '%s' from executor is pending", id);
982 if (request) {
983 struct pending_deletion_op_s *op = NULL;
984 char *ref = crm_element_value_copy(request->msg, PCMK_XA_REFERENCE);
985
986 op = pcmk__assert_alloc(1, sizeof(struct pending_deletion_op_s));
987 op->rsc = pcmk__str_copy(rsc->id);
988 op->input = copy_ha_msg_input(request);
989 g_hash_table_insert(lrm_state->deletion_ops, ref, op);
990 }
991 return;
992 } else {
993 crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
994 CRM_XS " rc=%d", id, sys, (user? " as " : ""),
995 (user? user : ""), pcmk_strerror(rc), rc);
996 }
997
998 delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib);
999 }
1000
1001 static int
1002 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
1003 {
1004 int call_id = 999999999;
1005 rsc_history_t *entry = NULL;
1006
1007 if(lrm_state) {
1008 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
1009 }
1010
1011
1012
1013
1014 if (entry) {
1015 call_id = entry->last_callid + 1;
1016 }
1017
1018 if (call_id < 0) {
1019 call_id = 1;
1020 }
1021 return call_id;
1022 }
1023
1024 static void
1025 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
1026 enum ocf_exitcode op_exitcode, const char *exit_reason)
1027 {
1028 op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
1029
1030 op->t_run = time(NULL);
1031 op->t_rcchange = op->t_run;
1032 lrmd__set_result(op, op_exitcode, op_status, exit_reason);
1033 }
1034
1035 static void
1036 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
1037 const char *from_host, const char *user_name,
1038 gboolean is_remote_node, bool reprobe_all_nodes)
1039 {
1040 GHashTableIter gIter;
1041 rsc_history_t *entry = NULL;
1042
1043 crm_info("Clearing resource history on node %s", lrm_state->node_name);
1044 g_hash_table_iter_init(&gIter, lrm_state->resource_history);
1045 while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
1046
1047
1048
1049 bool unregister = true;
1050
1051 if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
1052 unregister = false;
1053
1054 if (reprobe_all_nodes) {
1055 lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
1056
1057 if (remote_lrm_state != NULL) {
1058
1059
1060
1061 force_reprobe(remote_lrm_state, from_sys, from_host,
1062 user_name, TRUE, reprobe_all_nodes);
1063 }
1064 }
1065 }
1066
1067
1068
1069
1070 delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
1071 user_name, NULL, unregister, false);
1072 }
1073
1074
1075 controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
1076 cib_scope_local);
1077
1078
1079 update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
1080 }
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096 static void
1097 synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action,
1098 int op_status, enum ocf_exitcode rc,
1099 const char *exit_reason)
1100 {
1101 lrmd_event_data_t *op = NULL;
1102 const char *operation = crm_element_value(action, PCMK_XA_OPERATION);
1103 const char *target_node = crm_element_value(action, PCMK__META_ON_NODE);
1104 xmlNode *xml_rsc = pcmk__xe_first_child(action, PCMK_XE_PRIMITIVE, NULL,
1105 NULL);
1106
1107 if ((xml_rsc == NULL) || (pcmk__xe_id(xml_rsc) == NULL)) {
1108
1109 crm_info("Can't fake %s failure (%d) on %s without resource configuration",
1110 crm_element_value(action, PCMK__XA_OPERATION_KEY), rc,
1111 target_node);
1112 return;
1113
1114 } else if(operation == NULL) {
1115
1116 crm_info("Can't fake %s failure (%d) on %s without operation",
1117 pcmk__xe_id(xml_rsc), rc, target_node);
1118 return;
1119 }
1120
1121 op = construct_op(lrm_state, action, pcmk__xe_id(xml_rsc), operation);
1122
1123 if (pcmk__str_eq(operation, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
1124
1125 fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL);
1126 } else {
1127 fake_op_status(lrm_state, op, op_status, rc, exit_reason);
1128 }
1129
1130 crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
1131 op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
1132
1133
1134 process_lrm_event(lrm_state, op, NULL, action);
1135 lrmd_free_event(op);
1136 }
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147 static const char *
1148 lrm_op_target(const xmlNode *xml)
1149 {
1150 const char *target = NULL;
1151
1152 if (xml) {
1153 target = crm_element_value(xml, PCMK__META_ON_NODE);
1154 }
1155 if (target == NULL) {
1156 target = controld_globals.our_nodename;
1157 }
1158 return target;
1159 }
1160
1161 static void
1162 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
1163 const char *from_host, const char *from_sys)
1164 {
1165 lrmd_event_data_t *op = NULL;
1166 lrmd_rsc_info_t *rsc = NULL;
1167 xmlNode *xml_rsc = pcmk__xe_first_child(xml, PCMK_XE_PRIMITIVE, NULL, NULL);
1168
1169 CRM_CHECK(xml_rsc != NULL, return);
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179 op = construct_op(lrm_state, xml, pcmk__xe_id(xml_rsc), "asyncmon");
1180
1181 free((char*) op->user_data);
1182 op->user_data = NULL;
1183 op->interval_ms = 0;
1184
1185 if (user_name && !pcmk__is_privileged(user_name)) {
1186 crm_err("%s does not have permission to fail %s",
1187 user_name, pcmk__xe_id(xml_rsc));
1188 fake_op_status(lrm_state, op, PCMK_EXEC_ERROR,
1189 PCMK_OCF_INSUFFICIENT_PRIV,
1190 "Unprivileged user cannot fail resources");
1191 controld_ack_event_directly(from_host, from_sys, NULL, op,
1192 pcmk__xe_id(xml_rsc));
1193 lrmd_free_event(op);
1194 return;
1195 }
1196
1197
1198 if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
1199 crm_info("Failing resource %s...", rsc->id);
1200 fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR,
1201 "Simulated failure");
1202 process_lrm_event(lrm_state, op, NULL, xml);
1203 op->rc = PCMK_OCF_OK;
1204 lrmd_free_rsc_info(rsc);
1205
1206 } else {
1207 crm_info("Cannot find/create resource in order to fail it...");
1208 crm_log_xml_warn(xml, "bad input");
1209 fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR,
1210 "Cannot fail unknown resource");
1211 }
1212
1213 controld_ack_event_directly(from_host, from_sys, NULL, op,
1214 pcmk__xe_id(xml_rsc));
1215 lrmd_free_event(op);
1216 }
1217
1218 static void
1219 handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
1220 const char *from_host, const char *user_name,
1221 gboolean is_remote_node, bool reprobe_all_nodes)
1222 {
1223 crm_notice("Forcing the status of all resources to be redetected");
1224 force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node,
1225 reprobe_all_nodes);
1226
1227 if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
1228
1229 xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
1230 from_sys, CRM_SYSTEM_LRMD,
1231 controld_globals.our_uuid);
1232
1233 crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
1234
1235 if (relay_message(reply, TRUE) == FALSE) {
1236 crm_log_xml_err(reply, "Unable to route reply");
1237 }
1238 free_xml(reply);
1239 }
1240 }
1241
1242 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
1243 lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
1244 {
1245 char *op_key = NULL;
1246 char *meta_key = NULL;
1247 int call = 0;
1248 const char *call_id = NULL;
1249 const char *op_task = NULL;
1250 guint interval_ms = 0;
1251 gboolean in_progress = FALSE;
1252 xmlNode *params = pcmk__xe_first_child(input->xml, PCMK__XE_ATTRIBUTES,
1253 NULL, NULL);
1254
1255 CRM_CHECK(params != NULL, return FALSE);
1256
1257 meta_key = crm_meta_name(PCMK_XA_OPERATION);
1258 op_task = crm_element_value(params, meta_key);
1259 free(meta_key);
1260 CRM_CHECK(op_task != NULL, return FALSE);
1261
1262 meta_key = crm_meta_name(PCMK_META_INTERVAL);
1263 if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
1264 free(meta_key);
1265 return FALSE;
1266 }
1267 free(meta_key);
1268
1269 op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
1270
1271 meta_key = crm_meta_name(PCMK__XA_CALL_ID);
1272 call_id = crm_element_value(params, meta_key);
1273 free(meta_key);
1274
1275 crm_debug("Scheduler requested op %s (call=%s) be cancelled",
1276 op_key, (call_id? call_id : "NA"));
1277 pcmk__scan_min_int(call_id, &call, 0);
1278 if (call == 0) {
1279
1280 in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
1281
1282 } else {
1283
1284 in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
1285 }
1286
1287
1288 if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1289 char *op_id = make_stop_id(rsc->id, call);
1290
1291 if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
1292 crm_info("Nothing known about operation %d for %s", call, op_key);
1293 }
1294 controld_delete_action_history_by_key(rsc->id, lrm_state->node_name,
1295 op_key, call);
1296 send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
1297 from_host, from_sys);
1298
1299
1300 if (lrm_state->active_ops != NULL) {
1301 g_hash_table_remove(lrm_state->active_ops, op_id);
1302 }
1303 free(op_id);
1304
1305 } else {
1306
1307
1308
1309
1310
1311
1312
1313 const char *peer_version = crm_element_value(params,
1314 PCMK_XA_CRM_FEATURE_SET);
1315
1316 if (compare_version(peer_version, "3.0.8") <= 0) {
1317 crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
1318 op_key, from_host, peer_version);
1319 send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
1320 from_host, from_sys);
1321 }
1322 }
1323
1324 free(op_key);
1325 return TRUE;
1326 }
1327
1328 static void
1329 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
1330 lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
1331 bool crm_rsc_delete, const char *user_name)
1332 {
1333 bool unregister = true;
1334 int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
1335 user_name,
1336 cib_dryrun|cib_sync_call);
1337
1338 if (cib_rc != pcmk_rc_ok) {
1339 lrmd_event_data_t *op = NULL;
1340
1341 op = construct_op(lrm_state, input->xml, rsc->id, PCMK_ACTION_DELETE);
1342
1343
1344
1345
1346 lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL);
1347 controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
1348 lrmd_free_event(op);
1349 return;
1350 }
1351
1352 if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1353 unregister = false;
1354 }
1355
1356 delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
1357 user_name, input, unregister, true);
1358 }
1359
1360
1361 struct metadata_cb_data {
1362 lrmd_rsc_info_t *rsc;
1363 xmlNode *input_xml;
1364 };
1365
1366 static struct metadata_cb_data *
1367 new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml)
1368 {
1369 struct metadata_cb_data *data = NULL;
1370
1371 data = pcmk__assert_alloc(1, sizeof(struct metadata_cb_data));
1372 data->input_xml = pcmk__xml_copy(NULL, input_xml);
1373 data->rsc = lrmd_copy_rsc_info(rsc);
1374 return data;
1375 }
1376
1377 static void
1378 free_metadata_cb_data(struct metadata_cb_data *data)
1379 {
1380 lrmd_free_rsc_info(data->rsc);
1381 free_xml(data->input_xml);
1382 free(data);
1383 }
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393 static void
1394 metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data)
1395 {
1396 struct metadata_cb_data *data = (struct metadata_cb_data *) user_data;
1397
1398 struct ra_metadata_s *md = NULL;
1399 lrm_state_t *lrm_state = lrm_state_find(lrm_op_target(data->input_xml));
1400
1401 if ((lrm_state != NULL) && pcmk__result_ok(result)) {
1402 md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc,
1403 result->action_stdout);
1404 }
1405 if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
1406 do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md);
1407 }
1408 free_metadata_cb_data(data);
1409 }
1410
1411
1412 void
1413 do_lrm_invoke(long long action,
1414 enum crmd_fsa_cause cause,
1415 enum crmd_fsa_state cur_state,
1416 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
1417 {
1418 lrm_state_t *lrm_state = NULL;
1419 const char *crm_op = NULL;
1420 const char *from_sys = NULL;
1421 const char *from_host = NULL;
1422 const char *operation = NULL;
1423 ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
1424 const char *user_name = NULL;
1425 const char *target_node = lrm_op_target(input->xml);
1426 gboolean is_remote_node = FALSE;
1427 bool crm_rsc_delete = FALSE;
1428
1429
1430 is_remote_node = !pcmk__str_eq(target_node, controld_globals.our_nodename,
1431 pcmk__str_casei);
1432
1433 lrm_state = lrm_state_find(target_node);
1434 if ((lrm_state == NULL) && is_remote_node) {
1435 crm_err("Failing action because local node has never had connection to remote node %s",
1436 target_node);
1437 synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
1438 PCMK_OCF_UNKNOWN_ERROR,
1439 "Local node has no connection to remote");
1440 return;
1441 }
1442 pcmk__assert(lrm_state != NULL);
1443
1444 user_name = pcmk__update_acl_user(input->msg, PCMK__XA_CRM_USER, NULL);
1445 crm_op = crm_element_value(input->msg, PCMK__XA_CRM_TASK);
1446 from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM);
1447 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
1448 from_host = crm_element_value(input->msg, PCMK__XA_SRC);
1449 }
1450
1451 if (pcmk__str_eq(crm_op, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
1452 if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
1453 crm_rsc_delete = TRUE;
1454 }
1455 operation = PCMK_ACTION_DELETE;
1456
1457 } else if (input->xml != NULL) {
1458 operation = crm_element_value(input->xml, PCMK_XA_OPERATION);
1459 }
1460
1461 CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return);
1462
1463 crm_trace("'%s' execution request from %s as %s user",
1464 pcmk__s(crm_op, operation),
1465 pcmk__s(from_sys, "unknown subsystem"),
1466 pcmk__s(user_name, "current"));
1467
1468 if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) {
1469 fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
1470 from_sys);
1471
1472 } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_none)) {
1473
1474
1475
1476
1477
1478 crm_notice("Ignoring refresh request from Pacemaker Remote 1.1.9 node");
1479
1480
1481 } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_none)) {
1482 update_attrd(lrm_state->node_name, CRM_OP_PROBED, PCMK_VALUE_TRUE,
1483 user_name, is_remote_node);
1484
1485 } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none)
1486 || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
1487 const char *raw_target = NULL;
1488
1489 if (input->xml != NULL) {
1490
1491 raw_target = crm_element_value(input->xml, PCMK__META_ON_NODE);
1492 }
1493 handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
1494 is_remote_node, (raw_target == NULL));
1495
1496 } else if (operation != NULL) {
1497 lrmd_rsc_info_t *rsc = NULL;
1498 xmlNode *xml_rsc = pcmk__xe_first_child(input->xml, PCMK_XE_PRIMITIVE,
1499 NULL, NULL);
1500 gboolean create_rsc = !pcmk__str_eq(operation, PCMK_ACTION_DELETE,
1501 pcmk__str_none);
1502 int rc;
1503
1504
1505 CRM_CHECK((xml_rsc != NULL) && (pcmk__xe_id(xml_rsc) != NULL), return);
1506
1507 rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
1508 if (rc == -ENOTCONN) {
1509 synthesize_lrmd_failure(lrm_state, input->xml,
1510 PCMK_EXEC_NOT_CONNECTED,
1511 PCMK_OCF_UNKNOWN_ERROR,
1512 "Not connected to remote executor");
1513 return;
1514
1515 } else if ((rc < 0) && !create_rsc) {
1516
1517
1518
1519 crm_notice("Not registering resource '%s' for a %s event "
1520 CRM_XS " get-rc=%d (%s) transition-key=%s",
1521 pcmk__xe_id(xml_rsc), operation,
1522 rc, pcmk_strerror(rc), pcmk__xe_id(input->xml));
1523 delete_rsc_entry(lrm_state, input, pcmk__xe_id(xml_rsc), NULL,
1524 pcmk_ok, user_name, true);
1525 return;
1526
1527 } else if (rc == -EINVAL) {
1528
1529 crm_err("Invalid resource definition for %s", pcmk__xe_id(xml_rsc));
1530 crm_log_xml_warn(input->msg, "invalid resource");
1531 synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
1532 PCMK_OCF_NOT_CONFIGURED,
1533 "Invalid resource definition");
1534 return;
1535
1536 } else if (rc < 0) {
1537
1538 crm_err("Could not register resource '%s' with executor: %s "
1539 CRM_XS " rc=%d",
1540 pcmk__xe_id(xml_rsc), pcmk_strerror(rc), rc);
1541 crm_log_xml_warn(input->msg, "failed registration");
1542 synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
1543 PCMK_OCF_INVALID_PARAM,
1544 "Could not register resource with executor");
1545 return;
1546 }
1547
1548 if (pcmk__str_eq(operation, PCMK_ACTION_CANCEL, pcmk__str_none)) {
1549 if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
1550 crm_log_xml_warn(input->xml, "Bad command");
1551 }
1552
1553 } else if (pcmk__str_eq(operation, PCMK_ACTION_DELETE,
1554 pcmk__str_none)) {
1555 do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
1556 crm_rsc_delete, user_name);
1557
1558 } else {
1559 struct ra_metadata_s *md = NULL;
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569 if (strcmp(operation, PCMK_ACTION_START) != 0) {
1570 md = controld_get_rsc_metadata(lrm_state, rsc,
1571 controld_metadata_from_cache);
1572 }
1573
1574 if ((md == NULL) && crm_op_needs_metadata(rsc->standard,
1575 operation)) {
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587 struct metadata_cb_data *data = NULL;
1588
1589 data = new_metadata_cb_data(rsc, input->xml);
1590 crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously",
1591 rsc->id, rsc->standard,
1592 ((rsc->provider == NULL)? "" : ":"),
1593 ((rsc->provider == NULL)? "" : rsc->provider),
1594 rsc->type);
1595 (void) lrmd__metadata_async(rsc, metadata_complete,
1596 (void *) data);
1597 } else {
1598 do_lrm_rsc_op(lrm_state, rsc, input->xml, md);
1599 }
1600 }
1601
1602 lrmd_free_rsc_info(rsc);
1603
1604 } else {
1605 crm_err("Invalid execution request: unknown command '%s' (bug?)",
1606 crm_op);
1607 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
1608 }
1609 }
1610
1611 static lrmd_event_data_t *
1612 construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op,
1613 const char *rsc_id, const char *operation)
1614 {
1615 lrmd_event_data_t *op = NULL;
1616 const char *op_delay = NULL;
1617 const char *op_timeout = NULL;
1618 GHashTable *params = NULL;
1619
1620 xmlNode *primitive = NULL;
1621 const char *class = NULL;
1622
1623 const char *transition = NULL;
1624
1625 pcmk__assert((rsc_id != NULL) && (operation != NULL));
1626
1627 op = lrmd_new_event(rsc_id, operation, 0);
1628 op->type = lrmd_event_exec_complete;
1629 op->timeout = 0;
1630 op->start_delay = 0;
1631 lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
1632
1633 if (rsc_op == NULL) {
1634 CRM_LOG_ASSERT(pcmk__str_eq(operation, PCMK_ACTION_STOP,
1635 pcmk__str_casei));
1636 op->user_data = NULL;
1637
1638
1639
1640
1641
1642 op->params = pcmk__strkey_table(free, free);
1643
1644 pcmk__insert_dup(op->params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
1645
1646 crm_trace("Constructed %s op for %s", operation, rsc_id);
1647 return op;
1648 }
1649
1650 params = xml2list(rsc_op);
1651 g_hash_table_remove(params, CRM_META "_" PCMK__META_OP_TARGET_RC);
1652
1653 op_delay = crm_meta_value(params, PCMK_META_START_DELAY);
1654 pcmk__scan_min_int(op_delay, &op->start_delay, 0);
1655
1656 op_timeout = crm_meta_value(params, PCMK_META_TIMEOUT);
1657 pcmk__scan_min_int(op_timeout, &op->timeout, 0);
1658
1659 if (pcmk__guint_from_hash(params, CRM_META "_" PCMK_META_INTERVAL, 0,
1660 &(op->interval_ms)) != pcmk_rc_ok) {
1661 op->interval_ms = 0;
1662 }
1663
1664
1665
1666 primitive = pcmk__xe_first_child(rsc_op, PCMK_XE_PRIMITIVE, NULL, NULL);
1667 class = crm_element_value(primitive, PCMK_XA_CLASS);
1668
1669 if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
1670 && pcmk__str_eq(operation, PCMK_ACTION_MONITOR, pcmk__str_casei)
1671 && (op->interval_ms > 0)) {
1672
1673 op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
1674 if (op_timeout != NULL) {
1675 long long timeout_ms = crm_get_msec(op_timeout);
1676
1677 op->timeout = (int) QB_MIN(timeout_ms, INT_MAX);
1678 }
1679 }
1680
1681 if (!pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)) {
1682 op->params = params;
1683
1684 } else {
1685 rsc_history_t *entry = NULL;
1686
1687 if (lrm_state) {
1688 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
1689 }
1690
1691
1692
1693 if (!entry || !entry->stop_params) {
1694 op->params = params;
1695 } else {
1696
1697
1698 op->params = pcmk__strkey_table(free, free);
1699
1700 g_hash_table_foreach(params, copy_meta_keys, op->params);
1701 g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
1702 g_hash_table_destroy(params);
1703 params = NULL;
1704 }
1705 }
1706
1707
1708 if (op->timeout <= 0) {
1709 op->timeout = op->interval_ms;
1710 }
1711 if (op->start_delay < 0) {
1712 op->start_delay = 0;
1713 }
1714
1715 transition = crm_element_value(rsc_op, PCMK__XA_TRANSITION_KEY);
1716 CRM_CHECK(transition != NULL, return op);
1717
1718 op->user_data = pcmk__str_copy(transition);
1719
1720 if (op->interval_ms != 0) {
1721 if (pcmk__strcase_any_of(operation, PCMK_ACTION_START, PCMK_ACTION_STOP,
1722 NULL)) {
1723 crm_err("Start and Stop actions cannot have an interval: %u",
1724 op->interval_ms);
1725 op->interval_ms = 0;
1726 }
1727 }
1728
1729 crm_trace("Constructed %s op for %s: interval=%u",
1730 operation, rsc_id, op->interval_ms);
1731
1732 return op;
1733 }
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748 void
1749 controld_ack_event_directly(const char *to_host, const char *to_sys,
1750 const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
1751 const char *rsc_id)
1752 {
1753 xmlNode *reply = NULL;
1754 xmlNode *update, *iter;
1755 crm_node_t *peer = NULL;
1756
1757 CRM_CHECK(op != NULL, return);
1758 if (op->rsc_id == NULL) {
1759
1760 pcmk__assert(rsc_id != NULL);
1761 op->rsc_id = pcmk__str_copy(rsc_id);
1762 }
1763 if (to_sys == NULL) {
1764 to_sys = CRM_SYSTEM_TENGINE;
1765 }
1766
1767 peer = pcmk__get_node(0, controld_globals.our_nodename, NULL,
1768 pcmk__node_search_cluster_member);
1769 update = create_node_state_update(peer, node_update_none, NULL,
1770 __func__);
1771
1772 iter = pcmk__xe_create(update, PCMK__XE_LRM);
1773 crm_xml_add(iter, PCMK_XA_ID, controld_globals.our_uuid);
1774 iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCES);
1775 iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCE);
1776
1777 crm_xml_add(iter, PCMK_XA_ID, op->rsc_id);
1778
1779 controld_add_resource_history_xml(iter, rsc, op,
1780 controld_globals.our_nodename);
1781 reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
1782
1783 crm_log_xml_trace(update, "[direct ACK]");
1784
1785 crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
1786 op->rsc_id, op->op_type, op->interval_ms, op->user_data,
1787 crm_element_value(reply, PCMK_XA_REFERENCE));
1788
1789 if (relay_message(reply, TRUE) == FALSE) {
1790 crm_log_xml_err(reply, "Unable to route reply");
1791 }
1792
1793 free_xml(update);
1794 free_xml(reply);
1795 }
1796
1797 gboolean
1798 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
1799 {
1800 gboolean res = TRUE;
1801 GList *lrm_state_list = lrm_state_get_list();
1802 GList *state_entry;
1803
1804 for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
1805 lrm_state_t *lrm_state = state_entry->data;
1806
1807 if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
1808
1809 res = FALSE;
1810 }
1811 }
1812
1813 controld_set_fsa_input_flags(R_SENT_RSC_STOP);
1814 g_list_free(lrm_state_list); lrm_state_list = NULL;
1815 return res;
1816 }
1817
1818 struct stop_recurring_action_s {
1819 lrmd_rsc_info_t *rsc;
1820 lrm_state_t *lrm_state;
1821 };
1822
1823 static gboolean
1824 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
1825 {
1826 gboolean remove = FALSE;
1827 struct stop_recurring_action_s *event = user_data;
1828 active_op_t *op = value;
1829
1830 if ((op->interval_ms != 0)
1831 && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
1832
1833 crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
1834 remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
1835 }
1836
1837 return remove;
1838 }
1839
1840 static gboolean
1841 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
1842 {
1843 gboolean remove = FALSE;
1844 lrm_state_t *lrm_state = user_data;
1845 active_op_t *op = value;
1846
1847 if (op->interval_ms != 0) {
1848 crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
1849 (const char *) key);
1850 remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
1851 }
1852
1853 return remove;
1854 }
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866 static bool
1867 should_cancel_recurring(const char *rsc_id, const char *action, guint interval_ms)
1868 {
1869 if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0)
1870 && (strcmp(action, PCMK_ACTION_MIGRATE_TO) == 0)) {
1871
1872
1873
1874
1875 return false;
1876 }
1877
1878
1879 return (interval_ms == 0)
1880 && !pcmk__str_any_of(action, PCMK_ACTION_MONITOR,
1881 PCMK_ACTION_NOTIFY, NULL);
1882 }
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893 static const char *
1894 should_nack_action(const char *action)
1895 {
1896 if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)
1897 && pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
1898
1899 register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
1900 return "Not attempting start due to shutdown in progress";
1901 }
1902
1903 switch (controld_globals.fsa_state) {
1904 case S_NOT_DC:
1905 case S_POLICY_ENGINE:
1906 case S_TRANSITION_ENGINE:
1907 break;
1908 default:
1909 if (!pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
1910 return "Controller cannot attempt actions at this time";
1911 }
1912 break;
1913 }
1914 return NULL;
1915 }
1916
1917 static void
1918 do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg,
1919 struct ra_metadata_s *md)
1920 {
1921 int rc;
1922 int call_id = 0;
1923 char *op_id = NULL;
1924 lrmd_event_data_t *op = NULL;
1925 fsa_data_t *msg_data = NULL;
1926 const char *transition = NULL;
1927 const char *operation = NULL;
1928 const char *nack_reason = NULL;
1929
1930 CRM_CHECK((rsc != NULL) && (msg != NULL), return);
1931
1932 operation = crm_element_value(msg, PCMK_XA_OPERATION);
1933 CRM_CHECK(!pcmk__str_empty(operation), return);
1934
1935 transition = crm_element_value(msg, PCMK__XA_TRANSITION_KEY);
1936 if (pcmk__str_empty(transition)) {
1937 crm_log_xml_err(msg, "Missing transition number");
1938 }
1939
1940 if (lrm_state == NULL) {
1941
1942 crm_err("Cannot execute %s of %s: No executor connection "
1943 CRM_XS " transition_key=%s",
1944 operation, rsc->id, pcmk__s(transition, ""));
1945 synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID,
1946 PCMK_OCF_UNKNOWN_ERROR,
1947 "No executor connection");
1948 return;
1949 }
1950
1951 if (pcmk__str_any_of(operation, PCMK_ACTION_RELOAD,
1952 PCMK_ACTION_RELOAD_AGENT, NULL)) {
1953
1954
1955
1956
1957
1958 if ((md != NULL)
1959 && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
1960 operation = PCMK_ACTION_RELOAD;
1961 } else {
1962 operation = PCMK_ACTION_RELOAD_AGENT;
1963 }
1964 }
1965
1966 op = construct_op(lrm_state, msg, rsc->id, operation);
1967 CRM_CHECK(op != NULL, return);
1968
1969 if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) {
1970 guint removed = 0;
1971 struct stop_recurring_action_s data;
1972
1973 data.rsc = rsc;
1974 data.lrm_state = lrm_state;
1975 removed = g_hash_table_foreach_remove(lrm_state->active_ops,
1976 stop_recurring_action_by_rsc,
1977 &data);
1978
1979 if (removed) {
1980 crm_debug("Stopped %u recurring operation%s in preparation for "
1981 PCMK__OP_FMT, removed, pcmk__plural_s(removed),
1982 rsc->id, operation, op->interval_ms);
1983 }
1984 }
1985
1986
1987 crm_notice("Requesting local execution of %s operation for %s on %s "
1988 CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
1989 pcmk__readable_action(op->op_type, op->interval_ms), rsc->id,
1990 lrm_state->node_name, pcmk__s(transition, ""), rsc->id,
1991 operation, op->interval_ms);
1992
1993 nack_reason = should_nack_action(operation);
1994 if (nack_reason != NULL) {
1995 crm_notice("Discarding attempt to perform action %s on %s in state %s "
1996 "(shutdown=%s)", operation, rsc->id,
1997 fsa_state2string(controld_globals.fsa_state),
1998 pcmk__flag_text(controld_globals.fsa_input_register,
1999 R_SHUTDOWN));
2000
2001 lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
2002 nack_reason);
2003 controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
2004 lrmd_free_event(op);
2005 free(op_id);
2006 return;
2007 }
2008
2009 controld_record_pending_op(lrm_state->node_name, rsc, op);
2010
2011 op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
2012
2013 if (op->interval_ms > 0) {
2014
2015 cancel_op_key(lrm_state, rsc, op_id, FALSE);
2016 }
2017
2018 rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
2019 op->user_data, op->interval_ms,
2020 op->timeout, op->start_delay,
2021 op->params, &call_id);
2022 if (rc == pcmk_rc_ok) {
2023
2024
2025
2026 char *call_id_s = make_stop_id(rsc->id, call_id);
2027 active_op_t *pending = NULL;
2028
2029 pending = pcmk__assert_alloc(1, sizeof(active_op_t));
2030 crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
2031
2032 pending->call_id = call_id;
2033 pending->interval_ms = op->interval_ms;
2034 pending->op_type = pcmk__str_copy(operation);
2035 pending->op_key = pcmk__str_copy(op_id);
2036 pending->rsc_id = pcmk__str_copy(rsc->id);
2037 pending->start_time = time(NULL);
2038 pending->user_data = pcmk__str_copy(op->user_data);
2039 if (crm_element_value_epoch(msg, PCMK_OPT_SHUTDOWN_LOCK,
2040 &(pending->lock_time)) != pcmk_ok) {
2041 pending->lock_time = 0;
2042 }
2043 g_hash_table_replace(lrm_state->active_ops, call_id_s, pending);
2044
2045 if ((op->interval_ms > 0)
2046 && (op->start_delay > START_DELAY_THRESHOLD)) {
2047 int target_rc = PCMK_OCF_OK;
2048
2049 crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
2050 decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
2051 lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
2052 controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
2053 }
2054
2055 pending->params = op->params;
2056 op->params = NULL;
2057
2058 } else if (lrm_state_is_local(lrm_state)) {
2059 crm_err("Could not initiate %s action for resource %s locally: %s "
2060 CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
2061 fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
2062 PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
2063 process_lrm_event(lrm_state, op, NULL, NULL);
2064 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
2065
2066 } else {
2067 crm_err("Could not initiate %s action for resource %s remotely on %s: "
2068 "%s " CRM_XS " rc=%d",
2069 operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
2070 fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
2071 PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
2072 process_lrm_event(lrm_state, op, NULL, NULL);
2073 }
2074
2075 free(op_id);
2076 lrmd_free_event(op);
2077 }
2078
2079 void
2080 do_lrm_event(long long action,
2081 enum crmd_fsa_cause cause,
2082 enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
2083 {
2084 CRM_CHECK(FALSE, return);
2085 }
2086
2087 static char *
2088 unescape_newlines(const char *string)
2089 {
2090 char *pch = NULL;
2091 char *ret = NULL;
2092 static const char *escaped_newline = "\\n";
2093
2094 if (!string) {
2095 return NULL;
2096 }
2097
2098 ret = pcmk__str_copy(string);
2099 pch = strstr(ret, escaped_newline);
2100 while (pch != NULL) {
2101
2102
2103
2104 pch[0] = '\n';
2105 pch[1] = ' ';
2106 pch = strstr(pch, escaped_newline);
2107 }
2108
2109 return ret;
2110 }
2111
2112 static bool
2113 did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
2114 const char * op_type, guint interval_ms)
2115 {
2116 rsc_history_t *entry = NULL;
2117
2118 CRM_CHECK(lrm_state != NULL, return FALSE);
2119 CRM_CHECK(rsc_id != NULL, return FALSE);
2120 CRM_CHECK(op_type != NULL, return FALSE);
2121
2122 entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
2123 if (entry == NULL || entry->failed == NULL) {
2124 return FALSE;
2125 }
2126
2127 if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
2128 && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
2129 && entry->failed->interval_ms == interval_ms) {
2130 return TRUE;
2131 }
2132
2133 return FALSE;
2134 }
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145 static void
2146 log_executor_event(const lrmd_event_data_t *op, const char *op_key,
2147 const char *node_name, gboolean confirmed)
2148 {
2149 int log_level = LOG_ERR;
2150 GString *str = g_string_sized_new(100);
2151
2152 pcmk__g_strcat(str,
2153 "Result of ",
2154 pcmk__readable_action(op->op_type, op->interval_ms),
2155 " operation for ", op->rsc_id, NULL);
2156
2157 if (node_name != NULL) {
2158 pcmk__g_strcat(str, " on ", node_name, NULL);
2159 }
2160
2161 switch (op->op_status) {
2162 case PCMK_EXEC_DONE:
2163 log_level = LOG_NOTICE;
2164 pcmk__g_strcat(str, ": ", services_ocf_exitcode_str(op->rc), NULL);
2165 break;
2166
2167 case PCMK_EXEC_TIMEOUT:
2168 pcmk__g_strcat(str,
2169 ": ", pcmk_exec_status_str(op->op_status), " after ",
2170 pcmk__readable_interval(op->timeout), NULL);
2171 break;
2172
2173 case PCMK_EXEC_CANCELLED:
2174 log_level = LOG_INFO;
2175
2176
2177
2178
2179
2180
2181
2182
2183 #ifdef __clang__
2184 #ifdef __has_c_attribute
2185 #if __has_attribute(fallthrough)
2186 __attribute__((fallthrough));
2187 #endif
2188 #endif
2189 #endif
2190
2191 default:
2192 pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status),
2193 NULL);
2194 }
2195
2196 if ((op->exit_reason != NULL)
2197 && ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) {
2198
2199 pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL);
2200 }
2201
2202 g_string_append(str, " " CRM_XS);
2203 g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s",
2204 (confirmed? "" : "un"), op->call_id, op_key);
2205 if (op->op_status == PCMK_EXEC_DONE) {
2206 g_string_append_printf(str, " rc=%d", op->rc);
2207 }
2208
2209 do_crm_log(log_level, "%s", str->str);
2210 g_string_free(str, TRUE);
2211
2212
2213
2214
2215 if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) {
2216 char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output",
2217 op->rsc_id, op->op_type,
2218 op->interval_ms, node_name);
2219
2220 crm_log_output(LOG_NOTICE, prefix, op->output);
2221 free(prefix);
2222 }
2223 }
2224
2225 void
2226 process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
2227 active_op_t *pending, const xmlNode *action_xml)
2228 {
2229 char *op_id = NULL;
2230 char *op_key = NULL;
2231
2232 gboolean remove = FALSE;
2233 gboolean removed = FALSE;
2234 bool need_direct_ack = FALSE;
2235 lrmd_rsc_info_t *rsc = NULL;
2236 const char *node_name = NULL;
2237
2238 CRM_CHECK(op != NULL, return);
2239 CRM_CHECK(op->rsc_id != NULL, return);
2240
2241
2242 if (compare_version(controld_globals.dc_version, "3.2.0") < 0) {
2243 switch (op->op_status) {
2244 case PCMK_EXEC_NOT_CONNECTED:
2245 lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
2246 PCMK_EXEC_ERROR, op->exit_reason);
2247 break;
2248 case PCMK_EXEC_INVALID:
2249 lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
2250 op->exit_reason);
2251 break;
2252 default:
2253 break;
2254 }
2255 }
2256
2257 op_id = make_stop_id(op->rsc_id, op->call_id);
2258 op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
2259
2260
2261 if (lrm_state) {
2262 rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
2263 }
2264 if ((rsc == NULL) && action_xml) {
2265 xmlNode *xml = pcmk__xe_first_child(action_xml, PCMK_XE_PRIMITIVE, NULL,
2266 NULL);
2267
2268 const char *standard = crm_element_value(xml, PCMK_XA_CLASS);
2269 const char *provider = crm_element_value(xml, PCMK_XA_PROVIDER);
2270 const char *type = crm_element_value(xml, PCMK_XA_TYPE);
2271
2272 if (standard && type) {
2273 crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
2274 op->rsc_id, standard,
2275 (provider? ":" : ""), (provider? provider : ""), type);
2276 rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
2277 } else {
2278 crm_err("Can't process %s result because %s agent information not cached or in XML",
2279 op_key, op->rsc_id);
2280 }
2281 }
2282
2283
2284 if (lrm_state) {
2285 node_name = lrm_state->node_name;
2286 } else if (action_xml) {
2287 node_name = crm_element_value(action_xml, PCMK__META_ON_NODE);
2288 }
2289
2290 if(pending == NULL) {
2291 remove = TRUE;
2292 if (lrm_state) {
2293 pending = g_hash_table_lookup(lrm_state->active_ops, op_id);
2294 }
2295 }
2296
2297 if (op->op_status == PCMK_EXEC_ERROR) {
2298 switch(op->rc) {
2299 case PCMK_OCF_NOT_RUNNING:
2300 case PCMK_OCF_RUNNING_PROMOTED:
2301 case PCMK_OCF_DEGRADED:
2302 case PCMK_OCF_DEGRADED_PROMOTED:
2303
2304 op->op_status = PCMK_EXEC_DONE;
2305 break;
2306 default:
2307
2308 break;
2309 }
2310 }
2311
2312 if (op->op_status != PCMK_EXEC_CANCELLED) {
2313
2314
2315
2316
2317 need_direct_ack = TRUE;
2318
2319 if (controld_action_is_recordable(op->op_type)) {
2320 if (node_name && rsc) {
2321
2322 time_t lock_time = (pending == NULL)? 0 : pending->lock_time;
2323
2324 controld_update_resource_history(node_name, rsc, op, lock_time);
2325 need_direct_ack = FALSE;
2326
2327 } else if (op->rsc_deleted) {
2328
2329
2330
2331
2332 crm_notice("Not recording %s result in CIB because "
2333 "resource information was removed since it was initiated",
2334 op_key);
2335 } else {
2336
2337
2338
2339
2340 crm_err("Unable to record %s result in CIB: %s", op_key,
2341 (node_name? "No resource information" : "No node name"));
2342 }
2343 }
2344
2345 } else if (op->interval_ms == 0) {
2346
2347
2348
2349
2350 need_direct_ack = TRUE;
2351
2352 } else if (pending == NULL) {
2353
2354
2355
2356
2357 } else if (op->user_data == NULL) {
2358
2359
2360
2361 crm_err("Recurring operation %s was cancelled without transition information",
2362 op_key);
2363
2364 } else if (pcmk_is_set(pending->flags, active_op_remove)) {
2365
2366
2367
2368 if (lrm_state) {
2369 controld_delete_action_history(op);
2370 }
2371
2372
2373
2374
2375
2376
2377
2378 if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
2379 pending->op_type, pending->interval_ms)) {
2380 need_direct_ack = TRUE;
2381 }
2382
2383 } else if (op->rsc_deleted) {
2384
2385
2386
2387
2388 crm_debug("Recurring op %s was cancelled due to resource deletion",
2389 op_key);
2390 need_direct_ack = TRUE;
2391
2392 } else {
2393
2394
2395
2396
2397 }
2398
2399 if (need_direct_ack) {
2400 controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
2401 }
2402
2403 if(remove == FALSE) {
2404
2405 removed = TRUE;
2406
2407 } else if (lrm_state && ((op->interval_ms == 0)
2408 || (op->op_status == PCMK_EXEC_CANCELLED))) {
2409
2410 gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id);
2411
2412 if (op->interval_ms != 0) {
2413 removed = TRUE;
2414 } else if (found) {
2415 removed = TRUE;
2416 crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
2417 op_key, op->call_id, op_id,
2418 g_hash_table_size(lrm_state->active_ops));
2419 }
2420 }
2421
2422 log_executor_event(op, op_key, node_name, removed);
2423
2424 if (lrm_state) {
2425 if (!pcmk__str_eq(op->op_type, PCMK_ACTION_META_DATA,
2426 pcmk__str_casei)) {
2427 crmd_alert_resource_op(lrm_state->node_name, op);
2428 } else if (rsc && (op->rc == PCMK_OCF_OK)) {
2429 char *metadata = unescape_newlines(op->output);
2430
2431 controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata);
2432 free(metadata);
2433 }
2434 }
2435
2436 if (op->rsc_deleted) {
2437 crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
2438 if (lrm_state) {
2439 delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL,
2440 true);
2441 }
2442 }
2443
2444
2445
2446
2447 controld_trigger_fsa();
2448 if (lrm_state && rsc) {
2449 update_history_cache(lrm_state, rsc, op);
2450 }
2451
2452 lrmd_free_rsc_info(rsc);
2453 free(op_key);
2454 free(op_id);
2455 }
![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
![[previous]](../icons/left.png)
![[last]](../icons/n_last.png)
![[bottom]](../icons/n_bottom.png){kind=icon}