This source file includes following definitions.
- init_dotfile
- do_fsa_action
- do_log
- controld_init_fsa_trigger
- controld_destroy_fsa_trigger
- controld_trigger_fsa_as
- s_crmd_fsa
- s_crmd_fsa_actions
- log_fsa_input
- check_join_counts
- do_state_transition
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <sys/param.h>
13 #include <stdio.h>
14 #include <stdint.h>
15 #include <string.h>
16 #include <time.h>
17
18 #include <crm/crm.h>
19 #include <crm/lrmd.h>
20 #include <crm/cib.h>
21 #include <crm/msg_xml.h>
22 #include <crm/common/xml.h>
23 #include <crm/cluster/election_internal.h>
24 #include <crm/cluster.h>
25
26 #include <pacemaker-controld.h>
27
28
29 static crm_trigger_t *fsa_trigger = NULL;
30
31 #define DOT_PREFIX "actions:trace: "
32 #define do_dot_log(fmt, args...) crm_trace( fmt, ##args)
33
34 static void do_state_transition(enum crmd_fsa_state cur_state,
35 enum crmd_fsa_state next_state,
36 fsa_data_t *msg_data);
37
38 void s_crmd_fsa_actions(fsa_data_t * fsa_data);
39 void log_fsa_input(fsa_data_t * stored_msg);
40 void init_dotfile(void);
41
42 void
43 init_dotfile(void)
44 {
45 do_dot_log(DOT_PREFIX "digraph \"g\" {");
46 do_dot_log(DOT_PREFIX " size = \"30,30\"");
47 do_dot_log(DOT_PREFIX " graph [");
48 do_dot_log(DOT_PREFIX " fontsize = \"12\"");
49 do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\"");
50 do_dot_log(DOT_PREFIX " fontcolor = \"black\"");
51 do_dot_log(DOT_PREFIX " bb = \"0,0,398.922306,478.927856\"");
52 do_dot_log(DOT_PREFIX " color = \"black\"");
53 do_dot_log(DOT_PREFIX " ]");
54 do_dot_log(DOT_PREFIX " node [");
55 do_dot_log(DOT_PREFIX " fontsize = \"12\"");
56 do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\"");
57 do_dot_log(DOT_PREFIX " fontcolor = \"black\"");
58 do_dot_log(DOT_PREFIX " shape = \"ellipse\"");
59 do_dot_log(DOT_PREFIX " color = \"black\"");
60 do_dot_log(DOT_PREFIX " ]");
61 do_dot_log(DOT_PREFIX " edge [");
62 do_dot_log(DOT_PREFIX " fontsize = \"12\"");
63 do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\"");
64 do_dot_log(DOT_PREFIX " fontcolor = \"black\"");
65 do_dot_log(DOT_PREFIX " color = \"black\"");
66 do_dot_log(DOT_PREFIX " ]");
67 do_dot_log(DOT_PREFIX "// special nodes");
68 do_dot_log(DOT_PREFIX " \"S_PENDING\" ");
69 do_dot_log(DOT_PREFIX " [");
70 do_dot_log(DOT_PREFIX " color = \"blue\"");
71 do_dot_log(DOT_PREFIX " fontcolor = \"blue\"");
72 do_dot_log(DOT_PREFIX " ]");
73 do_dot_log(DOT_PREFIX " \"S_TERMINATE\" ");
74 do_dot_log(DOT_PREFIX " [");
75 do_dot_log(DOT_PREFIX " color = \"red\"");
76 do_dot_log(DOT_PREFIX " fontcolor = \"red\"");
77 do_dot_log(DOT_PREFIX " ]");
78 do_dot_log(DOT_PREFIX "// DC only nodes");
79 do_dot_log(DOT_PREFIX " \"S_INTEGRATION\" [ fontcolor = \"green\" ]");
80 do_dot_log(DOT_PREFIX " \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]");
81 do_dot_log(DOT_PREFIX " \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]");
82 do_dot_log(DOT_PREFIX " \"S_RELEASE_DC\" [ fontcolor = \"green\" ]");
83 do_dot_log(DOT_PREFIX " \"S_IDLE\" [ fontcolor = \"green\" ]");
84 }
85
86 static void
87 do_fsa_action(fsa_data_t * fsa_data, long long an_action,
88 void (*function) (long long action,
89 enum crmd_fsa_cause cause,
90 enum crmd_fsa_state cur_state,
91 enum crmd_fsa_input cur_input, fsa_data_t * msg_data))
92 {
93 controld_clear_fsa_action_flags(an_action);
94 crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action));
95 function(an_action, fsa_data->fsa_cause, controld_globals.fsa_state,
96 fsa_data->fsa_input, fsa_data);
97 }
98
99 static const uint64_t startup_actions =
100 A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG |
101 A_STARTED | A_CL_JOIN_QUERY;
102
103
104 void
105 do_log(long long action, enum crmd_fsa_cause cause,
106 enum crmd_fsa_state cur_state,
107 enum crmd_fsa_input current_input, fsa_data_t *msg_data)
108 {
109 unsigned log_type = LOG_TRACE;
110
111 if (action & A_LOG) {
112 log_type = LOG_INFO;
113 } else if (action & A_WARN) {
114 log_type = LOG_WARNING;
115 } else if (action & A_ERROR) {
116 log_type = LOG_ERR;
117 }
118
119 do_crm_log(log_type, "Input %s received in state %s from %s",
120 fsa_input2string(msg_data->fsa_input),
121 fsa_state2string(cur_state), msg_data->origin);
122
123 if (msg_data->data_type == fsa_dt_ha_msg) {
124 ha_msg_input_t *input = fsa_typed_data(msg_data->data_type);
125
126 crm_log_xml_debug(input->msg, __func__);
127
128 } else if (msg_data->data_type == fsa_dt_xml) {
129 xmlNode *input = fsa_typed_data(msg_data->data_type);
130
131 crm_log_xml_debug(input, __func__);
132
133 } else if (msg_data->data_type == fsa_dt_lrm) {
134 lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type);
135
136 do_crm_log(log_type,
137 "Resource %s: Call ID %d returned %d (%d)."
138 " New status if rc=0: %s",
139 input->rsc_id, input->call_id, input->rc,
140 input->op_status, (char *)input->user_data);
141 }
142 }
143
144
145
146
147
148 void
149 controld_init_fsa_trigger(void)
150 {
151 fsa_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
152 }
153
154
155
156
157
158 void
159 controld_destroy_fsa_trigger(void)
160 {
161
162 mainloop_destroy_trigger(fsa_trigger);
163 fsa_trigger = NULL;
164 }
165
166
167
168
169
170
171
172
173 void
174 controld_trigger_fsa_as(const char *fn, int line)
175 {
176 if (fsa_trigger != NULL) {
177 crm_trace("%s:%d - Triggered FSA invocation", fn, line);
178 mainloop_set_trigger(fsa_trigger);
179 }
180 }
181
182 enum crmd_fsa_state
183 s_crmd_fsa(enum crmd_fsa_cause cause)
184 {
185 controld_globals_t *globals = &controld_globals;
186 fsa_data_t *fsa_data = NULL;
187 uint64_t register_copy = controld_globals.fsa_input_register;
188 uint64_t new_actions = A_NOTHING;
189 enum crmd_fsa_state last_state;
190
191 crm_trace("FSA invoked with Cause: %s\tState: %s",
192 fsa_cause2string(cause),
193 fsa_state2string(globals->fsa_state));
194
195 fsa_dump_actions(controld_globals.fsa_actions, "Initial");
196
197 controld_clear_global_flags(controld_fsa_is_stalled);
198 if ((controld_globals.fsa_message_queue == NULL)
199 && (controld_globals.fsa_actions != A_NOTHING)) {
200
201 fsa_data = calloc(1, sizeof(fsa_data_t));
202 fsa_data->fsa_input = I_NULL;
203 fsa_data->fsa_cause = C_FSA_INTERNAL;
204 fsa_data->origin = __func__;
205 fsa_data->data_type = fsa_dt_none;
206 controld_globals.fsa_message_queue
207 = g_list_append(controld_globals.fsa_message_queue, fsa_data);
208 }
209 while ((controld_globals.fsa_message_queue != NULL)
210 && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
211 crm_trace("Checking messages (%d remaining)",
212 g_list_length(controld_globals.fsa_message_queue));
213
214 fsa_data = get_message();
215 if(fsa_data == NULL) {
216 continue;
217 }
218
219 log_fsa_input(fsa_data);
220
221
222 controld_set_fsa_action_flags(fsa_data->actions);
223 fsa_dump_actions(fsa_data->actions, "Restored actions");
224
225
226 new_actions = controld_fsa_get_action(fsa_data->fsa_input);
227 controld_set_fsa_action_flags(new_actions);
228 fsa_dump_actions(new_actions, "New actions");
229
230 if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
231 crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
232 fsa_input2string(fsa_data->fsa_input),
233 fsa_state2string(globals->fsa_state),
234 fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
235 }
236
237
238 if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
239 do_fsa_action(fsa_data, A_ERROR, do_log);
240 }
241 if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
242 do_fsa_action(fsa_data, A_WARN, do_log);
243 }
244 if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
245 do_fsa_action(fsa_data, A_LOG, do_log);
246 }
247
248
249 last_state = globals->fsa_state;
250 globals->fsa_state = controld_fsa_get_next_state(fsa_data->fsa_input);
251
252
253
254
255 if ((globals->fsa_state == S_STOPPING)
256 || pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
257 controld_clear_fsa_action_flags(startup_actions);
258 }
259
260
261
262
263
264 if (last_state != globals->fsa_state) {
265 do_state_transition(last_state, globals->fsa_state, fsa_data);
266 } else {
267 do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
268 " \tInput=%s \tOrigin=%s() \tid=%d",
269 fsa_state2string(globals->fsa_state),
270 fsa_cause2string(fsa_data->fsa_cause),
271 fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
272 }
273
274
275 s_crmd_fsa_actions(fsa_data);
276 delete_fsa_input(fsa_data);
277 }
278
279 if ((controld_globals.fsa_message_queue != NULL)
280 || (controld_globals.fsa_actions != A_NOTHING)
281 || pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
282
283 crm_debug("Exiting the FSA: queue=%d, fsa_actions=%#llx, stalled=%s",
284 g_list_length(controld_globals.fsa_message_queue),
285 (unsigned long long) controld_globals.fsa_actions,
286 pcmk__btoa(pcmk_is_set(controld_globals.flags,
287 controld_fsa_is_stalled)));
288 } else {
289 crm_trace("Exiting the FSA");
290 }
291
292
293 if (register_copy != controld_globals.fsa_input_register) {
294 uint64_t same = register_copy & controld_globals.fsa_input_register;
295
296 fsa_dump_inputs(LOG_DEBUG, "Added",
297 controld_globals.fsa_input_register ^ same);
298 fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
299 }
300
301 fsa_dump_actions(controld_globals.fsa_actions, "Remaining");
302 fsa_dump_queue(LOG_DEBUG);
303
304 return globals->fsa_state;
305 }
306
307 void
308 s_crmd_fsa_actions(fsa_data_t * fsa_data)
309 {
310
311
312
313
314 CRM_CHECK(fsa_data != NULL, return);
315 while ((controld_globals.fsa_actions != A_NOTHING)
316 && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
317
318
319
320
321
322
323 if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
324 do_fsa_action(fsa_data, A_ERROR, do_log);
325 } else if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
326 do_fsa_action(fsa_data, A_WARN, do_log);
327 } else if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
328 do_fsa_action(fsa_data, A_LOG, do_log);
329
330
331 } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_1)) {
332 do_fsa_action(fsa_data, A_EXIT_1, do_exit);
333
334
335 } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
336 O_LRM_RECONNECT)) {
337 do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control);
338
339 } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
340 O_CIB_RESTART)) {
341 do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control);
342
343 } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
344 O_PE_RESTART)) {
345 do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control);
346
347 } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
348 O_TE_RESTART)) {
349 do_fsa_action(fsa_data, O_TE_RESTART, do_te_control);
350
351
352 } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTUP)) {
353 do_fsa_action(fsa_data, A_STARTUP, do_startup);
354 } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_START)) {
355 do_fsa_action(fsa_data, A_CIB_START, do_cib_control);
356 } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_CONNECT)) {
357 do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control);
358 } else if (pcmk_is_set(controld_globals.fsa_actions, A_READCONFIG)) {
359 do_fsa_action(fsa_data, A_READCONFIG, do_read_config);
360
361
362 } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_CONNECT)) {
363 do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control);
364 } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_START)) {
365 do_fsa_action(fsa_data, A_TE_START, do_te_control);
366 } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_START)) {
367 do_fsa_action(fsa_data, A_PE_START, do_pe_control);
368
369
370 } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TIMER_STOP)) {
371 do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control);
372 } else if (pcmk_is_set(controld_globals.fsa_actions,
373 A_INTEGRATE_TIMER_STOP)) {
374 do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control);
375 } else if (pcmk_is_set(controld_globals.fsa_actions,
376 A_INTEGRATE_TIMER_START)) {
377 do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control);
378 } else if (pcmk_is_set(controld_globals.fsa_actions,
379 A_FINALIZE_TIMER_STOP)) {
380 do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control);
381 } else if (pcmk_is_set(controld_globals.fsa_actions,
382 A_FINALIZE_TIMER_START)) {
383 do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control);
384
385
386
387
388 } else if (pcmk_is_set(controld_globals.fsa_actions, A_MSG_ROUTE)) {
389 do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route);
390 } else if (pcmk_is_set(controld_globals.fsa_actions, A_RECOVER)) {
391 do_fsa_action(fsa_data, A_RECOVER, do_recover);
392 } else if (pcmk_is_set(controld_globals.fsa_actions,
393 A_CL_JOIN_RESULT)) {
394 do_fsa_action(fsa_data, A_CL_JOIN_RESULT,
395 do_cl_join_finalize_respond);
396
397 } else if (pcmk_is_set(controld_globals.fsa_actions,
398 A_CL_JOIN_REQUEST)) {
399 do_fsa_action(fsa_data, A_CL_JOIN_REQUEST,
400 do_cl_join_offer_respond);
401
402 } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN_REQ)) {
403 do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req);
404 } else if (pcmk_is_set(controld_globals.fsa_actions, A_ELECTION_VOTE)) {
405 do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote);
406 } else if (pcmk_is_set(controld_globals.fsa_actions,
407 A_ELECTION_COUNT)) {
408 do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote);
409
410 } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_EVENT)) {
411 do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event);
412
413
414
415
416 } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTED)) {
417 do_fsa_action(fsa_data, A_STARTED, do_started);
418 } else if (pcmk_is_set(controld_globals.fsa_actions, A_CL_JOIN_QUERY)) {
419 do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query);
420 } else if (pcmk_is_set(controld_globals.fsa_actions,
421 A_DC_TIMER_START)) {
422 do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control);
423
424
425
426
427
428 } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TAKEOVER)) {
429 do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover);
430 } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASE)) {
431 do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release);
432 } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_JOIN_FINAL)) {
433 do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final);
434 } else if (pcmk_is_set(controld_globals.fsa_actions,
435 A_ELECTION_CHECK)) {
436 do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check);
437
438 } else if (pcmk_is_set(controld_globals.fsa_actions,
439 A_ELECTION_START)) {
440 do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote);
441
442 } else if (pcmk_is_set(controld_globals.fsa_actions,
443 A_DC_JOIN_OFFER_ALL)) {
444 do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all);
445
446 } else if (pcmk_is_set(controld_globals.fsa_actions,
447 A_DC_JOIN_OFFER_ONE)) {
448 do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one);
449
450 } else if (pcmk_is_set(controld_globals.fsa_actions,
451 A_DC_JOIN_PROCESS_REQ)) {
452 do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ,
453 do_dc_join_filter_offer);
454
455 } else if (pcmk_is_set(controld_globals.fsa_actions,
456 A_DC_JOIN_PROCESS_ACK)) {
457 do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack);
458
459 } else if (pcmk_is_set(controld_globals.fsa_actions,
460 A_DC_JOIN_FINALIZE)) {
461 do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize);
462
463 } else if (pcmk_is_set(controld_globals.fsa_actions,
464 A_CL_JOIN_ANNOUNCE)) {
465 do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce);
466
467
468
469
470
471
472 } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_HALT)) {
473 do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke);
474 } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_CANCEL)) {
475 do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke);
476 } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_INVOKE)) {
477 do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke);
478 } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_INVOKE)) {
479 do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke);
480 } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_INVOKE)) {
481 do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke);
482
483
484 } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASED)) {
485 do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release);
486 } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_STOP)) {
487 do_fsa_action(fsa_data, A_PE_STOP, do_pe_control);
488 } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_STOP)) {
489 do_fsa_action(fsa_data, A_TE_STOP, do_te_control);
490 } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN)) {
491 do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown);
492 } else if (pcmk_is_set(controld_globals.fsa_actions,
493 A_LRM_DISCONNECT)) {
494 do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control);
495
496 } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_DISCONNECT)) {
497 do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control);
498 } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_STOP)) {
499 do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control);
500 } else if (pcmk_is_set(controld_globals.fsa_actions, A_STOP)) {
501 do_fsa_action(fsa_data, A_STOP, do_stop);
502
503
504 } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_0)) {
505 do_fsa_action(fsa_data, A_EXIT_0, do_exit);
506
507
508 } else {
509 crm_err("Action %s not supported "CRM_XS" %#llx",
510 fsa_action2string(controld_globals.fsa_actions),
511 (unsigned long long) controld_globals.fsa_actions);
512 register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL,
513 __func__);
514 }
515 }
516 }
517
518 void
519 log_fsa_input(fsa_data_t * stored_msg)
520 {
521 CRM_ASSERT(stored_msg);
522 crm_trace("Processing queued input %d", stored_msg->id);
523 if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
524 crm_trace("FSA processing LRM callback from %s", stored_msg->origin);
525
526 } else if (stored_msg->data == NULL) {
527 crm_trace("FSA processing input from %s", stored_msg->origin);
528
529 } else {
530 ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg,
531 __func__);
532
533 crm_trace("FSA processing XML message from %s", stored_msg->origin);
534 crm_log_xml_trace(ha_input->xml, "FSA message data");
535 }
536 }
537
538 static void
539 check_join_counts(fsa_data_t *msg_data)
540 {
541 int count;
542 guint npeers;
543
544 count = crmd_join_phase_count(crm_join_finalized);
545 if (count > 0) {
546 crm_err("%d cluster node%s failed to confirm join",
547 count, pcmk__plural_s(count));
548 crmd_join_phase_log(LOG_NOTICE);
549 return;
550 }
551
552 npeers = crm_active_peers();
553 count = crmd_join_phase_count(crm_join_confirmed);
554 if (count == npeers) {
555 if (npeers == 1) {
556 crm_debug("Sole active cluster node is fully joined");
557 } else {
558 crm_debug("All %d active cluster nodes are fully joined", count);
559 }
560
561 } else if (count > npeers) {
562 crm_err("New election needed because more nodes confirmed join "
563 "than are in membership (%d > %u)", count, npeers);
564 register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
565
566 } else if (controld_globals.membership_id != crm_peer_seq) {
567 crm_info("New join needed because membership changed (%llu -> %llu)",
568 controld_globals.membership_id, crm_peer_seq);
569 register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
570
571 } else {
572 crm_warn("Only %d of %u active cluster nodes fully joined "
573 "(%d did not respond to offer)",
574 count, npeers, crmd_join_phase_count(crm_join_welcomed));
575 }
576 }
577
578 static void
579 do_state_transition(enum crmd_fsa_state cur_state,
580 enum crmd_fsa_state next_state, fsa_data_t *msg_data)
581 {
582 int level = LOG_INFO;
583 int count = 0;
584 gboolean clear_recovery_bit = TRUE;
585 #if 0
586 uint64_t original_fsa_actions = controld_globals.fsa_actions;
587 #endif
588
589 enum crmd_fsa_cause cause = msg_data->fsa_cause;
590 enum crmd_fsa_input current_input = msg_data->fsa_input;
591
592 const char *state_from = fsa_state2string(cur_state);
593 const char *state_to = fsa_state2string(next_state);
594 const char *input = fsa_input2string(current_input);
595
596 CRM_LOG_ASSERT(cur_state != next_state);
597
598 do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]",
599 state_from, state_to, input, fsa_cause2string(cause), msg_data->origin);
600
601 if (cur_state == S_IDLE || next_state == S_IDLE) {
602 level = LOG_NOTICE;
603 } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) {
604 level = LOG_NOTICE;
605 } else if (cur_state == S_ELECTION) {
606 level = LOG_NOTICE;
607 } else if (cur_state == S_STARTING) {
608 level = LOG_NOTICE;
609 } else if (next_state == S_RECOVERY) {
610 level = LOG_WARNING;
611 }
612
613 do_crm_log(level, "State transition %s -> %s "
614 CRM_XS " input=%s cause=%s origin=%s",
615 state_from, state_to, input, fsa_cause2string(cause),
616 msg_data->origin);
617
618 if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) {
619 controld_stop_current_election_timeout();
620 }
621 if (next_state == S_INTEGRATION) {
622 controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START);
623 } else {
624 controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP);
625 }
626
627 if (next_state == S_FINALIZE_JOIN) {
628 controld_set_fsa_action_flags(A_FINALIZE_TIMER_START);
629 } else {
630 controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP);
631 }
632
633 if (next_state != S_PENDING) {
634 controld_set_fsa_action_flags(A_DC_TIMER_STOP);
635 }
636 if (next_state != S_IDLE) {
637 controld_stop_recheck_timer();
638 }
639
640 if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
641 populate_cib_nodes(node_update_quick|node_update_all, __func__);
642 }
643
644 switch (next_state) {
645 case S_PENDING:
646 {
647 cib_t *cib_conn = controld_globals.cib_conn;
648 cib_conn->cmds->set_secondary(cib_conn, cib_scope_local);
649 }
650 update_dc(NULL);
651 break;
652
653 case S_ELECTION:
654 update_dc(NULL);
655 break;
656
657 case S_NOT_DC:
658 controld_reset_counter_election_timer();
659 purge_stonith_cleanup();
660
661 if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
662 crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
663 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
664 }
665 CRM_LOG_ASSERT(controld_globals.dc_name != NULL);
666 if (controld_globals.dc_name == NULL) {
667 crm_err("Reached S_NOT_DC without a DC" " being recorded");
668 }
669 break;
670
671 case S_RECOVERY:
672 clear_recovery_bit = FALSE;
673 break;
674
675 case S_FINALIZE_JOIN:
676 CRM_LOG_ASSERT(AM_I_DC);
677 if (cause == C_TIMER_POPPED) {
678 crm_warn("Progressed to state %s after %s",
679 fsa_state2string(next_state), fsa_cause2string(cause));
680 }
681 count = crmd_join_phase_count(crm_join_welcomed);
682 if (count > 0) {
683 crm_warn("%d cluster node%s failed to respond to join offer",
684 count, pcmk__plural_s(count));
685 crmd_join_phase_log(LOG_NOTICE);
686
687 } else {
688 crm_debug("All cluster nodes (%d) responded to join offer",
689 crmd_join_phase_count(crm_join_integrated));
690 }
691 break;
692
693 case S_POLICY_ENGINE:
694 controld_reset_counter_election_timer();
695 CRM_LOG_ASSERT(AM_I_DC);
696 if (cause == C_TIMER_POPPED) {
697 crm_info("Progressed to state %s after %s",
698 fsa_state2string(next_state), fsa_cause2string(cause));
699 }
700 check_join_counts(msg_data);
701 break;
702
703 case S_STOPPING:
704 case S_TERMINATE:
705
706 controld_set_fsa_input_flags(R_SHUTDOWN);
707 break;
708
709 case S_IDLE:
710 CRM_LOG_ASSERT(AM_I_DC);
711 if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
712 crm_info("(Re)Issuing shutdown request now" " that we are the DC");
713 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
714 }
715 controld_start_recheck_timer();
716 break;
717
718 default:
719 break;
720 }
721
722 if (clear_recovery_bit && next_state != S_PENDING) {
723 controld_clear_fsa_action_flags(A_RECOVER);
724 } else if (clear_recovery_bit == FALSE) {
725 controld_set_fsa_action_flags(A_RECOVER);
726 }
727
728 #if 0
729 if (original_fsa_actions != controld_globals.fsa_actions) {
730 fsa_dump_actions(original_fsa_actions ^ controld_globals.fsa_actions,
731 "New actions");
732 }
733 #endif
734 }