This source file includes following definitions.
- do_ha_control
- do_shutdown
- do_shutdown_req
- crmd_fast_exit
- crmd_exit
- do_exit
- sigpipe_ignore
- do_startup
- accept_controller_client
- dispatch_controller_ipc
- ipc_client_disconnected
- ipc_connection_destroyed
- do_stop
- do_started
- do_recover
- config_query_callback
- controld_trigger_config_as
- crm_read_options
- do_read_config
- crm_shutdown
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <sys/param.h>
13 #include <sys/types.h>
14 #include <sys/stat.h>
15
16 #include <crm/crm.h>
17 #include <crm/common/xml.h>
18 #include <crm/pengine/rules.h>
19 #include <crm/cluster/internal.h>
20 #include <crm/cluster/election_internal.h>
21 #include <crm/common/ipc_internal.h>
22
23 #include <pacemaker-controld.h>
24
25 static qb_ipcs_service_t *ipcs = NULL;
26
27 static crm_trigger_t *config_read_trigger = NULL;
28
29 #if SUPPORT_COROSYNC
30 extern gboolean crm_connect_corosync(pcmk_cluster_t *cluster);
31 #endif
32
33 static void crm_shutdown(int nsig);
34 static gboolean crm_read_options(gpointer user_data);
35
36
37 void
38 do_ha_control(long long action,
39 enum crmd_fsa_cause cause,
40 enum crmd_fsa_state cur_state,
41 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
42 {
43 gboolean registered = FALSE;
44
45 if (controld_globals.cluster == NULL) {
46 controld_globals.cluster = pcmk_cluster_new();
47 }
48
49 if (action & A_HA_DISCONNECT) {
50 pcmk_cluster_disconnect(controld_globals.cluster);
51 crm_info("Disconnected from the cluster");
52
53 controld_set_fsa_input_flags(R_HA_DISCONNECTED);
54 }
55
56 if (action & A_HA_CONNECT) {
57 pcmk__cluster_set_status_callback(&peer_update_callback);
58 pcmk__cluster_set_autoreap(false);
59
60 #if SUPPORT_COROSYNC
61 if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) {
62 registered = crm_connect_corosync(controld_globals.cluster);
63 }
64 #endif
65
66 if (registered) {
67 pcmk__node_status_t *node = controld_get_local_node_status();
68
69 controld_election_init();
70
71 free(controld_globals.our_uuid);
72 controld_globals.our_uuid =
73 pcmk__str_copy(pcmk__cluster_node_uuid(node));
74
75 if (controld_globals.our_uuid == NULL) {
76 crm_err("Could not obtain local uuid");
77 registered = FALSE;
78 }
79 }
80
81 if (!registered) {
82 controld_set_fsa_input_flags(R_HA_DISCONNECTED);
83 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
84 return;
85 }
86
87 populate_cib_nodes(node_update_none, __func__);
88 controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
89 crm_info("Connected to the cluster");
90 }
91
92 if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) {
93 crm_err("Unexpected action %s in %s", fsa_action2string(action),
94 __func__);
95 }
96 }
97
98
99 void
100 do_shutdown(long long action,
101 enum crmd_fsa_cause cause,
102 enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
103 {
104
105 controld_set_fsa_input_flags(R_SHUTDOWN);
106 controld_disconnect_fencer(FALSE);
107 }
108
109
110 void
111 do_shutdown_req(long long action,
112 enum crmd_fsa_cause cause,
113 enum crmd_fsa_state cur_state,
114 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
115 {
116 xmlNode *msg = NULL;
117
118 controld_set_fsa_input_flags(R_SHUTDOWN);
119
120 crm_info("Sending shutdown request to all peers (DC is %s)",
121 pcmk__s(controld_globals.dc_name, "not set"));
122 msg = pcmk__new_request(pcmk_ipc_controld, CRM_SYSTEM_CRMD, NULL,
123 CRM_SYSTEM_CRMD, CRM_OP_SHUTDOWN_REQ, NULL);
124
125 if (!pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg)) {
126 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
127 }
128 pcmk__xml_free(msg);
129 }
130
131 void
132 crmd_fast_exit(crm_exit_t exit_code)
133 {
134 if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) {
135 crm_warn("Inhibiting respawn " QB_XS " remapping exit code %d to %d",
136 exit_code, CRM_EX_FATAL);
137 exit_code = CRM_EX_FATAL;
138
139 } else if ((exit_code == CRM_EX_OK)
140 && pcmk_is_set(controld_globals.fsa_input_register,
141 R_IN_RECOVERY)) {
142 crm_err("Could not recover from internal error");
143 exit_code = CRM_EX_ERROR;
144 }
145
146 if (controld_globals.logger_out != NULL) {
147 controld_globals.logger_out->finish(controld_globals.logger_out,
148 exit_code, true, NULL);
149 pcmk__output_free(controld_globals.logger_out);
150 controld_globals.logger_out = NULL;
151 }
152
153 crm_exit(exit_code);
154 }
155
156 crm_exit_t
157 crmd_exit(crm_exit_t exit_code)
158 {
159 GMainLoop *mloop = controld_globals.mainloop;
160
161 static bool in_progress = FALSE;
162
163 if (in_progress && (exit_code == CRM_EX_OK)) {
164 crm_debug("Exit is already in progress");
165 return exit_code;
166
167 } else if(in_progress) {
168 crm_notice("Error during shutdown process, exiting now with status %d (%s)",
169 exit_code, crm_exit_str(exit_code));
170 crm_write_blackbox(SIGTRAP, NULL);
171 crmd_fast_exit(exit_code);
172 }
173
174 in_progress = TRUE;
175 crm_trace("Preparing to exit with status %d (%s)",
176 exit_code, crm_exit_str(exit_code));
177
178
179 controld_set_fsa_input_flags(R_HA_DISCONNECTED);
180
181
182
183 if(ipcs) {
184 crm_trace("Closing IPC server");
185 mainloop_del_ipc_server(ipcs);
186 ipcs = NULL;
187 }
188
189 controld_close_attrd_ipc();
190 controld_shutdown_schedulerd_ipc();
191 controld_disconnect_fencer(TRUE);
192
193 if ((exit_code == CRM_EX_OK) && (controld_globals.mainloop == NULL)) {
194 crm_debug("No mainloop detected");
195 exit_code = CRM_EX_ERROR;
196 }
197
198
199
200
201
202
203
204 if (exit_code != CRM_EX_OK) {
205 crm_notice("Forcing immediate exit with status %d (%s)",
206 exit_code, crm_exit_str(exit_code));
207 crm_write_blackbox(SIGTRAP, NULL);
208 crmd_fast_exit(exit_code);
209 }
210
211
212
213 for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
214 iter = iter->next) {
215 fsa_data_t *fsa_data = (fsa_data_t *) iter->data;
216
217 crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
218 fsa_input2string(fsa_data->fsa_input),
219 fsa_state2string(controld_globals.fsa_state),
220 fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
221 delete_fsa_input(fsa_data);
222 }
223
224 controld_clear_fsa_input_flags(R_MEMBERSHIP);
225
226 g_list_free(controld_globals.fsa_message_queue);
227 controld_globals.fsa_message_queue = NULL;
228
229 controld_free_node_pending_timers();
230 election_reset(controld_globals.cluster);
231
232
233
234
235
236 controld_disconnect_cib_manager();
237
238 verify_stopped(controld_globals.fsa_state, LOG_WARNING);
239 controld_clear_fsa_input_flags(R_LRM_CONNECTED);
240 lrm_state_destroy_all();
241
242 mainloop_destroy_trigger(config_read_trigger);
243 config_read_trigger = NULL;
244
245 controld_destroy_fsa_trigger();
246 controld_destroy_transition_trigger();
247
248 pcmk__client_cleanup();
249 pcmk__cluster_destroy_node_caches();
250
251 controld_free_fsa_timers();
252 te_cleanup_stonith_history_sync(NULL, TRUE);
253 controld_free_sched_timer();
254
255 free(controld_globals.our_uuid);
256 controld_globals.our_uuid = NULL;
257
258 free(controld_globals.dc_name);
259 controld_globals.dc_name = NULL;
260
261 free(controld_globals.dc_version);
262 controld_globals.dc_version = NULL;
263
264 free(controld_globals.cluster_name);
265 controld_globals.cluster_name = NULL;
266
267 free(controld_globals.te_uuid);
268 controld_globals.te_uuid = NULL;
269
270 free_max_generation();
271 controld_destroy_failed_sync_table();
272 controld_destroy_outside_events_table();
273
274 mainloop_destroy_signal(SIGPIPE);
275 mainloop_destroy_signal(SIGUSR1);
276 mainloop_destroy_signal(SIGTERM);
277 mainloop_destroy_signal(SIGTRAP);
278
279
280 if (mloop) {
281 GMainContext *ctx = g_main_loop_get_context(controld_globals.mainloop);
282
283
284 controld_globals.mainloop = NULL;
285
286
287 mainloop_destroy_signal(SIGCHLD);
288
289 crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
290
291 {
292 int lpc = 0;
293
294 while((g_main_context_pending(ctx) && lpc < 10)) {
295 lpc++;
296 crm_trace("Iteration %d", lpc);
297 g_main_context_dispatch(ctx);
298 }
299 }
300
301 crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
302 g_main_loop_quit(mloop);
303
304
305 g_main_loop_unref(mloop);
306 } else {
307 mainloop_destroy_signal(SIGCHLD);
308 }
309
310 cib_delete(controld_globals.cib_conn);
311 controld_globals.cib_conn = NULL;
312
313 throttle_fini();
314
315 pcmk_cluster_free(controld_globals.cluster);
316 controld_globals.cluster = NULL;
317
318
319 crm_trace("Done preparing for exit with status %d (%s)",
320 exit_code, crm_exit_str(exit_code));
321 return exit_code;
322 }
323
324
325 void
326 do_exit(long long action,
327 enum crmd_fsa_cause cause,
328 enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
329 {
330 crm_exit_t exit_code = CRM_EX_OK;
331
332 if (pcmk_is_set(action, A_EXIT_1)) {
333 exit_code = CRM_EX_ERROR;
334 crm_err("Exiting now due to errors");
335 }
336 verify_stopped(cur_state, LOG_ERR);
337 crmd_exit(exit_code);
338 }
339
340 static void sigpipe_ignore(int nsig) { return; }
341
342
343 void
344 do_startup(long long action,
345 enum crmd_fsa_cause cause,
346 enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
347 {
348 crm_debug("Registering Signal Handlers");
349 mainloop_add_signal(SIGTERM, crm_shutdown);
350 mainloop_add_signal(SIGPIPE, sigpipe_ignore);
351
352 config_read_trigger = mainloop_add_trigger(G_PRIORITY_HIGH,
353 crm_read_options, NULL);
354
355 controld_init_fsa_trigger();
356 controld_init_transition_trigger();
357
358 crm_debug("Creating CIB manager and executor objects");
359 controld_globals.cib_conn = cib_new();
360
361 lrm_state_init_local();
362 if (controld_init_fsa_timers() == FALSE) {
363 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
364 }
365 }
366
367
368 static int32_t
369 accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
370 {
371 crm_trace("Accepting new IPC client connection");
372 if (pcmk__new_client(c, uid, gid) == NULL) {
373 return -ENOMEM;
374 }
375 return 0;
376 }
377
378
379 static int32_t
380 dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
381 {
382 uint32_t id = 0;
383 uint32_t flags = 0;
384 pcmk__client_t *client = pcmk__find_client(c);
385
386 xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);
387
388 if (msg == NULL) {
389 pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL,
390 CRM_EX_PROTOCOL);
391 return 0;
392 }
393 pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL,
394 CRM_EX_INDETERMINATE);
395
396 pcmk__assert(client->user != NULL);
397 pcmk__update_acl_user(msg, PCMK__XA_CRM_USER, client->user);
398
399 crm_xml_add(msg, PCMK__XA_CRM_SYS_FROM, client->id);
400 if (controld_authorize_ipc_message(msg, client, NULL)) {
401 crm_trace("Processing IPC message from client %s",
402 pcmk__client_name(client));
403 route_message(C_IPC_MESSAGE, msg);
404 }
405
406 controld_trigger_fsa();
407 pcmk__xml_free(msg);
408 return 0;
409 }
410
411 static int32_t
412 ipc_client_disconnected(qb_ipcs_connection_t *c)
413 {
414 pcmk__client_t *client = pcmk__find_client(c);
415
416 if (client) {
417 crm_trace("Disconnecting %sregistered client %s (%p/%p)",
418 (client->userdata? "" : "un"), pcmk__client_name(client),
419 c, client);
420 free(client->userdata);
421 pcmk__free_client(client);
422 controld_trigger_fsa();
423 }
424 return 0;
425 }
426
427 static void
428 ipc_connection_destroyed(qb_ipcs_connection_t *c)
429 {
430 crm_trace("Connection %p", c);
431 ipc_client_disconnected(c);
432 }
433
434
435 void
436 do_stop(long long action,
437 enum crmd_fsa_cause cause,
438 enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
439 {
440 crm_trace("Closing IPC server");
441 mainloop_del_ipc_server(ipcs); ipcs = NULL;
442 register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
443 }
444
445
446 void
447 do_started(long long action,
448 enum crmd_fsa_cause cause,
449 enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
450 {
451 static struct qb_ipcs_service_handlers crmd_callbacks = {
452 .connection_accept = accept_controller_client,
453 .connection_created = NULL,
454 .msg_process = dispatch_controller_ipc,
455 .connection_closed = ipc_client_disconnected,
456 .connection_destroyed = ipc_connection_destroyed
457 };
458
459 if (cur_state != S_STARTING) {
460 crm_err("Start cancelled... %s", fsa_state2string(cur_state));
461 return;
462
463 } else if (!pcmk_is_set(controld_globals.fsa_input_register,
464 R_MEMBERSHIP)) {
465 crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);
466
467 crmd_fsa_stall(TRUE);
468 return;
469
470 } else if (!pcmk_is_set(controld_globals.fsa_input_register,
471 R_LRM_CONNECTED)) {
472 crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);
473
474 crmd_fsa_stall(TRUE);
475 return;
476
477 } else if (!pcmk_is_set(controld_globals.fsa_input_register,
478 R_CIB_CONNECTED)) {
479 crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);
480
481 crmd_fsa_stall(TRUE);
482 return;
483
484 } else if (!pcmk_is_set(controld_globals.fsa_input_register,
485 R_READ_CONFIG)) {
486 crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);
487
488 crmd_fsa_stall(TRUE);
489 return;
490
491 } else if (!pcmk_is_set(controld_globals.fsa_input_register, R_PEER_DATA)) {
492
493 crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
494 crmd_fsa_stall(TRUE);
495 return;
496 }
497
498 crm_debug("Init server comms");
499 ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
500 if (ipcs == NULL) {
501 crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
502 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
503 } else {
504 crm_notice("Pacemaker controller successfully started and accepting connections");
505 }
506 controld_set_fsa_input_flags(R_ST_REQUIRED);
507 controld_timer_fencer_connect(GINT_TO_POINTER(TRUE));
508
509 controld_clear_fsa_input_flags(R_STARTING);
510 register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
511 }
512
513
514 void
515 do_recover(long long action,
516 enum crmd_fsa_cause cause,
517 enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
518 {
519 controld_set_fsa_input_flags(R_IN_RECOVERY);
520 crm_warn("Fast-tracking shutdown in response to errors");
521
522 register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
523 }
524
525 static void
526 config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
527 {
528 const char *value = NULL;
529 GHashTable *config_hash = NULL;
530 crm_time_t *now = crm_time_new(NULL);
531 xmlNode *crmconfig = NULL;
532 xmlNode *alerts = NULL;
533
534 if (rc != pcmk_ok) {
535 fsa_data_t *msg_data = NULL;
536
537 crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
538 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
539
540 if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
541 crm_err("The cluster is mis-configured - shutting down and staying down");
542 controld_set_fsa_input_flags(R_STAYDOWN);
543 }
544 goto bail;
545 }
546
547 crmconfig = output;
548 if ((crmconfig != NULL) && !pcmk__xe_is(crmconfig, PCMK_XE_CRM_CONFIG)) {
549 crmconfig = pcmk__xe_first_child(crmconfig, PCMK_XE_CRM_CONFIG, NULL,
550 NULL);
551 }
552 if (!crmconfig) {
553 fsa_data_t *msg_data = NULL;
554
555 crm_err("Local CIB query for " PCMK_XE_CRM_CONFIG " section failed");
556 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
557 goto bail;
558 }
559
560 crm_debug("Call %d : Parsing CIB options", call_id);
561 config_hash = pcmk__strkey_table(free, free);
562 pe_unpack_nvpairs(crmconfig, crmconfig, PCMK_XE_CLUSTER_PROPERTY_SET, NULL,
563 config_hash, PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, FALSE, now,
564 NULL);
565
566
567 pcmk__validate_cluster_options(config_hash);
568
569
570
571
572
573
574
575
576
577
578
579 value = g_hash_table_lookup(config_hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
580 controld_verify_stonith_watchdog_timeout(value);
581
582 value = g_hash_table_lookup(config_hash, PCMK_OPT_NO_QUORUM_POLICY);
583 if (pcmk__strcase_any_of(value, PCMK_VALUE_FENCE, PCMK_VALUE_FENCE_LEGACY,
584 NULL)
585 && (pcmk__locate_sbd() != 0)) {
586 controld_set_global_flags(controld_no_quorum_panic);
587 }
588
589 value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK);
590 if (crm_is_true(value)) {
591 controld_set_global_flags(controld_shutdown_lock_enabled);
592 } else {
593 controld_clear_global_flags(controld_shutdown_lock_enabled);
594 }
595
596 value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
597 pcmk_parse_interval_spec(value, &controld_globals.shutdown_lock_limit);
598 controld_globals.shutdown_lock_limit /= 1000;
599
600 value = g_hash_table_lookup(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
601 pcmk_parse_interval_spec(value, &controld_globals.node_pending_timeout);
602 controld_globals.node_pending_timeout /= 1000;
603
604 value = g_hash_table_lookup(config_hash, PCMK_OPT_CLUSTER_NAME);
605 pcmk__str_update(&(controld_globals.cluster_name), value);
606
607
608 controld_configure_election(config_hash);
609 controld_configure_fencing(config_hash);
610 controld_configure_fsa_timers(config_hash);
611 controld_configure_throttle(config_hash);
612
613 alerts = pcmk__xe_first_child(output, PCMK_XE_ALERTS, NULL, NULL);
614 crmd_unpack_alerts(alerts);
615
616 controld_set_fsa_input_flags(R_READ_CONFIG);
617 controld_trigger_fsa();
618
619 g_hash_table_destroy(config_hash);
620 bail:
621 crm_time_free(now);
622 }
623
624
625
626
627
628
629
630
631 void
632 controld_trigger_config_as(const char *fn, int line)
633 {
634 if (config_read_trigger != NULL) {
635 crm_trace("%s:%d - Triggered config processing", fn, line);
636 mainloop_set_trigger(config_read_trigger);
637 }
638 }
639
640 gboolean
641 crm_read_options(gpointer user_data)
642 {
643 cib_t *cib_conn = controld_globals.cib_conn;
644 int call_id = cib_conn->cmds->query(cib_conn,
645 "//" PCMK_XE_CRM_CONFIG
646 " | //" PCMK_XE_ALERTS,
647 NULL, cib_xpath);
648
649 fsa_register_cib_callback(call_id, NULL, config_query_callback);
650 crm_trace("Querying the CIB... call %d", call_id);
651 return TRUE;
652 }
653
654
655 void
656 do_read_config(long long action,
657 enum crmd_fsa_cause cause,
658 enum crmd_fsa_state cur_state,
659 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
660 {
661 throttle_init();
662 controld_trigger_config();
663 }
664
665 static void
666 crm_shutdown(int nsig)
667 {
668 const char *value = NULL;
669 guint default_period_ms = 0;
670
671 if ((controld_globals.mainloop == NULL)
672 || !g_main_loop_is_running(controld_globals.mainloop)) {
673 crmd_exit(CRM_EX_OK);
674 return;
675 }
676
677 if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
678 crm_err("Escalating shutdown");
679 register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
680 return;
681 }
682
683 controld_set_fsa_input_flags(R_SHUTDOWN);
684 register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
685
686
687
688
689
690
691
692 value = pcmk__cluster_option(NULL, PCMK_OPT_SHUTDOWN_ESCALATION);
693 pcmk_parse_interval_spec(value, &default_period_ms);
694 controld_shutdown_start_countdown(default_period_ms);
695 }