daemons/pacemakerd/pacemakerd.c

/* */
This source file includes following definitions.
pcmk_process_exit
pcmk_child_exit
stop_child
start_child
escalate_shutdown
pcmk_shutdown_worker
pcmk_ignore
pcmk_sigquit
pcmk_shutdown
pcmk_ipc_accept
pcmk_handle_ping_request
pcmk_ipc_dispatch
pcmk_ipc_closed
pcmk_ipc_destroy
mcp_chown
child_liveness
check_active_before_startup_processes
find_and_track_existing_processes
init_children_processes
remove_core_file_limit
request_shutdown
main
   1 /*
   2  * Copyright 2010-2020 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 #include "pacemakerd.h"
  12 
  13 #include <pwd.h>
  14 #include <grp.h>
  15 #include <poll.h>
  16 #include <errno.h>
  17 #include <stdio.h>
  18 #include <stdbool.h>
  19 #include <sys/stat.h>
  20 #include <sys/types.h>
  21 #include <sys/time.h>
  22 #include <sys/resource.h>
  23 #include <sys/reboot.h>
  24 
  25 #include <crm/crm.h>  /* indirectly: CRM_EX_* */
  26 #include <crm/cib/internal.h>  /* cib_channel_ro */
  27 #include <crm/msg_xml.h>
  28 #include <crm/common/ipc_internal.h>
  29 #include <crm/common/mainloop.h>
  30 #include <crm/cluster/internal.h>
  31 #include <crm/cluster.h>
  32 
  33 #include <dirent.h>
  34 #include <ctype.h>
  35 
  36 static gboolean fatal_error = FALSE;
  37 static GMainLoop *mainloop = NULL;
  38 static bool global_keep_tracking = false;
  39 
  40 #define PCMK_PROCESS_CHECK_INTERVAL 5
  41 
  42 static crm_trigger_t *shutdown_trigger = NULL;
  43 static crm_trigger_t *startup_trigger = NULL;
  44 static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid";
  45 
  46 /* state we report when asked via pacemakerd-api status-ping */
  47 static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
  48 static gboolean running_with_sbd = FALSE; /* local copy */
  49 /* When contacted via pacemakerd-api by a client having sbd in
  50  * the name we assume it is sbd-daemon which wants to know
  51  * if pacemakerd shutdown gracefully.
  52  * Thus when everything is shutdown properly pacemakerd
  53  * waits till it has reported the graceful completion of
  54  * shutdown to sbd and just when sbd-client closes the
  55  * connection we can assume that the report has arrived
  56  * properly so that pacemakerd can finally exit.
  57  * Following two variables are used to track that handshake.
  58  */
  59 static unsigned int shutdown_complete_state_reported_to = 0;
  60 static gboolean shutdown_complete_state_reported_client_closed = FALSE;
  61 
  62 typedef struct pcmk_child_s {
  63     pid_t pid;
  64     long flag;
  65     int start_seq;
  66     int respawn_count;
  67     gboolean respawn;
  68     const char *name;
  69     const char *uid;
  70     const char *command;
  71     const char *endpoint;  /* IPC server name */
  72 
  73     gboolean active_before_startup;
  74 } pcmk_child_t;
  75 
  76 /* Index into the array below */
  77 #define PCMK_CHILD_CONTROLD  3
  78 
  79 static pcmk_child_t pcmk_children[] = {
  80     {
  81         0, crm_proc_none,       0, 0, FALSE, "none",
  82         NULL, NULL
  83     },
  84     {
  85         0, crm_proc_execd,      3, 0, TRUE,  "pacemaker-execd",
  86         NULL, CRM_DAEMON_DIR "/pacemaker-execd",
  87         CRM_SYSTEM_LRMD
  88     },
  89     {
  90         0, crm_proc_based,      1, 0, TRUE,  "pacemaker-based",
  91         CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-based",
  92         PCMK__SERVER_BASED_RO
  93     },
  94     {
  95         0, crm_proc_controld,   6, 0, TRUE, "pacemaker-controld",
  96         CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-controld",
  97         CRM_SYSTEM_CRMD
  98     },
  99     {
 100         0, crm_proc_attrd,      4, 0, TRUE, "pacemaker-attrd",
 101         CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-attrd",
 102         T_ATTRD
 103     },
 104     {
 105         0, crm_proc_schedulerd, 5, 0, TRUE, "pacemaker-schedulerd",
 106         CRM_DAEMON_USER, CRM_DAEMON_DIR "/pacemaker-schedulerd",
 107         CRM_SYSTEM_PENGINE
 108     },
 109     {
 110         0, crm_proc_fenced,     2, 0, TRUE, "pacemaker-fenced",
 111         NULL, CRM_DAEMON_DIR "/pacemaker-fenced",
 112         "stonith-ng"
 113     },
 114 };
 115 
 116 static gboolean check_active_before_startup_processes(gpointer user_data);
 117 static int child_liveness(pcmk_child_t *child);
 118 static gboolean start_child(pcmk_child_t * child);
 119 
 120 static void
 121 pcmk_process_exit(pcmk_child_t * child)
     /*  */
 122 {
 123     child->pid = 0;
 124     child->active_before_startup = FALSE;
 125 
 126     child->respawn_count += 1;
 127     if (child->respawn_count > MAX_RESPAWN) {
 128         crm_err("Child respawn count exceeded by %s", child->name);
 129         child->respawn = FALSE;
 130     }
 131 
 132     if (shutdown_trigger) {
 133         /* resume step-wise shutdown (returned TRUE yields no parallelizing) */
 134         mainloop_set_trigger(shutdown_trigger);
 135 
 136     } else if (!child->respawn) {
 137         /* nothing to do */
 138 
 139     } else if (crm_is_true(getenv("PCMK_fail_fast"))) {
 140         crm_err("Rebooting system because of %s", child->name);
 141         pcmk__panic(__func__);
 142 
 143     } else if (child_liveness(child) == pcmk_rc_ok) {
 144         crm_warn("One-off suppressing strict respawning of a child process %s,"
 145                  " appears alright per %s IPC end-point",
 146                  child->name, child->endpoint);
 147         /* need to monitor how it evolves, and start new process if badly */
 148         child->active_before_startup = TRUE;
 149         if (!global_keep_tracking) {
 150             global_keep_tracking = true;
 151             g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
 152                                   check_active_before_startup_processes, NULL);
 153         }
 154 
 155     } else {
 156         crm_notice("Respawning failed child process: %s", child->name);
 157         start_child(child);
 158     }
 159 }
 160 
 161 static void
 162 pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
     /*  */
 163 {
 164     pcmk_child_t *child = mainloop_child_userdata(p);
 165     const char *name = mainloop_child_name(p);
 166 
 167     if (signo) {
 168         do_crm_log(((signo == SIGKILL)? LOG_WARNING : LOG_ERR),
 169                    "%s[%d] terminated with signal %d (core=%d)",
 170                    name, pid, signo, core);
 171 
 172     } else {
 173         switch(exitcode) {
 174             case CRM_EX_OK:
 175                 crm_info("%s[%d] exited with status %d (%s)",
 176                          name, pid, exitcode, crm_exit_str(exitcode));
 177                 break;
 178 
 179             case CRM_EX_FATAL:
 180                 crm_warn("Shutting cluster down because %s[%d] had fatal failure",
 181                          name, pid);
 182                 child->respawn = FALSE;
 183                 fatal_error = TRUE;
 184                 pcmk_shutdown(SIGTERM);
 185                 break;
 186 
 187             case CRM_EX_PANIC:
 188                 crm_emerg("%s[%d] instructed the machine to reset", name, pid);
 189                 child->respawn = FALSE;
 190                 fatal_error = TRUE;
 191                 pcmk__panic(__func__);
 192                 pcmk_shutdown(SIGTERM);
 193                 break;
 194 
 195             default:
 196                 crm_err("%s[%d] exited with status %d (%s)",
 197                         name, pid, exitcode, crm_exit_str(exitcode));
 198                 break;
 199         }
 200     }
 201 
 202     pcmk_process_exit(child);
 203 }
 204 
 205 static gboolean
 206 stop_child(pcmk_child_t * child, int signal)
     /*  */
 207 {
 208     if (signal == 0) {
 209         signal = SIGTERM;
 210     }
 211 
 212     /* why to skip PID of 1?
 213        - FreeBSD ~ how untrackable process behind IPC is masqueraded as
 214        - elsewhere: how "init" task is designated; in particular, in systemd
 215          arrangement of socket-based activation, this is pretty real */
 216     if (child->command == NULL || child->pid == PCMK__SPECIAL_PID) {
 217         crm_debug("Nothing to do for child \"%s\" (process %lld)",
 218                   child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid));
 219         return TRUE;
 220     }
 221 
 222     if (child->pid <= 0) {
 223         crm_trace("Client %s not running", child->name);
 224         return TRUE;
 225     }
 226 
 227     errno = 0;
 228     if (kill(child->pid, signal) == 0) {
 229         crm_notice("Stopping %s "CRM_XS" sent signal %d to process %lld",
 230                    child->name, signal, (long long) child->pid);
 231 
 232     } else {
 233         crm_err("Could not stop %s (process %lld) with signal %d: %s",
 234                 child->name, (long long) child->pid, signal, strerror(errno));
 235     }
 236 
 237     return TRUE;
 238 }
 239 
 240 static char *opts_default[] = { NULL, NULL };
 241 static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL };
 242 
 243 /* TODO once libqb is taught to juggle with IPC end-points carried over as
 244         bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325)
 245         it shall hand over these descriptors here if/once they are successfully
 246         pre-opened in (presumably) child_liveness(), to avoid any remaining
 247         room for races */
 248 static gboolean
 249 start_child(pcmk_child_t * child)
     /*  */
 250 {
 251     uid_t uid = 0;
 252     gid_t gid = 0;
 253     gboolean use_valgrind = FALSE;
 254     gboolean use_callgrind = FALSE;
 255     const char *env_valgrind = getenv("PCMK_valgrind_enabled");
 256     const char *env_callgrind = getenv("PCMK_callgrind_enabled");
 257 
 258     child->active_before_startup = FALSE;
 259 
 260     if (child->command == NULL) {
 261         crm_info("Nothing to do for child \"%s\"", child->name);
 262         return TRUE;
 263     }
 264 
 265     if (env_callgrind != NULL && crm_is_true(env_callgrind)) {
 266         use_callgrind = TRUE;
 267         use_valgrind = TRUE;
 268 
 269     } else if (env_callgrind != NULL && strstr(env_callgrind, child->name)) {
 270         use_callgrind = TRUE;
 271         use_valgrind = TRUE;
 272 
 273     } else if (env_valgrind != NULL && crm_is_true(env_valgrind)) {
 274         use_valgrind = TRUE;
 275 
 276     } else if (env_valgrind != NULL && strstr(env_valgrind, child->name)) {
 277         use_valgrind = TRUE;
 278     }
 279 
 280     if (use_valgrind && strlen(VALGRIND_BIN) == 0) {
 281         crm_warn("Cannot enable valgrind for %s:"
 282                  " The location of the valgrind binary is unknown", child->name);
 283         use_valgrind = FALSE;
 284     }
 285 
 286     if (child->uid) {
 287         if (crm_user_lookup(child->uid, &uid, &gid) < 0) {
 288             crm_err("Invalid user (%s) for %s: not found", child->uid, child->name);
 289             return FALSE;
 290         }
 291         crm_info("Using uid=%u and group=%u for process %s", uid, gid, child->name);
 292     }
 293 
 294     child->pid = fork();
 295     CRM_ASSERT(child->pid != -1);
 296 
 297     if (child->pid > 0) {
 298         /* parent */
 299         mainloop_child_add(child->pid, 0, child->name, child, pcmk_child_exit);
 300 
 301         crm_info("Forked child %lld for process %s%s",
 302                  (long long) child->pid, child->name,
 303                  use_valgrind ? " (valgrind enabled: " VALGRIND_BIN ")" : "");
 304         return TRUE;
 305 
 306     } else {
 307         /* Start a new session */
 308         (void)setsid();
 309 
 310         /* Setup the two alternate arg arrays */
 311         opts_vgrind[0] = strdup(VALGRIND_BIN);
 312         if (use_callgrind) {
 313             opts_vgrind[1] = strdup("--tool=callgrind");
 314             opts_vgrind[2] = strdup("--callgrind-out-file=" CRM_STATE_DIR "/callgrind.out.%p");
 315             opts_vgrind[3] = strdup(child->command);
 316             opts_vgrind[4] = NULL;
 317         } else {
 318             opts_vgrind[1] = strdup(child->command);
 319             opts_vgrind[2] = NULL;
 320             opts_vgrind[3] = NULL;
 321             opts_vgrind[4] = NULL;
 322         }
 323         opts_default[0] = strdup(child->command);
 324 
 325         if(gid) {
 326             // Whether we need root group access to talk to cluster layer
 327             bool need_root_group = TRUE;
 328 
 329             if (is_corosync_cluster()) {
 330                 /* Corosync clusters can drop root group access, because we set
 331                  * uidgid.gid.${gid}=1 via CMAP, which allows these processes to
 332                  * connect to corosync.
 333                  */
 334                 need_root_group = FALSE;
 335             }
 336 
 337             // Drop root group access if not needed
 338             if (!need_root_group && (setgid(gid) < 0)) {
 339                 crm_warn("Could not set group to %d: %s", gid, strerror(errno));
 340             }
 341 
 342             /* Initialize supplementary groups to only those always granted to
 343              * the user, plus haclient (so we can access IPC).
 344              */
 345             if (initgroups(child->uid, gid) < 0) {
 346                 crm_err("Cannot initialize groups for %s: %s (%d)", child->uid, pcmk_strerror(errno), errno);
 347             }
 348         }
 349 
 350         if (uid && setuid(uid) < 0) {
 351             crm_warn("Could not set user to %s (id %d): %s",
 352                      child->uid, uid, strerror(errno));
 353         }
 354 
 355         pcmk__close_fds_in_child(true);
 356 
 357         pcmk__open_devnull(O_RDONLY);   // stdin (fd 0)
 358         pcmk__open_devnull(O_WRONLY);   // stdout (fd 1)
 359         pcmk__open_devnull(O_WRONLY);   // stderr (fd 2)
 360 
 361         if (use_valgrind) {
 362             (void)execvp(VALGRIND_BIN, opts_vgrind);
 363         } else {
 364             (void)execvp(child->command, opts_default);
 365         }
 366         crm_crit("Could not execute %s: %s", child->command, strerror(errno));
 367         crm_exit(CRM_EX_FATAL);
 368     }
 369     return TRUE;                /* never reached */
 370 }
 371 
 372 static gboolean
 373 escalate_shutdown(gpointer data)
     /*  */
 374 {
 375 
 376     pcmk_child_t *child = data;
 377 
 378     if (child->pid == PCMK__SPECIAL_PID) {
 379         pcmk_process_exit(child);
 380 
 381     } else if (child->pid != 0) {
 382         /* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */
 383         crm_err("Child %s not terminating in a timely manner, forcing", child->name);
 384         stop_child(child, SIGSEGV);
 385     }
 386     return FALSE;
 387 }
 388 
 389 #define SHUTDOWN_ESCALATION_PERIOD 180000  /* 3m */
 390 
 391 static gboolean
 392 pcmk_shutdown_worker(gpointer user_data)
     /*  */
 393 {
 394     static int phase = SIZEOF(pcmk_children);
 395     static time_t next_log = 0;
 396 
 397     int lpc = 0;
 398 
 399     if (phase == SIZEOF(pcmk_children)) {
 400         crm_notice("Shutting down Pacemaker");
 401         pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
 402     }
 403 
 404     for (; phase > 0; phase--) {
 405         /* Don't stop anything with start_seq < 1 */
 406 
 407         for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) {
 408             pcmk_child_t *child = &(pcmk_children[lpc]);
 409 
 410             if (phase != child->start_seq) {
 411                 continue;
 412             }
 413 
 414             if (child->pid != 0) {
 415                 time_t now = time(NULL);
 416 
 417                 if (child->respawn) {
 418                     if (child->pid == PCMK__SPECIAL_PID) {
 419                         crm_warn("The process behind %s IPC cannot be"
 420                                  " terminated, so either wait the graceful"
 421                                  " period of %ld s for its native termination"
 422                                  " if it vitally depends on some other daemons"
 423                                  " going down in a controlled way already,"
 424                                  " or locate and kill the correct %s process"
 425                                  " on your own; set PCMK_fail_fast=1 to avoid"
 426                                  " this altogether next time around",
 427                                  child->name, (long) SHUTDOWN_ESCALATION_PERIOD,
 428                                  child->command);
 429                     }
 430                     next_log = now + 30;
 431                     child->respawn = FALSE;
 432                     stop_child(child, SIGTERM);
 433                     if (phase < pcmk_children[PCMK_CHILD_CONTROLD].start_seq) {
 434                         g_timeout_add(SHUTDOWN_ESCALATION_PERIOD,
 435                                       escalate_shutdown, child);
 436                     }
 437 
 438                 } else if (now >= next_log) {
 439                     next_log = now + 30;
 440                     crm_notice("Still waiting for %s to terminate "
 441                                CRM_XS " pid=%lld seq=%d",
 442                                child->name, (long long) child->pid,
 443                                child->start_seq);
 444                 }
 445                 return TRUE;
 446             }
 447 
 448             /* cleanup */
 449             crm_debug("%s confirmed stopped", child->name);
 450             child->pid = 0;
 451         }
 452     }
 453 
 454     crm_notice("Shutdown complete");
 455     pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
 456     if (!fatal_error && running_with_sbd &&
 457         pcmk__get_sbd_sync_resource_startup() &&
 458         !shutdown_complete_state_reported_client_closed) {
 459         crm_notice("Waiting for SBD to pick up shutdown-complete-state.");
 460         return TRUE;
 461     }
 462 
 463     {
 464         const char *delay = pcmk__env_option("shutdown_delay");
 465         if(delay) {
 466             sync();
 467             sleep(crm_get_msec(delay) / 1000);
 468         }
 469     }
 470 
 471     g_main_loop_quit(mainloop);
 472 
 473     if (fatal_error) {
 474         crm_notice("Shutting down and staying down after fatal error");
 475 #ifdef SUPPORT_COROSYNC
 476         pcmkd_shutdown_corosync();
 477 #endif
 478         crm_exit(CRM_EX_FATAL);
 479     }
 480 
 481     return TRUE;
 482 }
 483 
 484 static void
 485 pcmk_ignore(int nsig)
     /*  */
 486 {
 487     crm_info("Ignoring signal %s (%d)", strsignal(nsig), nsig);
 488 }
 489 
 490 static void
 491 pcmk_sigquit(int nsig)
     /*  */
 492 {
 493     pcmk__panic(__func__);
 494 }
 495 
 496 void
 497 pcmk_shutdown(int nsig)
     /*  */
 498 {
 499     if (shutdown_trigger == NULL) {
 500         shutdown_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, pcmk_shutdown_worker, NULL);
 501     }
 502     mainloop_set_trigger(shutdown_trigger);
 503 }
 504 
 505 static int32_t
 506 pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
 507 {
 508     crm_trace("Connection %p", c);
 509     if (pcmk__new_client(c, uid, gid) == NULL) {
 510         return -EIO;
 511     }
 512     return 0;
 513 }
 514 
 515 static void
 516 pcmk_handle_ping_request(pcmk__client_t *c, xmlNode *msg, uint32_t id)
     /*  */
 517 {
 518     const char *value = NULL;
 519     xmlNode *ping = NULL;
 520     xmlNode *reply = NULL;
 521     time_t pinged = time(NULL);
 522     const char *from = crm_element_value(msg, F_CRM_SYS_FROM);
 523 
 524     /* Pinged for status */
 525     crm_trace("Pinged from %s.%s",
 526               crm_str(crm_element_value(msg, F_CRM_ORIGIN)),
 527               from?from:"unknown");
 528     ping = create_xml_node(NULL, XML_CRM_TAG_PING);
 529     value = crm_element_value(msg, F_CRM_SYS_TO);
 530     crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
 531     crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state);
 532     crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged);
 533     crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
 534     reply = create_reply(msg, ping);
 535     free_xml(ping);
 536     if (reply) {
 537         if (pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event) !=
 538                 pcmk_rc_ok) {
 539             crm_err("Failed sending ping-reply");
 540         }
 541         free_xml(reply);
 542     } else {
 543         crm_err("Failed building ping-reply");
 544     }
 545     /* just proceed state on sbd pinging us */
 546     if (from && strstr(from, "sbd")) {
 547         if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, pcmk__str_none)) {
 548             if (pcmk__get_sbd_sync_resource_startup()) {
 549                 crm_notice("Shutdown-complete-state passed to SBD.");
 550             }
 551             shutdown_complete_state_reported_to = c->pid;
 552         } else if (pcmk__str_eq(pacemakerd_state, XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, pcmk__str_none)) {
 553             crm_notice("Received startup-trigger from SBD.");
 554             pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
 555             mainloop_set_trigger(startup_trigger);
 556         }
 557     }
 558 }
 559 
 560 /* Exit code means? */
 561 static int32_t
 562 pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
     /*  */
 563 {
 564     uint32_t id = 0;
 565     uint32_t flags = 0;
 566     const char *task = NULL;
 567     xmlNode *msg = NULL;
 568     pcmk__client_t *c = pcmk__find_client(qbc);
 569 
 570     CRM_CHECK(c != NULL, return 0);
 571 
 572     msg = pcmk__client_data2xml(c, data, &id, &flags);
 573     if (msg == NULL) {
 574         pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_PROTOCOL);
 575         return 0;
 576     }
 577 
 578     task = crm_element_value(msg, F_CRM_TASK);
 579     if (pcmk__str_eq(task, CRM_OP_QUIT, pcmk__str_none)) {
 580 #if ENABLE_ACL
 581         /* Only allow privileged users (i.e. root or hacluster)
 582          * to shut down Pacemaker from the command line (or direct IPC).
 583          *
 584          * We only check when ACLs are enabled, because without them, any client
 585          * with IPC access could shut down Pacemaker via the CIB anyway.
 586          */
 587         bool allowed = pcmk_is_set(c->flags, pcmk__client_privileged);
 588 #else
 589         bool allowed = true;
 590 #endif
 591         if (allowed) {
 592             crm_notice("Shutting down in response to IPC request %s from %s",
 593                        crm_element_value(msg, F_CRM_REFERENCE),
 594                        crm_element_value(msg, F_CRM_ORIGIN));
 595             pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_OK);
 596             pcmk_shutdown(15);
 597         } else {
 598             crm_warn("Ignoring shutdown request from unprivileged client %s",
 599                      pcmk__client_name(c));
 600             pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_INSUFFICIENT_PRIV);
 601         }
 602 
 603     } else if (pcmk__str_eq(task, CRM_OP_RM_NODE_CACHE, pcmk__str_none)) {
 604         crm_trace("Ignoring IPC request to purge node "
 605                   "because peer cache is not used");
 606         pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_OK);
 607 
 608     } else if (pcmk__str_eq(task, CRM_OP_PING, pcmk__str_none)) {
 609         pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_INDETERMINATE);
 610         pcmk_handle_ping_request(c, msg, id);
 611 
 612     } else {
 613         crm_debug("Unrecognized IPC command '%s' sent to pacemakerd",
 614                   crm_str(task));
 615         pcmk__ipc_send_ack(c, id, flags, "ack", CRM_EX_INVALID_PARAM);
 616     }
 617 
 618     free_xml(msg);
 619     return 0;
 620 }
 621 
 622 /* Error code means? */
 623 static int32_t
 624 pcmk_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 625 {
 626     pcmk__client_t *client = pcmk__find_client(c);
 627 
 628     if (client == NULL) {
 629         return 0;
 630     }
 631     crm_trace("Connection %p", c);
 632     if (shutdown_complete_state_reported_to == client->pid) {
 633         shutdown_complete_state_reported_client_closed = TRUE;
 634         if (shutdown_trigger) {
 635             mainloop_set_trigger(shutdown_trigger);
 636         }
 637     }
 638     pcmk__free_client(client);
 639     return 0;
 640 }
 641 
 642 static void
 643 pcmk_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 644 {
 645     crm_trace("Connection %p", c);
 646     pcmk_ipc_closed(c);
 647 }
 648 
 649 struct qb_ipcs_service_handlers mcp_ipc_callbacks = {
 650     .connection_accept = pcmk_ipc_accept,
 651     .connection_created = NULL,
 652     .msg_process = pcmk_ipc_dispatch,
 653     .connection_closed = pcmk_ipc_closed,
 654     .connection_destroyed = pcmk_ipc_destroy
 655 };
 656 
 657 static pcmk__cli_option_t long_options[] = {
 658     // long option, argument type, storage, short option, description, flags
 659     {
 660         "help", no_argument, NULL, '?',
 661         "\tThis text", pcmk__option_default
 662     },
 663     {
 664         "version", no_argument, NULL, '$',
 665         "\tVersion information", pcmk__option_default
 666     },
 667     {
 668         "verbose", no_argument, NULL, 'V',
 669         "\tIncrease debug output", pcmk__option_default
 670     },
 671     {
 672         "shutdown", no_argument, NULL, 'S',
 673         "\tInstruct Pacemaker to shutdown on this machine", pcmk__option_default
 674     },
 675     {
 676         "features", no_argument, NULL, 'F',
 677         "\tDisplay full version and list of features Pacemaker was built with",
 678         pcmk__option_default
 679     },
 680     {
 681         "-spacer-", no_argument, NULL, '-',
 682         "\nAdditional Options:", pcmk__option_default
 683     },
 684     {
 685         "foreground", no_argument, NULL, 'f',
 686         "\t(Ignored) Pacemaker always runs in the foreground",
 687         pcmk__option_default
 688     },
 689     {
 690         "pid-file", required_argument, NULL, 'p',
 691         "\t(Ignored) Daemon pid file location", pcmk__option_default
 692     },
 693     {
 694         "standby", no_argument, NULL, 's',
 695         "\tStart node in standby state", pcmk__option_default
 696     },
 697     { 0, 0, 0, 0 }
 698 };
 699 
 700 static void
 701 mcp_chown(const char *path, uid_t uid, gid_t gid)
     /*  */
 702 {
 703     int rc = chown(path, uid, gid);
 704 
 705     if (rc < 0) {
 706         crm_warn("Cannot change the ownership of %s to user %s and gid %d: %s",
 707                  path, CRM_DAEMON_USER, gid, pcmk_strerror(errno));
 708     }
 709 }
 710 
 711 /*!
 712  * \internal
 713  * \brief Check the liveness of the child based on IPC name and PID if tracked
 714  *
 715  * \param[inout] child  Child tracked data
 716  *
 717  * \return Standard Pacemaker return code
 718  *
 719  * \note Return codes of particular interest include pcmk_rc_ipc_unresponsive
 720  *       indicating that no trace of IPC liveness was detected,
 721  *       pcmk_rc_ipc_unauthorized indicating that the IPC endpoint is blocked by
 722  *       an unauthorized process, and pcmk_rc_ipc_pid_only indicating that
 723  *       the child is up by PID but not IPC end-point (possibly starting).
 724  * \note This function doesn't modify any of \p child members but \c pid,
 725  *       and is not actively toying with processes as such but invoking
 726  *       \c stop_child in one particular case (there's for some reason
 727  *       a different authentic holder of the IPC end-point).
 728  */
 729 static int
 730 child_liveness(pcmk_child_t *child)
     /*  */
 731 {
 732     uid_t cl_uid = 0;
 733     gid_t cl_gid = 0;
 734     const uid_t root_uid = 0;
 735     const gid_t root_gid = 0;
 736     const uid_t *ref_uid;
 737     const gid_t *ref_gid;
 738     int rc = pcmk_rc_ipc_unresponsive;
 739     pid_t ipc_pid = 0;
 740 
 741     if (child->endpoint == NULL
 742             && (child->pid <= 0 || child->pid == PCMK__SPECIAL_PID)) {
 743         crm_err("Cannot track child %s for missing both API end-point and PID",
 744                 child->name);
 745         rc = EINVAL; // Misuse of function when child is not trackable
 746 
 747     } else if (child->endpoint != NULL) {
 748         int legacy_rc = pcmk_ok;
 749 
 750         if (child->uid == NULL) {
 751             ref_uid = &root_uid;
 752             ref_gid = &root_gid;
 753         } else {
 754             ref_uid = &cl_uid;
 755             ref_gid = &cl_gid;
 756             legacy_rc = pcmk_daemon_user(&cl_uid, &cl_gid);
 757         }
 758 
 759         if (legacy_rc < 0) {
 760             rc = pcmk_legacy2rc(legacy_rc);
 761             crm_err("Could not find user and group IDs for user %s: %s "
 762                     CRM_XS " rc=%d", CRM_DAEMON_USER, pcmk_rc_str(rc), rc);
 763         } else {
 764             rc = pcmk__ipc_is_authentic_process_active(child->endpoint,
 765                                                        *ref_uid, *ref_gid,
 766                                                        &ipc_pid);
 767             if ((rc == pcmk_rc_ok) || (rc == pcmk_rc_ipc_unresponsive)) {
 768                 if (child->pid <= 0) {
 769                     /* If rc is pcmk_rc_ok, ipc_pid is nonzero and this
 770                      * initializes a new child. If rc is
 771                      * pcmk_rc_ipc_unresponsive, ipc_pid is zero, and we will
 772                      * investigate further.
 773                      */
 774                     child->pid = ipc_pid;
 775                 } else if ((ipc_pid != 0) && (child->pid != ipc_pid)) {
 776                     /* An unexpected (but authorized) process is responding to
 777                      * IPC. Investigate further.
 778                      */
 779                     rc = pcmk_rc_ipc_unresponsive;
 780                 }
 781             }
 782         }
 783     }
 784 
 785     if (rc == pcmk_rc_ipc_unresponsive) {
 786         /* If we get here, a child without IPC is being tracked, no IPC liveness
 787          * has been detected, or IPC liveness has been detected with an
 788          * unexpected (but authorized) process. This is safe on FreeBSD since
 789          * the only change possible from a proper child's PID into "special" PID
 790          * of 1 behind more loosely related process.
 791          */
 792         int ret = pcmk__pid_active(child->pid, child->name);
 793 
 794         if (ipc_pid && ((ret != pcmk_rc_ok)
 795                         || ipc_pid == PCMK__SPECIAL_PID
 796                         || (pcmk__pid_active(ipc_pid,
 797                                              child->name) == pcmk_rc_ok))) {
 798             /* An unexpected (but authorized) process was detected at the IPC
 799              * endpoint, and either it is active, or the child we're tracking is
 800              * not.
 801              */
 802 
 803             if (ret == pcmk_rc_ok) {
 804                 /* The child we're tracking is active. Kill it, and adopt the
 805                  * detected process. This assumes that our children don't fork
 806                  * (thus getting a different PID owning the IPC), but rather the
 807                  * tracking got out of sync because of some means external to
 808                  * Pacemaker, and adopting the detected process is better than
 809                  * killing it and possibly having to spawn a new child.
 810                  */
 811                 /* not possessing IPC, afterall (what about corosync CPG?) */
 812                 stop_child(child, SIGKILL);
 813             }
 814             rc = pcmk_rc_ok;
 815             child->pid = ipc_pid;
 816         } else if (ret == pcmk_rc_ok) {
 817             // Our tracked child's PID was found active, but not its IPC
 818             rc = pcmk_rc_ipc_pid_only;
 819         } else if ((child->pid == 0) && (ret == EINVAL)) {
 820             // FreeBSD can return EINVAL
 821             rc = pcmk_rc_ipc_unresponsive;
 822         } else {
 823             switch (ret) {
 824                 case EACCES:
 825                     rc = pcmk_rc_ipc_unauthorized;
 826                     break;
 827                 case ESRCH:
 828                     rc = pcmk_rc_ipc_unresponsive;
 829                     break;
 830                 default:
 831                     rc = ret;
 832                     break;
 833             }
 834         }
 835     }
 836     return rc;
 837 }
 838 
 839 static gboolean
 840 check_active_before_startup_processes(gpointer user_data)
     /*  */
 841 {
 842     int start_seq = 1, lpc = 0;
 843     static int max = SIZEOF(pcmk_children);
 844     gboolean keep_tracking = FALSE;
 845 
 846     for (start_seq = 1; start_seq < max; start_seq++) {
 847         for (lpc = 0; lpc < max; lpc++) {
 848             if (pcmk_children[lpc].active_before_startup == FALSE) {
 849                 /* we are already tracking it as a child process. */
 850                 continue;
 851             } else if (start_seq != pcmk_children[lpc].start_seq) {
 852                 continue;
 853             } else {
 854                 int rc = child_liveness(&pcmk_children[lpc]);
 855 
 856                 switch (rc) {
 857                     case pcmk_rc_ok:
 858                         break;
 859                     case pcmk_rc_ipc_unresponsive:
 860                     case pcmk_rc_ipc_pid_only: // This case: it was previously OK
 861                         if (pcmk_children[lpc].respawn == TRUE) {
 862                             crm_err("%s[%lld] terminated%s", pcmk_children[lpc].name,
 863                                     (long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid),
 864                                     (rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
 865                         } else {
 866                             /* orderly shutdown */
 867                             crm_notice("%s[%lld] terminated%s", pcmk_children[lpc].name,
 868                                        (long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[lpc].pid),
 869                                        (rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
 870                         }
 871                         pcmk_process_exit(&(pcmk_children[lpc]));
 872                         continue;
 873                     default:
 874                         crm_exit(CRM_EX_FATAL);
 875                         break;  /* static analysis/noreturn */
 876                 }
 877             }
 878             /* at least one of the processes found at startup
 879              * is still going, so keep this recurring timer around */
 880             keep_tracking = TRUE;
 881         }
 882     }
 883 
 884     global_keep_tracking = keep_tracking;
 885     return keep_tracking;
 886 }
 887 
 888 /*!
 889  * \internal
 890  * \brief Initial one-off check of the pre-existing "child" processes
 891  *
 892  * With "child" process, we mean the subdaemon that defines an API end-point
 893  * (all of them do as of the comment) -- the possible complement is skipped
 894  * as it is deemed it has no such shared resources to cause conflicts about,
 895  * hence it can presumably be started anew without hesitation.
 896  * If that won't hold true in the future, the concept of a shared resource
 897  * will have to be generalized beyond the API end-point.
 898  *
 899  * For boundary cases that the "child" is still starting (IPC end-point is yet
 900  * to be witnessed), or more rarely (practically FreeBSD only), when there's
 901  * a pre-existing "untrackable" authentic process, we give the situation some
 902  * time to possibly unfold in the right direction, meaning that said socket
 903  * will appear or the unattainable process will disappear per the observable
 904  * IPC, respectively.
 905  *
 906  * \return Standard Pacemaker return code
 907  *
 908  * \note Since this gets run at the very start, \c respawn_count fields
 909  *       for particular children get temporarily overloaded with "rounds
 910  *       of waiting" tracking, restored once we are about to finish with
 911  *       success (i.e. returning value >=0) and will remain unrestored
 912  *       otherwise.  One way to suppress liveness detection logic for
 913  *       particular child is to set the said value to a negative number.
 914  */
 915 #define WAIT_TRIES 4  /* together with interleaved sleeps, worst case ~ 1s */
 916 static int
 917 find_and_track_existing_processes(void)
     /*  */
 918 {
 919     bool tracking = false;
 920     bool wait_in_progress;
 921     int rc;
 922     size_t i, rounds;
 923 
 924     for (rounds = 1; rounds <= WAIT_TRIES; rounds++) {
 925         wait_in_progress = false;
 926         for (i = 0; i < SIZEOF(pcmk_children); i++) {
 927 
 928             if ((pcmk_children[i].endpoint == NULL)
 929                 || (pcmk_children[i].respawn_count < 0)) {
 930                 continue;
 931             }
 932 
 933             rc = child_liveness(&pcmk_children[i]);
 934             if (rc == pcmk_rc_ipc_unresponsive) {
 935                 /* As a speculation, don't give up if there are more rounds to
 936                  * come for other reasons, but don't artificially wait just
 937                  * because of this, since we would preferably start ASAP.
 938                  */
 939                 continue;
 940             }
 941 
 942             pcmk_children[i].respawn_count = rounds;
 943             switch (rc) {
 944                 case pcmk_rc_ok:
 945                     if (pcmk_children[i].pid == PCMK__SPECIAL_PID) {
 946                         if (crm_is_true(getenv("PCMK_fail_fast"))) {
 947                             crm_crit("Cannot reliably track pre-existing"
 948                                      " authentic process behind %s IPC on this"
 949                                      " platform and PCMK_fail_fast requested",
 950                                      pcmk_children[i].endpoint);
 951                             return EOPNOTSUPP;
 952                         } else if (pcmk_children[i].respawn_count == WAIT_TRIES) {
 953                             crm_notice("Assuming pre-existing authentic, though"
 954                                        " on this platform untrackable, process"
 955                                        " behind %s IPC is stable (was in %d"
 956                                        " previous samples) so rather than"
 957                                        " bailing out (PCMK_fail_fast not"
 958                                        " requested), we just switch to a less"
 959                                        " optimal IPC liveness monitoring"
 960                                        " (not very suitable for heavy load)",
 961                                        pcmk_children[i].name, WAIT_TRIES - 1);
 962                             crm_warn("The process behind %s IPC cannot be"
 963                                      " terminated, so the overall shutdown"
 964                                      " will get delayed implicitly (%ld s),"
 965                                      " which serves as a graceful period for"
 966                                      " its native termination if it vitally"
 967                                      " depends on some other daemons going"
 968                                      " down in a controlled way already",
 969                                      pcmk_children[i].name,
 970                                      (long) SHUTDOWN_ESCALATION_PERIOD);
 971                         } else {
 972                             wait_in_progress = true;
 973                             crm_warn("Cannot reliably track pre-existing"
 974                                      " authentic process behind %s IPC on this"
 975                                      " platform, can still disappear in %d"
 976                                      " attempt(s)", pcmk_children[i].endpoint,
 977                                      WAIT_TRIES - pcmk_children[i].respawn_count);
 978                             continue;
 979                         }
 980                     }
 981                     crm_notice("Tracking existing %s process (pid=%lld)",
 982                                pcmk_children[i].name,
 983                                (long long) PCMK__SPECIAL_PID_AS_0(
 984                                                pcmk_children[i].pid));
 985                     pcmk_children[i].respawn_count = -1;  /* 0~keep watching */
 986                     pcmk_children[i].active_before_startup = TRUE;
 987                     tracking = true;
 988                     break;
 989                 case pcmk_rc_ipc_pid_only:
 990                     if (pcmk_children[i].respawn_count == WAIT_TRIES) {
 991                         crm_crit("%s IPC end-point for existing authentic"
 992                                  " process %lld did not (re)appear",
 993                                  pcmk_children[i].endpoint,
 994                                  (long long) PCMK__SPECIAL_PID_AS_0(
 995                                                  pcmk_children[i].pid));
 996                         return rc;
 997                     }
 998                     wait_in_progress = true;
 999                     crm_warn("Cannot find %s IPC end-point for existing"
1000                              " authentic process %lld, can still (re)appear"
1001                              " in %d attempts (?)",
1002                              pcmk_children[i].endpoint,
1003                              (long long) PCMK__SPECIAL_PID_AS_0(
1004                                              pcmk_children[i].pid),
1005                              WAIT_TRIES - pcmk_children[i].respawn_count);
1006                     continue;
1007                 default:
1008                     crm_crit("Checked liveness of %s: %s " CRM_XS " rc=%d",
1009                              pcmk_children[i].name, pcmk_rc_str(rc), rc);
1010                     return rc;
1011             }
1012         }
1013         if (!wait_in_progress) {
1014             break;
1015         }
1016         (void) poll(NULL, 0, 250);  /* a bit for changes to possibly happen */
1017     }
1018     for (i = 0; i < SIZEOF(pcmk_children); i++) {
1019         pcmk_children[i].respawn_count = 0;  /* restore pristine state */
1020     }
1021 
1022     if (tracking) {
1023         g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
1024                               check_active_before_startup_processes, NULL);
1025     }
1026     return pcmk_rc_ok;
1027 }
1028 
1029 static gboolean
1030 init_children_processes(void *user_data)
     /*  */
1031 {
1032     int start_seq = 1, lpc = 0;
1033     static int max = SIZEOF(pcmk_children);
1034 
1035     /* start any children that have not been detected */
1036     for (start_seq = 1; start_seq < max; start_seq++) {
1037         /* don't start anything with start_seq < 1 */
1038         for (lpc = 0; lpc < max; lpc++) {
1039             if (pcmk_children[lpc].pid != 0) {
1040                 /* we are already tracking it */
1041                 continue;
1042             }
1043 
1044             if (start_seq == pcmk_children[lpc].start_seq) {
1045                 start_child(&(pcmk_children[lpc]));
1046             }
1047         }
1048     }
1049 
1050     /* From this point on, any daemons being started will be due to
1051      * respawning rather than node start.
1052      *
1053      * This may be useful for the daemons to know
1054      */
1055     setenv("PCMK_respawned", "true", 1);
1056     pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
1057     return TRUE;
1058 }
1059 
1060 static void
1061 remove_core_file_limit(void)
     /*  */
1062 {
1063     struct rlimit cores;
1064     int rc = getrlimit(RLIMIT_CORE, &cores);
1065 
1066     if (rc < 0) {
1067         crm_warn("Cannot determine current maximum core file size: %s",
1068                  strerror(errno));
1069         return;
1070     }
1071 
1072     if ((cores.rlim_max == 0) && (geteuid() == 0)) {
1073         cores.rlim_max = RLIM_INFINITY;
1074     } else {
1075         crm_info("Maximum core file size is %llu bytes",
1076                  (unsigned long long) cores.rlim_max);
1077     }
1078     cores.rlim_cur = cores.rlim_max;
1079 
1080     rc = setrlimit(RLIMIT_CORE, &cores);
1081     if (rc < 0) {
1082         crm_warn("Cannot raise system limit on core file size "
1083                  "(consider doing so manually)");
1084     }
1085 }
1086 
1087 static crm_exit_t
1088 request_shutdown(crm_ipc_t *ipc)
     /*  */
1089 {
1090     xmlNode *request = NULL;
1091     xmlNode *reply = NULL;
1092     int rc = 0;
1093     crm_exit_t status = CRM_EX_OK;
1094 
1095     request = create_request(CRM_OP_QUIT, NULL, NULL, CRM_SYSTEM_MCP,
1096                              CRM_SYSTEM_MCP, NULL);
1097     if (request == NULL) {
1098         crm_err("Unable to create shutdown request"); // Probably memory error
1099         status = CRM_EX_TEMPFAIL;
1100         goto done;
1101     }
1102 
1103     crm_notice("Requesting shutdown of existing Pacemaker instance");
1104     rc = crm_ipc_send(ipc, request, crm_ipc_client_response, 0, &reply);
1105     if (rc < 0) {
1106         crm_err("Could not send shutdown request");
1107         status = crm_errno2exit(rc);
1108         goto done;
1109     }
1110 
1111     if ((rc == 0) || (reply == NULL)) {
1112         crm_err("Unrecognized response to shutdown request");
1113         status = CRM_EX_PROTOCOL;
1114         goto done;
1115     }
1116 
1117     if ((crm_element_value_int(reply, "status", &rc) == 0)
1118         && (rc != CRM_EX_OK)) {
1119         crm_err("Shutdown request failed: %s", crm_exit_str(rc));
1120         status = rc;
1121         goto done;
1122     }
1123 
1124     // Wait for pacemakerd to shut down IPC (with 30-minute timeout)
1125     status = CRM_EX_TIMEOUT;
1126     for (int i = 0; i < 900; ++i) {
1127         if (!crm_ipc_connected(ipc)) {
1128             status = CRM_EX_OK;
1129             break;
1130         }
1131         sleep(2);
1132     }
1133 
1134 done:
1135     free_xml(request);
1136     crm_ipc_close(ipc);
1137     crm_ipc_destroy(ipc);
1138     return status;
1139 }
1140 
1141 int
1142 main(int argc, char **argv)
     /*  */
1143 {
1144     int flag;
1145     int argerr = 0;
1146 
1147     int option_index = 0;
1148     gboolean shutdown = FALSE;
1149 
1150     uid_t pcmk_uid = 0;
1151     gid_t pcmk_gid = 0;
1152     crm_ipc_t *old_instance = NULL;
1153     qb_ipcs_service_t *ipcs = NULL;
1154 
1155     crm_log_preinit(NULL, argc, argv);
1156     pcmk__set_cli_options(NULL, "[options]", long_options,
1157                           "primary Pacemaker daemon that launches and "
1158                           "monitors all subsidiary Pacemaker daemons");
1159     mainloop_add_signal(SIGHUP, pcmk_ignore);
1160     mainloop_add_signal(SIGQUIT, pcmk_sigquit);
1161 
1162     while (1) {
1163         flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
1164         if (flag == -1)
1165             break;
1166 
1167         switch (flag) {
1168             case 'V':
1169                 crm_bump_log_level(argc, argv);
1170                 break;
1171             case 'f':
1172                 /* Legacy */
1173                 break;
1174             case 'p':
1175                 pid_file = optarg;
1176                 break;
1177             case 's':
1178                 pcmk__set_env_option("node_start_state", "standby");
1179                 break;
1180             case '$':
1181             case '?':
1182                 pcmk__cli_help(flag, CRM_EX_OK);
1183                 break;
1184             case 'S':
1185                 shutdown = TRUE;
1186                 break;
1187             case 'F':
1188                 printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n", PACEMAKER_VERSION, BUILD_VERSION,
1189                        CRM_FEATURE_SET, CRM_FEATURES);
1190                 crm_exit(CRM_EX_OK);
1191             default:
1192                 printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
1193                 ++argerr;
1194                 break;
1195         }
1196     }
1197 
1198     if (optind < argc) {
1199         printf("non-option ARGV-elements: ");
1200         while (optind < argc)
1201             printf("%s ", argv[optind++]);
1202         printf("\n");
1203     }
1204     if (argerr) {
1205         pcmk__cli_help('?', CRM_EX_USAGE);
1206     }
1207 
1208 
1209     setenv("LC_ALL", "C", 1);
1210 
1211     pcmk__set_env_option("mcp", "true");
1212 
1213     crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
1214 
1215     crm_debug("Checking for existing Pacemaker instance");
1216     old_instance = crm_ipc_new(CRM_SYSTEM_MCP, 0);
1217     (void) crm_ipc_connect(old_instance);
1218 
1219     if (shutdown) {
1220         crm_exit(request_shutdown(old_instance));
1221 
1222     } else if (crm_ipc_connected(old_instance)) {
1223         crm_ipc_close(old_instance);
1224         crm_ipc_destroy(old_instance);
1225         crm_err("Aborting start-up because active Pacemaker instance found");
1226         crm_exit(CRM_EX_FATAL);
1227     }
1228 
1229     crm_ipc_close(old_instance);
1230     crm_ipc_destroy(old_instance);
1231 
1232 #ifdef SUPPORT_COROSYNC
1233     if (mcp_read_config() == FALSE) {
1234         crm_exit(CRM_EX_UNAVAILABLE);
1235     }
1236 #endif
1237 
1238     // OCF shell functions and cluster-glue need facility under different name
1239     {
1240         const char *facility = pcmk__env_option("logfacility");
1241 
1242         if (facility && !pcmk__str_eq(facility, "none", pcmk__str_casei)) {
1243             setenv("HA_LOGFACILITY", facility, 1);
1244         }
1245     }
1246 
1247     crm_notice("Starting Pacemaker %s "CRM_XS" build=%s features:%s",
1248                PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
1249     mainloop = g_main_loop_new(NULL, FALSE);
1250 
1251     remove_core_file_limit();
1252 
1253     if (pcmk_daemon_user(&pcmk_uid, &pcmk_gid) < 0) {
1254         crm_err("Cluster user %s does not exist, aborting Pacemaker startup", CRM_DAEMON_USER);
1255         crm_exit(CRM_EX_NOUSER);
1256     }
1257 
1258     // Used by some resource agents
1259     if ((mkdir(CRM_STATE_DIR, 0750) < 0) && (errno != EEXIST)) {
1260         crm_warn("Could not create " CRM_STATE_DIR ": %s", pcmk_strerror(errno));
1261     } else {
1262         mcp_chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid);
1263     }
1264 
1265     /* Used to store core/blackbox/scheduler/cib files in */
1266     crm_build_path(CRM_PACEMAKER_DIR, 0750);
1267     mcp_chown(CRM_PACEMAKER_DIR, pcmk_uid, pcmk_gid);
1268 
1269     /* Used to store core files in */
1270     crm_build_path(CRM_CORE_DIR, 0750);
1271     mcp_chown(CRM_CORE_DIR, pcmk_uid, pcmk_gid);
1272 
1273     /* Used to store blackbox dumps in */
1274     crm_build_path(CRM_BLACKBOX_DIR, 0750);
1275     mcp_chown(CRM_BLACKBOX_DIR, pcmk_uid, pcmk_gid);
1276 
1277     // Used to store scheduler inputs in
1278     crm_build_path(PE_STATE_DIR, 0750);
1279     mcp_chown(PE_STATE_DIR, pcmk_uid, pcmk_gid);
1280 
1281     /* Used to store the cluster configuration */
1282     crm_build_path(CRM_CONFIG_DIR, 0750);
1283     mcp_chown(CRM_CONFIG_DIR, pcmk_uid, pcmk_gid);
1284 
1285     // Don't build CRM_RSCTMP_DIR, pacemaker-execd will do it
1286 
1287     ipcs = mainloop_add_ipc_server(CRM_SYSTEM_MCP, QB_IPC_NATIVE, &mcp_ipc_callbacks);
1288     if (ipcs == NULL) {
1289         crm_err("Couldn't start IPC server");
1290         crm_exit(CRM_EX_OSERR);
1291     }
1292 
1293 #ifdef SUPPORT_COROSYNC
1294     /* Allows us to block shutdown */
1295     if (!cluster_connect_cfg()) {
1296         crm_exit(CRM_EX_PROTOCOL);
1297     }
1298 #endif
1299 
1300     if (pcmk__locate_sbd() > 0) {
1301         setenv("PCMK_watchdog", "true", 1);
1302         running_with_sbd = TRUE;
1303     } else {
1304         setenv("PCMK_watchdog", "false", 1);
1305     }
1306 
1307     switch (find_and_track_existing_processes()) {
1308         case pcmk_rc_ok:
1309             break;
1310         case pcmk_rc_ipc_unauthorized:
1311             crm_exit(CRM_EX_CANTCREAT);
1312         default:
1313             crm_exit(CRM_EX_FATAL);
1314     };
1315 
1316     mainloop_add_signal(SIGTERM, pcmk_shutdown);
1317     mainloop_add_signal(SIGINT, pcmk_shutdown);
1318 
1319     if ((running_with_sbd) && pcmk__get_sbd_sync_resource_startup()) {
1320         crm_notice("Waiting for startup-trigger from SBD.");
1321         pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING;
1322         startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL);
1323     } else {
1324         if (running_with_sbd) {
1325             crm_warn("Enabling SBD_SYNC_RESOURCE_STARTUP would (if supported "
1326                      "by your SBD version) improve reliability of "
1327                      "interworking between SBD & pacemaker.");
1328         }
1329         pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
1330         init_children_processes(NULL);
1331     }
1332 
1333     crm_notice("Pacemaker daemon successfully started and accepting connections");
1334     g_main_loop_run(mainloop);
1335 
1336     if (ipcs) {
1337         crm_trace("Closing IPC server");
1338         mainloop_del_ipc_server(ipcs);
1339         ipcs = NULL;
1340     }
1341 
1342     g_main_loop_unref(mainloop);
1343 #ifdef SUPPORT_COROSYNC
1344     cluster_disconnect_cfg();
1345 #endif
1346     crm_exit(CRM_EX_OK);
1347 }
/* */
root/daemons/pacemakerd/pacemakerd.c

DEFINITIONS
