root/lrmd/main.c

/* */

DEFINITIONS

1. stonith_connection_destroy_cb
2. get_stonith_connection
3. lrmd_ipc_accept
4. lrmd_ipc_created
5. lrmd_ipc_dispatch
6. lrmd_client_destroy
7. lrmd_ipc_closed
8. lrmd_ipc_destroy
9. lrmd_server_send_reply
10. lrmd_server_send_notify
11. lrmd_exit
12. lrmd_shutdown
13. handle_shutdown_ack
14. handle_shutdown_nack
15. sigdone
16. sigreap
17. spawn_pidone
18. main

   1 /*
   2  * Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  *
  18  */
  19 
  20 #include <crm_internal.h>
  21 
  22 #include <glib.h>
  23 #include <unistd.h>
  24 #include <signal.h>
  25 
  26 #include <sys/types.h>
  27 #include <sys/wait.h>
  28 
  29 #include <crm/crm.h>
  30 #include <crm/msg_xml.h>
  31 #include <crm/services.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc.h>
  34 #include <crm/common/ipcs.h>
  35 
  36 #include <lrmd_private.h>
  37 
  38 #if defined(HAVE_GNUTLS_GNUTLS_H) && defined(SUPPORT_REMOTE)
  39 #  define ENABLE_PCMK_REMOTE
  40 #endif
  41 
  42 GMainLoop *mainloop = NULL;
  43 static qb_ipcs_service_t *ipcs = NULL;
  44 stonith_t *stonith_api = NULL;
  45 int lrmd_call_id = 0;
  46 
  47 #ifdef ENABLE_PCMK_REMOTE
  48 /* whether shutdown request has been sent */
  49 static volatile sig_atomic_t shutting_down = FALSE;
  50 
  51 /* timer for waiting for acknowledgment of shutdown request */
  52 static volatile guint shutdown_ack_timer = 0;
  53 
  54 static gboolean lrmd_exit(gpointer data);
  55 #endif
  56 
  57 static void
  58 stonith_connection_destroy_cb(stonith_t * st, stonith_event_t * e)
     /*  */
  59 {
  60     stonith_api->state = stonith_disconnected;
  61     crm_err("LRMD lost STONITH connection");
  62     stonith_connection_failed();
  63 }
  64 
  65 stonith_t *
  66 get_stonith_connection(void)
     /*  */
  67 {
  68     if (stonith_api && stonith_api->state == stonith_disconnected) {
  69         stonith_api_delete(stonith_api);
  70         stonith_api = NULL;
  71     }
  72 
  73     if (!stonith_api) {
  74         int rc = 0;
  75         int tries = 10;
  76 
  77         stonith_api = stonith_api_new();
  78         do {
  79             rc = stonith_api->cmds->connect(stonith_api, "lrmd", NULL);
  80             if (rc == pcmk_ok) {
  81                 stonith_api->cmds->register_notification(stonith_api,
  82                                                          T_STONITH_NOTIFY_DISCONNECT,
  83                                                          stonith_connection_destroy_cb);
  84                 break;
  85             }
  86             sleep(1);
  87             tries--;
  88         } while (tries);
  89 
  90         if (rc) {
  91             crm_err("Unable to connect to stonith daemon to execute command. error: %s",
  92                     pcmk_strerror(rc));
  93             stonith_api_delete(stonith_api);
  94             stonith_api = NULL;
  95         }
  96     }
  97     return stonith_api;
  98 }
  99 
 100 static int32_t
 101 lrmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
 102 {
 103     crm_trace("Connection %p", c);
 104     if (crm_client_new(c, uid, gid) == NULL) {
 105         return -EIO;
 106     }
 107     return 0;
 108 }
 109 
 110 static void
 111 lrmd_ipc_created(qb_ipcs_connection_t * c)
     /*  */
 112 {
 113     crm_client_t *new_client = crm_client_get(c);
 114 
 115     crm_trace("Connection %p", c);
 116     CRM_ASSERT(new_client != NULL);
 117     /* Now that the connection is offically established, alert
 118      * the other clients a new connection exists. */
 119 
 120     notify_of_new_client(new_client);
 121 }
 122 
 123 static int32_t
 124 lrmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
 125 {
 126     uint32_t id = 0;
 127     uint32_t flags = 0;
 128     crm_client_t *client = crm_client_get(c);
 129     xmlNode *request = crm_ipcs_recv(client, data, size, &id, &flags);
 130 
 131     CRM_CHECK(client != NULL, crm_err("Invalid client");
 132               return FALSE);
 133     CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
 134               return FALSE);
 135 
 136     CRM_CHECK(flags & crm_ipc_client_response, crm_err("Invalid client request: %p", client);
 137               return FALSE);
 138 
 139     if (!request) {
 140         return 0;
 141     }
 142 
 143     if (!client->name) {
 144         const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
 145 
 146         if (value == NULL) {
 147             client->name = crm_itoa(crm_ipcs_client_pid(c));
 148         } else {
 149             client->name = strdup(value);
 150         }
 151     }
 152 
 153     lrmd_call_id++;
 154     if (lrmd_call_id < 1) {
 155         lrmd_call_id = 1;
 156     }
 157 
 158     crm_xml_add(request, F_LRMD_CLIENTID, client->id);
 159     crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
 160     crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
 161 
 162     process_lrmd_message(client, id, request);
 163 
 164     free_xml(request);
 165     return 0;
 166 }
 167 
 168 /*!
 169  * \internal
 170  * \brief Free a client connection, and exit if appropriate
 171  *
 172  * \param[in] client  Client connection to free
 173  */
 174 void
 175 lrmd_client_destroy(crm_client_t *client)
     /*  */
 176 {
 177     crm_client_destroy(client);
 178 
 179 #ifdef ENABLE_PCMK_REMOTE
 180     /* If we were waiting to shut down, we can now safely do so
 181      * if there are no more proxied IPC providers
 182      */
 183     if (shutting_down && (ipc_proxy_get_provider() == NULL)) {
 184         lrmd_exit(NULL);
 185     }
 186 #endif
 187 }
 188 
 189 static int32_t
 190 lrmd_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 191 {
 192     crm_client_t *client = crm_client_get(c);
 193 
 194     if (client == NULL) {
 195         return 0;
 196     }
 197 
 198     crm_trace("Connection %p", c);
 199     client_disconnect_cleanup(client->id);
 200 #ifdef ENABLE_PCMK_REMOTE
 201     ipc_proxy_remove_provider(client);
 202 #endif
 203     lrmd_client_destroy(client);
 204     return 0;
 205 }
 206 
 207 static void
 208 lrmd_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 209 {
 210     lrmd_ipc_closed(c);
 211     crm_trace("Connection %p", c);
 212 }
 213 
 214 static struct qb_ipcs_service_handlers lrmd_ipc_callbacks = {
 215     .connection_accept = lrmd_ipc_accept,
 216     .connection_created = lrmd_ipc_created,
 217     .msg_process = lrmd_ipc_dispatch,
 218     .connection_closed = lrmd_ipc_closed,
 219     .connection_destroyed = lrmd_ipc_destroy
 220 };
 221 
 222 int
 223 lrmd_server_send_reply(crm_client_t * client, uint32_t id, xmlNode * reply)
     /*  */
 224 {
 225 
 226     crm_trace("Sending reply (%d) to client (%s)", id, client->id);
 227     switch (client->kind) {
 228         case CRM_CLIENT_IPC:
 229             return crm_ipcs_send(client, id, reply, FALSE);
 230 #ifdef ENABLE_PCMK_REMOTE
 231         case CRM_CLIENT_TLS:
 232             return lrmd_tls_send_msg(client->remote, reply, id, "reply");
 233 #endif
 234         default:
 235             crm_err("Could not send reply: unknown client type %d",
 236                     client->kind);
 237     }
 238     return -ENOTCONN;
 239 }
 240 
 241 int
 242 lrmd_server_send_notify(crm_client_t * client, xmlNode * msg)
     /*  */
 243 {
 244     crm_trace("Sending notification to client (%s)", client->id);
 245     switch (client->kind) {
 246         case CRM_CLIENT_IPC:
 247             if (client->ipcs == NULL) {
 248                 crm_trace("Could not notify local client: disconnected");
 249                 return -ENOTCONN;
 250             }
 251             return crm_ipcs_send(client, 0, msg, crm_ipc_server_event);
 252 #ifdef ENABLE_PCMK_REMOTE
 253         case CRM_CLIENT_TLS:
 254             if (client->remote == NULL) {
 255                 crm_trace("Could not notify remote client: disconnected");
 256                 return -ENOTCONN;
 257             }
 258             return lrmd_tls_send_msg(client->remote, msg, 0, "notify");
 259 #endif
 260         default:
 261             crm_err("Could not notify client: unknown type %d", client->kind);
 262     }
 263     return -ENOTCONN;
 264 }
 265 
 266 /*!
 267  * \internal
 268  * \brief Clean up and exit immediately
 269  *
 270  * \param[in] data  Ignored
 271  *
 272  * \return Doesn't return
 273  * \note   This can be used as a timer callback.
 274  */
 275 static gboolean
 276 lrmd_exit(gpointer data)
     /*  */
 277 {
 278     crm_info("Terminating with %d clients",
 279              crm_hash_table_size(client_connections));
 280 
 281     if (stonith_api) {
 282         stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
 283         stonith_api->cmds->disconnect(stonith_api);
 284         stonith_api_delete(stonith_api);
 285     }
 286     if (ipcs) {
 287         mainloop_del_ipc_server(ipcs);
 288     }
 289 
 290 #ifdef ENABLE_PCMK_REMOTE
 291     lrmd_tls_server_destroy();
 292     ipc_proxy_cleanup();
 293 #endif
 294 
 295     crm_client_cleanup();
 296     g_hash_table_destroy(rsc_list);
 297 
 298     if (mainloop) {
 299         lrmd_drain_alerts(g_main_loop_get_context(mainloop));
 300     }
 301 
 302     crm_exit(pcmk_ok);
 303     return FALSE;
 304 }
 305 
 306 /*!
 307  * \internal
 308  * \brief Request cluster shutdown if appropriate, otherwise exit immediately
 309  *
 310  * \param[in] nsig  Signal that caused invocation (ignored)
 311  */
 312 static void
 313 lrmd_shutdown(int nsig)
     /*  */
 314 {
 315 #ifdef ENABLE_PCMK_REMOTE
 316     crm_client_t *ipc_proxy = ipc_proxy_get_provider();
 317 
 318     /* If there are active proxied IPC providers, then we may be running
 319      * resources, so notify the cluster that we wish to shut down.
 320      */
 321     if (ipc_proxy) {
 322         if (shutting_down) {
 323             crm_notice("Waiting for cluster to stop resources before exiting");
 324             return;
 325         }
 326 
 327         crm_info("Sending shutdown request to cluster");
 328         if (ipc_proxy_shutdown_req(ipc_proxy) < 0) {
 329             crm_crit("Shutdown request failed, exiting immediately");
 330 
 331         } else {
 332             /* We requested a shutdown. Now, we need to wait for an
 333              * acknowledgement from the proxy host (which ensures the proxy host
 334              * supports shutdown requests), then wait for all proxy hosts to
 335              * disconnect (which ensures that all resources have been stopped).
 336              */
 337             shutting_down = TRUE;
 338 
 339             /* Stop accepting new proxy connections */
 340             lrmd_tls_server_destroy();
 341 
 342             /* Older crmd versions will never acknowledge our request, so set a
 343              * fairly short timeout to exit quickly in that case. If we get the
 344              * ack, we'll defuse this timer.
 345              */
 346             shutdown_ack_timer = g_timeout_add_seconds(20, lrmd_exit, NULL);
 347 
 348             /* Currently, we let the OS kill us if the clients don't disconnect
 349              * in a reasonable time. We could instead set a long timer here
 350              * (shorter than what the OS is likely to use) and exit immediately
 351              * if it pops.
 352              */
 353             return;
 354         }
 355     }
 356 #endif
 357     lrmd_exit(NULL);
 358 }
 359 
 360 /*!
 361  * \internal
 362  * \brief Defuse short exit timer if shutting down
 363  */
 364 void handle_shutdown_ack()
     /*  */
 365 {
 366 #ifdef ENABLE_PCMK_REMOTE
 367     if (shutting_down) {
 368         crm_info("Received shutdown ack");
 369         if (shutdown_ack_timer > 0) {
 370             g_source_remove(shutdown_ack_timer);
 371             shutdown_ack_timer = 0;
 372         }
 373         return;
 374     }
 375 #endif
 376     crm_debug("Ignoring unexpected shutdown ack");
 377 }
 378 
 379 /*!
 380  * \internal
 381  * \brief Make short exit timer fire immediately
 382  */
 383 void handle_shutdown_nack()
     /*  */
 384 {
 385 #ifdef ENABLE_PCMK_REMOTE
 386     if (shutting_down) {
 387         crm_info("Received shutdown nack");
 388         if (shutdown_ack_timer > 0) {
 389             g_source_remove(shutdown_ack_timer);
 390             shutdown_ack_timer = g_timeout_add(0, lrmd_exit, NULL);
 391         }
 392         return;
 393     }
 394 #endif
 395     crm_debug("Ignoring unexpected shutdown nack");
 396 }
 397 
 398 
 399 static pid_t main_pid = 0;
 400 static void
 401 sigdone(void)
     /*  */
 402 {
 403     exit(0);
 404 }
 405 
 406 static void
 407 sigreap(void)
     /*  */
 408 {
 409     pid_t pid = 0;
 410     int status;
 411     do {
 412         /*
 413          * Opinions seem to differ as to what to put here:
 414          *  -1, any child process
 415          *  0,  any child process whose process group ID is equal to that of the calling process
 416          */
 417         pid = waitpid(-1, &status, WNOHANG);
 418         if(pid == main_pid) {
 419             /* Exit when pacemaker-remote exits and use the same return code */
 420             if (WIFEXITED(status)) {
 421                 exit(WEXITSTATUS(status));
 422             }
 423             exit(1);
 424         }
 425 
 426     } while (pid > 0);
 427 }
 428 
 429 static struct {
 430         int sig;
 431         void (*handler)(void);
 432 } sigmap[] = {
 433         { SIGCHLD, sigreap },
 434         { SIGINT,  sigdone },
 435 };
 436 
 437 static void spawn_pidone(int argc, char **argv, char **envp)
     /*  */
 438 {
 439     sigset_t set;
 440 
 441     if (getpid() != 1) {
 442         return;
 443     }
 444 
 445     sigfillset(&set);
 446     sigprocmask(SIG_BLOCK, &set, 0);
 447 
 448     main_pid = fork();
 449     switch (main_pid) {
 450         case 0:
 451             sigprocmask(SIG_UNBLOCK, &set, NULL);
 452             setsid();
 453             setpgid(0, 0);
 454 
 455             /* Child remains as pacemaker_remoted */
 456             return;
 457         case -1:
 458             perror("fork");
 459     }
 460 
 461     /* Parent becomes the reaper of zombie processes */
 462     /* Safe to initialize logging now if needed */
 463 
 464 #ifdef HAVE___PROGNAME
 465     /* Differentiate ourselves in the 'ps' output */
 466     {
 467         char *p;
 468         int i, maxlen;
 469         char *LastArgv = NULL;
 470         const char *name = "pcmk-init";
 471 
 472         for(i = 0; i < argc; i++) {
 473                 if(!i || (LastArgv + 1 == argv[i]))
 474                         LastArgv = argv[i] + strlen(argv[i]);
 475         }
 476 
 477         for(i = 0; envp[i] != NULL; i++) {
 478                 if((LastArgv + 1) == envp[i]) {
 479                         LastArgv = envp[i] + strlen(envp[i]);
 480                 }
 481         }
 482 
 483         maxlen = (LastArgv - argv[0]) - 2;
 484 
 485         i = strlen(name);
 486         /* We can overwrite individual argv[] arguments */
 487         snprintf(argv[0], maxlen, "%s", name);
 488 
 489         /* Now zero out everything else */
 490         p = &argv[0][i];
 491         while(p < LastArgv)
 492             *p++ = '\0';
 493         argv[1] = NULL;
 494     }
 495 #endif /* HAVE___PROGNAME */
 496 
 497     while (1) {
 498         int sig;
 499         size_t i;
 500 
 501         sigwait(&set, &sig);
 502         for (i = 0; i < DIMOF(sigmap); i++) {
 503             if (sigmap[i].sig == sig) {
 504                 sigmap[i].handler();
 505                 break;
 506             }
 507         }
 508     }
 509 }
 510 
 511 /* *INDENT-OFF* */
 512 static struct crm_option long_options[] = {
 513     /* Top-level Options */
 514     {"help",    0, 0,    '?', "\tThis text"},
 515     {"version", 0, 0,    '$', "\tVersion information"  },
 516     {"verbose", 0, 0,    'V', "\tIncrease debug output"},
 517 
 518     {"logfile", 1, 0,    'l', "\tSend logs to the additional named logfile"},
 519 #ifdef ENABLE_PCMK_REMOTE
 520     {"port", 1, 0,       'p', "\tPort to listen on"},
 521 #endif
 522 
 523     /* For compatibility with the original lrmd */
 524     {"dummy",  0, 0, 'r', NULL, 1},
 525     {0, 0, 0, 0}
 526 };
 527 /* *INDENT-ON* */
 528 
 529 int
 530 main(int argc, char **argv, char **envp)
     /*  */
 531 {
 532     int flag = 0;
 533     int index = 0;
 534     int bump_log_num = 0;
 535     const char *option = NULL;
 536 
 537     /* If necessary, create PID1 now before any FDs are opened */
 538     spawn_pidone(argc, argv, envp);
 539 
 540 #ifndef ENABLE_PCMK_REMOTE
 541     crm_log_preinit("lrmd", argc, argv);
 542     crm_set_options(NULL, "[options]", long_options,
 543                     "Daemon for controlling services confirming to different standards");
 544 #else
 545     crm_log_preinit("pacemaker_remoted", argc, argv);
 546     crm_set_options(NULL, "[options]", long_options,
 547                     "Pacemaker Remote daemon for extending pacemaker functionality to remote nodes.");
 548 #endif
 549 
 550     while (1) {
 551         flag = crm_get_option(argc, argv, &index);
 552         if (flag == -1) {
 553             break;
 554         }
 555 
 556         switch (flag) {
 557             case 'r':
 558                 crm_warn("The -r option to lrmd is deprecated (and ignored) "
 559                          "and will be removed in a future release");
 560                 break;
 561             case 'l':
 562                 crm_add_logfile(optarg);
 563                 break;
 564             case 'p':
 565                 setenv("PCMK_remote_port", optarg, 1);
 566                 break;
 567             case 'V':
 568                 bump_log_num++;
 569                 break;
 570             case '?':
 571             case '$':
 572                 crm_help(flag, EX_OK);
 573                 break;
 574             default:
 575                 crm_help('?', EX_USAGE);
 576                 break;
 577         }
 578     }
 579 
 580     crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
 581 
 582     while (bump_log_num > 0) {
 583         crm_bump_log_level(argc, argv);
 584         bump_log_num--;
 585     }
 586 
 587     option = daemon_option("logfacility");
 588     if(option && safe_str_neq(option, "none")) {
 589         setenv("HA_LOGFACILITY", option, 1);  /* Used by the ocf_log/ha_log OCF macro */
 590     }
 591 
 592     option = daemon_option("logfile");
 593     if(option && safe_str_neq(option, "none")) {
 594         setenv("HA_LOGFILE", option, 1);      /* Used by the ocf_log/ha_log OCF macro */
 595 
 596         if (daemon_option_enabled(crm_system_name, "debug")) {
 597             setenv("HA_DEBUGLOG", option, 1); /* Used by the ocf_log/ha_debug OCF macro */
 598         }
 599     }
 600 
 601     /* The presence of this variable allegedly controls whether child
 602      * processes like httpd will try and use Systemd's sd_notify
 603      * API
 604      */
 605     unsetenv("NOTIFY_SOCKET");
 606 
 607     /* Used by RAs - Leave owned by root */
 608     crm_build_path(CRM_RSCTMP_DIR, 0755);
 609 
 610     /* Legacy: Used by RAs - Leave owned by root */
 611     crm_build_path(HA_STATE_DIR"/heartbeat/rsctmp", 0755);
 612 
 613     rsc_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_rsc);
 614     ipcs = mainloop_add_ipc_server(CRM_SYSTEM_LRMD, QB_IPC_SHM, &lrmd_ipc_callbacks);
 615     if (ipcs == NULL) {
 616         crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
 617         crm_exit(DAEMON_RESPAWN_STOP);
 618     }
 619 
 620 #ifdef ENABLE_PCMK_REMOTE
 621     if (lrmd_init_remote_tls_server() < 0) {
 622         crm_err("Failed to create TLS listener: shutting down and staying down");
 623         crm_exit(DAEMON_RESPAWN_STOP);
 624     }
 625     ipc_proxy_init();
 626 #endif
 627 
 628     mainloop_add_signal(SIGTERM, lrmd_shutdown);
 629     mainloop = g_main_new(FALSE);
 630     crm_info("Starting");
 631     g_main_run(mainloop);
 632 
 633     /* should never get here */
 634     lrmd_exit(NULL);
 635     return pcmk_ok;
 636 }

/* */