daemons/execd/pacemaker-execd.c

/* */
This source file includes following definitions.
stonith_connection_destroy_cb
get_stonith_connection
lrmd_ipc_accept
lrmd_ipc_created
lrmd_ipc_dispatch
lrmd_client_destroy
lrmd_ipc_closed
lrmd_ipc_destroy
lrmd_server_send_reply
lrmd_server_send_notify
lrmd_exit
lrmd_shutdown
handle_shutdown_ack
handle_shutdown_nack
build_arg_context
main
   1 /*
   2  * Copyright 2012-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <glib.h>
  13 #include <signal.h>
  14 #include <sys/types.h>
  15 
  16 #include <crm/crm.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/services.h>
  19 #include <crm/common/cmdline_internal.h>
  20 #include <crm/common/ipc.h>
  21 #include <crm/common/ipc_internal.h>
  22 #include <crm/common/mainloop.h>
  23 #include <crm/common/output_internal.h>
  24 #include <crm/common/remote_internal.h>
  25 #include <crm/lrmd_internal.h>
  26 
  27 #include "pacemaker-execd.h"
  28 
  29 #ifdef PCMK__COMPILE_REMOTE
  30 #  define EXECD_TYPE "remote"
  31 #  define EXECD_NAME "pacemaker-remoted"
  32 #  define SUMMARY "resource agent executor daemon for Pacemaker Remote nodes"
  33 #else
  34 #  define EXECD_TYPE "local"
  35 #  define EXECD_NAME "pacemaker-execd"
  36 #  define SUMMARY "resource agent executor daemon for Pacemaker cluster nodes"
  37 #endif
  38 
  39 static GMainLoop *mainloop = NULL;
  40 static qb_ipcs_service_t *ipcs = NULL;
  41 static stonith_t *stonith_api = NULL;
  42 int lrmd_call_id = 0;
  43 time_t start_time;
  44 
  45 static struct {
  46     gchar **log_files;
  47 #ifdef PCMK__COMPILE_REMOTE
  48     gchar *port;
  49 #endif  // PCMK__COMPILE_REMOTE
  50 } options;
  51 
  52 #ifdef PCMK__COMPILE_REMOTE
  53 /* whether shutdown request has been sent */
  54 static gboolean shutting_down = FALSE;
  55 
  56 /* timer for waiting for acknowledgment of shutdown request */
  57 static guint shutdown_ack_timer = 0;
  58 
  59 static gboolean lrmd_exit(gpointer data);
  60 #endif
  61 
  62 static void
  63 stonith_connection_destroy_cb(stonith_t * st, stonith_event_t * e)
     /*  */
  64 {
  65     stonith_api->state = stonith_disconnected;
  66     stonith_connection_failed();
  67 }
  68 
  69 stonith_t *
  70 get_stonith_connection(void)
     /*  */
  71 {
  72     if (stonith_api && stonith_api->state == stonith_disconnected) {
  73         stonith_api_delete(stonith_api);
  74         stonith_api = NULL;
  75     }
  76 
  77     if (stonith_api == NULL) {
  78         int rc = pcmk_ok;
  79 
  80         stonith_api = stonith_api_new();
  81         if (stonith_api == NULL) {
  82             crm_err("Could not connect to fencer: API memory allocation failed");
  83             return NULL;
  84         }
  85         rc = stonith_api_connect_retry(stonith_api, crm_system_name, 10);
  86         if (rc != pcmk_ok) {
  87             crm_err("Could not connect to fencer in 10 attempts: %s "
  88                     CRM_XS " rc=%d", pcmk_strerror(rc), rc);
  89             stonith_api_delete(stonith_api);
  90             stonith_api = NULL;
  91         } else {
  92             stonith_api->cmds->register_notification(stonith_api,
  93                                                      T_STONITH_NOTIFY_DISCONNECT,
  94                                                      stonith_connection_destroy_cb);
  95         }
  96     }
  97     return stonith_api;
  98 }
  99 
 100 static int32_t
 101 lrmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
 102 {
 103     crm_trace("Connection %p", c);
 104     if (pcmk__new_client(c, uid, gid) == NULL) {
 105         return -EIO;
 106     }
 107     return 0;
 108 }
 109 
 110 static void
 111 lrmd_ipc_created(qb_ipcs_connection_t * c)
     /*  */
 112 {
 113     pcmk__client_t *new_client = pcmk__find_client(c);
 114 
 115     crm_trace("Connection %p", c);
 116     CRM_ASSERT(new_client != NULL);
 117     /* Now that the connection is offically established, alert
 118      * the other clients a new connection exists. */
 119 
 120     notify_of_new_client(new_client);
 121 }
 122 
 123 static int32_t
 124 lrmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
 125 {
 126     uint32_t id = 0;
 127     uint32_t flags = 0;
 128     pcmk__client_t *client = pcmk__find_client(c);
 129     xmlNode *request = pcmk__client_data2xml(client, data, &id, &flags);
 130 
 131     CRM_CHECK(client != NULL, crm_err("Invalid client");
 132               return FALSE);
 133     CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
 134               return FALSE);
 135 
 136     CRM_CHECK(flags & crm_ipc_client_response, crm_err("Invalid client request: %p", client);
 137               return FALSE);
 138 
 139     if (!request) {
 140         return 0;
 141     }
 142 
 143     if (!client->name) {
 144         const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
 145 
 146         if (value == NULL) {
 147             client->name = pcmk__itoa(pcmk__client_pid(c));
 148         } else {
 149             client->name = strdup(value);
 150         }
 151     }
 152 
 153     lrmd_call_id++;
 154     if (lrmd_call_id < 1) {
 155         lrmd_call_id = 1;
 156     }
 157 
 158     crm_xml_add(request, F_LRMD_CLIENTID, client->id);
 159     crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
 160     crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
 161 
 162     process_lrmd_message(client, id, request);
 163 
 164     free_xml(request);
 165     return 0;
 166 }
 167 
 168 /*!
 169  * \internal
 170  * \brief Free a client connection, and exit if appropriate
 171  *
 172  * \param[in,out] client  Client connection to free
 173  */
 174 void
 175 lrmd_client_destroy(pcmk__client_t *client)
     /*  */
 176 {
 177     pcmk__free_client(client);
 178 
 179 #ifdef PCMK__COMPILE_REMOTE
 180     /* If we were waiting to shut down, we can now safely do so
 181      * if there are no more proxied IPC providers
 182      */
 183     if (shutting_down && (ipc_proxy_get_provider() == NULL)) {
 184         lrmd_exit(NULL);
 185     }
 186 #endif
 187 }
 188 
 189 static int32_t
 190 lrmd_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 191 {
 192     pcmk__client_t *client = pcmk__find_client(c);
 193 
 194     if (client == NULL) {
 195         return 0;
 196     }
 197 
 198     crm_trace("Connection %p", c);
 199     client_disconnect_cleanup(client->id);
 200 #ifdef PCMK__COMPILE_REMOTE
 201     ipc_proxy_remove_provider(client);
 202 #endif
 203     lrmd_client_destroy(client);
 204     return 0;
 205 }
 206 
 207 static void
 208 lrmd_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 209 {
 210     lrmd_ipc_closed(c);
 211     crm_trace("Connection %p", c);
 212 }
 213 
 214 static struct qb_ipcs_service_handlers lrmd_ipc_callbacks = {
 215     .connection_accept = lrmd_ipc_accept,
 216     .connection_created = lrmd_ipc_created,
 217     .msg_process = lrmd_ipc_dispatch,
 218     .connection_closed = lrmd_ipc_closed,
 219     .connection_destroyed = lrmd_ipc_destroy
 220 };
 221 
 222 // \return Standard Pacemaker return code
 223 int
 224 lrmd_server_send_reply(pcmk__client_t *client, uint32_t id, xmlNode *reply)
     /*  */
 225 {
 226     crm_trace("Sending reply (%d) to client (%s)", id, client->id);
 227     switch (PCMK__CLIENT_TYPE(client)) {
 228         case pcmk__client_ipc:
 229             return pcmk__ipc_send_xml(client, id, reply, FALSE);
 230 #ifdef PCMK__COMPILE_REMOTE
 231         case pcmk__client_tls:
 232             return lrmd__remote_send_xml(client->remote, reply, id, "reply");
 233 #endif
 234         default:
 235             crm_err("Could not send reply: unknown type for client %s "
 236                     CRM_XS " flags=%#llx",
 237                     pcmk__client_name(client), client->flags);
 238     }
 239     return ENOTCONN;
 240 }
 241 
 242 // \return Standard Pacemaker return code
 243 int
 244 lrmd_server_send_notify(pcmk__client_t *client, xmlNode *msg)
     /*  */
 245 {
 246     crm_trace("Sending notification to client (%s)", client->id);
 247     switch (PCMK__CLIENT_TYPE(client)) {
 248         case pcmk__client_ipc:
 249             if (client->ipcs == NULL) {
 250                 crm_trace("Could not notify local client: disconnected");
 251                 return ENOTCONN;
 252             }
 253             return pcmk__ipc_send_xml(client, 0, msg, crm_ipc_server_event);
 254 #ifdef PCMK__COMPILE_REMOTE
 255         case pcmk__client_tls:
 256             if (client->remote == NULL) {
 257                 crm_trace("Could not notify remote client: disconnected");
 258                 return ENOTCONN;
 259             } else {
 260                 return lrmd__remote_send_xml(client->remote, msg, 0, "notify");
 261             }
 262 #endif
 263         default:
 264             crm_err("Could not notify client %s with unknown transport "
 265                     CRM_XS " flags=%#llx",
 266                     pcmk__client_name(client), client->flags);
 267     }
 268     return ENOTCONN;
 269 }
 270 
 271 /*!
 272  * \internal
 273  * \brief Clean up and exit immediately
 274  *
 275  * \param[in] data  Ignored
 276  *
 277  * \return Doesn't return
 278  * \note   This can be used as a timer callback.
 279  */
 280 static gboolean
 281 lrmd_exit(gpointer data)
     /*  */
 282 {
 283     crm_info("Terminating with %d clients", pcmk__ipc_client_count());
 284     if (stonith_api) {
 285         stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
 286         stonith_api->cmds->disconnect(stonith_api);
 287         stonith_api_delete(stonith_api);
 288     }
 289     if (ipcs) {
 290         mainloop_del_ipc_server(ipcs);
 291     }
 292 
 293 #ifdef PCMK__COMPILE_REMOTE
 294     execd_stop_tls_server();
 295     ipc_proxy_cleanup();
 296 #endif
 297 
 298     pcmk__client_cleanup();
 299     g_hash_table_destroy(rsc_list);
 300 
 301     if (mainloop) {
 302         lrmd_drain_alerts(mainloop);
 303     }
 304 
 305     crm_exit(CRM_EX_OK);
 306     return FALSE;
 307 }
 308 
 309 /*!
 310  * \internal
 311  * \brief Request cluster shutdown if appropriate, otherwise exit immediately
 312  *
 313  * \param[in] nsig  Signal that caused invocation (ignored)
 314  */
 315 static void
 316 lrmd_shutdown(int nsig)
     /*  */
 317 {
 318 #ifdef PCMK__COMPILE_REMOTE
 319     pcmk__client_t *ipc_proxy = ipc_proxy_get_provider();
 320 
 321     /* If there are active proxied IPC providers, then we may be running
 322      * resources, so notify the cluster that we wish to shut down.
 323      */
 324     if (ipc_proxy) {
 325         if (shutting_down) {
 326             crm_notice("Waiting for cluster to stop resources before exiting");
 327             return;
 328         }
 329 
 330         crm_info("Sending shutdown request to cluster");
 331         if (ipc_proxy_shutdown_req(ipc_proxy) < 0) {
 332             crm_crit("Shutdown request failed, exiting immediately");
 333 
 334         } else {
 335             /* We requested a shutdown. Now, we need to wait for an
 336              * acknowledgement from the proxy host (which ensures the proxy host
 337              * supports shutdown requests), then wait for all proxy hosts to
 338              * disconnect (which ensures that all resources have been stopped).
 339              */
 340             shutting_down = TRUE;
 341 
 342             /* Stop accepting new proxy connections */
 343             execd_stop_tls_server();
 344 
 345             /* Older controller versions will never acknowledge our request, so
 346              * set a fairly short timeout to exit quickly in that case. If we
 347              * get the ack, we'll defuse this timer.
 348              */
 349             shutdown_ack_timer = g_timeout_add_seconds(20, lrmd_exit, NULL);
 350 
 351             /* Currently, we let the OS kill us if the clients don't disconnect
 352              * in a reasonable time. We could instead set a long timer here
 353              * (shorter than what the OS is likely to use) and exit immediately
 354              * if it pops.
 355              */
 356             return;
 357         }
 358     }
 359 #endif
 360     lrmd_exit(NULL);
 361 }
 362 
 363 /*!
 364  * \internal
 365  * \brief Defuse short exit timer if shutting down
 366  */
 367 void
 368 handle_shutdown_ack(void)
     /*  */
 369 {
 370 #ifdef PCMK__COMPILE_REMOTE
 371     if (shutting_down) {
 372         crm_info("Received shutdown ack");
 373         if (shutdown_ack_timer > 0) {
 374             g_source_remove(shutdown_ack_timer);
 375             shutdown_ack_timer = 0;
 376         }
 377         return;
 378     }
 379 #endif
 380     crm_debug("Ignoring unexpected shutdown ack");
 381 }
 382 
 383 /*!
 384  * \internal
 385  * \brief Make short exit timer fire immediately
 386  */
 387 void
 388 handle_shutdown_nack(void)
     /*  */
 389 {
 390 #ifdef PCMK__COMPILE_REMOTE
 391     if (shutting_down) {
 392         crm_info("Received shutdown nack");
 393         if (shutdown_ack_timer > 0) {
 394             g_source_remove(shutdown_ack_timer);
 395             shutdown_ack_timer = g_timeout_add(0, lrmd_exit, NULL);
 396         }
 397         return;
 398     }
 399 #endif
 400     crm_debug("Ignoring unexpected shutdown nack");
 401 }
 402 
 403 static GOptionEntry entries[] = {
 404     { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
 405       &options.log_files, "Send logs to the additional named logfile", NULL },
 406 
 407 #ifdef PCMK__COMPILE_REMOTE
 408 
 409     { "port", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.port,
 410       "Port to listen on (defaults to " G_STRINGIFY(DEFAULT_REMOTE_PORT) ")", NULL },
 411 #endif  // PCMK__COMPILE_REMOTE
 412 
 413     { NULL }
 414 };
 415 
 416 static pcmk__supported_format_t formats[] = {
 417     PCMK__SUPPORTED_FORMAT_NONE,
 418     PCMK__SUPPORTED_FORMAT_TEXT,
 419     PCMK__SUPPORTED_FORMAT_XML,
 420     { NULL, NULL, NULL }
 421 };
 422 
 423 static GOptionContext *
 424 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
     /*  */
 425 {
 426     GOptionContext *context = NULL;
 427 
 428     context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
 429     pcmk__add_main_args(context, entries);
 430     return context;
 431 }
 432 
 433 int
 434 main(int argc, char **argv, char **envp)
     /*  */
 435 {
 436     int rc = pcmk_rc_ok;
 437     crm_exit_t exit_code = CRM_EX_OK;
 438 
 439     const char *option = NULL;
 440 
 441     pcmk__output_t *out = NULL;
 442 
 443     GError *error = NULL;
 444 
 445     GOptionGroup *output_group = NULL;
 446     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
 447 #ifdef PCMK__COMPILE_REMOTE
 448     gchar **processed_args = pcmk__cmdline_preproc(argv, "lp");
 449 #else
 450     gchar **processed_args = pcmk__cmdline_preproc(argv, "l");
 451 #endif  // PCMK__COMPILE_REMOTE
 452     GOptionContext *context = build_arg_context(args, &output_group);
 453 
 454 #ifdef PCMK__COMPILE_REMOTE
 455     // If necessary, create PID 1 now before any file descriptors are opened
 456     remoted_spawn_pidone(argc, argv, envp);
 457 #endif
 458 
 459     crm_log_preinit(EXECD_NAME, argc, argv);
 460 
 461     pcmk__register_formats(output_group, formats);
 462     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
 463         exit_code = CRM_EX_USAGE;
 464         goto done;
 465     }
 466 
 467     rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
 468     if (rc != pcmk_rc_ok) {
 469         exit_code = CRM_EX_ERROR;
 470         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
 471                     "Error creating output format %s: %s",
 472                     args->output_ty, pcmk_rc_str(rc));
 473         goto done;
 474     }
 475 
 476     if (args->version) {
 477         out->version(out, false);
 478         goto done;
 479     }
 480 
 481     // Open additional log files
 482     if (options.log_files != NULL) {
 483         for (gchar **fname = options.log_files; *fname != NULL; fname++) {
 484             rc = pcmk__add_logfile(*fname);
 485 
 486             if (rc != pcmk_rc_ok) {
 487                 out->err(out, "Logging to %s is disabled: %s",
 488                          *fname, pcmk_rc_str(rc));
 489             }
 490         }
 491     }
 492 
 493     pcmk__cli_init_logging(EXECD_NAME, args->verbosity);
 494     crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
 495 
 496     // ocf_log() (in resource-agents) uses the capitalized env options below
 497     option = pcmk__env_option(PCMK__ENV_LOGFACILITY);
 498     if (!pcmk__str_eq(option, PCMK__VALUE_NONE,
 499                       pcmk__str_casei|pcmk__str_null_matches)
 500         && !pcmk__str_eq(option, "/dev/null", pcmk__str_none)) {
 501 
 502         pcmk__set_env_option("LOGFACILITY", option, true);
 503     }
 504 
 505     option = pcmk__env_option(PCMK__ENV_LOGFILE);
 506     if (!pcmk__str_eq(option, PCMK__VALUE_NONE,
 507                       pcmk__str_casei|pcmk__str_null_matches)) {
 508         pcmk__set_env_option("LOGFILE", option, true);
 509 
 510         if (pcmk__env_option_enabled(crm_system_name, PCMK__ENV_DEBUG)) {
 511             pcmk__set_env_option("DEBUGLOG", option, true);
 512         }
 513     }
 514 
 515 #ifdef PCMK__COMPILE_REMOTE
 516     if (options.port != NULL) {
 517         pcmk__set_env_option(PCMK__ENV_REMOTE_PORT, options.port, false);
 518     }
 519 #endif  // PCMK__COMPILE_REMOTE
 520 
 521     start_time = time(NULL);
 522 
 523     crm_notice("Starting Pacemaker " EXECD_TYPE " executor");
 524 
 525     /* The presence of this variable allegedly controls whether child
 526      * processes like httpd will try and use Systemd's sd_notify
 527      * API
 528      */
 529     unsetenv("NOTIFY_SOCKET");
 530 
 531     {
 532         // Temporary directory for resource agent use (leave owned by root)
 533         int rc = pcmk__build_path(CRM_RSCTMP_DIR, 0755);
 534 
 535         if (rc != pcmk_rc_ok) {
 536             crm_warn("Could not create resource agent temporary directory "
 537                      CRM_RSCTMP_DIR ": %s", pcmk_rc_str(rc));
 538         }
 539     }
 540 
 541     rsc_list = pcmk__strkey_table(NULL, free_rsc);
 542     ipcs = mainloop_add_ipc_server(CRM_SYSTEM_LRMD, QB_IPC_SHM, &lrmd_ipc_callbacks);
 543     if (ipcs == NULL) {
 544         crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
 545         exit_code = CRM_EX_FATAL;
 546         goto done;
 547     }
 548 
 549 #ifdef PCMK__COMPILE_REMOTE
 550     if (lrmd_init_remote_tls_server() < 0) {
 551         crm_err("Failed to create TLS listener: shutting down and staying down");
 552         exit_code = CRM_EX_FATAL;
 553         goto done;
 554     }
 555     ipc_proxy_init();
 556 #endif
 557 
 558     mainloop_add_signal(SIGTERM, lrmd_shutdown);
 559     mainloop = g_main_loop_new(NULL, FALSE);
 560     crm_notice("Pacemaker " EXECD_TYPE " executor successfully started and accepting connections");
 561     crm_notice("OCF resource agent search path is %s", OCF_RA_PATH);
 562     g_main_loop_run(mainloop);
 563 
 564     /* should never get here */
 565     lrmd_exit(NULL);
 566 
 567 done:
 568     g_strfreev(options.log_files);
 569 #ifdef PCMK__COMPILE_REMOTE
 570     g_free(options.port);
 571 #endif  // PCMK__COMPILE_REMOTE
 572 
 573     g_strfreev(processed_args);
 574     pcmk__free_arg_context(context);
 575 
 576     pcmk__output_and_clear_error(&error, out);
 577 
 578     if (out != NULL) {
 579         out->finish(out, exit_code, true, NULL);
 580         pcmk__output_free(out);
 581     }
 582     pcmk__unregister_formats();
 583     crm_exit(exit_code);
 584 }
/* */
root/daemons/execd/pacemaker-execd.c

DEFINITIONS
